diff --git a/.gitignore b/.gitignore index 955a362..6e09010 100644 --- a/.gitignore +++ b/.gitignore @@ -55,6 +55,7 @@ temp/ docs/superpowers/ docs/plans/ docs/specs/ +.superpowers/ # Skill-optimizer generated artifacts .skill-optimizer/ @@ -67,3 +68,6 @@ cli-commands.json tools.json tasks.json .worktrees/ + +# Auto-improve-skill wrapper run logs +examples/workbench/*/.run.log diff --git a/CLAUDE.md b/CLAUDE.md index e91b35e..5bd3d80 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -23,6 +23,7 @@ npx tsx src/cli.ts run-suite --help - `src/workbench/`: workbench case loading, suite loading, Docker runner, Pi agent, graders, and traces - `docker/workbench-runner.Dockerfile`: generic non-root container image for setup, agent, grade, and cleanup phases - `skills/skill-optimizer/SKILL.md`: canonical distributable Agent Skill +- `skills/auto-improve-orchestrator/SKILL.md`: Claude Code skill that orchestrates auto-improvement of public agent skills. Operator dispatches the orchestrator subagent (via Agent tool with `isolation: "worktree"`) which manages research / eval-iteration / skill-iteration end-to-end for one skill. See `docs/auto-improve-skill-v1.3-spec.md` for the architecture. - `skills/skill-optimizer/references/workbench.md`: detailed workbench schema and usage reference - `.claude-plugin/`, `.codex-plugin/`, `.cursor-plugin/`, `.opencode/`: cross-agent plugin manifests and install support - `.agents/plugins/marketplace.json`: Codex repo marketplace entry for the root plugin diff --git a/docs/auto-improve-skill-v1.3-design.md b/docs/auto-improve-skill-v1.3-design.md new file mode 100644 index 0000000..33f7aa4 --- /dev/null +++ b/docs/auto-improve-skill-v1.3-design.md @@ -0,0 +1,237 @@ +# auto-improve-skill v1.3 — design proposal + +**Status:** draft, written 2026-05-12 during the v1.2.1 PR-prep session. +**Audience:** team review before implementation. +**Tracking:** the in-flight v1.2.1 pilot work (web-design-guidelines / +agent-browser / supabase) is the empirical basis for this proposal. + +## Executive summary + +v1.3 adds two structural phases to the auto-improve-skill pipeline, +both motivated by failure modes observed across 4 v1.2.1 pilots: + +1. **Phase 0 — Research-first context.** A research subagent reads the + target upstream repo's contribution conventions, frontmatter spec, + prefix taxonomy, and merged-PR shape patterns, and writes a context + file that v1.2.1's `--context` flag consumes. Without this, the + auto-pilot produces output that requires manual reformulation + before submission. +2. **Phase 3.5 — Eval-readiness loop.** The pipeline iterates on the + eval (seed harder/simpler cases) until baseline lands in the + "interesting zone" `(0.50, 0.95)`. Without this, baselines saturate + at 1.00 (no headroom to demonstrate uplift) or floor at <0.50 (skill + shape blocks measurement). + +The skill-iteration loop (current Phase 4) is unchanged. + +## Lesson 1 — Research-first context is mandatory + +### Evidence (4 pilots this session) + +| Skill | Without context | With researched context | +|---|---|---| +| web-design-guidelines | Manual proposal needed retargeting (SKILL.md→command.md), reformulation across 3 stylistic siblings, frontmatter mismatch. Manual labor: ~2 hours per PR. | Auto-pilot produced a clean, mergeable diff to the right file in the right voice. Manual labor: ~10 min mirror to AGENTS.md/README.md. | +| agent-browser | Auto-pilot proposed editing `skills/agent-browser/SKILL.md`. Per upstream `AGENTS.md`, that file is intentionally a discovery stub; real content lives at `skill-data/core/SKILL.md`. Manual retarget required. | (Pending — pilot in flight; context file says edit `agent-browser-core.md` and produced output names `before-skill-data-core-SKILL.md`.) | +| supabase (batch-1) | Produced shape-novel `references/review-...md` with non-existent prefix (`review-`), missing `impactDescription` frontmatter field, philosophical-style content (MEDIUM-HIGH rejection risk per CONTRIBUTING patterns). | Auto-pilot reshaped into convention-perfect SQL anti-pattern under correct prefix (`monitor-`), full 4-field frontmatter, `**Incorrect**`/`**Correct**` SQL blocks per `_template.md`, trailing `Reference:` link. Zero manual reformulation needed. | + +### Generalization + +The auto-pilot is good at *finding what to change* (which rules, which +files, which absence-type gaps). It is bad at *fitting upstream +conventions*: frontmatter schemas, file-location norms, prefix +taxonomies, additive-only rules, "Discussion-first" gates, voice +consistency. Conventions are repo-specific tribal knowledge that +cannot be inferred from reading the SKILL.md alone. + +### Phase 0 design + +```text +Phase 0 — Research upstream (NEW, runs before Phase 1) + +Inputs: + - target slug // + +Subtasks (executed by a research subagent): + 1. Repo metadata: license, CLA, default branch, recent activity + 2. Read CONTRIBUTING.md, AGENTS.md, .github/PULL_REQUEST_TEMPLATE.md, + CODEOWNERS, .github/workflows/*.yml + 3. Read skill-specific convention files: _contributing.md, + _template.md, _sections.md (or equivalents) + 4. Read sanity-test source if present (don't trust prior assumptions + about what CI validates) + 5. Sample last 10 merged PRs to the target skill (or repo) for shape: + file count, body shape, conventional-commit usage, scope sizing + 6. Sample last 5 closed-without-merge PRs for rejection signals: + "Discussion-first gate violated", "shape-novel content rejected", + etc. + 7. Identify other consumers (gh search for raw URL references; check + for install scripts; check repo's own README for distribution + channels) + +Output: + tools/auto-improve-contexts/-.md + - Repository facts (license, CI, maintainers, merge style) + - Hard constraints (additive-only, file-location, prefix taxonomy, + forbidden modifications) + - Frontmatter spec (exact required fields + allowed values) + - Content shape template (copy-and-fill) + - Optimization target file (where the skill change should land) + - Risk profile (LOW/MEDIUM/HIGH + reasons) + - Pre-submit checklist (what auto-pilot must verify) + - Useful URLs + +Cost: ~$0.50–$1.00 per skill (single subagent invocation). + +Caching: context files are committed to the repo. Re-running on the +same skill within 30 days: skip Phase 0, reuse cached context (with +explicit `--refresh-context` flag to force re-research). + +Operator override: `--context ` flag continues to work; if +provided, Phase 0 is skipped. +``` + +## Lesson 2 — Two-loop iteration: eval AND skill + +### Evidence + +| Skill | Initial baseline | Failure mode | Manual fix | +|---|---|---|---| +| agent-browser (Tier-0 only) | 0.97 | Shallow eval — only graded command-presence, not the skill's actual value prop (ref-based interaction, snapshot interpretation, multi-step state) | Built Tier-1 cases via subagent (~half-day): pre-recorded fixtures, stateful fake CLI, 4 new cases targeting the differentiator | +| supabase (calibrated graders, frontier models) | 1.00 | Eval saturated; calibrated graders + capable models perfect-detect the 9 seeded violations | Built deeper eval via subagent (~30 min): 3 new cases with absence-type violations requiring enumeration across multi-statement files | + +In both cases, the **eval was the bug, not the skill**. The skill- +iteration loop in Phase 4 can't escape the dead zone — it just exits +"baseline >= 0.95, success" with no measurement. + +### Phase 3.5 design + +```text +Phase 3.5 — Eval-readiness loop (NEW, between Phase 3 and Phase 4) + +while baseline NOT IN (0.50, 0.95): + if baseline >= 0.95: + dispatch eval-iteration subagent with prompt: + "Add 2-3 cases targeting absence-type rules / failure modes + not yet exercised. Realistic seedings, force enumeration. + Don't touch existing cases." + elif baseline < 0.50: + options (operator-decided or auto-judged): + a) Grader miscalibrated → run grader-vs-skill check (existing + in Phase 4); if grader bug, fix and re-baseline + b) Cases too contrived → simplify (remove ambiguous violations, + tighten task descriptions) + c) Skill genuinely doesn't address this shape → exit + "blocked-by-skill-shape" honestly + re-measure baseline + abort if iteration count > 3 (eval is harder to converge than skill) + +Then proceed to Phase 4 unchanged. + +Cost: ~$1.00 per eval iteration (subagent + smoke check). Bounded at +3 iterations. + +Convergence criterion: baseline in (0.50, 0.95). The interesting zone. +``` + +### Why these bounds? + +- **>= 0.95**: ceiling effect; can't measure uplift because there's no + headroom. Even +0.04 wouldn't clear our existing 0.05 success + threshold. +- **< 0.50**: floor effect; either the eval is broken (grader bugs, + ambiguous tasks) or the skill genuinely doesn't address the seeded + rules. In either case, the optimizer can't reliably improve. +- **(0.50, 0.95)**: the optimizer has clear signal. Both successful + iteration and lack-of-improvement are interpretable. + +## Combined v1.3 architecture + +```text +0. Research upstream → context file (NEW) +1. Discover skill, classify +2. Build initial suite +3. Measure baseline +3.5 Eval-readiness loop (NEW): + while baseline NOT IN (0.50, 0.95): iterate eval +4. Skill-iteration loop (existing): + while uplift < 0.05 AND iterations < 2: iterate skill +5. Re-check baseline (did eval drift after skill change?) +6. Package +``` + +## Implementation cost + +| Component | Effort | Cost per pilot run | +|---|---|---| +| Phase 0 research subagent | ~1 day to write the prompt template + repo-detection logic | +$0.50–$1.00 | +| Phase 3.5 eval-iteration subagent | ~2 days to write the subagent prompt + integration into the wrapper loop | +$1.00 per eval iteration (bounded at 3) | +| Wrapper integration | ~1 day for new flags (`--refresh-context`, `--max-eval-iterations`), result aggregation, telemetry | n/a | +| Testing on 5 representative skills | ~1 day | ~$10 total | + +**Total v1.3 build cost:** ~5 days of work + ~$15 of pilot runs to +validate. + +**Per-pilot incremental cost:** ~$1.50–$5.00 over v1.2.1, depending on +how many eval iterations are needed (most skills will converge in 0–1). + +## Migration / backwards compatibility + +- v1.2.1 wrapper continues to work standalone (`--context` flag is + preserved). +- v1.3 is opt-in via a new flag, e.g. `--research` to enable Phase 0 + and `--auto-eval` to enable Phase 3.5. Default off until validated. +- Once validated, defaults flip to on; operator can opt out via + `--no-research` / `--no-auto-eval`. + +## Open questions + +1. **Research-subagent prompt template** — should the Phase 0 subagent + prompt be skill-classification-aware? E.g. ask different questions + for code-reviewer vs tool-use vs document-producer skills. Probably + yes, but adds template branching complexity. +2. **Eval-iteration subagent prompt template** — same question. The + "what makes a harder case" guidance differs sharply by skill type. +3. **When to refuse eval iteration** — if baseline is at 1.00 because + the skill genuinely is excellent at its job, we shouldn't fabricate + harder cases. How does Phase 3.5 distinguish "ceiling because skill + is good" from "ceiling because eval is shallow"? + - One heuristic: if the existing eval already exercises the skill's + stated value prop (per the SKILL.md description), assume good. If + it tests only mechanical command presence, assume shallow. + - This needs a "value-prop coverage" check in Phase 3.5, ideally + read from the skill's frontmatter description. +4. **Cost ceiling** — Phase 0 + Phase 3.5 each cost ~$1; Phase 4 costs + $1–3. v1.3 raises typical pilot cost from ~$2 (v1.2.1) to ~$3–6. + Still within the $10 wrapper budget but worth keeping under + observation. +5. **When to accept lossy reshape** — supabase v1.2.1 forced reshape + from "two-pass meta-workflow" into "concrete SQL anti-pattern with + `**Incorrect**`/`**Correct**` blocks". Worked beautifully. Will + this transfer to other skills, or did we get lucky with supabase's + tight `_template.md`? Probably needs more pilots before generalizing. + +## Open architectural questions (longer-term) + +- **Should the auto-pilot also produce the AGENTS.md/README.md mirrors + for repos with multi-file convention (PR #23 shape)?** Currently + manual at PR-draft time. Could be a separate "packaging" subagent. +- **Should we treat upstream PR-submission as a phase too (Phase 6)?** + i.e. fork-clone-push-create-PR automation. Operator-gated for high- + visibility actions, but otherwise plausible. +- **Can the research subagent be made repo-agnostic?** Right now we + assumed a "skill repo" structure. For repos with non-standard layout + (vendored skills, monorepos, etc.) the research needs different + patterns. + +## Provenance + +This design is grounded in the v1.2.1 pilot session captured in: + +- `docs/pilot-runs/upstream-pr-drafts/1-vercel-labs-web-interface-guidelines.md` +- (pending) `docs/pilot-runs/upstream-pr-drafts/3-vercel-labs-agent-browser-*.md` +- (pending) `docs/pilot-runs/upstream-pr-drafts/4-supabase-agent-skills-*.md` +- `tools/auto-improve-contexts/{vercel-web-interface-guidelines, + vercel-agent-browser, supabase-postgres-best-practices}.md` +- Eval branches: `eval/auto-pilot/web-design-guidelines`, + `eval/auto-pilot/agent-browser` (in flight), + `eval/auto-pilot/supabase-postgres-best-practices-v2` (in flight). diff --git a/docs/auto-improve-skill-v1.3-plan.md b/docs/auto-improve-skill-v1.3-plan.md new file mode 100644 index 0000000..b93ea3b --- /dev/null +++ b/docs/auto-improve-skill-v1.3-plan.md @@ -0,0 +1,1654 @@ +# auto-improve-orchestrator v1.3 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace the v1.2.1 wrapper-spawned `claude -p` autonomous pilot with a Claude Code skill (`auto-improve-orchestrator`) that an operator's CC session invokes via the Agent tool. Each orchestrator subagent owns one skill end-to-end and dispatches its own sub-subagents. + +**Architecture:** Skill-optimizer stays lean (eval engine only). Orchestration lives in `skills/auto-improve-orchestrator/` as a Claude Code skill containing 4 prompt templates (orchestrator + 3 sub-subagents) plus reference material (workflow doc, lessons, per-skill context library). Operator dispatches the orchestrator via Agent tool with `isolation: "worktree"` for parallelism. + +**Tech Stack:** Markdown (skill prompts), bash (validation scripts), gray-matter (frontmatter parsing), Claude Code Agent tool. + +**Working dir:** `.claude/worktrees/v1.3-impl/` (branch `feat/auto-improve-skill-v1.3`). + +**Spec:** `docs/auto-improve-skill-v1.3-spec.md` (already committed at `5289092`). + +--- + +## File Structure + +After this plan executes: + +```text +skills/auto-improve-orchestrator/ # NEW skill + SKILL.md # discovery + invocation guide + prompts/ + orchestrator.md # main subagent prompt template + research-upstream.md # sub-subagent: Phase 0 research + eval-iterate.md # sub-subagent: Phase 3.5 eval iteration + skill-iterate.md # sub-subagent: Phase 4 skill iteration + references/ + workflow.md # human-readable orchestrator algorithm + lessons.md # MOVED from tools/ + contexts/ # MOVED from tools/ + vercel-web-interface-guidelines.md + vercel-agent-browser.md + supabase-postgres-best-practices.md + +tools/ # DELETIONS + auto-improve-skill.mjs # DELETED + auto-improve-skill-prompt.md # DELETED + auto-improve-skill-lessons.md # MOVED OUT (see above) + auto-improve-contexts/ # MOVED OUT (see above) + +CLAUDE.md # UPDATED to point at new skill +``` + +Each file's responsibility: + +- **SKILL.md** — frontmatter (name, description) + ~1 page of "how the operator invokes this" guidance. Always loaded by Claude Code when the skill is referenced. +- **prompts/orchestrator.md** — self-contained prompt template the operator dispatches via the Agent tool. Embeds the full algorithm. Reads context, dispatches sub-subagents, calls `run-suite`, packages, commits. +- **prompts/research-upstream.md** — self-contained sub-subagent prompt. Researches one upstream repo. Writes a context file. Returns short report. +- **prompts/eval-iterate.md** — self-contained sub-subagent prompt. Adds harder cases / simplifies cases / fixes grader bugs. Runs smoke check. Commits. +- **prompts/skill-iterate.md** — self-contained sub-subagent prompt. Applies one recipe (A–E) additively to the target file. Re-runs suite. Commits if uplift. +- **references/workflow.md** — human-readable description of the algorithm (mirror of what's in `prompts/orchestrator.md`, but for humans, not AI). +- **references/lessons.md** — recipes A–E + grader patterns G1–G6. Read by `prompts/skill-iterate.md` and `prompts/eval-iterate.md`. +- **references/contexts/** — per-skill upstream-research output. Read by `prompts/skill-iterate.md`. Written by `prompts/research-upstream.md`. + +--- + +## Task 1: Move `lessons.md` into the new skill + +**Files:** + +- Move: `tools/auto-improve-skill-lessons.md` → `skills/auto-improve-orchestrator/references/lessons.md` + +- [ ] **Step 1: Create destination dirs** + +```bash +mkdir -p skills/auto-improve-orchestrator/references +``` + +- [ ] **Step 2: `git mv` the lessons doc** + +```bash +git mv tools/auto-improve-skill-lessons.md \ + skills/auto-improve-orchestrator/references/lessons.md +``` + +- [ ] **Step 3: Verify the move + content unchanged** + +```bash +test -f skills/auto-improve-orchestrator/references/lessons.md +test ! -f tools/auto-improve-skill-lessons.md +head -3 skills/auto-improve-orchestrator/references/lessons.md +``` + +Expected: file exists at new path, gone from old path, content starts with the heading from the original lessons doc. + +- [ ] **Step 4: Commit** + +```bash +git commit -m "refactor(orchestrator): move lessons.md into new skill" +``` + +--- + +## Task 2: Move `contexts/` into the new skill + +**Files:** + +- Move: `tools/auto-improve-contexts/` → `skills/auto-improve-orchestrator/references/contexts/` + +- [ ] **Step 1: `git mv` the entire contexts directory** + +```bash +git mv tools/auto-improve-contexts \ + skills/auto-improve-orchestrator/references/contexts +``` + +- [ ] **Step 2: Verify the move** + +```bash +ls skills/auto-improve-orchestrator/references/contexts/ +test ! -d tools/auto-improve-contexts +``` + +Expected: 3 files at new path (`vercel-web-interface-guidelines.md`, `vercel-agent-browser.md`, `supabase-postgres-best-practices.md`), nothing at old path. + +- [ ] **Step 3: Commit** + +```bash +git commit -m "refactor(orchestrator): move contexts/ into new skill" +``` + +--- + +## Task 3: Delete v1.2.1 wrapper files + +**Files:** + +- Delete: `tools/auto-improve-skill.mjs` +- Delete: `tools/auto-improve-skill-prompt.md` + +- [ ] **Step 1: Confirm files exist (sanity check before deletion)** + +```bash +ls tools/auto-improve-skill.mjs tools/auto-improve-skill-prompt.md +``` + +Expected: both files listed. + +- [ ] **Step 2: `git rm` both** + +```bash +git rm tools/auto-improve-skill.mjs tools/auto-improve-skill-prompt.md +``` + +- [ ] **Step 3: Verify removal + tools/ dir state** + +```bash +test ! -f tools/auto-improve-skill.mjs +test ! -f tools/auto-improve-skill-prompt.md +ls tools/ +``` + +Expected: both files gone. `tools/` may now contain other unrelated files (don't touch them). + +- [ ] **Step 4: Commit** + +```bash +git commit -m "refactor(orchestrator): remove v1.2.1 wrapper + embedded prompt + +The wrapper-spawned claude -p autonomous pilot is replaced by the new +auto-improve-orchestrator Claude Code skill (built in subsequent commits). +Operator now dispatches the orchestrator subagent via the Agent tool +instead of running a Node wrapper." +``` + +--- + +## Task 4: Update CLAUDE.md to remove old wrapper references + +**Files:** + +- Modify: `CLAUDE.md` + +- [ ] **Step 1: Read the current CLAUDE.md to find references to the old wrapper** + +```bash +grep -n "auto-improve" CLAUDE.md +``` + +Expected: zero matches in the current file (the wrapper wasn't documented in CLAUDE.md as of the v1.2.1 work). If matches exist, note line numbers for editing in Step 2. + +- [ ] **Step 2: Add a brief pointer to the new orchestrator skill in CLAUDE.md's "Important Files" section** + +Find the "## Important Files" section in `CLAUDE.md`. After the existing `skills/skill-optimizer/SKILL.md` bullet, add a new bullet: + +```markdown +- `skills/auto-improve-orchestrator/SKILL.md`: Claude Code skill that orchestrates auto-improvement of public agent skills. Operator dispatches the orchestrator subagent (via Agent tool with `isolation: "worktree"`) which manages research / eval-iteration / skill-iteration end-to-end for one skill. See `docs/auto-improve-skill-v1.3-spec.md` for the architecture. +``` + +- [ ] **Step 3: Verify the edit landed** + +```bash +grep -A 1 "auto-improve-orchestrator/SKILL.md" CLAUDE.md +``` + +Expected: the new bullet appears. + +- [ ] **Step 4: Commit** + +```bash +git add CLAUDE.md +git commit -m "docs(CLAUDE.md): point at new auto-improve-orchestrator skill" +``` + +--- + +## Task 5: Create `skills/auto-improve-orchestrator/SKILL.md` + +**Files:** + +- Create: `skills/auto-improve-orchestrator/SKILL.md` + +- [ ] **Step 1: Write the SKILL.md with frontmatter + invocation guide** + +Create `skills/auto-improve-orchestrator/SKILL.md` with this exact content: + +````markdown +--- +name: auto-improve-orchestrator +description: Use when an operator wants to auto-improve a public agent skill — research upstream conventions, iterate the eval if it's saturated/floored, iterate the skill itself, and package proposed upstream changes. Dispatch the orchestrator subagent via the Agent tool with isolation:"worktree" for one or more skills (parallel-safe). +--- + +# auto-improve-orchestrator + +This skill manages the end-to-end auto-improvement workflow for one public +agent skill: research the upstream repo's conventions, measure baseline +on an existing eval workbench, iterate the eval if it's saturated/floored, +iterate the skill content with measured uplift, and package the proposed +upstream change. + +The skill-optimizer stays lean — it ships the eval engine (`run-suite`, +`run-case`, graders, Docker harness). This orchestrator skill contains +the *workflow* logic that uses the engine. + +## When to use + +- Operator says "auto-improve " or "improve " +- Operator says "run auto-improve on these N skills" (batch) +- Operator wants to extend the orchestration logic itself + +## How to invoke + +The operator's CC session dispatches the orchestrator subagent via the +Agent tool. For a single skill: + +``` +Agent({ + description: "auto-improve ", + subagent_type: "general-purpose", + isolation: "worktree", + run_in_background: true, + prompt: +}) +``` + +For a batch of N skills, dispatch N Agent calls in a single message — +they run in parallel, each in its own worktree. + +The orchestrator subagent: + +1. Reads/dispatches research subagent (Phase 0) → produces context file +2. Measures baseline via `run-suite` (Phase 3) — resume-aware +3. Iterates eval via dispatch (Phase 3.5) until baseline ∈ (0.50, 0.95) +4. Iterates skill via dispatch (Phase 4) until uplift ≥ +0.05 or 2 iters +5. Packages `proposed-upstream-changes/` + `analysis.md`, commits to + `eval/auto-pilot/` branch, returns summary + +## Sub-subagent prompts + +- `prompts/research-upstream.md` — Phase 0 research subagent template +- `prompts/eval-iterate.md` — Phase 3.5 eval-iteration subagent template +- `prompts/skill-iterate.md` — Phase 4 skill-iteration subagent template + +## Reference material + +- `references/workflow.md` — human-readable description of the algorithm +- `references/lessons.md` — recipes A-E + grader patterns G1-G6 (the + skill-iterate subagent reads this) +- `references/contexts/` — per-skill upstream-research outputs (the + skill-iterate subagent reads these; the research subagent writes them) + +## Pre-conditions + +- The skill's eval workbench must already exist at + `examples/workbench//`. Building initial workbenches is + out of scope for v1.3 — operator builds them manually. +- `OPENROUTER_API_KEY` must be set in `.env` at the repo root. +- Worktree must be created via `isolation: "worktree"` for parallelism + + isolation. Do NOT run the orchestrator subagent on the main + worktree. + +## Out of scope (deferred) + +- Initial workbench construction (v1.4) +- Autonomous CI mode (no operator session) +- Automatic PR submission (operator-driven) + +See `docs/auto-improve-skill-v1.3-spec.md` for the full design. +```` + +- [ ] **Step 2: Verify frontmatter parses** + +```bash +node -e " +const matter = require('gray-matter'); +const fs = require('fs'); +const content = fs.readFileSync('skills/auto-improve-orchestrator/SKILL.md', 'utf-8'); +const parsed = matter(content); +console.log('name:', parsed.data.name); +console.log('description length:', parsed.data.description.length); +if (!parsed.data.name || !parsed.data.description) { + console.error('MISSING required frontmatter fields'); + process.exit(1); +} +console.log('OK'); +" +``` + +Expected output: + +```text +name: auto-improve-orchestrator +description length: 33[0-9] +OK +``` + +(`gray-matter` is already a project dep — it's used by the workbench loader.) + +- [ ] **Step 3: Commit** + +```bash +git add skills/auto-improve-orchestrator/SKILL.md +git commit -m "feat(orchestrator): create SKILL.md with invocation guide" +``` + +--- + +## Task 6: Write `references/workflow.md` + +**Files:** + +- Create: `skills/auto-improve-orchestrator/references/workflow.md` + +This is the human-readable mirror of the orchestrator algorithm. The orchestrator subagent embeds the same logic in its prompt template (Task 10), but humans read this file to understand what the orchestrator does. + +- [ ] **Step 1: Write the workflow doc** + +Create `skills/auto-improve-orchestrator/references/workflow.md` with this exact content: + +````markdown +# auto-improve-orchestrator workflow + +This is a human-readable description of the algorithm the orchestrator +subagent follows. The same logic is embedded in +`../prompts/orchestrator.md` (the AI-targeted prompt template). + +## Inputs + +- `${SLUG}`: `//` (e.g. `supabase/agent-skills/supabase-postgres-best-practices`) +- `${REFRESH_CONTEXT}`: optional flag, default false. If true, force re-research even if cached context file exists. + +## Setup + +1. Parse `SLUG` into `OWNER`, `REPO`, `SKILL_ID`. +2. Verify the orchestrator is running in a git worktree (the `isolation: "worktree"` requirement). +3. Verify the skill's workbench exists at `examples/workbench//`. If not, exit `blocked-by-missing-workbench`. +4. Define paths: + - `WORKBENCH = examples/workbench//` + - `CONTEXT_FILE = skills/auto-improve-orchestrator/references/contexts/-.md` +5. Verify `OPENROUTER_API_KEY` is set (source `.env` from the main repo if needed). + +## Phase 0: Research + +If `CONTEXT_FILE` exists AND not `REFRESH_CONTEXT`: + +- Read `CONTEXT_FILE`. Use it. + +Else: + +- Dispatch sub-subagent `prompts/research-upstream.md` with input `${SLUG}`. +- Wait for completion. Verify `CONTEXT_FILE` was written. +- Read it. + +Extract from `CONTEXT_FILE`: + +- `target_file` (the file the skill-iterate subagent will edit) +- `packaging_dir` (where `proposed-upstream-changes/` subdir should go) +- `additive_only_constraint` and any other hard constraints + +## Phase 3: Baseline measurement + +Look for the most recent `.results//suite-result.json` in `WORKBENCH`. + +If found AND its case set matches current `suite.yml`: + +- Read it (resume — skip baseline run). + +Else: + +- Run from `WORKBENCH`: `npx tsx ../../../src/cli.ts run-suite ./suite.yml --trials 3` +- Wait for completion (long-running, ~30-60 min depending on suite size). +- Read the new `suite-result.json`. + +Compute `per_case_min` from `suite-result.json`: + +- Group `results[]` by `caseName`. +- Per-case mean = average of trial scores for that case (across all models and trials). +- `per_case_min = min(per-case means across all cases)`. + +## Phase 3.5: Eval-readiness loop + +```text +EVAL_ITER = 0 +While per_case_min OUTSIDE (0.50, 0.95) AND EVAL_ITER < 3: + If per_case_min >= 0.95: + DIRECTION = "add-harder" + Else if per_case_min < 0.50: + Run grader-vs-skill check: + - Read failed trials' findings.txt + - If models DID find the violations but grader scored wrong: + DIRECTION = "fix-graders" + - Else: DIRECTION = "simplify" + + Dispatch sub-subagent prompts/eval-iterate.md with: + SKILL_ID, latest suite-result path, DIRECTION + Wait for completion. Verify it committed workbench changes. + + Re-run baseline measurement (Phase 3, no resume — case set changed). + EVAL_ITER += 1 +``` + +Exit conditions: + +- Still `per_case_min >= 0.95` after 3 iterations: exit `skill-genuinely-good`. +- Still `per_case_min < 0.50` after 3 iterations: exit `blocked-by-skill-shape`. + +## Phase 4: Skill iteration + +```text +baseline_per_case_min = per_case_min +ITER = 0 + +While ITER < 2: + ITER += 1 + Dispatch sub-subagent prompts/skill-iterate.md with: + SKILL_ID, latest suite-result path, target_file from context, ITER + Wait for completion. Verify it committed changes. + + Read the new suite-result.json from the latest .results//. + new_per_case_min = recompute + + If new_per_case_min - baseline_per_case_min >= 0.05: + Success — break. +``` + +Exit if no iteration cleared +0.05: status `uplift-too-small`. Still package the changes (they're additive — the operator can decide whether to ship). + +## Phase 5: Package + +Compose `WORKBENCH/proposed-upstream-changes//`: + +- `before-` — original target file (read from upstream's vendored copy, before any v1.3 modifications) +- `after-` — current target file content +- `README.md` — description, evidence (per-case breakdown table), how to apply + +Write `WORKBENCH/analysis.md`: + +```markdown +--- +skill: +status: success | uplift-too-small | skill-genuinely-good | blocked-by-skill-shape | blocked-by-error +classification: +baseline_per_case_min: 0.NN +final_per_case_min: 0.NN +iterations: eval=N, skill=N +total_cost_usd: NN.NN +--- + +# Auto-pilot run for + +[3-6 short bullets covering: classification, what the eval surfaced, +what was changed and why, uplift result, any judgment calls.] +``` + +## Phase 6: Commit + +```bash +git checkout -b eval/auto-pilot/ +git add WORKBENCH/analysis.md +git add WORKBENCH/proposed-upstream-changes/ # if exists +git add WORKBENCH/.results//suite-result.json +# Skill modifications were committed by the skill-iterate subagent already +git commit -m "eval(auto-pilot): — status=, baseline=→" +``` + +DO NOT `git push`. + +## Cost tracking + +The orchestrator tracks cumulative `metrics.cost.total` from each `run-suite` invocation. + +- Soft warning at $5: print to stderr but continue. +- Hard stop at $10: write `analysis.md` immediately with `status: budget-exceeded` and exit. + +Sub-subagent dispatches (research / eval-iterate / skill-iterate) use the operator's Claude Code session under their plan and have no marginal cost. + +## Return summary + +The orchestrator returns to the caller (operator's CC session): + +- `branch`: `eval/auto-pilot/` +- `commit`: `` +- `status`: one of the exit statuses above +- `baseline_per_case_min` → `final_per_case_min` +- per-case breakdown table +- `proposed-upstream-changes/` path (if status is `success` or `uplift-too-small`) + +## Rules of engagement + +- NEVER ask the operator a question mid-run. Decide based on the algorithm. +- NEVER `git push` to a remote. +- NEVER modify files outside `WORKBENCH` or `references/contexts/` (the research subagent writes context files; the orchestrator never modifies them directly). +- Always commit before exiting (atomic with `analysis.md` write — write analysis, then commit, then exit). +- Cost guard: track cumulative cost from each `run-suite`'s `metrics.cost.total`. +```` + +- [ ] **Step 2: Verify the workflow doc is well-formed markdown** + +```bash +node -e " +const fs = require('fs'); +const content = fs.readFileSync('skills/auto-improve-orchestrator/references/workflow.md', 'utf-8'); +const headings = content.match(/^##? .*/gm) || []; +console.log('heading count:', headings.length); +console.log('first 3:', headings.slice(0, 3)); +if (headings.length < 8) { + console.error('Expected at least 8 headings (one per phase + sub-sections)'); + process.exit(1); +} +console.log('OK'); +" +``` + +Expected: at least 8 headings (phases + sub-sections), `OK` printed. + +- [ ] **Step 3: Commit** + +```bash +git add skills/auto-improve-orchestrator/references/workflow.md +git commit -m "feat(orchestrator): add workflow.md (human-readable algorithm)" +``` + +--- + +## Task 7: Write `prompts/research-upstream.md` + +**Files:** + +- Create: `skills/auto-improve-orchestrator/prompts/research-upstream.md` + +- [ ] **Step 1: Create prompts dir** + +```bash +mkdir -p skills/auto-improve-orchestrator/prompts +``` + +- [ ] **Step 2: Write the research subagent prompt template** + +Create `skills/auto-improve-orchestrator/prompts/research-upstream.md` with this exact content: + +````markdown +# Sub-subagent prompt: research upstream conventions + +You are a research subagent dispatched to study a single upstream +public-skill repo's contribution conventions. You produce a context +file that downstream subagents will use to ensure their proposed +changes fit the upstream's expectations and merge cleanly. + +## Inputs (templated) + +- `${SLUG}` — `//`. Example: `supabase/agent-skills/supabase-postgres-best-practices`. +- `${OUTPUT_PATH}` — where to write the context file. Default: `skills/auto-improve-orchestrator/references/contexts/-.md`. + +## Your job + +Read the target upstream repo's contribution conventions, frontmatter +spec, prefix taxonomy, and merged-PR shape patterns. Write a verbatim- +pastable context block to `${OUTPUT_PATH}` that the orchestrator and +skill-iterate subagents will consume. + +## Method + +Use `gh` CLI heavily (PR list/view, file API, search, repo-files API). +Use `WebFetch` sparingly for any README or external docs (e.g., +`docs..com` if a clear lead suggests external consumption). +Don't clone the repo — use the GitHub API and raw URLs. + +## Questions to answer + +For each, explain in your own words; cite source files/PRs you read. + +1. **On-disk inventory.** What's at `skills//`? List + `SKILL.md` plus every reference file under `references/`. For each + reference: filename, frontmatter values, content type. The + downstream subagents need a complete inventory to pick a non- + colliding `{prefix}` matching the existing taxonomy. + +2. **Frontmatter spec — exact schema.** Read the actual sanity-test + source code (e.g., under `tests/` or `scripts/`) and document the + EXACT required fields, allowed values for each enum field, and any + other validators. Don't assume from prior research — verify. + +3. **Reference file content conventions.** Pick 3 representative + existing references and document their structure: section headers, + code-block language tags, narrative-vs-list ratio, length range. + +4. **Concept-fit assessment.** If a downstream `target_file` doesn't + match the existing template (e.g., a meta-workflow file when all + existing references are single-rule transformations), flag this as + "shape-novel" with a rejection-risk estimate (LOW / MEDIUM / HIGH). + +5. **Prefix taxonomy.** What `{prefix}-` values exist? Are they locked + to a section taxonomy file (e.g., `_sections.md`)? Adding a new + prefix may require modifying that file (which violates additive- + only). + +6. **Recent merged additive PRs.** Look at the last 5–10 merged PRs + that added/modified content for THIS skill (or similar skills if + this one has few). Document: typical file count, body shape, + commit-message convention, time-to-merge, maintainer. + +7. **Closed-without-merge PRs.** Look at the last 3–5 closed PRs + that DIDN'T merge. What was the rejection signal? "Discussion- + first gate violated", "shape-novel", "duplicates X", etc. + +8. **Release Please / version bumping.** Is `metadata.version` in + `SKILL.md` auto-managed by Release Please? If yes, downstream + subagents must NOT manually bump it. + +9. **Architecture intent for SKILL.md vs `references/` split.** Why + split? Token economy? Per-rule contributions? Independent + versioning? The downstream skill-iterate subagent needs to know + whether to add new rules to `SKILL.md` or as a new `references/` + file. + +10. **Other consumers.** Is this skill referenced/installed/fetched by + anything outside the upstream repo? Install scripts, blog posts, + docs sites, downstream forks. Affects how additive-only the + proposed changes must be. + +11. **License + CLA.** What license? Any CLA bot? Affects whether + contributors need extra setup. + +12. **CI gates.** What does CI check? Frontmatter validators, format + checkers, test runners? + +## Output format + +Write `${OUTPUT_PATH}` with this structure: + +```markdown +# Auto-pilot context: / — + +## Repository facts + +- Repo: +- License: , CLA +- Maintainers: +- Merge style: , conventional commits enforced by +- CI: +- Discovery index / downstream sync: + +## Hard constraints (additive-only PR) + +- Add EXACTLY ONE new file at +- DO NOT modify +- Use only existing prefixes: +- DO NOT bump version () +- Other don'ts: + +## Frontmatter spec + +```yaml +--- +: +... +--- +``` + +## Content shape template + +[copy-and-fill template matching upstream's existing references] + +## Optimization target file + +**Edit:** `` +**Do NOT edit:** `` + +## Architecture intent + +[2-3 sentences explaining the upstream's design rationale] + +## Risk profile + +- HIGH/MEDIUM/LOW for + +## Pre-submit checklist + +1. +2. +... + +## Useful URLs + +- +``` + +## Commit + +After writing `${OUTPUT_PATH}`, commit on the current branch: + +```bash +git add ${OUTPUT_PATH} +git commit -m "docs(contexts): research upstream for ${SLUG}" +``` + +DO NOT push. + +## Return report + +Return to caller (orchestrator subagent) under 400 words: + +- On-disk inventory summary (file count + frontmatter overview) +- Frontmatter spec (the exact required fields) +- Content conventions (1 example structure) +- Concept fit (if applicable) +- Prefix recommendation +- Recent PR shape pattern +- Net rejection risk: LOW / MEDIUM / HIGH + rationale +- The verbatim context block path + +If a question genuinely can't be answered from public signals, say so +explicitly. Don't speculate. +```` + +- [ ] **Step 3: Verify the prompt has the expected templated variables** + +```bash +grep -E '\$\{(SLUG|OUTPUT_PATH)\}' \ + skills/auto-improve-orchestrator/prompts/research-upstream.md \ + | wc -l +``` + +Expected: at least 4 occurrences (each var used at least twice). + +- [ ] **Step 4: Commit** + +```bash +git add skills/auto-improve-orchestrator/prompts/research-upstream.md +git commit -m "feat(orchestrator): add research-upstream sub-subagent prompt" +``` + +--- + +## Task 8: Write `prompts/eval-iterate.md` + +**Files:** + +- Create: `skills/auto-improve-orchestrator/prompts/eval-iterate.md` + +- [ ] **Step 1: Write the eval-iterate subagent prompt template** + +Create `skills/auto-improve-orchestrator/prompts/eval-iterate.md` with this exact content: + +````markdown +# Sub-subagent prompt: iterate the eval workbench + +You are a sub-subagent dispatched to make a focused change to one +skill's eval workbench so that downstream skill-iteration has real +headroom (baseline lands in `(0.50, 0.95)`). + +## Inputs (templated) + +- `${SKILL_ID}` — leaf id, e.g. `supabase-postgres-best-practices`. +- `${WORKBENCH_DIR}` — `examples/workbench/${SKILL_ID}/`. +- `${SUITE_RESULT_PATH}` — path to the most recent `.results//suite-result.json`. +- `${DIRECTION}` — one of: + - `add-harder` — add 2-3 new cases that surface absence-type + violations the existing graders don't catch + - `simplify` — remove ambiguous violations from existing workspace + files; tighten task descriptions + - `fix-graders` — apply recipes G1-G6 from `references/lessons.md` + (line-tolerance, fuzzy keywords, etc.) +- `${LESSONS_PATH}` — `skills/auto-improve-orchestrator/references/lessons.md`. Read this for grader-pattern recipes G1-G6 (relevant for `fix-graders`) and the load-bearing prior on absence-type rules (relevant for `add-harder`). + +## What to do (per direction) + +### `add-harder` + +1. Read the current `${SUITE_RESULT_PATH}` to identify which rules are + ALREADY at ceiling (per-case score == 1.00). New cases should + target absence-type variants of these rules that the existing + workspace files don't exercise. +2. Read `${LESSONS_PATH}` § "The load-bearing prior" — absence-type + rules are 5-10× harder than presence-type. New cases should force + enumeration (multi-statement files, mixed correct + incorrect + patterns, invariants that span statements). +3. Write 2-3 NEW workspace files under `${WORKBENCH_DIR}/workspace/`. + Each file = one new case. Realistic content (not contrived). + Lowercase SQL keywords if applicable; semantic table/column names. +4. Write 2-3 NEW grader files under `${WORKBENCH_DIR}/checks/`. Each + grader = one new case. Use `_grader-utils.mjs` helpers + (`looseRange`, `fuzzyKeyword`, `tolerantKeyword`). +5. Update `${WORKBENCH_DIR}/suite.yml` to include the new cases. Don't + touch existing cases. +6. Run a smoke check at `${WORKBENCH_DIR}/checks/smoke-graders.mjs`: + - Hand-craft GOOD `findings.txt` per new grader → assert + `pass=true score=1` + - Hand-craft BAD `findings.txt` (missing 1-2 violations) → assert + `pass=false score<1` + - Hand-craft EMPTY → assert `pass=false score=0` + - Run: `node ${WORKBENCH_DIR}/checks/smoke-graders.mjs` + - All assertions must pass. + +### `simplify` + +1. Read `${SUITE_RESULT_PATH}` to identify which cases are scoring + `< 0.50`. Read failed trials' `findings.txt` to understand WHY: + ambiguous tasks, contrived violations, multiple valid answers. +2. Edit existing workspace files to remove ambiguity (clearer task + description, fewer red-herring statements, tighter line ranges). +3. Update graders if the violations themselves changed. +4. Run the smoke check (same shape as `add-harder`). + +### `fix-graders` + +1. Read failed trials' `findings.txt` (paths in `${SUITE_RESULT_PATH}`'s + `results[].trials[].resultPath`). If models DID find the violations + but the grader scored wrong (line off by ±5, keyword mismatch, + format variant), this is a grader bug. +2. Apply recipes G1-G6 from `${LESSONS_PATH}`: + - G1: widen `looseRange` from default 8 to 10-12 if line drift is + systematic + - G2: replace hand-written keyword regex with `fuzzyKeyword` + - G4: replace `/exact-stem/i` with `tolerantKeyword('stem')` + - G6: split per-finding-line check (don't credit cross-finding keyword matches) +3. Run the smoke check. + +## Tools allowed + +Read, Edit, Write, Bash, Glob. + +## Constraints + +- Additive ONLY for `add-harder` (no changes to existing cases or + graders unless their helpers need extension). +- Edits ONLY for `simplify` and `fix-graders` (no new cases). +- Smoke check MUST pass before commit. +- DO NOT run `npx tsx ../../../src/cli.ts run-suite` (that's the + orchestrator's job; you just modify the workbench). +- DO NOT modify the `references/` (vendored skill content) — those are + fixed inputs. + +## Commit + +After smoke check passes: + +```bash +git add ${WORKBENCH_DIR} +git commit -m "feat(eval): ${DIRECTION} for ${SKILL_ID} (cases: )" +``` + +DO NOT push. + +## Return report + +Return to caller (orchestrator subagent) under 300 words: + +- Direction: `add-harder` | `simplify` | `fix-graders` +- Cases added/modified (names + 1-line description each) +- Predicted baseline impact (rough: "frontier models will likely miss + X% of new checks because Y") +- Smoke-check result (N/N assertions pass) +- Blockers, if any +- Branch + commit SHA +```` + +- [ ] **Step 2: Verify the prompt has expected template variables** + +```bash +grep -E '\$\{(SKILL_ID|WORKBENCH_DIR|SUITE_RESULT_PATH|DIRECTION|LESSONS_PATH)\}' \ + skills/auto-improve-orchestrator/prompts/eval-iterate.md \ + | wc -l +``` + +Expected: at least 8 occurrences (5 vars, most used multiple times). + +- [ ] **Step 3: Commit** + +```bash +git add skills/auto-improve-orchestrator/prompts/eval-iterate.md +git commit -m "feat(orchestrator): add eval-iterate sub-subagent prompt" +``` + +--- + +## Task 9: Write `prompts/skill-iterate.md` + +**Files:** + +- Create: `skills/auto-improve-orchestrator/prompts/skill-iterate.md` + +- [ ] **Step 1: Write the skill-iterate subagent prompt template** + +Create `skills/auto-improve-orchestrator/prompts/skill-iterate.md` with this exact content: + +````markdown +# Sub-subagent prompt: iterate the skill content + +You are a sub-subagent dispatched to apply ONE additive recipe to a +target file in a skill's vendored content, then re-measure to see if +the change moves per-case-min by ≥+0.05. + +## Inputs (templated) + +- `${SKILL_ID}` — leaf id (e.g. `supabase-postgres-best-practices`). +- `${WORKBENCH_DIR}` — `examples/workbench/${SKILL_ID}/`. +- `${SUITE_RESULT_PATH}` — path to the most recent `.results//suite-result.json`. +- `${TARGET_FILE}` — path to the file to edit (extracted from the + `${CONTEXT_FILE}`'s "Optimization target file" directive). Example: + `${WORKBENCH_DIR}/references//.md`. +- `${CONTEXT_FILE}` — `skills/auto-improve-orchestrator/references/contexts/-.md`. +- `${LESSONS_PATH}` — `skills/auto-improve-orchestrator/references/lessons.md`. +- `${ITERATION}` — `1` or `2`. + +## What to do + +1. **Read the inputs.** Read `${SUITE_RESULT_PATH}`, `${CONTEXT_FILE}`, + `${LESSONS_PATH}`, `${TARGET_FILE}`. + +2. **Diagnose missed rules.** From `${SUITE_RESULT_PATH}`, identify: + - Per-case scores (group `results[]` by `caseName`, average trial + scores per case). + - Per-rule miss frequency (read failed trials' `findings.txt`, + identify which violation IDs were missed across trials/models). + - Categorize each missed rule: visible-pattern / absence-of-attribute / + state-machine / subjective. + +3. **Match to a recipe.** From `${LESSONS_PATH}` § "Optimization + patterns": + - **Recipe A** (two-pass workflow) — code-reviewer skills with mixed + presence/absence rules + - **Recipe B** (verify-tool-installed nudge) — tool-use skills where + models fall back to `curl`/`npm i` + - **Recipe C** (per-element checklists) — skills with rules grouped + by element type + - **Recipe D** (BAD/GOOD examples) — anti-patterns where the bad + pattern looks idiomatic + - **Recipe E** (rationale + bug-story) — state-machine violations + + Pick the recipe that best matches the dominant failure mode for + THIS iteration. If iteration 2 and recipe X was tried in iteration + 1 with insufficient uplift, pick a DIFFERENT recipe. + +4. **Apply the recipe ADDITIVELY to `${TARGET_FILE}`.** Read + `${CONTEXT_FILE}`'s "Hard constraints" — your edit must comply + (e.g., additive-only, terse imperative bullets, specific style). + Match the surrounding voice in `${TARGET_FILE}`. + +5. **Re-run the suite from `${WORKBENCH_DIR}`:** + + ```bash + cd ${WORKBENCH_DIR} + set -a; . ../../../.env; set +a + npx tsx ../../../src/cli.ts run-suite ./suite.yml --trials 3 \ + 2>&1 | tee /tmp/skill-iter-${ITERATION}-${SKILL_ID}.log + ``` + + Wait for completion (~30-60 min depending on suite size). + +6. **Compute new per_case_min.** Read the new `.results//suite-result.json`. Group by `caseName`. Per-case mean. Min across cases. + +7. **Compare to baseline.** The orchestrator passed you `${SUITE_RESULT_PATH}` (the previous result). Compute: + - `prev_per_case_min` from `${SUITE_RESULT_PATH}` + - `new_per_case_min` from the new run + - `uplift = new_per_case_min - prev_per_case_min` + +8. **Commit if uplift OR final iteration.** Even if uplift is small, + commit (the orchestrator may want the additive change as + `uplift-too-small` packaging): + + ```bash + git add ${TARGET_FILE} + git commit -m "feat(${SKILL_ID}): iterate ${ITERATION} — Recipe " + ``` + + Skip pushing. + +## Tools allowed + +Read, Edit, Bash, Glob. + +## Constraints + +- Additive ONLY. No deletions, no rewording of existing target-file + content. The `${CONTEXT_FILE}` may specify additional constraints + (e.g., specific frontmatter fields, prefix restrictions); honor all + of them. +- DO NOT touch `${WORKBENCH_DIR}/checks/`, `${WORKBENCH_DIR}/workspace/`, + `${WORKBENCH_DIR}/suite.yml` — those are eval harness, not skill + content. (Eval changes are the eval-iterate subagent's job.) +- DO NOT modify `${LESSONS_PATH}` or other reference material outside + `${TARGET_FILE}`. + +## Cost guard + +If the suite re-run fails or takes longer than the wrapper's default +timeout, exit with status `blocked-by-error` and a brief explanation. + +## Return report + +Return to caller (orchestrator subagent) under 300 words: + +- Recipe applied (A/B/C/D/E) +- Diff summary (what was added, ~lines) +- Per-case scores: prev → new (table) +- per_case_min: prev → new +- uplift: ±N.NN +- Verdict: `success` (uplift ≥ +0.05) | `uplift-too-small` (less) +- Branch + commit SHA + new `.results//` path +```` + +- [ ] **Step 2: Verify template variables** + +```bash +grep -E '\$\{(SKILL_ID|WORKBENCH_DIR|SUITE_RESULT_PATH|TARGET_FILE|CONTEXT_FILE|LESSONS_PATH|ITERATION)\}' \ + skills/auto-improve-orchestrator/prompts/skill-iterate.md \ + | wc -l +``` + +Expected: at least 12 occurrences (7 vars, most used multiple times). + +- [ ] **Step 3: Commit** + +```bash +git add skills/auto-improve-orchestrator/prompts/skill-iterate.md +git commit -m "feat(orchestrator): add skill-iterate sub-subagent prompt" +``` + +--- + +## Task 10: Write `prompts/orchestrator.md` + +**Files:** + +- Create: `skills/auto-improve-orchestrator/prompts/orchestrator.md` + +This is the main orchestrator prompt the operator dispatches via the Agent tool. + +- [ ] **Step 1: Write the orchestrator prompt template** + +Create `skills/auto-improve-orchestrator/prompts/orchestrator.md` with this exact content: + +````markdown +# Orchestrator subagent: end-to-end auto-improve for one skill + +You are dispatched as an autonomous orchestrator for a single public +agent skill: `${SLUG}`. You own this skill end-to-end. You make +decisions about when to research, when to iterate the eval, when to +iterate the skill, and when to package. You are running in your own +git worktree (created by `isolation: "worktree"`). + +## Inputs (templated) + +- `${SLUG}` — `//`. Example: `supabase/agent-skills/supabase-postgres-best-practices`. +- `${MAIN_REPO_PATH}` — absolute path to the operator's main repo (your worktree's parent). Example: `/home/yuqing/Documents/Code/skill-optimizer`. Used for `.env` access. +- `${REFRESH_CONTEXT}` — optional flag, default `false`. If `true`, force re-research even if cached context file exists. + +## Setup + +1. Parse `${SLUG}` into `OWNER`, `REPO`, `SKILL_ID`: + + ```bash + IFS=/ read -r OWNER REPO SKILL_ID <<< "${SLUG}" + ``` + +2. Verify you're in a git worktree (not the main repo): + + ```bash + GIT_DIR=$(cd "$(git rev-parse --git-dir)" 2>/dev/null && pwd -P) + GIT_COMMON=$(cd "$(git rev-parse --git-common-dir)" 2>/dev/null && pwd -P) + if [ "$GIT_DIR" = "$GIT_COMMON" ]; then + echo "ERROR: not running in a worktree. The orchestrator requires isolation:'worktree'." >&2 + exit 1 + fi + ``` + +3. Verify the skill's workbench exists: + + ```bash + WORKBENCH=examples/workbench/${SKILL_ID}/ + if [ ! -d "$WORKBENCH" ]; then + # Write analysis.md with status:blocked-by-missing-workbench, then exit + mkdir -p "$WORKBENCH" + cat > "${WORKBENCH}analysis.md" <&2 + # Write analysis.md with status:blocked-by-error and exit + exit 1 + fi + ``` + +6. Initialize cost tracker: + + ```bash + CUMULATIVE_COST=0 + ``` + +## Phase 0: Research + +```bash +if [ -f "$CONTEXT_FILE" ] && [ "${REFRESH_CONTEXT:-false}" != "true" ]; then + echo "Phase 0: using cached context at $CONTEXT_FILE" +else + echo "Phase 0: dispatching research-upstream subagent for ${SLUG}" + # Dispatch via Agent tool (load skills/auto-improve-orchestrator/prompts/research-upstream.md, + # substitute ${SLUG} and ${OUTPUT_PATH}=${CONTEXT_FILE}). + # Wait for completion. Verify CONTEXT_FILE was written. +fi + +# Read CONTEXT_FILE; extract: target_file, packaging_dir, additive_only_constraint +``` + +When dispatching the research subagent, use: + +- `subagent_type: "general-purpose"` +- `description: "research "` +- (no `isolation`) — research subagent runs in YOUR worktree (writes the context file) +- `prompt`: load `skills/auto-improve-orchestrator/prompts/research-upstream.md` and substitute `${SLUG}` and `${OUTPUT_PATH}` + +## Phase 3: Baseline measurement + +Find the most recent `.results//suite-result.json`: + +```bash +LATEST_RESULTS=$(ls -td ${WORKBENCH}.results/*/ 2>/dev/null | head -1) +LATEST_SUITE_JSON=${LATEST_RESULTS}suite-result.json +``` + +If `LATEST_SUITE_JSON` exists AND its `cases` field matches the current `${WORKBENCH}suite.yml`'s case names: + +- Read it (resume — skip baseline run). + +Else: + +```bash +cd ${WORKBENCH} +set -a; . ${MAIN_REPO_PATH}/.env; set +a +npx tsx ../../../src/cli.ts run-suite ./suite.yml --trials 3 \ + 2>&1 | tee /tmp/orchestrator-baseline-${SKILL_ID}.log +cd - +LATEST_RESULTS=$(ls -td ${WORKBENCH}.results/*/ | head -1) +LATEST_SUITE_JSON=${LATEST_RESULTS}suite-result.json +``` + +Read `LATEST_SUITE_JSON` (use `node -e "console.log(JSON.parse(...))"` or `jq`). + +Compute `per_case_min`: + +```bash +node -e " +const fs = require('fs'); +const data = JSON.parse(fs.readFileSync('$LATEST_SUITE_JSON', 'utf-8')); +const byCase = {}; +for (const r of data.results) { + if (!byCase[r.caseName]) byCase[r.caseName] = []; + byCase[r.caseName].push(r.meanScore); +} +const perCaseMeans = Object.entries(byCase).map(([c, scores]) => ({ + case: c, + mean: scores.reduce((a, b) => a + b, 0) / scores.length, +})); +perCaseMeans.sort((a, b) => a.mean - b.mean); +console.log(JSON.stringify({ + perCaseMin: perCaseMeans[0].mean, + perCaseMinName: perCaseMeans[0].case, + perCaseBreakdown: perCaseMeans, +}, null, 2)); +" +``` + +Track the cost: + +```bash +COST_THIS_RUN=$(node -e " +const data = JSON.parse(require('fs').readFileSync('$LATEST_SUITE_JSON', 'utf-8')); +const sum = (data.metrics?.cost?.total) || 0; +console.log(sum); +") +CUMULATIVE_COST=$(node -e "console.log($CUMULATIVE_COST + $COST_THIS_RUN)") +if (( $(echo "$CUMULATIVE_COST > 10" | bc -l) )); then + echo "ERROR: cumulative cost \$$CUMULATIVE_COST > \$10 hard cap" >&2 + # Write analysis.md status:budget-exceeded and exit + exit 1 +fi +if (( $(echo "$CUMULATIVE_COST > 5" | bc -l) )); then + echo "WARN: cumulative cost \$$CUMULATIVE_COST > \$5 soft warning" >&2 +fi +``` + +## Phase 3.5: Eval-readiness loop + +```bash +EVAL_ITER=0 +while [ "$EVAL_ITER" -lt 3 ]; do + if (( $(echo "$PER_CASE_MIN >= 0.95" | bc -l) )); then + DIRECTION="add-harder" + elif (( $(echo "$PER_CASE_MIN < 0.50" | bc -l) )); then + # Run grader-vs-skill check: read failed trials' findings.txt; + # if models DID find the violations but grader scored wrong, + # DIRECTION="fix-graders"; else "simplify". + DIRECTION="..." + else + break # in (0.50, 0.95) — proceed to skill iteration + fi + + # Dispatch eval-iterate subagent (load prompts/eval-iterate.md, substitute vars) + # Wait. Verify it committed. + + # Re-run baseline (no resume — case set changed) + # Recompute PER_CASE_MIN + EVAL_ITER=$((EVAL_ITER + 1)) +done + +# Exit conditions +if (( $(echo "$PER_CASE_MIN >= 0.95" | bc -l) )); then + # Write analysis.md status:skill-genuinely-good and commit + exit 0 +fi +if (( $(echo "$PER_CASE_MIN < 0.50" | bc -l) )); then + # Write analysis.md status:blocked-by-skill-shape and commit + exit 0 +fi +``` + +When dispatching eval-iterate, use: + +- `subagent_type: "general-purpose"` +- `description: "eval-iterate ${DIRECTION} ${SKILL_ID}"` +- `prompt`: load `prompts/eval-iterate.md` and substitute `${SKILL_ID}`, `${WORKBENCH_DIR}`, `${SUITE_RESULT_PATH}`, `${DIRECTION}`, `${LESSONS_PATH}` + +## Phase 4: Skill iteration + +```bash +BASELINE_PER_CASE_MIN=$PER_CASE_MIN +ITER=0 +SUCCESS=false + +while [ "$ITER" -lt 2 ]; do + ITER=$((ITER + 1)) + + # Dispatch skill-iterate subagent + # Wait. Verify it committed. + + # Find the new latest .results//suite-result.json + # Recompute NEW_PER_CASE_MIN + + UPLIFT=$(node -e "console.log($NEW_PER_CASE_MIN - $BASELINE_PER_CASE_MIN)") + if (( $(echo "$UPLIFT >= 0.05" | bc -l) )); then + SUCCESS=true + break + fi + + # Track cost again — re-run-suite happened + # If cost > 10, exit budget-exceeded +done +``` + +When dispatching skill-iterate: + +- `subagent_type: "general-purpose"` +- `description: "skill-iterate ${ITER} ${SKILL_ID}"` +- `prompt`: load `prompts/skill-iterate.md` and substitute all 7 inputs + +## Phase 5: Package + +If `SUCCESS=true` or `uplift-too-small` (the additive change is still +worth packaging): + +```bash +# Read CONTEXT_FILE for packaging_dir +PACKAGING_DIR="${WORKBENCH}proposed-upstream-changes/${PACKAGING_DIR_FROM_CONTEXT}" +mkdir -p "$PACKAGING_DIR" + +# Read the original target file (before any v1.3 mods). +# This is the version on the FIRST commit on this orchestrator's branch. +git show HEAD~${ITER}:${TARGET_FILE} > "${PACKAGING_DIR}/before-$(basename ${TARGET_FILE})" + +# Current target file is the after version +cp "${TARGET_FILE}" "${PACKAGING_DIR}/after-$(basename ${TARGET_FILE})" + +# Write a packaging README +cat > "${PACKAGING_DIR}/README.md" < +baseline_per_case_min: 0.NN +final_per_case_min: 0.NN +iterations: eval=N, skill=N +total_cost_usd: NN.NN +--- + +# Auto-pilot run for ${SLUG} + +[3-6 short bullets covering: what was done, why, evidence, judgment calls] +``` + +## Phase 6: Final commit + +```bash +git checkout -b eval/auto-pilot/${SKILL_ID} 2>/dev/null || git checkout eval/auto-pilot/${SKILL_ID} +git add ${WORKBENCH}analysis.md +[ -d "$PACKAGING_DIR" ] && git add "$PACKAGING_DIR" +git add ${LATEST_RESULTS}suite-result.json +git commit -m "eval(auto-pilot): ${SKILL_ID} — status=$STATUS, baseline=$BASELINE_PER_CASE_MIN→$PER_CASE_MIN" +``` + +DO NOT push. + +## Return summary + +Return to caller (operator's CC session) under 300 words: + +- Branch: `eval/auto-pilot/${SKILL_ID}` +- Final commit SHA: `` +- Status: one of the exit statuses +- Baseline → final per-case-min +- Per-case breakdown (table) +- Proposed-upstream-changes path (if applicable) +- Cumulative cost: $N.NN + +## Hard rules + +- NEVER ask the operator a question mid-run. +- NEVER `git push`. +- NEVER modify files outside `${WORKBENCH}` or `${CONTEXT_FILE}`. +- Always commit before exiting (even on error: write `analysis.md` with the right status, then commit). +- Cost guard: hard stop at $10 cumulative `metrics.cost.total`. +```` + +- [ ] **Step 2: Verify template variables** + +```bash +grep -E '\$\{(SLUG|MAIN_REPO_PATH|REFRESH_CONTEXT)\}' \ + skills/auto-improve-orchestrator/prompts/orchestrator.md \ + | wc -l +``` + +Expected: at least 5 occurrences. + +- [ ] **Step 3: Commit** + +```bash +git add skills/auto-improve-orchestrator/prompts/orchestrator.md +git commit -m "feat(orchestrator): add orchestrator main prompt template" +``` + +--- + +## Task 11: Smoke validation script + +Validates that the new skill files are well-formed before any end-to-end test. + +**Files:** + +- Create: `skills/auto-improve-orchestrator/.smoke-check.mjs` + +- [ ] **Step 1: Write the smoke check** + +Create `skills/auto-improve-orchestrator/.smoke-check.mjs` with this exact content: + +```js +#!/usr/bin/env node +// Smoke check for the auto-improve-orchestrator skill. +// Validates: SKILL.md frontmatter, prompt template variables, file existence. + +import { readFileSync, existsSync } from 'node:fs'; +import matter from 'gray-matter'; + +const skillRoot = 'skills/auto-improve-orchestrator'; +let failures = 0; + +function check(condition, msg) { + if (condition) { + console.log(`OK: ${msg}`); + } else { + console.error(`FAIL: ${msg}`); + failures++; + } +} + +// 1. SKILL.md exists + frontmatter parses +const skillMdPath = `${skillRoot}/SKILL.md`; +check(existsSync(skillMdPath), `${skillMdPath} exists`); +if (existsSync(skillMdPath)) { + const parsed = matter(readFileSync(skillMdPath, 'utf-8')); + check(parsed.data.name === 'auto-improve-orchestrator', 'SKILL.md name = "auto-improve-orchestrator"'); + check(typeof parsed.data.description === 'string' && parsed.data.description.length > 50, 'SKILL.md description is non-trivial'); +} + +// 2. All four prompt files exist +for (const name of ['orchestrator.md', 'research-upstream.md', 'eval-iterate.md', 'skill-iterate.md']) { + check(existsSync(`${skillRoot}/prompts/${name}`), `prompts/${name} exists`); +} + +// 3. workflow.md + lessons.md + at least one context exist +check(existsSync(`${skillRoot}/references/workflow.md`), 'references/workflow.md exists'); +check(existsSync(`${skillRoot}/references/lessons.md`), 'references/lessons.md exists'); +check(existsSync(`${skillRoot}/references/contexts`), 'references/contexts/ exists'); + +// 4. Each prompt has its expected templated variables +const expectedVars = { + 'orchestrator.md': ['SLUG', 'MAIN_REPO_PATH'], + 'research-upstream.md': ['SLUG', 'OUTPUT_PATH'], + 'eval-iterate.md': ['SKILL_ID', 'WORKBENCH_DIR', 'SUITE_RESULT_PATH', 'DIRECTION', 'LESSONS_PATH'], + 'skill-iterate.md': ['SKILL_ID', 'WORKBENCH_DIR', 'SUITE_RESULT_PATH', 'TARGET_FILE', 'CONTEXT_FILE', 'LESSONS_PATH', 'ITERATION'], +}; +for (const [file, vars] of Object.entries(expectedVars)) { + const content = readFileSync(`${skillRoot}/prompts/${file}`, 'utf-8'); + for (const v of vars) { + check(content.includes(`\${${v}}`), `prompts/${file} contains \${${v}}`); + } +} + +// 5. Old wrapper files are gone +check(!existsSync('tools/auto-improve-skill.mjs'), 'tools/auto-improve-skill.mjs is gone'); +check(!existsSync('tools/auto-improve-skill-prompt.md'), 'tools/auto-improve-skill-prompt.md is gone'); +check(!existsSync('tools/auto-improve-skill-lessons.md'), 'tools/auto-improve-skill-lessons.md moved out of tools/'); +check(!existsSync('tools/auto-improve-contexts'), 'tools/auto-improve-contexts/ moved out of tools/'); + +// 6. Lessons + contexts at new paths +check(existsSync(`${skillRoot}/references/lessons.md`), 'lessons at new path'); +check(existsSync(`${skillRoot}/references/contexts/supabase-postgres-best-practices.md`), 'supabase context at new path'); + +if (failures > 0) { + console.error(`\n${failures} smoke checks failed`); + process.exit(1); +} +console.log(`\nAll smoke checks passed`); +``` + +- [ ] **Step 2: Run the smoke check** + +```bash +node skills/auto-improve-orchestrator/.smoke-check.mjs +``` + +Expected: all checks pass, final line `All smoke checks passed`. Exit code 0. + +- [ ] **Step 3: Commit the smoke script** + +```bash +git add skills/auto-improve-orchestrator/.smoke-check.mjs +git commit -m "test(orchestrator): smoke validation script for skill structure" +``` + +--- + +## Task 12: End-to-end validation on supabase + +Validates that the orchestrator subagent actually works by dispatching it on the supabase skill (which has an existing workbench at `examples/workbench/supabase-postgres-best-practices/` from this session's prior work). + +This task is partially manual — the operator's CC session does the dispatch. The plan specifies what to dispatch and how to verify. + +- [ ] **Step 1: Verify pre-conditions for the test run** + +```bash +# Workbench exists (from supabase-pilot-v2 worktree's prior commits) +ls examples/workbench/supabase-postgres-best-practices/suite.yml 2>&1 || echo "MISSING — bring in from another worktree" + +# Context file exists (already there from v1.2.1 work) +ls skills/auto-improve-orchestrator/references/contexts/supabase-postgres-best-practices.md + +# Recent suite-result.json exists (resume should kick in) +ls examples/workbench/supabase-postgres-best-practices/.results/*/suite-result.json 2>&1 | head -1 +``` + +If the workbench is missing on `feat/auto-improve-skill-v1.3`, cherry-pick from the supabase-pilot-v2 worktree's branch: + +```bash +# From the v1.3-impl worktree: +git checkout eval/auto-pilot/supabase-postgres-best-practices-v2 -- \ + examples/workbench/supabase-postgres-best-practices/ +git commit -m "test(e2e): import supabase workbench for v1.3 validation" +``` + +- [ ] **Step 2: Dispatch the orchestrator subagent (operator's CC session does this)** + +The operator dispatches via the Agent tool: + +``` +Agent({ + description: "auto-improve supabase-postgres-best-practices", + subagent_type: "general-purpose", + isolation: "worktree", + run_in_background: true, + prompt: +}) +``` + +- [ ] **Step 3: Wait for orchestrator completion (long-running, 30-60 min)** + +The orchestrator runs in its own worktree. You'll get a notification when it completes. + +Expected behavior: + +- **Phase 0:** finds existing context file at `skills/auto-improve-orchestrator/references/contexts/supabase-postgres-best-practices.md` — no re-research. +- **Phase 3:** finds existing recent `suite-result.json` from v1.2.1 v2 work — RESUMES (no re-run). +- **Phase 3.5:** computes `per_case_min` from the existing results. The supabase v2 baseline had `update-without-where` at 77.8% (per the spec § "Validation"), so `per_case_min ≈ 0.78`. This is in `(0.50, 0.95)` — NO eval iteration needed; proceed to Phase 4. +- **Phase 4:** dispatches `skill-iterate` subagent with `target_file = monitor-two-pass-review.md` (from context). The subagent applies a recipe additively, re-runs the suite, computes uplift. +- **Phase 5/6:** if uplift ≥ +0.05 on `update-without-where`, packages + commits as `success`. Otherwise commits as `uplift-too-small`. + +- [ ] **Step 4: Verify the orchestrator's output** + +After completion, check: + +```bash +# A new branch exists +git branch | grep eval/auto-pilot/supabase-postgres-best-practices + +# A commit on it +git log -1 eval/auto-pilot/supabase-postgres-best-practices --oneline + +# analysis.md has a real status (not "pending") +git show eval/auto-pilot/supabase-postgres-best-practices:examples/workbench/supabase-postgres-best-practices/analysis.md | head -10 +``` + +Expected: branch exists, commit message includes status, analysis frontmatter has `status: success` or `status: uplift-too-small` (not `pending` or `blocked-by-error`). + +- [ ] **Step 5: If validation passed, commit a note** + +```bash +cat > docs/auto-improve-skill-v1.3-validation.md <<'EOF' +# v1.3 validation result + +Date: 2026-05-12 +Skill tested: supabase/agent-skills/supabase-postgres-best-practices + +The orchestrator subagent was dispatched on the existing supabase +workbench. Result: , baseline → final . + +The dispatched orchestrator: +- Found cached context — no re-research. +- Found recent suite-result — resumed from cached baseline. +- per_case_min was X.XX → triggered Phase 4 skill iteration (or: + was at ceiling/floor, triggered eval iteration). +- skill-iterate subagent applied Recipe ; uplift was ±N.NN. +- Packaged proposed-upstream-changes/ at . + +Branch: eval/auto-pilot/supabase-postgres-best-practices (commit ). + +v1.3 is validated end-to-end. +EOF + +git add docs/auto-improve-skill-v1.3-validation.md +git commit -m "test(e2e): document v1.3 validation result on supabase" +``` + +(Fill in the actual values after the orchestrator completes — this template is what gets committed at the end.) + +--- + +## Self-Review + +After writing the plan, I checked it against the spec for coverage, placeholders, and consistency. + +**Spec coverage:** + +- §"Architecture overview" — Tasks 1-5 (deletions/moves) and 5-10 (new skill files) cover all listed deliverables. ✓ +- §"Sub-subagent designs" — Tasks 7, 8, 9 implement the three sub-subagent prompts with the inputs/outputs specified in the spec. ✓ +- §"Orchestrator algorithm" — Task 10 embeds the algorithm. ✓ +- §"Parallelism guarantees" — Worktree isolation is enforced by Task 10's setup check (steps 2 of orchestrator.md content). ✓ +- §"Cost model" — Task 10's orchestrator template includes cost tracking with $5 warn / $10 hard stop. ✓ +- §"Cleanup of v1.2.1 artifacts" — Tasks 1-4 do all the deletes/moves/CLAUDE.md update. ✓ +- §"Validation / acceptance criteria" — Tasks 11-12 cover validation #1-#5. ✓ + +**Placeholder scan:** No "TBD", "TODO", "fill in details", or vague-error-handling instructions in any task. Each step has either complete code or an exact bash command. ✓ + +**Type / path consistency:** + +- `skills/auto-improve-orchestrator/` is the new skill root, used identically across all tasks. ✓ +- `${SKILL_ID}`, `${SLUG}`, `${OWNER}`, `${REPO}` follow the same parsing convention everywhere. ✓ +- Sub-subagent input names match between Task 8/9/10 (e.g., `${WORKBENCH_DIR}` is the same in all places). ✓ +- `per_case_min` is the consistent metric throughout (not aliased as `perCaseMinimum` or `min_score`). ✓ + +No issues found. + +--- + +## Execution Handoff + +Plan complete and saved to `docs/auto-improve-skill-v1.3-plan.md` (in the `feat/auto-improve-skill-v1.3` worktree at `.claude/worktrees/v1.3-impl/`). + +Two execution options: + +1. **Subagent-Driven (recommended)** — I dispatch a fresh subagent per task, review between tasks, fast iteration. Each task is self-contained enough to be one focused subagent. + +2. **Inline Execution** — Execute tasks in this session using executing-plans, batch execution with checkpoints for review. + +Which approach? diff --git a/docs/auto-improve-skill-v1.3-spec.md b/docs/auto-improve-skill-v1.3-spec.md new file mode 100644 index 0000000..5046c8b --- /dev/null +++ b/docs/auto-improve-skill-v1.3-spec.md @@ -0,0 +1,407 @@ +# auto-improve-orchestrator v1.3 — design spec + +**Status:** approved (brainstormed 2026-05-12) +**Replaces:** the v1.2.1 wrapper (`tools/auto-improve-skill.mjs`) and its +embedded 5-phase prompt template. +**Empirical basis:** four pilots run in the v1.2.1 PR-prep session +(web-design-guidelines, agent-browser, supabase v1+v2). The lessons +distilled from those pilots are documented in `docs/auto-improve-skill-v1.3-design.md` +(the design draft preceding this spec). + +## Goal + +Convert the auto-improve-skill workflow from a wrapper-spawned `claude -p` +autonomous pilot into a Claude Code skill (`auto-improve-orchestrator`) +that an operator's CC session invokes via the Agent tool. Each +orchestrator subagent owns one skill end-to-end, runs in its own +worktree, and dispatches sub-subagents for research / eval-iteration / +skill-iteration tasks. Multiple orchestrators can run in parallel for +batch operation. + +The orchestrator subagent layer captures the operator's workflow logic +(when to research, when to iterate eval, when to iterate skill, when +to package). The skill-optimizer itself stays lean — it ships the +eval engine (`run-suite`, `run-case`, graders, Docker harness) and +nothing else. + +## Motivation + +Two structural lessons from v1.2.1 motivate this design (full evidence +in `docs/auto-improve-skill-v1.3-design.md`): + +1. **Research-first context is mandatory.** The auto-pilot is good at + finding what to change but bad at fitting upstream conventions + (frontmatter schemas, file-location norms, prefix taxonomies). + Without an upstream-research-derived context file, output requires + manual reformulation before submission. +2. **Two-loop iteration on eval AND skill.** v1.2.1 only iterates the + skill (Phase 4). When baseline saturates at the ceiling (≥0.95) or + floors (<0.50), the optimizer can't escape — we manually built + harder/simpler eval suites via subagent dispatches twice this + session. + +Plus two implementation-detail bugs found during v1.2.1 work: + +1. **Per-case-minimum threshold.** The auto-pilot's `baseline ≥ 0.95 + → exit success` logic uses the OVERALL average, masking weak cases. + Supabase v2 had `update-without-where` at 77.8% but exited because + overall was 0.97. +2. **Resume-on-timeout.** When the wrapper's 90-min hard cap killed + the agent-browser pilot mid-baseline (50/54 trials done), 30+ min + of model work was discarded because nothing knew to pick up where + it left off. + +In v1.3's orchestrated architecture, all four are addressed naturally: +research becomes a sub-subagent, eval-iteration becomes a sub-subagent, +per-case-min is computed by the orchestrator from `suite-result.json`, +and resume-on-timeout falls out for free because every phase's +artifacts are persistent on disk and the orchestrator is resume-aware. + +## Architecture overview + +**Skill-optimizer (the tool) ships:** + +- `src/cli.ts`, `src/workbench/` — the eval engine (`run-suite`, + `run-case`, Docker harness, graders, `suite-result.json` aggregation) +- `examples/workbench/` — example evals +- `skills/skill-optimizer/SKILL.md` — the existing canonical skill +- `skills/auto-improve-orchestrator/` — **NEW** Claude Code skill + (the v1.3 deliverable) +- `docker/`, plugin manifests — unchanged + +**Skill-optimizer no longer ships:** + +- `tools/auto-improve-skill.mjs` — DELETED +- `tools/auto-improve-skill-prompt.md` — DELETED + +**Skill-optimizer file moves:** + +- `tools/auto-improve-skill-lessons.md` → `skills/auto-improve-orchestrator/references/lessons.md` +- `tools/auto-improve-contexts/` → `skills/auto-improve-orchestrator/references/contexts/` + +**The new skill `skills/auto-improve-orchestrator/`:** + +```text +skills/auto-improve-orchestrator/ + SKILL.md # discovery + invocation guide + prompts/ + orchestrator.md # orchestrator subagent prompt template + # (operator dispatches this via Agent tool) + research-upstream.md # sub-subagent prompt template (Phase 0) + eval-iterate.md # sub-subagent prompt template (Phase 3.5) + skill-iterate.md # sub-subagent prompt template (Phase 4) + references/ + workflow.md # human-readable doc of the orchestration algorithm + lessons.md # recipes A-E + grader patterns G1-G6 + # (read by skill-iterate sub-subagent) + contexts/ # per-skill context library + vercel-web-interface-guidelines.md + vercel-agent-browser.md + supabase-postgres-best-practices.md + ... +``` + +## Invocation flow + +```text +Operator (in their CC session): + "auto-improve supabase/agent-skills/supabase-postgres-best-practices" + (or for batch: "auto-improve all 10 skills from the top-N list") + +My CC session: + Read skills/auto-improve-orchestrator/SKILL.md to confirm invocation pattern. + For each skill in the request, dispatch one Agent call: + Agent({ + description: "auto-improve ", + subagent_type: "general-purpose", + isolation: "worktree", # auto-creates per-orchestrator worktree + run_in_background: true, + prompt: + }) + All N dispatches happen in a single message → parallel execution. + +Each orchestrator subagent: + 1. Owns its own worktree (isolation: "worktree") + 2. Owns its own skill end-to-end + 3. Follows the algorithm in its loaded prompt (workflow.md mirror) + 4. Dispatches its own sub-subagents for research / eval / skill iteration + 5. Commits to its own branch (eval/auto-pilot/) + 6. Returns final summary report + +My CC session: + Receives N completion notifications (async). + Reports each result to operator with branch name + summary. +``` + +## Sub-subagent designs + +### `prompts/research-upstream.md` + +- **Inputs (templated):** `${SLUG}` (`//`) +- **Tools:** Bash (gh CLI), WebFetch, Read, Write, Glob +- **What it does:** + - Reads CONTRIBUTING.md, AGENTS.md, .github/workflows/*.yml, CODEOWNERS + - Reads skill-specific convention files (`_contributing.md`, + `_template.md`, `_sections.md` if present) + - Reads sanity-test source code (don't trust prior assumptions about + what CI validates) + - Samples last 10 merged PRs to the target skill (or repo) for shape + - Samples last 5 closed-without-merge PRs for rejection signals + - Identifies other consumers (gh search for raw URL refs, install + scripts, repo's own README for distribution channels) +- **Output:** Writes + `skills/auto-improve-orchestrator/references/contexts/-.md` + with verbatim-pastable context block. Returns under-400-word report + covering: target file, risk profile, frontmatter spec, content shape + template, pre-submit checklist. +- **Commit:** `docs(contexts): research upstream for ` on the + orchestrator's worktree branch. + +### `prompts/eval-iterate.md` + +- **Inputs (templated):** `${SKILL_ID}`, `${SUITE_RESULT_PATH}`, + `${DIRECTION}` ∈ {`add-harder`, `simplify`, `fix-graders`} +- **Tools:** Read, Edit, Write, Bash, Glob +- **What it does:** + - `add-harder`: writes 2–3 new workspace files seeded with + absence-type violations the existing graders don't catch; writes + corresponding graders; updates `suite.yml`; runs smoke check + - `simplify`: removes ambiguous violations from existing workspace + files; tightens task descriptions; runs smoke check + - `fix-graders`: applies recipes G1–G6 from `references/lessons.md` + to grader files (line-tolerance, fuzzy keywords); runs smoke check +- **Output:** modified workbench files + smoke-check verification. + Returns under-300-word report: cases added/changed, predicted + baseline impact, blockers. +- **Commit:** `feat(eval): for (cases: )` + on the orchestrator's worktree branch. + +### `prompts/skill-iterate.md` + +- **Inputs (templated):** `${SKILL_ID}`, `${SUITE_RESULT_PATH}`, + `${TARGET_FILE}` (from context's "optimization target" directive), + `${ITERATION}` (1 or 2) +- **Tools:** Read, Edit, Bash, Glob +- **What it does:** + - Diagnoses missed rules from `suite-result.json` (per-case, per-rule) + - Reads `references/lessons.md` and + `references/contexts/-.md` + - Applies one recipe (A/B/C/D/E) to the target file — additive only + - Runs `npx tsx src/cli.ts run-suite ./suite.yml --trials 3` to + re-measure + - Computes uplift using **per-case-minimum**, not overall mean +- **Output:** modified target file + new `.results//` dir. Returns + under-300-word report: which recipe applied, diff summary, baseline + → final per-case-min, success/uplift-too-small verdict. +- **Commit:** `feat(): iterate — ` on the + orchestrator's worktree branch. + +## Orchestrator algorithm (`prompts/orchestrator.md`) + +The orchestrator subagent's prompt embeds this algorithm. Resume-aware +at every phase. + +```text +ORCHESTRATOR(slug, options): + + ────── Setup ────── + Parse SLUG = // + workbench_dir = examples/workbench// + context_file = skills/auto-improve-orchestrator/references/contexts/-.md + + Assert workbench_dir exists. + (v1.3 scope: initial workbench is operator-built; orchestrator + only iterates existing ones. workbench-build sub-subagent is + out of scope — see "Out of scope" below.) + + ────── Phase 0: Research ────── + IF context_file exists AND not --refresh-context: + use it + ELSE: + dispatch prompts/research-upstream.md with SLUG + → writes context_file, commits + Read context_file. Extract: target_file, packaging_dir, + additive_only_constraint. + + ────── Phase 3: Baseline ────── + latest_results = most recent .results//suite-result.json in + workbench_dir + IF latest_results exists AND its case set matches current suite.yml: + read it (resume — no re-run needed) + ELSE: + run `npx tsx src/cli.ts run-suite ./suite.yml --trials 3` + read the new suite-result.json + + ────── Phase 3.5: Eval-readiness loop ────── + per_case_min = min(per-case mean scores in suite-result.json results[]) + eval_iter = 0 + + WHILE per_case_min OUTSIDE (0.50, 0.95) AND eval_iter < 3: + IF per_case_min >= 0.95: + direction = "add-harder" + ELSE IF per_case_min < 0.50: + direction = "fix-graders" if grader-vs-skill check shows grader bug + else "simplify" + dispatch prompts/eval-iterate.md with (SKILL_ID, suite-result path, + direction) + → modifies workbench (new cases or grader fixes), commits + re-run `run-suite`, read new suite-result.json + per_case_min = recompute + eval_iter += 1 + + IF per_case_min still >= 0.95 after 3 iterations: + exit "skill-genuinely-good" — no PR proposed + IF per_case_min still < 0.50 after 3 iterations: + exit "blocked-by-skill-shape" + + ────── Phase 4: Skill iteration ────── + baseline_per_case_min = per_case_min + iter = 0 + + WHILE iter < 2: + iter += 1 + dispatch prompts/skill-iterate.md with (SKILL_ID, suite-result path, + target_file from context, + iter) + → applies one recipe (A-E) additively to target_file, re-runs + suite, commits if uplift, returns report with new per-case-min + new_per_case_min = read new suite-result.json + IF new_per_case_min - baseline_per_case_min >= 0.05: + success — break + + IF no iteration cleared +0.05: + exit "uplift-too-small" + + ────── Phase 5: Package ────── + Compose workbench_dir/proposed-upstream-changes// + - before- + - after- + - README.md (description + evidence + per-case breakdown) + Write workbench_dir/analysis.md with full report + Commit on a branch eval/auto-pilot/ + + ────── Done ────── + Return summary to caller (my CC session): branch name, commit SHA, + baseline → final per-case-min, per-case breakdown, draft PR location, + exit status. +``` + +## Parallelism guarantees + +| Concern | Mitigation | +|---|---| +| File conflicts | Each orchestrator gets own worktree (`isolation: "worktree"`) | +| Git ref races | Different branches per skill (`eval/auto-pilot/`); ref updates atomic | +| OpenRouter rate limits | Each `run-suite` may queue; observable but not a correctness issue | +| Docker resource pressure | Each `run-suite` spawns containers; OS handles; soft cap recommendation: max 5 concurrent orchestrators if running on a typical dev machine | +| `lessons.md` updates from siblings | Each writes to its OWN worktree's lessons.md; operator merges manually post-batch (or skip lessons-update if running in parallel mode) | +| Context-file conflicts | Different skill = different context file; no conflict | + +## Cost model + +- Sub-subagent dispatches (research / eval-iterate / skill-iterate) + use the operator's Claude Code session under their plan (Opus Max, + zero marginal cost). +- `run-suite` calls use OpenRouter for the trial models — paid. +- Per-orchestrator soft warning at $5 cumulative `metrics.cost.total`, + hard stop at $10. Orchestrator tracks the running total across all + `run-suite` invocations in its lifecycle. +- Batch of N orchestrators: total OpenRouter cost ≤ N × $10. Operator + visibility via per-orchestrator summary reports. + +## Cleanup of v1.2.1 artifacts + +**Files to delete (with `git rm`):** + +- `tools/auto-improve-skill.mjs` +- `tools/auto-improve-skill-prompt.md` + +**Files to move (with `git mv`):** + +- `tools/auto-improve-skill-lessons.md` → + `skills/auto-improve-orchestrator/references/lessons.md` +- `tools/auto-improve-contexts/` (entire dir) → + `skills/auto-improve-orchestrator/references/contexts/` + +**Files to update:** + +- `CLAUDE.md` — remove references to the old wrapper, add a brief note + pointing at `skills/auto-improve-orchestrator/SKILL.md` for the new + workflow. + +## Validation / acceptance criteria + +For v1.3 to be considered done: + +1. **Skill is discoverable.** `skills/auto-improve-orchestrator/SKILL.md` + exists with valid frontmatter and is invocable via the Skill tool. +2. **Sub-subagent prompts are self-contained.** Each prompt template + in `prompts/` runs cleanly when dispatched via Agent tool with the + templated inputs substituted. +3. **End-to-end test on a real skill.** Re-run #4 supabase + (`supabase/agent-skills/supabase-postgres-best-practices`) using + the new orchestrator. Expected outcome: + - Phase 0 finds existing context file (already on disk from v1.2.1 + work) — no re-research + - Phase 3 reads existing `.results/` from `supabase-pilot-v2` + worktree's recent baseline — no re-run (resume) + - Phase 3.5 detects per-case-min at 0.778 (the + `update-without-where` case), proceeds to skill iteration + - Phase 4 dispatches skill-iterate sub-subagent which adds an + additive Recipe-D BAD/GOOD example to the existing + `monitor-two-pass-review.md` reference + - Re-runs suite, computes uplift on the weak case + - Packages + commits +4. **Cleanup is complete.** `tools/auto-improve-skill.mjs` and + `tools/auto-improve-skill-prompt.md` are gone; lessons.md and + contexts/ are at the new paths; CLAUDE.md updated. +5. **Documentation matches reality.** `docs/auto-improve-skill-v1.3-design.md` + (the predecessor design doc) gets a note pointing at this spec. + This spec is committed. + +## Out of scope (deferred to v1.4 or later) + +- **Workbench-build sub-subagent.** v1.3 assumes the workbench at + `examples/workbench//` already exists. Building initial + workbenches from scratch (as the v1.2.1 wrapper's Phase 2 did) is + deferred — the v1.2.1 work this session built four workbenches + manually + via subagents and that's a reasonable starting library. + When operators want to auto-pilot a NEW skill (not in + `examples/workbench/`), they manually build the workbench first + (or invoke a future `workbench-build.md` sub-subagent). +- **Autonomous batch wrapper.** No CI-style "fire 10 pilots from + command line, walk away, come back to results" mode. The v1.3 model + requires an operator's CC session to dispatch orchestrators. If a + CI use case emerges, add a thin `tools/auto-improve-batch.mjs` later + that just dispatches via the Agent SDK programmatically. +- **PR submission automation.** v1.3 stops at packaging + (proposed-upstream-changes/ + analysis.md + commit). The fork-clone- + push-create-PR step remains operator-driven per v1.2.1 convention. +- **Skill-classification-aware sub-subagent prompts.** All three + sub-subagents use uniform prompts for all skill types. If empirical + evidence shows skill-type branching is needed (e.g. tool-use evals + need different "harder case" templates than code-reviewer evals), + add classification branching in v1.4. + +## Open questions (deferred but worth tracking) + +1. **Lessons.md merge strategy** when multiple parallel orchestrators + want to append run-record entries. v1.3: each writes to its own + worktree; operator merges post-batch. v1.4 might want a structured + append-only log. +2. **Cost ceiling escalation.** Should the orchestrator pause at $7.50 + and ping the operator for confirmation before continuing? v1.3: + no, just hard-stop at $10 and exit. +3. **Sub-subagent retry on transient failures.** v1.3: if a + sub-subagent fails (e.g., `gh` CLI unavailable, OpenRouter outage), + the orchestrator surfaces the failure and exits. No automatic + retry. v1.4 might add bounded retry. + +## Provenance + +- Predecessor design draft: + [`docs/auto-improve-skill-v1.3-design.md`](../../auto-improve-skill-v1.3-design.md) +- Brainstorming session: 2026-05-12 (this spec is the output) +- Empirical basis: v1.2.1 PR-prep session (4 pilots) and the lessons + doc at `tools/auto-improve-skill-lessons.md` (to be moved to + `references/lessons.md`) diff --git a/docs/auto-improve-skill-v1.3-validation.md b/docs/auto-improve-skill-v1.3-validation.md new file mode 100644 index 0000000..5c7b128 --- /dev/null +++ b/docs/auto-improve-skill-v1.3-validation.md @@ -0,0 +1,92 @@ +# v1.3 validation — deferred to operator + +**Date:** 2026-05-12 +**Status:** implementation complete, end-to-end orchestrator dispatch +deferred to operator's next session. + +## What's already validated (in this implementation session) + +The smoke check at `skills/auto-improve-orchestrator/.smoke-check.mjs` +passes 36/36 structural checks: + +- `SKILL.md` frontmatter parses (gray-matter); `name` and `description` present +- All 4 prompt files exist in `prompts/` (orchestrator + 3 sub-subagents) +- All required template variables present in each prompt +- `references/workflow.md`, `references/lessons.md`, `references/contexts/` exist +- Old wrapper files (`tools/auto-improve-skill.mjs`, + `tools/auto-improve-skill-prompt.md`) are deleted +- Lessons + contexts at new paths (moved from `tools/`) + +`workflow.md` has 14 section headings (Setup, Phase 0, Phase 3, 3.5, +4, 5, 6, Cost tracking, Return summary, Rules of engagement, plus +sub-sections). + +The orchestrator prompt (`prompts/orchestrator.md`, 321 lines) embeds +the full algorithm with all 7 phases and dispatch instructions for +sub-subagents. + +## What's deferred (operator-driven, ~30-60 min wall-clock) + +End-to-end dispatch of the orchestrator subagent on the supabase +skill (re-run of #4 from the v1.2.1 PR-prep session). The supabase +workbench has been imported into this branch +(`examples/workbench/supabase-postgres-best-practices/`) and the +context file is at the new path. The operator can dispatch: + +``` +Agent({ + description: "auto-improve supabase-postgres-best-practices", + subagent_type: "general-purpose", + isolation: "worktree", + run_in_background: true, + prompt: +}) +``` + +### Expected behavior + +- **Phase 0:** finds existing context file at `skills/auto-improve-orchestrator/references/contexts/supabase-postgres-best-practices.md` — no re-research dispatched. +- **Phase 3:** no `.results/` exists (gitignored, didn't carry over from the v2 worktree). Runs fresh baseline. Cost: ~$3 (matching the v2 pilot's prior `$3.15` for 45 trials). +- **Phase 3.5:** computes per-case-min from the new baseline. Expect close to the v2 result (overall ~0.97, with `update-without-where` case at ~0.78). Per-case-min ≈ 0.78 — in `(0.50, 0.95)`, NO eval iteration; proceed to skill iteration. +- **Phase 4:** dispatches `skill-iterate` sub-subagent with target file `references/supabase-postgres-best-practices/references/monitor-two-pass-review.md` (from context). The sub-subagent applies a recipe (likely Recipe D — BAD/GOOD example for `update-without-where`), re-runs the suite (~$3 more), computes uplift on the weak case. +- **Phase 5/6:** packages + commits to `eval/auto-pilot/supabase-postgres-best-practices` branch with status `success` (if uplift ≥ +0.05 on the weak case) or `uplift-too-small` (otherwise). Total cost: ~$6. + +### Acceptance criteria + +After the orchestrator returns: + +```bash +# A new branch exists +git branch | grep eval/auto-pilot/supabase-postgres-best-practices + +# A commit on it +git log -1 eval/auto-pilot/supabase-postgres-best-practices --oneline + +# analysis.md has a real status +git show eval/auto-pilot/supabase-postgres-best-practices:examples/workbench/supabase-postgres-best-practices/analysis.md | head -10 +``` + +Expected: branch exists, commit message includes status, analysis +frontmatter has `status: success` or `status: uplift-too-small` +(NOT `pending` or `blocked-by-error`). + +If validation passes, append to this file with the actual numbers. +If validation reveals a bug in the orchestrator, file an issue and +patch. + +## Why deferred + +The v1.3 implementation session this validates was already long +(brainstorm + spec + plan + 11 implementation tasks) and the operator +had several other background pilots in flight. Dispatching the +orchestrator end-to-end requires: + +1. Switching the operator's main worktree to `feat/auto-improve-skill-v1.3` (so the dispatched orchestrator's auto-created worktree branches from a tip that has the v1.3 prompts), OR +2. Operator can fire from any worktree on the `feat/auto-improve-skill-v1.3` branch (this v1.3-impl worktree works). + +The dispatch + ~30-60 min wait + post-run verification is best done as +a focused next session. diff --git a/docs/pilot-runs/2026-05-08-auto-improve-pilot-summary.md b/docs/pilot-runs/2026-05-08-auto-improve-pilot-summary.md new file mode 100644 index 0000000..e639c1f --- /dev/null +++ b/docs/pilot-runs/2026-05-08-auto-improve-pilot-summary.md @@ -0,0 +1,109 @@ +# Auto-improve-skill pilot summary — 2026-05-08 + +## Setup + +Built a `tools/auto-improve-skill.mjs` wrapper + `tools/auto-improve-skill-prompt.md` template. +Operator says "optimize ``"; orchestrator runs the wrapper via `Bash run_in_background`, +the inner `claude -p` agent does the entire find → eval → diagnose → improve → package loop, +writes `examples/workbench//analysis.md`, exits. + +Branch: `feat/auto-improve-skill` (wrapper + prompt). Per-pilot output on `eval/auto-pilot/`. + +## Three pilot runs + +Run sequentially-ish: pilot #1 in main worktree, pilots #2 and #3 in parallel via `git worktree` +in separate working folders. Three providers × three trials × N cases per pilot. + +| Skill | Classification | Status | Baseline | Final | Uplift | Iter | Plan-cost | OpenRouter | +|---|---|---|---|---|---|---|---|---| +| `vercel-labs/agent-browser/agent-browser` | tool-use | success | 0.56 | 1.00 | +0.44 | 1 | $3.15 | ~$2.80 | +| `supabase/agent-skills/supabase-postgres-best-practices` | code-reviewer | success | 0.54 | 0.86 | +0.32 | 1 | $0 | ~$2.40 | +| `anthropics/skills/pdf` | document-producer | success | 1.00 | 1.00 | +0 | 0 | $0 | ~$1.40 | + +3/3 succeeded. Each surfaced a distinct success path: + +- **agent-browser**: auto-pilot diagnosed that its own grader was over-specified (required `snapshot` for non-interactive ops, but the skill says CSS selectors are valid). Demoted the grader, +0.44 uplift mostly from grader correction. Also proposed a small additive "Quick task reference" section to upstream SKILL.md. +- **supabase**: 9 SQL violations seeded (FK indexes, RLS, covering indexes, etc.). Auto-pilot first self-corrected its grader (line tolerance ±3 → ±8, added keyword variants), then independently rediscovered the same **two-pass workflow** pattern we found manually for web-design-guidelines (pass 1 = visible token misuse, pass 2 = absence checks). Real upstream proposal generated. +- **pdf**: baseline already 1.00, auto-pilot triggered the "≥0.95 → exit clean, no proposal" path correctly. Did NOT manufacture problems. Noticed and noted that upstream's REFERENCE.md / FORMS.md links are 404. + +## Costs + +- OpenRouter (matrix runs): ~$6.60 total across 3 pilots. +- Plan budget (the inner `claude -p` self-reported `total_cost_usd`): only #1 hit the cap. + Pilot #1 first attempt blocked at $3.42 from the docker-permissions issue. Pilot #1c with + `--budget 15` settled at $3.15. Pilots #2 and #3 reported $0 (likely under tracking floor + or didn't iterate enough to register). +- Wall clock: ~50 min for 3 parallel pilots (vs ~150 min sequential). + +## Auto-pilot capabilities validated + +1. **Correct skill-shape classification** in all 3 cases (`tool-use`, `code-reviewer`, `document-producer`). +2. **Self-correction of own grader bugs** before diagnosing the underlying skill — happened in 2 of 3 pilots without operator nudging. Same patterns we manually applied (line-tolerance widening, hyphenated regex variants, keyword alternations). +3. **Pattern transfer**: the auto-pilot rediscovered the "two-pass workflow for absence-type rules" insight on supabase — a different skill in a different rule space — confirming the pattern generalizes. +4. **Clean exit on already-good skills**: pdf ran 36/36 trials passing at baseline; auto-pilot did not manufacture changes. +5. **Distinguishing skill problem from grader problem**: agent-browser caught grader-over-specification, separated it from skill quality. + +## Issues found in v1 of the auto-pilot + +1. **"Always: commit" step unreliable.** Pilots #1b and #2 didn't reach it — case files were left untracked in the worktree. Fix: hoist the commit step earlier (right after analysis.md is written), or split the prompt into two `claude -p` invocations (build + analyze). +2. **`--max-budget-usd 3.50` is too tight** for runs that need any real iteration. Pilot #1's first real-data attempt hit the cap mid-modification. Bumping to $15 worked. Sensible default for v2: $7-10. +3. **Phase 4 grader-fix iteration eats one of the two iteration slots.** The agent often spends iteration 1 fixing graders and only has one shot at modifying the skill. Fix: pre-bake known grader-tuning patterns into `_grader-utils.mjs` so the agent doesn't have to discover them, or count grader-only fixes separately from skill-modification iterations. + +## Patterns we should bake into v2 + +From pilots and prior manual runs, these recurring techniques are stable enough to embed as defaults: + +**Optimizing patterns** (bake into prompt as Phase-4 priors): + +- Two-pass workflow (pass 1 visible / pass 2 absence) for code-reviewer skills +- Per-element checklists for skills with rule-by-element structure +- BAD/GOOD examples for anti-pattern and absence-type rules +- "Verify-tool-installed" nudge for tool-use skills (agents fall back to `curl`/`npm i`) + +**Grader-reliability patterns** (bake into `_grader-utils.mjs`): + +- Default `±5–8` line tolerance +- Hyphen-tolerant regex (`/empty[-\s]+state/`) +- Per-finding-line keyword matching +- Multiple keyword variants (`/cover/i` for both "covering" and "does not cover") + +**Default seeded violation types** (bake into Phase-2 instructions): + +- For code-reviewer: ≥1 visible-token, ≥1 missing-attribute, ≥1 missing-branch, ≥1 anti-pattern, ≥1 state-machine +- For tool-use: ≥1 reaches-for-fallback, ≥1 wrong-flag, ≥1 missing-step +- For document-producer: ≥1 missing-field, ≥1 wrong-format, ≥1 edge-case-input + +## Decision points for the team + +1. **Continue scaling.** With these results, "optimize 10 skills" is a sequential loop the + orchestrator already supports (just call the wrapper N times). With worktrees, N=3 in + parallel is also straightforward. Cost per skill ~$2-3 OpenRouter + plan-tokens. + +2. **Tighten the prompt before scaling.** The "Always: commit" issue and the budget-too-tight + issue are real and would cost a fraction of one pilot to fix. ~30 min of work for v2. + +3. **Build the lessons doc.** A `tools/auto-improve-skill-lessons.md` referenced by the + prompt as Phase-4 prior, updated after every pilot. Compounds: pilot N benefits from + patterns 1..N-1. Not started; sub-project for after the next batch. + +4. **Skill-batch parallelism.** Worktree-per-pilot worked. For 10 skills, 3-way parallel + would land in ~3-4 batches (~3 hours). 5-way is also feasible if the dev machine has + the resources. + +## Reproducing the pilots + +```bash +cd /home/yuqing/Documents/Code/skill-optimizer +git checkout feat/auto-improve-skill +node tools/auto-improve-skill.mjs // [--budget 15] + +# Output: examples/workbench//{analysis.md, suite.yml, ...} +# Branch: eval/auto-pilot/ +``` + +For parallel runs, use git worktrees: + +```bash +git worktree add ../wt-pilot-2 -b auto-pilot/wt-2 feat/auto-improve-skill +cd ../wt-pilot-2 && node tools/auto-improve-skill.mjs --budget 15 +``` diff --git a/docs/pilot-runs/2026-05-09-auto-improve-batch-2-summary.md b/docs/pilot-runs/2026-05-09-auto-improve-batch-2-summary.md new file mode 100644 index 0000000..de6ee50 --- /dev/null +++ b/docs/pilot-runs/2026-05-09-auto-improve-batch-2-summary.md @@ -0,0 +1,100 @@ +# Auto-improve-skill batch 2 summary — 10 pilots, 8 success, 0 failures + +## Setup + +- **Wrapper version:** v1.1 + #3 (atomic write-and-commit, $10 default budget, lessons.md, pre-baked grader helpers) +- **Skills:** ranks 5–14 from the prioritized top-N list (skips the 4 already covered in batch 1: web-design-guidelines, agent-browser, supabase, pdf) +- **Parallelism:** 10 git worktrees, hardlinked `node_modules`, fired simultaneously +- **Wall clock:** ~50 min (slowest pilot to longest), down from estimated ~150 min sequential + +## Headline results + +| # | Skill | Classification | Status | Coverage | Mods | Notes | +|---|---|---|---|---|---|---| +| 1 | `anthropics/skills/pptx` | document-producer | ✅ success | 0.85 → 0.85 | 0 | grader cal raised raw 0.74 → 0.85; gpt-4o-mini fails entirely (model gap) | +| 2 | `vercel-labs/next-skills/next-best-practices` | code-reviewer | ✅ success | 0.80 → 0.975 | 0 | grader cal only — skill already strong | +| 3 | `firebase/agent-skills/firebase-auth-basics` | code-reviewer | ✅ success | 1.00 → 1.00 | 0 | reclassified from prior `tool-use` | +| 4 | `firebase/agent-skills/firebase-hosting-basics` | code-patterns | ✅ success | 0.89 → 1.00 | 1 | Recipe A + E added a Configuration Review section | +| 5 | `expo/skills/building-native-ui` | code-patterns | ✅ success | 0.99 → 0.99 | 0 | 17/18 trials — single gpt-5-mini miss accepted as noise | +| 6 | `google-labs-code/stitch-skills/shadcn-ui` | code-patterns | ✅ success | 0.82 → 0.89 | 1 | Recipe A + D — Gemini's wrong-location miss rate dropped 100% → 0% | +| 7 | `expo/skills/native-data-fetching` | code-reviewer | ✅ success | 1.00 → 1.00 | 0 | already-good | +| 8 | `firecrawl/skills/firecrawl-build-scrape` | code-patterns | ⚠️ uplift-too-small | 0.84 → 0.89 | 2 | +0.05, exactly on threshold; gpt-4o-mini verbosity floor caps it | +| 9 | `vercel-labs/next-skills/next-upgrade` | code-reviewer | ⚠️ uplift-too-small | **0.83 → 0.76** | 2 | **regression** — modifications hurt; new failure mode surfaced | +| 10 | `github/awesome-copilot/prd` | document-producer | ✅ success | 1.00 → 1.00 | 0 | sonnet API errors, judged on 12 valid trials from gpt-5-mini + gemini | + +**8/10 success • 2/10 uplift-too-small • 0/10 blocked or budget-exceeded** + +## Cost + +- OpenRouter spend during batch: **~$21.30** ($40.65 used – $19.35 prior to batch start) +- Per-pilot avg: **$2.13** (well under the $3.50 budgeted) +- Plan-token spend (inner `claude -p`): each pilot reported between $0.00 and $1.00 — no pilot hit the $10 wrapper cap + +## What v1.1 + #3 actually delivered + +The pilots demonstrate the prompt improvements working as intended: + +1. **"Atomic write-analysis-and-commit" worked.** **All 10 inner agents committed cleanly.** No manual recovery needed (vs batch 1 where 2 of 3 needed manual commits). +2. **Recipe citations by letter.** Pilots 4, 6, 8 explicitly cited Recipe A / D / E from `lessons.md` in their analysis bullets. They didn't rediscover the patterns from scratch. +3. **"Grader-vs-skill check first" worked.** Pilots 1, 2, 4, 6, 8, 9 all did iteration 0 grader calibration before counting against their iteration budget. Saved meaningful budget on pilots 2, 4, 6. +4. **`looseRange` / `tolerantKeyword` pre-baked helpers** — used in graders the auto-pilot wrote without rediscovering the patterns. Several pilots had to widen specifically for gpt-4o-mini drift (range 8 → 12 or 16) which is new signal worth adding to lessons.md. +5. **"Don't manufacture problems"** worked in all 5 already-good cases (3, 5, 7, 10, plus pilot 1 after grader cal). None proposed unnecessary changes. + +## New patterns surfaced — worth adding to `lessons.md` + +### Optimization patterns + +- **(NEW) Recipe F? — Don't add bash commands for small models.** Pilot 9 added bash grep commands to `next-upgrade`'s SKILL.md. gpt-4o-mini tried to *execute* them rather than reading files, dropping coverage from 0.83 to 0.69. **Anti-pattern.** When skill is aimed at small/cheap models, prefer pure declarative wording over executable commands. + +### Failure modes + +- **CLI fabrication on "upgrade-style" skills.** gpt-4o-mini will hallucinate a `npx -upgrade` CLI for any skill whose name suggests transformation/upgrade work, then write the error message as findings. Distinct from the agent-browser `curl` fallback (where the CLI exists but the model picks the wrong tool). Worth its own anti-pattern entry. +- **Verbosity floor on gpt-4o-mini.** Confirmed across pilots 8, 9 — emits 3-4 line responses, sometimes drops trailing rules entirely. Rules requiring multi-finding output above this floor are systematically under-detected. + +### Grader patterns + +- **(NEW) Per-model line tolerance.** sonnet/gemini drift 0–3 lines; gpt-4o-mini drifts 6–15 lines. The `looseRange` default of ±8 is calibrated for the first two but undertuned for the third. Future graders should default to `looseRange(N, 12)` or use per-model tolerance maps. + +### Skill-shape edge cases + +- **Repo path conventions vary.** `expo/skills` uses `plugins/expo/skills//SKILL.md` (not the canonical `skills//SKILL.md`). Pilots 5 and 7 both surfaced this and adapted. Worth noting in Phase-1 instructions. + +## Branches pushed + +- `eval/auto-pilot/batch-2-2026-05-09` (consolidated, all 10 cherry-picked) +- 10 individual `eval/auto-pilot/` branches (for per-skill review) + +## What to PR upstream + +Three pilots produced real, additive proposals: + +| Skill | Uplift | Where the change goes | +|---|---|---| +| firebase-hosting-basics | 0.89 → 1.00 | `firebase/agent-skills` | +| shadcn-ui | 0.82 → 0.89 | `google-labs-code/stitch-skills` | +| firecrawl-build-scrape | 0.84 → 0.89 | `firecrawl/skills` | + +**Skip from PR queue:** + +- All 5 baseline-already-good skills (no changes warranted) +- pilot 9 (next-upgrade) — modifications regressed; needs human review or a different approach (probably "drop bash commands, use BAD/GOOD only") +- pilot 8 (firecrawl-build-scrape) is on the bubble at +0.05 — judgment call + +## Decision points for the team + +1. **Scale further.** With v1.1+#3 working, batch 3 of 10 skills should land in another ~50 min for ~$25 OpenRouter. Plenty of remaining slugs in the top-N (15–47). +2. **Lessons.md v1.2 update.** Add the patterns from this batch (CLI fabrication, gpt-4o-mini line drift, repo-path variants, "don't add bash for small models"). 30 min of doc work that compounds for batch 3. +3. **Drop gpt-4o-mini from default matrix.** Repeated capability gap (verbosity floor + CLI fabrication + line drift) is dragging multiple pilots' scores. Switching the matrix to sonnet/gemini/another-mid-tier would likely lift batch coverage by 5-10pp without any skill changes. Worth piloting. + +## Reproducing + +```bash +# This batch can be reproduced from a fresh checkout of feat/auto-improve-skill: +cd /home/yuqing/Documents/Code/skill-optimizer +git checkout feat/auto-improve-skill +node tools/auto-improve-skill.mjs + +# For parallel batches, use git worktrees (see batch script in this commit's Setup section) +``` + +Cumulative spend: $40.65 of $60 OpenRouter credits. diff --git a/docs/pilot-runs/README.md b/docs/pilot-runs/README.md new file mode 100644 index 0000000..dd9fcc8 --- /dev/null +++ b/docs/pilot-runs/README.md @@ -0,0 +1,41 @@ +# Auto-improve-skill pilot runs + +> **Note (v1.3):** The `tools/auto-improve-skill.mjs` wrapper referenced in this README has been removed. The current invocation is via the `skills/auto-improve-orchestrator/` Claude Code skill — operator dispatches the orchestrator subagent via the Agent tool. See `docs/auto-improve-skill-v1.3-spec.md` and `skills/auto-improve-orchestrator/SKILL.md` for current usage. The historical commands in this file are preserved for reference but should not be used. + +Summaries of batched runs of the `tools/auto-improve-skill.mjs` auto-pilot +against public agent skills from our prioritized top-N list. Each summary +documents what skills ran, what the auto-pilot proposed, what worked, what +didn't, and what changes we should make to the prompt before the next batch. + +The per-skill eval artifacts (suite, graders, vendored upstream, proposed-upstream-changes/) +live on `eval/auto-pilot/` branches and the consolidated +`eval/auto-pilot/batch--` branches. + +## Index + +- [`2026-05-08-auto-improve-pilot-summary.md`](./2026-05-08-auto-improve-pilot-summary.md) + — Batch 1, 3 skills (agent-browser, supabase-postgres-best-practices, pdf). + Validated end-to-end. 3/3 success. +- [`2026-05-09-auto-improve-batch-2-summary.md`](./2026-05-09-auto-improve-batch-2-summary.md) + — Batch 2, 10 skills (pptx, next-best-practices, firebase-auth-basics, + firebase-hosting-basics, building-native-ui, shadcn-ui, native-data-fetching, + firecrawl-build-scrape, next-upgrade, prd). 8/10 success, 2/10 uplift-too-small. + +## How to run a new batch + +```bash +# Single skill from the main repo: +node tools/auto-improve-skill.mjs // [--budget 10] + +# Parallel batch via git worktrees: +for i in {1..N}; do + git worktree add ../wt-pilot-$i -b auto-pilot/wt-batch-$i feat/auto-improve-skill + cp -al node_modules dist ../wt-pilot-$i/ + cp .env ../wt-pilot-$i/ +done + +# Then fire one wrapper invocation per worktree in parallel. +``` + +After all pilots complete, cherry-pick each `eval/auto-pilot/` onto +a consolidated batch branch and open a PR. diff --git a/docs/pilot-runs/upstream-pr-conventions.md b/docs/pilot-runs/upstream-pr-conventions.md new file mode 100644 index 0000000..4d62611 --- /dev/null +++ b/docs/pilot-runs/upstream-pr-conventions.md @@ -0,0 +1,128 @@ +# Upstream PR conventions for skill repositories + +Operational guide for submitting skill-improvement PRs to upstream +maintainers. Each row was verified by reading the repo's +`AGENTS.md` / `CONTRIBUTING.md` / `.github/workflows/` + scanning the +last 5–10 merged PRs. Update this doc when we observe new patterns. + +## Quick reference + +| Repo | License | Style | Title format | Body | CI gates | CLA | +|---|---|---|---|---|---|---| +| `vercel-labs/agent-skills` | (no LICENSE) | casual | `{skill}: ` | `## Summary` + `## Test plan` | path-filtered (only fires for react-best-practices changes) | no | +| `vercel-labs/web-interface-guidelines` | MIT | terse | sentence-case freeform, optional `feat:`/`fix:` | 1–2 sentences | none (no workflows) | no | +| `vercel-labs/agent-browser` | Apache-2.0 | formal | `feat/fix/docs(scope): description` (conventional commits) | `## Summary` + `## Test plan` | Rust fmt/clippy/test + dashboard build + version-sync | no CLA bot observed | +| `supabase/agent-skills` | MIT | formal | `feat/fix/docs: description` (conventional commits, used by Release Please) | terse `## Summary` bullets | `pnpm test:sanity` only | no (CONTRIBUTING.md states MIT auto-license) | + +## Per-repo notes + +### `vercel-labs/agent-skills` + +- **Title**: `{skill-name}: ` — skill name as the scope, no + conventional-commit prefix needed. +- **Body**: Multi-section. Use `## Summary` bullets + `## Test plan` + checkboxes. 600–2500 chars is the observed norm. Claude Code footer + (`🤖 Generated with Claude Code`) is fully normalized — appears in + multiple merged PRs. +- **CI**: One workflow (`react-best-practices-ci.yml`) is path-filtered; + unless our change touches `skills/react-best-practices/**`, it won't + fire. Vercel deploy preview is cosmetic, not blocking. +- **Merge style**: Squash. Maintainer (`bhrigu123`) approves silently and + same-day for clean PRs. +- **PR scope**: Tight per-skill (one skill per PR). Improvements to + existing skills merge faster than new-skill additions (PR #238 + proposing a brand-new skill has sat for weeks). +- **Gotcha**: Some skills have a `.zip` alongside the directory. Not + blocking but a known convention. + +### `vercel-labs/web-interface-guidelines` + +- **Title**: Freeform sentence (e.g., `Add translate="no" guideline for + verbatim content`) or `feat:`/`fix:` prefix — both merged. +- **Body**: Minimal. PR #20 is exemplary: two sentences of rationale, no + headers. 0–400 chars is the observed norm. +- **CI**: No workflows. Zero automated checks. +- **Merge style**: Silent approve from `JohnPhamous` (Vercel staff). +- **Sync constraint**: `README.md` and `AGENTS.md` are dual copies of + the same content (one human-readable, one agent-readable). If we add + or change a guideline, **touch both files** in the same PR. PR #20 + did this; ours should too. +- **Pace**: Repo is low-traffic (48 forks, last merge ~5 weeks ago). + Expect slow response. Don't optimize for immediate merge. + +### `vercel-labs/agent-browser` + +- **Title**: Strict conventional commits — `feat(scope): description`, + `fix(scope): description`, `docs: description`. Scope is the + subsystem (`docs`, `doctor`, `native`, etc.). +- **Body**: `## Summary` (2 bullets) + `## Test plan` (2 checkboxes). + PR #1305 is a reference template. +- **CI**: Strict. Three blocking jobs (Rust fmt+clippy+test, dashboard + pnpm build, version-sync). **Docs-only and skill-data-only changes + should pass automatically**; anything touching Rust will trigger + expensive checks. +- **Merge style**: `ctate` is sole maintainer; very active, merges + same-day silently for clean PRs. +- **Critical gotcha**: Skill content lives at + `skill-data/core/SKILL.md`, **not** at `skills/agent-browser/SKILL.md` + (which is intentionally a thin stub per AGENTS.md). Any meaningful + skill change touches: + + 1. `skill-data/core/SKILL.md` + 2. `skill-data/core/references/*.md` (the per-rule reference docs) + 3. `README.md` + 4. The docs MDX pages + + Per AGENTS.md, omitting any of these is grounds for rejection. Use + HTML `` syntax in MDX (not markdown pipe tables). +- **PR scope**: Tight per subsystem. Docs-only changes are the + lowest-friction path — they bypass the Rust CI gates. + +### `supabase/agent-skills` + +- **Title**: Strict conventional commits — `feat: `, + `fix: `, `docs: `. Release Please uses these + to determine semver bumps. **Do not** bump `metadata.version` + manually in SKILL.md — Release Please handles it post-merge. +- **Body**: Short `## Summary` with 1–4 bullets. Link issues with + `Resolves AI-NNN` if applicable. No template. +- **CI**: One job — `Skills CI` runs `pnpm test:sanity`. Sanity tests + check that new reference files follow the `{prefix}-{name}.md` + naming convention with valid frontmatter (`title`, `impact`, `tags`). + Run `pnpm test:sanity` locally before submitting. +- **Merge style**: Squash. `gregnr` (Supabase staff) and `Rodriguespn` + (sole active community maintainer) merge in under 30 min for clean + PRs by core team members; external PRs may need a single LGTM. +- **PR scope**: Additive file change only. Add a new reference file + under `skills//references/{prefix}-{name}.md` with proper + frontmatter + Incorrect/Correct examples. CONTRIBUTING.md says + significant new skills need a prior GitHub Discussion; reference + additions don't. + +## Process for our own PRs + +For each PR we submit: + +1. **Branch** off a fresh local clone of the upstream repo, NOT off our + `examples/workbench//proposed-upstream-changes/`. Copy the + `after-*.md` content into the actual upstream file paths. +2. **Run any local checks** the repo requires (e.g., `pnpm test:sanity` + for supabase). +3. **Title and body** per the table above. +4. **Add the Claude Code footer** unless the repo's style sheet objects + (vercel-labs repos accept it; supabase hasn't shown a precedent + either way). +5. **Cap each PR to one skill**. If a skill has both a SKILL.md change + and a rules-doc change (as web-design-guidelines does, spanning two + repos), open two PRs and reference each from the other. + +## Reference: which repo each skill lives in + +| Our top-N skill | SKILL.md repo | Rules doc repo (if separate) | +|---|---|---| +| `vercel-labs/agent-skills/web-design-guidelines` | `vercel-labs/agent-skills` | `vercel-labs/web-interface-guidelines` | +| `vercel-labs/agent-browser/agent-browser` | `vercel-labs/agent-browser` (`skill-data/core/SKILL.md`) | n/a (inline) | +| `supabase/agent-skills/supabase-postgres-best-practices` | `supabase/agent-skills` | n/a (inline via `references/`) | + +Future skills we run on will surface their own conventions. Append +them here. diff --git a/docs/pilot-runs/upstream-pr-drafts/1-vercel-labs-agent-skills-web-design-guidelines.md b/docs/pilot-runs/upstream-pr-drafts/1-vercel-labs-agent-skills-web-design-guidelines.md new file mode 100644 index 0000000..d1bdc23 --- /dev/null +++ b/docs/pilot-runs/upstream-pr-drafts/1-vercel-labs-agent-skills-web-design-guidelines.md @@ -0,0 +1,133 @@ +# PR #1 — vercel-labs/agent-skills: web-design-guidelines + +**Target:** `vercel-labs/agent-skills` +**File:** `skills/web-design-guidelines/SKILL.md` +**Base branch:** `main` +**Title:** `web-design-guidelines: add explicit two-pass workflow` + +## Body + +```markdown +## Summary + +- Adds an explicit "Pass 1 — visible anti-patterns / Pass 2 — absences" workflow to the SKILL.md, so reviewing agents do a structured per-element absence check after scanning for visible bad patterns. +- The skill's rules are mostly about *what's missing* (a missing `alt`, a missing `aria-label`, a missing focus replacement). Models reliably catch the visible patterns but skip the absence checks unless explicitly told to look for them. +- Diff vs upstream is purely additive: no rule deletions, no wording changes to existing rules. Adds ~15 lines under "How It Works" plus a tightened "Usage" block. The WebFetch behavior and the rules URL are unchanged. + +## Evidence + +Built a workbench of 4 sample React/TSX components seeded with 20 known violations across a11y / focus / forms / typography / animation rule families, then ran a 3-model matrix (`claude-sonnet-4.6`, `openai/gpt-5-mini`, `google/gemini-2.5-pro`) × 3 trials. + +| Model | Before | After | +|---|---|---| +| `claude-sonnet-4.6` | 10/12 (83%) | 12/12 (100%) | +| `openai/gpt-5-mini` | 9/12 (75%) | 10/12 (83%) | +| `google/gemini-2.5-pro` | 7/12 (58%) | 9/12 (75%) | +| **Total** | **26/36 (72%)** | **31/36 (86%)** | + +`gpt-5-mini`'s gains come almost entirely from the new per-element checklist surfacing absence rules. Two rules (`no-empty-state-handling`, `input-missing-autocomplete`) were eliminated entirely. + +A companion PR to `vercel-labs/web-interface-guidelines` adds matching per-element checklists + 5 BAD/GOOD code blocks to `command.md`. Both PRs land independently but are most useful merged together. + +## Test plan + +- [ ] Read the diff — confirm additive only, no existing rules touched +- [ ] Verify the SKILL.md still parses correctly as a Claude Code skill +- [ ] Optional: re-run with your preferred review test files +``` + +## File diff + +**Before** (`skills/web-design-guidelines/SKILL.md`, 39 lines): + +The current upstream version. No changes needed before applying the diff below. + +**After** (54 lines, +15 net): adds explicit Pass 1 / Pass 2 sections to "How It Works" and tightens the "Usage" numbered list to reflect the two-pass workflow. + +The full proposed file is checked into our repo at: + +- [`examples/workbench/web-design-guidelines/proposed-upstream-changes/agent-skills--web-design-guidelines/after-SKILL.md`](../../../examples/workbench/web-design-guidelines/proposed-upstream-changes/agent-skills--web-design-guidelines/after-SKILL.md) + +A unified diff against the upstream: + +```diff +--- skills/web-design-guidelines/SKILL.md (current upstream) ++++ skills/web-design-guidelines/SKILL.md (proposed) +@@ metadata block @@ + author: vercel +- version: "1.0.0" ++ version: "1.1.0" + argument-hint: + +@@ "How It Works" section @@ + ## How It Works + + 1. Fetch the latest guidelines from the source URL below. + 2. Read the specified files (or prompt user for files/pattern). +-3. Check against all rules in the fetched guidelines +-4. Output findings in the terse `file:line` format ++3. Review each file in **TWO passes** — both passes are required. ++4. Output findings in the terse `file:line ` format. ++ ++### Pass 1 — Visible anti-patterns ++ ++Scan each file for literal patterns that appear in the code: ++`
` for actions, `transition: all`, `outline-none` className, ++`onPaste={(e) => e.preventDefault()}`, `"..."` (three dots), straight ++`"..."` quotes, etc. The full list is in the fetched guidelines. One ++finding per match. ++ ++### Pass 2 — Absences (per-element checklist) ++ ++The most-missed rules are about *what's missing*. After Pass 1, walk ++each ``, ``, `
has no supporting index + + Name the specific FK column. Reference the line number of the ALTER TABLE + that declares the unsupported FK. Cover all violations you find, one per + line. + graders: + - name: fk-index-audit-findings + command: node $CASE/checks/grade-fk-index-audit-findings.mjs + + - name: review-update-without-where + task: | + You are a Postgres expert applying the supabase-postgres-best-practices skill. + + Read the skill at supabase-postgres-best-practices/SKILL.md, then read all + relevant rule files under supabase-postgres-best-practices/references/ to + understand the complete set of rules. + + Review the file /work/data_migration.sql, a one-shot DML backfill script. + Identify any UPDATE or DELETE statement that would mutate more rows than + intended (for example, an UPDATE without a WHERE clause). + + For each violation, write one line to /work/findings.txt in the format: + + data_migration.sql: - : + + Name the specific table being mutated. Reference the line number of the + dangerous statement. Cover all violations you find, one per line. + graders: + - name: update-without-where-findings + command: node $CASE/checks/grade-update-without-where-findings.mjs diff --git a/examples/workbench/supabase-postgres-best-practices/workspace/data_migration.sql b/examples/workbench/supabase-postgres-best-practices/workspace/data_migration.sql new file mode 100644 index 0000000..2ec5808 --- /dev/null +++ b/examples/workbench/supabase-postgres-best-practices/workspace/data_migration.sql @@ -0,0 +1,42 @@ +-- data_migration.sql: one-shot backfill script. Wrap in a transaction +-- so we can review all DML before committing. Each statement targets a +-- specific subset of rows. + +begin; + +-- 1. Backfill missing display names from email local-part. +update users + set display_name = split_part(email, '@', 1) + where display_name is null; + + +-- 2. Mark stale draft posts as archived. +update posts + set status = 'archived', + archived_at = now() + where status = 'draft' + and updated_at < now() - interval '180 days'; + + +-- 3. Recompute order totals after pricing rule fix. +update orders + set total = subtotal + tax + shipping; + + +-- 4. Delete orphaned cart items left over from the v1 checkout flow. +delete from cart_items + where cart_id not in (select id from carts); + + +-- 5. Set notification_preference default for legacy users. +update users + set notification_preference = 'daily' + where notification_preference is null + and created_at < '2024-01-01'; + + +-- 6. Remove expired email verification tokens. +delete from email_verification_tokens + where expires_at < now(); + +commit; diff --git a/examples/workbench/supabase-postgres-best-practices/workspace/migrations.sql b/examples/workbench/supabase-postgres-best-practices/workspace/migrations.sql new file mode 100644 index 0000000..8153bc5 --- /dev/null +++ b/examples/workbench/supabase-postgres-best-practices/workspace/migrations.sql @@ -0,0 +1,78 @@ +-- migrations.sql: a sequence of ALTER TABLE statements that add foreign +-- keys to an existing schema. Each ALTER may or may not be followed by a +-- CREATE INDEX on the FK column. + +-- ===================================================================== +-- Migration 001: link orders to customers +-- ===================================================================== +alter table orders + add constraint orders_customer_id_fkey + foreign key (customer_id) references customers(id) on delete restrict; + +create index orders_customer_id_idx on orders (customer_id); + + +-- ===================================================================== +-- Migration 002: link order_items to orders +-- ===================================================================== +alter table order_items + add constraint order_items_order_id_fkey + foreign key (order_id) references orders(id) on delete cascade; + +-- (no supporting index) + + +-- ===================================================================== +-- Migration 003: link order_items to products +-- ===================================================================== +alter table order_items + add constraint order_items_product_id_fkey + foreign key (product_id) references products(id) on delete restrict; + +create index order_items_product_id_idx on order_items (product_id); + + +-- ===================================================================== +-- Migration 004: link invoices to orders +-- ===================================================================== +alter table invoices + add constraint invoices_order_id_fkey + foreign key (order_id) references orders(id) on delete restrict; + +-- (no supporting index) + + +-- ===================================================================== +-- Migration 005: link shipments to orders + carriers +-- ===================================================================== +alter table shipments + add constraint shipments_order_id_fkey + foreign key (order_id) references orders(id) on delete cascade; + +create index shipments_order_id_idx on shipments (order_id); + +alter table shipments + add constraint shipments_carrier_id_fkey + foreign key (carrier_id) references carriers(id) on delete set null; + +-- (no supporting index) + + +-- ===================================================================== +-- Migration 006: link refunds to invoices +-- ===================================================================== +alter table refunds + add constraint refunds_invoice_id_fkey + foreign key (invoice_id) references invoices(id) on delete restrict; + +create index refunds_invoice_id_idx on refunds (invoice_id); + + +-- ===================================================================== +-- Migration 007: link audit_log entries to users +-- ===================================================================== +alter table audit_log + add constraint audit_log_actor_id_fkey + foreign key (actor_id) references users(id) on delete set null; + +create index audit_log_actor_id_idx on audit_log (actor_id); diff --git a/examples/workbench/supabase-postgres-best-practices/workspace/multi_table_schema.sql b/examples/workbench/supabase-postgres-best-practices/workspace/multi_table_schema.sql new file mode 100644 index 0000000..68ea034 --- /dev/null +++ b/examples/workbench/supabase-postgres-best-practices/workspace/multi_table_schema.sql @@ -0,0 +1,87 @@ +-- multi_table_schema.sql: multi-tenant SaaS schema for a small social app. +-- Six tables: four hold per-user data (users, posts, comments, messages) +-- and two are global reference data (countries, currencies). + +-- ===================================================================== +-- 1. users — primary account table +-- ===================================================================== +create table users ( + id uuid primary key default gen_random_uuid(), + email text not null unique, + display_name text, + created_at timestamptz default now() +); + +alter table users enable row level security; +alter table users force row level security; + +create policy users_self_read on users + for select using ((select auth.uid()) = id); + + +-- ===================================================================== +-- 2. posts — user-authored content +-- ===================================================================== +create table posts ( + id bigint generated always as identity primary key, + user_id uuid not null references users(id) on delete cascade, + title text not null, + body text, + published_at timestamptz +); + +create index posts_user_id_idx on posts (user_id); + +alter table posts enable row level security; + +create policy posts_owner_all on posts + for all using ((select auth.uid()) = user_id); + + +-- ===================================================================== +-- 3. comments — replies to posts +-- ===================================================================== +create table comments ( + id bigint generated always as identity primary key, + post_id bigint not null references posts(id) on delete cascade, + author_id uuid not null references users(id) on delete cascade, + body text not null, + created_at timestamptz default now() +); + +create index comments_post_id_idx on comments (post_id); +create index comments_author_id_idx on comments (author_id); + + +-- ===================================================================== +-- 4. countries — global reference data +-- ===================================================================== +create table countries ( + code char(2) primary key, + name text not null +); + + +-- ===================================================================== +-- 5. messages — direct user-to-user messages +-- ===================================================================== +create table messages ( + id bigint generated always as identity primary key, + sender_id uuid not null references users(id) on delete cascade, + recipient_id uuid not null references users(id) on delete cascade, + body text not null, + sent_at timestamptz default now() +); + +create index messages_sender_id_idx on messages (sender_id); +create index messages_recipient_id_idx on messages (recipient_id); + + +-- ===================================================================== +-- 6. currencies — global reference data +-- ===================================================================== +create table currencies ( + code char(3) primary key, + name text not null, + symbol text +); diff --git a/examples/workbench/supabase-postgres-best-practices/workspace/rls_policies.sql b/examples/workbench/supabase-postgres-best-practices/workspace/rls_policies.sql new file mode 100644 index 0000000..551bc14 --- /dev/null +++ b/examples/workbench/supabase-postgres-best-practices/workspace/rls_policies.sql @@ -0,0 +1,25 @@ +-- rls_policies.sql: Row-Level Security setup with intentional best-practice violations +-- for supabase-postgres-best-practices eval + +-- === RLS enabled, but FORCE not applied === +-- VIOLATION (security-rls-basics): table owner can bypass RLS; FORCE not set +ALTER TABLE orders ENABLE ROW LEVEL SECURITY; +-- FIX: also run ALTER TABLE orders FORCE ROW LEVEL SECURITY; + +-- === Policy with per-row function call === +-- VIOLATION (security-rls-performance): auth.uid() called per row, not wrapped in SELECT +CREATE POLICY orders_user_policy ON orders + FOR ALL + USING (user_id = auth.uid()); +-- FIX: USING ((select auth.uid()) = user_id) + +-- === RLS policy column has no index === +-- VIOLATION (security-rls-performance): user_id used in RLS USING clause has no index +-- Every request triggers a sequential scan on orders for the authenticated user. +-- FIX: CREATE INDEX orders_user_id_idx ON orders (user_id); + +-- === Covering index missing INCLUDE === +-- VIOLATION (query-covering-indexes): status index does not cover fetched columns +CREATE INDEX orders_status_idx ON orders (status); +-- Query: SELECT status, customer_id, total FROM orders WHERE status = 'shipped' +-- FIX: CREATE INDEX orders_status_idx ON orders (status) INCLUDE (customer_id, total); diff --git a/examples/workbench/supabase-postgres-best-practices/workspace/schema.sql b/examples/workbench/supabase-postgres-best-practices/workspace/schema.sql new file mode 100644 index 0000000..be812e0 --- /dev/null +++ b/examples/workbench/supabase-postgres-best-practices/workspace/schema.sql @@ -0,0 +1,33 @@ +-- schema.sql: E-commerce schema with intentional best-practice violations +-- for supabase-postgres-best-practices eval + +-- Customers table (clean reference) +CREATE TABLE customers ( + id bigint generated always as identity primary key, + email text not null, + deleted_at timestamptz +); + +-- Orders table +-- VIOLATION (schema-foreign-key-indexes): customer_id FK column has no index +CREATE TABLE orders ( + id bigint generated always as identity primary key, + customer_id bigint references customers(id) on delete cascade, + status text not null default 'pending', + created_at timestamptz default now(), + total numeric(10,2) +); + +-- VIOLATION (schema-constraints): ADD CONSTRAINT IF NOT EXISTS is invalid Postgres syntax +ALTER TABLE orders + ADD CONSTRAINT IF NOT EXISTS orders_amount_valid CHECK (total > 0); + +-- VIOLATION (query-partial-indexes): full index on email instead of partial (ignores deleted_at) +CREATE INDEX customers_email_idx ON customers (email); + +-- VIOLATION (query-composite-indexes): range column before equality column breaks leftmost-prefix rule +CREATE INDEX orders_date_status_idx ON orders (created_at, status); + +-- VIOLATION (security-rls-basics): orders table stores user data but has no RLS enabled +-- The application filters by user_id in SQL but database enforces no tenant isolation. +-- Missing: ALTER TABLE orders ENABLE ROW LEVEL SECURITY; diff --git a/package-lock.json b/package-lock.json index d985d28..9580a6f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,7 +8,6 @@ "name": "skill-optimizer", "version": "2.0.0", "license": "MIT", - "main": ".opencode/plugins/skill-optimizer.js", "dependencies": { "@mariozechner/pi-agent-core": "^0.66.1", "@mariozechner/pi-ai": "^0.66.1", @@ -22,6 +21,7 @@ }, "devDependencies": { "@types/node": "^22.12.0", + "gray-matter": "^4.0.3", "tsx": "^4.19.0", "typescript": "^5.7.0" }, @@ -2773,6 +2773,16 @@ "integrity": "sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==", "license": "MIT" }, + "node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "license": "MIT", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, "node_modules/ast-types": { "version": "0.13.4", "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", @@ -3502,6 +3512,19 @@ "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", "license": "MIT" }, + "node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/extract-zip": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", @@ -3902,6 +3925,22 @@ "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", "license": "ISC" }, + "node_modules/gray-matter": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/gray-matter/-/gray-matter-4.0.3.tgz", + "integrity": "sha512-5v6yZd4JK3eMI3FqqCouswVqwugaA9r4dNZB1wwcmrD02QkV5H0y7XBQW8QwQqEaZY1pM9aqORSORhJRdNK44Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "js-yaml": "^3.13.1", + "kind-of": "^6.0.2", + "section-matter": "^1.0.0", + "strip-bom-string": "^1.0.0" + }, + "engines": { + "node": ">=6.0" + } + }, "node_modules/has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -4080,6 +4119,16 @@ "node": ">= 0.10" } }, + "node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", @@ -4134,6 +4183,20 @@ "url": "https://github.com/sponsors/panva" } }, + "node_modules/js-yaml": { + "version": "3.14.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", + "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", + "dev": true, + "license": "MIT", + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/json-bigint": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", @@ -4195,6 +4258,16 @@ "safe-buffer": "^5.0.1" } }, + "node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/koffi": { "version": "2.16.0", "resolved": "https://registry.npmjs.org/koffi/-/koffi-2.16.0.tgz", @@ -4978,6 +5051,20 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "license": "MIT" }, + "node_modules/section-matter": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/section-matter/-/section-matter-1.0.0.tgz", + "integrity": "sha512-vfD3pmTzGpufjScBh50YHKzEu2lxBWhVEHsNGoEXmCmn2hKGfeNLYMzCJpe8cD7gqX7TJluOVpBkAequ6dgMmA==", + "dev": true, + "license": "MIT", + "dependencies": { + "extend-shallow": "^2.0.1", + "kind-of": "^6.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/send": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", @@ -5176,6 +5263,13 @@ "node": ">=0.10.0" } }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", + "dev": true, + "license": "BSD-3-Clause" + }, "node_modules/statuses": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", @@ -5253,6 +5347,16 @@ "url": "https://github.com/chalk/strip-ansi?sponsor=1" } }, + "node_modules/strip-bom-string": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/strip-bom-string/-/strip-bom-string-1.0.0.tgz", + "integrity": "sha512-uCC2VHvQRYu+lMh4My/sFNmF2klFymLX1wHJeXnbEJERpV/ZsVuonzerjfrGpIGF7LBVa1O7i9kjiWvJiFck8g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/strnum": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.2.3.tgz", diff --git a/package.json b/package.json index 1f1be4c..acefde6 100644 --- a/package.json +++ b/package.json @@ -91,6 +91,7 @@ }, "devDependencies": { "@types/node": "^22.12.0", + "gray-matter": "^4.0.3", "tsx": "^4.19.0", "typescript": "^5.7.0" }, diff --git a/skills/auto-improve-orchestrator/.smoke-check.mjs b/skills/auto-improve-orchestrator/.smoke-check.mjs new file mode 100644 index 0000000..e4ecc52 --- /dev/null +++ b/skills/auto-improve-orchestrator/.smoke-check.mjs @@ -0,0 +1,67 @@ +#!/usr/bin/env node +// Smoke check for the auto-improve-orchestrator skill. +// Validates: SKILL.md frontmatter, prompt template variables, file existence. + +import { readFileSync, existsSync } from 'node:fs'; +import matter from 'gray-matter'; + +const skillRoot = 'skills/auto-improve-orchestrator'; +let failures = 0; + +function check(condition, msg) { + if (condition) { + console.log(`OK: ${msg}`); + } else { + console.error(`FAIL: ${msg}`); + failures++; + } +} + +// 1. SKILL.md exists + frontmatter parses +const skillMdPath = `${skillRoot}/SKILL.md`; +check(existsSync(skillMdPath), `${skillMdPath} exists`); +if (existsSync(skillMdPath)) { + const parsed = matter(readFileSync(skillMdPath, 'utf-8')); + check(parsed.data.name === 'auto-improve-orchestrator', 'SKILL.md name = "auto-improve-orchestrator"'); + check(typeof parsed.data.description === 'string' && parsed.data.description.length > 50, 'SKILL.md description is non-trivial'); +} + +// 2. All four prompt files exist +for (const name of ['orchestrator.md', 'research-upstream.md', 'eval-iterate.md', 'skill-iterate.md']) { + check(existsSync(`${skillRoot}/prompts/${name}`), `prompts/${name} exists`); +} + +// 3. workflow.md + lessons.md + at least one context exist +check(existsSync(`${skillRoot}/references/workflow.md`), 'references/workflow.md exists'); +check(existsSync(`${skillRoot}/references/lessons.md`), 'references/lessons.md exists'); +check(existsSync(`${skillRoot}/references/contexts`), 'references/contexts/ exists'); + +// 4. Each prompt has its expected templated variables +const expectedVars = { + 'orchestrator.md': ['SLUG', 'MAIN_REPO_PATH'], + 'research-upstream.md': ['SLUG', 'OUTPUT_PATH'], + 'eval-iterate.md': ['SKILL_ID', 'WORKBENCH_DIR', 'SUITE_RESULT_PATH', 'DIRECTION', 'LESSONS_PATH'], + 'skill-iterate.md': ['SKILL_ID', 'WORKBENCH_DIR', 'SUITE_RESULT_PATH', 'TARGET_FILE', 'CONTEXT_FILE', 'LESSONS_PATH', 'ITERATION'], +}; +for (const [file, vars] of Object.entries(expectedVars)) { + const content = readFileSync(`${skillRoot}/prompts/${file}`, 'utf-8'); + for (const v of vars) { + check(content.includes(`\${${v}}`), `prompts/${file} contains \${${v}}`); + } +} + +// 5. Old wrapper files are gone +check(!existsSync('tools/auto-improve-skill.mjs'), 'tools/auto-improve-skill.mjs is gone'); +check(!existsSync('tools/auto-improve-skill-prompt.md'), 'tools/auto-improve-skill-prompt.md is gone'); +check(!existsSync('tools/auto-improve-skill-lessons.md'), 'tools/auto-improve-skill-lessons.md moved out of tools/'); +check(!existsSync('tools/auto-improve-contexts'), 'tools/auto-improve-contexts/ moved out of tools/'); + +// 6. Lessons + contexts at new paths +check(existsSync(`${skillRoot}/references/lessons.md`), 'lessons at new path'); +check(existsSync(`${skillRoot}/references/contexts/supabase-supabase-postgres-best-practices.md`), 'supabase context at new path'); + +if (failures > 0) { + console.error(`\n${failures} smoke checks failed`); + process.exit(1); +} +console.log(`\nAll smoke checks passed`); diff --git a/skills/auto-improve-orchestrator/SKILL.md b/skills/auto-improve-orchestrator/SKILL.md new file mode 100644 index 0000000..09ba10b --- /dev/null +++ b/skills/auto-improve-orchestrator/SKILL.md @@ -0,0 +1,84 @@ +--- +name: auto-improve-orchestrator +description: Use when an operator wants to auto-improve a public agent skill — research upstream conventions, iterate the eval if it's saturated/floored, iterate the skill itself, and package proposed upstream changes. Dispatch the orchestrator subagent via the Agent tool with isolation:"worktree" for one or more skills (parallel-safe). +--- + +# auto-improve-orchestrator + +This skill manages the end-to-end auto-improvement workflow for one public +agent skill: research the upstream repo's conventions, measure baseline +on an existing eval workbench, iterate the eval if it's saturated/floored, +iterate the skill content with measured uplift, and package the proposed +upstream change. + +The skill-optimizer stays lean — it ships the eval engine (`run-suite`, +`run-case`, graders, Docker harness). This orchestrator skill contains +the *workflow* logic that uses the engine. + +## When to use + +- Operator says "auto-improve " or "improve " +- Operator says "run auto-improve on these N skills" (batch) +- Operator wants to extend the orchestration logic itself + +## How to invoke + +The operator's CC session dispatches the orchestrator subagent via the +Agent tool. For a single skill: + +``` +Agent({ + description: "auto-improve ", + subagent_type: "general-purpose", + isolation: "worktree", + run_in_background: true, + prompt: +}) +``` + +The orchestrator also accepts an optional `${REFRESH_CONTEXT}` template variable. Set to `"true"` (default `"false"`) to force the research sub-subagent to re-fetch upstream conventions even if a cached context file exists. Use when upstream conventions have changed (e.g., new sanity-test rules, new prefix taxonomy). + +For a batch of N skills, dispatch N Agent calls in a single message — +they run in parallel, each in its own worktree. + +The orchestrator subagent: + +1. Reads/dispatches research subagent (Phase 0) → produces context file +2. Measures baseline via `run-suite` (Phase 3) — resume-aware +3. Iterates eval via dispatch (Phase 3.5) until baseline ∈ (0.50, 0.95) +4. Iterates skill via dispatch (Phase 4) until uplift ≥ +0.05 or 2 iters +5. Packages `proposed-upstream-changes/` + `analysis.md`, commits to + `eval/auto-pilot/` branch, returns summary + +## Sub-subagent prompts + +- `prompts/research-upstream.md` — Phase 0 research subagent template +- `prompts/eval-iterate.md` — Phase 3.5 eval-iteration subagent template +- `prompts/skill-iterate.md` — Phase 4 skill-iteration subagent template + +## Reference material + +- `references/workflow.md` — human-readable description of the algorithm +- `references/lessons.md` — recipes A-E + grader patterns G1-G6 (the + skill-iterate subagent reads this) +- `references/contexts/` — per-skill upstream-research outputs (the + skill-iterate subagent reads these; the research subagent writes them) + +## Pre-conditions + +- The skill's eval workbench must already exist at + `examples/workbench//`. Building initial workbenches is + out of scope for v1.3 — operator builds them manually. +- `OPENROUTER_API_KEY` must be set in `.env` at the repo root. +- Worktree must be created via `isolation: "worktree"` for parallelism + - isolation. Do NOT run the orchestrator subagent on the main + worktree. + +## Out of scope (deferred) + +- Initial workbench construction (v1.4) +- Autonomous CI mode (no operator session) +- Automatic PR submission (operator-driven) + +See `docs/auto-improve-skill-v1.3-spec.md` for the full design. diff --git a/skills/auto-improve-orchestrator/prompts/eval-iterate.md b/skills/auto-improve-orchestrator/prompts/eval-iterate.md new file mode 100644 index 0000000..e4c1319 --- /dev/null +++ b/skills/auto-improve-orchestrator/prompts/eval-iterate.md @@ -0,0 +1,110 @@ +# Sub-subagent prompt: iterate the eval workbench + +You are a sub-subagent dispatched to make a focused change to one +skill's eval workbench so that downstream skill-iteration has real +headroom (baseline lands in `(0.50, 0.95)`). + +## Inputs (templated) + +- `${SKILL_ID}` — leaf id, e.g. `supabase-postgres-best-practices`. +- `${WORKBENCH_DIR}` — `examples/workbench/${SKILL_ID}/`. +- `${SUITE_RESULT_PATH}` — path to the most recent `.results//suite-result.json`. +- `${DIRECTION}` — one of: + - `add-harder` — add 2-3 new cases that surface absence-type + violations the existing graders don't catch + - `simplify` — remove ambiguous violations from existing workspace + files; tighten task descriptions + - `fix-graders` — apply recipes G1-G6 from `references/lessons.md` + (line-tolerance, fuzzy keywords, etc.) +- `${LESSONS_PATH}` — `skills/auto-improve-orchestrator/references/lessons.md`. Read this for grader-pattern recipes G1-G6 (relevant for `fix-graders`) and the load-bearing prior on absence-type rules (relevant for `add-harder`). + +## What to do (per direction) + +### `add-harder` + +1. Read the current `${SUITE_RESULT_PATH}` to identify which rules are + ALREADY at ceiling (per-case score == 1.00). New cases should + target absence-type variants of these rules that the existing + workspace files don't exercise. +2. Read `${LESSONS_PATH}` § "The load-bearing prior" — absence-type + rules are 5-10× harder than presence-type. New cases should force + enumeration (multi-statement files, mixed correct + incorrect + patterns, invariants that span statements). +3. Write 2-3 NEW workspace files under `${WORKBENCH_DIR}/workspace/`. + Each file = one new case. Realistic content (not contrived). + Lowercase SQL keywords if applicable; semantic table/column names. +4. Write 2-3 NEW grader files under `${WORKBENCH_DIR}/checks/`. Each + grader = one new case. Use `_grader-utils.mjs` helpers + (`looseRange`, `fuzzyKeyword`, `tolerantKeyword`). +5. Update `${WORKBENCH_DIR}/suite.yml` to include the new cases. Don't + touch existing cases. +6. Run a smoke check at `${WORKBENCH_DIR}/checks/smoke-graders.mjs`: + - Hand-craft GOOD `findings.txt` per new grader → assert + `pass=true score=1` + - Hand-craft BAD `findings.txt` (missing 1-2 violations) → assert + `pass=false score<1` + - Hand-craft EMPTY → assert `pass=false score=0` + - Run: `node ${WORKBENCH_DIR}/checks/smoke-graders.mjs` + - All assertions must pass. + +### `simplify` + +1. Read `${SUITE_RESULT_PATH}` to identify which cases are scoring + `< 0.50`. Read failed trials' `findings.txt` to understand WHY: + ambiguous tasks, contrived violations, multiple valid answers. +2. Edit existing workspace files to remove ambiguity (clearer task + description, fewer red-herring statements, tighter line ranges). +3. Update graders if the violations themselves changed. +4. Run the smoke check (same shape as `add-harder`). + +### `fix-graders` + +1. Read failed trials' `findings.txt` (paths in `${SUITE_RESULT_PATH}`'s + `results[].trials[].resultPath`). If models DID find the violations + but the grader scored wrong (line off by ±5, keyword mismatch, + format variant), this is a grader bug. +2. Apply recipes G1-G6 from `${LESSONS_PATH}`: + - G1: widen `looseRange` from default 8 to 10-12 if line drift is + systematic + - G2: replace hand-written keyword regex with `fuzzyKeyword` + - G4: replace `/exact-stem/i` with `tolerantKeyword('stem')` + - G6: split per-finding-line check (don't credit cross-finding keyword matches) +3. Run the smoke check. + +## Tools allowed + +Read, Edit, Write, Bash, Glob. + +## Constraints + +- Additive ONLY for `add-harder` (no changes to existing cases or + graders unless their helpers need extension). +- Edits ONLY for `simplify` and `fix-graders` (no new cases). +- Smoke check MUST pass before commit. +- DO NOT run `npx tsx ../../../src/cli.ts run-suite` (that's the + orchestrator's job; you just modify the workbench). +- DO NOT modify the `references/` (vendored skill content) — those are + fixed inputs. + +## Commit + +After smoke check passes: + +```bash +git add ${WORKBENCH_DIR} +git commit -m "feat(eval): ${DIRECTION} for ${SKILL_ID} (cases: )" +``` + +DO NOT push. + +## Return report + +Return to caller (orchestrator subagent) under 300 words: + +- Direction: `add-harder` | `simplify` | `fix-graders` +- Cases added/modified (names + 1-line description each) +- Predicted baseline impact (rough: "frontier models will likely miss + X% of new checks because Y") +- Smoke-check result (N/N assertions pass) +- Blockers, if any +- Branch + commit SHA diff --git a/skills/auto-improve-orchestrator/prompts/orchestrator.md b/skills/auto-improve-orchestrator/prompts/orchestrator.md new file mode 100644 index 0000000..93aa072 --- /dev/null +++ b/skills/auto-improve-orchestrator/prompts/orchestrator.md @@ -0,0 +1,321 @@ +# Orchestrator subagent: end-to-end auto-improve for one skill + +You are dispatched as an autonomous orchestrator for a single public +agent skill: `${SLUG}`. You own this skill end-to-end. You make +decisions about when to research, when to iterate the eval, when to +iterate the skill, and when to package. You are running in your own +git worktree (created by `isolation: "worktree"`). + +## Inputs (templated) + +- `${SLUG}` — `//`. Example: `supabase/agent-skills/supabase-postgres-best-practices`. +- `${MAIN_REPO_PATH}` — absolute path to the operator's main repo (your worktree's parent). Example: `/home/yuqing/Documents/Code/skill-optimizer`. Used for `.env` access. +- `${REFRESH_CONTEXT}` — optional flag, default `false`. If `true`, force re-research even if cached context file exists. + +## Setup + +1. Parse `${SLUG}` into `OWNER`, `REPO`, `SKILL_ID`: + + ```bash + IFS=/ read -r OWNER REPO SKILL_ID <<< "${SLUG}" + ``` + +2. Verify you're in a git worktree (not the main repo): + + ```bash + GIT_DIR=$(cd "$(git rev-parse --git-dir)" 2>/dev/null && pwd -P) + GIT_COMMON=$(cd "$(git rev-parse --git-common-dir)" 2>/dev/null && pwd -P) + if [ "$GIT_DIR" = "$GIT_COMMON" ]; then + echo "ERROR: not running in a worktree. The orchestrator requires isolation:'worktree'." >&2 + exit 1 + fi + ``` + +3. Verify the skill's workbench exists: + + ```bash + WORKBENCH=examples/workbench/${SKILL_ID}/ + if [ ! -d "$WORKBENCH" ]; then + # Write analysis.md with status:blocked-by-missing-workbench, then exit + mkdir -p "$WORKBENCH" + cat > "${WORKBENCH}analysis.md" <&2 + # Write analysis.md with status:blocked-by-error and exit + exit 1 + fi + ``` + +6. Initialize cost tracker: + + ```bash + CUMULATIVE_COST=0 + ``` + +## Phase 0: Research + +```bash +if [ -f "$CONTEXT_FILE" ] && [ "${REFRESH_CONTEXT:-false}" != "true" ]; then + echo "Phase 0: using cached context at $CONTEXT_FILE" +else + echo "Phase 0: dispatching research-upstream subagent for ${SLUG}" + # Dispatch via Agent tool (load skills/auto-improve-orchestrator/prompts/research-upstream.md, + # substitute ${SLUG} and ${OUTPUT_PATH}=${CONTEXT_FILE}). + # Wait for completion. Verify CONTEXT_FILE was written. +fi + +# Read CONTEXT_FILE; extract: target_file, packaging_dir, additive_only_constraint +``` + +When dispatching the research subagent, use: + +- `subagent_type: "general-purpose"` +- `description: "research "` +- (no `isolation`) — research subagent runs in YOUR worktree (writes the context file) +- `prompt`: load `skills/auto-improve-orchestrator/prompts/research-upstream.md` and substitute `${SLUG}` and `${OUTPUT_PATH}` + +## Phase 3: Baseline measurement + +Find the most recent `.results//suite-result.json`: + +```bash +LATEST_RESULTS=$(ls -td ${WORKBENCH}.results/*/ 2>/dev/null | head -1) +LATEST_SUITE_JSON=${LATEST_RESULTS}suite-result.json +``` + +If `LATEST_SUITE_JSON` exists AND its `cases` field matches the current `${WORKBENCH}suite.yml`'s case names: + +- Read it (resume — skip baseline run). + +Else: + +```bash +cd ${WORKBENCH} +set -a; . ${MAIN_REPO_PATH}/.env; set +a +npx tsx ../../../src/cli.ts run-suite ./suite.yml --trials 3 \ + 2>&1 | tee /tmp/orchestrator-baseline-${SKILL_ID}.log +cd - +LATEST_RESULTS=$(ls -td ${WORKBENCH}.results/*/ | head -1) +LATEST_SUITE_JSON=${LATEST_RESULTS}suite-result.json +``` + +Read `LATEST_SUITE_JSON` (use `node -e "console.log(JSON.parse(...))"` or `jq`). + +Compute `per_case_min`: + +```bash +node -e " +const fs = require('fs'); +const data = JSON.parse(fs.readFileSync('$LATEST_SUITE_JSON', 'utf-8')); +const byCase = {}; +for (const r of data.results) { + if (!byCase[r.caseName]) byCase[r.caseName] = []; + byCase[r.caseName].push(r.meanScore); +} +const perCaseMeans = Object.entries(byCase).map(([c, scores]) => ({ + case: c, + mean: scores.reduce((a, b) => a + b, 0) / scores.length, +})); +perCaseMeans.sort((a, b) => a.mean - b.mean); +console.log(JSON.stringify({ + perCaseMin: perCaseMeans[0].mean, + perCaseMinName: perCaseMeans[0].case, + perCaseBreakdown: perCaseMeans, +}, null, 2)); +" +``` + +Track the cost: + +```bash +COST_THIS_RUN=$(node -e " +const data = JSON.parse(require('fs').readFileSync('$LATEST_SUITE_JSON', 'utf-8')); +const sum = (data.metrics?.cost?.total) || 0; +console.log(sum); +") +CUMULATIVE_COST=$(node -e "console.log($CUMULATIVE_COST + $COST_THIS_RUN)") +if (( $(echo "$CUMULATIVE_COST > 10" | bc -l) )); then + echo "ERROR: cumulative cost \$$CUMULATIVE_COST > \$10 hard cap" >&2 + # Write analysis.md status:budget-exceeded and exit + exit 1 +fi +if (( $(echo "$CUMULATIVE_COST > 5" | bc -l) )); then + echo "WARN: cumulative cost \$$CUMULATIVE_COST > \$5 soft warning" >&2 +fi +``` + +## Phase 3.5: Eval-readiness loop + +```bash +EVAL_ITER=0 +while [ "$EVAL_ITER" -lt 3 ]; do + if (( $(echo "$PER_CASE_MIN >= 0.95" | bc -l) )); then + DIRECTION="add-harder" + elif (( $(echo "$PER_CASE_MIN < 0.50" | bc -l) )); then + # Run grader-vs-skill check: read failed trials' findings.txt; + # if models DID find the violations but grader scored wrong, + # DIRECTION="fix-graders"; else "simplify". + DIRECTION="..." + else + break # in (0.50, 0.95) — proceed to skill iteration + fi + + # Dispatch eval-iterate subagent (load prompts/eval-iterate.md, substitute vars) + # Wait. Verify it committed. + + # Re-run baseline (no resume — case set changed) + # Recompute PER_CASE_MIN + EVAL_ITER=$((EVAL_ITER + 1)) +done + +# Exit conditions +if (( $(echo "$PER_CASE_MIN >= 0.95" | bc -l) )); then + # Write analysis.md status:skill-genuinely-good and commit + exit 0 +fi +if (( $(echo "$PER_CASE_MIN < 0.50" | bc -l) )); then + # Write analysis.md status:blocked-by-skill-shape and commit + exit 0 +fi +``` + +When dispatching eval-iterate, use: + +- `subagent_type: "general-purpose"` +- `description: "eval-iterate ${DIRECTION} ${SKILL_ID}"` +- `prompt`: load `prompts/eval-iterate.md` and substitute `${SKILL_ID}`, `${WORKBENCH_DIR}`, `${SUITE_RESULT_PATH}`, `${DIRECTION}`, `${LESSONS_PATH}` + +## Phase 4: Skill iteration + +```bash +BASELINE_PER_CASE_MIN=$PER_CASE_MIN +ITER=0 +SUCCESS=false + +while [ "$ITER" -lt 2 ]; do + ITER=$((ITER + 1)) + + # Dispatch skill-iterate subagent + # Wait. Verify it committed. + + # Find the new latest .results//suite-result.json + # Recompute NEW_PER_CASE_MIN + + UPLIFT=$(node -e "console.log($NEW_PER_CASE_MIN - $BASELINE_PER_CASE_MIN)") + if (( $(echo "$UPLIFT >= 0.05" | bc -l) )); then + SUCCESS=true + break + fi + + # Track cost again — re-run-suite happened + # If cost > 10, exit budget-exceeded +done +``` + +When dispatching skill-iterate: + +- `subagent_type: "general-purpose"` +- `description: "skill-iterate ${ITER} ${SKILL_ID}"` +- `prompt`: load `prompts/skill-iterate.md` and substitute all 7 inputs + +## Phase 5: Package + +If `SUCCESS=true` or `uplift-too-small` (the additive change is still +worth packaging): + +```bash +# Read CONTEXT_FILE for packaging_dir +PACKAGING_DIR="${WORKBENCH}proposed-upstream-changes/${PACKAGING_DIR_FROM_CONTEXT}" +mkdir -p "$PACKAGING_DIR" + +# Read the original target file (before any v1.3 mods). +# This is the version on the FIRST commit on this orchestrator's branch. +git show HEAD~${ITER}:${TARGET_FILE} > "${PACKAGING_DIR}/before-$(basename ${TARGET_FILE})" + +# Current target file is the after version +cp "${TARGET_FILE}" "${PACKAGING_DIR}/after-$(basename ${TARGET_FILE})" + +# Write a packaging README +cat > "${PACKAGING_DIR}/README.md" < +baseline_per_case_min: 0.NN +final_per_case_min: 0.NN +iterations: eval=N, skill=N +total_cost_usd: NN.NN +--- + +# Auto-pilot run for ${SLUG} + +[3-6 short bullets covering: what was done, why, evidence, judgment calls] +``` + +## Phase 6: Final commit + +```bash +git checkout -b eval/auto-pilot/${SKILL_ID} 2>/dev/null || git checkout eval/auto-pilot/${SKILL_ID} +git add ${WORKBENCH}analysis.md +[ -d "$PACKAGING_DIR" ] && git add "$PACKAGING_DIR" +git add ${LATEST_RESULTS}suite-result.json +git commit -m "eval(auto-pilot): ${SKILL_ID} — status=$STATUS, baseline=$BASELINE_PER_CASE_MIN→$PER_CASE_MIN" +``` + +DO NOT push. + +## Return summary + +Return to caller (operator's CC session) under 300 words: + +- Branch: `eval/auto-pilot/${SKILL_ID}` +- Final commit SHA: `` +- Status: one of the exit statuses +- Baseline → final per-case-min +- Per-case breakdown (table) +- Proposed-upstream-changes path (if applicable) +- Cumulative cost: $N.NN + +## Hard rules + +- NEVER ask the operator a question mid-run. +- NEVER `git push`. +- NEVER modify files outside `${WORKBENCH}`. Context files (`${CONTEXT_FILE}`) are written by the research sub-subagent, not by you. +- Always commit before exiting (even on error: write `analysis.md` with the right status, then commit). +- Cost guard: hard stop at $10 cumulative `metrics.cost.total`. diff --git a/skills/auto-improve-orchestrator/prompts/research-upstream.md b/skills/auto-improve-orchestrator/prompts/research-upstream.md new file mode 100644 index 0000000..cd551bc --- /dev/null +++ b/skills/auto-improve-orchestrator/prompts/research-upstream.md @@ -0,0 +1,172 @@ +# Sub-subagent prompt: research upstream conventions + +You are a research subagent dispatched to study a single upstream +public-skill repo's contribution conventions. You produce a context +file that downstream subagents will use to ensure their proposed +changes fit the upstream's expectations and merge cleanly. + +## Inputs (templated) + +- `${SLUG}` — `//`. Example: `supabase/agent-skills/supabase-postgres-best-practices`. +- `${OUTPUT_PATH}` — where to write the context file. Default: `skills/auto-improve-orchestrator/references/contexts/-.md`. + +## Your job + +Read the target upstream repo's contribution conventions, frontmatter +spec, prefix taxonomy, and merged-PR shape patterns. Write a verbatim- +pastable context block to `${OUTPUT_PATH}` that the orchestrator and +skill-iterate subagents will consume. + +## Method + +Use `gh` CLI heavily (PR list/view, file API, search, repo-files API). +Use `WebFetch` sparingly for any README or external docs (e.g., +`docs..com` if a clear lead suggests external consumption). +Don't clone the repo — use the GitHub API and raw URLs. + +## Questions to answer + +For each, explain in your own words; cite source files/PRs you read. + +1. **On-disk inventory.** What's at `skills//`? List + `SKILL.md` plus every reference file under `references/`. For each + reference: filename, frontmatter values, content type. The + downstream subagents need a complete inventory to pick a non- + colliding `{prefix}` matching the existing taxonomy. + +2. **Frontmatter spec — exact schema.** Read the actual sanity-test + source code (e.g., under `tests/` or `scripts/`) and document the + EXACT required fields, allowed values for each enum field, and any + other validators. Don't assume from prior research — verify. + +3. **Reference file content conventions.** Pick 3 representative + existing references and document their structure: section headers, + code-block language tags, narrative-vs-list ratio, length range. + +4. **Concept-fit assessment.** If a downstream `target_file` doesn't + match the existing template (e.g., a meta-workflow file when all + existing references are single-rule transformations), flag this as + "shape-novel" with a rejection-risk estimate (LOW / MEDIUM / HIGH). + +5. **Prefix taxonomy.** What `{prefix}-` values exist? Are they locked + to a section taxonomy file (e.g., `_sections.md`)? Adding a new + prefix may require modifying that file (which violates additive- + only). + +6. **Recent merged additive PRs.** Look at the last 5–10 merged PRs + that added/modified content for THIS skill (or similar skills if + this one has few). Document: typical file count, body shape, + commit-message convention, time-to-merge, maintainer. + +7. **Closed-without-merge PRs.** Look at the last 3–5 closed PRs + that DIDN'T merge. What was the rejection signal? "Discussion- + first gate violated", "shape-novel", "duplicates X", etc. + +8. **Release Please / version bumping.** Is `metadata.version` in + `SKILL.md` auto-managed by Release Please? If yes, downstream + subagents must NOT manually bump it. + +9. **Architecture intent for SKILL.md vs `references/` split.** Why + split? Token economy? Per-rule contributions? Independent + versioning? The downstream skill-iterate subagent needs to know + whether to add new rules to `SKILL.md` or as a new `references/` + file. + +10. **Other consumers.** Is this skill referenced/installed/fetched by + anything outside the upstream repo? Install scripts, blog posts, + docs sites, downstream forks. Affects how additive-only the + proposed changes must be. + +11. **License + CLA.** What license? Any CLA bot? Affects whether + contributors need extra setup. + +12. **CI gates.** What does CI check? Frontmatter validators, format + checkers, test runners? + +## Output format + +Write `${OUTPUT_PATH}` with this structure: + +```markdown +# Auto-pilot context: / — + +## Repository facts + +- Repo: +- License: , CLA +- Maintainers: +- Merge style: , conventional commits enforced by +- CI: +- Discovery index / downstream sync: + +## Hard constraints (additive-only PR) + +- Add EXACTLY ONE new file at +- DO NOT modify +- Use only existing prefixes: +- DO NOT bump version () +- Other don'ts: + +## Frontmatter spec + +```yaml +--- +: +... +--- +``` + +## Content shape template + +[copy-and-fill template matching upstream's existing references] + +## Optimization target file + +**Edit:** `` +**Do NOT edit:** `` + +## Architecture intent + +[2-3 sentences explaining the upstream's design rationale] + +## Risk profile + +- HIGH/MEDIUM/LOW for + +## Pre-submit checklist + +1. +2. +... + +## Useful URLs + +- +``` + +## Commit + +After writing `${OUTPUT_PATH}`, commit on the current branch: + +```bash +git add ${OUTPUT_PATH} +git commit -m "docs(contexts): research upstream for ${SLUG}" +``` + +DO NOT push. + +## Return report + +Return to caller (orchestrator subagent) under 400 words: + +- On-disk inventory summary (file count + frontmatter overview) +- Frontmatter spec (the exact required fields) +- Content conventions (1 example structure) +- Concept fit (if applicable) +- Prefix recommendation +- Recent PR shape pattern +- Net rejection risk: LOW / MEDIUM / HIGH + rationale +- The verbatim context block path + +If a question genuinely can't be answered from public signals, say so +explicitly. Don't speculate. diff --git a/skills/auto-improve-orchestrator/prompts/skill-iterate.md b/skills/auto-improve-orchestrator/prompts/skill-iterate.md new file mode 100644 index 0000000..bd03796 --- /dev/null +++ b/skills/auto-improve-orchestrator/prompts/skill-iterate.md @@ -0,0 +1,113 @@ +# Sub-subagent prompt: iterate the skill content + +You are a sub-subagent dispatched to apply ONE additive recipe to a +target file in a skill's vendored content, then re-measure to see if +the change moves per-case-min by ≥+0.05. + +## Inputs (templated) + +- `${SKILL_ID}` — leaf id (e.g. `supabase-postgres-best-practices`). +- `${WORKBENCH_DIR}` — `examples/workbench/${SKILL_ID}/`. +- `${SUITE_RESULT_PATH}` — path to the most recent `.results//suite-result.json`. +- `${TARGET_FILE}` — path to the file to edit (extracted from the + `${CONTEXT_FILE}`'s "Optimization target file" directive). Example: + `${WORKBENCH_DIR}/references//.md`. +- `${CONTEXT_FILE}` — `skills/auto-improve-orchestrator/references/contexts/-.md`. +- `${LESSONS_PATH}` — `skills/auto-improve-orchestrator/references/lessons.md`. +- `${ITERATION}` — `1` or `2`. + +## What to do + +1. **Read the inputs.** Read `${SUITE_RESULT_PATH}`, `${CONTEXT_FILE}`, + `${LESSONS_PATH}`, `${TARGET_FILE}`. + +2. **Diagnose missed rules.** From `${SUITE_RESULT_PATH}`, identify: + - Per-case scores (group `results[]` by `caseName`, average trial + scores per case). + - Per-rule miss frequency (read failed trials' `findings.txt`, + identify which violation IDs were missed across trials/models). + - Categorize each missed rule: visible-pattern / absence-of-attribute / + state-machine / subjective. + +3. **Match to a recipe.** From `${LESSONS_PATH}` § "Optimization + patterns": + - **Recipe A** (two-pass workflow) — code-reviewer skills with mixed + presence/absence rules + - **Recipe B** (verify-tool-installed nudge) — tool-use skills where + models fall back to `curl`/`npm i` + - **Recipe C** (per-element checklists) — skills with rules grouped + by element type + - **Recipe D** (BAD/GOOD examples) — anti-patterns where the bad + pattern looks idiomatic + - **Recipe E** (rationale + bug-story) — state-machine violations + + Pick the recipe that best matches the dominant failure mode for + THIS iteration. If iteration 2 and recipe X was tried in iteration + 1 with insufficient uplift, pick a DIFFERENT recipe. + +4. **Apply the recipe ADDITIVELY to `${TARGET_FILE}`.** Read + `${CONTEXT_FILE}`'s "Hard constraints" — your edit must comply + (e.g., additive-only, terse imperative bullets, specific style). + Match the surrounding voice in `${TARGET_FILE}`. + +5. **Re-run the suite from `${WORKBENCH_DIR}`:** + + ```bash + cd ${WORKBENCH_DIR} + set -a; . ../../../.env; set +a + npx tsx ../../../src/cli.ts run-suite ./suite.yml --trials 3 \ + 2>&1 | tee /tmp/skill-iter-${ITERATION}-${SKILL_ID}.log + ``` + + Wait for completion (~30-60 min depending on suite size). + +6. **Compute new per_case_min.** Read the new `.results//suite-result.json`. Group by `caseName`. Per-case mean. Min across cases. + +7. **Compare to baseline.** The orchestrator passed you `${SUITE_RESULT_PATH}` (the previous result). Compute: + - `prev_per_case_min` from `${SUITE_RESULT_PATH}` + - `new_per_case_min` from the new run + - `uplift = new_per_case_min - prev_per_case_min` + +8. **Commit if uplift OR final iteration.** Even if uplift is small, + commit (the orchestrator may want the additive change as + `uplift-too-small` packaging): + + ```bash + git add ${TARGET_FILE} + git commit -m "feat(${SKILL_ID}): iterate ${ITERATION} — Recipe " + ``` + + Skip pushing. + +## Tools allowed + +Read, Edit, Bash, Glob. + +## Constraints + +- Additive ONLY. No deletions, no rewording of existing target-file + content. The `${CONTEXT_FILE}` may specify additional constraints + (e.g., specific frontmatter fields, prefix restrictions); honor all + of them. +- DO NOT touch `${WORKBENCH_DIR}/checks/`, `${WORKBENCH_DIR}/workspace/`, + `${WORKBENCH_DIR}/suite.yml` — those are eval harness, not skill + content. (Eval changes are the eval-iterate subagent's job.) +- DO NOT modify `${LESSONS_PATH}` or other reference material outside + `${TARGET_FILE}`. + +## Cost guard + +If the suite re-run fails or takes longer than the wrapper's default +timeout, exit with status `blocked-by-error` and a brief explanation. + +## Return report + +Return to caller (orchestrator subagent) under 300 words: + +- Recipe applied (A/B/C/D/E) +- Diff summary (what was added, ~lines) +- Per-case scores: prev → new (table) +- per_case_min: prev → new +- uplift: ±N.NN +- Verdict: `success` (uplift ≥ +0.05) | `uplift-too-small` (less) +- Branch + commit SHA + new `.results//` path diff --git a/skills/auto-improve-orchestrator/references/contexts/firebase-firebase-hosting-basics.md b/skills/auto-improve-orchestrator/references/contexts/firebase-firebase-hosting-basics.md new file mode 100644 index 0000000..657ecde --- /dev/null +++ b/skills/auto-improve-orchestrator/references/contexts/firebase-firebase-hosting-basics.md @@ -0,0 +1,81 @@ +# Auto-pilot context: firebase/agent-skills — firebase-hosting-basics + +## Repository facts + +- Repo: firebase/agent-skills +- License: Apache License 2.0, CLA required (Google CLA at cla.developers.google.com) +- Maintainers: @joehan (primary), Google Firebase team +- Merge style: squash merge; no Release Please detected; version in SKILL.md frontmatter is NOT auto-managed +- CI: GitHub Actions — `sync-genkit-skills.yml` (syncs genkit skills); no frontmatter validator CI found for hosting skills; no automated test runner in CI for this skill +- Discovery index / downstream sync: syncs to firebase-tools eval pipeline (`firebase/firebase-tools/scripts/agent-evals`); also installed via `npx skills add firebase/skills` + +## Hard constraints (additive-only PR) + +- Add content to `skills/firebase-hosting-basics/SKILL.md` OR add a new reference file under `skills/firebase-hosting-basics/references/` +- DO NOT modify `README.md`, `CONTRIBUTING.md`, `.github/`, or other skill directories +- Use existing structure: SKILL.md + references/configuration.md + references/deploying.md +- DO NOT bump `metadata.version` (no Release Please detected, but do not modify frontmatter unless a field is required) +- Incremental improvements → PR to `main` branch; new skills or significant changes → PR to `next` +- CLA required before PR can merge + +## Frontmatter spec + +No dedicated frontmatter validator script found in CI for this skill. Based on SKILL.md observed structure: + +```yaml +--- +name: firebase-hosting-basics +description: +--- +``` + +Two fields observed: `name` (string, matches dir name) and `description` (string). No enum constraints found. + +## Content shape template + +SKILL.md structure: +- H1 heading (skill name) +- `## Overview` — bullet list of key features +- Feature-comparison section (e.g., `## Hosting vs App Hosting`) +- `## Instructions` — numbered sub-sections linking to reference files +- Optional review/workflow section (already added: `## Configuration Review`) + +reference file structure (e.g., configuration.md, deploying.md): +- H1 heading +- `## Key Attributes` or `## Standard Deployment` — sections with JSON code blocks +- Code blocks tagged ` ```json ` or ` ```bash ` +- Mix of prose and code; mostly code-forward +- Length: ~60-115 lines per reference file + +## Optimization target file + +**Edit:** `skills/firebase-hosting-basics/SKILL.md` (for skill content improvements) +**Also editable:** `skills/firebase-hosting-basics/references/configuration.md`, `skills/firebase-hosting-basics/references/deploying.md` +**Do NOT edit:** other skill directories, CI files, CONTRIBUTING.md, README.md + +## Architecture intent + +The split between SKILL.md and references/ exists to allow per-topic contribution without touching the main skill entry point. SKILL.md is the agent's first read; it links to reference files for deeper specifics. Token economy is a secondary concern — the primary intent is per-topic versioning and contribution granularity. + +## Risk profile + +- LOW: incremental additions to SKILL.md or existing reference files for an established skill +- MEDIUM: adding new reference file (novel surface) +- Note: PR #120 was closed because it was submitted by an automated tool without a CLA; not a shape rejection. The change itself (improvements to a Dart skill) was substantive. Rejection was likely process-gate (CLA), not content quality. + +## Pre-submit checklist + +1. Ensure Google CLA is signed for the submitting account +2. Point PR to `main` branch (incremental improvement to existing skill) +3. Do not modify frontmatter `name` or bump version +4. Match existing voice: terse imperative bullets, JSON code blocks, second-person +5. Additive only — no deletions of existing content +6. Add test case to firebase-tools eval pipeline if adding new behavioral guidance + +## Useful URLs + +- SKILL.md: https://github.com/firebase/agent-skills/blob/main/skills/firebase-hosting-basics/SKILL.md +- configuration.md: https://github.com/firebase/agent-skills/blob/main/skills/firebase-hosting-basics/references/configuration.md +- deploying.md: https://github.com/firebase/agent-skills/blob/main/skills/firebase-hosting-basics/references/deploying.md +- CONTRIBUTING.md: https://github.com/firebase/agent-skills/blob/main/CONTRIBUTING.md +- Recent merged PRs: https://github.com/firebase/agent-skills/pulls?q=is%3Apr+is%3Amerged diff --git a/skills/auto-improve-orchestrator/references/contexts/google-labs-code-shadcn-ui.md b/skills/auto-improve-orchestrator/references/contexts/google-labs-code-shadcn-ui.md new file mode 100644 index 0000000..ab45a72 --- /dev/null +++ b/skills/auto-improve-orchestrator/references/contexts/google-labs-code-shadcn-ui.md @@ -0,0 +1,105 @@ +# Auto-pilot context: google-labs-code/stitch-skills — shadcn-ui + +## Repository facts + +- Repo: `google-labs-code/stitch-skills` +- License: Apache-2.0, Google CLA required (contributors must sign) +- Maintainers: google-labs-code org (Google Labs) +- Merge style: squash, conventional commits (no Release Please observed) +- CI: validates `react-components/` subtree only; shadcn-ui skill changes bypass CI gating +- Discovery index / downstream sync: listed in stitch-skills catalog; no external sync observed + +## Hard constraints (additive-only PR) + +- Add content ONLY to `skills/shadcn-ui/SKILL.md` (additive — new sections, examples, checklists) +- DO NOT modify other skill files or the `react-components/` source +- DO NOT bump `metadata.version` manually (no Release Please; version managed by maintainers) +- DO NOT delete or reword existing rules in `SKILL.md` +- DO NOT reorder existing sections +- Keep voice: imperative bullets, terse, code examples in TSX fenced blocks + +## Frontmatter spec + +```yaml +--- +name: shadcn-ui +description: +allowed-tools: + - "shadcn*:*" + - "mcp_shadcn*" + - "Read" + - "Write" + - "Bash" + - "web_fetch" +--- +``` + +## Content shape template + +Existing sections use h2 (`##`) for major topics, h3 (`###`) for sub-topics. +Code blocks use tsx/bash language tags. Narrative is imperative, concise bullet lists. +A representative addition looks like: + +```markdown +## Code Review Checklist + +When reviewing existing code for shadcn/ui best-practice compliance, scan each file in two passes: + +### Pass 1 — File placement and visible anti-patterns + +- [ ] **File location**: Custom/composed components must NOT be in `components/ui/`. +- [ ] **Class merging**: Every dynamic `className` must use `cn()`. + +### Pass 2 — Absence checks (per element) + +**Every interactive element** (`
`, etc.): +- Has `role="button"` +- Has `onKeyDown` keyboard handler +- Has `tabIndex={0}` +``` + +## Optimization target file + +**Edit:** `examples/workbench/shadcn-ui/references/shadcn-ui/SKILL.md` +(This is the vendored copy of the upstream `skills/shadcn-ui/SKILL.md`) + +**Do NOT edit:** +- `examples/workbench/shadcn-ui/suite.yml` +- `examples/workbench/shadcn-ui/checks/` +- `examples/workbench/shadcn-ui/workspace/` + +## Architecture intent + +The stitch-skills repo provides a catalog of agent skills for Google's Stitch platform. +`SKILL.md` for shadcn-ui is the complete, self-contained skill — no split between +discovery stub and content doc (unlike vercel-labs/agent-browser pattern). +All additions go directly into `SKILL.md`. + +## Risk profile + +- LOW: shadcn-ui CI path is not gated; additive changes to `SKILL.md` merge quickly +- MEDIUM: Google CLA adds a one-time contributor step for new contributors +- No Release Please; maintainer decides version bumps manually + +## Pre-submit checklist + +1. Verify the file path comment is correct (additive-only, no deletions) +2. Verify no existing rules were deleted or reworded +3. Verify new sections appear after existing content (append pattern) +4. Confirm TSX code blocks use `tsx` language tag +5. Confirm CLA signed if submitting externally + +## Previous iteration history (for re-fire context) + +The prior batch-2 run (with gpt-4o-mini, May 2026) applied: +- BAD/GOOD tsx example for V1-wrong-location (added under `### Extending Components`) +- Code Review Checklist section (Pass 1 + Pass 2) at end of SKILL.md + +These changes are already in `references/shadcn-ui/SKILL.md` in this workbench. +The re-fire (with gpt-5) measures whether frontier models with the improved SKILL.md +already score ≥ 0.95 (exit clean) or still have headroom for further improvement. + +## Useful URLs + +- Upstream repo: `https://github.com/google-labs-code/stitch-skills` +- Skill file: `https://github.com/google-labs-code/stitch-skills/blob/main/skills/shadcn-ui/SKILL.md` diff --git a/skills/auto-improve-orchestrator/references/contexts/supabase-supabase-postgres-best-practices.md b/skills/auto-improve-orchestrator/references/contexts/supabase-supabase-postgres-best-practices.md new file mode 100644 index 0000000..41c7d35 --- /dev/null +++ b/skills/auto-improve-orchestrator/references/contexts/supabase-supabase-postgres-best-practices.md @@ -0,0 +1,183 @@ +# Auto-pilot context: supabase/agent-skills — supabase-postgres-best-practices + +## Repository facts + +- Repo: supabase/agent-skills (default branch: main) +- License: MIT, no CLA +- Maintainers: gregnr (Supabase staff), Rodriguespn (active community maintainer) +- Merge style: squash, conventional commits enforced by Release Please +- CI: `Skills CI` runs `pnpm test:sanity` which executes `npx skills add` to + confirm install — does NOT validate per-reference frontmatter; convention + is enforced by maintainer review only +- Discovery index published at `.well-known/agent-skills/index.json` on every + release +- Downstream sync: supabase-community/supabase-plugin receives + workflow_dispatch on release + +## Hard constraints (additive-only PR) + +- Add EXACTLY ONE new file: + `skills/supabase-postgres-best-practices/references/{prefix}-{name}.md` +- DO NOT modify `SKILL.md` — Release Please owns `metadata.version`. Manual + edits cause merge conflicts with the bot's release PR. +- DO NOT modify `_sections.md`, `_template.md`, `_contributing.md`, the + SKILL.md "Rule Categories by Priority" table, `release-please-config.json`, + `package.json`, or `CHANGELOG.md` +- DO NOT add a new prefix. Use only the existing 8: `query-`, `conn-`, + `security-`, `schema-`, `lock-`, `data-`, `monitor-`, `advanced-` +- DO NOT bump `metadata.version` in SKILL.md +- DO NOT add README.md, INSTALLATION_GUIDE.md, QUICK_REFERENCE.md, or + CHANGELOG.md inside the skill (AGENTS.md explicitly forbids) + +## Frontmatter spec for the new reference file + +Required fields (exact form, comma-separated tags as a STRING, not a YAML +list): + +```yaml +--- +title: +impact: +impactDescription: +tags: <3-6 hyphenated-keywords, comma-separated, e.g. "indexes, performance, query-optimization"> +--- +``` + +## Content shape template (copy and fill) + +```markdown +--- +title: +impact: <CRITICAL|HIGH|MEDIUM-HIGH|MEDIUM|LOW-MEDIUM|LOW> +impactDescription: <quantified benefit> +tags: <comma, separated, keywords> +--- + +## <Same title as frontmatter> + +<1-2 sentence explanation of the problem and why it matters.> + +**Incorrect (<short parenthetical naming the problem>):** + +\`\`\`sql +-- comment explaining what makes this slow/wrong +<bad SQL> +\`\`\` + +**Correct (<short parenthetical naming the fix>):** + +\`\`\`sql +-- comment explaining why this is better +<good SQL> +\`\`\` + +<Optional: 1 follow-up subsection with another correct variant or trade-off note.> + +Reference: [<Link Text>](<https URL to postgres or supabase docs>) +``` + +Target length: 40–80 lines, 1.2–1.9 KB. Code blocks must be tagged `sql` +(lowercase keywords). Comments explain WHY not WHAT. Use semantic +table/column names (`users`, `orders`, `customer_id`). + +## Two-pass-review proposal — required reshaping + +The proposed content (two-pass review, presence vs absence violations) does +NOT fit the existing single-rule SQL-transformation convention. All 28 +existing references are concrete SQL anti-pattern fixes, not meta-workflow +guidance. `_contributing.md` Key Principle #1: "Show exact SQL rewrites. +Avoid philosophical advice." Key Principle #2: "Error-First Structure." + +**Reshape strategy (REQUIRED before writing):** Pick the single +highest-impact concrete SQL anti-pattern that two-pass review catches and +the single-pass workflow misses. Frame the reference around that +anti-pattern. Example framing: + +- Filename: `monitor-two-pass-review.md` (prefix `monitor-` because + diagnostic workflow) +- title: "Run Two Passes on Generated SQL Reviews" +- Incorrect block: a single-pass review that approves SQL missing a + `WHERE` (absence violation) or containing `DROP` (presence violation) +- Correct block: a two-pass review that catches both classes +- impact: MEDIUM (matches monitor-* siblings) +- impactDescription: "Catch absence-class bugs (missing WHERE, missing + index) that single-pass review skips" +- tags: review, diagnostics, code-review, sql-review + +If the reshape makes the SQL examples feel contrived, ABORT and surface a +`needs-discussion` signal in `analysis.md` (use status: +`blocked-by-skill-shape` and explain) instead of opening a borderline PR. +Open a GitHub Discussion under +<https://github.com/orgs/supabase/discussions> as the next manual step. + +## PR composition (for downstream packaging) + +- Branch name: `feat/{short-kebab-name}` (matches Rodriguespn convention) +- Title: `feat: <short imperative summary>` (use `feat:` for additive + content; `fix:` only for corrections — both currently bump patch under + bump-patch-for-minor-pre-major) +- Body shape (no PR template enforced — ignore the stale template at + `.github/`): + + ```markdown + ## Summary + + - <1-line what> + - <1-line why> + - <optional: 1 line on which prefix/section it slots into> + ``` + +- Optionally append `Resolves AI-NNN` if a Linear ticket exists; otherwise + omit. +- Single commit, single file, no co-authoring trailer required by repo + (their merge is squash). +- DO NOT include a "Test plan" section — no merged PR uses one. + +## Pre-submit checklist (auto-pilot must verify before declaring success) + +1. Exactly 1 file added under + `skills/supabase-postgres-best-practices/references/` +2. Filename matches `{existing-prefix}-{kebab-name}.md` +3. Frontmatter has `title`, `impact` (allowed enum), `impactDescription`, + `tags` (comma-separated string) +4. Body has `## <Title>`, `**Incorrect (...):**` block with ` ```sql `, + `**Correct (...):**` block with ` ```sql `, trailing + `Reference: [...](https://...)` link +5. Total file size 1.0–2.0 KB, 35–90 lines +6. SKILL.md, _sections.md,_template.md, _contributing.md, + release-please-config.json, package.json all UNCHANGED +7. `metadata.version` in SKILL.md UNCHANGED (currently "1.1.1" — Release + Please owns it) +8. No README.md/INSTALLATION_GUIDE.md/CHANGELOG.md added anywhere + +## Optimization target file + +**Edit:** `references/supabase-postgres-best-practices/{new-reference}.md` +(create it as a new file under the workbench's vendored references dir, then +package it as the proposed upstream addition). + +**Do NOT edit:** `references/supabase-postgres-best-practices/SKILL.md`. + +## Risk flags + +- HIGH: a meta-workflow reference is shape-novel for this skill; expect + "fit-the-convention" pushback from gregnr/Rodriguespn. Reshape to + concrete SQL anti-pattern as above OR open a Discussion first. +- MEDIUM: `npx skills add supabase/agent-skills` is publicly consumed; + additive-only is mandatory. +- LOW: external small `feat:` PRs do merge same-day if convention is + followed. + +## Useful URLs + +- Convention source of truth: + `skills/supabase-postgres-best-practices/references/_contributing.md` +- Section taxonomy: + `skills/supabase-postgres-best-practices/references/_sections.md` +- Reference template: + `skills/supabase-postgres-best-practices/references/_template.md` +- Frontmatter format spec: `AGENTS.md` (symlinked as `CLAUDE.md`) +- Release config: `release-please-config.json` +- Sanity test (does NOT validate frontmatter): `test/sanity.test.ts` +- CONTRIBUTING gate: `CONTRIBUTING.md` ("open a Discussion first" for + major changes) diff --git a/skills/auto-improve-orchestrator/references/contexts/vercel-labs-agent-browser.md b/skills/auto-improve-orchestrator/references/contexts/vercel-labs-agent-browser.md new file mode 100644 index 0000000..4c67f2c --- /dev/null +++ b/skills/auto-improve-orchestrator/references/contexts/vercel-labs-agent-browser.md @@ -0,0 +1,140 @@ +# Auto-pilot context: vercel-labs/agent-browser + +## Workbench is ALREADY BUILT (Tier-1 deeper eval) — skip rebuilding + +`examples/workbench/agent-browser/` is already populated with a +hand-built Tier-1 eval (4 cases beyond the 2 inherited Tier-0 cases = +6 cases total) that uses **pre-recorded snapshots played back by a +stateful fake CLI**. DO NOT rebuild it. Specifically: + +- **Phase 1 (Discover):** classify the skill (`tool-use`) but DO NOT + WebFetch upstream SKILL.md or rules docs. The vendored copies at + `references/agent-browser/SKILL.md` and + `references/agent-browser/agent-browser-core.md` are authoritative + for this pilot. +- **Phase 2 (Build suite):** SKIP ENTIRELY. Verify the existing + `suite.yml`, `workspace/`, `bin/agent-browser`, `references/`, and + `checks/` files are present and proceed. DO NOT overwrite ANY of + them. If a file is missing, exit `status: blocked-by-error` — + something has gone wrong with the cherry-pick, not your fault. +- **Phase 3 (Baseline):** run normally. Use the existing 6-case suite + with the standard model matrix. + +## Optimization target file + +**Edit:** `references/agent-browser/agent-browser-core.md` + +This is the vendored copy of upstream `skill-data/core/SKILL.md` — the +**actual workflow content** that teaches the agent how to use +agent-browser (navigate, snapshot, click @eN, type @eN, etc.). When the +agent runs `agent-browser skills get core`, the fake CLI emits this +file's contents. + +**Do NOT edit:** + +- `references/agent-browser/SKILL.md` — that's the discovery stub. Per + upstream `AGENTS.md`, it's intentionally thin and should not contain + workflow content. +- `bin/agent-browser` (the fake CLI), `suite.yml`, `workspace/`, or any + file under `checks/` — those are the eval harness and must stay + fixed; modifying them invalidates the measurement. + +## Architecture intent (from prior research) + +- Upstream repo is `vercel-labs/agent-browser` (Rust CLI for + Chrome/Chromium automation via CDP, designed for AI agents). +- The split is intentional: `skills/agent-browser/SKILL.md` is a thin + discovery stub; the real workflow content lives at + `skill-data/core/SKILL.md` and is loaded by the agent at runtime via + `agent-browser skills get core`. This keeps the SKILL.md token-cheap + and lets the workflow doc evolve with the CLI version. +- License: Apache-2.0, no CLA observed. +- Maintainer: `ctate` (sole, very active; same-day merges for clean + PRs). +- Strict CI: Rust fmt + clippy + test + dashboard `pnpm build` + + version-sync. **Docs-only changes (changes confined to + `skill-data/core/SKILL.md` or its references) pass automatically** — + do not touch any Rust file or dashboard code. +- Conventional commits required: `feat(scope):`, `fix(scope):`, + `docs(scope): description`. Scope is the subsystem (`docs`, + `doctor`, `native`, etc.). +- Per upstream AGENTS.md: "Any skill improvement PR must touch + `skill-data/core/SKILL.md` and its `references/` files, plus + `README` and the docs MDX pages." This 4-file mirror is a packaging + concern at PR-draft time, not auto-pilot scope. Auto-pilot should + produce just the proposed change to `skill-data/core/SKILL.md`; the + PR-draft step manually mirrors the relevant additions to README and + MDX. + +## What the deeper eval tests (informs which additions are likely valuable) + +| Tier-0 (existing) | Tier-1 (new) | +|---|---| +| Tool-was-invoked-at-all | **Ref correctness** — agent must `click @eN` where `@eN` is the right element from the recorded snapshot | +| `skills get core` was called first | **Snapshot-first discipline** — must `snapshot` before any `click`/`type` | +| `navigate` (not `curl`/`wget`) | **No CSS selectors** — `click "#button"` fails; `click @e3` passes | +| Snapshot/screenshot was called | **Content correctness** — `title.txt` must equal the actual title from the recording, not just non-empty | +| Output file is non-empty | **State-machine path completeness** — multi-step flows: `type @e5 → type @e6 → click @e7 → re-snapshot → extract` | + +Likely failure modes (and where additive guidance helps): + +- Agents fall back to CSS selectors when an element name "looks + obvious" → recipe: explicit "NEVER use CSS selectors. Always use + `@eN` refs from the most recent snapshot." (Recipe D — BAD/GOOD + example showing wrong vs right.) +- Agents skip `snapshot` when they "know" what's on the page → recipe: + "Always `snapshot` immediately after `navigate`, and again after any + `click`/`type` that changes state. The snapshot is your only source + of valid `@eN` refs." +- Agents pick the wrong `@eN` when multiple visually-similar elements + exist → recipe: per-action checklist "Read the snapshot's role + + label fields before choosing a ref." +- Agents extract content from the wrong recording field (kicker vs h1 + vs byline) → recipe: explicit "When asked for the article title, + use the `<h1>` text, not the kicker or byline." + +## Hard constraints + +1. **Additive only.** No deletions, no rewording of existing core.md + content. +2. **Style:** match the existing `agent-browser-core.md` voice — terse, + command-oriented bullet lists. Examples are encouraged (BAD/GOOD + blocks). No prose paragraphs. +3. **Length budget:** the existing `agent-browser-core.md` is ~90 lines. + Additions of 20–40 lines are reasonable; >60 lines is suspect (means + you're rewriting, not augmenting). +4. **Do not modify the fake CLI or the eval harness.** If you find a + genuine grader bug (e.g. graders mismark a correct trace), fix the + GRADER (per the prompt's grader-vs-skill check, free retry not + counted against iteration budget) — do NOT change the skill content + to satisfy a buggy grader. +5. **Fake-CLI awareness.** The fake CLI is stateful — it tracks which + page the agent is on (`/work/.ab-state`) and which post-action + snapshot to serve next. Recordings define which `@eN` refs exist on + each page+state. Your skill changes should encourage agents to + actually USE the snapshot's refs, not invent them. + +## Packaging + +When Phase 5 packages the proposed change: + +- Name files `before-skill-data-core-SKILL.md` / + `after-skill-data-core-SKILL.md` (the upstream target file is at + `skill-data/core/SKILL.md`) +- Put them under + `proposed-upstream-changes/vercel-labs-agent-browser/` (matches + prior pilot's directory layout) +- Per upstream AGENTS.md, the human PR-draft step (separate from this + pilot) will also mirror the relevant additions into upstream + `README.md` and the docs MDX pages. Auto-pilot is NOT responsible + for those mirrors. + +## Risk profile + +- LOW for additive changes to `skill-data/core/SKILL.md` if the diff + is small and matches existing voice. ctate ships docs-only PRs + same-day. +- MEDIUM if the diff is large or rewords existing content (slight + drift from "additive only" trips clippy-style review). +- HIGH if any non-docs file is touched (Rust changes trigger expensive + CI; not in scope here). diff --git a/skills/auto-improve-orchestrator/references/contexts/vercel-labs-web-design-guidelines.md b/skills/auto-improve-orchestrator/references/contexts/vercel-labs-web-design-guidelines.md new file mode 100644 index 0000000..6a264db --- /dev/null +++ b/skills/auto-improve-orchestrator/references/contexts/vercel-labs-web-design-guidelines.md @@ -0,0 +1,93 @@ +# Upstream context: vercel-labs/web-interface-guidelines + +This pilot targets the **rules doc** consumed by the +`vercel-labs/agent-skills/web-design-guidelines` skill, not the skill +itself. The SKILL.md is a thin Claude-Code-specific adapter and is NOT +the right optimization target for this pilot. + +## Optimization target file + +**Edit:** `references/web-design-guidelines/command.md` +**Do NOT edit:** `references/web-design-guidelines/SKILL.md` + +The SKILL.md is essentially untouched in upstream history (the last +substantive change was its initial commit). All meaningful improvements +should land in `command.md`, which is the canonical Vercel design +artifact distributed natively to 7 agent tools (Amp Code, Claude Code, +Cursor, OpenCode, Windsurf, Antigravity, Gemini CLI) via `install.sh`, +plus consumed by 10+ downstream repos via raw GitHub URL fetch. + +## Architecture intent (from upstream research) + +- `command.md` is the **canonical source of truth**. The skill is one + of many thin downstream adapters (others: 7 native tool installs + + the `vercel-labs/agent-skills` wrapper). +- `command.md`, `AGENTS.md`, and `README.md` are three stylistic + reformulations of the same rule set, each distributed through a + different channel. The auto-pilot only needs to optimize `command.md` + here; the AGENTS.md / README.md mirrors are produced manually at + PR-draft time. +- The skill always WebFetches `main` (no commit pinning), confirming + the rules doc is expected to evolve independently and downstream + consumers ride latest. + +## Hard constraints + +1. **Additive only.** Every merged PR in the last year is additive + (add rules, reword rules, fix links, add tool installers). Zero + restructure / reorganization PRs have been merged. **Do not delete, + reorder, or substantively reword existing rules.** +2. **Do not consolidate or restructure.** Merging `command.md` content + into the SKILL.md, or splitting `command.md` into multiple files, + would break `install.sh` and sever the canonical URL that 10+ + external repos and vercel.com/design link to. Risk of rejection: + HIGH. +3. **Two-pass workflow goes in `command.md` only.** Meta-instructions + about "how to apply the rules" (e.g. Pass 1 = visible / Pass 2 = + absences) fit `command.md` because it's the file consumed at audit + time. They do NOT fit `AGENTS.md` (ambient project context, read at + every coding action — the agent isn't "doing a review"). Out of + scope here, but worth knowing the scope limit. +4. **Rule additions / clarifications are in scope.** Per-element + checklists, BAD/GOOD examples, and explicit "missing X" rules are + the kinds of changes the merged-PR pattern welcomes (PR #23 is the + canonical precedent — adds `translate="no"` guideline as an additive + rule). +5. **Maintain frontmatter.** `command.md` starts with YAML frontmatter + (`description:`, `argument-hint:`). Preserve it. +6. **Style:** terse imperative bullets (e.g. `- Icon-only buttons need + \`aria-label\``). Match the existing voice. No prose, no rationale + in-line, no "MUST/SHOULD/NEVER" (that's the AGENTS.md voice). + +## Where headroom likely lies (prior from manual eval) + +The load-bearing prior from `auto-improve-skill-lessons.md` applies: +absence-type rules ("a missing attribute", "a missing branch") are +5-10x harder than presence-type rules ("a wrong token in code"). Prior +manual eval on this skill showed the biggest uplift came from: + +- Per-element absence checklists (`<img>`, `<input>`, `<button>` — + walk each one, flag missing attributes) +- BAD/GOOD code examples for anti-patterns where the bad pattern looks + idiomatic (`disabled={!form.valid}`, `onPaste={(e) => e.preventDefault()}`) +- Explicit "missing X" rules where the rule is currently phrased only + as a presence check + +When seeding violations, lean toward absence-type — that's where the +existing `command.md` likely has gaps and where additive rules will +create measurable uplift. + +## Packaging + +When Phase 5 packages the proposed change: + +- Name files `before-command.md` / `after-command.md` (not + `before-SKILL.md`) +- Put them under `proposed-upstream-changes/vercel-labs-web-interface-guidelines/` + (the rules-doc repo) +- Do not produce a `before-SKILL.md` / `after-SKILL.md` for the + `vercel-labs-agent-skills` repo — we're not changing it +- The PR-draft step (manual, separate from this pilot) will produce + `AGENTS.md` (MUST/SHOULD/NEVER style) and `README.md` (prose style) + reformulations of the same change set, following the PR #23 + precedent of touching all 3 files in one PR diff --git a/skills/auto-improve-orchestrator/references/lessons.md b/skills/auto-improve-orchestrator/references/lessons.md new file mode 100644 index 0000000..ae3a27f --- /dev/null +++ b/skills/auto-improve-orchestrator/references/lessons.md @@ -0,0 +1,362 @@ +# Auto-improve-skill — Lessons learned + +This is a **living doc**. The auto-pilot reads it during Phase 4 (Diagnose + ++ Modify). Every pilot adds patterns it discovered to the relevant +section. Pilot N benefits from patterns surfaced in pilots 1..N-1; the +auto-pilot doesn't have to rediscover them from zero. + +**How to use this from the prompt:** Phase 4 reads this file before +choosing what to modify. Match the failure pattern you observe to a +recipe below; if no recipe matches, do the diagnosis from first +principles, then add a new entry here at the end of the run. + +--- + +## The load-bearing prior + +> **Rules about *absence* (a missing attribute, a missing branch, a +> missing focus replacement) are 5–10× harder for models than rules +> about *presence* (a literal token in the code).** + +Source: manual web-design-guidelines run + auto-pilot supabase pilot +(2026-05-08) — both surfaced this independently. Use it to categorize +every missed rule before deciding what to modify. + +| Rule pattern | Relative miss rate | What helps | +|---|---|---| +| Visible bad pattern (literal token in code) | low | Often catches itself; the rule wording is enough | +| Anti-pattern that "looks normal" (e.g., `<button disabled={!form.valid}>`) | medium-high | BAD/GOOD example + rationale + bug-story | +| Missing attribute (e.g., `<img>` without `alt`) | high | Per-element checklist | +| Missing branch (e.g., empty-state, error-handling) | high | Per-call-site checklist | +| State-machine violation (e.g., disabled-then-enabled timing) | very high | Inline trace narration | +| Subjective / judgment-based rules | depends on phrasing | Often skip; LLM-as-judge is out of scope | + +--- + +## Optimization patterns (Phase-4 modify recipes) + +### A. Two-pass workflow + +**When to use:** code-reviewer skills with mixed presence/absence rules. + +**Recipe:** add a new section near the top of `SKILL.md`: + +```markdown +## How It Works + +Review each file in TWO passes — both are required. + +### Pass 1 — Visible anti-patterns + +Scan for literal patterns: `<div onClick>` for actions, `transition: all`, +`outline-none` className, `onPaste={(e) => e.preventDefault()}`, etc. + +### Pass 2 — Absences (per-element checklist) + +The most-missed rules are about *what's missing*. After Pass 1, walk +each `<img>`, `<input>`, `<button>`, etc. once and run the per-element +checklist in <rules-doc>. +``` + +**Empirical evidence:** + ++ Manual web-design-guidelines: 26/36 → 31/36 (+14pp) on the 4-case + suite, sonnet 83% → 100% ++ Auto-pilot supabase: 0.54 → 0.86 (+32pp) on 9 SQL violations + +**When NOT to use:** document-producer skills (the agent builds output, +isn't reviewing). Tool-use skills usually want a different fix (see B). + +### B. Verify-tool-installed nudge + +**When to use:** tool-use skills where models reach for `curl`, `npm i`, +or other fallbacks instead of the prescribed CLI. + +**Recipe:** add a "Setup" or "Pre-flight" section to `SKILL.md`: + +```markdown +## Pre-flight + +The CLI is pre-installed. Verify with `which agent-browser` before +starting. **Do not** `npm install` it; **do not** fall back to `curl` +for HTTP fetches. +``` + +**Empirical evidence:** auto-pilot agent-browser pilot — gpt-5-mini and +gemini both fell back to `curl` or tried `npm i -g` before the nudge. + +### C. Per-element checklists + +**When to use:** code-reviewer skills with rules grouped by element type +(every `<img>`, every `<input>`, every `<form>`). + +**Recipe:** in the rules doc, add a "Per-element review" section +listing checks per element type. Format that worked for +web-design-guidelines: + +```markdown +## Per-element review (Pass 2) + +**Every `<img>`:** +- explicit `width` AND `height` (prevents CLS) +- above-fold critical → `priority` or `fetchpriority="high"` +- below-fold → `loading="lazy"` + +**Every `<input>`:** +- `autoComplete` set +- correct `type` +- `<label htmlFor>` or wrapping `<label>` +- emails/codes → `spellCheck={false}` +``` + +**Empirical evidence:** closed most absence-type misses on +web-design-guidelines after the two-pass workflow was added. + +### D. BAD / GOOD code examples + +**When to use:** anti-pattern rules where the bad pattern looks +idiomatic; rules that depend on a state-machine; rules that contradict +common React/JS patterns the model has seen in training data. + +**Recipe:** under the rule, add a fenced JSX block with BAD and GOOD +side-by-side. Example structure (rendered into the rules doc verbatim): + +````text +**Submit button stays enabled until request starts.** + +```jsx +// BAD: button disables based on form state. User types → deletes → +// button flickers off → autofill races with state. +<button type="submit" disabled={!email}>Submit</button> + +// GOOD: stays enabled. Spinner appears during the request. +<button type="submit" disabled={submitting}> + {submitting ? <Spinner /> : 'Submit'} +</button> +``` +```` + +**Empirical evidence:** manual web-design-guidelines run — the rules +that needed examples (submit-disabled, paste-blocking, missing +autoComplete, image priority hint) all closed their miss rates by 60-100% +after the example was added. + +### E. Rationale + bug-story + +**When to use:** state-machine violations, lifecycle bugs, +non-obvious-failure rules. + +**Recipe:** narrate the failure case inline with the rule. Example: + +```markdown +NEVER `disabled={!form.valid}` — the user types, then deletes a +character to fix a typo, the button flickers off, and the paste-fill +races with state. Tested users will assume the button is broken. +``` + +The narration gives the model a "why this rule matters" hook that pure +declarative rules don't provide. + +--- + +## Grader-reliability patterns (Phase-2 build-suite recipes) + +These are common ways graders go wrong on first build. Pre-tune your +graders to avoid them; if you see the failure mode at baseline, fix the +grader as iteration 1 (do not propose a skill change yet). + +### G1. Line tolerance ±5–8 (not ±0–3) + +LLM line-counting is unreliable. Models report violations 1-3 lines off +from the actual line in multi-line JSX/SQL/code. Use the `looseRange` +helper (default tolerance ±8): + +```javascript +{ id: 'rule-id', lines: looseRange(18), keywords: [/.../i] } +// Accepts lines 10-26. +``` + +`looseRange(N, tolerance)` is defined in `_grader-utils.mjs`. Prefer it +over hand-rolling `range(N-3, N+3)` — the default already absorbs the +common drift width seen across all 4 prior pilots. + +### G2. Hyphen-tolerant keyword regex + +Models output "empty-state" when the rule says "empty state", or +"clickable-handler" when the rule says "clickable handler". Use the +`fuzzyKeyword` helper: + +```javascript +keywords: [fuzzyKeyword('empty state')] // matches "empty state" and "empty-state" +keywords: [fuzzyKeyword('aria label')] // matches "aria-label" and "aria label" +``` + +`fuzzyKeyword(phrase)` is defined in `_grader-utils.mjs`. It escapes +regex metacharacters and replaces internal whitespace with `[-\s]*`, +so callers don't have to hand-roll the regex. + +### G3. Per-finding-line keyword matching (not whole-text) + +Don't `keywords.some(re => re.test(fullText))` — that produces spurious +cross-matches when keyword X appears in a different rule's finding line. +Use `_grader-utils.mjs`'s built-in per-finding-line matcher (split +findings.txt by line, match within each line). + +### G4. Multiple keyword variants + +Models phrase the same concept several ways: + ++ "covering" / "does not cover" / "missing covering index" ++ "label" / "aria-label" / "labeled" ++ "hover" / "hover state" / "hover:bg-*" + +Use the `tolerantKeyword` helper for word-stem matching: + +```javascript +keywords: [tolerantKeyword('cover')] // matches "cover", "covering", "covered" +keywords: [tolerantKeyword('label')] // matches "label", "labeled", "labels" +``` + +For multiple distinct stems on the same rule, use an array — the grader +treats them as alternatives: + +```javascript +keywords: [tolerantKeyword('hover'), fuzzyKeyword('hover state')] +``` + +Both `tolerantKeyword` and `fuzzyKeyword` are defined in `_grader-utils.mjs`. + +### G5. Set-semantics for sibling/list assertions + +When the grader checks a list of items, sort and compare — the model +emits items in different orders. + +```javascript +const names = pdf.repo_siblings_in_cohort_names.split(' | '); +assert.deepEqual(names.sort(), ['docx', 'xlsx']); // not deepEqual to ordered array +``` + +### G6. Verbosity floor for terse models + +Gemini sometimes outputs 3-4 line responses. Don't grade strict-pass on +"all 5 violations found" — many gemini failures are *truncated output*, +not missed rules. Compute rule-coverage rate (sum-found / sum-expected) +as the load-bearing metric instead of binary pass. + +--- + +## Default seeded violation types per skill shape + +When the auto-pilot builds a case in Phase 2, seed at least one +violation from each category for the skill's shape. This ensures +coverage of the absence-vs-presence axis and exposes whether the skill +needs Pattern A (two-pass workflow), Pattern C (per-element +checklists), or something else. + +### code-reviewer + +Seed at least one of each: + +1. Visible token misuse (e.g., `<div onClick>` for action) +2. Missing attribute (e.g., `<input>` without `autoComplete`) +3. Missing branch / no-empty-state (e.g., `array.map()` with no fallback for `[]`) +4. Anti-pattern that "looks normal" (e.g., `disabled={!form.valid}`) +5. State-machine violation (e.g., submit timing, focus on error) + +### tool-use / mcp-driver + +Seed at least one of each: + +1. Reaches-for-fallback (model uses `curl`/`npm i` instead of the prescribed CLI) +2. Wrong tool flag (passes `--user` when the skill calls for `--principal`) +3. Missing required step (skips snapshot, skips re-snapshot after action) +4. Output not validated (returns trace.jsonl without checking required artifacts) + +### document-producer + +Seed at least one of each: + +1. Missing required field in output (e.g., `answer.json` has no `risk_flags` key) +2. Wrong format (e.g., `2025-01-15` when the skill says `Intl.DateTimeFormat`) +3. Edge-case input (e.g., empty input, very long input, pre-corrupted file) +4. Format-only-correct: output validates but is unusable (e.g., PDF renders blank) + +### code-patterns + +Seed at least one of each: + +1. Wrong convention applied (skill says use 2-space indent, output uses 4) +2. Pattern not applied at all (skill says use `useReducer`, output uses `useState`) +3. Incorrect composition (uses prescribed pattern but in the wrong order) + +--- + +## Failure modes / known anti-patterns to avoid (Phase-4 don'ts) + +### Don't manufacture problems + +If baseline rule-coverage is ≥ 0.95, *exit clean*. Do not propose +modifications to a skill that already works. The goal is upstream PR +quality, not modification volume. + +**Source:** auto-pilot pdf pilot — baseline 1.00, no modifications +proposed. Maintainers will lose trust in our PRs if we open them for +non-issues. + +### Don't make breaking changes + +All proposed modifications must be **additive**: new sections, new +examples, new checklists. Never: + ++ Delete an existing rule ++ Change the wording of an existing rule ++ Reorder existing sections ++ Remove URLs or references in the skill + +This keeps the diff vs upstream small and the PR low-risk. + +### Don't burn iteration 1 on the wrong problem + +When baseline scores low, *first* check: is the grader the problem? Look +at the actual `findings.txt` from failed trials. If models *did* identify +the violations but the grader scored them wrong (line numbers off, +keyword mismatch, format variant), fix the grader as iteration 0 (don't +count it against the 2-iteration budget). + +**Source:** auto-pilot supabase + agent-browser pilots — both spent +iteration 1 on grader fixes before reaching skill modification. + +--- + +## Run-record protocol + +Every pilot adds an entry to one of these tables when it discovers +something new. Format: + +```markdown +**[skill-name] (date):** what was new — link to commit. +``` + +### Patterns added by pilots + ++ **manual web-design-guidelines (2026-05-06):** Two-pass workflow + per-element + checklists + 5 BAD/GOOD examples. Lifted 4-case suite from 72% → 86%. ++ **auto-pilot supabase (2026-05-08):** Independently rediscovered two-pass + workflow. Added it to a SQL skill. 0.54 → 0.86. ++ **auto-pilot agent-browser (2026-05-08):** Found that grader was over-strict + for non-interactive ops. Demoted snapshot from required to evidence-only. + Also surfaced "Verify-tool-installed nudge" pattern. ++ **auto-pilot pdf (2026-05-08):** Validated "exit clean on already-good skill" + — no modifications proposed; baseline 1.00. + +### Grader patterns added by pilots + ++ **manual web-design-guidelines (2026-05-06):** ±5-8 line tolerance, hyphen + regex, per-finding-line matching, keyword variants. ++ **auto-pilot supabase (2026-05-08):** "covering" / "does not cover" alternation + pattern. Confirmed ±3 → ±8 line widening is needed by default. + ++ **auto-pilot supabase v2 (2026-05-12):** Upstream constraints required adding a new reference file (`monitor-two-pass-review.md`) instead of editing SKILL.md. Baseline was already 1.00 (calibrated graders from prior run). Pattern: when a re-run starts from calibrated graders, the Phase 3 exit condition fires before Phase 4 — the "modification" step then serves purely as upstream PR packaging rather than eval improvement. + +(Future pilots: append your additions here.) diff --git a/skills/auto-improve-orchestrator/references/workflow.md b/skills/auto-improve-orchestrator/references/workflow.md new file mode 100644 index 0000000..c5728e3 --- /dev/null +++ b/skills/auto-improve-orchestrator/references/workflow.md @@ -0,0 +1,174 @@ +# auto-improve-orchestrator workflow + +This is a human-readable description of the algorithm the orchestrator +subagent follows. The same logic is embedded in +`../prompts/orchestrator.md` (the AI-targeted prompt template). + +## Inputs + +- `${SLUG}`: `<owner>/<repo>/<skill-id>` (e.g. `supabase/agent-skills/supabase-postgres-best-practices`) +- `${REFRESH_CONTEXT}`: optional flag, default false. If true, force re-research even if cached context file exists. + +## Setup + +1. Parse `SLUG` into `OWNER`, `REPO`, `SKILL_ID`. +2. Verify the orchestrator is running in a git worktree (the `isolation: "worktree"` requirement). +3. Verify the skill's workbench exists at `examples/workbench/<SKILL_ID>/`. If not, exit `blocked-by-missing-workbench`. +4. Define paths: + - `WORKBENCH = examples/workbench/<SKILL_ID>/` + - `CONTEXT_FILE = skills/auto-improve-orchestrator/references/contexts/<OWNER>-<SKILL_ID>.md` +5. Verify `OPENROUTER_API_KEY` is set (source `.env` from the main repo if needed). + +## Phase 0: Research + +If `CONTEXT_FILE` exists AND not `REFRESH_CONTEXT`: + +- Read `CONTEXT_FILE`. Use it. + +Else: + +- Dispatch sub-subagent `prompts/research-upstream.md` with input `${SLUG}`. +- Wait for completion. Verify `CONTEXT_FILE` was written. +- Read it. + +Extract from `CONTEXT_FILE`: + +- `target_file` (the file the skill-iterate subagent will edit) +- `packaging_dir` (where `proposed-upstream-changes/` subdir should go) +- `additive_only_constraint` and any other hard constraints + +## Phase 3: Baseline measurement + +Look for the most recent `.results/<ts>/suite-result.json` in `WORKBENCH`. + +If found AND its case set matches current `suite.yml`: + +- Read it (resume — skip baseline run). + +Else: + +- Run from `WORKBENCH`: `npx tsx ../../../src/cli.ts run-suite ./suite.yml --trials 3` +- Wait for completion (long-running, ~30-60 min depending on suite size). +- Read the new `suite-result.json`. + +Compute `per_case_min` from `suite-result.json`: + +- Group `results[]` by `caseName`. +- Per-case mean = average of trial scores for that case (across all models and trials). +- `per_case_min = min(per-case means across all cases)`. + +## Phase 3.5: Eval-readiness loop + +```text +EVAL_ITER = 0 +While per_case_min OUTSIDE (0.50, 0.95) AND EVAL_ITER < 3: + If per_case_min >= 0.95: + DIRECTION = "add-harder" + Else if per_case_min < 0.50: + Run grader-vs-skill check: + - Read failed trials' findings.txt + - If models DID find the violations but grader scored wrong: + DIRECTION = "fix-graders" + - Else: DIRECTION = "simplify" + + Dispatch sub-subagent prompts/eval-iterate.md with: + SKILL_ID, WORKBENCH_DIR, SUITE_RESULT_PATH, DIRECTION, LESSONS_PATH + Wait for completion. Verify it committed workbench changes. + + Re-run baseline measurement (Phase 3, no resume — case set changed). + EVAL_ITER += 1 +``` + +Exit conditions: + +- Still `per_case_min >= 0.95` after 3 iterations: exit `skill-genuinely-good`. +- Still `per_case_min < 0.50` after 3 iterations: exit `blocked-by-skill-shape`. + +## Phase 4: Skill iteration + +```text +baseline_per_case_min = per_case_min +ITER = 0 + +While ITER < 2: + ITER += 1 + Dispatch sub-subagent prompts/skill-iterate.md with: + SKILL_ID, WORKBENCH_DIR, SUITE_RESULT_PATH, TARGET_FILE (from context), CONTEXT_FILE, LESSONS_PATH, ITER + Wait for completion. Verify it committed changes. + + Read the new suite-result.json from the latest .results/<ts>/. + new_per_case_min = recompute + + If new_per_case_min - baseline_per_case_min >= 0.05: + Success — break. +``` + +Exit if no iteration cleared +0.05: status `uplift-too-small`. Still package the changes (they're additive — the operator can decide whether to ship). + +## Phase 5: Package + +Compose `WORKBENCH/proposed-upstream-changes/<packaging_dir>/`: + +- `before-<target_file_basename>` — original target file (read from upstream's vendored copy, before any v1.3 modifications) +- `after-<target_file_basename>` — current target file content +- `README.md` — description, evidence (per-case breakdown table), how to apply + +Write `WORKBENCH/analysis.md`: + +```markdown +--- +skill: <SLUG> +status: success | uplift-too-small | skill-genuinely-good | blocked-by-skill-shape | blocked-by-error +classification: <from CONTEXT_FILE> +baseline_per_case_min: 0.NN +final_per_case_min: 0.NN +iterations: eval=N, skill=N +total_cost_usd: NN.NN +--- + +# Auto-pilot run for <SLUG> + +[3-6 short bullets covering: classification, what the eval surfaced, +what was changed and why, uplift result, any judgment calls.] +``` + +## Phase 6: Commit + +```bash +git checkout -b eval/auto-pilot/<SKILL_ID> +git add WORKBENCH/analysis.md +git add WORKBENCH/proposed-upstream-changes/ # if exists +git add WORKBENCH/.results/<latest>/suite-result.json +# Skill modifications were committed by the skill-iterate subagent already +git commit -m "eval(auto-pilot): <SKILL_ID> — status=<S>, baseline=<B>→<F>" +``` + +DO NOT `git push`. + +## Cost tracking + +The orchestrator tracks cumulative `metrics.cost.total` from each `run-suite` invocation. + +- Soft warning at $5: print to stderr but continue. +- Hard stop at $10: write `analysis.md` immediately with `status: budget-exceeded` and exit. + +Sub-subagent dispatches (research / eval-iterate / skill-iterate) use the operator's Claude Code session under their plan and have no marginal cost. + +## Return summary + +The orchestrator returns to the caller (operator's CC session): + +- `branch`: `eval/auto-pilot/<SKILL_ID>` +- `commit`: `<SHA>` +- `status`: one of the exit statuses above +- `baseline_per_case_min` → `final_per_case_min` +- per-case breakdown table +- `proposed-upstream-changes/` path (if status is `success` or `uplift-too-small`) + +## Rules of engagement + +- NEVER ask the operator a question mid-run. Decide based on the algorithm. +- NEVER `git push` to a remote. +- NEVER modify files outside `WORKBENCH` or `references/contexts/` (the research subagent writes context files; the orchestrator never modifies them directly). +- Always commit before exiting (atomic with `analysis.md` write — write analysis, then commit, then exit). +- Cost guard: track cumulative cost from each `run-suite`'s `metrics.cost.total`. diff --git a/src/workbench/docker-runner.ts b/src/workbench/docker-runner.ts index 292ce6a..b0f5451 100644 --- a/src/workbench/docker-runner.ts +++ b/src/workbench/docker-runner.ts @@ -1,4 +1,4 @@ -import { cpSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { chmodSync, cpSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -212,6 +212,8 @@ async function copyAgentResults(containerName: string, resultsDir: string, repoR copy.stderr.trim(), ].filter(Boolean).join('\n\n')); } + + await runShellCommand(`chmod -R a+rw ${shellQuote(resultsDir)}`, { cwd: repoRoot }); } async function removeContainer(containerName: string, repoRoot: string): Promise<void> { @@ -408,6 +410,8 @@ export function prepareDockerWorkbenchRun( mkdirSync(referencesDir, { recursive: true }); mkdirSync(workDir, { recursive: true }); mkdirSync(resultsDir, { recursive: true }); + chmodSync(workDir, 0o777); + chmodSync(resultsDir, 0o777); copyDirectoryContents(resolvedCase.referencesDir, referencesDir); copyCaseSupportDirs(resolvedCase.configDir, caseDir); @@ -434,7 +438,15 @@ export function prepareDockerWorkbenchRun( resultPath: join(resultsDir, 'result.json'), tracePath: join(resultsDir, 'trace.jsonl'), ...(mcpConfigPath ? { mcpConfigPath } : {}), - cleanup: () => rmSync(tempDir, { recursive: true, force: true }), + cleanup: () => { + try { + rmSync(tempDir, { recursive: true, force: true }); + } catch (error) { + // The container (uid 10001) may write subdirs (.cache, .venv) that the host user + // cannot delete. Don't let cleanup failures kill the run; tmpfiles.d will sweep /tmp later. + console.warn(`workbench: could not remove ${tempDir}: ${error instanceof Error ? error.message : String(error)}`); + } + }, }; }