diff --git a/Makefile b/Makefile index dfd4418..a70e003 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: sandbox-setup sandbox-clean dev build web update-analytics-data test clear-data sandbox-setup: build - ./build/skillwalker sandbox setup + ./build/skillwalker sandbox create sandbox-clean: sbx rm --force claude-skills-skillwalker diff --git a/README.md b/README.md index 37d6e95..7fe365e 100644 --- a/README.md +++ b/README.md @@ -40,12 +40,16 @@ All commands run from the **repository root**. ```bash sbx login - ./build/skillwalker sandbox setup + ./build/skillwalker sandbox create ``` Complete the login in the Claude TUI. If you aren't prompted, run `/login`. When setup finishes, exit with `/exit`. + To move to a newer Claude Code release later, run + `./build/skillwalker sandbox update`. It deletes the sandbox and recreates it + from the latest template, so you will log in again. + You're ready. Now choose what you want to measure. ## What do you want to measure? diff --git a/docs/agent-call-improvement-loop.md b/docs/agent-call-improvement-loop.md index 4bd3556..c99d80f 100644 --- a/docs/agent-call-improvement-loop.md +++ b/docs/agent-call-improvement-loop.md @@ -24,7 +24,7 @@ At the end of every iteration, ACIL prints a progress summary. When the loop exi ```bash make build -./build/skillwalker sandbox setup +./build/skillwalker sandbox create ``` ## Eval Requirements diff --git a/docs/cli.md b/docs/cli.md index e0838f7..1c2b723 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -12,7 +12,7 @@ The `@testdouble/skillwalker-cli` package is the command-line entry point for Sk ## Summary -- Six top-level commands exposed via the `skillwalker` binary: `test-run`, `test-eval`, `scil`, `acil`, `update-analytics-data`, and `sandbox`. The Test Sandbox lifecycle lives under `sandbox` as three sub-commands: `sandbox setup`, `sandbox clean`, and `sandbox shell` +- Six top-level commands exposed via the `skillwalker` binary: `test-run`, `test-eval`, `scil`, `acil`, `update-analytics-data`, and `sandbox`. The Test Sandbox lifecycle lives under `sandbox` as four sub-commands: `sandbox create`, `sandbox update`, `sandbox clean`, and `sandbox shell` - All test execution happens inside a Test Sandbox via `@testdouble/sandbox-integration`, with Claude invoked through `@testdouble/claude-integration` - Two test runner types handle different test kinds: prompt tests (full Claude sessions) and skill-call tests (trigger detection with temporary stripped-down plugins) - The SCIL (Skill Call Improvement Loop) command iteratively improves skill descriptions by running evaluation cycles and using Claude to generate better descriptions @@ -38,7 +38,8 @@ flowchart TB acil["acil"] analytics["update-analytics"] sandbox["sandbox"] - setup["sandbox setup"] + create["sandbox create"] + update["sandbox update"] clean["sandbox clean"] shell["sandbox shell"] @@ -46,7 +47,7 @@ flowchart TB evals["skillwalker-evals
evaluate TestRun"] scilsteps["scil steps 1-10
loop.ts"] acilsteps["acil steps 1-10
loop.ts"] - si["sandbox-integration
openShell · removeSandbox · createSandbox"] + si["sandbox-integration
openShell · removeSandbox · createSandbox · updateSandbox"] data["@testdouble/skillwalker-data
types, config, JSONL I/O, analytics, SCIL, ACIL"] @@ -57,7 +58,8 @@ flowchart TB dispatch --> acil --> acilsteps --> data dispatch --> analytics --> data dispatch --> sandbox - sandbox --> setup --> si + sandbox --> create --> si + sandbox --> update --> si sandbox --> clean --> si sandbox --> shell --> si ``` @@ -73,10 +75,11 @@ flowchart TB | `packages/cli/src/commands/scil.ts` | `scil` command — entry point for the Skill Call Improvement Loop | | `packages/cli/src/commands/acil.ts` | `acil` command — entry point for the Agent Call Improvement Loop | | `packages/cli/src/commands/update-analytics.ts` | `update-analytics-data` command — imports JSONL to Parquet | -| `packages/cli/src/commands/sandbox.ts` | `sandbox` parent command — registers the three sub-commands below | +| `packages/cli/src/commands/sandbox.ts` | `sandbox` parent command — registers the four sub-commands below | | `packages/cli/src/commands/sandbox/shell.ts` | `sandbox shell` sub-command — opens interactive shell in Test Sandbox | | `packages/cli/src/commands/sandbox/clean.ts` | `sandbox clean` sub-command — removes the Test Sandbox | -| `packages/cli/src/commands/sandbox/setup.ts` | `sandbox setup` sub-command — creates sandbox and authenticates via OAuth | +| `packages/cli/src/commands/sandbox/create.ts` | `sandbox create` sub-command — creates sandbox and authenticates via OAuth | +| `packages/cli/src/commands/sandbox/update.ts` | `sandbox update` sub-command — deletes the sandbox and recreates it from the latest Claude Code template | ## Core Types @@ -93,7 +96,7 @@ Each command module in `packages/cli/src/commands/` is a thin Yargs wrapper that - **scil** — Calls `runScilLoop()` from the execution package for iterative skill description improvement - **acil** — Calls `runAcilLoop()` from the execution package for iterative agent description improvement - **update-analytics-data** — Calls the execution package's analytics ingestion -- **sandbox setup** / **sandbox clean** / **sandbox shell** — Delegate to `@testdouble/sandbox-integration` for sandbox lifecycle. The `sandbox` parent holds no logic of its own; it registers the three sub-commands and requires one of them +- **sandbox create** / **sandbox update** / **sandbox clean** / **sandbox shell** — Delegate to `@testdouble/sandbox-integration` for sandbox lifecycle. The `sandbox` parent holds no logic of its own; it registers the four sub-commands and requires one of them See [execution.md](./execution.md) for implementation details of each pipeline (test-run steps, test-eval steps, SCIL loop, ACIL loop, concurrency pool, scoring, error hierarchy). @@ -128,7 +131,8 @@ The CLI catches `SkillwalkerError` at the top level (`index.ts`) and writes the - `packages/cli/src/commands/scil.test.ts` — Tests `scil` command builder and handler - `packages/cli/src/commands/acil.test.ts` — Tests `acil` command builder and handler - `packages/cli/src/commands/update-analytics.test.ts` — Tests `update-analytics-data` command -- `packages/cli/src/commands/sandbox/setup.test.ts` — Tests `sandbox setup` sub-command +- `packages/cli/src/commands/sandbox/create.test.ts` — Tests `sandbox create` sub-command +- `packages/cli/src/commands/sandbox/update.test.ts` — Tests `sandbox update` sub-command - `packages/cli/src/commands/sandbox/clean.test.ts` — Tests `sandbox clean` sub-command - `packages/cli/src/commands/sandbox/shell.test.ts` — Tests `sandbox shell` sub-command diff --git a/docs/sandbox-integration-package.md b/docs/sandbox-integration-package.md index 98b731e..459adc4 100644 --- a/docs/sandbox-integration-package.md +++ b/docs/sandbox-integration-package.md @@ -16,7 +16,7 @@ The `@testdouble/sandbox-integration` package is the single point of contact for No other package in Skillwalker spawns `sbx` processes directly. All Sandbox CLI access is funneled through this package, which provides two categories of functionality: 1. **Sandbox execution** -- verifying the sandbox exists and running commands inside it (`ensureSandboxExists`, `execInSandbox`) -2. **Lifecycle management** -- creating, removing, and opening interactive shells in the sandbox (`createSandbox`, `removeSandbox`, `openShell`) +2. **Lifecycle management** -- creating, removing, and opening interactive shells in the sandbox (`createSandbox`, `updateSandbox`, `removeSandbox`, `openShell`) The package returns a clean `SandboxResult` type instead of exposing raw `Bun.spawn` process handles, giving consumers a stable interface decoupled from the process spawning implementation. @@ -26,7 +26,7 @@ All public symbols are re-exported from the barrel file `index.ts`: ```typescript export { SANDBOX_NAME, ensureSandboxExists, execInSandbox } from './src/sandbox.js' -export { createSandbox, removeSandbox, openShell } from './src/lifecycle.js' +export { createSandbox, openShell, removeSandbox, updateSandbox } from './src/lifecycle.js' export { SandboxError } from './src/errors.js' export type { SandboxResult } from './src/types.js' ``` @@ -72,7 +72,7 @@ class SandboxError extends Error { async function ensureSandboxExists(): Promise ``` -Pre-flight check that the sandbox is running. Runs `sbx ls --quiet` and verifies `SANDBOX_NAME` exactly matches one output line. Throws `SandboxError` with `exitCode: null` if the sandbox is not found, with a message directing the user to run `./build/skillwalker sandbox setup`. +Pre-flight check that the sandbox is running. Runs `sbx ls --quiet` and verifies `SANDBOX_NAME` exactly matches one output line. Throws `SandboxError` with `exitCode: null` if the sandbox is not found, with a message directing the user to run `./build/skillwalker sandbox create`. **Consumers:** - `cli/src/commands/test-run.ts` -- before the per-eval test loop @@ -110,7 +110,7 @@ async function createSandbox(repoRoot: string): Promise Checks whether the sandbox already exists via an internal `sandboxExists()` helper (runs `sbx ls --quiet`). If found, prints a help message to stderr explaining how to recreate it, and returns early. Otherwise, spawns `sbx run --name claude-skills-skillwalker claude ` with inherited stdio for interactive OAuth login. Prints progress messages to stderr. -**Consumer:** `cli/src/commands/sandbox/setup.ts` +**Consumer:** `cli/src/commands/sandbox/create.ts` #### removeSandbox() @@ -122,6 +122,22 @@ Runs `sbx rm --force claude-skills-skillwalker`. Drains stdout and stderr in par **Consumer:** `cli/src/commands/sandbox/clean.ts` -- catches `SandboxError` and re-throws as `SkillwalkerError` +#### updateSandbox() + +```typescript +async function updateSandbox(repoRoot: string): Promise +``` + +Replaces the sandbox with one built from the latest Claude Code template. `sbx` has no pull command and reuses a cached template image, so this function: + +1. Removes the sandbox with `removeSandbox()`, if it exists. +2. Lists templates with `sbx template ls` and removes each cached image whose repository is `docker/sandbox-templates` and whose tag starts with `claude-code`, using `sbx template rm `. +3. Calls `createSandbox(repoRoot)`, which makes `sbx run` fetch the current template. + +`sbx template ls` can list one image under several IDs, and removing the first ID removes them all. A later `rm` that reports `no template image` is therefore treated as already removed. Any other listing or removal failure throws `SandboxError`. + +**Consumer:** `cli/src/commands/sandbox/update.ts` -- catches `SandboxError` and re-throws as `SkillwalkerError` + #### openShell() ```typescript @@ -146,7 +162,7 @@ Contains the `SandboxResult` interface (see Core Types above). flowchart TB subgraph cli["@testdouble/skillwalker-cli"] direction LR - commands["commands/
sandbox/setup · sandbox/clean
sandbox/shell · test-run"] + commands["commands/
sandbox/create · sandbox/clean
sandbox/shell · test-run"] scil["scil/
loop"] end @@ -155,7 +171,7 @@ flowchart TB subgraph si["@testdouble/sandbox-integration"] direction TB sandboxts["sandbox.ts
ensureSandboxExists()
execInSandbox()
SANDBOX_NAME"] - lifecycle["lifecycle.ts
createSandbox()
removeSandbox()
openShell()"] + lifecycle["lifecycle.ts
createSandbox()
updateSandbox()
removeSandbox()
openShell()"] types["types.ts
SandboxResult"] errors["errors.ts
SandboxError"] @@ -178,7 +194,8 @@ flowchart TB | Consumer | Imports | |----------|---------| -| `cli/src/commands/sandbox/setup.ts` | `createSandbox` | +| `cli/src/commands/sandbox/create.ts` | `createSandbox` | +| `cli/src/commands/sandbox/update.ts` | `updateSandbox`, `SandboxError` | | `cli/src/commands/sandbox/clean.ts` | `removeSandbox`, `SANDBOX_NAME`, `SandboxError` | | `cli/src/commands/sandbox/shell.ts` | `openShell` | | `cli/src/commands/test-run.ts` | `ensureSandboxExists` | @@ -189,7 +206,7 @@ flowchart TB | Scenario | Error Type | Behavior | |----------|------------|----------| -| Sandbox not found by `ensureSandboxExists` | `SandboxError` (exitCode: `null`) | Thrown with message suggesting `./build/skillwalker sandbox setup` | +| Sandbox not found by `ensureSandboxExists` | `SandboxError` (exitCode: `null`) | Thrown with message suggesting `./build/skillwalker sandbox create` | | `sbx rm` fails | `SandboxError` (exitCode: process code) | Thrown with stdout+stderr in message | | Non-zero exit from `execInSandbox` | No error thrown | Returned in `SandboxResult.exitCode`; caller decides | | `proc.exitCode` is null in `execInSandbox` | No error thrown | Defaults to `1` in `SandboxResult` | @@ -202,7 +219,7 @@ Three test files with full coverage of the public API: |------|--------| | `src/errors.test.ts` | `SandboxError` construction, properties, inheritance | | `src/sandbox.test.ts` | `ensureSandboxExists`, `execInSandbox` | -| `src/lifecycle.test.ts` | `removeSandbox`, `createSandbox`, `openShell` | +| `src/lifecycle.test.ts` | `removeSandbox`, `createSandbox`, `updateSandbox`, `openShell` | ### Test Patterns @@ -245,7 +262,7 @@ function makeStream(content: string): ReadableStream { | `src/types.ts` | `SandboxResult` interface | | `src/errors.ts` | `SandboxError` class | | `src/sandbox.ts` | `SANDBOX_NAME`, `ensureSandboxExists`, `execInSandbox` | -| `src/lifecycle.ts` | `createSandbox`, `removeSandbox`, `openShell` | +| `src/lifecycle.ts` | `createSandbox`, `updateSandbox`, `removeSandbox`, `openShell` | | `src/errors.test.ts` | Unit tests for `SandboxError` | | `src/sandbox.test.ts` | Unit tests for sandbox execution functions | | `src/lifecycle.test.ts` | Unit tests for lifecycle management functions | diff --git a/docs/sandbox-integration.md b/docs/sandbox-integration.md index 94a246f..b3d07b9 100644 --- a/docs/sandbox-integration.md +++ b/docs/sandbox-integration.md @@ -13,14 +13,14 @@ Centralized package for all Test Sandbox interactions in Skillwalker — creatin ## Summary - The `@testdouble/sandbox-integration` package is the single point of contact for all Sandbox CLI commands in Skillwalker. No other package spawns `sbx` processes directly. -- Provides two categories of functions: **sandbox execution** (`ensureSandboxExists`, `execInSandbox`) for running Claude inside the sandbox, and **lifecycle management** (`createSandbox`, `removeSandbox`, `openShell`) for managing the sandbox itself. +- Provides two categories of functions: **sandbox execution** (`ensureSandboxExists`, `execInSandbox`) for running Claude inside the sandbox, and **lifecycle management** (`createSandbox`, `updateSandbox`, `removeSandbox`, `openShell`) for managing the sandbox itself. - Uses Docker Desktop sandboxes (not traditional containers) via the `sbx` CLI subcommands. - Returns a clean `SandboxResult` type instead of exposing raw `Bun.spawn` process handles. Key files: - `packages/sandbox-integration/index.ts` — Public API barrel export - `packages/sandbox-integration/src/sandbox.ts` — `ensureSandboxExists`, `execInSandbox`, `SANDBOX_NAME` -- `packages/sandbox-integration/src/lifecycle.ts` — `createSandbox`, `removeSandbox`, `openShell` +- `packages/sandbox-integration/src/lifecycle.ts` — `createSandbox`, `updateSandbox`, `removeSandbox`, `openShell` - `packages/sandbox-integration/sandbox-run.sh` — Shell script executed inside the sandbox to prepare the working directory and invoke Claude ## Architecture @@ -29,7 +29,7 @@ Key files: flowchart TB subgraph cli["@testdouble/skillwalker-cli"] direction LR - commands["commands/
sandbox/clean · sandbox/shell
sandbox/setup · test-run"] + commands["commands/
sandbox/clean · sandbox/shell
sandbox/create · test-run"] runners["test-runners/
prompt/ · skill-call/"] scil["scil/
loop · step-5 · step-7"] end @@ -37,7 +37,7 @@ flowchart TB subgraph si["@testdouble/sandbox-integration"] direction TB sandboxts["sandbox.ts
ensureSandboxExists()
execInSandbox()
SANDBOX_NAME"] - lifecycle["lifecycle.ts
createSandbox()
removeSandbox()
openShell()"] + lifecycle["lifecycle.ts
createSandbox()
updateSandbox()
removeSandbox()
openShell()"] runsh["sandbox-run.sh
(runs inside Docker)"] lifecycle --> sandboxts @@ -59,7 +59,7 @@ flowchart TB | `packages/sandbox-integration/package.json` | Package metadata (`@testdouble/sandbox-integration`) | | `packages/sandbox-integration/index.ts` | Barrel re-export of all public symbols | | `packages/sandbox-integration/src/sandbox.ts` | `SANDBOX_NAME`, `ensureSandboxExists`, `execInSandbox` | -| `packages/sandbox-integration/src/lifecycle.ts` | `createSandbox`, `removeSandbox`, `openShell` | +| `packages/sandbox-integration/src/lifecycle.ts` | `createSandbox`, `updateSandbox`, `removeSandbox`, `openShell` | | `packages/sandbox-integration/src/types.ts` | `SandboxResult` interface | | `packages/sandbox-integration/src/errors.ts` | `SandboxError` class | | `packages/sandbox-integration/sandbox-run.sh` | Scaffold setup and Claude invocation inside the sandbox | @@ -171,7 +171,17 @@ export async function createSandbox(repoRoot: string): Promise Checks if the sandbox already exists via an internal `sandboxExists()` helper. If it does, prints a help message to stderr and returns. Otherwise, spawns `sbx run --name claude-skills-skillwalker claude ` with inherited stdio for interactive OAuth login. -Called by `commands/sandbox/setup.ts`. +Called by `commands/sandbox/create.ts`. + +#### updateSandbox + +```typescript +export async function updateSandbox(repoRoot: string): Promise +``` + +Removes the sandbox if it exists, then removes every cached `docker/sandbox-templates` image tagged `claude-code*` (found with `sbx template ls`). Finally it calls `createSandbox`, so `sbx run` fetches the latest Claude Code template. `sbx` has no pull command, so deleting the cached image is the only way to get a newer one. An `rm` that reports `no template image` counts as already removed, because `sbx template ls` can list one image under several IDs. Any other listing or removal failure throws `SandboxError`. + +Called by `commands/sandbox/update.ts`, which catches `SandboxError` and re-throws as `SkillwalkerError`. #### removeSandbox @@ -212,7 +222,7 @@ See [Cross-Runtime Meta Property Resolution](coding-standards/cross-runtime-meta | Scenario | Error Type | Behavior | |----------|------------|----------| -| Sandbox not found by `ensureSandboxExists` | `SandboxError` (exitCode: `null`) | Thrown with message suggesting `./build/skillwalker sandbox setup` | +| Sandbox not found by `ensureSandboxExists` | `SandboxError` (exitCode: `null`) | Thrown with message suggesting `./build/skillwalker sandbox create` | | `sbx rm` fails | `SandboxError` (exitCode: process code) | Thrown with stdout+stderr in message | | Non-zero exit code from `execInSandbox` | No error thrown | Returned in `SandboxResult.exitCode`; caller decides | | `execInSandbox` with `proc.exitCode` null | No error thrown | `exitCode` defaults to `1` in `SandboxResult` | @@ -231,7 +241,7 @@ See [Cross-Runtime Meta Property Resolution](coding-standards/cross-runtime-meta - `packages/sandbox-integration/src/errors.test.ts` — `SandboxError` construction and properties - `packages/sandbox-integration/src/sandbox.test.ts` — `ensureSandboxExists` and `execInSandbox` with mocked `Bun.spawn` -- `packages/sandbox-integration/src/lifecycle.test.ts` — `removeSandbox`, `createSandbox`, `openShell` with mocked `Bun.spawn` and `sandbox.js` +- `packages/sandbox-integration/src/lifecycle.test.ts` — `removeSandbox`, `createSandbox`, `updateSandbox`, `openShell` with mocked `Bun.spawn` and `sandbox.js` ### Test Patterns @@ -268,7 +278,7 @@ Modules under test are imported dynamically inside each `it` block via `await im If `ensureSandboxExists` throws `SandboxError`, run: -1. `./build/skillwalker sandbox setup` — creates the sandbox and completes OAuth +1. `./build/skillwalker sandbox create` — creates the sandbox and completes OAuth 2. Verify with `sbx ls --quiet` — should list `claude-skills-skillwalker` ### Sandbox already exists during setup @@ -276,7 +286,9 @@ If `ensureSandboxExists` throws `SandboxError`, run: `createSandbox` returns early with a help message. To recreate: 1. `sbx rm --force claude-skills-skillwalker` -2. `./build/skillwalker sandbox setup` +2. `./build/skillwalker sandbox create` + +To recreate it from the latest Claude Code template instead, run `./build/skillwalker sandbox update`. ### Tests fail with "Cannot read properties of undefined (reading 'exited')" diff --git a/docs/skill-call-improvement-loop.md b/docs/skill-call-improvement-loop.md index 678350f..70f7d5d 100644 --- a/docs/skill-call-improvement-loop.md +++ b/docs/skill-call-improvement-loop.md @@ -24,7 +24,7 @@ At the end of every iteration, SCIL prints a progress summary. When the loop exi ```bash make build -./build/skillwalker sandbox setup +./build/skillwalker sandbox create ``` ## Eval Requirements diff --git a/docs/skillwalker-architecture.md b/docs/skillwalker-architecture.md index 3800e95..3c00a7b 100644 --- a/docs/skillwalker-architecture.md +++ b/docs/skillwalker-architecture.md @@ -78,7 +78,7 @@ flowchart LR cli --> exec cli -->|"update-analytics command"| data - cli -->|"sandbox setup/clean/shell sub-commands"| sandbox + cli -->|"sandbox create/update/clean/shell sub-commands"| sandbox exec --> data exec --> evals @@ -115,7 +115,8 @@ The command-line entry point. A thin Yargs wrapper that parses arguments, resolv | `scil` | Iterative skill-call description improvement loop | `runScilLoop()` | | `acil` | Iterative agent-call description improvement loop | `runAcilLoop()` | | `update-analytics` | Import JSONL output to Parquet via DuckDB | `skillwalker-data` directly | -| `sandbox setup` | Create the Test Sandbox | `sandbox-integration` directly | +| `sandbox create` | Create the Test Sandbox | `sandbox-integration` directly | +| `sandbox update` | Recreate the Test Sandbox from the latest Claude Code template | `sandbox-integration` directly | | `sandbox clean` | Remove the Test Sandbox | `sandbox-integration` directly | | `sandbox shell` | Open an interactive bash session in the Test Sandbox | `sandbox-integration` directly | @@ -228,6 +229,7 @@ The sandbox execution layer. Manages Test Sandbox lifecycle and runs commands in | `execInSandbox(args, scaffoldPath, debug)` | Execute a command in sandbox with optional scaffold directory | | `ensureSandboxExists()` | Verify the named sandbox is running | | `createSandbox(repoRoot)` | Create a new Test Sandbox with repo mount | +| `updateSandbox(repoRoot)` | Remove the sandbox and cached Claude Code templates, then recreate it | | `removeSandbox()` | Remove the Test Sandbox | | `openShell()` | Open interactive bash in sandbox | | `SANDBOX_NAME` | `'claude-skills-skillwalker'` constant | diff --git a/packages/cli/src/command-registration.integration.test.ts b/packages/cli/src/command-registration.integration.test.ts index b1fff85..e6dffc8 100644 --- a/packages/cli/src/command-registration.integration.test.ts +++ b/packages/cli/src/command-registration.integration.test.ts @@ -42,10 +42,11 @@ describe('top-level command registration', () => { }) describe('sandbox sub-command registration', () => { - it('lists all three sub-commands and fails when none is given', () => { + it('lists all four sub-commands and fails when none is given', () => { const { status, output } = runCli('sandbox') expect(status).toBe(1) - expect(output).toContain('skillwalker sandbox setup') + expect(output).toContain('skillwalker sandbox create') + expect(output).toContain('skillwalker sandbox update') expect(output).toContain('skillwalker sandbox clean') expect(output).toContain('skillwalker sandbox shell') }) diff --git a/packages/cli/src/commands/sandbox.ts b/packages/cli/src/commands/sandbox.ts index 2c17410..42bdda1 100644 --- a/packages/cli/src/commands/sandbox.ts +++ b/packages/cli/src/commands/sandbox.ts @@ -1,13 +1,14 @@ import type { Argv } from 'yargs' import * as clean from './sandbox/clean.js' -import * as setup from './sandbox/setup.js' +import * as create from './sandbox/create.js' import * as shell from './sandbox/shell.js' +import * as update from './sandbox/update.js' export const command = 'sandbox' export const describe = 'Manage the Test Sandbox' export function builder(yargs: Argv): Argv { - return yargs.command(setup).command(clean).command(shell).demandCommand(1) + return yargs.command(create).command(update).command(clean).command(shell).demandCommand(1) } // Unreachable: demandCommand(1) above rejects a bare `sandbox` invocation before diff --git a/packages/cli/src/commands/sandbox/setup.test.ts b/packages/cli/src/commands/sandbox/create.test.ts similarity index 87% rename from packages/cli/src/commands/sandbox/setup.test.ts rename to packages/cli/src/commands/sandbox/create.test.ts index abf2b2b..e2b4c08 100644 --- a/packages/cli/src/commands/sandbox/setup.test.ts +++ b/packages/cli/src/commands/sandbox/create.test.ts @@ -5,7 +5,7 @@ vi.mock('@testdouble/sandbox-integration', () => ({ })) import { createSandbox } from '@testdouble/sandbox-integration' -import { builder, command, describe as commandDescribe, handler } from './setup.js' +import { builder, command, describe as commandDescribe, handler } from './create.js' beforeEach(() => { vi.clearAllMocks() @@ -16,9 +16,9 @@ afterEach(() => { vi.restoreAllMocks() }) -describe('sandbox setup command exports', () => { +describe('sandbox create command exports', () => { it('exports the correct command string', () => { - expect(command).toBe('setup') + expect(command).toBe('create') }) it('exports a non-empty describe string', () => { @@ -27,7 +27,7 @@ describe('sandbox setup command exports', () => { }) }) -describe('sandbox setup builder', () => { +describe('sandbox create builder', () => { function buildOptions() { const options: Record = {} const fakeYargs = { @@ -46,7 +46,7 @@ describe('sandbox setup builder', () => { }) }) -describe('sandbox setup handler', () => { +describe('sandbox create handler', () => { it('calls createSandbox with the resolved repo-root', async () => { await handler({ 'repo-root': '/repo/root' }) expect(vi.mocked(createSandbox)).toHaveBeenCalledWith('/repo/root') diff --git a/packages/cli/src/commands/sandbox/setup.ts b/packages/cli/src/commands/sandbox/create.ts similarity index 94% rename from packages/cli/src/commands/sandbox/setup.ts rename to packages/cli/src/commands/sandbox/create.ts index 74e297c..04533f2 100644 --- a/packages/cli/src/commands/sandbox/setup.ts +++ b/packages/cli/src/commands/sandbox/create.ts @@ -1,7 +1,7 @@ import { createSandbox } from '@testdouble/sandbox-integration' import type { Argv } from 'yargs' -export const command = 'setup' +export const command = 'create' export const describe = 'Create a Test Sandbox and authenticate via OAuth for test runs' export function builder(yargs: Argv): Argv { diff --git a/packages/cli/src/commands/sandbox/update.test.ts b/packages/cli/src/commands/sandbox/update.test.ts new file mode 100644 index 0000000..c24fdf2 --- /dev/null +++ b/packages/cli/src/commands/sandbox/update.test.ts @@ -0,0 +1,69 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@testdouble/sandbox-integration', () => ({ + updateSandbox: vi.fn(), + SandboxError: class SandboxError extends Error { + exitCode: number | null + constructor(message: string, exitCode: number | null) { + super(message) + this.name = 'SandboxError' + this.exitCode = exitCode + } + }, +})) + +import { SandboxError, updateSandbox } from '@testdouble/sandbox-integration' +import { SkillwalkerError } from '@testdouble/skillwalker-execution' +import { builder, command, describe as commandDescribe, handler } from './update.js' + +beforeEach(() => { + vi.clearAllMocks() + vi.mocked(updateSandbox).mockResolvedValue(undefined) +}) + +afterEach(() => { + vi.restoreAllMocks() +}) + +describe('sandbox update command exports', () => { + it('exports the correct command string', () => { + expect(command).toBe('update') + }) + + it('exports a non-empty describe string', () => { + expect(typeof commandDescribe).toBe('string') + expect(commandDescribe.length).toBeGreaterThan(0) + }) +}) + +describe('sandbox update builder', () => { + function buildOptions() { + const options: Record = {} + const fakeYargs = { + option(name: string, opts: unknown) { + options[name] = opts + return fakeYargs + }, + } as any + builder(fakeYargs) + return options + } + + it('configures repo-root with default process.cwd()', () => { + const options = buildOptions() + expect(options['repo-root']).toMatchObject({ type: 'string', default: process.cwd() }) + }) +}) + +describe('sandbox update handler', () => { + it('calls updateSandbox with the resolved repo-root', async () => { + await handler({ 'repo-root': '/repo/root' }) + expect(vi.mocked(updateSandbox)).toHaveBeenCalledWith('/repo/root') + }) + + it('throws SkillwalkerError when updateSandbox throws SandboxError', async () => { + vi.mocked(updateSandbox).mockRejectedValue(new SandboxError('sbx template rm failed (exit code 1): in use', 1)) + + await expect(handler({ 'repo-root': '/repo/root' })).rejects.toThrow(SkillwalkerError) + }) +}) diff --git a/packages/cli/src/commands/sandbox/update.ts b/packages/cli/src/commands/sandbox/update.ts new file mode 100644 index 0000000..e7c8509 --- /dev/null +++ b/packages/cli/src/commands/sandbox/update.ts @@ -0,0 +1,25 @@ +import { SandboxError, updateSandbox } from '@testdouble/sandbox-integration' +import { SkillwalkerError } from '@testdouble/skillwalker-execution' +import type { Argv } from 'yargs' + +export const command = 'update' +export const describe = 'Delete the Test Sandbox and recreate it from the latest Claude Code sandbox template' + +export function builder(yargs: Argv): Argv { + return yargs.option('repo-root', { + type: 'string', + default: process.cwd(), + describe: 'Target repo root to mount in the sandbox (defaults to current working directory)', + }) +} + +export async function handler(argv: Record): Promise { + try { + await updateSandbox(argv['repo-root'] as string) + } catch (error) { + if (error instanceof SandboxError) { + throw new SkillwalkerError(error.message) + } + throw error + } +} diff --git a/packages/sandbox-integration/index.ts b/packages/sandbox-integration/index.ts index 0e68212..1fc1dff 100644 --- a/packages/sandbox-integration/index.ts +++ b/packages/sandbox-integration/index.ts @@ -1,4 +1,4 @@ export { SandboxError } from './src/errors.js' -export { createSandbox, openShell, removeSandbox } from './src/lifecycle.js' +export { createSandbox, openShell, removeSandbox, updateSandbox } from './src/lifecycle.js' export { ensureSandboxExists, execInSandbox, SANDBOX_NAME } from './src/sandbox.js' export type { SandboxResult } from './src/types.js' diff --git a/packages/sandbox-integration/src/lifecycle.test.ts b/packages/sandbox-integration/src/lifecycle.test.ts index 65837ac..7664082 100644 --- a/packages/sandbox-integration/src/lifecycle.test.ts +++ b/packages/sandbox-integration/src/lifecycle.test.ts @@ -106,6 +106,117 @@ describe('createSandbox', () => { }) }) +describe('updateSandbox', () => { + const templateList = [ + 'REPOSITORY TAG IMAGE ID FLAVOR CREATED', + 'docker/sandbox-templates claude-code-docker 6f873d7e6093 claude-code-docker 4 months ago', + 'docker/sandbox-templates claude-code-docker 94670d5b2a24 claude-code-docker 4 months ago', + 'docker/sandbox-templates codex-docker aaaaaaaaaaaa codex-docker 4 months ago', + 'myimage claude-code-custom bbbbbbbbbbbb custom 1 day ago', + '', + ].join('\n') + + function makeCapturedProc(stdout: string, exitCode = 0) { + return { stdout: makeStream(stdout), stderr: makeStream(''), exited: Promise.resolve(), exitCode } + } + + function spawnedArgs(): string[][] { + return (globalThis as any).Bun.spawn.mock.calls.map((call: unknown[]) => call[0]) + } + + it('removes the sandbox and cached Claude Code templates, then creates a new sandbox', async () => { + ;(globalThis as any).Bun.spawn + .mockReturnValueOnce(makeCapturedProc('claude-skills-skillwalker\n')) + .mockReturnValueOnce(makeCapturedProc('')) + .mockReturnValueOnce(makeCapturedProc(templateList)) + .mockReturnValueOnce(makeCapturedProc('')) + .mockReturnValueOnce(makeCapturedProc('')) + .mockReturnValueOnce(makeCapturedProc('')) + .mockReturnValueOnce({ exited: Promise.resolve() }) + + const stderrSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true) + + const { updateSandbox } = await import('./lifecycle.js') + await updateSandbox('/repo/root') + + expect(spawnedArgs()).toEqual([ + ['sbx', 'ls', '--quiet'], + ['sbx', 'rm', '--force', 'claude-skills-skillwalker'], + ['sbx', 'template', 'ls'], + ['sbx', 'template', 'rm', '6f873d7e6093'], + ['sbx', 'template', 'rm', '94670d5b2a24'], + ['sbx', 'ls', '--quiet'], + ['sbx', 'run', '--name', 'claude-skills-skillwalker', 'claude', '/repo/root'], + ]) + + stderrSpy.mockRestore() + }) + + it('skips sandbox removal when no sandbox exists', async () => { + ;(globalThis as any).Bun.spawn + .mockReturnValueOnce(makeCapturedProc('other-sandbox\n')) + .mockReturnValueOnce(makeCapturedProc(templateList.split('\n')[0])) + .mockReturnValueOnce(makeCapturedProc('other-sandbox\n')) + .mockReturnValueOnce({ exited: Promise.resolve() }) + + const stderrSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true) + + const { updateSandbox } = await import('./lifecycle.js') + await updateSandbox('/repo/root') + + expect(spawnedArgs()).toEqual([ + ['sbx', 'ls', '--quiet'], + ['sbx', 'template', 'ls'], + ['sbx', 'ls', '--quiet'], + ['sbx', 'run', '--name', 'claude-skills-skillwalker', 'claude', '/repo/root'], + ]) + + stderrSpy.mockRestore() + }) + + it('continues when a template image was already removed along with an earlier one', async () => { + ;(globalThis as any).Bun.spawn + .mockReturnValueOnce(makeCapturedProc('other-sandbox\n')) + .mockReturnValueOnce(makeCapturedProc(templateList)) + .mockReturnValueOnce(makeCapturedProc('')) + .mockReturnValueOnce(makeCapturedProc("ERROR: sandboxd error: status 404: no template image '94670d5b2a24'", 1)) + .mockReturnValueOnce(makeCapturedProc('other-sandbox\n')) + .mockReturnValueOnce({ exited: Promise.resolve() }) + + const stderrSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true) + + const { updateSandbox } = await import('./lifecycle.js') + await updateSandbox('/repo/root') + + expect(spawnedArgs().at(-1)).toEqual(['sbx', 'run', '--name', 'claude-skills-skillwalker', 'claude', '/repo/root']) + + stderrSpy.mockRestore() + }) + + it('throws SandboxError when a template image cannot be removed', async () => { + ;(globalThis as any).Bun.spawn + .mockReturnValueOnce(makeCapturedProc('other-sandbox\n')) + .mockReturnValueOnce(makeCapturedProc(templateList)) + .mockReturnValueOnce(makeCapturedProc('image in use', 1)) + + const stderrSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true) + + const { updateSandbox } = await import('./lifecycle.js') + await expect(updateSandbox('/repo/root')).rejects.toThrow(SandboxError) + + stderrSpy.mockRestore() + }) + + it('throws SandboxError when template listing fails', async () => { + ;(globalThis as any).Bun.spawn + .mockReturnValueOnce(makeCapturedProc('other-sandbox\n')) + .mockReturnValueOnce(makeCapturedProc('not logged in', 1)) + + const { updateSandbox } = await import('./lifecycle.js') + await expect(updateSandbox('/repo/root')).rejects.toThrow(SandboxError) + }) +}) + describe('openShell', () => { it('calls ensureSandboxExists before spawning', async () => { ;(globalThis as any).Bun.spawn.mockReturnValue({ diff --git a/packages/sandbox-integration/src/lifecycle.ts b/packages/sandbox-integration/src/lifecycle.ts index 5716c4e..319f780 100644 --- a/packages/sandbox-integration/src/lifecycle.ts +++ b/packages/sandbox-integration/src/lifecycle.ts @@ -1,6 +1,10 @@ import { SandboxError } from './errors.js' import { ensureSandboxExists, listSandboxNames, SANDBOX_NAME, spawnSbx } from './sandbox.js' +const CLAUDE_TEMPLATE_REPOSITORY = 'docker/sandbox-templates' +const CLAUDE_TEMPLATE_TAG_PREFIX = 'claude-code' +const TEMPLATE_ALREADY_REMOVED_MESSAGE = 'no template image' + async function sandboxExists(): Promise { return (await listSandboxNames()).includes(SANDBOX_NAME) } @@ -17,8 +21,8 @@ async function drainStream(stream: ReadableStream): Promise return result } -export async function removeSandbox(): Promise { - const proc = spawnSbx(['rm', '--force', SANDBOX_NAME], { stdout: 'pipe', stderr: 'pipe' }) +async function runSbxCaptured(args: string[]): Promise<{ exitCode: number | null; output: string }> { + const proc = spawnSbx(args, { stdout: 'pipe', stderr: 'pipe' }) const [stdoutCapture, stderrCapture] = await Promise.all([ drainStream(proc.stdout as ReadableStream), @@ -26,19 +30,59 @@ export async function removeSandbox(): Promise { ]) await proc.exited - if (proc.exitCode !== 0) { - throw new SandboxError( - `sbx rm failed (exit code ${proc.exitCode ?? 1}): ${stdoutCapture}${stderrCapture}`, - proc.exitCode, - ) + return { exitCode: proc.exitCode, output: `${stdoutCapture}${stderrCapture}` } +} + +export async function removeSandbox(): Promise { + const { exitCode, output } = await runSbxCaptured(['rm', '--force', SANDBOX_NAME]) + + if (exitCode !== 0) { + throw new SandboxError(`sbx rm failed (exit code ${exitCode ?? 1}): ${output}`, exitCode) } } +/** + * Image IDs of the cached Claude Code sandbox templates. `sbx` has no pull + * command, so removing these is what makes the next `sbx run` fetch the latest. + */ +async function listClaudeTemplateImageIds(): Promise { + const { exitCode, output } = await runSbxCaptured(['template', 'ls']) + + if (exitCode !== 0) { + throw new SandboxError(`sbx template ls failed (exit code ${exitCode ?? 1}): ${output}`, exitCode) + } + + const ids = output + .split('\n') + .slice(1) + .map((line) => line.trim().split(/\s+/)) + .filter( + ([repository, tag]) => repository === CLAUDE_TEMPLATE_REPOSITORY && tag?.startsWith(CLAUDE_TEMPLATE_TAG_PREFIX), + ) + .map(([, , imageId]) => imageId) + .filter((imageId): imageId is string => imageId !== undefined) + + return [...new Set(ids)] +} + +async function removeTemplateImage(imageId: string): Promise { + const { exitCode, output } = await runSbxCaptured(['template', 'rm', imageId]) + + // `sbx template ls` can list one image under several IDs. Removing the first + // removes them all, so a later ID reports that the image no longer exists. + if (exitCode === 0 || output.includes(TEMPLATE_ALREADY_REMOVED_MESSAGE)) return + + throw new SandboxError( + `sbx template rm ${imageId} failed (exit code ${exitCode ?? 1}): ${output}\nRetry with \`./build/skillwalker sandbox update\`.`, + exitCode, + ) +} + export async function createSandbox(repoRoot: string): Promise { if (await sandboxExists()) { process.stderr.write(`Sandbox "${SANDBOX_NAME}" already exists. To recreate, run:\n`) process.stderr.write(` sbx rm --force ${SANDBOX_NAME}\n`) - process.stderr.write(` ./build/skillwalker sandbox setup\n`) + process.stderr.write(` ./build/skillwalker sandbox create\n`) return } @@ -55,6 +99,20 @@ export async function createSandbox(repoRoot: string): Promise { process.stderr.write(`\nSandbox "${SANDBOX_NAME}" is ready. You can now run tests.\n`) } +export async function updateSandbox(repoRoot: string): Promise { + if (await sandboxExists()) { + process.stderr.write(`Removing sandbox "${SANDBOX_NAME}"...\n`) + await removeSandbox() + } + + for (const imageId of await listClaudeTemplateImageIds()) { + process.stderr.write(`Removing cached Claude Code template image ${imageId}...\n`) + await removeTemplateImage(imageId) + } + + await createSandbox(repoRoot) +} + export async function openShell(): Promise { await ensureSandboxExists() diff --git a/packages/sandbox-integration/src/sandbox.ts b/packages/sandbox-integration/src/sandbox.ts index 0a389a4..5ea2e24 100644 --- a/packages/sandbox-integration/src/sandbox.ts +++ b/packages/sandbox-integration/src/sandbox.ts @@ -27,7 +27,7 @@ export async function listSandboxNames(): Promise { if (proc.exitCode !== 0) { throw new SandboxError( - `Unable to list sandboxes with sbx (exit code ${proc.exitCode ?? 1}): ${stdout}${stderr}\nRun \`sbx login\`, then retry \`./build/skillwalker sandbox setup\`.`, + `Unable to list sandboxes with sbx (exit code ${proc.exitCode ?? 1}): ${stdout}${stderr}\nRun \`sbx login\`, then retry \`./build/skillwalker sandbox create\`.`, proc.exitCode, ) } @@ -42,7 +42,7 @@ export async function ensureSandboxExists(): Promise { const sandboxes = await listSandboxNames() if (!sandboxes.includes(SANDBOX_NAME)) { - throw new SandboxError(`Sandbox "${SANDBOX_NAME}" not found. Run './build/skillwalker sandbox setup' first.`, null) + throw new SandboxError(`Sandbox "${SANDBOX_NAME}" not found. Run './build/skillwalker sandbox create' first.`, null) } }