From 0fc9b15c1516da9fcae5ca3f37b0c9a372802127 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 04:15:18 -0700 Subject: [PATCH 01/38] ci: onboard Blacksmith Testbox --- .../ci-workflow-guard-tests-testbox.yml | 53 ++++++++ skills/blacksmith-testbox/SKILL.md | 126 ++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 .github/workflows/ci-workflow-guard-tests-testbox.yml create mode 100644 skills/blacksmith-testbox/SKILL.md diff --git a/.github/workflows/ci-workflow-guard-tests-testbox.yml b/.github/workflows/ci-workflow-guard-tests-testbox.yml new file mode 100644 index 00000000000..2e889d07100 --- /dev/null +++ b/.github/workflows/ci-workflow-guard-tests-testbox.yml @@ -0,0 +1,53 @@ +name: Blacksmith Testbox + +on: + workflow_dispatch: + inputs: + testbox_id: + type: string + description: "Testbox session ID" + required: true + pull_request: + paths: + - '.github/workflows/**' + +permissions: + contents: read + +jobs: + workflow-guard-tests: + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 40 + steps: + - name: Begin Testbox + uses: useblacksmith/begin-testbox@233448af4bfdc6fca509a7f0974411ac6d8a8043 # v2 + with: + testbox_id: ${{ inputs.testbox_id }} + + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + + - name: Set up Bun for Claude launch environment policy test + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: "1.3.6" + + - name: Set up Python 3.9 for nightly prune compatibility + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + with: + python-version: "3.9" + + - name: Install workflow guard Python dependencies + run: python3 -m pip install --disable-pip-version-check --no-input PyYAML==6.0.3 bashlex==0.18 + + - name: Initialize Ghostty for Zig version guard + run: git submodule update --init --depth 1 ghostty + + - name: Initialize Bonsplit for deferred-work ownership guard + run: git submodule update --init --depth 1 vendor/bonsplit + + - name: Run Testbox + uses: useblacksmith/run-testbox@5ca05834db1d3813554d1dd109e5f2087a8d7cbc # v2 + if: always() diff --git a/skills/blacksmith-testbox/SKILL.md b/skills/blacksmith-testbox/SKILL.md new file mode 100644 index 00000000000..2ba8db4f942 --- /dev/null +++ b/skills/blacksmith-testbox/SKILL.md @@ -0,0 +1,126 @@ +--- +name: blacksmith-testbox +description: > + Run Linux-compatible cmux checks in a Blacksmith Testbox. Use for Blacksmith + onboarding, remote Linux validation, Python workflow guards, and fast + repeatable checks against the CI environment. +--- + +# Blacksmith Testbox + +This repository's Testbox is a Linux environment derived from +`.github/workflows/ci.yml` and its `workflow-guard-tests` job. It hydrates Bun +1.3.6, Python 3.9, PyYAML, bashlex, and shallow `ghostty` and `vendor/bonsplit` +submodules. It does not provide macOS, Xcode, a GUI, or the full cmux CI suite. +Use the repository's cloud or hosted macOS workflow for Swift, XCTest, app-host, +and UI verification. + +## Prerequisites + +Install and authenticate the CLI once per machine: + +```bash +curl -fsSL https://get.blacksmith.sh | sh +blacksmith auth whoami +``` + +If authentication is missing, use the browser flow. The organization slug for +this repository is `manaflow-ai`: + +```bash +blacksmith auth login --non-interactive --organization manaflow-ai +``` + +Do not print or commit `~/.blacksmith/credentials`. Do not pass an API token +when browser authentication is requested. + +## Warm up + +Run every Testbox command from the root of the current git worktree. Push the +workflow file before warming up, because Blacksmith dispatches the workflow +from GitHub rather than reading an unpushed local file. + +```bash +cd "$(git rev-parse --show-toplevel)" +blacksmith testbox warmup .github/workflows/ci-workflow-guard-tests-testbox.yml \ + --ref \ + --idle-timeout 30 +``` + +The CLI defaults `--ref` to the current branch. Use an explicit branch when +the shell's checkout and the intended workflow revision could differ. Save the +returned `tbx_...` ID and use one ID per worktree or agent. + +The workflow has two modes: + +* A Testbox warmup supplies `testbox_id`, starts hydration, and keeps the VM + alive. +* A normal pull request run has no `testbox_id`; the begin/run actions validate + the setup without claiming a VM. + +## Run checks + +`run` waits for hydration automatically and returns the remote command's exit +status: + +```bash +blacksmith testbox run --id \ + "python3 tests/test_ci_change_areas.py" + +blacksmith testbox run --id \ + "cd agent-chat && bun test test/claude-environment.test.ts" +``` + +This workflow installs only the dependencies listed above. Before web, Go, Rust, +or other checks, run the repository's matching install/setup command in the +same Testbox, then run the check. A changed dependency manifest requires a +fresh install on the Testbox: + +```bash +blacksmith testbox run --id \ + "cd web && bun install --frozen-lockfile && bun run typecheck" +``` + +Do not assume that a local `node_modules`, build directory, Swift package cache, +or other ignored/generated directory was transferred. The CLI synchronizes the +worktree with checksum-based deletion semantics; execute required installs and +builds remotely. + +## Sync and safety rules + +* Invoke `blacksmith testbox run` from the repository root. Put `cd` only + inside the quoted remote command. Running the CLI from a subdirectory can + mirror the wrong tree and delete unrelated remote files. +* Treat the Testbox as a disposable mirror of the current worktree. Do not + store unique state there without downloading it first. +* Reuse the same ID for iterative runs. A new worktree needs its own warmup. +* Never use this Linux Testbox for macOS or iOS compilation, XCTest, app-host + tests, UI tests, or GUI dogfood. +* Local formatting and static inspection are allowed. Follow the repository + testing rules for every behavioral test and build. +* Do not put credentials, private keys, or generated Testbox state in the + repository. + +## Status, artifacts, and cleanup + +Use a blocking status check when needed, rather than a sleep loop: + +```bash +blacksmith testbox status --id --wait --wait-timeout 15m +``` + +Download artifacts relative to the remote worktree: + +```bash +blacksmith testbox download --id test-results/ ./test-results/ +blacksmith testbox download --id build/output.tar.gz ./output.tar.gz +``` + +Stop the VM after the task: + +```bash +blacksmith testbox stop --id +``` + +The default idle timeout is 30 minutes. Stopping explicitly avoids paying for +unused runner minutes. From e40704611ac35f0e3a806841a9eae383f4ffa153 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 04:31:27 -0700 Subject: [PATCH 02/38] ci: provision cmux-tui Rust Testbox lane --- .github/actionlint.yaml | 1 + .../ci-workflow-guard-tests-testbox.yml | 105 ++++++++-- .gitignore | 3 + scripts/blacksmith-cmux-tui-testbox-stage.sh | 124 +++++++++++ skills/blacksmith-testbox/SKILL.md | 198 +++++++++++------- skills/blacksmith-testbox/benchmark.md | 198 ++++++++++++++++++ 6 files changed, 527 insertions(+), 102 deletions(-) create mode 100755 scripts/blacksmith-cmux-tui-testbox-stage.sh create mode 100644 skills/blacksmith-testbox/benchmark.md diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index a9ac6b01ed9..4402dc219bd 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -19,4 +19,5 @@ self-hosted-runner: # Linux: Blacksmith primary (LINUX_RUNNER), WarpBuild overflow fallback. - blacksmith-4vcpu-ubuntu-2404 - blacksmith-8vcpu-ubuntu-2404 + - blacksmith-32vcpu-ubuntu-2404 - warp-ubuntu-latest-x64-4x diff --git a/.github/workflows/ci-workflow-guard-tests-testbox.yml b/.github/workflows/ci-workflow-guard-tests-testbox.yml index 2e889d07100..49db99ef294 100644 --- a/.github/workflows/ci-workflow-guard-tests-testbox.yml +++ b/.github/workflows/ci-workflow-guard-tests-testbox.yml @@ -1,53 +1,114 @@ -name: Blacksmith Testbox +name: cmux-tui Rust Testbox setup on: workflow_dispatch: inputs: testbox_id: - type: string - description: "Testbox session ID" + description: "Testbox session ID supplied by blacksmith testbox warmup" required: true - pull_request: - paths: - - '.github/workflows/**' + type: string permissions: contents: read jobs: - workflow-guard-tests: - runs-on: blacksmith-4vcpu-ubuntu-2404 - timeout-minutes: 40 + cmux-tui-rust: + name: cmux-tui Rust setup + runs-on: blacksmith-32vcpu-ubuntu-2404 + timeout-minutes: 45 steps: + # begin-testbox must be the first step. It attaches the VM requested by + # `blacksmith testbox warmup` and leaves it alive after this setup job. - name: Begin Testbox uses: useblacksmith/begin-testbox@233448af4bfdc6fca509a7f0974411ac6d8a8043 # v2 with: testbox_id: ${{ inputs.testbox_id }} - - name: Checkout + - name: Checkout exact dispatch commit uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 + persist-credentials: false + ref: ${{ github.sha }} + + - name: Require exact checkout + env: + EXPECTED_SHA: ${{ github.sha }} + shell: bash + run: | + actual_sha="$(git rev-parse HEAD)" + if [[ "$actual_sha" != "$EXPECTED_SHA" ]]; then + echo "::error::checked out $actual_sha, expected dispatch SHA $EXPECTED_SHA" >&2 + exit 1 + fi + printf 'testbox setup commit: %s\n' "$actual_sha" + + - name: Initialize Ghostty source submodule + shell: bash + run: | + git submodule update --init --depth 1 ghostty + test -f ghostty/build.zig.zon - - name: Set up Bun for Claude launch environment policy test - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + - name: Install Linux build dependencies + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y clang libclang-dev pkg-config + + - name: Cache Zig package downloads + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: - bun-version: "1.3.6" + path: ~/.cache/zig + key: cmux-tui-zig-${{ hashFiles('ghostty/build.zig.zon', 'ghostty/build.zig.zon.json') }} + restore-keys: | + cmux-tui-zig- + + - name: Install repository-pinned Zig + shell: bash + run: ./scripts/install-zig-ci.sh + + - name: Fetch Ghostty Zig dependencies without compiling + working-directory: ghostty + shell: bash + run: | + # `--fetch` hydrates the package cache and exits before a build. + "$CMUX_ZIG" build --fetch - - name: Set up Python 3.9 for nightly prune compatibility - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + - name: Cache Cargo registry and git dependencies + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: - python-version: "3.9" + path: | + ~/.cargo/registry + ~/.cargo/git + key: cmux-tui-cargo-${{ runner.os }}-${{ hashFiles('cmux-tui/Cargo.lock', 'cmux-tui/rust-toolchain.toml') }} + restore-keys: | + cmux-tui-cargo-${{ runner.os }}- - - name: Install workflow guard Python dependencies - run: python3 -m pip install --disable-pip-version-check --no-input PyYAML==6.0.3 bashlex==0.18 + - name: Set up repository-pinned cmux-tui Rust + uses: ./.github/actions/setup-cmux-tui-rust - - name: Initialize Ghostty for Zig version guard - run: git submodule update --init --depth 1 ghostty + - name: Fetch Cargo dependencies without compiling + working-directory: cmux-tui + shell: bash + run: cargo fetch --locked - - name: Initialize Bonsplit for deferred-work ownership guard - run: git submodule update --init --depth 1 vendor/bonsplit + - name: Record hydrated toolchain and workspace metadata + working-directory: cmux-tui + shell: bash + run: | + printf 'rust-toolchain: ' + rustup show active-toolchain + printf 'cargo: ' + cargo --version + printf 'rustc: ' + rustc --version + printf 'zig: ' + "$CMUX_ZIG" version + cargo metadata --locked --no-deps --format-version 1 > "$RUNNER_TEMP/cmux-tui-cargo-metadata.json" + test -s "$RUNNER_TEMP/cmux-tui-cargo-metadata.json" + # Always hand control back to Testbox. Warmup is setup-only: all actual + # Rust builds are issued later with `blacksmith testbox run`. - name: Run Testbox uses: useblacksmith/run-testbox@5ca05834db1d3813554d1dd109e5f2087a8d7cbc # v2 if: always() diff --git a/.gitignore b/.gitignore index ae1b9c3c139..1d2f3efd6c9 100644 --- a/.gitignore +++ b/.gitignore @@ -73,3 +73,6 @@ artifacts/ # tmux verbose debug logs (tmux -v) that land in the cwd tmux-*.log + +# Remote-only Blacksmith cmux-tui benchmark output +testbox-benchmark/ diff --git a/scripts/blacksmith-cmux-tui-testbox-stage.sh b/scripts/blacksmith-cmux-tui-testbox-stage.sh new file mode 100755 index 00000000000..ac5ae23b311 --- /dev/null +++ b/scripts/blacksmith-cmux-tui-testbox-stage.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +set -euo pipefail + +# This helper is intentionally remote-only. The local benchmark plan invokes it +# through `blacksmith testbox run`; the guard makes an accidental local launch a +# no-op instead of running a Rust build on the developer's Mac. +if [[ "${CMUX_TESTBOX_REMOTE:-}" != "1" ]]; then + echo "refusing to run outside a Blacksmith Testbox (set CMUX_TESTBOX_REMOTE=1 only in the remote command)" >&2 + exit 64 +fi + +if [[ $# -ne 1 ]]; then + echo "usage: CMUX_TESTBOX_REMOTE=1 $0 {first-clean|incremental-noop|changed-file}" >&2 + exit 64 +fi + +stage="$1" +case "$stage" in + first-clean|incremental-noop|changed-file) ;; + *) + echo "unsupported benchmark stage: $stage" >&2 + exit 64 + ;; +esac + +repo_root="$(git rev-parse --show-toplevel)" +cd "$repo_root" +if [[ ! -f cmux-tui/Cargo.toml || ! -f ghostty/.git || ! -f ghostty/build.zig.zon ]]; then + echo "cmux-tui and its Ghostty source submodule must be initialized" >&2 + exit 65 +fi + +benchmark_dir="$repo_root/testbox-benchmark" +mkdir -p "$benchmark_dir" +log_path="$benchmark_dir/$stage.log" +time_path="$benchmark_dir/$stage.time" +json_path="$benchmark_dir/$stage.json" +changed_file="cmux-tui/crates/cmux-tui/src/main.rs" +changed_backup="" + +# shellcheck disable=SC2329 # invoked indirectly by the EXIT trap +restore_changed_file() { + if [[ -n "$changed_backup" && -f "$changed_backup" ]]; then + cp "$changed_backup" "$repo_root/$changed_file" + rm -f "$changed_backup" + fi +} +trap restore_changed_file EXIT + +case "$stage" in + first-clean) + rm -rf "$repo_root/cmux-tui/target" + ;; + changed-file) + changed_backup="$(mktemp "${TMPDIR:-/tmp}/cmux-tui-testbox-source.XXXXXX")" + cp "$repo_root/$changed_file" "$changed_backup" + printf '\n// Blacksmith Testbox changed-file timing marker.\n' >> "$repo_root/$changed_file" + ;; +esac + +start_epoch="$(python3 -c 'import time; print(time.time())')" +set +e +( + cd "$repo_root/cmux-tui" + /usr/bin/time -p -o "$time_path" cargo build -p cmux-tui --locked +) >"$log_path" 2>&1 +status=$? +set -e +end_epoch="$(python3 -c 'import time; print(time.time())')" + +python3 - "$stage" "$start_epoch" "$end_epoch" "$status" "$time_path" "$changed_file" >"$json_path" <<'PY' +import datetime as dt +import json +import os +import platform +import subprocess +import sys + +stage, start, end, status, time_path, changed_file = sys.argv[1:] +start = float(start) +end = float(end) +status = int(status) +remote_time = {} +with open(time_path, encoding="utf-8") as handle: + for line in handle: + key, _, value = line.strip().partition(" ") + if key in {"real", "user", "sys"}: + remote_time[f"time_{key}_seconds"] = float(value) + +try: + git_sha = subprocess.check_output( + ["git", "rev-parse", "HEAD"], text=True, stderr=subprocess.DEVNULL + ).strip() +except subprocess.CalledProcessError: + git_sha = "unknown" + +record = { + "schema": 1, + "stage": stage, + "command": "cargo build -p cmux-tui --locked", + "exit_code": status, + "ok": status == 0, + "started_at": dt.datetime.fromtimestamp(start, dt.timezone.utc).isoformat(), + "finished_at": dt.datetime.fromtimestamp(end, dt.timezone.utc).isoformat(), + "wall_seconds": round(end - start, 3), + "git_sha": git_sha, + "runner": { + "hostname": platform.node(), + "arch": platform.machine(), + "cpu_count": os.cpu_count(), + "uname": " ".join(platform.uname()), + }, + "changed_file": changed_file if stage == "changed-file" else None, + **remote_time, +} +print(json.dumps(record, sort_keys=True)) +PY + +cat "$log_path" +printf '\n--- /usr/bin/time -p (%s) ---\n' "$stage" +cat "$time_path" +printf '\n--- structured timing (%s) ---\n' "$stage" +cat "$json_path" +exit "$status" diff --git a/skills/blacksmith-testbox/SKILL.md b/skills/blacksmith-testbox/SKILL.md index 2ba8db4f942..9df23a516fc 100644 --- a/skills/blacksmith-testbox/SKILL.md +++ b/skills/blacksmith-testbox/SKILL.md @@ -1,126 +1,164 @@ --- name: blacksmith-testbox description: > - Run Linux-compatible cmux checks in a Blacksmith Testbox. Use for Blacksmith - onboarding, remote Linux validation, Python workflow guards, and fast - repeatable checks against the CI environment. + Provision and reuse a beefy Blacksmith Testbox for cmux-tui Rust builds, + capture remote timings, download raw evidence, and clean up safely. Never + run cargo, rustc, or Zig builds on the local Mac. --- -# Blacksmith Testbox - -This repository's Testbox is a Linux environment derived from -`.github/workflows/ci.yml` and its `workflow-guard-tests` job. It hydrates Bun -1.3.6, Python 3.9, PyYAML, bashlex, and shallow `ghostty` and `vendor/bonsplit` -submodules. It does not provide macOS, Xcode, a GUI, or the full cmux CI suite. -Use the repository's cloud or hosted macOS workflow for Swift, XCTest, app-host, -and UI verification. - -## Prerequisites - -Install and authenticate the CLI once per machine: +# cmux-tui Blacksmith Testbox + +This lane is Linux-only. It uses +`.github/workflows/ci-workflow-guard-tests-testbox.yml`, job +`cmux-tui-rust`, on `blacksmith-32vcpu-ubuntu-2404`. The workflow is a +setup-only entrypoint for a reusable Testbox. It checks out the exact dispatch +SHA, initializes the `ghostty` source submodule, installs Linux C/LLVM headers, +installs the repository-pinned Zig and Rust toolchains, fetches Zig and Cargo +dependencies, and then hands control back to Testbox. It does not run Rust tests +or Rust compilation during warmup. `zig build --fetch` only hydrates Zig +packages and exits before compilation. + +The repository's single Rust toolchain source is +`cmux-tui/rust-toolchain.toml`. The workflow invokes +`./.github/actions/setup-cmux-tui-rust`, so a workflow-specific Rust version +must never be added. + +## Hard safety boundary + +* Never run `cargo`, `rustc`, `rustup`, `zig build`, or another Rust/Zig build + command on Lawrence's Mac. This includes local fallback builds and local + test commands. +* Run every Blacksmith CLI command from the root of the intended isolated + worktree. The CLI synchronizes that directory and can delete remote files + that are not represented locally. +* Put remote build commands inside `blacksmith testbox run`. The benchmark + helper also requires `CMUX_TESTBOX_REMOTE=1`, so an accidental local launch + exits before invoking a compiler. +* Use this Testbox only for Linux-compatible cmux-tui work. Use the hosted + macOS workflows for Swift, Xcode, XCTest, GUI, and app-host verification. + +## Authentication and root guard + +Check authentication without printing credentials: ```bash -curl -fsSL https://get.blacksmith.sh | sh blacksmith auth whoami ``` -If authentication is missing, use the browser flow. The organization slug for -this repository is `manaflow-ai`: +If the CLI is missing, install it once with: ```bash -blacksmith auth login --non-interactive --organization manaflow-ai +curl -fsSL https://get.blacksmith.sh | sh ``` -Do not print or commit `~/.blacksmith/credentials`. Do not pass an API token -when browser authentication is requested. - -## Warm up - -Run every Testbox command from the root of the current git worktree. Push the -workflow file before warming up, because Blacksmith dispatches the workflow -from GitHub rather than reading an unpushed local file. +If authentication or dispatch is blocked, use the Blacksmith Console at +https://app.blacksmith.sh. The exact retry command, after the workflow is +available on the selected ref, is: ```bash cd "$(git rev-parse --show-toplevel)" +SHA="$(git rev-parse HEAD)" blacksmith testbox warmup .github/workflows/ci-workflow-guard-tests-testbox.yml \ - --ref \ - --idle-timeout 30 + --ref "$SHA" --job cmux-tui-rust --idle-timeout 30 ``` -The CLI defaults `--ref` to the current branch. Use an explicit branch when -the shell's checkout and the intended workflow revision could differ. Save the -returned `tbx_...` ID and use one ID per worktree or agent. +A workflow that exists only on a feature branch can be rejected by GitHub's +workflow-dispatch API. If Blacksmith reports `workflow not found` or a 404, +do not change the ref or fall back to a local build. Open the Console, make the +workflow available on the repository's dispatchable default-branch revision, +and rerun the exact command above with the full SHA recorded in the evidence. -The workflow has two modes: +Use this guard before any warmup or run: -* A Testbox warmup supplies `testbox_id`, starts hydration, and keeps the VM - alive. -* A normal pull request run has no `testbox_id`; the begin/run actions validate - the setup without claiming a VM. +```bash +cd "$(git rev-parse --show-toplevel)" +SHA="$(git rev-parse HEAD)" +case "$SHA" in + [0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]*) ;; + *) echo "HEAD is not a full commit SHA" >&2; exit 1 ;; +esac +``` -## Run checks +The branch must be pushed before warmup. Record `git status --short --branch` +and the full SHA in the scratch evidence before dispatching. -`run` waits for hydration automatically and returns the remote command's exit -status: +## Warmup, run, and stop -```bash -blacksmith testbox run --id \ - "python3 tests/test_ci_change_areas.py" +Warm the exact branch head with the stable workflow/job pair: -blacksmith testbox run --id \ - "cd agent-chat && bun test test/claude-environment.test.ts" +```bash +cd "$(git rev-parse --show-toplevel)" +SHA="$(git rev-parse HEAD)" +blacksmith testbox warmup .github/workflows/ci-workflow-guard-tests-testbox.yml \ + --ref "$SHA" \ + --job cmux-tui-rust \ + --idle-timeout 30 ``` -This workflow installs only the dependencies listed above. Before web, Go, Rust, -or other checks, run the repository's matching install/setup command in the -same Testbox, then run the check. A changed dependency manifest requires a -fresh install on the Testbox: +Save the returned `tbx_...` ID. One ID belongs to one worktree or agent. Wait +for setup if needed: ```bash -blacksmith testbox run --id \ - "cd web && bun install --frozen-lockfile && bun run typecheck" +blacksmith testbox status --id --wait --wait-timeout 15m ``` -Do not assume that a local `node_modules`, build directory, Swift package cache, -or other ignored/generated directory was transferred. The CLI synchronizes the -worktree with checksum-based deletion semantics; execute required installs and -builds remotely. +Run harmless remote identity and dependency checks from the repository root: -## Sync and safety rules - -* Invoke `blacksmith testbox run` from the repository root. Put `cd` only - inside the quoted remote command. Running the CLI from a subdirectory can - mirror the wrong tree and delete unrelated remote files. -* Treat the Testbox as a disposable mirror of the current worktree. Do not - store unique state there without downloading it first. -* Reuse the same ID for iterative runs. A new worktree needs its own warmup. -* Never use this Linux Testbox for macOS or iOS compilation, XCTest, app-host - tests, UI tests, or GUI dogfood. -* Local formatting and static inspection are allowed. Follow the repository - testing rules for every behavioral test and build. -* Do not put credentials, private keys, or generated Testbox state in the - repository. - -## Status, artifacts, and cleanup +```bash +blacksmith testbox run --id --debug \ + 'set -euo pipefail; printf "sha="; git rev-parse HEAD; printf "runner="; uname -a; printf "cpus="; nproc; rustup show active-toolchain; rustc --version; cargo --version; "${CMUX_ZIG:-zig}" version' +``` -Use a blocking status check when needed, rather than a sleep loop: +Run a build only through the remote-only timing helper: ```bash -blacksmith testbox status --id --wait --wait-timeout 15m +blacksmith testbox run --id --debug \ + 'CMUX_TESTBOX_REMOTE=1 ./scripts/blacksmith-cmux-tui-testbox-stage.sh first-clean' +blacksmith testbox run --id --debug \ + 'CMUX_TESTBOX_REMOTE=1 ./scripts/blacksmith-cmux-tui-testbox-stage.sh incremental-noop' +blacksmith testbox run --id --debug \ + 'CMUX_TESTBOX_REMOTE=1 ./scripts/blacksmith-cmux-tui-testbox-stage.sh changed-file' ``` -Download artifacts relative to the remote worktree: +The complete capture procedure, including warmup wall time, run transcripts, +and aggregation, is in `skills/blacksmith-testbox/benchmark.md`. + +Download raw remote timing files before stopping: ```bash -blacksmith testbox download --id test-results/ ./test-results/ -blacksmith testbox download --id build/output.tar.gz ./output.tar.gz +mkdir -p .cmux-scratch/testbox-benchmark-raw +blacksmith testbox download --id testbox-benchmark/ \ + .cmux-scratch/testbox-benchmark-raw/ ``` -Stop the VM after the task: +Stop disposable capacity and prove cleanup: ```bash -blacksmith testbox stop --id +blacksmith testbox stop --id +blacksmith testbox list --all ``` -The default idle timeout is 30 minutes. Stopping explicitly avoids paying for -unused runner minutes. +`list --all` must report no active Testbox. Keep the warmup log, setup/job/run +IDs, status output, downloaded `*.json`/`*.time`/`*.log` files, stop output, and +final list output in a separate `.cmux-scratch/` evidence artifact. Never add +credentials or private keys to that artifact. + +## Timing interpretation + +The benchmark reports two clocks. The local CLI transcript measures sync, +transport, queueing, and the remote command. The downloaded `/usr/bin/time -p` +record measures the remote `cargo build -p cmux-tui --locked` command. Compare +remote `real` or `time_real_seconds` values for build performance, and retain +CLI wall time when evaluating Testbox overhead. + +`first-clean` is target-clean but dependency-warm: warmup runs `cargo fetch` and +`zig build --fetch`, and the workflow may restore registry, git, and Zig caches. +`incremental-noop` measures a second build on the same VM. `changed-file` adds a +comment to `cmux-tui/crates/cmux-tui/src/main.rs`, builds, and restores the +original file. These are deliberately different from a cold-VM benchmark. + +Do not invent a local baseline. The cmux-tui instructions prohibit local Rust +builds, so local evidence is explicitly unavailable. Prior hosted correctness +runs without Cargo durations are provenance only. Prior Blacksmith macOS +Swift/Xcode timing artifacts use a different OS, architecture, runner, cache, +and workload, so they are context rather than a comparable Rust baseline. diff --git a/skills/blacksmith-testbox/benchmark.md b/skills/blacksmith-testbox/benchmark.md new file mode 100644 index 00000000000..974934b0f36 --- /dev/null +++ b/skills/blacksmith-testbox/benchmark.md @@ -0,0 +1,198 @@ +# cmux-tui Testbox timing plan + +Run this plan from the root of the isolated cmux worktree. It never invokes a +Rust tool on the local Mac. Every `cargo`, `rustc`, and `zig` command below is +inside a quoted command passed to `blacksmith testbox run`, or inside the +setup-only GitHub job on the remote Linux runner. + +## Fixed lane contract + +| Item | Value | +| --- | --- | +| Workflow | `.github/workflows/ci-workflow-guard-tests-testbox.yml` | +| Job | `cmux-tui-rust` | +| Runner | `blacksmith-32vcpu-ubuntu-2404` | +| Rust source of truth | `cmux-tui/rust-toolchain.toml`, via `./.github/actions/setup-cmux-tui-rust` | +| Remote build helper | `scripts/blacksmith-cmux-tui-testbox-stage.sh` | +| Remote output | `testbox-benchmark/`, ignored by the repository so repeated runs retain it | + +The warmup job only checks out the exact dispatch SHA, initializes `ghostty`, +installs Linux headers/tools, installs the pinned Zig and Rust toolchains, and +fetches Cargo and Zig dependencies. `zig build --fetch` is the only build-system +operation in warmup, and it exits before compiling. Rust builds happen only in +the three explicit benchmark runs. + +The current Blacksmith catalog reports the requested x64 label as 32 vCPU and +121.6 GB, while the ARM label with the same vCPU count reports 96 GB. Keep the +requested `blacksmith-32vcpu-ubuntu-2404` label unless the repository's Linux +constraints make x64 impossible, and record the catalog result with the run. + +## Warmup and identity capture + +```bash +set -euo pipefail +cd "$(git rev-parse --show-toplevel)" +SHA="$(git rev-parse HEAD)" +WORKFLOW=.github/workflows/ci-workflow-guard-tests-testbox.yml +JOB=cmux-tui-rust +OUT="$PWD/.cmux-scratch/blacksmith-testbox-cmux-tui-$SHA" +mkdir -p "$OUT/raw" + +blacksmith auth whoami + +now() { python3 -c 'import time; print(f"{time.time():.6f}")'; } +TBX="" +cleanup() { + local result=$? + trap - EXIT + if [[ -n "$TBX" ]]; then + blacksmith testbox stop --id "$TBX" >"$OUT/stop.log" 2>&1 || true + fi + blacksmith testbox list --all >"$OUT/list-after-stop.log" 2>&1 || true + exit "$result" +} +trap cleanup EXIT + +warmup_started="$(now)" +set +e +blacksmith testbox warmup "$WORKFLOW" \ + --ref "$SHA" \ + --job "$JOB" \ + --idle-timeout 30 2>&1 | tee "$OUT/warmup.log" +warmup_status=${PIPESTATUS[0]} +set -e +warmup_finished="$(now)" +python3 - "$SHA" "$JOB" "$warmup_started" "$warmup_finished" "$warmup_status" "$OUT/warmup.json" <<'PY' +import json +import sys + +sha, job, started, finished, status, output = sys.argv[1:] +record = { + "sha": sha, + "job": job, + "started_epoch": float(started), + "finished_epoch": float(finished), + "wall_seconds": round(float(finished) - float(started), 3), + "exit_code": int(status), +} +with open(output, "w", encoding="utf-8") as handle: + json.dump(record, handle, indent=2, sort_keys=True) + handle.write("\n") +PY +if [[ "$warmup_status" -ne 0 ]]; then + echo "warmup failed; inspect $OUT/warmup.log and use https://app.blacksmith.sh" >&2 + exit "$warmup_status" +fi +TBX="$(python3 - "$OUT/warmup.log" <<'PY' +import re +import sys + +text = open(sys.argv[1], encoding="utf-8").read() +ids = re.findall(r"\btbx_[A-Za-z0-9_-]+\b", text) +if not ids: + raise SystemExit("warmup output did not contain a tbx_ ID") +print(ids[-1]) +PY +)" +printf 'Testbox ID: %s\n' "$TBX" | tee "$OUT/testbox-id.txt" + +blacksmith testbox status --id "$TBX" --wait --wait-timeout 15m \ + 2>&1 | tee "$OUT/status-ready.log" +blacksmith testbox run --id "$TBX" --debug \ + 'set -euo pipefail; printf "remote_sha="; git rev-parse HEAD; printf "runner="; uname -a; printf "cpus="; nproc; rustup show active-toolchain; rustc --version; cargo --version; "$CMUX_ZIG" version' \ + 2>&1 | tee "$OUT/identity.run.log" +``` + +The warmup transcript is the source for the warmup ID and the GitHub job/run +IDs. Preserve it verbatim. Do not substitute a branch name for `SHA`: the +checkout guard and the evidence file are intended to prove the exact 40-byte +revision tested. + +## Three remote build timings + +The helper creates a structured JSON record, a raw Cargo log, and the raw +`/usr/bin/time -p` output for each stage. Its environment guard prevents a +local accidental launch. + +```bash +run_stage() { + local stage="$1" + set +e + blacksmith testbox run --id "$TBX" --debug \ + "CMUX_TESTBOX_REMOTE=1 ./scripts/blacksmith-cmux-tui-testbox-stage.sh $stage" \ + 2>&1 | tee "$OUT/$stage.run.log" + local status=${PIPESTATUS[0]} + set -e + return "$status" +} + +benchmark_status=0 +for stage in first-clean incremental-noop changed-file; do + if ! run_stage "$stage"; then + benchmark_status=1 + break + fi +done + +mkdir -p "$OUT/raw" +blacksmith testbox download --id "$TBX" testbox-benchmark/ "$OUT/raw/" \ + 2>&1 | tee "$OUT/download.log" + +python3 - "$OUT" "$SHA" "$TBX" <<'PY' +import json +import pathlib +import sys + +out = pathlib.Path(sys.argv[1]) +sha, testbox_id = sys.argv[2:] +records = [] +for path in sorted((out / "raw").glob("*.json")): + records.append(json.loads(path.read_text(encoding="utf-8"))) +with (out / "timings.json").open("w", encoding="utf-8") as handle: + json.dump({"sha": sha, "testbox_id": testbox_id, "stages": records}, handle, indent=2, sort_keys=True) + handle.write("\n") +PY +exit "$benchmark_status" +``` + +`first-clean` removes only the remote `cmux-tui/target` directory before a +debug `cargo build -p cmux-tui --locked`. `incremental-noop` repeats that exact +command without changing source. `changed-file` appends a comment to +`cmux-tui/crates/cmux-tui/src/main.rs`, builds, and restores the original bytes +before the remote command exits. The local worktree is never mutated by the +helper. + +The local `*.run.log` files include Testbox run IDs and CLI wall time. The + downloaded `raw/*.json`, `raw/*.log`, and `raw/*.time` files are the remote + timing evidence. Keep both clocks: CLI wall time includes sync/transport, +while `time_real_seconds` measures the remote Cargo command. + +## Historical comparison and cleanup + +Record the following alongside `timings.json`: + +1. The exact runner label and the output of `blacksmith runners catalog`. +2. The warmup ID, setup job/run ID, and each `blacksmith testbox run` ID from the + raw transcripts. +3. Any prior hosted cmux-tui timing evidence found under the hq scratch/artifact + directories. Local Rust timing evidence is intentionally absent because this + Mac is prohibited from running Rust builds. +4. Whether the comparison was target-clean, registry/git-cache warm, Zig-cache + warm, or a genuinely cold VM. The setup workflow deliberately warms the + dependency caches, so `first-clean` is target-cold and dependency-warm. + +Prior Blacksmith macOS reload timing artifacts, if used for context, are not a +Rust baseline: they measure a Swift/Xcode build on a different OS, architecture, +runner SKU, cache state, and workload. Hosted cmux-tui correctness runs without +recorded Cargo durations are provenance, not performance comparisons. + +Always download before stopping, then stop and verify there is no active box: + +```bash +blacksmith testbox stop --id "$TBX" +blacksmith testbox list --all +``` + +The final list must report no active Testboxes. Keep `warmup.log`, all run +transcripts, `download.log`, `stop.log`, `list-after-stop.log`, and the raw +remote directory in the separate `.cmux-scratch/` evidence artifact. From 67f398f9489bfa3e287e0ed2c7e22f559d6b7744 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 04:49:40 -0700 Subject: [PATCH 03/38] ci: serialize Testbox benchmark stages --- .gitignore | 1 + scripts/blacksmith-cmux-tui-testbox-stage.sh | 5 +++++ skills/blacksmith-testbox/SKILL.md | 3 ++- skills/blacksmith-testbox/benchmark.md | 8 +++++--- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 1d2f3efd6c9..6e942c1ae9c 100644 --- a/.gitignore +++ b/.gitignore @@ -76,3 +76,4 @@ tmux-*.log # Remote-only Blacksmith cmux-tui benchmark output testbox-benchmark/ +.cmux-scratch/ diff --git a/scripts/blacksmith-cmux-tui-testbox-stage.sh b/scripts/blacksmith-cmux-tui-testbox-stage.sh index ac5ae23b311..c71d3bab158 100755 --- a/scripts/blacksmith-cmux-tui-testbox-stage.sh +++ b/scripts/blacksmith-cmux-tui-testbox-stage.sh @@ -32,6 +32,11 @@ fi benchmark_dir="$repo_root/testbox-benchmark" mkdir -p "$benchmark_dir" +# Testbox can acknowledge a run while its remote shell is still flushing +# output. Serialize stages and hold the lock through artifact writes so a +# subsequent run cannot overwrite a prior stage's timing file. +exec 9>"$benchmark_dir/.stage.lock" +flock -x 9 log_path="$benchmark_dir/$stage.log" time_path="$benchmark_dir/$stage.time" json_path="$benchmark_dir/$stage.json" diff --git a/skills/blacksmith-testbox/SKILL.md b/skills/blacksmith-testbox/SKILL.md index 9df23a516fc..6ad339a000b 100644 --- a/skills/blacksmith-testbox/SKILL.md +++ b/skills/blacksmith-testbox/SKILL.md @@ -155,7 +155,8 @@ CLI wall time when evaluating Testbox overhead. `zig build --fetch`, and the workflow may restore registry, git, and Zig caches. `incremental-noop` measures a second build on the same VM. `changed-file` adds a comment to `cmux-tui/crates/cmux-tui/src/main.rs`, builds, and restores the -original file. These are deliberately different from a cold-VM benchmark. +original file. The helper serializes stages with a remote lock. These are +deliberately different from a cold-VM benchmark. Do not invent a local baseline. The cmux-tui instructions prohibit local Rust builds, so local evidence is explicitly unavailable. Prior hosted correctness diff --git a/skills/blacksmith-testbox/benchmark.md b/skills/blacksmith-testbox/benchmark.md index 974934b0f36..54f1d2f1cfa 100644 --- a/skills/blacksmith-testbox/benchmark.md +++ b/skills/blacksmith-testbox/benchmark.md @@ -158,9 +158,11 @@ exit "$benchmark_status" `first-clean` removes only the remote `cmux-tui/target` directory before a debug `cargo build -p cmux-tui --locked`. `incremental-noop` repeats that exact command without changing source. `changed-file` appends a comment to -`cmux-tui/crates/cmux-tui/src/main.rs`, builds, and restores the original bytes -before the remote command exits. The local worktree is never mutated by the -helper. +`cmux-tui/crates/cmux-tui/src/main.rs` immediately before the build, which +changes Cargo's fingerprint, then restores the original bytes before the remote +command exits. The helper holds a remote stage lock so concurrent Testbox run +requests cannot overwrite timing files. The local worktree is never mutated by +the helper. The local `*.run.log` files include Testbox run IDs and CLI wall time. The downloaded `raw/*.json`, `raw/*.log`, and `raw/*.time` files are the remote From 5e31e66f804115beea66694c0ae6675141fc0aab Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 05:28:16 -0700 Subject: [PATCH 04/38] ci: harden trusted Blacksmith Testbox lane --- .../ci-workflow-guard-tests-testbox.yml | 218 ++++++++++- .gitignore | 4 +- scripts/blacksmith-cmux-tui-testbox-stage.sh | 360 ++++++++++++++++-- scripts/blacksmith-testbox-cleanup.sh | 81 ++++ skills/blacksmith-testbox/SKILL.md | 300 ++++++++++----- skills/blacksmith-testbox/benchmark.md | 286 ++++++++------ 6 files changed, 989 insertions(+), 260 deletions(-) create mode 100755 scripts/blacksmith-testbox-cleanup.sh diff --git a/.github/workflows/ci-workflow-guard-tests-testbox.yml b/.github/workflows/ci-workflow-guard-tests-testbox.yml index 49db99ef294..029014bd77f 100644 --- a/.github/workflows/ci-workflow-guard-tests-testbox.yml +++ b/.github/workflows/ci-workflow-guard-tests-testbox.yml @@ -7,14 +7,34 @@ on: description: "Testbox session ID supplied by blacksmith testbox warmup" required: true type: string + source_sha: + description: "Optional full source SHA; if set, it must equal the pushed branch head" + required: false + default: "" + type: string + +# The Blacksmith CLI supplies testbox_id and dispatches the selected pushed +# branch. It does not expose arbitrary workflow inputs, so source_sha is an +# optional manual-dispatch assertion; the benchmark's remote guard is required +# for the CLI path below. +permissions: {} -permissions: - contents: read +concurrency: + # A Testbox is a mutable shared workspace. Queue every request that names the + # same box/source instead of allowing two syncs or stage commands to race. + group: cmux-tui-testbox-${{ inputs.testbox_id }}-${{ inputs.source_sha || github.sha }} + cancel-in-progress: false jobs: cmux-tui-rust: name: cmux-tui Rust setup runs-on: blacksmith-32vcpu-ubuntu-2404 + environment: + # Configure this environment with required reviewers and no secrets. + # Approval must happen before begin-testbox exposes its auth token. + name: blacksmith-testbox-trusted + permissions: + contents: read timeout-minutes: 45 steps: # begin-testbox must be the first step. It attaches the VM requested by @@ -24,6 +44,49 @@ jobs: with: testbox_id: ${{ inputs.testbox_id }} + - name: Validate trusted branch dispatch and source identity + env: + EXPECTED_INPUT_SHA: ${{ inputs.source_sha }} + DISPATCH_SHA: ${{ github.sha }} + DISPATCH_REF: ${{ github.ref }} + TESTBOX_ID: ${{ inputs.testbox_id }} + REPOSITORY: ${{ github.repository }} + shell: bash + run: | + set -euo pipefail + [[ "$REPOSITORY" == "manaflow-ai/cmux" ]] || { + echo "::error::this Testbox lane is only valid for manaflow-ai/cmux" >&2 + exit 1 + } + [[ "$TESTBOX_ID" =~ ^tbx_[A-Za-z0-9_-]+$ ]] || { + echo "::error::malformed Testbox ID" >&2 + exit 1 + } + [[ "$DISPATCH_SHA" =~ ^[0-9a-f]{40}$ ]] || { + echo "::error::github.sha must be a lowercase full commit SHA" >&2 + exit 1 + } + if [[ ! "$DISPATCH_REF" =~ ^refs/heads/[A-Za-z0-9._/-]+$ || "$DISPATCH_REF" == *..* || "$DISPATCH_REF" == */ || "$DISPATCH_REF" == *//* ]]; then + echo "::error::Testbox warmup must dispatch a pushed branch ref, not a raw SHA, tag, or malformed ref" >&2 + exit 1 + fi + if [[ -n "$EXPECTED_INPUT_SHA" ]]; then + [[ "$EXPECTED_INPUT_SHA" =~ ^[0-9a-f]{40}$ ]] || { + echo "::error::source_sha must be a lowercase full commit SHA" >&2 + exit 1 + } + [[ "$EXPECTED_INPUT_SHA" == "$DISPATCH_SHA" ]] || { + echo "::error::source_sha does not equal the workflow dispatch SHA" >&2 + exit 1 + } + fi + remote_sha="$(git ls-remote --exit-code "https://github.com/${REPOSITORY}.git" "$DISPATCH_REF" | awk 'NR == 1 { print $1 }')" + [[ "$remote_sha" == "$DISPATCH_SHA" ]] || { + echo "::error::the pushed branch moved during dispatch: remote=$remote_sha workflow=$DISPATCH_SHA" >&2 + exit 1 + } + printf 'trusted source ref: %s\ntrusted source SHA: %s\n' "$DISPATCH_REF" "$DISPATCH_SHA" + - name: Checkout exact dispatch commit uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -31,27 +94,57 @@ jobs: persist-credentials: false ref: ${{ github.sha }} - - name: Require exact checkout + - name: Require exact checkout and clean source env: EXPECTED_SHA: ${{ github.sha }} shell: bash run: | + set -euo pipefail actual_sha="$(git rev-parse HEAD)" - if [[ "$actual_sha" != "$EXPECTED_SHA" ]]; then + [[ "$actual_sha" == "$EXPECTED_SHA" ]] || { echo "::error::checked out $actual_sha, expected dispatch SHA $EXPECTED_SHA" >&2 exit 1 - fi - printf 'testbox setup commit: %s\n' "$actual_sha" + } + [[ -z "$(git status --porcelain=v1 --untracked-files=normal)" ]] || { + echo "::error::source checkout is dirty before hydration" >&2 + git status --short >&2 + exit 1 + } + source_tree_sha="$(git rev-parse 'HEAD^{tree}')" + ghostty_gitlink_sha="$(git rev-parse 'HEAD:ghostty')" + [[ "$ghostty_gitlink_sha" =~ ^[0-9a-f]{40}$ ]] || { + echo "::error::HEAD:ghostty is not a gitlink SHA" >&2 + exit 1 + } + printf 'source_sha=%s\nsource_tree_sha=%s\nghostty_gitlink_sha=%s\n' \ + "$actual_sha" "$source_tree_sha" "$ghostty_gitlink_sha" - name: Initialize Ghostty source submodule shell: bash run: | + set -euo pipefail git submodule update --init --depth 1 ghostty + [[ "$(git -C ghostty rev-parse --show-toplevel)" == "$GITHUB_WORKSPACE/ghostty" ]] || { + echo "::error::ghostty did not initialize as its own submodule checkout" >&2 + exit 1 + } + expected_ghostty_sha="$(git rev-parse HEAD:ghostty)" + actual_ghostty_sha="$(git -C ghostty rev-parse HEAD)" + [[ "$actual_ghostty_sha" == "$expected_ghostty_sha" ]] || { + echo "::error::Ghostty checkout $actual_ghostty_sha does not match gitlink $expected_ghostty_sha" >&2 + exit 1 + } + [[ -z "$(git -C ghostty status --porcelain=v1 --untracked-files=normal)" ]] || { + echo "::error::Ghostty submodule is dirty after initialization" >&2 + git -C ghostty status --short >&2 + exit 1 + } test -f ghostty/build.zig.zon - name: Install Linux build dependencies shell: bash run: | + set -euo pipefail sudo apt-get update sudo apt-get install -y clang libclang-dev pkg-config @@ -71,6 +164,7 @@ jobs: working-directory: ghostty shell: bash run: | + set -euo pipefail # `--fetch` hydrates the package cache and exits before a build. "$CMUX_ZIG" build --fetch @@ -90,22 +184,112 @@ jobs: - name: Fetch Cargo dependencies without compiling working-directory: cmux-tui shell: bash - run: cargo fetch --locked + run: | + set -euo pipefail + cargo fetch --locked - - name: Record hydrated toolchain and workspace metadata - working-directory: cmux-tui + - name: Record runner, toolchain, and Ghostty identity + env: + SOURCE_SHA: ${{ github.sha }} + SOURCE_REF: ${{ github.ref }} + TESTBOX_ID: ${{ inputs.testbox_id }} + RUNNER_LABEL: blacksmith-32vcpu-ubuntu-2404 shell: bash run: | - printf 'rust-toolchain: ' - rustup show active-toolchain - printf 'cargo: ' - cargo --version - printf 'rustc: ' - rustc --version - printf 'zig: ' - "$CMUX_ZIG" version + set -euo pipefail + source_tree_sha="$(git rev-parse 'HEAD^{tree}')" + ghostty_gitlink_sha="$(git rev-parse 'HEAD:ghostty')" + ghostty_head_sha="$(git -C ghostty rev-parse HEAD)" + [[ "$SOURCE_SHA" == "$(git rev-parse HEAD)" ]] || exit 1 + [[ "$ghostty_gitlink_sha" == "$ghostty_head_sha" ]] || exit 1 + [[ -z "$(git status --porcelain=v1 --untracked-files=normal)" ]] || exit 1 + [[ -z "$(git -C ghostty status --porcelain=v1 --untracked-files=normal)" ]] || exit 1 + pushd cmux-tui >/dev/null cargo metadata --locked --no-deps --format-version 1 > "$RUNNER_TEMP/cmux-tui-cargo-metadata.json" test -s "$RUNNER_TEMP/cmux-tui-cargo-metadata.json" + RUST_TOOLCHAIN="$(rustup show active-toolchain)" + RUSTC_VERSION="$(rustc --version)" + CARGO_VERSION="$(cargo --version)" + popd >/dev/null + mkdir -p testbox-benchmark + SOURCE_TREE_SHA="$source_tree_sha" + GHOSTTY_GITLINK_SHA="$ghostty_gitlink_sha" + GHOSTTY_HEAD_SHA="$ghostty_head_sha" + ZIG_VERSION="$("$CMUX_ZIG" version)" + ZIG_PATH="$CMUX_ZIG" + RUNNER_UNAME="$(uname -a)" + RUNNER_CPU_COUNT="$(nproc)" + CARGO_METADATA_SHA256="$(sha256sum "$RUNNER_TEMP/cmux-tui-cargo-metadata.json" | cut -d ' ' -f 1)" + RUST_TOOLCHAIN_FILE_SHA256="$(sha256sum cmux-tui/rust-toolchain.toml | cut -d ' ' -f 1)" + CARGO_LOCK_SHA256="$(sha256sum cmux-tui/Cargo.lock | cut -d ' ' -f 1)" + GHOSTTY_ZON_SHA256="$(sha256sum ghostty/build.zig.zon | cut -d ' ' -f 1)" + export SOURCE_TREE_SHA GHOSTTY_GITLINK_SHA GHOSTTY_HEAD_SHA RUST_TOOLCHAIN RUSTC_VERSION CARGO_VERSION ZIG_VERSION ZIG_PATH RUNNER_UNAME RUNNER_CPU_COUNT CARGO_METADATA_SHA256 RUST_TOOLCHAIN_FILE_SHA256 CARGO_LOCK_SHA256 GHOSTTY_ZON_SHA256 + python3 - <<'PY' > testbox-benchmark/setup-identity.json + import json + import os + import platform + + print(json.dumps({ + "schema": 2, + "source": { + "ref": os.environ["SOURCE_REF"], + "commit_sha": os.environ["SOURCE_SHA"], + "tree_sha": os.environ["SOURCE_TREE_SHA"], + "ghostty_gitlink_sha": os.environ["GHOSTTY_GITLINK_SHA"], + "ghostty_head_sha": os.environ["GHOSTTY_HEAD_SHA"], + }, + "testbox": { + "id": os.environ["TESTBOX_ID"], + "setup_workflow_run_id": os.environ["GITHUB_RUN_ID"], + }, + "runner": { + "label": os.environ["RUNNER_LABEL"], + "name": os.environ.get("RUNNER_NAME"), + "os": os.environ.get("RUNNER_OS"), + "arch": os.environ.get("RUNNER_ARCH"), + "hostname": platform.node(), + "uname": os.environ["RUNNER_UNAME"], + "cpu_count": int(os.environ["RUNNER_CPU_COUNT"]), + }, + "toolchain": { + "rust_toolchain": os.environ["RUST_TOOLCHAIN"], + "rustc": os.environ["RUSTC_VERSION"], + "cargo": os.environ["CARGO_VERSION"], + "rust_toolchain_file_sha256": os.environ["RUST_TOOLCHAIN_FILE_SHA256"], + "cargo_lock_sha256": os.environ["CARGO_LOCK_SHA256"], + "cargo_metadata_sha256": os.environ["CARGO_METADATA_SHA256"], + "zig_path": os.environ["ZIG_PATH"], + "zig": os.environ["ZIG_VERSION"], + "ghostty_build_zig_zon_sha256": os.environ["GHOSTTY_ZON_SHA256"], + }, + }, sort_keys=True, indent=2)) + PY + test -s testbox-benchmark/setup-identity.json + cat testbox-benchmark/setup-identity.json + + - name: Require clean hydrated source + shell: bash + run: | + set -euo pipefail + [[ -z "$(git status --porcelain=v1 --untracked-files=normal)" ]] || { + echo "::error::source became dirty during hydration" >&2 + git status --short >&2 + exit 1 + } + [[ -z "$(git -C ghostty status --porcelain=v1 --untracked-files=normal)" ]] || { + echo "::error::Ghostty became dirty during hydration" >&2 + git -C ghostty status --short >&2 + exit 1 + } + + - name: Upload setup identity JSON + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cmux-tui-testbox-setup-${{ github.run_id }} + path: testbox-benchmark/setup-identity.json + if-no-files-found: warn + retention-days: 14 # Always hand control back to Testbox. Warmup is setup-only: all actual # Rust builds are issued later with `blacksmith testbox run`. diff --git a/.gitignore b/.gitignore index 6e942c1ae9c..db6f2afc864 100644 --- a/.gitignore +++ b/.gitignore @@ -75,5 +75,5 @@ artifacts/ tmux-*.log # Remote-only Blacksmith cmux-tui benchmark output -testbox-benchmark/ -.cmux-scratch/ +/testbox-benchmark/ +/.cmux-scratch/ diff --git a/scripts/blacksmith-cmux-tui-testbox-stage.sh b/scripts/blacksmith-cmux-tui-testbox-stage.sh index c71d3bab158..df3451d941d 100755 --- a/scripts/blacksmith-cmux-tui-testbox-stage.sh +++ b/scripts/blacksmith-cmux-tui-testbox-stage.sh @@ -1,20 +1,28 @@ #!/usr/bin/env bash set -euo pipefail +export LC_ALL=C # This helper is intentionally remote-only. The local benchmark plan invokes it -# through `blacksmith testbox run`; the guard makes an accidental local launch a -# no-op instead of running a Rust build on the developer's Mac. +# through `blacksmith testbox run`; the environment flag is only a first guard. +# The Blacksmith kernel marker and /tmp/.testbox state below are the stronger +# signals that this command is running in the prepared Testbox VM. if [[ "${CMUX_TESTBOX_REMOTE:-}" != "1" ]]; then echo "refusing to run outside a Blacksmith Testbox (set CMUX_TESTBOX_REMOTE=1 only in the remote command)" >&2 exit 64 fi +if [[ ! -r /proc/cmdline ]] || ! grep -Eq '(^|[[:space:]])metadata_port=[^[:space:]]+' /proc/cmdline; then + echo "refusing to run without the Blacksmith Testbox metadata marker" >&2 + exit 64 +fi -if [[ $# -ne 1 ]]; then - echo "usage: CMUX_TESTBOX_REMOTE=1 $0 {first-clean|incremental-noop|changed-file}" >&2 +if [[ $# -ne 3 ]]; then + echo "usage: CMUX_TESTBOX_REMOTE=1 CMUX_TESTBOX_ID=tbx_... $0 {first-clean|incremental-noop|changed-file} " >&2 exit 64 fi stage="$1" +expected_source_sha="$2" +expected_ghostty_sha="$3" case "$stage" in first-clean|incremental-noop|changed-file) ;; *) @@ -23,99 +31,369 @@ case "$stage" in ;; esac +for value_name in expected_source_sha expected_ghostty_sha; do + value="${!value_name}" + if [[ ! "$value" =~ ^[0-9a-f]{40}$ ]]; then + echo "$value_name must be a lowercase 40-character commit SHA" >&2 + exit 64 + fi +done + +testbox_id="${CMUX_TESTBOX_ID:-}" +if [[ ! "$testbox_id" =~ ^tbx_[A-Za-z0-9_-]+$ ]]; then + echo "CMUX_TESTBOX_ID must identify the claimed Testbox" >&2 + exit 64 +fi + +state_dir=/tmp/.testbox +if [[ ! -d "$state_dir" || ! -s "$state_dir/auth_token" || ! -f "$state_dir/testbox_id" ]]; then + echo "refusing to run without the Testbox state files" >&2 + exit 64 +fi +state_testbox_id="$(tr -d '\r\n' <"$state_dir/testbox_id")" +if [[ "$state_testbox_id" != "$testbox_id" ]]; then + echo "Testbox state belongs to $state_testbox_id, expected $testbox_id" >&2 + exit 66 +fi + repo_root="$(git rev-parse --show-toplevel)" cd "$repo_root" -if [[ ! -f cmux-tui/Cargo.toml || ! -f ghostty/.git || ! -f ghostty/build.zig.zon ]]; then +ghostty_root="$repo_root/ghostty" +if [[ ! -f cmux-tui/Cargo.toml || ! -f ghostty/build.zig.zon ]]; then echo "cmux-tui and its Ghostty source submodule must be initialized" >&2 exit 65 fi +if [[ "$(git -C ghostty rev-parse --show-toplevel 2>/dev/null || true)" != "$ghostty_root" ]]; then + echo "ghostty is not an initialized submodule checkout" >&2 + exit 65 +fi +expected_tree_sha="$(git rev-parse "${expected_source_sha}^{tree}")" benchmark_dir="$repo_root/testbox-benchmark" +command -v flock >/dev/null || { + echo "flock is required for serialized Testbox stages" >&2 + exit 65 +} mkdir -p "$benchmark_dir" # Testbox can acknowledge a run while its remote shell is still flushing -# output. Serialize stages and hold the lock through artifact writes so a +# output. Serialize stages and hold the lock through all artifact writes so a # subsequent run cannot overwrite a prior stage's timing file. exec 9>"$benchmark_dir/.stage.lock" flock -x 9 + log_path="$benchmark_dir/$stage.log" time_path="$benchmark_dir/$stage.time" json_path="$benchmark_dir/$stage.json" +pre_identity_path="$benchmark_dir/.$stage.pre-identity.json" +post_identity_path="$benchmark_dir/.$stage.post-identity.json" changed_file="cmux-tui/crates/cmux-tui/src/main.rs" changed_backup="" -# shellcheck disable=SC2329 # invoked indirectly by the EXIT trap restore_changed_file() { - if [[ -n "$changed_backup" && -f "$changed_backup" ]]; then - cp "$changed_backup" "$repo_root/$changed_file" - rm -f "$changed_backup" + if [[ -n "$changed_backup" ]]; then + if [[ ! -f "$changed_backup" ]]; then + echo "changed-file backup disappeared: $changed_backup" >&2 + return 1 + fi + if ! cp "$changed_backup" "$repo_root/$changed_file"; then + return 1 + fi + if ! rm -f "$changed_backup"; then + return 1 + fi + changed_backup="" fi } -trap restore_changed_file EXIT + +# Always restore the deliberately changed source, including when Cargo exits +# non-zero. Do not let cleanup replace the build result unless restoration +# itself fails. +# shellcheck disable=SC2329 # invoked indirectly by the EXIT trap +finish_source() { + local result=$? + if [[ -n "$changed_backup" ]]; then + if ! restore_changed_file; then + echo "failed to restore $changed_file" >&2 + (( result == 0 )) && result=67 + fi + fi + exit "$result" +} +trap finish_source EXIT + +clean_status() { + local top_status ghostty_status + top_status="$(git status --porcelain=v1 --untracked-files=normal)" + if [[ -n "$top_status" ]]; then + printf '%s\n' "top-level source is dirty:" "$top_status" >&2 + return 1 + fi + ghostty_status="$(git -C ghostty status --porcelain=v1 --untracked-files=normal)" + if [[ -n "$ghostty_status" ]]; then + printf '%s\n' "Ghostty submodule is dirty:" "$ghostty_status" >&2 + return 1 + fi +} + +capture_identity() { + python3 - "$expected_source_sha" "$expected_tree_sha" "$expected_ghostty_sha" "$testbox_id" "$repo_root" <<'PY' +import json +import os +import pathlib +import platform +import subprocess +import sys + +expected_source_sha, expected_tree_sha, expected_ghostty_sha, testbox_id, repo_root = sys.argv[1:] +repo = pathlib.Path(repo_root) +ghostty = repo / "ghostty" + +def run(command, cwd=repo): + return subprocess.check_output(command, cwd=cwd, text=True, stderr=subprocess.STDOUT).strip() + +def optional_file(path): + try: + return path.read_text(encoding="utf-8").strip() + except OSError: + return None + +def status(cwd=repo): + return subprocess.check_output( + ["git", "status", "--porcelain=v1", "--untracked-files=normal"], + cwd=cwd, + text=True, + ).splitlines() + +source_sha = run(["git", "rev-parse", "HEAD"]) +source_tree_sha = run(["git", "rev-parse", "HEAD^{tree}"]) +ghostty_gitlink_sha = run(["git", "rev-parse", "HEAD:ghostty"]) +ghostty_head_sha = run(["git", "-C", "ghostty", "rev-parse", "HEAD"]) +record = { + "commit_sha": source_sha, + "tree_sha": source_tree_sha, + "expected_commit_sha": expected_source_sha, + "expected_tree_sha": expected_tree_sha, + "dirty_files": status(), + "ghostty": { + "gitlink_sha": ghostty_gitlink_sha, + "expected_gitlink_sha": expected_ghostty_sha, + "head_sha": ghostty_head_sha, + "dirty_files": status(ghostty), + }, + "testbox_id": testbox_id, + "testbox_state": { + "adopted_run_id": optional_file(pathlib.Path("/tmp/.testbox/adopted_run_id")), + "runner_host": optional_file(pathlib.Path("/tmp/.testbox/runner_host")), + "runner_ssh_port": optional_file(pathlib.Path("/tmp/.testbox/runner_ssh_port")), + }, +} +print(json.dumps(record, sort_keys=True)) +PY +} + +verify_identity() { + local identity_path="$1" + python3 - "$identity_path" "$expected_source_sha" "$expected_tree_sha" "$expected_ghostty_sha" "$testbox_id" <<'PY' +import json +import pathlib +import sys + +path, expected_source_sha, expected_tree_sha, expected_ghostty_sha, expected_testbox_id = sys.argv[1:] +record = json.loads(pathlib.Path(path).read_text(encoding="utf-8")) +errors = [] +if record.get("commit_sha") != expected_source_sha: + errors.append(f"source commit {record.get('commit_sha')} != {expected_source_sha}") +if record.get("expected_commit_sha") != expected_source_sha: + errors.append("source expectation was not recorded") +if record.get("tree_sha") != expected_tree_sha: + errors.append(f"source tree {record.get('tree_sha')} != {expected_tree_sha}") +if record.get("expected_tree_sha") != expected_tree_sha: + errors.append("source tree expectation was not recorded") +if record.get("dirty_files"): + errors.append("top-level source is dirty") +ghostty = record.get("ghostty", {}) +if ghostty.get("gitlink_sha") != expected_ghostty_sha: + errors.append(f"Ghostty gitlink {ghostty.get('gitlink_sha')} != {expected_ghostty_sha}") +if ghostty.get("expected_gitlink_sha") != expected_ghostty_sha: + errors.append("Ghostty expectation was not recorded") +if ghostty.get("head_sha") != expected_ghostty_sha: + errors.append(f"Ghostty checkout {ghostty.get('head_sha')} != {expected_ghostty_sha}") +if ghostty.get("dirty_files"): + errors.append("Ghostty submodule is dirty") +if record.get("testbox_id") != expected_testbox_id: + errors.append("Testbox identity mismatch") +if errors: + for error in errors: + print(f"source guard: {error}", file=sys.stderr) + raise SystemExit(66) +PY +} + +# Verify the immutable source and submodule before every stage. The changed-file +# stage is allowed to become dirty only after this check and must be clean again +# before its JSON record is emitted. +clean_status +capture_identity >"$pre_identity_path" +verify_identity "$pre_identity_path" + +runner_label="blacksmith-32vcpu-ubuntu-2404" +zig_bin="${CMUX_ZIG:-$(command -v zig)}" +rust_toolchain="$(rustup show active-toolchain)" +rustc_version="$(rustc --version)" +cargo_version="$(cargo --version)" +zig_version="$("$zig_bin" version)" +rust_toolchain_file_sha256="$(sha256sum cmux-tui/rust-toolchain.toml | cut -d ' ' -f 1)" +cargo_lock_sha256="$(sha256sum cmux-tui/Cargo.lock | cut -d ' ' -f 1)" +ghostty_zon_sha256="$(sha256sum ghostty/build.zig.zon | cut -d ' ' -f 1)" case "$stage" in first-clean) rm -rf "$repo_root/cmux-tui/target" ;; changed-file) + if [[ ! -f "$repo_root/$changed_file" ]]; then + echo "changed-file target is missing: $changed_file" >&2 + exit 65 + fi changed_backup="$(mktemp "${TMPDIR:-/tmp}/cmux-tui-testbox-source.XXXXXX")" cp "$repo_root/$changed_file" "$changed_backup" - printf '\n// Blacksmith Testbox changed-file timing marker.\n' >> "$repo_root/$changed_file" + printf '\n// Blacksmith Testbox changed-file timing marker.\n' >>"$repo_root/$changed_file" ;; esac start_epoch="$(python3 -c 'import time; print(time.time())')" +rm -f "$time_path" "$log_path" "$json_path" "$post_identity_path" set +e ( cd "$repo_root/cmux-tui" /usr/bin/time -p -o "$time_path" cargo build -p cmux-tui --locked ) >"$log_path" 2>&1 -status=$? +build_status=$? set -e end_epoch="$(python3 -c 'import time; print(time.time())')" -python3 - "$stage" "$start_epoch" "$end_epoch" "$status" "$time_path" "$changed_file" >"$json_path" <<'PY' +restore_status=0 +if ! restore_changed_file; then + echo "failed to restore $changed_file" >&2 + restore_status=67 +fi + +post_identity_status=0 +if ! capture_identity >"$post_identity_path"; then + echo "failed to capture post-stage source identity" >&2 + post_identity_status=66 + printf '{}\n' >"$post_identity_path" +fi +if (( post_identity_status == 0 )); then + if ! verify_identity "$post_identity_path"; then + post_identity_status=66 + fi +fi + +final_status="$build_status" +if (( final_status == 0 && restore_status != 0 )); then + final_status="$restore_status" +elif (( final_status == 0 && post_identity_status != 0 )); then + final_status="$post_identity_status" +fi + +python3 - "$stage" "$start_epoch" "$end_epoch" "$build_status" "$final_status" "$time_path" "$pre_identity_path" "$post_identity_path" "$changed_file" "$expected_source_sha" "$expected_tree_sha" "$expected_ghostty_sha" "$testbox_id" "$runner_label" "$rust_toolchain" "$rustc_version" "$cargo_version" "$zig_bin" "$zig_version" "$rust_toolchain_file_sha256" "$cargo_lock_sha256" "$ghostty_zon_sha256" >"$json_path" <<'PY' import datetime as dt import json import os +import pathlib import platform -import subprocess import sys -stage, start, end, status, time_path, changed_file = sys.argv[1:] -start = float(start) -end = float(end) -status = int(status) +( + stage, + start, + end, + build_status, + final_status, + time_path, + pre_identity_path, + post_identity_path, + changed_file, + expected_source_sha, + expected_tree_sha, + expected_ghostty_sha, + testbox_id, + runner_label, + rust_toolchain, + rustc_version, + cargo_version, + zig_bin, + zig_version, + rust_toolchain_file_sha256, + cargo_lock_sha256, + ghostty_zon_sha256, +) = sys.argv[1:] + +def read_json(path): + try: + return json.loads(pathlib.Path(path).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return {} + remote_time = {} -with open(time_path, encoding="utf-8") as handle: - for line in handle: - key, _, value = line.strip().partition(" ") +try: + for line in pathlib.Path(time_path).read_text(encoding="utf-8").splitlines(): + key, _, value = line.partition(" ") if key in {"real", "user", "sys"}: remote_time[f"time_{key}_seconds"] = float(value) +except OSError: + pass -try: - git_sha = subprocess.check_output( - ["git", "rev-parse", "HEAD"], text=True, stderr=subprocess.DEVNULL - ).strip() -except subprocess.CalledProcessError: - git_sha = "unknown" - +pre = read_json(pre_identity_path) +post = read_json(post_identity_path) record = { - "schema": 1, + "schema": 2, "stage": stage, "command": "cargo build -p cmux-tui --locked", - "exit_code": status, - "ok": status == 0, - "started_at": dt.datetime.fromtimestamp(start, dt.timezone.utc).isoformat(), - "finished_at": dt.datetime.fromtimestamp(end, dt.timezone.utc).isoformat(), - "wall_seconds": round(end - start, 3), - "git_sha": git_sha, + "build_exit_code": int(build_status), + "exit_code": int(final_status), + "ok": int(final_status) == 0, + "started_at": dt.datetime.fromtimestamp(float(start), dt.timezone.utc).isoformat(), + "finished_at": dt.datetime.fromtimestamp(float(end), dt.timezone.utc).isoformat(), + "wall_seconds": round(float(end) - float(start), 3), + "source": { + "expected_commit_sha": expected_source_sha, + "expected_tree_sha": expected_tree_sha, + "before": pre, + "after": post, + "changed_file": changed_file if stage == "changed-file" else None, + "restored": stage != "changed-file" or not post.get("dirty_files"), + }, + "ghostty": { + "expected_gitlink_sha": expected_ghostty_sha, + "before_gitlink_sha": pre.get("ghostty", {}).get("gitlink_sha"), + "before_head_sha": pre.get("ghostty", {}).get("head_sha"), + "after_gitlink_sha": post.get("ghostty", {}).get("gitlink_sha"), + "after_head_sha": post.get("ghostty", {}).get("head_sha"), + }, + "testbox": { + "id": testbox_id, + "adopted_run_id": pre.get("testbox_state", {}).get("adopted_run_id"), + }, "runner": { + "label": runner_label, "hostname": platform.node(), "arch": platform.machine(), "cpu_count": os.cpu_count(), "uname": " ".join(platform.uname()), + "host_from_testbox_state": pre.get("testbox_state", {}).get("runner_host"), + }, + "toolchain": { + "rust_toolchain": rust_toolchain, + "rustc": rustc_version, + "cargo": cargo_version, + "rust_toolchain_file_sha256": rust_toolchain_file_sha256, + "cargo_lock_sha256": cargo_lock_sha256, + "zig_path": zig_bin, + "zig": zig_version, + "ghostty_build_zig_zon_sha256": ghostty_zon_sha256, }, - "changed_file": changed_file if stage == "changed-file" else None, **remote_time, } print(json.dumps(record, sort_keys=True)) @@ -123,7 +401,11 @@ PY cat "$log_path" printf '\n--- /usr/bin/time -p (%s) ---\n' "$stage" -cat "$time_path" +if [[ -f "$time_path" ]]; then + cat "$time_path" +else + echo "time output unavailable" >&2 +fi printf '\n--- structured timing (%s) ---\n' "$stage" cat "$json_path" -exit "$status" +exit "$final_status" diff --git a/scripts/blacksmith-testbox-cleanup.sh b/scripts/blacksmith-testbox-cleanup.sh new file mode 100755 index 00000000000..196e7b20f75 --- /dev/null +++ b/scripts/blacksmith-testbox-cleanup.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 2 ]]; then + echo "usage: $0 " >&2 + exit 64 +fi + +testbox_id="$1" +evidence_dir="$2" +if [[ ! "$testbox_id" =~ ^tbx_[A-Za-z0-9_-]+$ ]]; then + echo "invalid Testbox ID: $testbox_id" >&2 + exit 64 +fi + +mkdir -p "$evidence_dir" +stop_log="$evidence_dir/stop.log" +status_log="$evidence_dir/status-after-stop.log" +list_log="$evidence_dir/list-after-stop.log" +cleanup_status=0 + +set +e +blacksmith testbox stop --id "$testbox_id" >"$stop_log" 2>&1 +stop_status=$? +set -e +if (( stop_status != 0 )); then + # A completed Testbox can race the explicit stop call. Tolerate only the + # documented terminal-state response, never an arbitrary stop failure. + if grep -Eiq 'already[[:space:]]+(stopped|completed)' "$stop_log"; then + printf 'stop already reached a terminal state for %s; continuing\n' "$testbox_id" >&2 + else + echo "failed to stop Testbox $testbox_id; see $stop_log" >&2 + cleanup_status=$stop_status + fi +fi + +# Status is diagnostic. The authoritative cleanup check below is the specific +# ID in `list --all`, because Blacksmith may remove completed boxes immediately. +set +e +blacksmith testbox status --id "$testbox_id" >"$status_log" 2>&1 +status_status=$? +set -e +if (( status_status == 0 )); then + status_value="$(awk -v id="$testbox_id" '$1 == id { print tolower($2); exit }' "$status_log")" + case "$status_value" in + completed|stopped|cancelled|failed|terminated) + ;; + ready|running|hydrating|in_progress|queued) + echo "Testbox $testbox_id is still active after cleanup" >&2 + (( cleanup_status == 0 )) && cleanup_status=1 + ;; + *) + if grep -Eiq 'status:[[:space:]]*(completed|stopped|cancelled|failed|terminated)' "$status_log"; then + : + else + echo "could not establish a terminal status for Testbox $testbox_id; see $status_log" >&2 + (( cleanup_status == 0 )) && cleanup_status=1 + fi + ;; + esac +elif ! grep -Eiq '(not found|already[[:space:]]+(stopped|completed)|HTTP[[:space:]]+409|status[[:space:]]+code[[:space:]]+409)' "$status_log"; then + echo "failed to inspect Testbox $testbox_id; see $status_log" >&2 + (( cleanup_status == 0 )) && cleanup_status=$status_status +fi + +set +e +blacksmith testbox list --all >"$list_log" 2>&1 +list_status=$? +set -e +if (( list_status != 0 )); then + echo "failed to list Testboxes after stopping $testbox_id; see $list_log" >&2 + (( cleanup_status == 0 )) && cleanup_status=$list_status +elif grep -Fq -- "$testbox_id" "$list_log"; then + echo "Testbox $testbox_id is still present in the active inventory; see $list_log" >&2 + (( cleanup_status == 0 )) && cleanup_status=1 +fi + +if (( cleanup_status != 0 )); then + exit "$cleanup_status" +fi +printf 'verified Testbox %s is no longer active\n' "$testbox_id" diff --git a/skills/blacksmith-testbox/SKILL.md b/skills/blacksmith-testbox/SKILL.md index 6ad339a000b..99b9c90db67 100644 --- a/skills/blacksmith-testbox/SKILL.md +++ b/skills/blacksmith-testbox/SKILL.md @@ -1,9 +1,9 @@ --- name: blacksmith-testbox description: > - Provision and reuse a beefy Blacksmith Testbox for cmux-tui Rust builds, - capture remote timings, download raw evidence, and clean up safely. Never - run cargo, rustc, or Zig builds on the local Mac. + Provision and reuse a trusted Blacksmith Testbox for cmux-tui Rust builds, + capture remote timings, download raw evidence, and clean up safely. Never run + cargo, rustc, or Zig builds on the local Mac. --- # cmux-tui Blacksmith Testbox @@ -11,137 +11,259 @@ description: > This lane is Linux-only. It uses `.github/workflows/ci-workflow-guard-tests-testbox.yml`, job `cmux-tui-rust`, on `blacksmith-32vcpu-ubuntu-2404`. The workflow is a -setup-only entrypoint for a reusable Testbox. It checks out the exact dispatch -SHA, initializes the `ghostty` source submodule, installs Linux C/LLVM headers, -installs the repository-pinned Zig and Rust toolchains, fetches Zig and Cargo -dependencies, and then hands control back to Testbox. It does not run Rust tests -or Rust compilation during warmup. `zig build --fetch` only hydrates Zig -packages and exits before compilation. +setup-only entrypoint for a reusable Testbox. It validates a pushed branch +ref, checks out and verifies the exact dispatch commit, initializes the +`ghostty` source submodule, installs Linux C/LLVM headers, installs the +repository-pinned Zig and Rust toolchains, fetches Zig and Cargo dependencies, +records runner/toolchain/Ghostty identity in JSON, and then hands control back +to Testbox. It does not run Rust tests or Rust compilation during warmup. +`zig build --fetch` only hydrates Zig packages and exits before compilation. The repository's single Rust toolchain source is `cmux-tui/rust-toolchain.toml`. The workflow invokes `./.github/actions/setup-cmux-tui-rust`, so a workflow-specific Rust version must never be added. -## Hard safety boundary +## Hard safety and trust boundary + +This is a trusted-maintainer lane, not a pull-request validation lane. +`useblacksmith/begin-testbox` writes `/tmp/.testbox/auth_token` into the +candidate command environment. Any command run through `blacksmith testbox +run` can read that token and can inspect anything else exposed to the GitHub +job. `permissions: contents: read` does not prevent token access, and this +workflow intentionally grants no other GitHub permissions or workflow secrets. + +Before using the lane, a repository administrator must create the +`blacksmith-testbox-trusted` GitHub environment, configure required reviewers +(or an equivalent manual approval rule), and leave the environment secret set +empty. GitHub evaluates that approval before the job's first step, including +`begin-testbox`. If the environment does not exist or has no required reviewer, +stop: the lane is not production-safe. Never dispatch it for an untrusted PR, +fork, branch containing unreviewed workflow/helper changes, or source supplied +by an external contributor. When trust changes, stop the old box and warm a +fresh one. + +The helper retains the `CMUX_TESTBOX_REMOTE=1` guard for accidental local +launches, and additionally requires the Blacksmith VM kernel metadata marker +and matching `/tmp/.testbox` state. The environment flag remains caller +controlled and is not an authentication mechanism. The protected environment +and trusted-maintainer policy are the security boundary. * Never run `cargo`, `rustc`, `rustup`, `zig build`, or another Rust/Zig build command on Lawrence's Mac. This includes local fallback builds and local test commands. * Run every Blacksmith CLI command from the root of the intended isolated - worktree. The CLI synchronizes that directory and can delete remote files - that are not represented locally. + worktree. The CLI synchronizes that directory with `rsync --delete`; it can + delete remote files that are not represented locally. * Put remote build commands inside `blacksmith testbox run`. The benchmark - helper also requires `CMUX_TESTBOX_REMOTE=1`, so an accidental local launch + helper also requires the Testbox VM guard, so an accidental local launch exits before invoking a compiler. -* Use this Testbox only for Linux-compatible cmux-tui work. Use the hosted - macOS workflows for Swift, Xcode, XCTest, GUI, and app-host verification. +* Use this Testbox only for Linux-compatible cmux-tui work. Use hosted macOS + workflows for Swift, Xcode, XCTest, GUI, and app-host verification. -## Authentication and root guard +## Authentication and CLI installation Check authentication without printing credentials: ```bash blacksmith auth whoami +blacksmith --version ``` -If the CLI is missing, install it once with: +Use the repository or organization-approved, pinned Blacksmith CLI artifact or +package-manager version. Record its version in the evidence. Do not install a +mutable remote script with `curl ... | sh`; if the approved pinned artifact is +not available, stop and ask the tooling owner rather than weakening this lane. -```bash -curl -fsSL https://get.blacksmith.sh | sh -``` - -If authentication or dispatch is blocked, use the Blacksmith Console at -https://app.blacksmith.sh. The exact retry command, after the workflow is -available on the selected ref, is: - -```bash -cd "$(git rev-parse --show-toplevel)" -SHA="$(git rev-parse HEAD)" -blacksmith testbox warmup .github/workflows/ci-workflow-guard-tests-testbox.yml \ - --ref "$SHA" --job cmux-tui-rust --idle-timeout 30 -``` +## Exact source contract -A workflow that exists only on a feature branch can be rejected by GitHub's -workflow-dispatch API. If Blacksmith reports `workflow not found` or a 404, -do not change the ref or fall back to a local build. Open the Console, make the -workflow available on the repository's dispatchable default-branch revision, -and rerun the exact command above with the full SHA recorded in the evidence. +Blacksmith/GitHub rejected the previous raw-SHA warmup with HTTP 422 (`No ref +found`). Do not pass a commit SHA to `blacksmith testbox warmup --ref`, and do +not document raw SHA refs as supported. Warm up a pushed branch ref, then carry +the full SHA separately and assert it remotely. -Use this guard before any warmup or run: +Run this preflight from the clean worktree root: ```bash +set -euo pipefail cd "$(git rev-parse --show-toplevel)" -SHA="$(git rev-parse HEAD)" -case "$SHA" in - [0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]*) ;; - *) echo "HEAD is not a full commit SHA" >&2; exit 1 ;; -esac +SOURCE_REF="$(git symbolic-ref --short HEAD)" +[[ -n "$SOURCE_REF" ]] || { echo "HEAD is detached" >&2; exit 1; } +SOURCE_SHA="$(git rev-parse HEAD)" +GHOSTTY_SHA="$(git rev-parse HEAD:ghostty)" +[[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]] +[[ "$GHOSTTY_SHA" =~ ^[0-9a-f]{40}$ ]] +[[ -z "$(git status --porcelain=v1 --untracked-files=normal)" ]] +remote_sha="$(git ls-remote --exit-code origin "refs/heads/$SOURCE_REF" | awk 'NR == 1 { print $1 }')" +[[ "$remote_sha" == "$SOURCE_SHA" ]] || { + echo "the pushed branch is not the exact local HEAD" >&2 + exit 1 +} +printf 'source_ref=%s\nsource_sha=%s\nghostty_gitlink_sha=%s\n' \ + "$SOURCE_REF" "$SOURCE_SHA" "$GHOSTTY_SHA" ``` -The branch must be pushed before warmup. Record `git status --short --branch` -and the full SHA in the scratch evidence before dispatching. +The branch can move after this check. Every remote stage rechecks the full +source SHA, commit tree, GitHub `ghostty` gitlink, initialized Ghostty HEAD, +and clean status before building. A moved branch or dirty/mismatched checkout +fails closed; do not silently warm another revision. + +The workflow has an optional `source_sha` workflow-dispatch input for a direct +GitHub dispatch. The Blacksmith CLI supplies `testbox_id` but does not expose +arbitrary workflow inputs, so the normal CLI path uses `github.sha` plus the +remote assertion in the helper. A supplied `source_sha` must be a lowercase +40-character SHA equal to the branch dispatch SHA. -## Warmup, run, and stop +## Warmup and identity capture -Warm the exact branch head with the stable workflow/job pair: +Warm the validated branch, never its SHA: ```bash -cd "$(git rev-parse --show-toplevel)" -SHA="$(git rev-parse HEAD)" -blacksmith testbox warmup .github/workflows/ci-workflow-guard-tests-testbox.yml \ - --ref "$SHA" \ - --job cmux-tui-rust \ +WORKFLOW=.github/workflows/ci-workflow-guard-tests-testbox.yml +JOB=cmux-tui-rust +blacksmith testbox warmup "$WORKFLOW" \ + --ref "$SOURCE_REF" \ + --job "$JOB" \ --idle-timeout 30 ``` -Save the returned `tbx_...` ID. One ID belongs to one worktree or agent. Wait -for setup if needed: +Save the returned `tbx_...` ID. One ID belongs to one worktree and one trust +context. The workflow concurrency group keys the Testbox ID and source, and +each remote stage holds `testbox-benchmark/.stage.lock` through its build and +artifact writes. Do not issue concurrent `run` commands against one ID. + +Wait for setup and capture the exact run identity: ```bash -blacksmith testbox status --id --wait --wait-timeout 15m +blacksmith testbox status --id "$TBX" --wait --wait-timeout 15m +blacksmith testbox run --id "$TBX" --debug \ + "set -euo pipefail; test -s /tmp/.testbox/auth_token; grep -Eq '(^|[[:space:]])metadata_port=[^[:space:]]+' /proc/cmdline; test \"\$(git rev-parse HEAD)\" = \"$SOURCE_SHA\"; test \"\$(git rev-parse HEAD:ghostty)\" = \"$GHOSTTY_SHA\"; test \"\$(git -C ghostty rev-parse HEAD)\" = \"$GHOSTTY_SHA\"; test -z \"\$(git status --porcelain=v1 --untracked-files=normal)\"; test -z \"\$(git -C ghostty status --porcelain=v1 --untracked-files=normal)\"; rustup show active-toolchain; rustc --version; cargo --version; \"$CMUX_ZIG\" version" ``` -Run harmless remote identity and dependency checks from the repository root: +The setup workflow also uploads `setup-identity.json`. It contains the +workflow run, source ref/SHA/tree, Ghostty gitlink and checkout SHA, runner +label/architecture/CPU identity, and pinned Rust/Zig/toolchain-file metadata. +Never print or download `/tmp/.testbox/auth_token`. -```bash -blacksmith testbox run --id --debug \ - 'set -euo pipefail; printf "sha="; git rev-parse HEAD; printf "runner="; uname -a; printf "cpus="; nproc; rustup show active-toolchain; rustc --version; cargo --version; "${CMUX_ZIG:-zig}" version' -``` +## Remote benchmark stages -Run a build only through the remote-only timing helper: +Before each stage, recompute `SOURCE_SHA` and `GHOSTTY_SHA` and repeat the +clean pushed-branch preflight. Pass the expected values as validated arguments; +the helper does not trust the remote checkout or a caller-supplied expected SHA +without comparing it to Git metadata: ```bash -blacksmith testbox run --id --debug \ - 'CMUX_TESTBOX_REMOTE=1 ./scripts/blacksmith-cmux-tui-testbox-stage.sh first-clean' -blacksmith testbox run --id --debug \ - 'CMUX_TESTBOX_REMOTE=1 ./scripts/blacksmith-cmux-tui-testbox-stage.sh incremental-noop' -blacksmith testbox run --id --debug \ - 'CMUX_TESTBOX_REMOTE=1 ./scripts/blacksmith-cmux-tui-testbox-stage.sh changed-file' +run_stage() { + local stage="$1" + local run_status download_status=0 + set +e + blacksmith testbox run --id "$TBX" --debug \ + "CMUX_TESTBOX_REMOTE=1 CMUX_TESTBOX_ID=$TBX CMUX_TESTBOX_SOURCE_REF=$SOURCE_REF ./scripts/blacksmith-cmux-tui-testbox-stage.sh $stage $SOURCE_SHA $GHOSTTY_SHA" \ + 2>&1 | tee "$OUT/$stage.run.log" + run_status=${PIPESTATUS[0]} + set -e + + # Download immediately. Blacksmith's next rsync may delete or replace remote + # files, so a one-time download after all stages is insufficient. + for suffix in json time log; do + if ! blacksmith testbox download --id "$TBX" \ + "testbox-benchmark/$stage.$suffix" "$OUT/raw/$stage.$suffix" \ + 2>&1 | tee -a "$OUT/$stage.download.log"; then + download_status=1 + fi + done + if (( run_status != 0 )); then + return "$run_status" + fi + return "$download_status" +} ``` -The complete capture procedure, including warmup wall time, run transcripts, -and aggregation, is in `skills/blacksmith-testbox/benchmark.md`. +The helper supports exactly `first-clean`, `incremental-noop`, and +`changed-file`. It records a schema-2 JSON object for each stage containing: -Download raw remote timing files before stopping: +* expected and observed source commit/tree identity before and after the build; +* expected and observed Ghostty gitlink and initialized submodule HEAD; +* clean/dirty file lists and source restoration status; +* Testbox ID and adopted workflow run ID; +* runner label, hostname, architecture, CPU count, and `uname`; +* active Rust toolchain, `rustc`, Cargo, Zig, lockfile/toolchain hashes, and + Ghostty package-manifest hash; and +* Cargo exit status, `/usr/bin/time -p` values, and CLI transcript timing. + +`first-clean` removes only the remote `cmux-tui/target` directory before a +`cargo build -p cmux-tui --locked`. `incremental-noop` repeats that command +without changing source. `changed-file` appends a comment to +`cmux-tui/crates/cmux-tui/src/main.rs`, builds, and restores the original bytes +before emitting its final record. A dirty or mismatched source before any +stage, after restoration, or in the Ghostty submodule aborts the stage. + +After all successful downloads, aggregate and verify the records: ```bash -mkdir -p .cmux-scratch/testbox-benchmark-raw -blacksmith testbox download --id testbox-benchmark/ \ - .cmux-scratch/testbox-benchmark-raw/ +python3 - "$OUT" "$SOURCE_SHA" "$GHOSTTY_SHA" "$TBX" <<'PY' +import json +import pathlib +import subprocess +import sys + +out = pathlib.Path(sys.argv[1]) +expected_source, expected_ghostty, testbox_id = sys.argv[2:] +expected_tree = subprocess.check_output( + ["git", "rev-parse", f"{expected_source}^{{tree}}"], text=True +).strip() +required = {"first-clean", "incremental-noop", "changed-file"} +records = [] +for path in sorted((out / "raw").glob("*.json")): + record = json.loads(path.read_text(encoding="utf-8")) + records.append(record) +if {record.get("stage") for record in records} != required: + raise SystemExit("timing evidence is missing one or more benchmark stages") +for record in records: + if record.get("testbox", {}).get("id") != testbox_id: + raise SystemExit(f"{record.get('stage')} has the wrong Testbox ID") + source = record.get("source", {}) + if source.get("expected_commit_sha") != expected_source or source.get("expected_tree_sha") != expected_tree: + raise SystemExit(f"{record.get('stage')} has the wrong expected source identity") + for side in ("before", "after"): + snapshot = source.get(side, {}) + if snapshot.get("commit_sha") != expected_source or snapshot.get("tree_sha") != expected_tree: + raise SystemExit(f"{record.get('stage')} has the wrong {side} source SHA") + if snapshot.get("dirty_files"): + raise SystemExit(f"{record.get('stage')} has dirty top-level source") + ghostty = snapshot.get("ghostty", {}) + if ghostty.get("gitlink_sha") != expected_ghostty or ghostty.get("head_sha") != expected_ghostty: + raise SystemExit(f"{record.get('stage')} has mismatched Ghostty identity") + if ghostty.get("dirty_files"): + raise SystemExit(f"{record.get('stage')} has dirty Ghostty source") + if not record.get("ok"): + raise SystemExit(f"{record.get('stage')} did not complete successfully") +with (out / "timings.json").open("w", encoding="utf-8") as handle: + json.dump({"schema": 2, "source_sha": expected_source, "ghostty_gitlink_sha": expected_ghostty, "testbox_id": testbox_id, "stages": records}, handle, indent=2, sort_keys=True) + handle.write("\n") +PY ``` -Stop disposable capacity and prove cleanup: +Keep `raw/*.json`, `raw/*.time`, `raw/*.log`, every `*.run.log` and download +log, the setup artifact, and the source manifest in the separate +`.cmux-scratch/` evidence directory. Do not add credentials or private keys. + +## Fail-safe cleanup + +Always download before cleanup. Use the checked-in cleanup helper rather than +ignoring errors with `|| true`: ```bash -blacksmith testbox stop --id -blacksmith testbox list --all +scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" ``` -`list --all` must report no active Testbox. Keep the warmup log, setup/job/run -IDs, status output, downloaded `*.json`/`*.time`/`*.log` files, stop output, and -final list output in a separate `.cmux-scratch/` evidence artifact. Never add -credentials or private keys to that artifact. +It records stop, post-stop status, and `list --all` output; verifies that the +specific Testbox ID is terminal or absent from the active inventory; and +accepts only the known race where stop returns a 409 saying the box is already +stopped or completed. Other stop, status, or list failures remain failures. +Put it in an `EXIT` trap that preserves the benchmark's original exit status +unless cleanup itself fails. If warmup never returned an ID, still run +`blacksmith testbox list --all` and retain its exit status and output. ## Timing interpretation @@ -153,13 +275,13 @@ CLI wall time when evaluating Testbox overhead. `first-clean` is target-clean but dependency-warm: warmup runs `cargo fetch` and `zig build --fetch`, and the workflow may restore registry, git, and Zig caches. -`incremental-noop` measures a second build on the same VM. `changed-file` adds a -comment to `cmux-tui/crates/cmux-tui/src/main.rs`, builds, and restores the -original file. The helper serializes stages with a remote lock. These are -deliberately different from a cold-VM benchmark. - -Do not invent a local baseline. The cmux-tui instructions prohibit local Rust -builds, so local evidence is explicitly unavailable. Prior hosted correctness -runs without Cargo durations are provenance only. Prior Blacksmith macOS -Swift/Xcode timing artifacts use a different OS, architecture, runner, cache, -and workload, so they are context rather than a comparable Rust baseline. +`incremental-noop` measures a second build on the same VM. `changed-file` is a +controlled source change on that same VM. These are deliberately different +from a cold-VM benchmark. + +The existing 32-vCPU evidence at +`.cmux-scratch/blacksmith-testbox-e40704611ac35f4ffa153/` remains historical +provenance for setup SHA `e40704611ac35f0e3a806841a9eae383f4ffa153`, Testbox +`tbx_01kzxebn91nhatkv4ygevh06vs`, and workflow run `31696013711`. Its raw +records and cleanup result must not be rewritten when validating this hardening +change. diff --git a/skills/blacksmith-testbox/benchmark.md b/skills/blacksmith-testbox/benchmark.md index 54f1d2f1cfa..28739a4d1d4 100644 --- a/skills/blacksmith-testbox/benchmark.md +++ b/skills/blacksmith-testbox/benchmark.md @@ -12,77 +12,89 @@ setup-only GitHub job on the remote Linux runner. | Workflow | `.github/workflows/ci-workflow-guard-tests-testbox.yml` | | Job | `cmux-tui-rust` | | Runner | `blacksmith-32vcpu-ubuntu-2404` | +| Protected environment | `blacksmith-testbox-trusted` | | Rust source of truth | `cmux-tui/rust-toolchain.toml`, via `./.github/actions/setup-cmux-tui-rust` | | Remote build helper | `scripts/blacksmith-cmux-tui-testbox-stage.sh` | -| Remote output | `testbox-benchmark/`, ignored by the repository so repeated runs retain it | +| Cleanup helper | `scripts/blacksmith-testbox-cleanup.sh` | +| Remote output | `testbox-benchmark/` | + +A repository administrator must configure the protected environment with +required reviewers and no secrets before this plan is usable. The lane must +never run untrusted PR or fork code. `begin-testbox` exposes its auth token to +commands in the Testbox, so `contents: read` is not a trust boundary. The warmup job only checks out the exact dispatch SHA, initializes `ghostty`, -installs Linux headers/tools, installs the pinned Zig and Rust toolchains, and -fetches Cargo and Zig dependencies. `zig build --fetch` is the only build-system -operation in warmup, and it exits before compiling. Rust builds happen only in -the three explicit benchmark runs. +installs Linux headers/tools, installs the pinned Zig and Rust toolchains, +fetches Cargo and Zig dependencies, and records JSON identity. `zig build +--fetch` is the only build-system operation in warmup, and it exits before +compiling. Rust builds happen only in the three explicit benchmark runs. The current Blacksmith catalog reports the requested x64 label as 32 vCPU and 121.6 GB, while the ARM label with the same vCPU count reports 96 GB. Keep the -requested `blacksmith-32vcpu-ubuntu-2404` label unless the repository's Linux +requested `blacksmith-32vcpu-ubuntu-2404` label unless repository Linux constraints make x64 impossible, and record the catalog result with the run. -## Warmup and identity capture +## Exact pushed source and Ghostty identity + +Blacksmith/GitHub rejected the previous raw-SHA `--ref` with HTTP 422 (`No ref +found`). A raw commit SHA is not a supported warmup ref for this lane. Use a +pushed branch ref and carry the full source SHA as an assertion: ```bash set -euo pipefail cd "$(git rev-parse --show-toplevel)" -SHA="$(git rev-parse HEAD)" -WORKFLOW=.github/workflows/ci-workflow-guard-tests-testbox.yml -JOB=cmux-tui-rust -OUT="$PWD/.cmux-scratch/blacksmith-testbox-cmux-tui-$SHA" -mkdir -p "$OUT/raw" - -blacksmith auth whoami - -now() { python3 -c 'import time; print(f"{time.time():.6f}")'; } -TBX="" -cleanup() { - local result=$? - trap - EXIT - if [[ -n "$TBX" ]]; then - blacksmith testbox stop --id "$TBX" >"$OUT/stop.log" 2>&1 || true - fi - blacksmith testbox list --all >"$OUT/list-after-stop.log" 2>&1 || true - exit "$result" +SOURCE_REF="$(git symbolic-ref --short HEAD)" +SOURCE_SHA="$(git rev-parse HEAD)" +GHOSTTY_SHA="$(git rev-parse HEAD:ghostty)" +[[ -n "$SOURCE_REF" ]] +[[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]] +[[ "$GHOSTTY_SHA" =~ ^[0-9a-f]{40}$ ]] +[[ -z "$(git status --porcelain=v1 --untracked-files=normal)" ]] +remote_sha="$(git ls-remote --exit-code origin "refs/heads/$SOURCE_REF" | awk 'NR == 1 { print $1 }')" +[[ "$remote_sha" == "$SOURCE_SHA" ]] || { + echo "push the exact clean branch head before warming Testbox" >&2 + exit 1 } -trap cleanup EXIT - -warmup_started="$(now)" -set +e -blacksmith testbox warmup "$WORKFLOW" \ - --ref "$SHA" \ - --job "$JOB" \ - --idle-timeout 30 2>&1 | tee "$OUT/warmup.log" -warmup_status=${PIPESTATUS[0]} -set -e -warmup_finished="$(now)" -python3 - "$SHA" "$JOB" "$warmup_started" "$warmup_finished" "$warmup_status" "$OUT/warmup.json" <<'PY' +mkdir -p ".cmux-scratch/blacksmith-testbox-$SOURCE_SHA/raw" +python3 - "$SOURCE_REF" "$SOURCE_SHA" "$GHOSTTY_SHA" > ".cmux-scratch/blacksmith-testbox-$SOURCE_SHA/source.json" <<'PY' import json +import subprocess import sys -sha, job, started, finished, status, output = sys.argv[1:] +ref, sha, ghostty = sys.argv[1:] record = { - "sha": sha, - "job": job, - "started_epoch": float(started), - "finished_epoch": float(finished), - "wall_seconds": round(float(finished) - float(started), 3), - "exit_code": int(status), + "source_ref": ref, + "source_sha": sha, + "source_tree_sha": subprocess.check_output(["git", "rev-parse", "HEAD^{tree}"], text=True).strip(), + "ghostty_gitlink_sha": ghostty, } -with open(output, "w", encoding="utf-8") as handle: - json.dump(record, handle, indent=2, sort_keys=True) - handle.write("\n") +print(json.dumps(record, indent=2, sort_keys=True)) PY -if [[ "$warmup_status" -ne 0 ]]; then - echo "warmup failed; inspect $OUT/warmup.log and use https://app.blacksmith.sh" >&2 - exit "$warmup_status" -fi +``` + +Before every stage, rerun the clean-branch check and recompute the expected +values. If the branch moved, the worktree became dirty, or the Ghostty pointer +changed, stop the box and start a new evidence directory. Do not silently +substitute the new SHA. + +## Warmup and setup identity + +Use the branch ref, not `SOURCE_SHA`, in warmup: + +```bash +WORKFLOW=.github/workflows/ci-workflow-guard-tests-testbox.yml +JOB=cmux-tui-rust +OUT="$PWD/.cmux-scratch/blacksmith-testbox-$SOURCE_SHA" +TBX="" +mkdir -p "$OUT/raw" +blacksmith auth whoami +blacksmith --version | tee "$OUT/blacksmith-version.txt" +blacksmith runners catalog > "$OUT/runner-catalog.json" +blacksmith testbox warmup "$WORKFLOW" \ + --ref "$SOURCE_REF" \ + --job "$JOB" \ + --idle-timeout 30 \ + 2>&1 | tee "$OUT/warmup.log" TBX="$(python3 - "$OUT/warmup.log" <<'PY' import re import sys @@ -90,40 +102,64 @@ import sys text = open(sys.argv[1], encoding="utf-8").read() ids = re.findall(r"\btbx_[A-Za-z0-9_-]+\b", text) if not ids: - raise SystemExit("warmup output did not contain a tbx_ ID") + raise SystemExit("warmup output did not contain a Testbox ID") print(ids[-1]) PY )" printf 'Testbox ID: %s\n' "$TBX" | tee "$OUT/testbox-id.txt" - blacksmith testbox status --id "$TBX" --wait --wait-timeout 15m \ 2>&1 | tee "$OUT/status-ready.log" +``` + +The workflow validates that the dispatch ref is a pushed branch, that +`github.sha` is a full SHA, and that the branch still resolves to that SHA. If +a direct GitHub dispatch supplies the optional `source_sha` input, it must equal +`github.sha`. The Blacksmith CLI path relies on the same full SHA passed to the +remote helper because the CLI only supplies `testbox_id` to workflow inputs. + +Capture a harmless remote identity transcript before builds: + +```bash blacksmith testbox run --id "$TBX" --debug \ - 'set -euo pipefail; printf "remote_sha="; git rev-parse HEAD; printf "runner="; uname -a; printf "cpus="; nproc; rustup show active-toolchain; rustc --version; cargo --version; "$CMUX_ZIG" version' \ + "set -euo pipefail; test -s /tmp/.testbox/auth_token; grep -Eq '(^|[[:space:]])metadata_port=[^[:space:]]+' /proc/cmdline; test \"\$(git rev-parse HEAD)\" = \"$SOURCE_SHA\"; test \"\$(git rev-parse HEAD:ghostty)\" = \"$GHOSTTY_SHA\"; test \"\$(git -C ghostty rev-parse HEAD)\" = \"$GHOSTTY_SHA\"; test -z \"\$(git status --porcelain=v1 --untracked-files=normal)\"; test -z \"\$(git -C ghostty status --porcelain=v1 --untracked-files=normal)\"; rustup show active-toolchain; rustc --version; cargo --version; \"$CMUX_ZIG\" version" \ 2>&1 | tee "$OUT/identity.run.log" ``` -The warmup transcript is the source for the warmup ID and the GitHub job/run -IDs. Preserve it verbatim. Do not substitute a branch name for `SHA`: the -checkout guard and the evidence file are intended to prove the exact 40-byte -revision tested. +The setup job's `setup-identity.json` is uploaded as a GitHub artifact. Keep +its artifact URL or download it into `$OUT`; it records runner/toolchain/ +Ghostty identity independently of the stage helper. ## Three remote build timings -The helper creates a structured JSON record, a raw Cargo log, and the raw -`/usr/bin/time -p` output for each stage. Its environment guard prevents a -local accidental launch. +The helper creates one structured JSON record, one raw Cargo log, and one raw +`/usr/bin/time -p` file per stage. It verifies source and submodule identity +before the stage, holds a remote `flock` through all writes, restores the +controlled changed file, and verifies clean identity again. ```bash run_stage() { local stage="$1" + local run_status download_status=0 set +e blacksmith testbox run --id "$TBX" --debug \ - "CMUX_TESTBOX_REMOTE=1 ./scripts/blacksmith-cmux-tui-testbox-stage.sh $stage" \ + "CMUX_TESTBOX_REMOTE=1 CMUX_TESTBOX_ID=$TBX CMUX_TESTBOX_SOURCE_REF=$SOURCE_REF ./scripts/blacksmith-cmux-tui-testbox-stage.sh $stage $SOURCE_SHA $GHOSTTY_SHA" \ 2>&1 | tee "$OUT/$stage.run.log" - local status=${PIPESTATUS[0]} + run_status=${PIPESTATUS[0]} set -e - return "$status" + + # rsync --delete can remove remote output before the next run. Download each + # stage immediately, before starting another stage. + for suffix in json time log; do + if ! blacksmith testbox download --id "$TBX" \ + "testbox-benchmark/$stage.$suffix" "$OUT/raw/$stage.$suffix" \ + 2>&1 | tee -a "$OUT/$stage.download.log"; then + download_status=1 + fi + done + if (( run_status != 0 )); then + return "$run_status" + fi + return "$download_status" } benchmark_status=0 @@ -133,68 +169,92 @@ for stage in first-clean incremental-noop changed-file; do break fi done +``` -mkdir -p "$OUT/raw" -blacksmith testbox download --id "$TBX" testbox-benchmark/ "$OUT/raw/" \ - 2>&1 | tee "$OUT/download.log" +`first-clean` is target-clean but dependency-warm. `incremental-noop` repeats +the exact build on the same VM. `changed-file` appends a comment to +`cmux-tui/crates/cmux-tui/src/main.rs`, builds, and restores the original bytes. +The local worktree is never mutated by the helper. + +Verify the downloaded records before accepting timings: -python3 - "$OUT" "$SHA" "$TBX" <<'PY' +```bash +python3 - "$OUT" "$SOURCE_SHA" "$GHOSTTY_SHA" "$TBX" <<'PY' import json import pathlib +import subprocess import sys out = pathlib.Path(sys.argv[1]) -sha, testbox_id = sys.argv[2:] -records = [] -for path in sorted((out / "raw").glob("*.json")): - records.append(json.loads(path.read_text(encoding="utf-8"))) +expected_source, expected_ghostty, testbox_id = sys.argv[2:] +expected_tree = subprocess.check_output( + ["git", "rev-parse", f"{expected_source}^{{tree}}"], text=True +).strip() +required = {"first-clean", "incremental-noop", "changed-file"} +records = [json.loads(path.read_text(encoding="utf-8")) for path in sorted((out / "raw").glob("*.json"))] +if {record.get("stage") for record in records} != required: + raise SystemExit("expected exactly three stage records") +for record in records: + stage = record.get("stage") + if record.get("testbox", {}).get("id") != testbox_id: + raise SystemExit(f"{stage}: wrong Testbox ID") + source_record = record.get("source", {}) + if source_record.get("expected_commit_sha") != expected_source or source_record.get("expected_tree_sha") != expected_tree: + raise SystemExit(f"{stage}: wrong expected source identity") + for side in ("before", "after"): + source = source_record.get(side, {}) + if source.get("commit_sha") != expected_source or source.get("tree_sha") != expected_tree or source.get("dirty_files"): + raise SystemExit(f"{stage}: source mismatch or dirty {side} checkout") + ghostty = source.get("ghostty", {}) + if ghostty.get("gitlink_sha") != expected_ghostty or ghostty.get("head_sha") != expected_ghostty or ghostty.get("dirty_files"): + raise SystemExit(f"{stage}: Ghostty mismatch or dirty {side} checkout") + if not record.get("ok"): + raise SystemExit(f"{stage}: build failed") with (out / "timings.json").open("w", encoding="utf-8") as handle: - json.dump({"sha": sha, "testbox_id": testbox_id, "stages": records}, handle, indent=2, sort_keys=True) + json.dump({"schema": 2, "source_sha": expected_source, "ghostty_gitlink_sha": expected_ghostty, "testbox_id": testbox_id, "stages": records}, handle, indent=2, sort_keys=True) handle.write("\n") PY -exit "$benchmark_status" ``` -`first-clean` removes only the remote `cmux-tui/target` directory before a -debug `cargo build -p cmux-tui --locked`. `incremental-noop` repeats that exact -command without changing source. `changed-file` appends a comment to -`cmux-tui/crates/cmux-tui/src/main.rs` immediately before the build, which -changes Cargo's fingerprint, then restores the original bytes before the remote -command exits. The helper holds a remote stage lock so concurrent Testbox run -requests cannot overwrite timing files. The local worktree is never mutated by -the helper. - -The local `*.run.log` files include Testbox run IDs and CLI wall time. The - downloaded `raw/*.json`, `raw/*.log`, and `raw/*.time` files are the remote - timing evidence. Keep both clocks: CLI wall time includes sync/transport, -while `time_real_seconds` measures the remote Cargo command. - -## Historical comparison and cleanup - -Record the following alongside `timings.json`: - -1. The exact runner label and the output of `blacksmith runners catalog`. -2. The warmup ID, setup job/run ID, and each `blacksmith testbox run` ID from the - raw transcripts. -3. Any prior hosted cmux-tui timing evidence found under the hq scratch/artifact - directories. Local Rust timing evidence is intentionally absent because this - Mac is prohibited from running Rust builds. -4. Whether the comparison was target-clean, registry/git-cache warm, Zig-cache - warm, or a genuinely cold VM. The setup workflow deliberately warms the - dependency caches, so `first-clean` is target-cold and dependency-warm. - -Prior Blacksmith macOS reload timing artifacts, if used for context, are not a -Rust baseline: they measure a Swift/Xcode build on a different OS, architecture, -runner SKU, cache state, and workload. Hosted cmux-tui correctness runs without -recorded Cargo durations are provenance, not performance comparisons. - -Always download before stopping, then stop and verify there is no active box: +## Cleanup and evidence + +Download all raw files and `timings.json` before cleanup. Then call the +fail-safe helper, which preserves an already-completed 409 but fails on other +stop/status/list errors and verifies this exact Testbox ID is no longer active: ```bash -blacksmith testbox stop --id "$TBX" -blacksmith testbox list --all +scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" ``` -The final list must report no active Testboxes. Keep `warmup.log`, all run -transcripts, `download.log`, `stop.log`, `list-after-stop.log`, and the raw -remote directory in the separate `.cmux-scratch/` evidence artifact. +A shell `EXIT` trap should call that helper while preserving the benchmark +status. If warmup did not return a Testbox ID, still run +`blacksmith testbox list --all` and save its output. Keep warmup/status/identity +transcripts, every stage run and download transcript, raw JSON/time/log files, +runner catalog, setup identity artifact, cleanup logs, and the final source +manifest in the separate `.cmux-scratch/` directory. Never store credentials, +private keys, or `/tmp/.testbox/auth_token`. + +Record these fields alongside `timings.json`: + +1. Exact source branch, full source SHA/tree SHA, Ghostty gitlink SHA, and the + clean-status result before each stage. +2. Requested runner label and catalog output. +3. Blacksmith CLI version, Testbox ID, setup workflow run/job IDs, identity run + ID, and each stage run/sync ID from raw transcripts. +4. Whether the comparison was target-clean, registry/git-cache warm, + Zig-cache warm, or a genuinely cold VM. Warmup deliberately hydrates + dependencies, so `first-clean` is target-cold and dependency-warm. +5. Cleanup stop/status/list output and whether the specific ID was absent from + the active inventory. + +The historical 32-vCPU evidence at +`.cmux-scratch/blacksmith-testbox-e40704611ac35f4ffa153/` remains unchanged: +setup SHA `e40704611ac35f0e3a806841a9eae383f4ffa153`, Testbox +`tbx_01kzxebn91nhatkv4ygevh06vs`, workflow run `31696013711`, first-clean +`161.47s`, incremental no-op `8.28s`, changed-file `9.13s`, and cleanup with no +active box. Do not rewrite it while validating this hardening change. + +Prior hosted cmux-tui correctness runs without Cargo durations are provenance, +not performance comparisons. Prior Blacksmith macOS Swift/Xcode artifacts use +a different OS, architecture, runner SKU, cache state, and workload, so they +are context rather than a Rust baseline. From 33b0dfe97d8b8a8f19e7810aeb6be71655084249 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 05:37:24 -0700 Subject: [PATCH 05/38] docs: tighten Testbox source preflight --- skills/blacksmith-testbox/SKILL.md | 8 ++++++-- skills/blacksmith-testbox/benchmark.md | 25 ++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/skills/blacksmith-testbox/SKILL.md b/skills/blacksmith-testbox/SKILL.md index 99b9c90db67..610bf69962c 100644 --- a/skills/blacksmith-testbox/SKILL.md +++ b/skills/blacksmith-testbox/SKILL.md @@ -82,10 +82,12 @@ found`). Do not pass a commit SHA to `blacksmith testbox warmup --ref`, and do not document raw SHA refs as supported. Warm up a pushed branch ref, then carry the full SHA separately and assert it remotely. -Run this preflight from the clean worktree root: +Initialize the public Ghostty submodule once, then run this preflight from the +clean worktree root: ```bash set -euo pipefail +git submodule update --init ghostty cd "$(git rev-parse --show-toplevel)" SOURCE_REF="$(git symbolic-ref --short HEAD)" [[ -n "$SOURCE_REF" ]] || { echo "HEAD is detached" >&2; exit 1; } @@ -93,7 +95,9 @@ SOURCE_SHA="$(git rev-parse HEAD)" GHOSTTY_SHA="$(git rev-parse HEAD:ghostty)" [[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]] [[ "$GHOSTTY_SHA" =~ ^[0-9a-f]{40}$ ]] +[[ "$(git -C ghostty rev-parse HEAD)" == "$GHOSTTY_SHA" ]] [[ -z "$(git status --porcelain=v1 --untracked-files=normal)" ]] +[[ -z "$(git -C ghostty status --porcelain=v1 --untracked-files=normal)" ]] remote_sha="$(git ls-remote --exit-code origin "refs/heads/$SOURCE_REF" | awk 'NR == 1 { print $1 }')" [[ "$remote_sha" == "$SOURCE_SHA" ]] || { echo "the pushed branch is not the exact local HEAD" >&2 @@ -158,7 +162,7 @@ run_stage() { local run_status download_status=0 set +e blacksmith testbox run --id "$TBX" --debug \ - "CMUX_TESTBOX_REMOTE=1 CMUX_TESTBOX_ID=$TBX CMUX_TESTBOX_SOURCE_REF=$SOURCE_REF ./scripts/blacksmith-cmux-tui-testbox-stage.sh $stage $SOURCE_SHA $GHOSTTY_SHA" \ + "CMUX_TESTBOX_REMOTE=1 CMUX_TESTBOX_ID=$TBX ./scripts/blacksmith-cmux-tui-testbox-stage.sh $stage $SOURCE_SHA $GHOSTTY_SHA" \ 2>&1 | tee "$OUT/$stage.run.log" run_status=${PIPESTATUS[0]} set -e diff --git a/skills/blacksmith-testbox/benchmark.md b/skills/blacksmith-testbox/benchmark.md index 28739a4d1d4..586dfa496eb 100644 --- a/skills/blacksmith-testbox/benchmark.md +++ b/skills/blacksmith-testbox/benchmark.md @@ -42,6 +42,7 @@ pushed branch ref and carry the full source SHA as an assertion: ```bash set -euo pipefail +git submodule update --init ghostty cd "$(git rev-parse --show-toplevel)" SOURCE_REF="$(git symbolic-ref --short HEAD)" SOURCE_SHA="$(git rev-parse HEAD)" @@ -49,7 +50,9 @@ GHOSTTY_SHA="$(git rev-parse HEAD:ghostty)" [[ -n "$SOURCE_REF" ]] [[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]] [[ "$GHOSTTY_SHA" =~ ^[0-9a-f]{40}$ ]] +[[ "$(git -C ghostty rev-parse HEAD)" == "$GHOSTTY_SHA" ]] [[ -z "$(git status --porcelain=v1 --untracked-files=normal)" ]] +[[ -z "$(git -C ghostty status --porcelain=v1 --untracked-files=normal)" ]] remote_sha="$(git ls-remote --exit-code origin "refs/heads/$SOURCE_REF" | awk 'NR == 1 { print $1 }')" [[ "$remote_sha" == "$SOURCE_SHA" ]] || { echo "push the exact clean branch head before warming Testbox" >&2 @@ -87,6 +90,26 @@ JOB=cmux-tui-rust OUT="$PWD/.cmux-scratch/blacksmith-testbox-$SOURCE_SHA" TBX="" mkdir -p "$OUT/raw" +cleanup() { + local result=$? cleanup_status + trap - EXIT + if [[ -n "$TBX" ]]; then + set +e + scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" + cleanup_status=$? + set -e + else + set +e + blacksmith testbox list --all >"$OUT/list-after-stop.log" 2>&1 + cleanup_status=$? + set -e + fi + if (( result == 0 && cleanup_status != 0 )); then + result="$cleanup_status" + fi + exit "$result" +} +trap cleanup EXIT blacksmith auth whoami blacksmith --version | tee "$OUT/blacksmith-version.txt" blacksmith runners catalog > "$OUT/runner-catalog.json" @@ -142,7 +165,7 @@ run_stage() { local run_status download_status=0 set +e blacksmith testbox run --id "$TBX" --debug \ - "CMUX_TESTBOX_REMOTE=1 CMUX_TESTBOX_ID=$TBX CMUX_TESTBOX_SOURCE_REF=$SOURCE_REF ./scripts/blacksmith-cmux-tui-testbox-stage.sh $stage $SOURCE_SHA $GHOSTTY_SHA" \ + "CMUX_TESTBOX_REMOTE=1 CMUX_TESTBOX_ID=$TBX ./scripts/blacksmith-cmux-tui-testbox-stage.sh $stage $SOURCE_SHA $GHOSTTY_SHA" \ 2>&1 | tee "$OUT/$stage.run.log" run_status=${PIPESTATUS[0]} set -e From 22800c17377d6768825d329c32f46547f519fc52 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 05:58:56 -0700 Subject: [PATCH 06/38] ci: serialize and recover trusted Testbox runs --- .../ci-workflow-guard-tests-testbox.yml | 21 +- scripts/blacksmith-cmux-tui-testbox-stage.sh | 12 +- scripts/blacksmith-testbox-recover-warmup.sh | 70 +++++++ skills/blacksmith-testbox/SKILL.md | 63 ++++-- skills/blacksmith-testbox/benchmark.md | 179 ++++++++++++++---- 5 files changed, 279 insertions(+), 66 deletions(-) create mode 100755 scripts/blacksmith-testbox-recover-warmup.sh diff --git a/.github/workflows/ci-workflow-guard-tests-testbox.yml b/.github/workflows/ci-workflow-guard-tests-testbox.yml index 029014bd77f..b29b82ec72f 100644 --- a/.github/workflows/ci-workflow-guard-tests-testbox.yml +++ b/.github/workflows/ci-workflow-guard-tests-testbox.yml @@ -20,9 +20,10 @@ on: permissions: {} concurrency: - # A Testbox is a mutable shared workspace. Queue every request that names the - # same box/source instead of allowing two syncs or stage commands to race. - group: cmux-tui-testbox-${{ inputs.testbox_id }}-${{ inputs.source_sha || github.sha }} + # A Testbox is a mutable shared workspace. Serialize every request for the + # same ID, even when callers name different source SHAs. Source identity is + # validated and recorded separately, so it must never widen this lock. + group: cmux-tui-testbox-${{ inputs.testbox_id }} cancel-in-progress: false jobs: @@ -111,11 +112,12 @@ jobs: exit 1 } source_tree_sha="$(git rev-parse 'HEAD^{tree}')" - ghostty_gitlink_sha="$(git rev-parse 'HEAD:ghostty')" - [[ "$ghostty_gitlink_sha" =~ ^[0-9a-f]{40}$ ]] || { - echo "::error::HEAD:ghostty is not a gitlink SHA" >&2 + ghostty_entry="$(git ls-tree HEAD ghostty)" + [[ "$ghostty_entry" =~ ^160000[[:space:]]commit[[:space:]][0-9a-f]{40}[[:space:]]ghostty$ ]] || { + echo "::error::HEAD:ghostty is not a gitlink" >&2 exit 1 } + ghostty_gitlink_sha="$(git rev-parse 'HEAD:ghostty')" printf 'source_sha=%s\nsource_tree_sha=%s\nghostty_gitlink_sha=%s\n' \ "$actual_sha" "$source_tree_sha" "$ghostty_gitlink_sha" @@ -128,6 +130,11 @@ jobs: echo "::error::ghostty did not initialize as its own submodule checkout" >&2 exit 1 } + ghostty_entry="$(git ls-tree HEAD ghostty)" + [[ "$ghostty_entry" =~ ^160000[[:space:]]commit[[:space:]][0-9a-f]{40}[[:space:]]ghostty$ ]] || { + echo "::error::HEAD:ghostty is not a gitlink" >&2 + exit 1 + } expected_ghostty_sha="$(git rev-parse HEAD:ghostty)" actual_ghostty_sha="$(git -C ghostty rev-parse HEAD)" [[ "$actual_ghostty_sha" == "$expected_ghostty_sha" ]] || { @@ -198,6 +205,8 @@ jobs: run: | set -euo pipefail source_tree_sha="$(git rev-parse 'HEAD^{tree}')" + ghostty_entry="$(git ls-tree HEAD ghostty)" + [[ "$ghostty_entry" =~ ^160000[[:space:]]commit[[:space:]][0-9a-f]{40}[[:space:]]ghostty$ ]] || exit 1 ghostty_gitlink_sha="$(git rev-parse 'HEAD:ghostty')" ghostty_head_sha="$(git -C ghostty rev-parse HEAD)" [[ "$SOURCE_SHA" == "$(git rev-parse HEAD)" ]] || exit 1 diff --git a/scripts/blacksmith-cmux-tui-testbox-stage.sh b/scripts/blacksmith-cmux-tui-testbox-stage.sh index df3451d941d..4a2958dfc71 100755 --- a/scripts/blacksmith-cmux-tui-testbox-stage.sh +++ b/scripts/blacksmith-cmux-tui-testbox-stage.sh @@ -67,6 +67,11 @@ if [[ "$(git -C ghostty rev-parse --show-toplevel 2>/dev/null || true)" != "$gho echo "ghostty is not an initialized submodule checkout" >&2 exit 65 fi +ghostty_entry="$(git ls-tree HEAD ghostty)" +if [[ ! "$ghostty_entry" =~ ^160000[[:space:]]commit[[:space:]][0-9a-f]{40}[[:space:]]ghostty$ ]]; then + echo "HEAD:ghostty is not a gitlink" >&2 + exit 65 +fi expected_tree_sha="$(git rev-parse "${expected_source_sha}^{tree}")" benchmark_dir="$repo_root/testbox-benchmark" @@ -290,11 +295,12 @@ if (( post_identity_status == 0 )); then fi fi -final_status="$build_status" -if (( final_status == 0 && restore_status != 0 )); then +if (( restore_status != 0 )); then final_status="$restore_status" -elif (( final_status == 0 && post_identity_status != 0 )); then +elif (( post_identity_status != 0 )); then final_status="$post_identity_status" +else + final_status="$build_status" fi python3 - "$stage" "$start_epoch" "$end_epoch" "$build_status" "$final_status" "$time_path" "$pre_identity_path" "$post_identity_path" "$changed_file" "$expected_source_sha" "$expected_tree_sha" "$expected_ghostty_sha" "$testbox_id" "$runner_label" "$rust_toolchain" "$rustc_version" "$cargo_version" "$zig_bin" "$zig_version" "$rust_toolchain_file_sha256" "$cargo_lock_sha256" "$ghostty_zon_sha256" >"$json_path" <<'PY' diff --git a/scripts/blacksmith-testbox-recover-warmup.sh b/scripts/blacksmith-testbox-recover-warmup.sh new file mode 100755 index 00000000000..abc9bbce415 --- /dev/null +++ b/scripts/blacksmith-testbox-recover-warmup.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 6 ]]; then + echo "usage: $0 " >&2 + exit 64 +fi + +before_list="$1" +after_list="$2" +workflow="$3" +job="$4" +branch_ref="$5" +evidence_dir="$6" +for path in "$before_list" "$after_list"; do + if [[ ! -f "$path" ]]; then + echo "inventory file is missing: $path" >&2 + exit 65 + fi +done +mkdir -p "$evidence_dir" +decision_log="$evidence_dir/warmup-recovery.log" + +set +e +candidate="$(python3 - "$before_list" "$after_list" "$workflow" "$job" "$branch_ref" <<'PY' +import re +import sys + +before_path, after_path, workflow, job, branch_ref = sys.argv[1:] +id_pattern = re.compile(r"tbx_[A-Za-z0-9_-]+$") + +def matching(path): + result = set() + with open(path, encoding="utf-8") as handle: + for line in handle: + fields = line.split() + # list --all is a whitespace table: ID STATUS IP WORKFLOW JOB REF ... + if len(fields) >= 6 and id_pattern.fullmatch(fields[0]): + if fields[3] == workflow and fields[4] == job and fields[5] == branch_ref: + result.add(fields[0]) + return result + +before = matching(before_path) +after = matching(after_path) +new = sorted(after - before) +if len(new) != 1: + print( + f"warmup recovery found {len(new)} uniquely matching new Testbox IDs " + f"(before={sorted(before)}, after={sorted(after)})", + file=sys.stderr, + ) + raise SystemExit(1) +print(new[0]) +PY +)" +parse_status=$? +set -e +if (( parse_status != 0 )); then + { + echo "No Testbox was stopped: warmup recovery was ambiguous or had no matching new ID." + echo "before=$before_list" + echo "after=$after_list" + } >"$decision_log" + cat "$decision_log" >&2 + exit "$parse_status" +fi + +printf 'warmup recovery selected uniquely new Testbox %s\n' "$candidate" | tee "$decision_log" +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +"$script_dir/blacksmith-testbox-cleanup.sh" "$candidate" "$evidence_dir" diff --git a/skills/blacksmith-testbox/SKILL.md b/skills/blacksmith-testbox/SKILL.md index 610bf69962c..bb110fd7112 100644 --- a/skills/blacksmith-testbox/SKILL.md +++ b/skills/blacksmith-testbox/SKILL.md @@ -74,6 +74,9 @@ Use the repository or organization-approved, pinned Blacksmith CLI artifact or package-manager version. Record its version in the evidence. Do not install a mutable remote script with `curl ... | sh`; if the approved pinned artifact is not available, stop and ask the tooling owner rather than weakening this lane. +The repository does not currently pin a checksum-verified CLI artifact, so CLI +provenance is a trusted-lane operational limitation: do not silently upgrade or +substitute a version, and retain `blacksmith --version` with each evidence set. ## Exact source contract @@ -90,8 +93,16 @@ set -euo pipefail git submodule update --init ghostty cd "$(git rev-parse --show-toplevel)" SOURCE_REF="$(git symbolic-ref --short HEAD)" -[[ -n "$SOURCE_REF" ]] || { echo "HEAD is detached" >&2; exit 1; } +if [[ ! "$SOURCE_REF" =~ ^[A-Za-z0-9._/-]+$ || "$SOURCE_REF" == *..* || "$SOURCE_REF" == */ || "$SOURCE_REF" == *//* ]]; then + echo "HEAD must name a supported pushed branch ref" >&2 + exit 1 +fi SOURCE_SHA="$(git rev-parse HEAD)" +ghostty_entry="$(git ls-tree HEAD ghostty)" +[[ "$ghostty_entry" =~ ^160000[[:space:]]commit[[:space:]][0-9a-f]{40}[[:space:]]ghostty$ ]] || { + echo "HEAD:ghostty is not a gitlink" >&2 + exit 1 +} GHOSTTY_SHA="$(git rev-parse HEAD:ghostty)" [[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]] [[ "$GHOSTTY_SHA" =~ ^[0-9a-f]{40}$ ]] @@ -132,22 +143,31 @@ blacksmith testbox warmup "$WORKFLOW" \ ``` Save the returned `tbx_...` ID. One ID belongs to one worktree and one trust -context. The workflow concurrency group keys the Testbox ID and source, and -each remote stage holds `testbox-benchmark/.stage.lock` through its build and -artifact writes. Do not issue concurrent `run` commands against one ID. +context. The workflow concurrency group keys every setup request by Testbox +ID, including requests for different source SHAs. Source identity is validated +separately, and each remote stage holds `testbox-benchmark/.stage.lock` through +its build and artifact writes. Blacksmith's CLI exposes no pre-rsync lease, so +that remote lock cannot serialize the CLI's initial sync. One Testbox ID must +therefore have one owning worktree/operator; do not issue concurrent `run` or +download commands from independent clients. This is an explicit trusted-lane +limitation, not a claim of multi-client Testbox isolation. Wait for setup and capture the exact run identity: ```bash blacksmith testbox status --id "$TBX" --wait --wait-timeout 15m -blacksmith testbox run --id "$TBX" --debug \ - "set -euo pipefail; test -s /tmp/.testbox/auth_token; grep -Eq '(^|[[:space:]])metadata_port=[^[:space:]]+' /proc/cmdline; test \"\$(git rev-parse HEAD)\" = \"$SOURCE_SHA\"; test \"\$(git rev-parse HEAD:ghostty)\" = \"$GHOSTTY_SHA\"; test \"\$(git -C ghostty rev-parse HEAD)\" = \"$GHOSTTY_SHA\"; test -z \"\$(git status --porcelain=v1 --untracked-files=normal)\"; test -z \"\$(git -C ghostty status --porcelain=v1 --untracked-files=normal)\"; rustup show active-toolchain; rustc --version; cargo --version; \"$CMUX_ZIG\" version" ``` -The setup workflow also uploads `setup-identity.json`. It contains the -workflow run, source ref/SHA/tree, Ghostty gitlink and checkout SHA, runner -label/architecture/CPU identity, and pinned Rust/Zig/toolchain-file metadata. -Never print or download `/tmp/.testbox/auth_token`. +The setup job's `setup-identity.json` artifact and each stage helper's +pre-build JSON record are the identity transcripts. The helper verifies the +Testbox VM marker, claimed Testbox ID, source commit/tree, Ghostty +gitlink/checkout, and clean status before it invokes Cargo, then repeats those +checks after the build. Keep the setup artifact URL or download it into `$OUT`; +it records runner/toolchain/Ghostty identity independently of the stage helper. +The setup JSON contains the workflow run, source ref/SHA/tree, Ghostty gitlink +and checkout SHA, runner label/architecture/CPU identity, and pinned +Rust/Zig/toolchain-file metadata. Never print or download +`/tmp/.testbox/auth_token`. ## Remote benchmark stages @@ -157,25 +177,32 @@ the helper does not trust the remote checkout or a caller-supplied expected SHA without comparing it to Git metadata: ```bash +# Run this orchestration block in Bash, not an interactive zsh session. run_stage() { local stage="$1" local run_status download_status=0 set +e + printf -v remote_command \ + 'CMUX_TESTBOX_REMOTE=1 CMUX_TESTBOX_ID=%q %q %q %q %q' \ + "$TBX" ./scripts/blacksmith-cmux-tui-testbox-stage.sh \ + "$stage" "$SOURCE_SHA" "$GHOSTTY_SHA" blacksmith testbox run --id "$TBX" --debug \ - "CMUX_TESTBOX_REMOTE=1 CMUX_TESTBOX_ID=$TBX ./scripts/blacksmith-cmux-tui-testbox-stage.sh $stage $SOURCE_SHA $GHOSTTY_SHA" \ - 2>&1 | tee "$OUT/$stage.run.log" - run_status=${PIPESTATUS[0]} + "$remote_command" >"$OUT/$stage.run.log" 2>&1 + run_status=$? set -e + cat "$OUT/$stage.run.log" # Download immediately. Blacksmith's next rsync may delete or replace remote # files, so a one-time download after all stages is insufficient. + : >"$OUT/$stage.download.log" for suffix in json time log; do if ! blacksmith testbox download --id "$TBX" \ "testbox-benchmark/$stage.$suffix" "$OUT/raw/$stage.$suffix" \ - 2>&1 | tee -a "$OUT/$stage.download.log"; then + >>"$OUT/$stage.download.log" 2>&1; then download_status=1 fi done + cat "$OUT/$stage.download.log" if (( run_status != 0 )); then return "$run_status" fi @@ -266,8 +293,12 @@ specific Testbox ID is terminal or absent from the active inventory; and accepts only the known race where stop returns a 409 saying the box is already stopped or completed. Other stop, status, or list failures remain failures. Put it in an `EXIT` trap that preserves the benchmark's original exit status -unless cleanup itself fails. If warmup never returned an ID, still run -`blacksmith testbox list --all` and retain its exit status and output. +unless cleanup itself fails. The detailed benchmark captures a pre-warmup +inventory and uses `scripts/blacksmith-testbox-recover-warmup.sh` when warmup +fails before printing an ID. That recovery path stops a box only when the +before/after inventory has exactly one new match for the workflow, job, and +branch; ambiguous or missing matches are reported without stopping an +unrelated box. ## Timing interpretation diff --git a/skills/blacksmith-testbox/benchmark.md b/skills/blacksmith-testbox/benchmark.md index 586dfa496eb..ce1cba502c2 100644 --- a/skills/blacksmith-testbox/benchmark.md +++ b/skills/blacksmith-testbox/benchmark.md @@ -21,7 +21,11 @@ setup-only GitHub job on the remote Linux runner. A repository administrator must configure the protected environment with required reviewers and no secrets before this plan is usable. The lane must never run untrusted PR or fork code. `begin-testbox` exposes its auth token to -commands in the Testbox, so `contents: read` is not a trust boundary. +commands in the Testbox, so `contents: read` is not a trust boundary. The +repository does not currently pin a checksum-verified Blacksmith CLI artifact; +that is a trusted-lane operational limitation. Use only the organization- +approved CLI, record `blacksmith --version`, and stop rather than silently +substituting a version. The warmup job only checks out the exact dispatch SHA, initializes `ghostty`, installs Linux headers/tools, installs the pinned Zig and Rust toolchains, @@ -45,7 +49,17 @@ set -euo pipefail git submodule update --init ghostty cd "$(git rev-parse --show-toplevel)" SOURCE_REF="$(git symbolic-ref --short HEAD)" +if [[ ! "$SOURCE_REF" =~ ^[A-Za-z0-9._/-]+$ || "$SOURCE_REF" == *..* || "$SOURCE_REF" == */ || "$SOURCE_REF" == *//* ]]; then + echo "HEAD must name a supported pushed branch ref" >&2 + exit 1 +fi SOURCE_SHA="$(git rev-parse HEAD)" +SOURCE_TREE_SHA="$(git rev-parse 'HEAD^{tree}')" +ghostty_entry="$(git ls-tree HEAD ghostty)" +[[ "$ghostty_entry" =~ ^160000[[:space:]]commit[[:space:]][0-9a-f]{40}[[:space:]]ghostty$ ]] || { + echo "HEAD:ghostty is not a gitlink" >&2 + exit 1 +} GHOSTTY_SHA="$(git rev-parse HEAD:ghostty)" [[ -n "$SOURCE_REF" ]] [[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]] @@ -59,26 +73,43 @@ remote_sha="$(git ls-remote --exit-code origin "refs/heads/$SOURCE_REF" | awk 'N exit 1 } mkdir -p ".cmux-scratch/blacksmith-testbox-$SOURCE_SHA/raw" -python3 - "$SOURCE_REF" "$SOURCE_SHA" "$GHOSTTY_SHA" > ".cmux-scratch/blacksmith-testbox-$SOURCE_SHA/source.json" <<'PY' +python3 - "$SOURCE_REF" "$SOURCE_SHA" "$SOURCE_TREE_SHA" "$GHOSTTY_SHA" > ".cmux-scratch/blacksmith-testbox-$SOURCE_SHA/source.json" <<'PY' import json -import subprocess import sys -ref, sha, ghostty = sys.argv[1:] -record = { +ref, sha, tree, ghostty = sys.argv[1:] +print(json.dumps({ "source_ref": ref, "source_sha": sha, - "source_tree_sha": subprocess.check_output(["git", "rev-parse", "HEAD^{tree}"], text=True).strip(), + "source_tree_sha": tree, "ghostty_gitlink_sha": ghostty, -} -print(json.dumps(record, indent=2, sort_keys=True)) +}, indent=2, sort_keys=True)) PY + +assert_source_unchanged() { + local current_ref current_sha current_tree current_ghostty remote_sha ghostty_entry + current_ref="$(git symbolic-ref --short HEAD)" + current_sha="$(git rev-parse HEAD)" + current_tree="$(git rev-parse 'HEAD^{tree}')" + ghostty_entry="$(git ls-tree HEAD ghostty)" + current_ghostty="$(git rev-parse HEAD:ghostty)" + remote_sha="$(git ls-remote --exit-code origin "refs/heads/$SOURCE_REF" | awk 'NR == 1 { print $1 }')" + if [[ "$current_ref" != "$SOURCE_REF" || "$current_sha" != "$SOURCE_SHA" || + "$current_tree" != "$SOURCE_TREE_SHA" || "$current_ghostty" != "$GHOSTTY_SHA" || + ! "$ghostty_entry" =~ ^160000[[:space:]]commit[[:space:]][0-9a-f]{40}[[:space:]]ghostty$ || + "$remote_sha" != "$SOURCE_SHA" || + -n "$(git status --porcelain=v1 --untracked-files=normal)" || + -n "$(git -C ghostty status --porcelain=v1 --untracked-files=normal)" ]]; then + echo "source branch, tree, Ghostty gitlink, remote head, or clean status changed" >&2 + return 1 + fi +} +assert_source_unchanged ``` -Before every stage, rerun the clean-branch check and recompute the expected -values. If the branch moved, the worktree became dirty, or the Ghostty pointer -changed, stop the box and start a new evidence directory. Do not silently -substitute the new SHA. +`assert_source_unchanged` runs before every stage below. If the branch moved, +the worktree became dirty, or the Ghostty pointer changed, stop the box and +start a new evidence directory. Do not silently substitute the new SHA. ## Warmup and setup identity @@ -89,9 +120,12 @@ WORKFLOW=.github/workflows/ci-workflow-guard-tests-testbox.yml JOB=cmux-tui-rust OUT="$PWD/.cmux-scratch/blacksmith-testbox-$SOURCE_SHA" TBX="" +before_list_status=125 mkdir -p "$OUT/raw" cleanup() { - local result=$? cleanup_status + local result=$? + local cleanup_status=0 + local after_list_status=125 trap - EXIT if [[ -n "$TBX" ]]; then set +e @@ -99,10 +133,23 @@ cleanup() { cleanup_status=$? set -e else + # If warmup failed before printing its ID, stop only a uniquely correlated + # new box. Never stop an arbitrary active box from a shared organization. set +e - blacksmith testbox list --all >"$OUT/list-after-stop.log" 2>&1 - cleanup_status=$? + blacksmith testbox list --all >"$OUT/list-after-warmup-failure.log" 2>&1 + after_list_status=$? set -e + if (( before_list_status == 0 && after_list_status == 0 )); then + set +e + scripts/blacksmith-testbox-recover-warmup.sh \ + "$OUT/list-before-warmup.log" "$OUT/list-after-warmup-failure.log" \ + "$WORKFLOW" "$JOB" "$SOURCE_REF" "$OUT" + cleanup_status=$? + set -e + else + cleanup_status=1 + echo "could not reconcile a failed warmup because inventory capture failed" >&2 + fi fi if (( result == 0 && cleanup_status != 0 )); then result="$cleanup_status" @@ -111,13 +158,28 @@ cleanup() { } trap cleanup EXIT blacksmith auth whoami -blacksmith --version | tee "$OUT/blacksmith-version.txt" -blacksmith runners catalog > "$OUT/runner-catalog.json" +blacksmith --version >"$OUT/blacksmith-version.txt" +cat "$OUT/blacksmith-version.txt" +blacksmith runners catalog >"$OUT/runner-catalog.json" +set +e +blacksmith testbox list --all >"$OUT/list-before-warmup.log" 2>&1 +before_list_status=$? +set -e +cat "$OUT/list-before-warmup.log" +if (( before_list_status != 0 )); then + echo "refusing to warm a Testbox without a baseline inventory" >&2 + exit "$before_list_status" +fi +set +e blacksmith testbox warmup "$WORKFLOW" \ --ref "$SOURCE_REF" \ --job "$JOB" \ --idle-timeout 30 \ - 2>&1 | tee "$OUT/warmup.log" + >"$OUT/warmup.log" 2>&1 +warmup_status=$? +set -e +cat "$OUT/warmup.log" +set +e TBX="$(python3 - "$OUT/warmup.log" <<'PY' import re import sys @@ -129,9 +191,24 @@ if not ids: print(ids[-1]) PY )" +parse_status=$? +set -e +if (( warmup_status != 0 )); then + exit "$warmup_status" +fi +if (( parse_status != 0 )); then + exit "$parse_status" +fi printf 'Testbox ID: %s\n' "$TBX" | tee "$OUT/testbox-id.txt" +set +e blacksmith testbox status --id "$TBX" --wait --wait-timeout 15m \ - 2>&1 | tee "$OUT/status-ready.log" + >"$OUT/status-ready.log" 2>&1 +status_ready=$? +set -e +cat "$OUT/status-ready.log" +if (( status_ready != 0 )); then + exit "$status_ready" +fi ``` The workflow validates that the dispatch ref is a pushed branch, that @@ -139,18 +216,20 @@ The workflow validates that the dispatch ref is a pushed branch, that a direct GitHub dispatch supplies the optional `source_sha` input, it must equal `github.sha`. The Blacksmith CLI path relies on the same full SHA passed to the remote helper because the CLI only supplies `testbox_id` to workflow inputs. - -Capture a harmless remote identity transcript before builds: - -```bash -blacksmith testbox run --id "$TBX" --debug \ - "set -euo pipefail; test -s /tmp/.testbox/auth_token; grep -Eq '(^|[[:space:]])metadata_port=[^[:space:]]+' /proc/cmdline; test \"\$(git rev-parse HEAD)\" = \"$SOURCE_SHA\"; test \"\$(git rev-parse HEAD:ghostty)\" = \"$GHOSTTY_SHA\"; test \"\$(git -C ghostty rev-parse HEAD)\" = \"$GHOSTTY_SHA\"; test -z \"\$(git status --porcelain=v1 --untracked-files=normal)\"; test -z \"\$(git -C ghostty status --porcelain=v1 --untracked-files=normal)\"; rustup show active-toolchain; rustc --version; cargo --version; \"$CMUX_ZIG\" version" \ - 2>&1 | tee "$OUT/identity.run.log" -``` - -The setup job's `setup-identity.json` is uploaded as a GitHub artifact. Keep -its artifact URL or download it into `$OUT`; it records runner/toolchain/ -Ghostty identity independently of the stage helper. +The workflow concurrency group serializes setup requests by Testbox ID, even +when source SHAs differ. The remote `flock` begins after Blacksmith's rsync, so +it protects stage/build/artifact writes only. Blacksmith exposes no pre-rsync +lease; one Testbox ID must have one owning worktree/operator, and independent +clients must not issue concurrent `run` or download commands. This is an +explicit trusted-lane limitation. + +Do not issue a separate interpolated identity command. The setup job's +`setup-identity.json` artifact and each stage helper's pre-build JSON record are +the identity transcripts. The helper verifies the Testbox VM marker, claimed +Testbox ID, source commit/tree, Ghostty gitlink/checkout, and clean status before +it invokes Cargo, then repeats those checks after the build. Keep the setup +artifact URL or download it into `$OUT`; it records runner/toolchain/Ghostty +identity independently of the stage helper. ## Three remote build timings @@ -160,25 +239,32 @@ before the stage, holds a remote `flock` through all writes, restores the controlled changed file, and verifies clean identity again. ```bash +# Run this orchestration block in Bash, not an interactive zsh session. run_stage() { local stage="$1" local run_status download_status=0 set +e + printf -v remote_command \ + 'CMUX_TESTBOX_REMOTE=1 CMUX_TESTBOX_ID=%q %q %q %q %q' \ + "$TBX" ./scripts/blacksmith-cmux-tui-testbox-stage.sh \ + "$stage" "$SOURCE_SHA" "$GHOSTTY_SHA" blacksmith testbox run --id "$TBX" --debug \ - "CMUX_TESTBOX_REMOTE=1 CMUX_TESTBOX_ID=$TBX ./scripts/blacksmith-cmux-tui-testbox-stage.sh $stage $SOURCE_SHA $GHOSTTY_SHA" \ - 2>&1 | tee "$OUT/$stage.run.log" - run_status=${PIPESTATUS[0]} + "$remote_command" >"$OUT/$stage.run.log" 2>&1 + run_status=$? set -e + cat "$OUT/$stage.run.log" # rsync --delete can remove remote output before the next run. Download each # stage immediately, before starting another stage. + : >"$OUT/$stage.download.log" for suffix in json time log; do if ! blacksmith testbox download --id "$TBX" \ "testbox-benchmark/$stage.$suffix" "$OUT/raw/$stage.$suffix" \ - 2>&1 | tee -a "$OUT/$stage.download.log"; then + >>"$OUT/$stage.download.log" 2>&1; then download_status=1 fi done + cat "$OUT/$stage.download.log" if (( run_status != 0 )); then return "$run_status" fi @@ -187,11 +273,18 @@ run_stage() { benchmark_status=0 for stage in first-clean incremental-noop changed-file; do + if ! assert_source_unchanged; then + benchmark_status=1 + break + fi if ! run_stage "$stage"; then benchmark_status=1 break fi done +if (( benchmark_status != 0 )); then + exit "$benchmark_status" +fi ``` `first-clean` is target-clean but dependency-warm. `incremental-noop` repeats @@ -250,12 +343,16 @@ scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" ``` A shell `EXIT` trap should call that helper while preserving the benchmark -status. If warmup did not return a Testbox ID, still run -`blacksmith testbox list --all` and save its output. Keep warmup/status/identity -transcripts, every stage run and download transcript, raw JSON/time/log files, -runner catalog, setup identity artifact, cleanup logs, and the final source -manifest in the separate `.cmux-scratch/` directory. Never store credentials, -private keys, or `/tmp/.testbox/auth_token`. +status. The benchmark captures a pre-warmup inventory. If warmup fails before +printing an ID, `scripts/blacksmith-testbox-recover-warmup.sh` compares the +before/after inventories and invokes cleanup only for exactly one new box +matching this workflow, job, and branch. Ambiguous or missing matches are +reported without stopping an unrelated box. Keep both inventories and their +command statuses. Keep warmup/status/identity transcripts, every stage run and +download transcript, raw JSON/time/log files, runner catalog, setup identity +artifact, cleanup logs, and the final source manifest in the separate +`.cmux-scratch/` directory. Never store credentials, private keys, or +`/tmp/.testbox/auth_token`. Record these fields alongside `timings.json`: From 42ab3daf29db34f6991e53426120ecd2fb1ae6ff Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 06:26:16 -0700 Subject: [PATCH 07/38] ci: bind Testbox cleanup to setup receipts --- .../ci-workflow-guard-tests-testbox.yml | 14 ++- scripts/blacksmith-cmux-tui-testbox-stage.sh | 33 +++++++ scripts/blacksmith-testbox-cleanup.sh | 58 ++++++++++-- scripts/blacksmith-testbox-recover-warmup.sh | 70 -------------- skills/blacksmith-testbox/SKILL.md | 28 ++++-- skills/blacksmith-testbox/benchmark.md | 94 ++++++++++++++----- 6 files changed, 183 insertions(+), 114 deletions(-) delete mode 100755 scripts/blacksmith-testbox-recover-warmup.sh diff --git a/.github/workflows/ci-workflow-guard-tests-testbox.yml b/.github/workflows/ci-workflow-guard-tests-testbox.yml index b29b82ec72f..7e771571d77 100644 --- a/.github/workflows/ci-workflow-guard-tests-testbox.yml +++ b/.github/workflows/ci-workflow-guard-tests-testbox.yml @@ -300,7 +300,19 @@ jobs: if-no-files-found: warn retention-days: 14 - # Always hand control back to Testbox. Warmup is setup-only: all actual + - name: Mark Testbox setup ready + if: success() + shell: bash + run: | + set -euo pipefail + marker=/tmp/.testbox/cmux-tui-rust-setup-identity.json + test -d /tmp/.testbox + install -m 600 testbox-benchmark/setup-identity.json "$marker" + test -s "$marker" + + # Always hand control back to Testbox. The pinned action receives the + # final job status and reports hydration_failed, rather than ready, when + # any setup step failed. The setup marker is a second guard for CLI runs. # Rust builds are issued later with `blacksmith testbox run`. - name: Run Testbox uses: useblacksmith/run-testbox@5ca05834db1d3813554d1dd109e5f2087a8d7cbc # v2 diff --git a/scripts/blacksmith-cmux-tui-testbox-stage.sh b/scripts/blacksmith-cmux-tui-testbox-stage.sh index 4a2958dfc71..d603818cc77 100755 --- a/scripts/blacksmith-cmux-tui-testbox-stage.sh +++ b/scripts/blacksmith-cmux-tui-testbox-stage.sh @@ -73,6 +73,39 @@ if [[ ! "$ghostty_entry" =~ ^160000[[:space:]]commit[[:space:]][0-9a-f]{40}[[:sp exit 65 fi expected_tree_sha="$(git rev-parse "${expected_source_sha}^{tree}")" +setup_identity_path="$state_dir/cmux-tui-rust-setup-identity.json" +if [[ ! -s "$setup_identity_path" ]]; then + echo "refusing to run without a successful Testbox setup identity marker" >&2 + exit 65 +fi +verify_setup_identity() { + python3 - "$setup_identity_path" "$expected_source_sha" "$expected_tree_sha" "$expected_ghostty_sha" "$testbox_id" <<'PY' +import json +import pathlib +import sys + +path, expected_source, expected_tree, expected_ghostty, expected_testbox = sys.argv[1:] +record = json.loads(pathlib.Path(path).read_text(encoding="utf-8")) +source = record.get("source", {}) +testbox = record.get("testbox", {}) +errors = [] +if source.get("commit_sha") != expected_source: + errors.append("setup source commit mismatch") +if source.get("tree_sha") != expected_tree: + errors.append("setup source tree mismatch") +if source.get("ghostty_gitlink_sha") != expected_ghostty: + errors.append("setup Ghostty gitlink mismatch") +if source.get("ghostty_head_sha") != expected_ghostty: + errors.append("setup Ghostty checkout mismatch") +if testbox.get("id") != expected_testbox: + errors.append("setup Testbox ID mismatch") +if errors: + for error in errors: + print(error, file=sys.stderr) + raise SystemExit(66) +PY +} +verify_setup_identity benchmark_dir="$repo_root/testbox-benchmark" command -v flock >/dev/null || { diff --git a/scripts/blacksmith-testbox-cleanup.sh b/scripts/blacksmith-testbox-cleanup.sh index 196e7b20f75..176c4d0017d 100755 --- a/scripts/blacksmith-testbox-cleanup.sh +++ b/scripts/blacksmith-testbox-cleanup.sh @@ -1,19 +1,47 @@ #!/usr/bin/env bash set -euo pipefail -if [[ $# -ne 2 ]]; then - echo "usage: $0 " >&2 +if [[ $# -ne 3 ]]; then + echo "usage: $0 " >&2 exit 64 fi testbox_id="$1" evidence_dir="$2" +confirmation_token="$3" if [[ ! "$testbox_id" =~ ^tbx_[A-Za-z0-9_-]+$ ]]; then echo "invalid Testbox ID: $testbox_id" >&2 exit 64 fi +if [[ ! "$confirmation_token" =~ ^[0-9a-f]{32}$ ]]; then + echo "confirmation token must be a 32-character lowercase hex value" >&2 + exit 64 +fi mkdir -p "$evidence_dir" +receipt_path="$evidence_dir/testbox-receipt.json" +if [[ ! -s "$receipt_path" ]]; then + echo "refusing cleanup without the warmup ownership receipt: $receipt_path" >&2 + exit 65 +fi +python3 - "$receipt_path" "$testbox_id" "$confirmation_token" <<'PY' +import json +import pathlib +import sys + +receipt_path, expected_id, expected_token = sys.argv[1:] +try: + receipt = json.loads(pathlib.Path(receipt_path).read_text(encoding="utf-8")) +except (OSError, json.JSONDecodeError) as error: + raise SystemExit(f"invalid warmup ownership receipt: {error}") +if receipt.get("testbox_id") != expected_id: + raise SystemExit("cleanup ID does not match the warmup ownership receipt") +if receipt.get("confirmation_token") != expected_token: + raise SystemExit("cleanup confirmation token does not match the warmup ownership receipt") +for field in ("workflow", "job", "source_ref", "source_sha", "source_tree_sha", "ghostty_gitlink_sha"): + if not receipt.get(field): + raise SystemExit(f"warmup ownership receipt is missing {field}") +PY stop_log="$evidence_dir/stop.log" status_log="$evidence_dir/status-after-stop.log" list_log="$evidence_dir/list-after-stop.log" @@ -34,8 +62,9 @@ if (( stop_status != 0 )); then fi fi -# Status is diagnostic. The authoritative cleanup check below is the specific -# ID in `list --all`, because Blacksmith may remove completed boxes immediately. +# Status is diagnostic. The authoritative cleanup check below parses the +# specific ID in `list --all`; terminal rows are accepted because --all may +# retain completed boxes, while active rows remain failures. set +e blacksmith testbox status --id "$testbox_id" >"$status_log" 2>&1 status_status=$? @@ -70,9 +99,24 @@ set -e if (( list_status != 0 )); then echo "failed to list Testboxes after stopping $testbox_id; see $list_log" >&2 (( cleanup_status == 0 )) && cleanup_status=$list_status -elif grep -Fq -- "$testbox_id" "$list_log"; then - echo "Testbox $testbox_id is still present in the active inventory; see $list_log" >&2 - (( cleanup_status == 0 )) && cleanup_status=1 +else + listed_status="$(awk -v id="$testbox_id" '$1 == id { print tolower($2); exit }' "$list_log")" + case "$listed_status" in + "") + # The CLI may remove terminal boxes from the inventory immediately. + ;; + completed|stopped|cancelled|failed|terminated) + # --all is the full inventory, so a terminal row is safe and expected. + ;; + ready|running|hydrating|in_progress|queued) + echo "Testbox $testbox_id is still active after cleanup; see $list_log" >&2 + (( cleanup_status == 0 )) && cleanup_status=1 + ;; + *) + echo "unknown status for Testbox $testbox_id in inventory: $listed_status" >&2 + (( cleanup_status == 0 )) && cleanup_status=1 + ;; + esac fi if (( cleanup_status != 0 )); then diff --git a/scripts/blacksmith-testbox-recover-warmup.sh b/scripts/blacksmith-testbox-recover-warmup.sh deleted file mode 100755 index abc9bbce415..00000000000 --- a/scripts/blacksmith-testbox-recover-warmup.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -if [[ $# -ne 6 ]]; then - echo "usage: $0 " >&2 - exit 64 -fi - -before_list="$1" -after_list="$2" -workflow="$3" -job="$4" -branch_ref="$5" -evidence_dir="$6" -for path in "$before_list" "$after_list"; do - if [[ ! -f "$path" ]]; then - echo "inventory file is missing: $path" >&2 - exit 65 - fi -done -mkdir -p "$evidence_dir" -decision_log="$evidence_dir/warmup-recovery.log" - -set +e -candidate="$(python3 - "$before_list" "$after_list" "$workflow" "$job" "$branch_ref" <<'PY' -import re -import sys - -before_path, after_path, workflow, job, branch_ref = sys.argv[1:] -id_pattern = re.compile(r"tbx_[A-Za-z0-9_-]+$") - -def matching(path): - result = set() - with open(path, encoding="utf-8") as handle: - for line in handle: - fields = line.split() - # list --all is a whitespace table: ID STATUS IP WORKFLOW JOB REF ... - if len(fields) >= 6 and id_pattern.fullmatch(fields[0]): - if fields[3] == workflow and fields[4] == job and fields[5] == branch_ref: - result.add(fields[0]) - return result - -before = matching(before_path) -after = matching(after_path) -new = sorted(after - before) -if len(new) != 1: - print( - f"warmup recovery found {len(new)} uniquely matching new Testbox IDs " - f"(before={sorted(before)}, after={sorted(after)})", - file=sys.stderr, - ) - raise SystemExit(1) -print(new[0]) -PY -)" -parse_status=$? -set -e -if (( parse_status != 0 )); then - { - echo "No Testbox was stopped: warmup recovery was ambiguous or had no matching new ID." - echo "before=$before_list" - echo "after=$after_list" - } >"$decision_log" - cat "$decision_log" >&2 - exit "$parse_status" -fi - -printf 'warmup recovery selected uniquely new Testbox %s\n' "$candidate" | tee "$decision_log" -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -"$script_dir/blacksmith-testbox-cleanup.sh" "$candidate" "$evidence_dir" diff --git a/skills/blacksmith-testbox/SKILL.md b/skills/blacksmith-testbox/SKILL.md index bb110fd7112..33a10c80a04 100644 --- a/skills/blacksmith-testbox/SKILL.md +++ b/skills/blacksmith-testbox/SKILL.md @@ -166,8 +166,10 @@ checks after the build. Keep the setup artifact URL or download it into `$OUT`; it records runner/toolchain/Ghostty identity independently of the stage helper. The setup JSON contains the workflow run, source ref/SHA/tree, Ghostty gitlink and checkout SHA, runner label/architecture/CPU identity, and pinned -Rust/Zig/toolchain-file metadata. Never print or download -`/tmp/.testbox/auth_token`. +Rust/Zig/toolchain-file metadata. A successful setup also copies that JSON to +`/tmp/.testbox/cmux-tui-rust-setup-identity.json`; the stage helper refuses a +missing or mismatched marker, so a failed hydration cannot be benchmarked. +Never print or download `/tmp/.testbox/auth_token`. ## Remote benchmark stages @@ -282,10 +284,16 @@ log, the setup artifact, and the source manifest in the separate ## Fail-safe cleanup Always download before cleanup. Use the checked-in cleanup helper rather than -ignoring errors with `|| true`: +ignoring errors with `|| true`. Pass the confirmation token generated with the +warmup receipt; never print it: ```bash -scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" +CLEANUP_TOKEN="${CLEANUP_TOKEN:-}" +[[ "$CLEANUP_TOKEN" =~ ^[0-9a-f]{32}$ ]] || { + echo "use the confirmation token emitted by the warmup receipt" >&2 + exit 64 +} +scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" "$CLEANUP_TOKEN" ``` It records stop, post-stop status, and `list --all` output; verifies that the @@ -293,12 +301,12 @@ specific Testbox ID is terminal or absent from the active inventory; and accepts only the known race where stop returns a 409 saying the box is already stopped or completed. Other stop, status, or list failures remain failures. Put it in an `EXIT` trap that preserves the benchmark's original exit status -unless cleanup itself fails. The detailed benchmark captures a pre-warmup -inventory and uses `scripts/blacksmith-testbox-recover-warmup.sh` when warmup -fails before printing an ID. That recovery path stops a box only when the -before/after inventory has exactly one new match for the workflow, job, and -branch; ambiguous or missing matches are reported without stopping an -unrelated box. +unless cleanup itself fails. The detailed benchmark writes a receipt and +confirmation token for the exact ID returned by warmup; cleanup refuses an ID +or token that is not bound to that receipt. If warmup fails before returning an +ID, retain before/after inventory but do not automatically stop a box, because +an inventory diff cannot prove ownership across concurrent operators. Reconcile +that orphan manually through the Blacksmith control plane. ## Timing interpretation diff --git a/skills/blacksmith-testbox/benchmark.md b/skills/blacksmith-testbox/benchmark.md index ce1cba502c2..58850c918db 100644 --- a/skills/blacksmith-testbox/benchmark.md +++ b/skills/blacksmith-testbox/benchmark.md @@ -120,6 +120,8 @@ WORKFLOW=.github/workflows/ci-workflow-guard-tests-testbox.yml JOB=cmux-tui-rust OUT="$PWD/.cmux-scratch/blacksmith-testbox-$SOURCE_SHA" TBX="" +warmup_testbox_id="" +cleanup_token="" before_list_status=125 mkdir -p "$OUT/raw" cleanup() { @@ -127,28 +129,24 @@ cleanup() { local cleanup_status=0 local after_list_status=125 trap - EXIT - if [[ -n "$TBX" ]]; then + if [[ -n "$TBX" && -n "$cleanup_token" ]]; then set +e - scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" + scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" "$cleanup_token" cleanup_status=$? set -e else - # If warmup failed before printing its ID, stop only a uniquely correlated - # new box. Never stop an arbitrary active box from a shared organization. + # Without the CLI receipt there is no proof that a newly listed box belongs + # to this invocation. Report inventory, but never stop another operator's box. set +e blacksmith testbox list --all >"$OUT/list-after-warmup-failure.log" 2>&1 after_list_status=$? set -e - if (( before_list_status == 0 && after_list_status == 0 )); then - set +e - scripts/blacksmith-testbox-recover-warmup.sh \ - "$OUT/list-before-warmup.log" "$OUT/list-after-warmup-failure.log" \ - "$WORKFLOW" "$JOB" "$SOURCE_REF" "$OUT" - cleanup_status=$? - set -e + if (( after_list_status != 0 )); then + cleanup_status="$after_list_status" + echo "could not capture post-failure Testbox inventory" >&2 else cleanup_status=1 - echo "could not reconcile a failed warmup because inventory capture failed" >&2 + echo "warmup returned no owned Testbox receipt; no automatic stop was attempted" >&2 fi fi if (( result == 0 && cleanup_status != 0 )); then @@ -180,7 +178,7 @@ warmup_status=$? set -e cat "$OUT/warmup.log" set +e -TBX="$(python3 - "$OUT/warmup.log" <<'PY' +warmup_testbox_id="$(python3 - "$OUT/warmup.log" <<'PY' import re import sys @@ -193,13 +191,48 @@ PY )" parse_status=$? set -e -if (( warmup_status != 0 )); then - exit "$warmup_status" -fi if (( parse_status != 0 )); then exit "$parse_status" fi +umask 077 +set +e +cleanup_token="$(python3 - "$OUT/testbox-receipt.json" "$warmup_testbox_id" "$WORKFLOW" "$JOB" "$SOURCE_REF" "$SOURCE_SHA" "$SOURCE_TREE_SHA" "$GHOSTTY_SHA" <<'PY' +import datetime as dt +import json +import pathlib +import secrets +import sys + +path, testbox_id, workflow, job, source_ref, source_sha, source_tree, ghostty_sha = sys.argv[1:] +token = secrets.token_hex(16) +path = pathlib.Path(path) +path.write_text(json.dumps({ + "schema": 1, + "testbox_id": testbox_id, + "workflow": workflow, + "job": job, + "source_ref": source_ref, + "source_sha": source_sha, + "source_tree_sha": source_tree, + "ghostty_gitlink_sha": ghostty_sha, + "confirmation_token": token, + "created_at": dt.datetime.now(dt.timezone.utc).isoformat(), +}, indent=2, sort_keys=True) + "\n", encoding="utf-8") +path.chmod(0o600) +print(token) +PY +)" +receipt_status=$? +set -e +if (( receipt_status != 0 )); then + echo "could not create the warmup ownership receipt" >&2 + exit "$receipt_status" +fi +TBX="$warmup_testbox_id" printf 'Testbox ID: %s\n' "$TBX" | tee "$OUT/testbox-id.txt" +if (( warmup_status != 0 )); then + exit "$warmup_status" +fi set +e blacksmith testbox status --id "$TBX" --wait --wait-timeout 15m \ >"$OUT/status-ready.log" 2>&1 @@ -229,7 +262,10 @@ the identity transcripts. The helper verifies the Testbox VM marker, claimed Testbox ID, source commit/tree, Ghostty gitlink/checkout, and clean status before it invokes Cargo, then repeats those checks after the build. Keep the setup artifact URL or download it into `$OUT`; it records runner/toolchain/Ghostty -identity independently of the stage helper. +identity independently of the stage helper. A successful setup copies the same +JSON to `/tmp/.testbox/cmux-tui-rust-setup-identity.json`; the stage helper +rejects a missing or mismatched marker, so failed hydration cannot be +benchmarked. ## Three remote build timings @@ -339,18 +375,24 @@ fail-safe helper, which preserves an already-completed 409 but fails on other stop/status/list errors and verifies this exact Testbox ID is no longer active: ```bash -scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" +cleanup_token="${cleanup_token:-}" +[[ "$cleanup_token" =~ ^[0-9a-f]{32}$ ]] || { + echo "use the confirmation token emitted by the warmup receipt" >&2 + exit 64 +} +scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" "$cleanup_token" ``` A shell `EXIT` trap should call that helper while preserving the benchmark -status. The benchmark captures a pre-warmup inventory. If warmup fails before -printing an ID, `scripts/blacksmith-testbox-recover-warmup.sh` compares the -before/after inventories and invokes cleanup only for exactly one new box -matching this workflow, job, and branch. Ambiguous or missing matches are -reported without stopping an unrelated box. Keep both inventories and their -command statuses. Keep warmup/status/identity transcripts, every stage run and -download transcript, raw JSON/time/log files, runner catalog, setup identity -artifact, cleanup logs, and the final source manifest in the separate +status. Warmup writes `testbox-receipt.json` and a confirmation token bound to +the exact returned ID; cleanup refuses a mismatched ID or token. If warmup +fails before returning an ID, retain before/after inventories but do not +automatically stop a box, because an inventory diff cannot prove ownership +across concurrent operators. Reconcile that orphan manually through the +Blacksmith control plane. Keep both inventories and their command statuses, +plus warmup/status/identity transcripts, every stage run and download +transcript, raw JSON/time/log files, runner catalog, setup identity artifact, +cleanup logs, the receipt, and the final source manifest in the separate `.cmux-scratch/` directory. Never store credentials, private keys, or `/tmp/.testbox/auth_token`. From 5735ee5d409e3c32945e9c85ac1747b050b233dc Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 06:33:04 -0700 Subject: [PATCH 08/38] ci: pin Testbox hydration failure reporting --- .github/workflows/ci-workflow-guard-tests-testbox.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-workflow-guard-tests-testbox.yml b/.github/workflows/ci-workflow-guard-tests-testbox.yml index 7e771571d77..4f823e1e65f 100644 --- a/.github/workflows/ci-workflow-guard-tests-testbox.yml +++ b/.github/workflows/ci-workflow-guard-tests-testbox.yml @@ -315,5 +315,5 @@ jobs: # any setup step failed. The setup marker is a second guard for CLI runs. # Rust builds are issued later with `blacksmith testbox run`. - name: Run Testbox - uses: useblacksmith/run-testbox@5ca05834db1d3813554d1dd109e5f2087a8d7cbc # v2 + uses: useblacksmith/run-testbox@3f60ff9ceb2c10c3feefa87dc0c6490cffae059d # v2 + hydration-failure reporting if: always() From bc7ff77e4daca26cf83698dbeb2087e56882bac5 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 07:03:44 -0700 Subject: [PATCH 09/38] ci: bound and verify remote Testbox builds --- .../ci-workflow-guard-tests-testbox.yml | 37 ++++++++++++++++-- scripts/blacksmith-cmux-tui-testbox-stage.sh | 38 +++++++++++++++++-- skills/blacksmith-testbox/SKILL.md | 9 +++-- skills/blacksmith-testbox/benchmark.md | 24 ++++++++---- 4 files changed, 92 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci-workflow-guard-tests-testbox.yml b/.github/workflows/ci-workflow-guard-tests-testbox.yml index 4f823e1e65f..5d846fa756e 100644 --- a/.github/workflows/ci-workflow-guard-tests-testbox.yml +++ b/.github/workflows/ci-workflow-guard-tests-testbox.yml @@ -300,13 +300,44 @@ jobs: if-no-files-found: warn retention-days: 14 - - name: Mark Testbox setup ready + - name: Confirm Testbox registration and mark setup ready if: success() + env: + EXPECTED_TESTBOX_ID: ${{ inputs.testbox_id }} shell: bash run: | set -euo pipefail - marker=/tmp/.testbox/cmux-tui-rust-setup-identity.json - test -d /tmp/.testbox + state=/tmp/.testbox + for required_file in testbox_id installation_model_id auth_token api_url runner_host runner_ssh_port adopted_run_id; do + test -s "$state/$required_file" + done + test -s "$state/working_directory" + test "$(<"$state/testbox_id")" = "$EXPECTED_TESTBOX_ID" + installation_model_id="$(<"$state/installation_model_id")" + [[ "$installation_model_id" =~ ^[0-9]+$ ]] + api_url="$(<"$state/api_url")" + auth_token="$(<"$state/auth_token")" + runner_host="$(<"$state/runner_host")" + runner_ssh_port="$(<"$state/runner_ssh_port")" + working_directory="$(<"$state/working_directory")" + adopted_run_id="$(<"$state/adopted_run_id")" + payload="$(jq -n \ + --arg testbox_id "$EXPECTED_TESTBOX_ID" \ + --arg runner_host "$runner_host" \ + --arg runner_ssh_port "$runner_ssh_port" \ + --arg working_directory "$working_directory" \ + --arg adopted_run_id "$adopted_run_id" \ + --argjson installation_model_id "$installation_model_id" \ + '{testbox_id: $testbox_id, installation_model_id: $installation_model_id, status: "ready", ip_address: $runner_host, ssh_port: $runner_ssh_port, working_directory: $working_directory, adopted_run_id: $adopted_run_id, metadata: {}}')" + # begin-testbox deliberately continues after phone-home errors. Make + # registration an explicit fail-closed prerequisite before publishing + # the marker consumed by the remote benchmark helper. + curl -fsS --connect-timeout 2 --max-time 10 \ + -X POST "$api_url/api/testbox/phone-home" \ + -H 'Content-Type: application/json' \ + -H "Authorization: Bearer $auth_token" \ + --data "$payload" >/dev/null + marker="$state/cmux-tui-rust-setup-identity.json" install -m 600 testbox-benchmark/setup-identity.json "$marker" test -s "$marker" diff --git a/scripts/blacksmith-cmux-tui-testbox-stage.sh b/scripts/blacksmith-cmux-tui-testbox-stage.sh index d603818cc77..3362d6a7318 100755 --- a/scripts/blacksmith-cmux-tui-testbox-stage.sh +++ b/scripts/blacksmith-cmux-tui-testbox-stage.sh @@ -73,6 +73,10 @@ if [[ ! "$ghostty_entry" =~ ^160000[[:space:]]commit[[:space:]][0-9a-f]{40}[[:sp exit 65 fi expected_tree_sha="$(git rev-parse "${expected_source_sha}^{tree}")" +if ! command -v timeout >/dev/null; then + echo "timeout is required for bounded remote builds" >&2 + exit 65 +fi setup_identity_path="$state_dir/cmux-tui-rust-setup-identity.json" if [[ ! -s "$setup_identity_path" ]]; then echo "refusing to run without a successful Testbox setup identity marker" >&2 @@ -126,6 +130,8 @@ pre_identity_path="$benchmark_dir/.$stage.pre-identity.json" post_identity_path="$benchmark_dir/.$stage.post-identity.json" changed_file="cmux-tui/crates/cmux-tui/src/main.rs" changed_backup="" +changed_backup_sha256="" +changed_backup_size="" restore_changed_file() { if [[ -n "$changed_backup" ]]; then @@ -133,9 +139,19 @@ restore_changed_file() { echo "changed-file backup disappeared: $changed_backup" >&2 return 1 fi + if [[ "$(wc -c <"$changed_backup")" != "$changed_backup_size" || + "$(sha256sum "$changed_backup" | cut -d ' ' -f 1)" != "$changed_backup_sha256" ]]; then + echo "changed-file backup failed integrity verification" >&2 + return 1 + fi if ! cp "$changed_backup" "$repo_root/$changed_file"; then return 1 fi + if [[ "$(wc -c <"$repo_root/$changed_file")" != "$changed_backup_size" || + "$(sha256sum "$repo_root/$changed_file" | cut -d ' ' -f 1)" != "$changed_backup_sha256" ]]; then + echo "restored source failed integrity verification" >&2 + return 1 + fi if ! rm -f "$changed_backup"; then return 1 fi @@ -280,6 +296,7 @@ rust_toolchain="$(rustup show active-toolchain)" rustc_version="$(rustc --version)" cargo_version="$(cargo --version)" zig_version="$("$zig_bin" version)" +export ZIG="$zig_bin" rust_toolchain_file_sha256="$(sha256sum cmux-tui/rust-toolchain.toml | cut -d ' ' -f 1)" cargo_lock_sha256="$(sha256sum cmux-tui/Cargo.lock | cut -d ' ' -f 1)" ghostty_zon_sha256="$(sha256sum ghostty/build.zig.zon | cut -d ' ' -f 1)" @@ -293,8 +310,22 @@ case "$stage" in echo "changed-file target is missing: $changed_file" >&2 exit 65 fi - changed_backup="$(mktemp "${TMPDIR:-/tmp}/cmux-tui-testbox-source.XXXXXX")" - cp "$repo_root/$changed_file" "$changed_backup" + backup_candidate="$(mktemp "${TMPDIR:-/tmp}/cmux-tui-testbox-source.XXXXXX")" + if ! cp "$repo_root/$changed_file" "$backup_candidate"; then + rm -f "$backup_candidate" + echo "failed to create a source backup" >&2 + exit 67 + fi + backup_sha256="$(sha256sum "$backup_candidate" | cut -d ' ' -f 1)" + backup_size="$(wc -c <"$backup_candidate")" + [[ "$backup_size" =~ ^[0-9]+$ && "$backup_size" -gt 0 ]] || { + rm -f "$backup_candidate" + echo "source backup is empty" >&2 + exit 67 + } + changed_backup="$backup_candidate" + changed_backup_sha256="$backup_sha256" + changed_backup_size="$backup_size" printf '\n// Blacksmith Testbox changed-file timing marker.\n' >>"$repo_root/$changed_file" ;; esac @@ -304,7 +335,8 @@ rm -f "$time_path" "$log_path" "$json_path" "$post_identity_path" set +e ( cd "$repo_root/cmux-tui" - /usr/bin/time -p -o "$time_path" cargo build -p cmux-tui --locked + timeout --foreground --kill-after=30s 20m \ + /usr/bin/time -p -o "$time_path" cargo build -p cmux-tui --locked ) >"$log_path" 2>&1 build_status=$? set -e diff --git a/skills/blacksmith-testbox/SKILL.md b/skills/blacksmith-testbox/SKILL.md index 33a10c80a04..9f9193a9283 100644 --- a/skills/blacksmith-testbox/SKILL.md +++ b/skills/blacksmith-testbox/SKILL.md @@ -213,7 +213,9 @@ run_stage() { ``` The helper supports exactly `first-clean`, `incremental-noop`, and -`changed-file`. It records a schema-2 JSON object for each stage containing: +`changed-file`. Each remote Cargo build is bounded to 20 minutes with a +30-second kill grace period. It records a schema-2 JSON object for each stage +containing: * expected and observed source commit/tree identity before and after the build; * expected and observed Ghostty gitlink and initialized submodule HEAD; @@ -278,8 +280,9 @@ PY ``` Keep `raw/*.json`, `raw/*.time`, `raw/*.log`, every `*.run.log` and download -log, the setup artifact, and the source manifest in the separate -`.cmux-scratch/` evidence directory. Do not add credentials or private keys. +log, the setup artifact, and the source manifest in a new, unique +`.cmux-scratch/` evidence directory. Never reuse a prior SHA-only directory; +refuse to overwrite historical records. Do not add credentials or private keys. ## Fail-safe cleanup diff --git a/skills/blacksmith-testbox/benchmark.md b/skills/blacksmith-testbox/benchmark.md index 58850c918db..12970711cc8 100644 --- a/skills/blacksmith-testbox/benchmark.md +++ b/skills/blacksmith-testbox/benchmark.md @@ -118,7 +118,12 @@ Use the branch ref, not `SOURCE_SHA`, in warmup: ```bash WORKFLOW=.github/workflows/ci-workflow-guard-tests-testbox.yml JOB=cmux-tui-rust -OUT="$PWD/.cmux-scratch/blacksmith-testbox-$SOURCE_SHA" +OUT_ROOT="$PWD/.cmux-scratch/blacksmith-testbox-$SOURCE_SHA" +if [[ -e "$OUT_ROOT" ]]; then + echo "evidence directory already exists; choose a new run path and preserve prior records: $OUT_ROOT" >&2 + exit 1 +fi +OUT="$OUT_ROOT" TBX="" warmup_testbox_id="" cleanup_token="" @@ -270,9 +275,12 @@ benchmarked. ## Three remote build timings The helper creates one structured JSON record, one raw Cargo log, and one raw -`/usr/bin/time -p` file per stage. It verifies source and submodule identity -before the stage, holds a remote `flock` through all writes, restores the -controlled changed file, and verifies clean identity again. +`/usr/bin/time -p` file per stage. Each remote Cargo build is bounded to 20 +minutes with a 30-second kill grace period. It verifies source and submodule +identity before the stage, exports and records the exact Zig binary used by +Cargo, holds a remote `flock` through all writes, restores the controlled +changed file from an integrity-checked backup, and verifies clean identity +again. ```bash # Run this orchestration block in Bash, not an interactive zsh session. @@ -392,8 +400,9 @@ across concurrent operators. Reconcile that orphan manually through the Blacksmith control plane. Keep both inventories and their command statuses, plus warmup/status/identity transcripts, every stage run and download transcript, raw JSON/time/log files, runner catalog, setup identity artifact, -cleanup logs, the receipt, and the final source manifest in the separate -`.cmux-scratch/` directory. Never store credentials, private keys, or +cleanup logs, the receipt, and the final source manifest in the new, unique +`.cmux-scratch/` directory. Never reuse a prior SHA-only directory or overwrite +historical records. Never store credentials, private keys, or `/tmp/.testbox/auth_token`. Record these fields alongside `timings.json`: @@ -410,7 +419,8 @@ Record these fields alongside `timings.json`: the active inventory. The historical 32-vCPU evidence at -`.cmux-scratch/blacksmith-testbox-e40704611ac35f4ffa153/` remains unchanged: +`.cmux-scratch/blacksmith-testbox-e40704611ac35f4ffa153/` remains unchanged and +must never be selected as a writable `OUT` directory: setup SHA `e40704611ac35f0e3a806841a9eae383f4ffa153`, Testbox `tbx_01kzxebn91nhatkv4ygevh06vs`, workflow run `31696013711`, first-clean `161.47s`, incremental no-op `8.28s`, changed-file `9.13s`, and cleanup with no From 5cdb5fc20d5bddc6e0c4dfe6236a435f9ef84452 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 07:13:19 -0700 Subject: [PATCH 10/38] ci: verify Testbox ready phone-home --- .github/workflows/ci-workflow-guard-tests-testbox.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-workflow-guard-tests-testbox.yml b/.github/workflows/ci-workflow-guard-tests-testbox.yml index 5d846fa756e..f4e42c0ce7b 100644 --- a/.github/workflows/ci-workflow-guard-tests-testbox.yml +++ b/.github/workflows/ci-workflow-guard-tests-testbox.yml @@ -332,11 +332,12 @@ jobs: # begin-testbox deliberately continues after phone-home errors. Make # registration an explicit fail-closed prerequisite before publishing # the marker consumed by the remote benchmark helper. - curl -fsS --connect-timeout 2 --max-time 10 \ + response="$(curl -fsS --connect-timeout 2 --max-time 10 \ -X POST "$api_url/api/testbox/phone-home" \ -H 'Content-Type: application/json' \ -H "Authorization: Bearer $auth_token" \ - --data "$payload" >/dev/null + --data "$payload")" + [[ -n "$response" ]] marker="$state/cmux-tui-rust-setup-identity.json" install -m 600 testbox-benchmark/setup-identity.json "$marker" test -s "$marker" From 94c618747b61e1d9987d8f54580693c103cba90d Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 07:33:10 -0700 Subject: [PATCH 11/38] ci: fence setup markers and preserve evidence --- .../ci-workflow-guard-tests-testbox.yml | 2 ++ scripts/blacksmith-cmux-tui-testbox-stage.sh | 18 ++++++++++++---- skills/blacksmith-testbox/benchmark.md | 21 ++++++++++++------- 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci-workflow-guard-tests-testbox.yml b/.github/workflows/ci-workflow-guard-tests-testbox.yml index f4e42c0ce7b..989348a16c1 100644 --- a/.github/workflows/ci-workflow-guard-tests-testbox.yml +++ b/.github/workflows/ci-workflow-guard-tests-testbox.yml @@ -308,6 +308,7 @@ jobs: run: | set -euo pipefail state=/tmp/.testbox + rm -f /tmp/.testbox/cmux-tui-rust-setup-identity.json for required_file in testbox_id installation_model_id auth_token api_url runner_host runner_ssh_port adopted_run_id; do test -s "$state/$required_file" done @@ -339,6 +340,7 @@ jobs: --data "$payload")" [[ -n "$response" ]] marker="$state/cmux-tui-rust-setup-identity.json" + rm -f "$marker" install -m 600 testbox-benchmark/setup-identity.json "$marker" test -s "$marker" diff --git a/scripts/blacksmith-cmux-tui-testbox-stage.sh b/scripts/blacksmith-cmux-tui-testbox-stage.sh index 3362d6a7318..f43ff9bac02 100755 --- a/scripts/blacksmith-cmux-tui-testbox-stage.sh +++ b/scripts/blacksmith-cmux-tui-testbox-stage.sh @@ -82,14 +82,22 @@ if [[ ! -s "$setup_identity_path" ]]; then echo "refusing to run without a successful Testbox setup identity marker" >&2 exit 65 fi +setup_run_id="$(tr -d '\r\n' <"$state_dir/adopted_run_id")" +if [[ ! "$setup_run_id" =~ ^[0-9]+$ ]]; then + echo "invalid Testbox setup workflow run ID" >&2 + exit 65 +fi verify_setup_identity() { - python3 - "$setup_identity_path" "$expected_source_sha" "$expected_tree_sha" "$expected_ghostty_sha" "$testbox_id" <<'PY' + python3 - "$setup_identity_path" "$expected_source_sha" "$expected_tree_sha" "$expected_ghostty_sha" "$testbox_id" "$setup_run_id" <<'PY' import json import pathlib import sys -path, expected_source, expected_tree, expected_ghostty, expected_testbox = sys.argv[1:] -record = json.loads(pathlib.Path(path).read_text(encoding="utf-8")) +path, expected_source, expected_tree, expected_ghostty, expected_testbox, expected_run_id = sys.argv[1:] +try: + record = json.loads(pathlib.Path(path).read_text(encoding="utf-8")) +except (OSError, json.JSONDecodeError) as error: + raise SystemExit(f"invalid setup identity marker: {error}") source = record.get("source", {}) testbox = record.get("testbox", {}) errors = [] @@ -103,6 +111,8 @@ if source.get("ghostty_head_sha") != expected_ghostty: errors.append("setup Ghostty checkout mismatch") if testbox.get("id") != expected_testbox: errors.append("setup Testbox ID mismatch") +if str(testbox.get("setup_workflow_run_id")) != expected_run_id: + errors.append("setup workflow run ID mismatch") if errors: for error in errors: print(error, file=sys.stderr) @@ -335,7 +345,7 @@ rm -f "$time_path" "$log_path" "$json_path" "$post_identity_path" set +e ( cd "$repo_root/cmux-tui" - timeout --foreground --kill-after=30s 20m \ + timeout --kill-after=30s 20m \ /usr/bin/time -p -o "$time_path" cargo build -p cmux-tui --locked ) >"$log_path" 2>&1 build_status=$? diff --git a/skills/blacksmith-testbox/benchmark.md b/skills/blacksmith-testbox/benchmark.md index 12970711cc8..2a3fa08b8b2 100644 --- a/skills/blacksmith-testbox/benchmark.md +++ b/skills/blacksmith-testbox/benchmark.md @@ -72,8 +72,19 @@ remote_sha="$(git ls-remote --exit-code origin "refs/heads/$SOURCE_REF" | awk 'N echo "push the exact clean branch head before warming Testbox" >&2 exit 1 } -mkdir -p ".cmux-scratch/blacksmith-testbox-$SOURCE_SHA/raw" -python3 - "$SOURCE_REF" "$SOURCE_SHA" "$SOURCE_TREE_SHA" "$GHOSTTY_SHA" > ".cmux-scratch/blacksmith-testbox-$SOURCE_SHA/source.json" <<'PY' +EVIDENCE_ROOT="$PWD/.cmux-scratch" +if [[ -e "$EVIDENCE_ROOT/blacksmith-testbox-$SOURCE_SHA" ]]; then + RUN_SUFFIX="$(date -u +%Y%m%dT%H%M%SZ)-$$" +else + RUN_SUFFIX="initial-$$" +fi +OUT_ROOT="$EVIDENCE_ROOT/blacksmith-testbox-$SOURCE_SHA-$RUN_SUFFIX" +if [[ -e "$OUT_ROOT" ]]; then + echo "evidence directory already exists; choose a new run path: $OUT_ROOT" >&2 + exit 1 +fi +mkdir -p "$OUT_ROOT/raw" +python3 - "$SOURCE_REF" "$SOURCE_SHA" "$SOURCE_TREE_SHA" "$GHOSTTY_SHA" > "$OUT_ROOT/source.json" <<'PY' import json import sys @@ -118,11 +129,7 @@ Use the branch ref, not `SOURCE_SHA`, in warmup: ```bash WORKFLOW=.github/workflows/ci-workflow-guard-tests-testbox.yml JOB=cmux-tui-rust -OUT_ROOT="$PWD/.cmux-scratch/blacksmith-testbox-$SOURCE_SHA" -if [[ -e "$OUT_ROOT" ]]; then - echo "evidence directory already exists; choose a new run path and preserve prior records: $OUT_ROOT" >&2 - exit 1 -fi +OUT_ROOT="${OUT_ROOT:?set by the exact-source preflight above}" OUT="$OUT_ROOT" TBX="" warmup_testbox_id="" From 5da140127ef73a05b3dcde168c0d90f45c4a2118 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 07:54:33 -0700 Subject: [PATCH 12/38] docs: state Testbox trust configuration limits --- .github/workflows/ci-workflow-guard-tests-testbox.yml | 3 ++- skills/blacksmith-testbox/SKILL.md | 6 +++++- skills/blacksmith-testbox/benchmark.md | 6 ++++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-workflow-guard-tests-testbox.yml b/.github/workflows/ci-workflow-guard-tests-testbox.yml index 989348a16c1..e49395f51a3 100644 --- a/.github/workflows/ci-workflow-guard-tests-testbox.yml +++ b/.github/workflows/ci-workflow-guard-tests-testbox.yml @@ -309,7 +309,7 @@ jobs: set -euo pipefail state=/tmp/.testbox rm -f /tmp/.testbox/cmux-tui-rust-setup-identity.json - for required_file in testbox_id installation_model_id auth_token api_url runner_host runner_ssh_port adopted_run_id; do + for required_file in testbox_id installation_model_id auth_token api_url runner_host runner_ssh_port adopted_run_id ssh_public_key; do test -s "$state/$required_file" done test -s "$state/working_directory" @@ -339,6 +339,7 @@ jobs: -H "Authorization: Bearer $auth_token" \ --data "$payload")" [[ -n "$response" ]] + test -s "$state/ssh_public_key" marker="$state/cmux-tui-rust-setup-identity.json" rm -f "$marker" install -m 600 testbox-benchmark/setup-identity.json "$marker" diff --git a/skills/blacksmith-testbox/SKILL.md b/skills/blacksmith-testbox/SKILL.md index 9f9193a9283..418f6447270 100644 --- a/skills/blacksmith-testbox/SKILL.md +++ b/skills/blacksmith-testbox/SKILL.md @@ -47,7 +47,11 @@ The helper retains the `CMUX_TESTBOX_REMOTE=1` guard for accidental local launches, and additionally requires the Blacksmith VM kernel metadata marker and matching `/tmp/.testbox` state. The environment flag remains caller controlled and is not an authentication mechanism. The protected environment -and trusted-maintainer policy are the security boundary. +and trusted-maintainer policy are the security boundary. Verify before every +use that `blacksmith-testbox-trusted` still has required reviewers, no secrets, +administrator bypass disabled, and no broad branch policy admitting unreviewed +refs. The workflow cannot manufacture those repository settings, so drift makes +the lane unavailable rather than safe. * Never run `cargo`, `rustc`, `rustup`, `zig build`, or another Rust/Zig build command on Lawrence's Mac. This includes local fallback builds and local diff --git a/skills/blacksmith-testbox/benchmark.md b/skills/blacksmith-testbox/benchmark.md index 2a3fa08b8b2..5c184e34af3 100644 --- a/skills/blacksmith-testbox/benchmark.md +++ b/skills/blacksmith-testbox/benchmark.md @@ -19,8 +19,10 @@ setup-only GitHub job on the remote Linux runner. | Remote output | `testbox-benchmark/` | A repository administrator must configure the protected environment with -required reviewers and no secrets before this plan is usable. The lane must -never run untrusted PR or fork code. `begin-testbox` exposes its auth token to +required reviewers, no secrets, administrator bypass disabled, and a trusted +branch policy before this plan is usable. Verify that configuration before each +run. The lane must never run untrusted PR or fork code. If the configuration is +missing or drifts, stop instead of treating the environment name as a guard. `begin-testbox` exposes its auth token to commands in the Testbox, so `contents: read` is not a trust boundary. The repository does not currently pin a checksum-verified Blacksmith CLI artifact; that is a trusted-lane operational limitation. Use only the organization- From bd89ec77ccf2c88df501350e54ce1948c3f2e386 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 08:10:45 -0700 Subject: [PATCH 13/38] ci: preview owned Testbox cleanup --- scripts/blacksmith-testbox-cleanup.sh | 44 ++++++++++++++++++++++++++ skills/blacksmith-testbox/SKILL.md | 10 +++--- skills/blacksmith-testbox/benchmark.md | 5 +-- 3 files changed, 53 insertions(+), 6 deletions(-) diff --git a/scripts/blacksmith-testbox-cleanup.sh b/scripts/blacksmith-testbox-cleanup.sh index 176c4d0017d..eca29f9f8dd 100755 --- a/scripts/blacksmith-testbox-cleanup.sh +++ b/scripts/blacksmith-testbox-cleanup.sh @@ -42,6 +42,50 @@ for field in ("workflow", "job", "source_ref", "source_sha", "source_tree_sha", if not receipt.get(field): raise SystemExit(f"warmup ownership receipt is missing {field}") PY +receipt_workflow="$(python3 - "$receipt_path" <<'PY' +import json +import pathlib +import sys +print(json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8"))["workflow"]) +PY +)" +receipt_job="$(python3 - "$receipt_path" <<'PY' +import json +import pathlib +import sys +print(json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8"))["job"]) +PY +)" +receipt_ref="$(python3 - "$receipt_path" <<'PY' +import json +import pathlib +import sys +print(json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8"))["source_ref"]) +PY +)" +pre_status_log="$evidence_dir/status-before-stop.log" +set +e +blacksmith testbox status --id "$testbox_id" >"$pre_status_log" 2>&1 +pre_status=$? +set -e +if (( pre_status == 0 )); then + pre_row="$(awk -v id="$testbox_id" '$1 == id { print; exit }' "$pre_status_log")" + if [[ -z "$pre_row" ]]; then + echo "status did not contain the owned Testbox row; refusing cleanup" >&2 + exit 66 + fi + pre_workflow="$(awk '{print $4}' <<<"$pre_row")" + pre_job="$(awk '{print $5}' <<<"$pre_row")" + pre_ref="$(awk '{print $6}' <<<"$pre_row")" + if [[ "$pre_workflow" != "$receipt_workflow" || "$pre_job" != "$receipt_job" || "$pre_ref" != "$receipt_ref" ]]; then + echo "owned Testbox context differs from the warmup receipt; refusing cleanup" >&2 + exit 66 + fi +elif ! grep -Eiq '(not found|already[[:space:]]+(stopped|completed)|HTTP[[:space:]]+409|status[[:space:]]+code[[:space:]]+409)' "$pre_status_log"; then + echo "failed to preview Testbox $testbox_id before cleanup; see $pre_status_log" >&2 + exit "$pre_status" +fi + stop_log="$evidence_dir/stop.log" status_log="$evidence_dir/status-after-stop.log" list_log="$evidence_dir/list-after-stop.log" diff --git a/skills/blacksmith-testbox/SKILL.md b/skills/blacksmith-testbox/SKILL.md index 418f6447270..071fbe749bb 100644 --- a/skills/blacksmith-testbox/SKILL.md +++ b/skills/blacksmith-testbox/SKILL.md @@ -303,10 +303,12 @@ CLEANUP_TOKEN="${CLEANUP_TOKEN:-}" scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" "$CLEANUP_TOKEN" ``` -It records stop, post-stop status, and `list --all` output; verifies that the -specific Testbox ID is terminal or absent from the active inventory; and -accepts only the known race where stop returns a 409 saying the box is already -stopped or completed. Other stop, status, or list failures remain failures. +It records a pre-stop status preview, stop result, post-stop status, and +`list --all` output. The preview must match the receipt's workflow, job, and +branch before any stop is attempted. It verifies that the specific Testbox ID +is terminal or absent from the active inventory, and accepts only the known race +where stop returns a 409 saying the box is already stopped or completed. Other +stop, status, or list failures remain failures. Put it in an `EXIT` trap that preserves the benchmark's original exit status unless cleanup itself fails. The detailed benchmark writes a receipt and confirmation token for the exact ID returned by warmup; cleanup refuses an ID diff --git a/skills/blacksmith-testbox/benchmark.md b/skills/blacksmith-testbox/benchmark.md index 5c184e34af3..7b8ca73be2f 100644 --- a/skills/blacksmith-testbox/benchmark.md +++ b/skills/blacksmith-testbox/benchmark.md @@ -388,8 +388,9 @@ PY ## Cleanup and evidence Download all raw files and `timings.json` before cleanup. Then call the -fail-safe helper, which preserves an already-completed 409 but fails on other -stop/status/list errors and verifies this exact Testbox ID is no longer active: +fail-safe helper, which previews the exact receipt context, preserves an +already-completed 409, fails on other stop/status/list errors, and verifies this +exact Testbox ID is no longer active: ```bash cleanup_token="${cleanup_token:-}" From c1999b8f5411bbc6a6d85a1ea417e864d4c94131 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 08:32:34 -0700 Subject: [PATCH 14/38] ci: assert trusted runner identity --- .github/workflows/ci-workflow-guard-tests-testbox.yml | 8 ++++++++ skills/blacksmith-testbox/SKILL.md | 3 ++- skills/blacksmith-testbox/benchmark.md | 11 ++++++++++- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-workflow-guard-tests-testbox.yml b/.github/workflows/ci-workflow-guard-tests-testbox.yml index e49395f51a3..ddba9522104 100644 --- a/.github/workflows/ci-workflow-guard-tests-testbox.yml +++ b/.github/workflows/ci-workflow-guard-tests-testbox.yml @@ -86,6 +86,10 @@ jobs: echo "::error::the pushed branch moved during dispatch: remote=$remote_sha workflow=$DISPATCH_SHA" >&2 exit 1 } + # The workflow's source SHA is the only attestation Blacksmith can + # carry through workflow_dispatch. The local preflight and every + # remote stage must compare against this exact value; never infer an + # older revision from the mutable branch after begin-testbox. printf 'trusted source ref: %s\ntrusted source SHA: %s\n' "$DISPATCH_REF" "$DISPATCH_SHA" - name: Checkout exact dispatch commit @@ -228,6 +232,10 @@ jobs: ZIG_PATH="$CMUX_ZIG" RUNNER_UNAME="$(uname -a)" RUNNER_CPU_COUNT="$(nproc)" + [[ "$RUNNER_ARCH" == "X64" && "$RUNNER_CPU_COUNT" == "32" ]] || { + echo "::error::expected x64 32-vCPU runner, got arch=$RUNNER_ARCH cpu_count=$RUNNER_CPU_COUNT" >&2 + exit 1 + } CARGO_METADATA_SHA256="$(sha256sum "$RUNNER_TEMP/cmux-tui-cargo-metadata.json" | cut -d ' ' -f 1)" RUST_TOOLCHAIN_FILE_SHA256="$(sha256sum cmux-tui/rust-toolchain.toml | cut -d ' ' -f 1)" CARGO_LOCK_SHA256="$(sha256sum cmux-tui/Cargo.lock | cut -d ' ' -f 1)" diff --git a/skills/blacksmith-testbox/SKILL.md b/skills/blacksmith-testbox/SKILL.md index 071fbe749bb..fc4d505ae08 100644 --- a/skills/blacksmith-testbox/SKILL.md +++ b/skills/blacksmith-testbox/SKILL.md @@ -225,7 +225,8 @@ containing: * expected and observed Ghostty gitlink and initialized submodule HEAD; * clean/dirty file lists and source restoration status; * Testbox ID and adopted workflow run ID; -* runner label, hostname, architecture, CPU count, and `uname`; +* runner label, hostname, architecture, CPU count, and `uname`; the setup + workflow fails closed unless the actual runner is x64 with 32 CPUs; * active Rust toolchain, `rustc`, Cargo, Zig, lockfile/toolchain hashes, and Ghostty package-manifest hash; and * Cargo exit status, `/usr/bin/time -p` values, and CLI transcript timing. diff --git a/skills/blacksmith-testbox/benchmark.md b/skills/blacksmith-testbox/benchmark.md index 7b8ca73be2f..b496accdb99 100644 --- a/skills/blacksmith-testbox/benchmark.md +++ b/skills/blacksmith-testbox/benchmark.md @@ -191,6 +191,11 @@ blacksmith testbox warmup "$WORKFLOW" \ warmup_status=$? set -e cat "$OUT/warmup.log" +if (( warmup_status != 0 )); then + # Do not parse IDs from a failed CLI transcript. It may contain a stale ID + # from an error message, and cleanup is intentionally receipt-bound. + exit "$warmup_status" +fi set +e warmup_testbox_id="$(python3 - "$OUT/warmup.log" <<'PY' import re @@ -365,6 +370,9 @@ if {record.get("stage") for record in records} != required: raise SystemExit("expected exactly three stage records") for record in records: stage = record.get("stage") + runner = record.get("runner", {}) + if runner.get("arch") != "x86_64" or runner.get("cpu_count") != 32: + raise SystemExit(f"{stage}: wrong runner identity {runner}") if record.get("testbox", {}).get("id") != testbox_id: raise SystemExit(f"{stage}: wrong Testbox ID") source_record = record.get("source", {}) @@ -419,7 +427,8 @@ Record these fields alongside `timings.json`: 1. Exact source branch, full source SHA/tree SHA, Ghostty gitlink SHA, and the clean-status result before each stage. -2. Requested runner label and catalog output. +2. Requested runner label and catalog output. The setup job rejects any + actual architecture or CPU count other than x64 and 32. 3. Blacksmith CLI version, Testbox ID, setup workflow run/job IDs, identity run ID, and each stage run/sync ID from raw transcripts. 4. Whether the comparison was target-clean, registry/git-cache warm, From ba2fd408e3c33bb45c9e3a698238438d3264929d Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 09:00:19 -0700 Subject: [PATCH 15/38] ci: finalize Testbox readiness and cleanup states --- .../ci-workflow-guard-tests-testbox.yml | 27 +++---------------- scripts/blacksmith-testbox-cleanup.sh | 12 ++++----- skills/blacksmith-testbox/SKILL.md | 7 ++--- 3 files changed, 14 insertions(+), 32 deletions(-) diff --git a/.github/workflows/ci-workflow-guard-tests-testbox.yml b/.github/workflows/ci-workflow-guard-tests-testbox.yml index ddba9522104..7c89e83c047 100644 --- a/.github/workflows/ci-workflow-guard-tests-testbox.yml +++ b/.github/workflows/ci-workflow-guard-tests-testbox.yml @@ -324,30 +324,11 @@ jobs: test "$(<"$state/testbox_id")" = "$EXPECTED_TESTBOX_ID" installation_model_id="$(<"$state/installation_model_id")" [[ "$installation_model_id" =~ ^[0-9]+$ ]] - api_url="$(<"$state/api_url")" - auth_token="$(<"$state/auth_token")" - runner_host="$(<"$state/runner_host")" - runner_ssh_port="$(<"$state/runner_ssh_port")" - working_directory="$(<"$state/working_directory")" - adopted_run_id="$(<"$state/adopted_run_id")" - payload="$(jq -n \ - --arg testbox_id "$EXPECTED_TESTBOX_ID" \ - --arg runner_host "$runner_host" \ - --arg runner_ssh_port "$runner_ssh_port" \ - --arg working_directory "$working_directory" \ - --arg adopted_run_id "$adopted_run_id" \ - --argjson installation_model_id "$installation_model_id" \ - '{testbox_id: $testbox_id, installation_model_id: $installation_model_id, status: "ready", ip_address: $runner_host, ssh_port: $runner_ssh_port, working_directory: $working_directory, adopted_run_id: $adopted_run_id, metadata: {}}')" - # begin-testbox deliberately continues after phone-home errors. Make - # registration an explicit fail-closed prerequisite before publishing - # the marker consumed by the remote benchmark helper. - response="$(curl -fsS --connect-timeout 2 --max-time 10 \ - -X POST "$api_url/api/testbox/phone-home" \ - -H 'Content-Type: application/json' \ - -H "Authorization: Bearer $auth_token" \ - --data "$payload")" - [[ -n "$response" ]] test -s "$state/ssh_public_key" + # begin-testbox deliberately continues after degraded phone-home. + # Validate the SSH handoff and install the identity marker before the + # pinned run-testbox action publishes ready. That action remains the + # sole readiness publisher, so a CLI run cannot race this marker. marker="$state/cmux-tui-rust-setup-identity.json" rm -f "$marker" install -m 600 testbox-benchmark/setup-identity.json "$marker" diff --git a/scripts/blacksmith-testbox-cleanup.sh b/scripts/blacksmith-testbox-cleanup.sh index eca29f9f8dd..f577cc6975d 100755 --- a/scripts/blacksmith-testbox-cleanup.sh +++ b/scripts/blacksmith-testbox-cleanup.sh @@ -81,7 +81,7 @@ if (( pre_status == 0 )); then echo "owned Testbox context differs from the warmup receipt; refusing cleanup" >&2 exit 66 fi -elif ! grep -Eiq '(not found|already[[:space:]]+(stopped|completed)|HTTP[[:space:]]+409|status[[:space:]]+code[[:space:]]+409)' "$pre_status_log"; then +elif ! grep -Eiq '(not found|already[[:space:]]+(stopped|completed)|hydration_failed|HTTP[[:space:]]+409|status[[:space:]]+code[[:space:]]+409)' "$pre_status_log"; then echo "failed to preview Testbox $testbox_id before cleanup; see $pre_status_log" >&2 exit "$pre_status" fi @@ -98,7 +98,7 @@ set -e if (( stop_status != 0 )); then # A completed Testbox can race the explicit stop call. Tolerate only the # documented terminal-state response, never an arbitrary stop failure. - if grep -Eiq 'already[[:space:]]+(stopped|completed)' "$stop_log"; then + if grep -Eiq 'already[[:space:]]+(stopped|completed)|hydration_failed' "$stop_log"; then printf 'stop already reached a terminal state for %s; continuing\n' "$testbox_id" >&2 else echo "failed to stop Testbox $testbox_id; see $stop_log" >&2 @@ -116,14 +116,14 @@ set -e if (( status_status == 0 )); then status_value="$(awk -v id="$testbox_id" '$1 == id { print tolower($2); exit }' "$status_log")" case "$status_value" in - completed|stopped|cancelled|failed|terminated) + completed|stopped|cancelled|failed|terminated|hydration_failed) ;; ready|running|hydrating|in_progress|queued) echo "Testbox $testbox_id is still active after cleanup" >&2 (( cleanup_status == 0 )) && cleanup_status=1 ;; *) - if grep -Eiq 'status:[[:space:]]*(completed|stopped|cancelled|failed|terminated)' "$status_log"; then + if grep -Eiq 'status:[[:space:]]*(completed|stopped|cancelled|failed|terminated|hydration_failed)' "$status_log"; then : else echo "could not establish a terminal status for Testbox $testbox_id; see $status_log" >&2 @@ -131,7 +131,7 @@ if (( status_status == 0 )); then fi ;; esac -elif ! grep -Eiq '(not found|already[[:space:]]+(stopped|completed)|HTTP[[:space:]]+409|status[[:space:]]+code[[:space:]]+409)' "$status_log"; then +elif ! grep -Eiq '(not found|already[[:space:]]+(stopped|completed)|hydration_failed|HTTP[[:space:]]+409|status[[:space:]]+code[[:space:]]+409)' "$status_log"; then echo "failed to inspect Testbox $testbox_id; see $status_log" >&2 (( cleanup_status == 0 )) && cleanup_status=$status_status fi @@ -149,7 +149,7 @@ else "") # The CLI may remove terminal boxes from the inventory immediately. ;; - completed|stopped|cancelled|failed|terminated) + completed|stopped|cancelled|failed|terminated|hydration_failed) # --all is the full inventory, so a terminal row is safe and expected. ;; ready|running|hydrating|in_progress|queued) diff --git a/skills/blacksmith-testbox/SKILL.md b/skills/blacksmith-testbox/SKILL.md index fc4d505ae08..91c5d673201 100644 --- a/skills/blacksmith-testbox/SKILL.md +++ b/skills/blacksmith-testbox/SKILL.md @@ -307,9 +307,10 @@ scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" "$CLEANUP_TOKEN" It records a pre-stop status preview, stop result, post-stop status, and `list --all` output. The preview must match the receipt's workflow, job, and branch before any stop is attempted. It verifies that the specific Testbox ID -is terminal or absent from the active inventory, and accepts only the known race -where stop returns a 409 saying the box is already stopped or completed. Other -stop, status, or list failures remain failures. +is terminal or absent from the active inventory, and accepts the known terminal +states `completed`, `stopped`, `cancelled`, `failed`, `terminated`, and +`hydration_failed`, plus a 409 saying the box is already stopped or completed. +Other stop, status, or list failures remain failures. Put it in an `EXIT` trap that preserves the benchmark's original exit status unless cleanup itself fails. The detailed benchmark writes a receipt and confirmation token for the exact ID returned by warmup; cleanup refuses an ID From dd3765c7900b2c7fc41ac8c2bba7234409624698 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 10:40:27 -0700 Subject: [PATCH 16/38] ci: make Testbox readiness and cleanup explicit --- .../ci-workflow-guard-tests-testbox.yml | 49 +++++++++++++ scripts/blacksmith-testbox-cleanup.sh | 72 +++++++++++-------- skills/blacksmith-testbox/SKILL.md | 21 +++--- skills/blacksmith-testbox/benchmark.md | 25 +++---- 4 files changed, 118 insertions(+), 49 deletions(-) diff --git a/.github/workflows/ci-workflow-guard-tests-testbox.yml b/.github/workflows/ci-workflow-guard-tests-testbox.yml index 7c89e83c047..cb1c3301e77 100644 --- a/.github/workflows/ci-workflow-guard-tests-testbox.yml +++ b/.github/workflows/ci-workflow-guard-tests-testbox.yml @@ -341,3 +341,52 @@ jobs: - name: Run Testbox uses: useblacksmith/run-testbox@3f60ff9ceb2c10c3feefa87dc0c6490cffae059d # v2 + hydration-failure reporting if: always() + + # run-testbox publishes ready best-effort. Only a successful, bounded + # acknowledgement allows a maintainer to consume this box. This check + # runs after the keepalive action only for observability, so it must not + # advertise readiness itself or require the auth token in branch code. + - name: Verify Testbox readiness acknowledgement + if: success() + env: + EXPECTED_TESTBOX_ID: ${{ inputs.testbox_id }} + shell: bash + run: | + set -euo pipefail + state=/tmp/.testbox + api_url="$(<"$state/api_url")" + auth_token="$(<"$state/auth_token")" + runner_host="$(<"$state/runner_host")" + runner_ssh_port="$(<"$state/runner_ssh_port")" + working_directory="$(<"$state/working_directory")" + adopted_run_id="$(<"$state/adopted_run_id")" + installation_model_id="$(<"$state/installation_model_id")" + payload="$(jq -n \ + --arg testbox_id "$EXPECTED_TESTBOX_ID" \ + --arg runner_host "$runner_host" \ + --arg runner_ssh_port "$runner_ssh_port" \ + --arg working_directory "$working_directory" \ + --arg adopted_run_id "$adopted_run_id" \ + --argjson installation_model_id "$installation_model_id" \ + '{testbox_id: $testbox_id, installation_model_id: $installation_model_id, status: "ready", ip_address: $runner_host, ssh_port: $runner_ssh_port, working_directory: $working_directory, adopted_run_id: $adopted_run_id, metadata: {}}')" + response="" + for attempt in 1 2 3 4 5; do + set +e + response="$(curl -fsS --connect-timeout 2 --max-time 10 \ + -X POST "$api_url/api/testbox/phone-home" \ + -H 'Content-Type: application/json' \ + -H "Authorization: Bearer $auth_token" \ + --data "$payload" 2>"$RUNNER_TEMP/testbox-ready-phone-home-$attempt.err")" + phone_home_status=$? + set -e + if (( phone_home_status == 0 )) && [[ -n "$response" ]]; then + printf '%s\n' "$response" >"$RUNNER_TEMP/testbox-ready-phone-home-response.json" + exit 0 + fi + if (( attempt < 5 )); then + sleep $((attempt * 2)) + fi + done + echo "::error::Testbox ready acknowledgement failed after bounded retries" >&2 + cat "$RUNNER_TEMP"/testbox-ready-phone-home-*.err >&2 || true + exit 1 diff --git a/scripts/blacksmith-testbox-cleanup.sh b/scripts/blacksmith-testbox-cleanup.sh index f577cc6975d..79187eff5b3 100755 --- a/scripts/blacksmith-testbox-cleanup.sh +++ b/scripts/blacksmith-testbox-cleanup.sh @@ -1,14 +1,19 @@ #!/usr/bin/env bash set -euo pipefail -if [[ $# -ne 3 ]]; then - echo "usage: $0 " >&2 +if [[ $# -ne 4 ]]; then + echo "usage: $0 " >&2 exit 64 fi testbox_id="$1" evidence_dir="$2" confirmation_token="$3" +operator_confirmation="$4" +if [[ "$operator_confirmation" != "STOP" ]]; then + echo "refusing destructive cleanup without explicit STOP confirmation" >&2 + exit 64 +fi if [[ ! "$testbox_id" =~ ^tbx_[A-Za-z0-9_-]+$ ]]; then echo "invalid Testbox ID: $testbox_id" >&2 exit 64 @@ -90,6 +95,8 @@ stop_log="$evidence_dir/stop.log" status_log="$evidence_dir/status-after-stop.log" list_log="$evidence_dir/list-after-stop.log" cleanup_status=0 +poll_deadline=$((SECONDS + 120)) +poll_attempt=0 set +e blacksmith testbox stop --id "$testbox_id" >"$stop_log" 2>&1 @@ -109,32 +116,41 @@ fi # Status is diagnostic. The authoritative cleanup check below parses the # specific ID in `list --all`; terminal rows are accepted because --all may # retain completed boxes, while active rows remain failures. -set +e -blacksmith testbox status --id "$testbox_id" >"$status_log" 2>&1 -status_status=$? -set -e -if (( status_status == 0 )); then - status_value="$(awk -v id="$testbox_id" '$1 == id { print tolower($2); exit }' "$status_log")" - case "$status_value" in - completed|stopped|cancelled|failed|terminated|hydration_failed) - ;; - ready|running|hydrating|in_progress|queued) - echo "Testbox $testbox_id is still active after cleanup" >&2 - (( cleanup_status == 0 )) && cleanup_status=1 - ;; - *) - if grep -Eiq 'status:[[:space:]]*(completed|stopped|cancelled|failed|terminated|hydration_failed)' "$status_log"; then - : - else - echo "could not establish a terminal status for Testbox $testbox_id; see $status_log" >&2 - (( cleanup_status == 0 )) && cleanup_status=1 - fi - ;; - esac -elif ! grep -Eiq '(not found|already[[:space:]]+(stopped|completed)|hydration_failed|HTTP[[:space:]]+409|status[[:space:]]+code[[:space:]]+409)' "$status_log"; then - echo "failed to inspect Testbox $testbox_id; see $status_log" >&2 - (( cleanup_status == 0 )) && cleanup_status=$status_status -fi +while :; do + poll_attempt=$((poll_attempt + 1)) + : >"$status_log" + set +e + blacksmith testbox status --id "$testbox_id" >"$status_log" 2>&1 + status_status=$? + set -e + terminal=0 + if (( status_status == 0 )); then + status_value="$(awk -v id="$testbox_id" '$1 == id { print tolower($2); exit }' "$status_log")" + case "$status_value" in + completed|stopped|cancelled|failed|terminated|hydration_failed) terminal=1 ;; + ready|running|hydrating|in_progress|queued) ;; + *) + grep -Eiq 'status:[[:space:]]*(completed|stopped|cancelled|failed|terminated|hydration_failed)' "$status_log" && terminal=1 + ;; + esac + elif grep -Eiq '(not found|already[[:space:]]+(stopped|completed)|hydration_failed|HTTP[[:space:]]+409|status[[:space:]]+code[[:space:]]+409)' "$status_log"; then + terminal=1 + else + echo "failed to inspect Testbox $testbox_id; see $status_log" >&2 + (( cleanup_status == 0 )) && cleanup_status=$status_status + break + fi + if (( terminal == 1 )); then + break + fi + if (( SECONDS >= poll_deadline )); then + echo "Testbox $testbox_id is still active after bounded cleanup polling" >&2 + (( cleanup_status == 0 )) && cleanup_status=1 + break + fi + sleep_seconds=$((poll_attempt < 6 ? poll_attempt * 2 : 10)) + sleep "$sleep_seconds" +done set +e blacksmith testbox list --all >"$list_log" 2>&1 diff --git a/skills/blacksmith-testbox/SKILL.md b/skills/blacksmith-testbox/SKILL.md index 91c5d673201..4e183a04c75 100644 --- a/skills/blacksmith-testbox/SKILL.md +++ b/skills/blacksmith-testbox/SKILL.md @@ -292,25 +292,28 @@ refuse to overwrite historical records. Do not add credentials or private keys. ## Fail-safe cleanup Always download before cleanup. Use the checked-in cleanup helper rather than -ignoring errors with `|| true`. Pass the confirmation token generated with the -warmup receipt; never print it: +ignoring errors with `|| true`. After an independent operator decides the exact +box may be destroyed, pass the ownership token generated with the warmup receipt +and the literal `STOP`; never print the token: ```bash CLEANUP_TOKEN="${CLEANUP_TOKEN:-}" [[ "$CLEANUP_TOKEN" =~ ^[0-9a-f]{32}$ ]] || { - echo "use the confirmation token emitted by the warmup receipt" >&2 + echo "use the ownership token emitted by the warmup receipt" >&2 exit 64 } -scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" "$CLEANUP_TOKEN" +scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" "$CLEANUP_TOKEN" STOP ``` It records a pre-stop status preview, stop result, post-stop status, and `list --all` output. The preview must match the receipt's workflow, job, and -branch before any stop is attempted. It verifies that the specific Testbox ID -is terminal or absent from the active inventory, and accepts the known terminal -states `completed`, `stopped`, `cancelled`, `failed`, `terminated`, and -`hydration_failed`, plus a 409 saying the box is already stopped or completed. -Other stop, status, or list failures remain failures. +branch before any stop is attempted. Cleanup is destructive and requires the +separate literal `STOP` operator confirmation in addition to the receipt token. +It verifies that the specific Testbox ID is terminal or absent from the active +inventory, accepts the known terminal states `completed`, `stopped`, `cancelled`, +`failed`, `terminated`, and `hydration_failed`, plus a 409 saying the box is +already stopped or completed, and polls for up to two minutes while cancellation +propagates. Other stop, status, or list failures remain failures. Put it in an `EXIT` trap that preserves the benchmark's original exit status unless cleanup itself fails. The detailed benchmark writes a receipt and confirmation token for the exact ID returned by warmup; cleanup refuses an ID diff --git a/skills/blacksmith-testbox/benchmark.md b/skills/blacksmith-testbox/benchmark.md index b496accdb99..5ae46555eec 100644 --- a/skills/blacksmith-testbox/benchmark.md +++ b/skills/blacksmith-testbox/benchmark.md @@ -145,7 +145,7 @@ cleanup() { trap - EXIT if [[ -n "$TBX" && -n "$cleanup_token" ]]; then set +e - scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" "$cleanup_token" + scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" "$cleanup_token" STOP cleanup_status=$? set -e else @@ -395,10 +395,11 @@ PY ## Cleanup and evidence -Download all raw files and `timings.json` before cleanup. Then call the -fail-safe helper, which previews the exact receipt context, preserves an -already-completed 409, fails on other stop/status/list errors, and verifies this -exact Testbox ID is no longer active: +Download all raw files and `timings.json` before cleanup. Then, after an +operator explicitly decides this exact box may be destroyed, call the fail-safe +helper. It previews the exact receipt context, preserves an already-completed +409, polls cancellation to a bounded deadline, fails on other stop/status/list +errors, and verifies this exact Testbox ID is no longer active: ```bash cleanup_token="${cleanup_token:-}" @@ -406,15 +407,15 @@ cleanup_token="${cleanup_token:-}" echo "use the confirmation token emitted by the warmup receipt" >&2 exit 64 } -scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" "$cleanup_token" +scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" "$cleanup_token" STOP ``` -A shell `EXIT` trap should call that helper while preserving the benchmark -status. Warmup writes `testbox-receipt.json` and a confirmation token bound to -the exact returned ID; cleanup refuses a mismatched ID or token. If warmup -fails before returning an ID, retain before/after inventories but do not -automatically stop a box, because an inventory diff cannot prove ownership -across concurrent operators. Reconcile that orphan manually through the +A shell `EXIT` trap may call that helper only after an independent operator +sets the explicit `STOP` confirmation; preserve the benchmark status. Warmup +writes `testbox-receipt.json` and an ownership token bound to the exact returned +ID; cleanup refuses a mismatched ID or token. If warmup fails before returning +an ID, retain before/after inventories but do not automatically stop a box, +because an inventory diff cannot prove ownership across concurrent operators. Reconcile that orphan manually through the Blacksmith control plane. Keep both inventories and their command statuses, plus warmup/status/identity transcripts, every stage run and download transcript, raw JSON/time/log files, runner catalog, setup identity artifact, From b0c5b80cfbdf11c028f7e8eabb6f007412bb26bf Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 11:00:31 -0700 Subject: [PATCH 17/38] ci: gate Testbox readiness and destructive cleanup --- .../ci-workflow-guard-tests-testbox.yml | 24 +++++++++---------- skills/blacksmith-testbox/benchmark.md | 16 +++++++------ 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci-workflow-guard-tests-testbox.yml b/.github/workflows/ci-workflow-guard-tests-testbox.yml index cb1c3301e77..48b2d1ad9d2 100644 --- a/.github/workflows/ci-workflow-guard-tests-testbox.yml +++ b/.github/workflows/ci-workflow-guard-tests-testbox.yml @@ -334,18 +334,10 @@ jobs: install -m 600 testbox-benchmark/setup-identity.json "$marker" test -s "$marker" - # Always hand control back to Testbox. The pinned action receives the - # final job status and reports hydration_failed, rather than ready, when - # any setup step failed. The setup marker is a second guard for CLI runs. - # Rust builds are issued later with `blacksmith testbox run`. - - name: Run Testbox - uses: useblacksmith/run-testbox@3f60ff9ceb2c10c3feefa87dc0c6490cffae059d # v2 + hydration-failure reporting - if: always() - - # run-testbox publishes ready best-effort. Only a successful, bounded - # acknowledgement allows a maintainer to consume this box. This check - # runs after the keepalive action only for observability, so it must not - # advertise readiness itself or require the auth token in branch code. + # run-testbox publishes ready best-effort and then keeps the VM alive. + # Publish and acknowledge readiness before entering that blocking action; + # if this bounded check fails, the always-run action sees a failed job and + # reports hydration_failed instead of advertising an unusable box. - name: Verify Testbox readiness acknowledgement if: success() env: @@ -390,3 +382,11 @@ jobs: echo "::error::Testbox ready acknowledgement failed after bounded retries" >&2 cat "$RUNNER_TEMP"/testbox-ready-phone-home-*.err >&2 || true exit 1 + + # Always hand control back to Testbox. The pinned action receives the + # final job status and reports hydration_failed, rather than ready, when + # any setup or readiness step failed. Rust builds are issued later with + # `blacksmith testbox run`. + - name: Run Testbox + uses: useblacksmith/run-testbox@3f60ff9ceb2c10c3feefa87dc0c6490cffae059d # v2 + hydration-failure reporting + if: always() diff --git a/skills/blacksmith-testbox/benchmark.md b/skills/blacksmith-testbox/benchmark.md index 5ae46555eec..a3aab044ee4 100644 --- a/skills/blacksmith-testbox/benchmark.md +++ b/skills/blacksmith-testbox/benchmark.md @@ -143,7 +143,7 @@ cleanup() { local cleanup_status=0 local after_list_status=125 trap - EXIT - if [[ -n "$TBX" && -n "$cleanup_token" ]]; then + if [[ -n "$TBX" && -n "$cleanup_token" && "${CONFIRM_TESTBOX_STOP:-}" == "STOP" ]]; then set +e scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" "$cleanup_token" STOP cleanup_status=$? @@ -410,12 +410,14 @@ cleanup_token="${cleanup_token:-}" scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" "$cleanup_token" STOP ``` -A shell `EXIT` trap may call that helper only after an independent operator -sets the explicit `STOP` confirmation; preserve the benchmark status. Warmup -writes `testbox-receipt.json` and an ownership token bound to the exact returned -ID; cleanup refuses a mismatched ID or token. If warmup fails before returning -an ID, retain before/after inventories but do not automatically stop a box, -because an inventory diff cannot prove ownership across concurrent operators. Reconcile that orphan manually through the +A shell `EXIT` trap may call that helper only when an independent operator has +exported `CONFIRM_TESTBOX_STOP=STOP`; otherwise it preserves the benchmark +status, records inventory, and leaves the box for explicit manual cleanup. +Warmup writes `testbox-receipt.json` and an ownership token bound to the exact +returned ID; cleanup refuses a mismatched ID or token. If warmup fails before +returning an ID, retain before/after inventories but do not automatically stop a +box, because an inventory diff cannot prove ownership across concurrent +operators. Reconcile that orphan manually through the Blacksmith control plane. Keep both inventories and their command statuses, plus warmup/status/identity transcripts, every stage run and download transcript, raw JSON/time/log files, runner catalog, setup identity artifact, From 4a05ae642c9e7231ea992d7c797b44a3453f072e Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 11:15:41 -0700 Subject: [PATCH 18/38] ci: verify Testbox ownership before cleanup --- .../ci-workflow-guard-tests-testbox.yml | 10 ++++-- scripts/blacksmith-testbox-cleanup.sh | 32 +++++++++++++------ skills/blacksmith-testbox/SKILL.md | 22 +++++++------ skills/blacksmith-testbox/benchmark.md | 2 +- 4 files changed, 43 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ci-workflow-guard-tests-testbox.yml b/.github/workflows/ci-workflow-guard-tests-testbox.yml index 48b2d1ad9d2..0af03a4216e 100644 --- a/.github/workflows/ci-workflow-guard-tests-testbox.yml +++ b/.github/workflows/ci-workflow-guard-tests-testbox.yml @@ -16,7 +16,8 @@ on: # The Blacksmith CLI supplies testbox_id and dispatches the selected pushed # branch. It does not expose arbitrary workflow inputs, so source_sha is an # optional manual-dispatch assertion; the benchmark's remote guard is required -# for the CLI path below. +# for the CLI path below. The protected environment is configured out of band; +# this workflow is intentionally unusable unless that repository policy exists. permissions: {} concurrency: @@ -38,8 +39,11 @@ jobs: contents: read timeout-minutes: 45 steps: - # begin-testbox must be the first step. It attaches the VM requested by - # `blacksmith testbox warmup` and leaves it alive after this setup job. + # begin-testbox must be the first step. GitHub environment approval and + # its trusted-branch policy are evaluated before this step; do not weaken + # those external controls or move this action into branch-controlled code. + # It attaches the VM requested by `blacksmith testbox warmup` and leaves + # it alive after this setup job. - name: Begin Testbox uses: useblacksmith/begin-testbox@233448af4bfdc6fca509a7f0974411ac6d8a8043 # v2 with: diff --git a/scripts/blacksmith-testbox-cleanup.sh b/scripts/blacksmith-testbox-cleanup.sh index 79187eff5b3..e4b7681f34d 100755 --- a/scripts/blacksmith-testbox-cleanup.sh +++ b/scripts/blacksmith-testbox-cleanup.sh @@ -70,21 +70,33 @@ PY )" pre_status_log="$evidence_dir/status-before-stop.log" set +e -blacksmith testbox status --id "$testbox_id" >"$pre_status_log" 2>&1 +blacksmith testbox list --all >"$pre_status_log" 2>&1 pre_status=$? set -e if (( pre_status == 0 )); then pre_row="$(awk -v id="$testbox_id" '$1 == id { print; exit }' "$pre_status_log")" if [[ -z "$pre_row" ]]; then - echo "status did not contain the owned Testbox row; refusing cleanup" >&2 - exit 66 - fi - pre_workflow="$(awk '{print $4}' <<<"$pre_row")" - pre_job="$(awk '{print $5}' <<<"$pre_row")" - pre_ref="$(awk '{print $6}' <<<"$pre_row")" - if [[ "$pre_workflow" != "$receipt_workflow" || "$pre_job" != "$receipt_job" || "$pre_ref" != "$receipt_ref" ]]; then - echo "owned Testbox context differs from the warmup receipt; refusing cleanup" >&2 - exit 66 + # --all can omit a terminal box. A missing row is safe only when the + # single-box status endpoint independently reports a known terminal state. + set +e + blacksmith testbox status --id "$testbox_id" >>"$pre_status_log" 2>&1 + pre_lookup_status=$? + set -e + if (( pre_lookup_status == 0 )) && ! grep -Eiq '(completed|stopped|cancelled|failed|terminated|hydration_failed)' "$pre_status_log"; then + echo "inventory omitted an active or unknown Testbox; refusing cleanup" >&2 + exit 66 + elif (( pre_lookup_status != 0 )) && ! grep -Eiq '(not found|already[[:space:]]+(stopped|completed)|hydration_failed|HTTP[[:space:]]+409|status[[:space:]]+code[[:space:]]+409)' "$pre_status_log"; then + echo "could not establish ownership before cleanup; refusing cleanup" >&2 + exit "$pre_lookup_status" + fi + else + pre_workflow="$(awk '{print $4}' <<<"$pre_row")" + pre_job="$(awk '{print $5}' <<<"$pre_row")" + pre_ref="$(awk '{print $6}' <<<"$pre_row")" + if [[ "$pre_workflow" != "$receipt_workflow" || "$pre_job" != "$receipt_job" || "$pre_ref" != "$receipt_ref" ]]; then + echo "owned Testbox context differs from the warmup receipt; refusing cleanup" >&2 + exit 66 + fi fi elif ! grep -Eiq '(not found|already[[:space:]]+(stopped|completed)|hydration_failed|HTTP[[:space:]]+409|status[[:space:]]+code[[:space:]]+409)' "$pre_status_log"; then echo "failed to preview Testbox $testbox_id before cleanup; see $pre_status_log" >&2 diff --git a/skills/blacksmith-testbox/SKILL.md b/skills/blacksmith-testbox/SKILL.md index 4e183a04c75..751a4f3a935 100644 --- a/skills/blacksmith-testbox/SKILL.md +++ b/skills/blacksmith-testbox/SKILL.md @@ -35,12 +35,15 @@ workflow intentionally grants no other GitHub permissions or workflow secrets. Before using the lane, a repository administrator must create the `blacksmith-testbox-trusted` GitHub environment, configure required reviewers -(or an equivalent manual approval rule), and leave the environment secret set -empty. GitHub evaluates that approval before the job's first step, including -`begin-testbox`. If the environment does not exist or has no required reviewer, -stop: the lane is not production-safe. Never dispatch it for an untrusted PR, -fork, branch containing unreviewed workflow/helper changes, or source supplied -by an external contributor. When trust changes, stop the old box and warm a +(or an equivalent manual approval rule), disable administrator bypass, and leave +the environment secret set empty. GitHub evaluates that approval before the +job's first step, including `begin-testbox`. The workflow cannot verify this +out-of-band policy after `begin-testbox` without exposing the same token, so a +missing or drifting environment is an operational stop condition, not a +recoverable workflow state. If the environment does not exist or has no +required reviewer, stop: the lane is not production-safe. Never dispatch it +for an untrusted PR, fork, branch containing unreviewed workflow/helper changes, +or source supplied by an external contributor. When trust changes, stop the old box and warm a fresh one. The helper retains the `CMUX_TESTBOX_REMOTE=1` guard for accidental local @@ -314,9 +317,10 @@ inventory, accepts the known terminal states `completed`, `stopped`, `cancelled` `failed`, `terminated`, and `hydration_failed`, plus a 409 saying the box is already stopped or completed, and polls for up to two minutes while cancellation propagates. Other stop, status, or list failures remain failures. -Put it in an `EXIT` trap that preserves the benchmark's original exit status -unless cleanup itself fails. The detailed benchmark writes a receipt and -confirmation token for the exact ID returned by warmup; cleanup refuses an ID +Put it in an `EXIT` trap only after an independent operator exports +`CONFIRM_TESTBOX_STOP=STOP`; otherwise preserve the benchmark's original exit +status and leave the box for manual cleanup. The detailed benchmark writes a +receipt and ownership token for the exact ID returned by warmup; cleanup refuses an ID or token that is not bound to that receipt. If warmup fails before returning an ID, retain before/after inventory but do not automatically stop a box, because an inventory diff cannot prove ownership across concurrent operators. Reconcile diff --git a/skills/blacksmith-testbox/benchmark.md b/skills/blacksmith-testbox/benchmark.md index a3aab044ee4..7da51ac38ce 100644 --- a/skills/blacksmith-testbox/benchmark.md +++ b/skills/blacksmith-testbox/benchmark.md @@ -163,7 +163,7 @@ cleanup() { echo "warmup returned no owned Testbox receipt; no automatic stop was attempted" >&2 fi fi - if (( result == 0 && cleanup_status != 0 )); then + if (( result == 0 && cleanup_status != 0 && "${CONFIRM_TESTBOX_STOP:-}" == "STOP" )); then result="$cleanup_status" fi exit "$result" From 1c95667efb71544da47ab9f5fadb01e5f934997e Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 11:36:43 -0700 Subject: [PATCH 19/38] ci: restrict Testbox token setup to main --- .../workflows/ci-workflow-guard-tests-testbox.yml | 7 +++++++ skills/blacksmith-testbox/SKILL.md | 15 +++++++++------ skills/blacksmith-testbox/benchmark.md | 2 +- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci-workflow-guard-tests-testbox.yml b/.github/workflows/ci-workflow-guard-tests-testbox.yml index 0af03a4216e..e51c732ee7e 100644 --- a/.github/workflows/ci-workflow-guard-tests-testbox.yml +++ b/.github/workflows/ci-workflow-guard-tests-testbox.yml @@ -44,6 +44,13 @@ jobs: # those external controls or move this action into branch-controlled code. # It attaches the VM requested by `blacksmith testbox warmup` and leaves # it alive after this setup job. + - name: Guard trusted source ref + if: github.ref != 'refs/heads/main' + shell: bash + run: | + echo "::error::Testbox token-bearing setup is restricted to reviewed main" >&2 + exit 1 + - name: Begin Testbox uses: useblacksmith/begin-testbox@233448af4bfdc6fca509a7f0974411ac6d8a8043 # v2 with: diff --git a/skills/blacksmith-testbox/SKILL.md b/skills/blacksmith-testbox/SKILL.md index 751a4f3a935..5de9458c9fe 100644 --- a/skills/blacksmith-testbox/SKILL.md +++ b/skills/blacksmith-testbox/SKILL.md @@ -37,10 +37,11 @@ Before using the lane, a repository administrator must create the `blacksmith-testbox-trusted` GitHub environment, configure required reviewers (or an equivalent manual approval rule), disable administrator bypass, and leave the environment secret set empty. GitHub evaluates that approval before the -job's first step, including `begin-testbox`. The workflow cannot verify this -out-of-band policy after `begin-testbox` without exposing the same token, so a -missing or drifting environment is an operational stop condition, not a -recoverable workflow state. If the environment does not exist or has no +job's first step, including `begin-testbox`. The workflow additionally rejects every ref except reviewed `main` before +`begin-testbox`. It cannot verify the environment's reviewer configuration +inside the token-bearing job without exposing the same token, so a missing or +drifting environment remains an operational stop condition, not a recoverable +workflow state. If the environment does not exist or has no required reviewer, stop: the lane is not production-safe. Never dispatch it for an untrusted PR, fork, branch containing unreviewed workflow/helper changes, or source supplied by an external contributor. When trust changes, stop the old box and warm a @@ -53,8 +54,10 @@ controlled and is not an authentication mechanism. The protected environment and trusted-maintainer policy are the security boundary. Verify before every use that `blacksmith-testbox-trusted` still has required reviewers, no secrets, administrator bypass disabled, and no broad branch policy admitting unreviewed -refs. The workflow cannot manufacture those repository settings, so drift makes -the lane unavailable rather than safe. +refs. The checked-in workflow independently permits only reviewed `main`; the +feature branch is for code review and cannot expose a Testbox token. The +workflow cannot manufacture the environment settings, so drift makes the lane +unavailable rather than safe. * Never run `cargo`, `rustc`, `rustup`, `zig build`, or another Rust/Zig build command on Lawrence's Mac. This includes local fallback builds and local diff --git a/skills/blacksmith-testbox/benchmark.md b/skills/blacksmith-testbox/benchmark.md index 7da51ac38ce..6c2f38bb2b1 100644 --- a/skills/blacksmith-testbox/benchmark.md +++ b/skills/blacksmith-testbox/benchmark.md @@ -163,7 +163,7 @@ cleanup() { echo "warmup returned no owned Testbox receipt; no automatic stop was attempted" >&2 fi fi - if (( result == 0 && cleanup_status != 0 && "${CONFIRM_TESTBOX_STOP:-}" == "STOP" )); then + if (( result == 0 && cleanup_status != 0 )) && [[ "${CONFIRM_TESTBOX_STOP:-}" == "STOP" ]]; then result="$cleanup_status" fi exit "$result" From 1dd0e2a9c195e3ce0a4f4479a85af89863439257 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 11:49:33 -0700 Subject: [PATCH 20/38] ci: validate Testbox inputs before cleanup --- .../ci-workflow-guard-tests-testbox.yml | 11 ++++++++ scripts/blacksmith-testbox-cleanup.sh | 27 +++++++++++++++---- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-workflow-guard-tests-testbox.yml b/.github/workflows/ci-workflow-guard-tests-testbox.yml index e51c732ee7e..9b9ba4ea7fb 100644 --- a/.github/workflows/ci-workflow-guard-tests-testbox.yml +++ b/.github/workflows/ci-workflow-guard-tests-testbox.yml @@ -51,6 +51,17 @@ jobs: echo "::error::Testbox token-bearing setup is restricted to reviewed main" >&2 exit 1 + - name: Validate Testbox ID before token-bearing action + env: + TESTBOX_ID: ${{ inputs.testbox_id }} + shell: bash + run: | + set -euo pipefail + [[ "$TESTBOX_ID" =~ ^tbx_[A-Za-z0-9_-]+$ ]] || { + echo "::error::malformed Testbox ID" >&2 + exit 1 + } + - name: Begin Testbox uses: useblacksmith/begin-testbox@233448af4bfdc6fca509a7f0974411ac6d8a8043 # v2 with: diff --git a/scripts/blacksmith-testbox-cleanup.sh b/scripts/blacksmith-testbox-cleanup.sh index e4b7681f34d..97eeb33908b 100755 --- a/scripts/blacksmith-testbox-cleanup.sh +++ b/scripts/blacksmith-testbox-cleanup.sh @@ -78,14 +78,31 @@ if (( pre_status == 0 )); then if [[ -z "$pre_row" ]]; then # --all can omit a terminal box. A missing row is safe only when the # single-box status endpoint independently reports a known terminal state. + pre_lookup_log="$evidence_dir/status-before-stop-id.log" set +e - blacksmith testbox status --id "$testbox_id" >>"$pre_status_log" 2>&1 + blacksmith testbox status --id "$testbox_id" >"$pre_lookup_log" 2>&1 pre_lookup_status=$? set -e - if (( pre_lookup_status == 0 )) && ! grep -Eiq '(completed|stopped|cancelled|failed|terminated|hydration_failed)' "$pre_status_log"; then - echo "inventory omitted an active or unknown Testbox; refusing cleanup" >&2 - exit 66 - elif (( pre_lookup_status != 0 )) && ! grep -Eiq '(not found|already[[:space:]]+(stopped|completed)|hydration_failed|HTTP[[:space:]]+409|status[[:space:]]+code[[:space:]]+409)' "$pre_status_log"; then + if (( pre_lookup_status == 0 )); then + pre_lookup_row="$(awk -v id="$testbox_id" '$1 == id { print; exit }' "$pre_lookup_log")" + if [[ -z "$pre_lookup_row" ]]; then + echo "single-box status omitted the owned Testbox; refusing cleanup" >&2 + exit 66 + fi + pre_lookup_status_value="$(awk '{print tolower($2)}' <<<"$pre_lookup_row")" + case "$pre_lookup_status_value" in + completed|stopped|cancelled|failed|terminated|hydration_failed) + ;; + ready|running|hydrating|in_progress|queued) + echo "owned Testbox is active but absent from inventory; refusing cleanup" >&2 + exit 66 + ;; + *) + echo "could not establish the owned Testbox terminal state; refusing cleanup" >&2 + exit 66 + ;; + esac + elif ! grep -Eiq '(not found|already[[:space:]]+(stopped|completed)|hydration_failed|HTTP[[:space:]]+409|status[[:space:]]+code[[:space:]]+409)' "$pre_lookup_log"; then echo "could not establish ownership before cleanup; refusing cleanup" >&2 exit "$pre_lookup_status" fi From 7e1e7cc79b6c146336da8be55a0803ab9d8276f5 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 12:00:46 -0700 Subject: [PATCH 21/38] ci: align Testbox cleanup with CLI inventory --- scripts/blacksmith-testbox-cleanup.sh | 16 ++++++++++------ skills/blacksmith-testbox/benchmark.md | 15 ++++++++++----- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/scripts/blacksmith-testbox-cleanup.sh b/scripts/blacksmith-testbox-cleanup.sh index 97eeb33908b..a16337b252e 100755 --- a/scripts/blacksmith-testbox-cleanup.sh +++ b/scripts/blacksmith-testbox-cleanup.sh @@ -107,9 +107,12 @@ if (( pre_status == 0 )); then exit "$pre_lookup_status" fi else - pre_workflow="$(awk '{print $4}' <<<"$pre_row")" - pre_job="$(awk '{print $5}' <<<"$pre_row")" - pre_ref="$(awk '{print $6}' <<<"$pre_row")" + # `list --all` currently exposes ID, STATUS, IP, WORKFLOW, JOB, REF, ...; + # derive the ownership fields relative to the stable ID/status columns and + # verify them against the receipt before issuing a destructive stop. + pre_workflow="$(awk '{print $5}' <<<"$pre_row")" + pre_job="$(awk '{print $6}' <<<"$pre_row")" + pre_ref="$(awk '{print $7}' <<<"$pre_row")" if [[ "$pre_workflow" != "$receipt_workflow" || "$pre_job" != "$receipt_job" || "$pre_ref" != "$receipt_ref" ]]; then echo "owned Testbox context differs from the warmup receipt; refusing cleanup" >&2 exit 66 @@ -132,9 +135,10 @@ blacksmith testbox stop --id "$testbox_id" >"$stop_log" 2>&1 stop_status=$? set -e if (( stop_status != 0 )); then - # A completed Testbox can race the explicit stop call. Tolerate only the - # documented terminal-state response, never an arbitrary stop failure. - if grep -Eiq 'already[[:space:]]+(stopped|completed)|hydration_failed' "$stop_log"; then + # A completed or already-absent Testbox can race the explicit stop call. + # Tolerate only the documented terminal/not-found response, never an + # arbitrary stop failure. + if grep -Eiq '(already[[:space:]]+(stopped|completed)|hydration_failed|not found|HTTP[[:space:]]+404)' "$stop_log"; then printf 'stop already reached a terminal state for %s; continuing\n' "$testbox_id" >&2 else echo "failed to stop Testbox $testbox_id; see $stop_log" >&2 diff --git a/skills/blacksmith-testbox/benchmark.md b/skills/blacksmith-testbox/benchmark.md index 6c2f38bb2b1..fa24131efee 100644 --- a/skills/blacksmith-testbox/benchmark.md +++ b/skills/blacksmith-testbox/benchmark.md @@ -51,6 +51,10 @@ set -euo pipefail git submodule update --init ghostty cd "$(git rev-parse --show-toplevel)" SOURCE_REF="$(git symbolic-ref --short HEAD)" +[[ "$SOURCE_REF" == "main" ]] || { + echo "the token-bearing Testbox lane only accepts reviewed main; use a PR workflow for feature branches" >&2 + exit 1 +} if [[ ! "$SOURCE_REF" =~ ^[A-Za-z0-9._/-]+$ || "$SOURCE_REF" == *..* || "$SOURCE_REF" == */ || "$SOURCE_REF" == *//* ]]; then echo "HEAD must name a supported pushed branch ref" >&2 exit 1 @@ -263,11 +267,12 @@ if (( status_ready != 0 )); then fi ``` -The workflow validates that the dispatch ref is a pushed branch, that -`github.sha` is a full SHA, and that the branch still resolves to that SHA. If -a direct GitHub dispatch supplies the optional `source_sha` input, it must equal -`github.sha`. The Blacksmith CLI path relies on the same full SHA passed to the -remote helper because the CLI only supplies `testbox_id` to workflow inputs. +The workflow validates that the dispatch ref is reviewed `main`, is a pushed +branch, that `github.sha` is a full SHA, and that the branch still resolves to +that SHA. If a direct GitHub dispatch supplies the optional `source_sha` input, +it must equal `github.sha`. The Blacksmith CLI path relies on the same full SHA +passed to the remote helper because the CLI only supplies `testbox_id` to +workflow inputs. The workflow concurrency group serializes setup requests by Testbox ID, even when source SHAs differ. The remote `flock` begins after Blacksmith's rsync, so it protects stage/build/artifact writes only. Blacksmith exposes no pre-rsync From 3c283ab6d20fbb1586f16e7c0825d52de9794640 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 12:10:56 -0700 Subject: [PATCH 22/38] ci: make Testbox cleanup status-specific --- .../ci-workflow-guard-tests-testbox.yml | 5 +- scripts/blacksmith-testbox-cleanup.sh | 98 ++++++++++--------- skills/blacksmith-testbox/SKILL.md | 4 + 3 files changed, 58 insertions(+), 49 deletions(-) diff --git a/.github/workflows/ci-workflow-guard-tests-testbox.yml b/.github/workflows/ci-workflow-guard-tests-testbox.yml index 9b9ba4ea7fb..dfcc3821e3c 100644 --- a/.github/workflows/ci-workflow-guard-tests-testbox.yml +++ b/.github/workflows/ci-workflow-guard-tests-testbox.yml @@ -37,7 +37,10 @@ jobs: name: blacksmith-testbox-trusted permissions: contents: read - timeout-minutes: 45 + # Hydration plus three sequential 20-minute bounded remote builds happen + # after setup. Keep the GitHub job alive long enough for all stages and + # cleanup; the Testbox itself still has its separate idle timeout. + timeout-minutes: 120 steps: # begin-testbox must be the first step. GitHub environment approval and # its trusted-branch policy are evaluated before this step; do not weaken diff --git a/scripts/blacksmith-testbox-cleanup.sh b/scripts/blacksmith-testbox-cleanup.sh index a16337b252e..6b45a06be11 100755 --- a/scripts/blacksmith-testbox-cleanup.sh +++ b/scripts/blacksmith-testbox-cleanup.sh @@ -68,58 +68,55 @@ import sys print(json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8"))["source_ref"]) PY )" +inventory_log="$evidence_dir/list-before-stop.log" +set +e +blacksmith testbox list --all >"$inventory_log" 2>&1 +inventory_status=$? +set -e +if (( inventory_status != 0 )); then + echo "failed to capture the Testbox inventory before cleanup; refusing stop" >&2 + exit "$inventory_status" +fi + +# `status --id` is the ownership source of truth. Its row has the stable +# columns ID STATUS IP WORKFLOW JOB REF, so compare only this exact ID with the +# warmup receipt. Never infer ownership from another row in --all output. pre_status_log="$evidence_dir/status-before-stop.log" set +e -blacksmith testbox list --all >"$pre_status_log" 2>&1 +blacksmith testbox status --id "$testbox_id" >"$pre_status_log" 2>&1 pre_status=$? set -e +skip_stop=0 if (( pre_status == 0 )); then pre_row="$(awk -v id="$testbox_id" '$1 == id { print; exit }' "$pre_status_log")" if [[ -z "$pre_row" ]]; then - # --all can omit a terminal box. A missing row is safe only when the - # single-box status endpoint independently reports a known terminal state. - pre_lookup_log="$evidence_dir/status-before-stop-id.log" - set +e - blacksmith testbox status --id "$testbox_id" >"$pre_lookup_log" 2>&1 - pre_lookup_status=$? - set -e - if (( pre_lookup_status == 0 )); then - pre_lookup_row="$(awk -v id="$testbox_id" '$1 == id { print; exit }' "$pre_lookup_log")" - if [[ -z "$pre_lookup_row" ]]; then - echo "single-box status omitted the owned Testbox; refusing cleanup" >&2 - exit 66 - fi - pre_lookup_status_value="$(awk '{print tolower($2)}' <<<"$pre_lookup_row")" - case "$pre_lookup_status_value" in - completed|stopped|cancelled|failed|terminated|hydration_failed) - ;; - ready|running|hydrating|in_progress|queued) - echo "owned Testbox is active but absent from inventory; refusing cleanup" >&2 - exit 66 - ;; - *) - echo "could not establish the owned Testbox terminal state; refusing cleanup" >&2 - exit 66 - ;; - esac - elif ! grep -Eiq '(not found|already[[:space:]]+(stopped|completed)|hydration_failed|HTTP[[:space:]]+409|status[[:space:]]+code[[:space:]]+409)' "$pre_lookup_log"; then - echo "could not establish ownership before cleanup; refusing cleanup" >&2 - exit "$pre_lookup_status" - fi - else - # `list --all` currently exposes ID, STATUS, IP, WORKFLOW, JOB, REF, ...; - # derive the ownership fields relative to the stable ID/status columns and - # verify them against the receipt before issuing a destructive stop. - pre_workflow="$(awk '{print $5}' <<<"$pre_row")" - pre_job="$(awk '{print $6}' <<<"$pre_row")" - pre_ref="$(awk '{print $7}' <<<"$pre_row")" - if [[ "$pre_workflow" != "$receipt_workflow" || "$pre_job" != "$receipt_job" || "$pre_ref" != "$receipt_ref" ]]; then - echo "owned Testbox context differs from the warmup receipt; refusing cleanup" >&2 - exit 66 - fi + echo "status did not contain the owned Testbox row; refusing cleanup" >&2 + exit 66 fi -elif ! grep -Eiq '(not found|already[[:space:]]+(stopped|completed)|hydration_failed|HTTP[[:space:]]+409|status[[:space:]]+code[[:space:]]+409)' "$pre_status_log"; then - echo "failed to preview Testbox $testbox_id before cleanup; see $pre_status_log" >&2 + pre_status_value="$(awk '{print tolower($2)}' <<<"$pre_row")" + pre_workflow="$(awk '{print $4}' <<<"$pre_row")" + pre_job="$(awk '{print $5}' <<<"$pre_row")" + pre_ref="$(awk '{print $6}' <<<"$pre_row")" + if [[ "$pre_workflow" != "$receipt_workflow" || "$pre_job" != "$receipt_job" || "$pre_ref" != "$receipt_ref" ]]; then + echo "owned Testbox context differs from the warmup receipt; refusing cleanup" >&2 + exit 66 + fi + case "$pre_status_value" in + completed|stopped|cancelled|failed|terminated|hydration_failed) + skip_stop=1 + ;; + ready|running|hydrating|in_progress|queued) + ;; + *) + echo "unknown status for owned Testbox $testbox_id; refusing cleanup" >&2 + exit 66 + ;; + esac +elif grep -Eiq '(not found|already[[:space:]]+(stopped|completed)|hydration_failed|HTTP[[:space:]]+409|status[[:space:]]+code[[:space:]]+409)' "$pre_status_log"; then + # An ID-specific terminal/not-found response proves there is nothing to stop. + skip_stop=1 +else + echo "failed to inspect owned Testbox $testbox_id before cleanup; refusing stop" >&2 exit "$pre_status" fi @@ -130,10 +127,15 @@ cleanup_status=0 poll_deadline=$((SECONDS + 120)) poll_attempt=0 -set +e -blacksmith testbox stop --id "$testbox_id" >"$stop_log" 2>&1 -stop_status=$? -set -e +if (( skip_stop == 1 )); then + printf 'Testbox %s is already terminal or absent; no stop request needed\n' "$testbox_id" >"$stop_log" + stop_status=0 +else + set +e + blacksmith testbox stop --id "$testbox_id" >"$stop_log" 2>&1 + stop_status=$? + set -e +fi if (( stop_status != 0 )); then # A completed or already-absent Testbox can race the explicit stop call. # Tolerate only the documented terminal/not-found response, never an diff --git a/skills/blacksmith-testbox/SKILL.md b/skills/blacksmith-testbox/SKILL.md index 5de9458c9fe..b278ce2567c 100644 --- a/skills/blacksmith-testbox/SKILL.md +++ b/skills/blacksmith-testbox/SKILL.md @@ -103,6 +103,10 @@ set -euo pipefail git submodule update --init ghostty cd "$(git rev-parse --show-toplevel)" SOURCE_REF="$(git symbolic-ref --short HEAD)" +[[ "$SOURCE_REF" == "main" ]] || { + echo "the token-bearing Testbox lane only accepts reviewed main; use a PR workflow for feature branches" >&2 + exit 1 +} if [[ ! "$SOURCE_REF" =~ ^[A-Za-z0-9._/-]+$ || "$SOURCE_REF" == *..* || "$SOURCE_REF" == */ || "$SOURCE_REF" == *//* ]]; then echo "HEAD must name a supported pushed branch ref" >&2 exit 1 From 4465525ebb768dccd9b3f82eeaddbf1517dc28ec Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 12:33:13 -0700 Subject: [PATCH 23/38] ci: parse Testbox status formats safely --- scripts/blacksmith-testbox-cleanup.sh | 60 ++++++++++++++++++++------- 1 file changed, 45 insertions(+), 15 deletions(-) diff --git a/scripts/blacksmith-testbox-cleanup.sh b/scripts/blacksmith-testbox-cleanup.sh index 6b45a06be11..00276063829 100755 --- a/scripts/blacksmith-testbox-cleanup.sh +++ b/scripts/blacksmith-testbox-cleanup.sh @@ -78,37 +78,67 @@ if (( inventory_status != 0 )); then exit "$inventory_status" fi -# `status --id` is the ownership source of truth. Its row has the stable -# columns ID STATUS IP WORKFLOW JOB REF, so compare only this exact ID with the -# warmup receipt. Never infer ownership from another row in --all output. +# The CLI has emitted both table-shaped and summary-shaped `status --id` +# responses over time. Extract state without treating either response as an +# ownership table. Ownership context comes only from the exact ID row in the +# successful inventory preview below. +status_from_log() { + local log_path="$1" + local value + value="$(awk -v id="$testbox_id" '$1 == id { print tolower($2); exit }' "$log_path")" + if [[ -z "$value" ]]; then + value="$(awk '{ + for (i = 1; i <= NF; i++) { + token = tolower($i) + if (token == "status:" && i < NF) { + print tolower($(i + 1)) + exit + } + if (token ~ /^status:/) { + sub(/^status:/, "", token) + if (token != "") { + print token + exit + } + } + } + }' "$log_path")" + fi + printf '%s' "$value" +} + +inventory_row="$(awk -v id="$testbox_id" '$1 == id { print; exit }' "$inventory_log")" pre_status_log="$evidence_dir/status-before-stop.log" set +e blacksmith testbox status --id "$testbox_id" >"$pre_status_log" 2>&1 pre_status=$? set -e skip_stop=0 -if (( pre_status == 0 )); then - pre_row="$(awk -v id="$testbox_id" '$1 == id { print; exit }' "$pre_status_log")" - if [[ -z "$pre_row" ]]; then - echo "status did not contain the owned Testbox row; refusing cleanup" >&2 - exit 66 - fi - pre_status_value="$(awk '{print tolower($2)}' <<<"$pre_row")" - pre_workflow="$(awk '{print $4}' <<<"$pre_row")" - pre_job="$(awk '{print $5}' <<<"$pre_row")" - pre_ref="$(awk '{print $6}' <<<"$pre_row")" +if [[ -n "$inventory_row" ]]; then + # `list --all` is ID STATUS IP WORKFLOW JOB REF ...; compare this exact row + # to the warmup receipt before any destructive operation. + pre_workflow="$(awk '{print $4}' <<<"$inventory_row")" + pre_job="$(awk '{print $5}' <<<"$inventory_row")" + pre_ref="$(awk '{print $6}' <<<"$inventory_row")" if [[ "$pre_workflow" != "$receipt_workflow" || "$pre_job" != "$receipt_job" || "$pre_ref" != "$receipt_ref" ]]; then echo "owned Testbox context differs from the warmup receipt; refusing cleanup" >&2 exit 66 fi +fi +if (( pre_status == 0 )); then + pre_status_value="$(status_from_log "$pre_status_log")" case "$pre_status_value" in completed|stopped|cancelled|failed|terminated|hydration_failed) skip_stop=1 ;; ready|running|hydrating|in_progress|queued) + [[ -n "$inventory_row" ]] || { + echo "owned Testbox is active but absent from inventory; refusing cleanup" >&2 + exit 66 + } ;; *) - echo "unknown status for owned Testbox $testbox_id; refusing cleanup" >&2 + echo "could not parse status for owned Testbox $testbox_id; refusing cleanup" >&2 exit 66 ;; esac @@ -160,7 +190,7 @@ while :; do set -e terminal=0 if (( status_status == 0 )); then - status_value="$(awk -v id="$testbox_id" '$1 == id { print tolower($2); exit }' "$status_log")" + status_value="$(status_from_log "$status_log")" case "$status_value" in completed|stopped|cancelled|failed|terminated|hydration_failed) terminal=1 ;; ready|running|hydrating|in_progress|queued) ;; From 9fc1b553148fa09306089ba5fc1f225633d5ffab Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 12:46:26 -0700 Subject: [PATCH 24/38] ci: require a current Testbox cleanup preview --- scripts/blacksmith-testbox-cleanup.sh | 334 ++++++++++++++++---------- 1 file changed, 213 insertions(+), 121 deletions(-) diff --git a/scripts/blacksmith-testbox-cleanup.sh b/scripts/blacksmith-testbox-cleanup.sh index 00276063829..0fc76408fe3 100755 --- a/scripts/blacksmith-testbox-cleanup.sh +++ b/scripts/blacksmith-testbox-cleanup.sh @@ -2,24 +2,24 @@ set -euo pipefail if [[ $# -ne 4 ]]; then - echo "usage: $0 " >&2 + echo "usage: $0 " >&2 exit 64 fi testbox_id="$1" evidence_dir="$2" -confirmation_token="$3" +ownership_token="$3" operator_confirmation="$4" -if [[ "$operator_confirmation" != "STOP" ]]; then - echo "refusing destructive cleanup without explicit STOP confirmation" >&2 - exit 64 -fi if [[ ! "$testbox_id" =~ ^tbx_[A-Za-z0-9_-]+$ ]]; then echo "invalid Testbox ID: $testbox_id" >&2 exit 64 fi -if [[ ! "$confirmation_token" =~ ^[0-9a-f]{32}$ ]]; then - echo "confirmation token must be a 32-character lowercase hex value" >&2 +if [[ ! "$ownership_token" =~ ^[0-9a-f]{32}$ ]]; then + echo "ownership token must be a 32-character lowercase hex value" >&2 + exit 64 +fi +if [[ "$operator_confirmation" != "PREVIEW" && ! "$operator_confirmation" =~ ^STOP:[0-9a-f]{64}$ ]]; then + echo "confirmation must be PREVIEW or STOP:<64-character preview SHA>" >&2 exit 64 fi @@ -29,7 +29,7 @@ if [[ ! -s "$receipt_path" ]]; then echo "refusing cleanup without the warmup ownership receipt: $receipt_path" >&2 exit 65 fi -python3 - "$receipt_path" "$testbox_id" "$confirmation_token" <<'PY' +python3 - "$receipt_path" "$testbox_id" "$ownership_token" <<'PY' import json import pathlib import sys @@ -42,7 +42,7 @@ except (OSError, json.JSONDecodeError) as error: if receipt.get("testbox_id") != expected_id: raise SystemExit("cleanup ID does not match the warmup ownership receipt") if receipt.get("confirmation_token") != expected_token: - raise SystemExit("cleanup confirmation token does not match the warmup ownership receipt") + raise SystemExit("ownership token does not match the warmup ownership receipt") for field in ("workflow", "job", "source_ref", "source_sha", "source_tree_sha", "ghostty_gitlink_sha"): if not receipt.get(field): raise SystemExit(f"warmup ownership receipt is missing {field}") @@ -68,6 +68,81 @@ import sys print(json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8"))["source_ref"]) PY )" +receipt_source_sha="$(python3 - "$receipt_path" <<'PY' +import json +import pathlib +import sys +print(json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8"))["source_sha"]) +PY +)" +receipt_source_tree_sha="$(python3 - "$receipt_path" <<'PY' +import json +import pathlib +import sys +print(json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8"))["source_tree_sha"]) +PY +)" +receipt_ghostty_sha="$(python3 - "$receipt_path" <<'PY' +import json +import pathlib +import sys +print(json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8"))["ghostty_gitlink_sha"]) +PY +)" + +# Parse either the table emitted by `list/status --id` or a summary response. +# The parser validates context only for a row containing this exact ID. It does +# not rely on fixed whitespace columns, because queued rows may have an empty IP. +parse_cli_output() { + local log_path="$1" + python3 - "$log_path" "$testbox_id" "$receipt_workflow" "$receipt_job" "$receipt_ref" <<'PY' +import pathlib +import re +import sys + +path, expected_id, expected_workflow, expected_job, expected_ref = sys.argv[1:] +text = pathlib.Path(path).read_text(encoding="utf-8", errors="replace") +for line in text.splitlines(): + fields = line.split() + if not fields or fields[0] != expected_id: + continue + if len(fields) < 2: + raise SystemExit(66) + status = fields[1].lower() + for index, field in enumerate(fields[2:], start=2): + if field == expected_workflow: + if fields[index + 1:index + 3] != [expected_job, expected_ref]: + raise SystemExit(66) + break + print(status) + raise SystemExit(0) + +# Some CLI versions use a summary such as `[tbx_...] Status: ready`. +if re.search(rf"\b{re.escape(expected_id)}\b", text): + match = re.search(r"\bstatus\s*:?\s*([A-Za-z_]+)", text, re.IGNORECASE) + if match: + print(match.group(1).lower()) + raise SystemExit(0) +raise SystemExit(3) +PY +} + +is_terminal() { + case "$1" in + completed|stopped|cancelled|failed|terminated|hydration_failed) return 0 ;; + *) return 1 ;; + esac +} +is_active() { + case "$1" in + ready|running|hydrating|in_progress|queued) return 0 ;; + *) return 1 ;; + esac +} +is_known_absence() { + grep -Eiq '(not found|already[[:space:]]+(stopped|completed)|hydration_failed|HTTP[[:space:]]+404|HTTP[[:space:]]+409|status[[:space:]]+code[[:space:]]+409)' "$1" +} + inventory_log="$evidence_dir/list-before-stop.log" set +e blacksmith testbox list --all >"$inventory_log" 2>&1 @@ -78,134 +153,144 @@ if (( inventory_status != 0 )); then exit "$inventory_status" fi -# The CLI has emitted both table-shaped and summary-shaped `status --id` -# responses over time. Extract state without treating either response as an -# ownership table. Ownership context comes only from the exact ID row in the -# successful inventory preview below. -status_from_log() { - local log_path="$1" - local value - value="$(awk -v id="$testbox_id" '$1 == id { print tolower($2); exit }' "$log_path")" - if [[ -z "$value" ]]; then - value="$(awk '{ - for (i = 1; i <= NF; i++) { - token = tolower($i) - if (token == "status:" && i < NF) { - print tolower($(i + 1)) - exit - } - if (token ~ /^status:/) { - sub(/^status:/, "", token) - if (token != "") { - print token - exit - } - } - } - }' "$log_path")" - fi - printf '%s' "$value" -} +inventory_row_present=0 +set +e +parse_cli_output "$inventory_log" >/dev/null +inventory_parse_status=$? +set -e +case "$inventory_parse_status" in + 0) inventory_row_present=1 ;; + 3) : ;; + 66) echo "inventory ownership context differs from the warmup receipt; refusing cleanup" >&2; exit 66 ;; + *) echo "could not parse the Testbox inventory; refusing cleanup" >&2; exit "$inventory_parse_status" ;; +esac -inventory_row="$(awk -v id="$testbox_id" '$1 == id { print; exit }' "$inventory_log")" -pre_status_log="$evidence_dir/status-before-stop.log" +status_log="$evidence_dir/status-before-stop.log" set +e -blacksmith testbox status --id "$testbox_id" >"$pre_status_log" 2>&1 -pre_status=$? +blacksmith testbox status --id "$testbox_id" >"$status_log" 2>&1 +status_command_status=$? set -e -skip_stop=0 -if [[ -n "$inventory_row" ]]; then - # `list --all` is ID STATUS IP WORKFLOW JOB REF ...; compare this exact row - # to the warmup receipt before any destructive operation. - pre_workflow="$(awk '{print $4}' <<<"$inventory_row")" - pre_job="$(awk '{print $5}' <<<"$inventory_row")" - pre_ref="$(awk '{print $6}' <<<"$inventory_row")" - if [[ "$pre_workflow" != "$receipt_workflow" || "$pre_job" != "$receipt_job" || "$pre_ref" != "$receipt_ref" ]]; then - echo "owned Testbox context differs from the warmup receipt; refusing cleanup" >&2 - exit 66 - fi -fi -if (( pre_status == 0 )); then - pre_status_value="$(status_from_log "$pre_status_log")" - case "$pre_status_value" in - completed|stopped|cancelled|failed|terminated|hydration_failed) - skip_stop=1 - ;; - ready|running|hydrating|in_progress|queued) - [[ -n "$inventory_row" ]] || { - echo "owned Testbox is active but absent from inventory; refusing cleanup" >&2 - exit 66 - } - ;; - *) - echo "could not parse status for owned Testbox $testbox_id; refusing cleanup" >&2 - exit 66 - ;; +status_value="" +status_absent=0 +if (( status_command_status == 0 )); then + set +e + status_value="$(parse_cli_output "$status_log")" + status_parse_status=$? + set -e + case "$status_parse_status" in + 0) ;; + 3) echo "status omitted the owned Testbox $testbox_id; refusing cleanup" >&2; exit 66 ;; + 66) echo "status ownership context differs from the warmup receipt; refusing cleanup" >&2; exit 66 ;; + *) echo "could not parse status for owned Testbox $testbox_id; refusing cleanup" >&2; exit "$status_parse_status" ;; esac -elif grep -Eiq '(not found|already[[:space:]]+(stopped|completed)|hydration_failed|HTTP[[:space:]]+409|status[[:space:]]+code[[:space:]]+409)' "$pre_status_log"; then - # An ID-specific terminal/not-found response proves there is nothing to stop. - skip_stop=1 +elif is_known_absence "$status_log"; then + status_absent=1 else echo "failed to inspect owned Testbox $testbox_id before cleanup; refusing stop" >&2 - exit "$pre_status" + exit "$status_command_status" +fi + +if (( status_absent == 0 )) && is_active "$status_value" && (( inventory_row_present == 0 )); then + echo "owned Testbox is active but absent from the inventory; refusing cleanup" >&2 + exit 66 +fi +if (( status_absent == 0 )) && ! is_active "$status_value" && ! is_terminal "$status_value"; then + echo "unknown status for owned Testbox $testbox_id; refusing cleanup" >&2 + exit 66 +fi + +preview_path="$evidence_dir/cleanup-preview.json" +python3 - "$preview_path" "$testbox_id" "${status_value:-absent}" "$inventory_row_present" "$receipt_workflow" "$receipt_job" "$receipt_ref" "$receipt_source_sha" "$receipt_source_tree_sha" "$receipt_ghostty_sha" <<'PY' +import json +import pathlib +import sys + +(path, testbox_id, status, inventory_present, workflow, job, ref, + source_sha, source_tree_sha, ghostty_sha) = sys.argv[1:] +payload = { + "schema": 1, + "testbox_id": testbox_id, + "status": status, + "inventory_row_present": bool(int(inventory_present)), + "workflow": workflow, + "job": job, + "source_ref": ref, + "source_sha": source_sha, + "source_tree_sha": source_tree_sha, + "ghostty_gitlink_sha": ghostty_sha, +} +out = pathlib.Path(path) +out.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") +out.chmod(0o600) +PY +preview_sha="$(sha256sum "$preview_path" | awk '{print $1}')" +printf 'Testbox cleanup preview: id=%s status=%s inventory_row=%s workflow=%s job=%s ref=%s\n' \ + "$testbox_id" "${status_value:-absent}" "$inventory_row_present" "$receipt_workflow" "$receipt_job" "$receipt_ref" +printf 'Preview SHA: %s\n' "$preview_sha" +if [[ "$operator_confirmation" == "PREVIEW" ]]; then + echo "Review the preview, then rerun with the same token and STOP:$preview_sha to authorize stop." >&2 + exit 75 +fi +expected_preview_sha="${operator_confirmation#STOP:}" +if [[ "$expected_preview_sha" != "$preview_sha" ]]; then + echo "current cleanup preview differs from the supplied confirmation; refusing stop" >&2 + exit 67 fi stop_log="$evidence_dir/stop.log" -status_log="$evidence_dir/status-after-stop.log" list_log="$evidence_dir/list-after-stop.log" cleanup_status=0 poll_deadline=$((SECONDS + 120)) poll_attempt=0 - -if (( skip_stop == 1 )); then +if (( status_absent == 1 )) || is_terminal "$status_value"; then printf 'Testbox %s is already terminal or absent; no stop request needed\n' "$testbox_id" >"$stop_log" - stop_status=0 else set +e blacksmith testbox stop --id "$testbox_id" >"$stop_log" 2>&1 stop_status=$? set -e -fi -if (( stop_status != 0 )); then - # A completed or already-absent Testbox can race the explicit stop call. - # Tolerate only the documented terminal/not-found response, never an - # arbitrary stop failure. - if grep -Eiq '(already[[:space:]]+(stopped|completed)|hydration_failed|not found|HTTP[[:space:]]+404)' "$stop_log"; then - printf 'stop already reached a terminal state for %s; continuing\n' "$testbox_id" >&2 - else - echo "failed to stop Testbox $testbox_id; see $stop_log" >&2 - cleanup_status=$stop_status + if (( stop_status != 0 )); then + if is_known_absence "$stop_log"; then + printf 'stop reached a known terminal or absent state for %s; continuing\n' "$testbox_id" >&2 + else + echo "failed to stop Testbox $testbox_id; see $stop_log" >&2 + cleanup_status=$stop_status + fi fi fi -# Status is diagnostic. The authoritative cleanup check below parses the -# specific ID in `list --all`; terminal rows are accepted because --all may -# retain completed boxes, while active rows remain failures. +# Poll the ID-specific endpoint until cancellation propagates. Never treat a +# different row in the global inventory as proof that this ID is terminal. while :; do poll_attempt=$((poll_attempt + 1)) : >"$status_log" set +e blacksmith testbox status --id "$testbox_id" >"$status_log" 2>&1 - status_status=$? + status_command_status=$? set -e - terminal=0 - if (( status_status == 0 )); then - status_value="$(status_from_log "$status_log")" - case "$status_value" in - completed|stopped|cancelled|failed|terminated|hydration_failed) terminal=1 ;; - ready|running|hydrating|in_progress|queued) ;; - *) - grep -Eiq 'status:[[:space:]]*(completed|stopped|cancelled|failed|terminated|hydration_failed)' "$status_log" && terminal=1 - ;; - esac - elif grep -Eiq '(not found|already[[:space:]]+(stopped|completed)|hydration_failed|HTTP[[:space:]]+409|status[[:space:]]+code[[:space:]]+409)' "$status_log"; then - terminal=1 - else - echo "failed to inspect Testbox $testbox_id; see $status_log" >&2 - (( cleanup_status == 0 )) && cleanup_status=$status_status + if (( status_command_status == 0 )); then + set +e + status_value="$(parse_cli_output "$status_log")" + status_parse_status=$? + set -e + if (( status_parse_status != 0 )); then + echo "could not parse post-stop status for $testbox_id; see $status_log" >&2 + (( cleanup_status == 0 )) && cleanup_status=66 + break + fi + if is_terminal "$status_value"; then + break + fi + if ! is_active "$status_value"; then + echo "unknown post-stop status for $testbox_id; see $status_log" >&2 + (( cleanup_status == 0 )) && cleanup_status=66 + break + fi + elif is_known_absence "$status_log"; then break - fi - if (( terminal == 1 )); then + else + echo "failed to inspect Testbox $testbox_id after cleanup; see $status_log" >&2 + (( cleanup_status == 0 )) && cleanup_status=$status_command_status break fi if (( SECONDS >= poll_deadline )); then @@ -225,21 +310,28 @@ if (( list_status != 0 )); then echo "failed to list Testboxes after stopping $testbox_id; see $list_log" >&2 (( cleanup_status == 0 )) && cleanup_status=$list_status else - listed_status="$(awk -v id="$testbox_id" '$1 == id { print tolower($2); exit }' "$list_log")" - case "$listed_status" in - "") - # The CLI may remove terminal boxes from the inventory immediately. - ;; - completed|stopped|cancelled|failed|terminated|hydration_failed) - # --all is the full inventory, so a terminal row is safe and expected. + set +e + listed_status="$(parse_cli_output "$list_log")" + listed_parse_status=$? + set -e + case "$listed_parse_status" in + 0) + if is_active "$listed_status"; then + echo "Testbox $testbox_id is still active after cleanup; see $list_log" >&2 + (( cleanup_status == 0 )) && cleanup_status=1 + elif ! is_terminal "$listed_status"; then + echo "unknown status for Testbox $testbox_id in inventory: $listed_status" >&2 + (( cleanup_status == 0 )) && cleanup_status=66 + fi ;; - ready|running|hydrating|in_progress|queued) - echo "Testbox $testbox_id is still active after cleanup; see $list_log" >&2 - (( cleanup_status == 0 )) && cleanup_status=1 + 3) ;; + 66) + echo "Testbox $testbox_id ownership changed in final inventory; see $list_log" >&2 + (( cleanup_status == 0 )) && cleanup_status=66 ;; *) - echo "unknown status for Testbox $testbox_id in inventory: $listed_status" >&2 - (( cleanup_status == 0 )) && cleanup_status=1 + echo "could not parse final Testbox inventory; see $list_log" >&2 + (( cleanup_status == 0 )) && cleanup_status=$listed_parse_status ;; esac fi From 63d74f31ea8a47554afdab4b8ccf11b0acd5c7ab Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 13:00:57 -0700 Subject: [PATCH 25/38] ci: close final Testbox trust gaps --- .../ci-workflow-guard-tests-testbox.yml | 6 +++--- scripts/blacksmith-cmux-tui-testbox-stage.sh | 6 ++++++ scripts/blacksmith-testbox-cleanup.sh | 18 +++++++++++------- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci-workflow-guard-tests-testbox.yml b/.github/workflows/ci-workflow-guard-tests-testbox.yml index dfcc3821e3c..253fe517153 100644 --- a/.github/workflows/ci-workflow-guard-tests-testbox.yml +++ b/.github/workflows/ci-workflow-guard-tests-testbox.yml @@ -47,11 +47,11 @@ jobs: # those external controls or move this action into branch-controlled code. # It attaches the VM requested by `blacksmith testbox warmup` and leaves # it alive after this setup job. - - name: Guard trusted source ref - if: github.ref != 'refs/heads/main' + - name: Guard trusted repository and source ref + if: github.repository != 'manaflow-ai/cmux' || github.ref != 'refs/heads/main' shell: bash run: | - echo "::error::Testbox token-bearing setup is restricted to reviewed main" >&2 + echo "::error::Testbox token-bearing setup is restricted to manaflow-ai/cmux reviewed main" >&2 exit 1 - name: Validate Testbox ID before token-bearing action diff --git a/scripts/blacksmith-cmux-tui-testbox-stage.sh b/scripts/blacksmith-cmux-tui-testbox-stage.sh index f43ff9bac02..ce6d77ba195 100755 --- a/scripts/blacksmith-cmux-tui-testbox-stage.sh +++ b/scripts/blacksmith-cmux-tui-testbox-stage.sh @@ -100,6 +100,8 @@ except (OSError, json.JSONDecodeError) as error: raise SystemExit(f"invalid setup identity marker: {error}") source = record.get("source", {}) testbox = record.get("testbox", {}) +runner = record.get("runner", {}) +toolchain = record.get("toolchain", {}) errors = [] if source.get("commit_sha") != expected_source: errors.append("setup source commit mismatch") @@ -113,6 +115,10 @@ if testbox.get("id") != expected_testbox: errors.append("setup Testbox ID mismatch") if str(testbox.get("setup_workflow_run_id")) != expected_run_id: errors.append("setup workflow run ID mismatch") +if runner.get("label") != "blacksmith-32vcpu-ubuntu-2404" or runner.get("arch") != "X64" or runner.get("cpu_count") != 32: + errors.append("setup runner identity mismatch") +if not toolchain.get("rust_toolchain") or not toolchain.get("rustc") or not toolchain.get("cargo") or not toolchain.get("zig"): + errors.append("setup toolchain identity is incomplete") if errors: for error in errors: print(error, file=sys.stderr) diff --git a/scripts/blacksmith-testbox-cleanup.sh b/scripts/blacksmith-testbox-cleanup.sh index 0fc76408fe3..9d5429ffb0f 100755 --- a/scripts/blacksmith-testbox-cleanup.sh +++ b/scripts/blacksmith-testbox-cleanup.sh @@ -18,8 +18,8 @@ if [[ ! "$ownership_token" =~ ^[0-9a-f]{32}$ ]]; then echo "ownership token must be a 32-character lowercase hex value" >&2 exit 64 fi -if [[ "$operator_confirmation" != "PREVIEW" && ! "$operator_confirmation" =~ ^STOP:[0-9a-f]{64}$ ]]; then - echo "confirmation must be PREVIEW or STOP:<64-character preview SHA>" >&2 +if [[ "$operator_confirmation" != "PREVIEW" && "$operator_confirmation" != "STOP" && ! "$operator_confirmation" =~ ^STOP:[0-9a-f]{64}$ ]]; then + echo "confirmation must be PREVIEW, STOP, or STOP:<64-character preview SHA>" >&2 exit 64 fi @@ -140,7 +140,7 @@ is_active() { esac } is_known_absence() { - grep -Eiq '(not found|already[[:space:]]+(stopped|completed)|hydration_failed|HTTP[[:space:]]+404|HTTP[[:space:]]+409|status[[:space:]]+code[[:space:]]+409)' "$1" + grep -Eiq '(not found|already[[:space:]]+(stopped|completed)|hydration_failed|HTTP[[:space:]]+404|status[[:space:]]+code[[:space:]]+404)' "$1" } inventory_log="$evidence_dir/list-before-stop.log" @@ -231,10 +231,14 @@ if [[ "$operator_confirmation" == "PREVIEW" ]]; then echo "Review the preview, then rerun with the same token and STOP:$preview_sha to authorize stop." >&2 exit 75 fi -expected_preview_sha="${operator_confirmation#STOP:}" -if [[ "$expected_preview_sha" != "$preview_sha" ]]; then - echo "current cleanup preview differs from the supplied confirmation; refusing stop" >&2 - exit 67 +if [[ "$operator_confirmation" == "STOP" ]]; then + echo "legacy STOP accepted for a trusted EXIT trap; use STOP:$preview_sha for interactive cleanup" >&2 +else + expected_preview_sha="${operator_confirmation#STOP:}" + if [[ "$expected_preview_sha" != "$preview_sha" ]]; then + echo "current cleanup preview differs from the supplied confirmation; refusing stop" >&2 + exit 67 + fi fi stop_log="$evidence_dir/stop.log" From d0e710866d0db113e57bc48bfb26b17ddd6d0a5f Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 13:16:05 -0700 Subject: [PATCH 26/38] ci: bound Testbox control-plane operations --- .../workflows/ci-workflow-guard-tests-testbox.yml | 6 ++++-- scripts/blacksmith-testbox-cleanup.sh | 14 +++++++++----- skills/blacksmith-testbox/benchmark.md | 14 ++++++++------ 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci-workflow-guard-tests-testbox.yml b/.github/workflows/ci-workflow-guard-tests-testbox.yml index 253fe517153..50a4b1cdf3a 100644 --- a/.github/workflows/ci-workflow-guard-tests-testbox.yml +++ b/.github/workflows/ci-workflow-guard-tests-testbox.yml @@ -396,8 +396,10 @@ jobs: --data "$payload" 2>"$RUNNER_TEMP/testbox-ready-phone-home-$attempt.err")" phone_home_status=$? set -e - if (( phone_home_status == 0 )) && [[ -n "$response" ]]; then - printf '%s\n' "$response" >"$RUNNER_TEMP/testbox-ready-phone-home-response.json" + if (( phone_home_status == 0 )); then + # The API may acknowledge a valid phone-home with 204 No Content. + # Curl success is the acknowledgement; a response body is optional. + printf '%s\n' "$response" >"$RUNNER_TEMP/testbox-ready-phone-home-response.txt" exit 0 fi if (( attempt < 5 )); then diff --git a/scripts/blacksmith-testbox-cleanup.sh b/scripts/blacksmith-testbox-cleanup.sh index 9d5429ffb0f..26f6c96c1cf 100755 --- a/scripts/blacksmith-testbox-cleanup.sh +++ b/scripts/blacksmith-testbox-cleanup.sh @@ -24,6 +24,10 @@ if [[ "$operator_confirmation" != "PREVIEW" && "$operator_confirmation" != "STOP fi mkdir -p "$evidence_dir" +command -v timeout >/dev/null || { + echo "timeout is required for bounded cleanup operations" >&2 + exit 65 +} receipt_path="$evidence_dir/testbox-receipt.json" if [[ ! -s "$receipt_path" ]]; then echo "refusing cleanup without the warmup ownership receipt: $receipt_path" >&2 @@ -145,7 +149,7 @@ is_known_absence() { inventory_log="$evidence_dir/list-before-stop.log" set +e -blacksmith testbox list --all >"$inventory_log" 2>&1 +timeout --foreground --kill-after=5s 20s blacksmith testbox list --all >"$inventory_log" 2>&1 inventory_status=$? set -e if (( inventory_status != 0 )); then @@ -167,7 +171,7 @@ esac status_log="$evidence_dir/status-before-stop.log" set +e -blacksmith testbox status --id "$testbox_id" >"$status_log" 2>&1 +timeout --foreground --kill-after=5s 20s blacksmith testbox status --id "$testbox_id" >"$status_log" 2>&1 status_command_status=$? set -e status_value="" @@ -250,7 +254,7 @@ if (( status_absent == 1 )) || is_terminal "$status_value"; then printf 'Testbox %s is already terminal or absent; no stop request needed\n' "$testbox_id" >"$stop_log" else set +e - blacksmith testbox stop --id "$testbox_id" >"$stop_log" 2>&1 + timeout --foreground --kill-after=5s 20s blacksmith testbox stop --id "$testbox_id" >"$stop_log" 2>&1 stop_status=$? set -e if (( stop_status != 0 )); then @@ -269,7 +273,7 @@ while :; do poll_attempt=$((poll_attempt + 1)) : >"$status_log" set +e - blacksmith testbox status --id "$testbox_id" >"$status_log" 2>&1 + timeout --foreground --kill-after=5s 20s blacksmith testbox status --id "$testbox_id" >"$status_log" 2>&1 status_command_status=$? set -e if (( status_command_status == 0 )); then @@ -307,7 +311,7 @@ while :; do done set +e -blacksmith testbox list --all >"$list_log" 2>&1 +timeout --foreground --kill-after=5s 20s blacksmith testbox list --all >"$list_log" 2>&1 list_status=$? set -e if (( list_status != 0 )); then diff --git a/skills/blacksmith-testbox/benchmark.md b/skills/blacksmith-testbox/benchmark.md index fa24131efee..003b3bf6175 100644 --- a/skills/blacksmith-testbox/benchmark.md +++ b/skills/blacksmith-testbox/benchmark.md @@ -295,11 +295,12 @@ benchmarked. The helper creates one structured JSON record, one raw Cargo log, and one raw `/usr/bin/time -p` file per stage. Each remote Cargo build is bounded to 20 -minutes with a 30-second kill grace period. It verifies source and submodule -identity before the stage, exports and records the exact Zig binary used by -Cargo, holds a remote `flock` through all writes, restores the controlled -changed file from an integrity-checked backup, and verifies clean identity -again. +minutes with a 30-second kill grace period, and the outer CLI invocation is +bounded to 25 minutes so rsync, SSH, or control-plane hangs cannot bypass the +benchmark's cleanup path. It verifies source and submodule identity before the +stage, exports and records the exact Zig binary used by Cargo, holds a remote +`flock` through all writes, restores the controlled changed file from an +integrity-checked backup, and verifies clean identity again. ```bash # Run this orchestration block in Bash, not an interactive zsh session. @@ -311,7 +312,8 @@ run_stage() { 'CMUX_TESTBOX_REMOTE=1 CMUX_TESTBOX_ID=%q %q %q %q %q' \ "$TBX" ./scripts/blacksmith-cmux-tui-testbox-stage.sh \ "$stage" "$SOURCE_SHA" "$GHOSTTY_SHA" - blacksmith testbox run --id "$TBX" --debug \ + timeout --foreground --kill-after=30s 25m \ + blacksmith testbox run --id "$TBX" --debug \ "$remote_command" >"$OUT/$stage.run.log" 2>&1 run_status=$? set -e From 905939f5738e671d5efcac89123d61378873c6e1 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 13:29:01 -0700 Subject: [PATCH 27/38] ci: use portable bounded Testbox commands --- scripts/blacksmith-bounded-command.sh | 40 ++++++++++++++++++++ scripts/blacksmith-cmux-tui-testbox-stage.sh | 4 +- scripts/blacksmith-testbox-cleanup.sh | 36 ++++++++++++------ skills/blacksmith-testbox/benchmark.md | 3 +- 4 files changed, 67 insertions(+), 16 deletions(-) create mode 100755 scripts/blacksmith-bounded-command.sh diff --git a/scripts/blacksmith-bounded-command.sh b/scripts/blacksmith-bounded-command.sh new file mode 100755 index 00000000000..c50c8a78920 --- /dev/null +++ b/scripts/blacksmith-bounded-command.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -lt 2 || ! "$1" =~ ^[0-9]+$ ]]; then + echo "usage: $0 [args...]" >&2 + exit 64 +fi +seconds="$1" +shift + +# Prefer GNU coreutils when present, while keeping the operator workflow +# usable on macOS hosts that have neither `timeout` nor `gtimeout`. +for candidate in gtimeout timeout; do + if command -v "$candidate" >/dev/null && "$candidate" --version 2>&1 | grep -q 'GNU coreutils'; then + exec "$candidate" --foreground --kill-after=5s "${seconds}s" "$@" + fi +done + +exec python3 - "$seconds" "$@" <<'PY' +import os +import signal +import subprocess +import sys + +seconds = float(sys.argv[1]) +argv = sys.argv[2:] +if not argv: + raise SystemExit("missing command") +process = subprocess.Popen(argv, start_new_session=True) +try: + raise SystemExit(process.wait(timeout=seconds)) +except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + raise SystemExit(124) +PY diff --git a/scripts/blacksmith-cmux-tui-testbox-stage.sh b/scripts/blacksmith-cmux-tui-testbox-stage.sh index ce6d77ba195..a725c6a4dab 100755 --- a/scripts/blacksmith-cmux-tui-testbox-stage.sh +++ b/scripts/blacksmith-cmux-tui-testbox-stage.sh @@ -178,7 +178,7 @@ restore_changed_file() { # Always restore the deliberately changed source, including when Cargo exits # non-zero. Do not let cleanup replace the build result unless restoration # itself fails. -# shellcheck disable=SC2329 # invoked indirectly by the EXIT trap +# shellcheck disable=SC2329 # invoked indirectly by the signal/EXIT traps finish_source() { local result=$? if [[ -n "$changed_backup" ]]; then @@ -189,7 +189,7 @@ finish_source() { fi exit "$result" } -trap finish_source EXIT +trap finish_source EXIT TERM INT HUP clean_status() { local top_status ghostty_status diff --git a/scripts/blacksmith-testbox-cleanup.sh b/scripts/blacksmith-testbox-cleanup.sh index 26f6c96c1cf..789e6abbc42 100755 --- a/scripts/blacksmith-testbox-cleanup.sh +++ b/scripts/blacksmith-testbox-cleanup.sh @@ -24,9 +24,19 @@ if [[ "$operator_confirmation" != "PREVIEW" && "$operator_confirmation" != "STOP fi mkdir -p "$evidence_dir" -command -v timeout >/dev/null || { - echo "timeout is required for bounded cleanup operations" >&2 - exit 65 +sha256_file() { + if command -v sha256sum >/dev/null; then + sha256sum "$1" | awk '{print $1}' + elif command -v shasum >/dev/null; then + shasum -a 256 "$1" | awk '{print $1}' + else + echo "sha256sum or shasum is required for cleanup preview hashing" >&2 + return 65 + fi +} +bounded_command() { + scripts_dir="$(CDPATH='' cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" + "$scripts_dir/blacksmith-bounded-command.sh" "$@" } receipt_path="$evidence_dir/testbox-receipt.json" if [[ ! -s "$receipt_path" ]]; then @@ -113,11 +123,13 @@ for line in text.splitlines(): if len(fields) < 2: raise SystemExit(66) status = fields[1].lower() + matching_context = False for index, field in enumerate(fields[2:], start=2): - if field == expected_workflow: - if fields[index + 1:index + 3] != [expected_job, expected_ref]: - raise SystemExit(66) + if field == expected_workflow and fields[index + 1:index + 3] == [expected_job, expected_ref]: + matching_context = True break + if not matching_context: + raise SystemExit(66) print(status) raise SystemExit(0) @@ -149,7 +161,7 @@ is_known_absence() { inventory_log="$evidence_dir/list-before-stop.log" set +e -timeout --foreground --kill-after=5s 20s blacksmith testbox list --all >"$inventory_log" 2>&1 +bounded_command 20 blacksmith testbox list --all >"$inventory_log" 2>&1 inventory_status=$? set -e if (( inventory_status != 0 )); then @@ -171,7 +183,7 @@ esac status_log="$evidence_dir/status-before-stop.log" set +e -timeout --foreground --kill-after=5s 20s blacksmith testbox status --id "$testbox_id" >"$status_log" 2>&1 +bounded_command 20 blacksmith testbox status --id "$testbox_id" >"$status_log" 2>&1 status_command_status=$? set -e status_value="" @@ -227,7 +239,7 @@ out = pathlib.Path(path) out.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") out.chmod(0o600) PY -preview_sha="$(sha256sum "$preview_path" | awk '{print $1}')" +preview_sha="$(sha256_file "$preview_path" | awk '{print $1}')" printf 'Testbox cleanup preview: id=%s status=%s inventory_row=%s workflow=%s job=%s ref=%s\n' \ "$testbox_id" "${status_value:-absent}" "$inventory_row_present" "$receipt_workflow" "$receipt_job" "$receipt_ref" printf 'Preview SHA: %s\n' "$preview_sha" @@ -254,7 +266,7 @@ if (( status_absent == 1 )) || is_terminal "$status_value"; then printf 'Testbox %s is already terminal or absent; no stop request needed\n' "$testbox_id" >"$stop_log" else set +e - timeout --foreground --kill-after=5s 20s blacksmith testbox stop --id "$testbox_id" >"$stop_log" 2>&1 + bounded_command 20 blacksmith testbox stop --id "$testbox_id" >"$stop_log" 2>&1 stop_status=$? set -e if (( stop_status != 0 )); then @@ -273,7 +285,7 @@ while :; do poll_attempt=$((poll_attempt + 1)) : >"$status_log" set +e - timeout --foreground --kill-after=5s 20s blacksmith testbox status --id "$testbox_id" >"$status_log" 2>&1 + bounded_command 20 blacksmith testbox status --id "$testbox_id" >"$status_log" 2>&1 status_command_status=$? set -e if (( status_command_status == 0 )); then @@ -311,7 +323,7 @@ while :; do done set +e -timeout --foreground --kill-after=5s 20s blacksmith testbox list --all >"$list_log" 2>&1 +bounded_command 20 blacksmith testbox list --all >"$list_log" 2>&1 list_status=$? set -e if (( list_status != 0 )); then diff --git a/skills/blacksmith-testbox/benchmark.md b/skills/blacksmith-testbox/benchmark.md index 003b3bf6175..a2f76c61e73 100644 --- a/skills/blacksmith-testbox/benchmark.md +++ b/skills/blacksmith-testbox/benchmark.md @@ -312,8 +312,7 @@ run_stage() { 'CMUX_TESTBOX_REMOTE=1 CMUX_TESTBOX_ID=%q %q %q %q %q' \ "$TBX" ./scripts/blacksmith-cmux-tui-testbox-stage.sh \ "$stage" "$SOURCE_SHA" "$GHOSTTY_SHA" - timeout --foreground --kill-after=30s 25m \ - blacksmith testbox run --id "$TBX" --debug \ + ./scripts/blacksmith-bounded-command.sh 1500 blacksmith testbox run --id "$TBX" --debug \ "$remote_command" >"$OUT/$stage.run.log" 2>&1 run_status=$? set -e From ad1206b84b4e84fde017a099f2fc0e772f7a8b43 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 13:43:03 -0700 Subject: [PATCH 28/38] ci: require preview-bound Testbox cleanup --- scripts/blacksmith-bounded-command.sh | 2 +- scripts/blacksmith-testbox-cleanup.sh | 16 ++++++---------- skills/blacksmith-testbox/SKILL.md | 15 +++++++++------ skills/blacksmith-testbox/benchmark.md | 14 +++++++++++--- 4 files changed, 27 insertions(+), 20 deletions(-) diff --git a/scripts/blacksmith-bounded-command.sh b/scripts/blacksmith-bounded-command.sh index c50c8a78920..ad9aaa88e4c 100755 --- a/scripts/blacksmith-bounded-command.sh +++ b/scripts/blacksmith-bounded-command.sh @@ -12,7 +12,7 @@ shift # usable on macOS hosts that have neither `timeout` nor `gtimeout`. for candidate in gtimeout timeout; do if command -v "$candidate" >/dev/null && "$candidate" --version 2>&1 | grep -q 'GNU coreutils'; then - exec "$candidate" --foreground --kill-after=5s "${seconds}s" "$@" + exec "$candidate" --kill-after=5s "${seconds}s" "$@" fi done diff --git a/scripts/blacksmith-testbox-cleanup.sh b/scripts/blacksmith-testbox-cleanup.sh index 789e6abbc42..7998a91b3be 100755 --- a/scripts/blacksmith-testbox-cleanup.sh +++ b/scripts/blacksmith-testbox-cleanup.sh @@ -18,8 +18,8 @@ if [[ ! "$ownership_token" =~ ^[0-9a-f]{32}$ ]]; then echo "ownership token must be a 32-character lowercase hex value" >&2 exit 64 fi -if [[ "$operator_confirmation" != "PREVIEW" && "$operator_confirmation" != "STOP" && ! "$operator_confirmation" =~ ^STOP:[0-9a-f]{64}$ ]]; then - echo "confirmation must be PREVIEW, STOP, or STOP:<64-character preview SHA>" >&2 +if [[ "$operator_confirmation" != "PREVIEW" && ! "$operator_confirmation" =~ ^STOP:[0-9a-f]{64}$ ]]; then + echo "confirmation must be PREVIEW or STOP:<64-character preview SHA>" >&2 exit 64 fi @@ -247,14 +247,10 @@ if [[ "$operator_confirmation" == "PREVIEW" ]]; then echo "Review the preview, then rerun with the same token and STOP:$preview_sha to authorize stop." >&2 exit 75 fi -if [[ "$operator_confirmation" == "STOP" ]]; then - echo "legacy STOP accepted for a trusted EXIT trap; use STOP:$preview_sha for interactive cleanup" >&2 -else - expected_preview_sha="${operator_confirmation#STOP:}" - if [[ "$expected_preview_sha" != "$preview_sha" ]]; then - echo "current cleanup preview differs from the supplied confirmation; refusing stop" >&2 - exit 67 - fi +expected_preview_sha="${operator_confirmation#STOP:}" +if [[ "$operator_confirmation" != "PREVIEW" && "$expected_preview_sha" != "$preview_sha" ]]; then + echo "current cleanup preview differs from the supplied confirmation; refusing stop" >&2 + exit 67 fi stop_log="$evidence_dir/stop.log" diff --git a/skills/blacksmith-testbox/SKILL.md b/skills/blacksmith-testbox/SKILL.md index b278ce2567c..de8078c5eb8 100644 --- a/skills/blacksmith-testbox/SKILL.md +++ b/skills/blacksmith-testbox/SKILL.md @@ -202,7 +202,8 @@ run_stage() { 'CMUX_TESTBOX_REMOTE=1 CMUX_TESTBOX_ID=%q %q %q %q %q' \ "$TBX" ./scripts/blacksmith-cmux-tui-testbox-stage.sh \ "$stage" "$SOURCE_SHA" "$GHOSTTY_SHA" - blacksmith testbox run --id "$TBX" --debug \ + ./scripts/blacksmith-bounded-command.sh 1500 \ + blacksmith testbox run --id "$TBX" --debug \ "$remote_command" >"$OUT/$stage.run.log" 2>&1 run_status=$? set -e @@ -312,21 +313,23 @@ CLEANUP_TOKEN="${CLEANUP_TOKEN:-}" echo "use the ownership token emitted by the warmup receipt" >&2 exit 64 } -scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" "$CLEANUP_TOKEN" STOP +scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" "$CLEANUP_TOKEN" PREVIEW +# Review cleanup-preview.json, then rerun with STOP:. ``` It records a pre-stop status preview, stop result, post-stop status, and `list --all` output. The preview must match the receipt's workflow, job, and -branch before any stop is attempted. Cleanup is destructive and requires the -separate literal `STOP` operator confirmation in addition to the receipt token. +branch before any stop is attempted. Cleanup is destructive and requires a fresh `STOP:` +confirmation after reviewing the current receipt-bound preview. It verifies that the specific Testbox ID is terminal or absent from the active inventory, accepts the known terminal states `completed`, `stopped`, `cancelled`, `failed`, `terminated`, and `hydration_failed`, plus a 409 saying the box is already stopped or completed, and polls for up to two minutes while cancellation propagates. Other stop, status, or list failures remain failures. Put it in an `EXIT` trap only after an independent operator exports -`CONFIRM_TESTBOX_STOP=STOP`; otherwise preserve the benchmark's original exit -status and leave the box for manual cleanup. The detailed benchmark writes a +`CONFIRM_TESTBOX_STOP=STOP`; the trap performs a preview and confirms its hash +before stopping. Otherwise preserve the benchmark's original exit status and +leave the box for manual cleanup. The detailed benchmark writes a receipt and ownership token for the exact ID returned by warmup; cleanup refuses an ID or token that is not bound to that receipt. If warmup fails before returning an ID, retain before/after inventory but do not automatically stop a box, because diff --git a/skills/blacksmith-testbox/benchmark.md b/skills/blacksmith-testbox/benchmark.md index a2f76c61e73..9c5ed8327f5 100644 --- a/skills/blacksmith-testbox/benchmark.md +++ b/skills/blacksmith-testbox/benchmark.md @@ -149,8 +149,15 @@ cleanup() { trap - EXIT if [[ -n "$TBX" && -n "$cleanup_token" && "${CONFIRM_TESTBOX_STOP:-}" == "STOP" ]]; then set +e - scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" "$cleanup_token" STOP - cleanup_status=$? + scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" "$cleanup_token" PREVIEW + preview_status=$? + if (( preview_status == 75 )); then + preview_sha="$(sha256sum "$OUT/cleanup-preview.json" | awk '{print $1}')" + scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" "$cleanup_token" "STOP:$preview_sha" + cleanup_status=$? + else + cleanup_status="$preview_status" + fi set -e else # Without the CLI receipt there is no proof that a newly listed box belongs @@ -413,7 +420,8 @@ cleanup_token="${cleanup_token:-}" echo "use the confirmation token emitted by the warmup receipt" >&2 exit 64 } -scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" "$cleanup_token" STOP +scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" "$cleanup_token" PREVIEW +# Review cleanup-preview.json, then rerun with STOP:. ``` A shell `EXIT` trap may call that helper only when an independent operator has From 53fbe799d0cf097d14f5c7eb3d5f5bd14d6ade46 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 13:54:04 -0700 Subject: [PATCH 29/38] docs: bound Testbox downloads and cleanup hashing --- skills/blacksmith-testbox/SKILL.md | 3 ++- skills/blacksmith-testbox/benchmark.md | 17 ++++++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/skills/blacksmith-testbox/SKILL.md b/skills/blacksmith-testbox/SKILL.md index de8078c5eb8..966b74aac78 100644 --- a/skills/blacksmith-testbox/SKILL.md +++ b/skills/blacksmith-testbox/SKILL.md @@ -213,7 +213,8 @@ run_stage() { # files, so a one-time download after all stages is insufficient. : >"$OUT/$stage.download.log" for suffix in json time log; do - if ! blacksmith testbox download --id "$TBX" \ + if ! ./scripts/blacksmith-bounded-command.sh 120 \ + blacksmith testbox download --id "$TBX" \ "testbox-benchmark/$stage.$suffix" "$OUT/raw/$stage.$suffix" \ >>"$OUT/$stage.download.log" 2>&1; then download_status=1 diff --git a/skills/blacksmith-testbox/benchmark.md b/skills/blacksmith-testbox/benchmark.md index 9c5ed8327f5..1103a467134 100644 --- a/skills/blacksmith-testbox/benchmark.md +++ b/skills/blacksmith-testbox/benchmark.md @@ -152,8 +152,18 @@ cleanup() { scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" "$cleanup_token" PREVIEW preview_status=$? if (( preview_status == 75 )); then - preview_sha="$(sha256sum "$OUT/cleanup-preview.json" | awk '{print $1}')" - scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" "$cleanup_token" "STOP:$preview_sha" + if command -v sha256sum >/dev/null; then + preview_sha="$(sha256sum "$OUT/cleanup-preview.json" | awk '{print $1}')" + elif command -v shasum >/dev/null; then + preview_sha="$(shasum -a 256 "$OUT/cleanup-preview.json" | awk '{print $1}')" + else + echo "no SHA-256 utility available for cleanup preview" >&2 + cleanup_status=65 + preview_sha="" + fi + if [[ -n "$preview_sha" ]]; then + scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" "$cleanup_token" "STOP:$preview_sha" + fi cleanup_status=$? else cleanup_status="$preview_status" @@ -329,7 +339,8 @@ run_stage() { # stage immediately, before starting another stage. : >"$OUT/$stage.download.log" for suffix in json time log; do - if ! blacksmith testbox download --id "$TBX" \ + if ! ./scripts/blacksmith-bounded-command.sh 120 \ + blacksmith testbox download --id "$TBX" \ "testbox-benchmark/$stage.$suffix" "$OUT/raw/$stage.$suffix" \ >>"$OUT/$stage.download.log" 2>&1; then download_status=1 From d2107463f2601a9b0f391e820042bd15f5163b1d Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 14:06:36 -0700 Subject: [PATCH 30/38] ci: handle Testbox metadata and CLI schema drift --- .../workflows/ci-workflow-guard-tests-testbox.yml | 14 ++++++++++++++ scripts/blacksmith-testbox-cleanup.sh | 15 +++++++++------ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-workflow-guard-tests-testbox.yml b/.github/workflows/ci-workflow-guard-tests-testbox.yml index 50a4b1cdf3a..0066266a942 100644 --- a/.github/workflows/ci-workflow-guard-tests-testbox.yml +++ b/.github/workflows/ci-workflow-guard-tests-testbox.yml @@ -333,6 +333,20 @@ jobs: if-no-files-found: warn retention-days: 14 + # Missing `/tmp/.testbox` means begin-testbox could not register the VM. + # No auth token or API URL exists, so neither this workflow nor the pinned + # action can report hydration_failed. Fail explicitly; the CLI's bounded + # readiness wait and receipt-aware cleanup remain the only safe recovery. + - name: Require Testbox registration state + if: always() + shell: bash + run: | + set -euo pipefail + if [[ ! -d /tmp/.testbox ]]; then + echo "::error::begin-testbox produced no registration state; hydration failure cannot be reported from this runner" >&2 + exit 1 + fi + - name: Confirm Testbox registration and mark setup ready if: success() env: diff --git a/scripts/blacksmith-testbox-cleanup.sh b/scripts/blacksmith-testbox-cleanup.sh index 7998a91b3be..7607e373f6d 100755 --- a/scripts/blacksmith-testbox-cleanup.sh +++ b/scripts/blacksmith-testbox-cleanup.sh @@ -123,12 +123,15 @@ for line in text.splitlines(): if len(fields) < 2: raise SystemExit(66) status = fields[1].lower() - matching_context = False - for index, field in enumerate(fields[2:], start=2): - if field == expected_workflow and fields[index + 1:index + 3] == [expected_job, expected_ref]: - matching_context = True - break - if not matching_context: + # Blacksmith CLI releases have emitted both ID/STATUS/REPO/WORKFLOW/CREATED + # and ID/STATUS/IP/WORKFLOW/JOB/REF/... schemas. Require the exact workflow + # in either schema; when job/ref columns exist, require those too. + try: + workflow_index = fields.index(expected_workflow, 2) + except ValueError: + raise SystemExit(66) + trailing = fields[workflow_index + 1:] + if expected_job in trailing and expected_ref not in trailing: raise SystemExit(66) print(status) raise SystemExit(0) From 229119dfe1be78c2b997fd829474ce978110aabd Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 14:19:53 -0700 Subject: [PATCH 31/38] ci: tighten Testbox ownership parsing --- scripts/blacksmith-testbox-cleanup.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/scripts/blacksmith-testbox-cleanup.sh b/scripts/blacksmith-testbox-cleanup.sh index 7607e373f6d..281d91f82d4 100755 --- a/scripts/blacksmith-testbox-cleanup.sh +++ b/scripts/blacksmith-testbox-cleanup.sh @@ -131,7 +131,14 @@ for line in text.splitlines(): except ValueError: raise SystemExit(66) trailing = fields[workflow_index + 1:] - if expected_job in trailing and expected_ref not in trailing: + # Known schemas either end after CREATED (no job/ref columns) or expose + # JOB and REF immediately after WORKFLOW. If either expected field appears, + # require the exact pair in the exact order; reject all ambiguous contexts. + if len(trailing) >= 2 and trailing[:2] == [expected_job, expected_ref]: + pass + elif expected_job in trailing or expected_ref in trailing: + raise SystemExit(66) + elif len(trailing) not in (1, 2): raise SystemExit(66) print(status) raise SystemExit(0) @@ -159,7 +166,7 @@ is_active() { esac } is_known_absence() { - grep -Eiq '(not found|already[[:space:]]+(stopped|completed)|hydration_failed|HTTP[[:space:]]+404|status[[:space:]]+code[[:space:]]+404)' "$1" + grep -Eiq '(not found|already[[:space:]]+(stopped|completed)|hydration_failed|HTTP[[:space:]]+404|status[[:space:]]+code[[:space:]]+404|HTTP[[:space:]]+409|status[[:space:]]+code[[:space:]]+409)' "$1" } inventory_log="$evidence_dir/list-before-stop.log" From 07c1856cd0f745ba8a881fd63b97551448d227c7 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 14:37:30 -0700 Subject: [PATCH 32/38] ci: enforce hydrated toolchain identity --- scripts/blacksmith-cmux-tui-testbox-stage.sh | 38 +++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/scripts/blacksmith-cmux-tui-testbox-stage.sh b/scripts/blacksmith-cmux-tui-testbox-stage.sh index a725c6a4dab..fcecf9c2806 100755 --- a/scripts/blacksmith-cmux-tui-testbox-stage.sh +++ b/scripts/blacksmith-cmux-tui-testbox-stage.sh @@ -126,6 +126,38 @@ if errors: PY } verify_setup_identity +setup_rust_toolchain="$(python3 - "$setup_identity_path" <<'PY' +import json +import pathlib +import sys +record=json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")) +print(record["toolchain"]["rust_toolchain"]) +PY +)" +setup_rustc="$(python3 - "$setup_identity_path" <<'PY' +import json +import pathlib +import sys +record=json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")) +print(record["toolchain"]["rustc"]) +PY +)" +setup_cargo="$(python3 - "$setup_identity_path" <<'PY' +import json +import pathlib +import sys +record=json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")) +print(record["toolchain"]["cargo"]) +PY +)" +setup_zig="$(python3 - "$setup_identity_path" <<'PY' +import json +import pathlib +import sys +record=json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")) +print(record["toolchain"]["zig"]) +PY +)" benchmark_dir="$repo_root/testbox-benchmark" command -v flock >/dev/null || { @@ -312,6 +344,10 @@ rust_toolchain="$(rustup show active-toolchain)" rustc_version="$(rustc --version)" cargo_version="$(cargo --version)" zig_version="$("$zig_bin" version)" +[[ "$rust_toolchain" == "$setup_rust_toolchain" && "$rustc_version" == "$setup_rustc" && "$cargo_version" == "$setup_cargo" && "$zig_version" == "$setup_zig" ]] || { + echo "active Rust/Cargo/Zig toolchain differs from the setup identity marker" >&2 + exit 66 +} export ZIG="$zig_bin" rust_toolchain_file_sha256="$(sha256sum cmux-tui/rust-toolchain.toml | cut -d ' ' -f 1)" cargo_lock_sha256="$(sha256sum cmux-tui/Cargo.lock | cut -d ' ' -f 1)" @@ -333,7 +369,7 @@ case "$stage" in exit 67 fi backup_sha256="$(sha256sum "$backup_candidate" | cut -d ' ' -f 1)" - backup_size="$(wc -c <"$backup_candidate")" + backup_size="$(wc -c <"$backup_candidate" | tr -d '[:space:]')" [[ "$backup_size" =~ ^[0-9]+$ && "$backup_size" -gt 0 ]] || { rm -f "$backup_candidate" echo "source backup is empty" >&2 From 2af052bd41cf6167d761ed6a2b2208f9faef3244 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 14:51:16 -0700 Subject: [PATCH 33/38] ci: bound warmup and preserve interrupted stages --- scripts/blacksmith-bounded-command.sh | 23 ++++++++++++++------ scripts/blacksmith-cmux-tui-testbox-stage.sh | 17 ++++++++++++++- skills/blacksmith-testbox/benchmark.md | 9 +++++--- 3 files changed, 38 insertions(+), 11 deletions(-) diff --git a/scripts/blacksmith-bounded-command.sh b/scripts/blacksmith-bounded-command.sh index ad9aaa88e4c..55614f58599 100755 --- a/scripts/blacksmith-bounded-command.sh +++ b/scripts/blacksmith-bounded-command.sh @@ -27,14 +27,23 @@ argv = sys.argv[2:] if not argv: raise SystemExit("missing command") process = subprocess.Popen(argv, start_new_session=True) -try: - raise SystemExit(process.wait(timeout=seconds)) -except subprocess.TimeoutExpired: - os.killpg(process.pid, signal.SIGTERM) + +def terminate(_signum, _frame): try: + os.killpg(process.pid, signal.SIGTERM) process.wait(timeout=5) - except subprocess.TimeoutExpired: - os.killpg(process.pid, signal.SIGKILL) + except (ProcessLookupError, subprocess.TimeoutExpired): + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass process.wait() - raise SystemExit(124) + raise SystemExit(128 + _signum) + +signal.signal(signal.SIGINT, terminate) +signal.signal(signal.SIGTERM, terminate) +try: + raise SystemExit(process.wait(timeout=seconds)) +except subprocess.TimeoutExpired: + terminate(signal.SIGTERM, None) PY diff --git a/scripts/blacksmith-cmux-tui-testbox-stage.sh b/scripts/blacksmith-cmux-tui-testbox-stage.sh index fcecf9c2806..d7cbb639365 100755 --- a/scripts/blacksmith-cmux-tui-testbox-stage.sh +++ b/scripts/blacksmith-cmux-tui-testbox-stage.sh @@ -221,7 +221,22 @@ finish_source() { fi exit "$result" } -trap finish_source EXIT TERM INT HUP +# Signals must remain failures even when they arrive after a successful command. +# The EXIT trap performs the actual restoration exactly once. +# shellcheck disable=SC2329 # invoked indirectly by signal traps +interrupt_source() { + local signal_name="$1" + trap - TERM INT HUP + case "$signal_name" in + TERM) exit 143 ;; + INT) exit 130 ;; + HUP) exit 129 ;; + esac +} +trap 'interrupt_source TERM' TERM +trap 'interrupt_source INT' INT +trap 'interrupt_source HUP' HUP +trap finish_source EXIT clean_status() { local top_status ghostty_status diff --git a/skills/blacksmith-testbox/benchmark.md b/skills/blacksmith-testbox/benchmark.md index 1103a467134..39bd3360227 100644 --- a/skills/blacksmith-testbox/benchmark.md +++ b/skills/blacksmith-testbox/benchmark.md @@ -173,7 +173,8 @@ cleanup() { # Without the CLI receipt there is no proof that a newly listed box belongs # to this invocation. Report inventory, but never stop another operator's box. set +e - blacksmith testbox list --all >"$OUT/list-after-warmup-failure.log" 2>&1 + ./scripts/blacksmith-bounded-command.sh 60 \ + blacksmith testbox list --all >"$OUT/list-after-warmup-failure.log" 2>&1 after_list_status=$? set -e if (( after_list_status != 0 )); then @@ -195,7 +196,8 @@ blacksmith --version >"$OUT/blacksmith-version.txt" cat "$OUT/blacksmith-version.txt" blacksmith runners catalog >"$OUT/runner-catalog.json" set +e -blacksmith testbox list --all >"$OUT/list-before-warmup.log" 2>&1 +./scripts/blacksmith-bounded-command.sh 60 \ + blacksmith testbox list --all >"$OUT/list-before-warmup.log" 2>&1 before_list_status=$? set -e cat "$OUT/list-before-warmup.log" @@ -204,7 +206,8 @@ if (( before_list_status != 0 )); then exit "$before_list_status" fi set +e -blacksmith testbox warmup "$WORKFLOW" \ +./scripts/blacksmith-bounded-command.sh 1200 \ + blacksmith testbox warmup "$WORKFLOW" \ --ref "$SOURCE_REF" \ --job "$JOB" \ --idle-timeout 30 \ From 89bd10d6859ed99a77fc80543dc6debb37283762 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 15:42:34 -0700 Subject: [PATCH 34/38] ci: compare toolchain identity in cmux-tui --- scripts/blacksmith-cmux-tui-testbox-stage.sh | 2 ++ skills/blacksmith-testbox/SKILL.md | 7 +++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/blacksmith-cmux-tui-testbox-stage.sh b/scripts/blacksmith-cmux-tui-testbox-stage.sh index d7cbb639365..f7294cb52d1 100755 --- a/scripts/blacksmith-cmux-tui-testbox-stage.sh +++ b/scripts/blacksmith-cmux-tui-testbox-stage.sh @@ -355,9 +355,11 @@ verify_identity "$pre_identity_path" runner_label="blacksmith-32vcpu-ubuntu-2404" zig_bin="${CMUX_ZIG:-$(command -v zig)}" +pushd cmux-tui >/dev/null rust_toolchain="$(rustup show active-toolchain)" rustc_version="$(rustc --version)" cargo_version="$(cargo --version)" +popd >/dev/null zig_version="$("$zig_bin" version)" [[ "$rust_toolchain" == "$setup_rust_toolchain" && "$rustc_version" == "$setup_rustc" && "$cargo_version" == "$setup_cargo" && "$zig_version" == "$setup_zig" ]] || { echo "active Rust/Cargo/Zig toolchain differs from the setup identity marker" >&2 diff --git a/skills/blacksmith-testbox/SKILL.md b/skills/blacksmith-testbox/SKILL.md index 966b74aac78..95c51f9a2fa 100644 --- a/skills/blacksmith-testbox/SKILL.md +++ b/skills/blacksmith-testbox/SKILL.md @@ -36,12 +36,15 @@ workflow intentionally grants no other GitHub permissions or workflow secrets. Before using the lane, a repository administrator must create the `blacksmith-testbox-trusted` GitHub environment, configure required reviewers (or an equivalent manual approval rule), disable administrator bypass, and leave -the environment secret set empty. GitHub evaluates that approval before the +the environment secret set empty. Treat this as a required repository control, +not a setting this workflow can create: if the environment is deleted, renamed, +or unconfigured, disable the workflow before any dispatch. GitHub evaluates that approval before the job's first step, including `begin-testbox`. The workflow additionally rejects every ref except reviewed `main` before `begin-testbox`. It cannot verify the environment's reviewer configuration inside the token-bearing job without exposing the same token, so a missing or drifting environment remains an operational stop condition, not a recoverable -workflow state. If the environment does not exist or has no +workflow state. Repository policy must prevent dispatch or merge of this lane +until that control is restored. If the environment does not exist or has no required reviewer, stop: the lane is not production-safe. Never dispatch it for an untrusted PR, fork, branch containing unreviewed workflow/helper changes, or source supplied by an external contributor. When trust changes, stop the old box and warm a From 5c529ba4bc0bee7a9d1acdfd6b5144e60730b9b1 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 15:57:54 -0700 Subject: [PATCH 35/38] ci: own bounded Testbox keepalive --- .../ci-workflow-guard-tests-testbox.yml | 64 ++--------- scripts/blacksmith-testbox-keepalive.sh | 108 ++++++++++++++++++ 2 files changed, 116 insertions(+), 56 deletions(-) create mode 100755 scripts/blacksmith-testbox-keepalive.sh diff --git a/.github/workflows/ci-workflow-guard-tests-testbox.yml b/.github/workflows/ci-workflow-guard-tests-testbox.yml index 0066266a942..1928c783ef9 100644 --- a/.github/workflows/ci-workflow-guard-tests-testbox.yml +++ b/.github/workflows/ci-workflow-guard-tests-testbox.yml @@ -373,61 +373,13 @@ jobs: install -m 600 testbox-benchmark/setup-identity.json "$marker" test -s "$marker" - # run-testbox publishes ready best-effort and then keeps the VM alive. - # Publish and acknowledge readiness before entering that blocking action; - # if this bounded check fails, the always-run action sees a failed job and - # reports hydration_failed instead of advertising an unusable box. - - name: Verify Testbox readiness acknowledgement - if: success() + # Keepalive is checked into this reviewed workflow instead of importing + # an upstream composite that performs an unbounded duplicate ready curl. + # It reports hydration_failed on any failed setup/readiness path and + # bounds every phone-home request before entering its idle loop. + - name: Run trusted Testbox keepalive + if: always() env: - EXPECTED_TESTBOX_ID: ${{ inputs.testbox_id }} + JOB_STATUS: ${{ job.status }} shell: bash - run: | - set -euo pipefail - state=/tmp/.testbox - api_url="$(<"$state/api_url")" - auth_token="$(<"$state/auth_token")" - runner_host="$(<"$state/runner_host")" - runner_ssh_port="$(<"$state/runner_ssh_port")" - working_directory="$(<"$state/working_directory")" - adopted_run_id="$(<"$state/adopted_run_id")" - installation_model_id="$(<"$state/installation_model_id")" - payload="$(jq -n \ - --arg testbox_id "$EXPECTED_TESTBOX_ID" \ - --arg runner_host "$runner_host" \ - --arg runner_ssh_port "$runner_ssh_port" \ - --arg working_directory "$working_directory" \ - --arg adopted_run_id "$adopted_run_id" \ - --argjson installation_model_id "$installation_model_id" \ - '{testbox_id: $testbox_id, installation_model_id: $installation_model_id, status: "ready", ip_address: $runner_host, ssh_port: $runner_ssh_port, working_directory: $working_directory, adopted_run_id: $adopted_run_id, metadata: {}}')" - response="" - for attempt in 1 2 3 4 5; do - set +e - response="$(curl -fsS --connect-timeout 2 --max-time 10 \ - -X POST "$api_url/api/testbox/phone-home" \ - -H 'Content-Type: application/json' \ - -H "Authorization: Bearer $auth_token" \ - --data "$payload" 2>"$RUNNER_TEMP/testbox-ready-phone-home-$attempt.err")" - phone_home_status=$? - set -e - if (( phone_home_status == 0 )); then - # The API may acknowledge a valid phone-home with 204 No Content. - # Curl success is the acknowledgement; a response body is optional. - printf '%s\n' "$response" >"$RUNNER_TEMP/testbox-ready-phone-home-response.txt" - exit 0 - fi - if (( attempt < 5 )); then - sleep $((attempt * 2)) - fi - done - echo "::error::Testbox ready acknowledgement failed after bounded retries" >&2 - cat "$RUNNER_TEMP"/testbox-ready-phone-home-*.err >&2 || true - exit 1 - - # Always hand control back to Testbox. The pinned action receives the - # final job status and reports hydration_failed, rather than ready, when - # any setup or readiness step failed. Rust builds are issued later with - # `blacksmith testbox run`. - - name: Run Testbox - uses: useblacksmith/run-testbox@3f60ff9ceb2c10c3feefa87dc0c6490cffae059d # v2 + hydration-failure reporting - if: always() + run: ./scripts/blacksmith-testbox-keepalive.sh diff --git a/scripts/blacksmith-testbox-keepalive.sh b/scripts/blacksmith-testbox-keepalive.sh new file mode 100755 index 00000000000..1c58926ba78 --- /dev/null +++ b/scripts/blacksmith-testbox-keepalive.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +set -euo pipefail + +# This is the trusted-main replacement for the upstream keepalive composite. +# It deliberately reads the Testbox token only after GitHub has evaluated the +# protected environment and the workflow's main/repository guard. +state=/tmp/.testbox +job_status="${JOB_STATUS:-failure}" + +if [[ ! -d "$state" ]]; then + if [[ "$job_status" == "success" ]]; then + echo "Testbox validation passed, but no registration state was returned" >&2 + exit 1 + fi + echo "Testbox registration state is absent after a failed setup; no phone-home is possible" >&2 + exit 0 +fi + +for required_file in testbox_id installation_model_id auth_token api_url runner_host runner_ssh_port adopted_run_id working_directory; do + test -s "$state/$required_file" || { + echo "missing Testbox state file: $state/$required_file" >&2 + exit 1 + } +done + +testbox_id="$(<"$state/testbox_id")" +installation_model_id="$(<"$state/installation_model_id")" +auth_token="$(<"$state/auth_token")" +api_url="$(<"$state/api_url")" +runner_host="$(<"$state/runner_host")" +runner_ssh_port="$(<"$state/runner_ssh_port")" +working_directory="$(<"$state/working_directory")" +adopted_run_id="$(<"$state/adopted_run_id")" +[[ "$testbox_id" =~ ^tbx_[A-Za-z0-9_-]+$ ]] +[[ "$installation_model_id" =~ ^[0-9]+$ ]] + +phone_home() { + local status="$1" + local payload + payload="$(jq -n \ + --arg testbox_id "$testbox_id" \ + --arg runner_host "$runner_host" \ + --arg runner_ssh_port "$runner_ssh_port" \ + --arg working_directory "$working_directory" \ + --arg adopted_run_id "$adopted_run_id" \ + --arg status "$status" \ + --argjson installation_model_id "$installation_model_id" \ + '{testbox_id: $testbox_id, installation_model_id: $installation_model_id, status: $status, ip_address: $runner_host, ssh_port: $runner_ssh_port, working_directory: $working_directory, adopted_run_id: $adopted_run_id, metadata: {}}')" + curl --fail --silent --show-error --connect-timeout 2 --max-time 10 \ + -X POST "$api_url/api/testbox/phone-home" \ + -H 'Content-Type: application/json' \ + -H "Authorization: Bearer $auth_token" \ + --data "$payload" >/dev/null +} + +phone_home_with_retry() { + local status="$1" + local attempt + for attempt in 1 2 3 4 5; do + if phone_home "$status"; then + return 0 + fi + if (( attempt < 5 )); then + sleep $((attempt * 2)) + fi + done + return 1 +} + +if [[ "$job_status" != "success" ]]; then + if ! phone_home_with_retry hydration_failed; then + echo "warning: could not report hydration_failed" >&2 + fi + echo "Testbox hydration failed; no ready state was published" >&2 + exit 0 +fi + +if ! phone_home_with_retry ready; then + echo "ready phone-home failed after bounded retries" >&2 + phone_home_with_retry hydration_failed || echo "warning: could not report hydration_failed" >&2 + exit 1 +fi + +printf 'Testbox ready: %s (%s)\n' "$testbox_id" "$runner_host" +idle_timeout_minutes="10" +if [[ -s "$state/idle_timeout" ]]; then + idle_timeout_minutes="$(cat "$state/idle_timeout")" +fi +[[ "$idle_timeout_minutes" =~ ^[0-9]+$ ]] || idle_timeout_minutes=10 +last_activity="$(date +%s)" +idle_timeout_seconds=$((idle_timeout_minutes * 60)) + +while :; do + sleep 30 + now="$(date +%s)" + if ss -tnp 2>/dev/null | grep -q ":${runner_ssh_port}[[:>:]]"; then + last_activity="$now" + elif [[ -f "$HOME/.testbox-last-activity" ]]; then + marker_mtime="$(stat -c %Y "$HOME/.testbox-last-activity" 2>/dev/null || stat -f %m "$HOME/.testbox-last-activity")" + if [[ "$marker_mtime" -gt "$last_activity" ]]; then + last_activity="$marker_mtime" + fi + fi + if (( now - last_activity >= idle_timeout_seconds )); then + phone_home_with_retry completed || echo "warning: could not report completed" >&2 + exit 0 + fi +done From 6042481a6ff3193b7ff2170af70d606030fcd1d8 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 16:04:49 -0700 Subject: [PATCH 36/38] ci: fix keepalive SSH activity detection --- scripts/blacksmith-testbox-keepalive.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/blacksmith-testbox-keepalive.sh b/scripts/blacksmith-testbox-keepalive.sh index 1c58926ba78..192e9825a0b 100755 --- a/scripts/blacksmith-testbox-keepalive.sh +++ b/scripts/blacksmith-testbox-keepalive.sh @@ -93,7 +93,7 @@ idle_timeout_seconds=$((idle_timeout_minutes * 60)) while :; do sleep 30 now="$(date +%s)" - if ss -tnp 2>/dev/null | grep -q ":${runner_ssh_port}[[:>:]]"; then + if ss -tnp 2>/dev/null | grep -Eq ":${runner_ssh_port}([^0-9]|$)"; then last_activity="$now" elif [[ -f "$HOME/.testbox-last-activity" ]]; then marker_mtime="$(stat -c %Y "$HOME/.testbox-last-activity" 2>/dev/null || stat -f %m "$HOME/.testbox-last-activity")" From 00634a26e8ffa924d8d5e688bcaede5737034542 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 16:14:51 -0700 Subject: [PATCH 37/38] docs: require independently reviewed cleanup preview --- skills/blacksmith-testbox/SKILL.md | 11 +++++++--- skills/blacksmith-testbox/benchmark.md | 30 ++++++-------------------- 2 files changed, 15 insertions(+), 26 deletions(-) diff --git a/skills/blacksmith-testbox/SKILL.md b/skills/blacksmith-testbox/SKILL.md index 95c51f9a2fa..4b091f2bce5 100644 --- a/skills/blacksmith-testbox/SKILL.md +++ b/skills/blacksmith-testbox/SKILL.md @@ -190,6 +190,11 @@ Never print or download `/tmp/.testbox/auth_token`. ## Remote benchmark stages +The detailed, receipt-producing orchestration in `benchmark.md` is the required +entry point for a complete benchmark. It creates the unique `OUT_ROOT`, receipt, +cleanup token, setup artifact capture, and cleanup preview state. Do not copy +only this stage loop into an ad hoc shell without those prerequisites. + Before each stage, recompute `SOURCE_SHA` and `GHOSTTY_SHA` and repeat the clean pushed-branch preflight. Pass the expected values as validated arguments; the helper does not trust the remote checkout or a caller-supplied expected SHA @@ -331,9 +336,9 @@ inventory, accepts the known terminal states `completed`, `stopped`, `cancelled` already stopped or completed, and polls for up to two minutes while cancellation propagates. Other stop, status, or list failures remain failures. Put it in an `EXIT` trap only after an independent operator exports -`CONFIRM_TESTBOX_STOP=STOP`; the trap performs a preview and confirms its hash -before stopping. Otherwise preserve the benchmark's original exit status and -leave the box for manual cleanup. The detailed benchmark writes a +`CONFIRM_TESTBOX_STOP_SHA` containing the SHA-256 of a separately reviewed +`cleanup-preview.json`; otherwise preserve the benchmark's original exit status +and leave the box for manual cleanup. The detailed benchmark writes a receipt and ownership token for the exact ID returned by warmup; cleanup refuses an ID or token that is not bound to that receipt. If warmup fails before returning an ID, retain before/after inventory but do not automatically stop a box, because diff --git a/skills/blacksmith-testbox/benchmark.md b/skills/blacksmith-testbox/benchmark.md index 39bd3360227..b2d62a1ad8f 100644 --- a/skills/blacksmith-testbox/benchmark.md +++ b/skills/blacksmith-testbox/benchmark.md @@ -147,27 +147,10 @@ cleanup() { local cleanup_status=0 local after_list_status=125 trap - EXIT - if [[ -n "$TBX" && -n "$cleanup_token" && "${CONFIRM_TESTBOX_STOP:-}" == "STOP" ]]; then + if [[ -n "$TBX" && -n "$cleanup_token" && -n "${CONFIRM_TESTBOX_STOP_SHA:-}" ]]; then set +e - scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" "$cleanup_token" PREVIEW - preview_status=$? - if (( preview_status == 75 )); then - if command -v sha256sum >/dev/null; then - preview_sha="$(sha256sum "$OUT/cleanup-preview.json" | awk '{print $1}')" - elif command -v shasum >/dev/null; then - preview_sha="$(shasum -a 256 "$OUT/cleanup-preview.json" | awk '{print $1}')" - else - echo "no SHA-256 utility available for cleanup preview" >&2 - cleanup_status=65 - preview_sha="" - fi - if [[ -n "$preview_sha" ]]; then - scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" "$cleanup_token" "STOP:$preview_sha" - fi - cleanup_status=$? - else - cleanup_status="$preview_status" - fi + scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" "$cleanup_token" "STOP:${CONFIRM_TESTBOX_STOP_SHA}" + cleanup_status=$? set -e else # Without the CLI receipt there is no proof that a newly listed box belongs @@ -185,7 +168,7 @@ cleanup() { echo "warmup returned no owned Testbox receipt; no automatic stop was attempted" >&2 fi fi - if (( result == 0 && cleanup_status != 0 )) && [[ "${CONFIRM_TESTBOX_STOP:-}" == "STOP" ]]; then + if (( result == 0 && cleanup_status != 0 )) && [[ -n "${CONFIRM_TESTBOX_STOP_SHA:-}" ]]; then result="$cleanup_status" fi exit "$result" @@ -439,8 +422,9 @@ scripts/blacksmith-testbox-cleanup.sh "$TBX" "$OUT" "$cleanup_token" PREVIEW ``` A shell `EXIT` trap may call that helper only when an independent operator has -exported `CONFIRM_TESTBOX_STOP=STOP`; otherwise it preserves the benchmark -status, records inventory, and leaves the box for explicit manual cleanup. +exported `CONFIRM_TESTBOX_STOP_SHA` with the SHA-256 of a separately reviewed +`cleanup-preview.json`; otherwise it preserves the benchmark status, records +inventory, and leaves the box for explicit manual cleanup. Warmup writes `testbox-receipt.json` and an ownership token bound to the exact returned ID; cleanup refuses a mismatched ID or token. If warmup fails before returning an ID, retain before/after inventories but do not automatically stop a From 1f21d216f91ffa234dd71dccac41537a909b22c8 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 13 Aug 2026 16:39:55 -0700 Subject: [PATCH 38/38] ci: pin reviewed Testbox branch and SHA --- .../ci-workflow-guard-tests-testbox.yml | 119 ++++++++++++------ skills/blacksmith-testbox/SKILL.md | 83 +++++++----- skills/blacksmith-testbox/benchmark.md | 38 ++++-- 3 files changed, 159 insertions(+), 81 deletions(-) diff --git a/.github/workflows/ci-workflow-guard-tests-testbox.yml b/.github/workflows/ci-workflow-guard-tests-testbox.yml index 1928c783ef9..c3a6639b873 100644 --- a/.github/workflows/ci-workflow-guard-tests-testbox.yml +++ b/.github/workflows/ci-workflow-guard-tests-testbox.yml @@ -8,16 +8,16 @@ on: required: true type: string source_sha: - description: "Optional full source SHA; if set, it must equal the pushed branch head" + description: "Optional SHA assertion; it cannot select a candidate revision" required: false default: "" type: string -# The Blacksmith CLI supplies testbox_id and dispatches the selected pushed -# branch. It does not expose arbitrary workflow inputs, so source_sha is an -# optional manual-dispatch assertion; the benchmark's remote guard is required -# for the CLI path below. The protected environment is configured out of band; -# this workflow is intentionally unusable unless that repository policy exists. +# This is an explicitly allowlisted reviewed-branch lane. Blacksmith supplies +# only testbox_id, so the selected --ref and github.sha identify the candidate. +# The protected environment pins both values in its non-secret +# BLACKSMITH_TESTBOX_REVIEWED_REF and BLACKSMITH_TESTBOX_REVIEWED_SHA variables. +# The workflow must never be dispatched for an untrusted PR, fork, or branch. permissions: {} concurrency: @@ -42,17 +42,59 @@ jobs: # cleanup; the Testbox itself still has its separate idle timeout. timeout-minutes: 120 steps: - # begin-testbox must be the first step. GitHub environment approval and - # its trusted-branch policy are evaluated before this step; do not weaken - # those external controls or move this action into branch-controlled code. - # It attaches the VM requested by `blacksmith testbox warmup` and leaves - # it alive after this setup job. - - name: Guard trusted repository and source ref - if: github.repository != 'manaflow-ai/cmux' || github.ref != 'refs/heads/main' + # Environment approval and its exact branch policy are evaluated before + # this preflight. Keep every candidate-controlled check before + # begin-testbox, and keep begin-testbox pinned in this reviewed workflow. + # The action attaches the VM requested by `blacksmith testbox warmup` and + # leaves it alive after this setup job. + - name: Validate reviewed branch and exact candidate SHA + env: + EXPECTED_INPUT_SHA: ${{ inputs.source_sha }} + REVIEWED_REF: ${{ vars.BLACKSMITH_TESTBOX_REVIEWED_REF }} + REVIEWED_SHA: ${{ vars.BLACKSMITH_TESTBOX_REVIEWED_SHA }} + DISPATCH_REF: ${{ github.ref }} + DISPATCH_SHA: ${{ github.sha }} + EVENT_NAME: ${{ github.event_name }} + REPOSITORY: ${{ github.repository }} shell: bash run: | - echo "::error::Testbox token-bearing setup is restricted to manaflow-ai/cmux reviewed main" >&2 - exit 1 + set -euo pipefail + [[ "$REPOSITORY" == "manaflow-ai/cmux" ]] || { + echo "::error::this Testbox lane is only valid for manaflow-ai/cmux" >&2 + exit 1 + } + [[ "$EVENT_NAME" == "workflow_dispatch" ]] || { + echo "::error::Testbox setup must be dispatched manually, never from a PR event" >&2 + exit 1 + } + [[ "$REVIEWED_REF" =~ ^[A-Za-z0-9._/-]+$ && "$REVIEWED_REF" != *..* && "$REVIEWED_REF" != */ && "$REVIEWED_REF" != *//* ]] || { + echo "::error::protected environment must pin one valid branch name in BLACKSMITH_TESTBOX_REVIEWED_REF" >&2 + exit 1 + } + [[ "$REVIEWED_SHA" =~ ^[0-9a-f]{40}$ ]] || { + echo "::error::protected environment must pin one lowercase full SHA in BLACKSMITH_TESTBOX_REVIEWED_SHA" >&2 + exit 1 + } + [[ "$DISPATCH_REF" == "refs/heads/$REVIEWED_REF" ]] || { + echo "::error::Testbox warmup ref is not the protected reviewed branch" >&2 + exit 1 + } + [[ "$DISPATCH_SHA" == "$REVIEWED_SHA" ]] || { + echo "::error::workflow SHA is not the protected reviewed candidate SHA" >&2 + exit 1 + } + remote_sha="$(git ls-remote --exit-code "https://github.com/${REPOSITORY}.git" "refs/heads/$REVIEWED_REF" | awk 'NR == 1 { print $1 }')" + [[ "$remote_sha" == "$REVIEWED_SHA" ]] || { + echo "::error::the reviewed branch moved during dispatch: remote=$remote_sha expected=$REVIEWED_SHA" >&2 + exit 1 + } + if [[ -n "$EXPECTED_INPUT_SHA" ]]; then + [[ "$EXPECTED_INPUT_SHA" =~ ^[0-9a-f]{40}$ && "$EXPECTED_INPUT_SHA" == "$REVIEWED_SHA" ]] || { + echo "::error::source_sha is only an assertion and must equal the protected reviewed SHA" >&2 + exit 1 + } + fi + printf 'reviewed source ref: %s\nreviewed source SHA: %s\n' "$REVIEWED_REF" "$REVIEWED_SHA" - name: Validate Testbox ID before token-bearing action env: @@ -70,11 +112,14 @@ jobs: with: testbox_id: ${{ inputs.testbox_id }} - - name: Validate trusted branch dispatch and source identity + - name: Revalidate reviewed branch and source identity env: EXPECTED_INPUT_SHA: ${{ inputs.source_sha }} + REVIEWED_REF: ${{ vars.BLACKSMITH_TESTBOX_REVIEWED_REF }} + REVIEWED_SHA: ${{ vars.BLACKSMITH_TESTBOX_REVIEWED_SHA }} DISPATCH_SHA: ${{ github.sha }} DISPATCH_REF: ${{ github.ref }} + EVENT_NAME: ${{ github.event_name }} TESTBOX_ID: ${{ inputs.testbox_id }} REPOSITORY: ${{ github.repository }} shell: bash @@ -84,38 +129,38 @@ jobs: echo "::error::this Testbox lane is only valid for manaflow-ai/cmux" >&2 exit 1 } + [[ "$EVENT_NAME" == "workflow_dispatch" ]] || { + echo "::error::Testbox setup must be dispatched manually, never from a PR event" >&2 + exit 1 + } [[ "$TESTBOX_ID" =~ ^tbx_[A-Za-z0-9_-]+$ ]] || { echo "::error::malformed Testbox ID" >&2 exit 1 } - [[ "$DISPATCH_SHA" =~ ^[0-9a-f]{40}$ ]] || { - echo "::error::github.sha must be a lowercase full commit SHA" >&2 + [[ "$REVIEWED_REF" =~ ^[A-Za-z0-9._/-]+$ && "$REVIEWED_REF" != *..* && "$REVIEWED_REF" != */ && "$REVIEWED_REF" != *//* ]] || { + echo "::error::protected environment reviewed ref is malformed" >&2 exit 1 } - if [[ ! "$DISPATCH_REF" =~ ^refs/heads/[A-Za-z0-9._/-]+$ || "$DISPATCH_REF" == *..* || "$DISPATCH_REF" == */ || "$DISPATCH_REF" == *//* ]]; then - echo "::error::Testbox warmup must dispatch a pushed branch ref, not a raw SHA, tag, or malformed ref" >&2 + [[ "$REVIEWED_SHA" =~ ^[0-9a-f]{40}$ ]] || { + echo "::error::protected environment reviewed SHA is malformed" >&2 exit 1 - fi + } + [[ "$DISPATCH_REF" == "refs/heads/$REVIEWED_REF" && "$DISPATCH_SHA" == "$REVIEWED_SHA" ]] || { + echo "::error::reviewed branch or candidate SHA changed during setup" >&2 + exit 1 + } + remote_sha="$(git ls-remote --exit-code "https://github.com/${REPOSITORY}.git" "refs/heads/$REVIEWED_REF" | awk 'NR == 1 { print $1 }')" + [[ "$remote_sha" == "$REVIEWED_SHA" ]] || { + echo "::error::the reviewed branch moved during setup: remote=$remote_sha expected=$REVIEWED_SHA" >&2 + exit 1 + } if [[ -n "$EXPECTED_INPUT_SHA" ]]; then - [[ "$EXPECTED_INPUT_SHA" =~ ^[0-9a-f]{40}$ ]] || { - echo "::error::source_sha must be a lowercase full commit SHA" >&2 - exit 1 - } - [[ "$EXPECTED_INPUT_SHA" == "$DISPATCH_SHA" ]] || { - echo "::error::source_sha does not equal the workflow dispatch SHA" >&2 + [[ "$EXPECTED_INPUT_SHA" == "$REVIEWED_SHA" ]] || { + echo "::error::source_sha does not equal the protected reviewed SHA" >&2 exit 1 } fi - remote_sha="$(git ls-remote --exit-code "https://github.com/${REPOSITORY}.git" "$DISPATCH_REF" | awk 'NR == 1 { print $1 }')" - [[ "$remote_sha" == "$DISPATCH_SHA" ]] || { - echo "::error::the pushed branch moved during dispatch: remote=$remote_sha workflow=$DISPATCH_SHA" >&2 - exit 1 - } - # The workflow's source SHA is the only attestation Blacksmith can - # carry through workflow_dispatch. The local preflight and every - # remote stage must compare against this exact value; never infer an - # older revision from the mutable branch after begin-testbox. - printf 'trusted source ref: %s\ntrusted source SHA: %s\n' "$DISPATCH_REF" "$DISPATCH_SHA" + printf 'reviewed source ref: %s\nreviewed source SHA: %s\n' "$REVIEWED_REF" "$REVIEWED_SHA" - name: Checkout exact dispatch commit uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/skills/blacksmith-testbox/SKILL.md b/skills/blacksmith-testbox/SKILL.md index 4b091f2bce5..1e7002e5b5c 100644 --- a/skills/blacksmith-testbox/SKILL.md +++ b/skills/blacksmith-testbox/SKILL.md @@ -35,32 +35,43 @@ workflow intentionally grants no other GitHub permissions or workflow secrets. Before using the lane, a repository administrator must create the `blacksmith-testbox-trusted` GitHub environment, configure required reviewers -(or an equivalent manual approval rule), disable administrator bypass, and leave -the environment secret set empty. Treat this as a required repository control, -not a setting this workflow can create: if the environment is deleted, renamed, -or unconfigured, disable the workflow before any dispatch. GitHub evaluates that approval before the -job's first step, including `begin-testbox`. The workflow additionally rejects every ref except reviewed `main` before -`begin-testbox`. It cannot verify the environment's reviewer configuration -inside the token-bearing job without exposing the same token, so a missing or -drifting environment remains an operational stop condition, not a recoverable -workflow state. Repository policy must prevent dispatch or merge of this lane -until that control is restored. If the environment does not exist or has no -required reviewer, stop: the lane is not production-safe. Never dispatch it -for an untrusted PR, fork, branch containing unreviewed workflow/helper changes, -or source supplied by an external contributor. When trust changes, stop the old box and warm a -fresh one. +(or an equivalent manual approval rule), disable administrator bypass, leave the +environment secret set empty, and set these two **environment configuration +variables**: + +* `BLACKSMITH_TESTBOX_REVIEWED_REF`, one exact branch name without the + `refs/heads/` prefix; +* `BLACKSMITH_TESTBOX_REVIEWED_SHA`, one exact lowercase 40-character commit + SHA for that branch. + +Configure the environment's deployment branch rule to the same exact branch, +with no wildcard or fork rule. Update the two variables only after an +independent maintainer has reviewed that exact commit, including this workflow +and the remote helper. Treat the environment, its variables, and its branch +rule as required repository controls, not settings this workflow can create. +GitHub evaluates reviewer approval and the branch rule before the job's first +step, including `begin-testbox`. The workflow then compares `github.ref`, +`github.sha`, and a fresh `git ls-remote` result to those exact pins before +exposing the Testbox token. If the environment is deleted, renamed, +unconfigured, or stale, disable the lane and stop rather than changing the +workflow to proceed. If it has no required reviewer, no exact branch rule, or +no exact ref/SHA pins, the lane is not production-safe. + +This is reviewed-branch mode, not PR validation. Never dispatch it for an +untrusted PR, fork, external contributor, wildcard branch, or commit whose +workflow/helper changes have not been independently reviewed. The Testbox +command environment can read `/tmp/.testbox/auth_token` and anything else +exposed to the GitHub job. The token is not sandboxed, and `permissions: +contents: read` does not change that trust boundary. When the reviewed commit +or trust context changes, stop the old box and warm a fresh one. The helper retains the `CMUX_TESTBOX_REMOTE=1` guard for accidental local launches, and additionally requires the Blacksmith VM kernel metadata marker and matching `/tmp/.testbox` state. The environment flag remains caller -controlled and is not an authentication mechanism. The protected environment -and trusted-maintainer policy are the security boundary. Verify before every -use that `blacksmith-testbox-trusted` still has required reviewers, no secrets, -administrator bypass disabled, and no broad branch policy admitting unreviewed -refs. The checked-in workflow independently permits only reviewed `main`; the -feature branch is for code review and cannot expose a Testbox token. The -workflow cannot manufacture the environment settings, so drift makes the lane -unavailable rather than safe. +controlled and is not an authentication mechanism. The protected environment, +its exact ref/SHA pins, and trusted-maintainer review are the security boundary. +The workflow cannot manufacture those controls, so configuration drift makes +the lane unavailable rather than safe. * Never run `cargo`, `rustc`, `rustup`, `zig build`, or another Rust/Zig build command on Lawrence's Mac. This includes local fallback builds and local @@ -106,8 +117,10 @@ set -euo pipefail git submodule update --init ghostty cd "$(git rev-parse --show-toplevel)" SOURCE_REF="$(git symbolic-ref --short HEAD)" -[[ "$SOURCE_REF" == "main" ]] || { - echo "the token-bearing Testbox lane only accepts reviewed main; use a PR workflow for feature branches" >&2 +REVIEWED_REF="$(gh variable get BLACKSMITH_TESTBOX_REVIEWED_REF --env blacksmith-testbox-trusted --repo manaflow-ai/cmux --json value --jq .value)" +REVIEWED_SHA="$(gh variable get BLACKSMITH_TESTBOX_REVIEWED_SHA --env blacksmith-testbox-trusted --repo manaflow-ai/cmux --json value --jq .value)" +[[ "$SOURCE_REF" == "$REVIEWED_REF" ]] || { + echo "HEAD is not the exact branch pinned by blacksmith-testbox-trusted" >&2 exit 1 } if [[ ! "$SOURCE_REF" =~ ^[A-Za-z0-9._/-]+$ || "$SOURCE_REF" == *..* || "$SOURCE_REF" == */ || "$SOURCE_REF" == *//* ]]; then @@ -115,6 +128,10 @@ if [[ ! "$SOURCE_REF" =~ ^[A-Za-z0-9._/-]+$ || "$SOURCE_REF" == *..* || "$SOURCE exit 1 fi SOURCE_SHA="$(git rev-parse HEAD)" +[[ "$REVIEWED_SHA" =~ ^[0-9a-f]{40}$ && "$SOURCE_SHA" == "$REVIEWED_SHA" ]] || { + echo "HEAD is not the exact commit pinned by blacksmith-testbox-trusted" >&2 + exit 1 +} ghostty_entry="$(git ls-tree HEAD ghostty)" [[ "$ghostty_entry" =~ ^160000[[:space:]]commit[[:space:]][0-9a-f]{40}[[:space:]]ghostty$ ]] || { echo "HEAD:ghostty is not a gitlink" >&2 @@ -140,15 +157,18 @@ source SHA, commit tree, GitHub `ghostty` gitlink, initialized Ghostty HEAD, and clean status before building. A moved branch or dirty/mismatched checkout fails closed; do not silently warm another revision. -The workflow has an optional `source_sha` workflow-dispatch input for a direct -GitHub dispatch. The Blacksmith CLI supplies `testbox_id` but does not expose -arbitrary workflow inputs, so the normal CLI path uses `github.sha` plus the -remote assertion in the helper. A supplied `source_sha` must be a lowercase -40-character SHA equal to the branch dispatch SHA. +The workflow's optional `source_sha` input is an assertion only; it cannot +select a candidate because Blacksmith supplies only `testbox_id`. The CLI +warmup selects `SOURCE_REF`, while GitHub supplies `github.sha`; the protected +environment pins both and the workflow checks the remote branch again. A +supplied `source_sha` must equal that same protected SHA. This is the mechanism +that permits a reviewed feature branch to validate its final PR head without +executing an unreviewed branch. ## Warmup and identity capture -Warm the validated branch, never its SHA: +Warm the exact branch pinned by the protected environment, never an +unreviewed branch or a raw SHA: ```bash WORKFLOW=.github/workflows/ci-workflow-guard-tests-testbox.yml @@ -196,7 +216,8 @@ cleanup token, setup artifact capture, and cleanup preview state. Do not copy only this stage loop into an ad hoc shell without those prerequisites. Before each stage, recompute `SOURCE_SHA` and `GHOSTTY_SHA` and repeat the -clean pushed-branch preflight. Pass the expected values as validated arguments; +clean pushed-branch preflight, including the protected reviewed ref/SHA pins. +Pass the expected values as validated arguments; the helper does not trust the remote checkout or a caller-supplied expected SHA without comparing it to Git metadata: diff --git a/skills/blacksmith-testbox/benchmark.md b/skills/blacksmith-testbox/benchmark.md index b2d62a1ad8f..3e7c5d851fc 100644 --- a/skills/blacksmith-testbox/benchmark.md +++ b/skills/blacksmith-testbox/benchmark.md @@ -19,11 +19,16 @@ setup-only GitHub job on the remote Linux runner. | Remote output | `testbox-benchmark/` | A repository administrator must configure the protected environment with -required reviewers, no secrets, administrator bypass disabled, and a trusted -branch policy before this plan is usable. Verify that configuration before each -run. The lane must never run untrusted PR or fork code. If the configuration is -missing or drifts, stop instead of treating the environment name as a guard. `begin-testbox` exposes its auth token to -commands in the Testbox, so `contents: read` is not a trust boundary. The +required reviewers, no secrets, administrator bypass disabled, and an exact +selected-branch policy before this plan is usable. The environment must also +set the non-secret variables `BLACKSMITH_TESTBOX_REVIEWED_REF` (one branch name +without `refs/heads/`) and `BLACKSMITH_TESTBOX_REVIEWED_SHA` (one lowercase +40-character SHA). Both variables and the environment branch rule must point to +the same exact reviewed commit. Verify that configuration before each run. The +lane must never run untrusted PR or fork code. If the configuration is missing +or drifts, stop instead of treating the environment name as a guard. +`begin-testbox` exposes its auth token to commands in the Testbox, so +`contents: read` is not a trust boundary and the token is not sandboxed. The repository does not currently pin a checksum-verified Blacksmith CLI artifact; that is a trusted-lane operational limitation. Use only the organization- approved CLI, record `blacksmith --version`, and stop rather than silently @@ -51,8 +56,10 @@ set -euo pipefail git submodule update --init ghostty cd "$(git rev-parse --show-toplevel)" SOURCE_REF="$(git symbolic-ref --short HEAD)" -[[ "$SOURCE_REF" == "main" ]] || { - echo "the token-bearing Testbox lane only accepts reviewed main; use a PR workflow for feature branches" >&2 +REVIEWED_REF="$(gh variable get BLACKSMITH_TESTBOX_REVIEWED_REF --env blacksmith-testbox-trusted --repo manaflow-ai/cmux --json value --jq .value)" +REVIEWED_SHA="$(gh variable get BLACKSMITH_TESTBOX_REVIEWED_SHA --env blacksmith-testbox-trusted --repo manaflow-ai/cmux --json value --jq .value)" +[[ "$SOURCE_REF" == "$REVIEWED_REF" ]] || { + echo "HEAD is not the exact branch pinned by blacksmith-testbox-trusted" >&2 exit 1 } if [[ ! "$SOURCE_REF" =~ ^[A-Za-z0-9._/-]+$ || "$SOURCE_REF" == *..* || "$SOURCE_REF" == */ || "$SOURCE_REF" == *//* ]]; then @@ -60,6 +67,10 @@ if [[ ! "$SOURCE_REF" =~ ^[A-Za-z0-9._/-]+$ || "$SOURCE_REF" == *..* || "$SOURCE exit 1 fi SOURCE_SHA="$(git rev-parse HEAD)" +[[ "$REVIEWED_SHA" =~ ^[0-9a-f]{40}$ && "$SOURCE_SHA" == "$REVIEWED_SHA" ]] || { + echo "HEAD is not the exact commit pinned by blacksmith-testbox-trusted" >&2 + exit 1 +} SOURCE_TREE_SHA="$(git rev-parse 'HEAD^{tree}')" ghostty_entry="$(git ls-tree HEAD ghostty)" [[ "$ghostty_entry" =~ ^160000[[:space:]]commit[[:space:]][0-9a-f]{40}[[:space:]]ghostty$ ]] || { @@ -270,12 +281,13 @@ if (( status_ready != 0 )); then fi ``` -The workflow validates that the dispatch ref is reviewed `main`, is a pushed -branch, that `github.sha` is a full SHA, and that the branch still resolves to -that SHA. If a direct GitHub dispatch supplies the optional `source_sha` input, -it must equal `github.sha`. The Blacksmith CLI path relies on the same full SHA -passed to the remote helper because the CLI only supplies `testbox_id` to -workflow inputs. +The workflow validates that the dispatch ref exactly matches the protected +reviewed branch, that `github.sha` exactly matches the protected reviewed SHA, +and that the remote branch still resolves to that SHA. If a direct GitHub +dispatch supplies the optional `source_sha` input, it is an assertion and must +equal the protected SHA. The Blacksmith CLI path needs no arbitrary workflow +input: it supplies `testbox_id`, while `--ref` selects the explicitly reviewed +branch and `github.sha` carries the pinned candidate. The workflow concurrency group serializes setup requests by Testbox ID, even when source SHAs differ. The remote `flock` begins after Blacksmith's rsync, so it protects stage/build/artifact writes only. Blacksmith exposes no pre-rsync