From 4e4c184b402abc1ac4f25b62c786503b9e623c02 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 12 Oct 2026 17:23:00 +0000 Subject: [PATCH] feat(sandbox): CI capabilities benchmark for Actions-product readiness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds benchmarks/sandbox/ci-capabilities.bench.ts: a real-fidelity gate that replays the benchmarks-platform CI executor path inside one sandbox — exec semantics, re-attach, concurrent + detached exec, workspace/runtime, egress reachability, git clone, resource floor, verbatim dockerd bring-up and act install scripts (ci-runtime.ts, ported from lib/ci/executor/ act-runtime.ts), runner-image pull, a detached act workflow run to green, and a second sandbox verifying the instance deadline is honored. Each check reports {passed, error} into a capability matrix in record.data and results/sandbox-ci-capabilities/latest.json so a provider's readiness (or exact gap) is visible at a glance. A provider passing every check can be dropped into the Actions product. Also extracts the shared LARGE_RESOURCE_OPTIONS sizing map from dax.bench.ts into resource-options.ts and repoints the sandbox-capabilities workflow at the new bench (120min job timeout). Co-Authored-By: Noah Kiser --- .github/workflows/sandbox-capabilities.yml | 22 +- benchmarks/sandbox/ci-capabilities-results.ts | 68 ++ benchmarks/sandbox/ci-capabilities.bench.ts | 626 ++++++++++++++++++ benchmarks/sandbox/ci-runtime.ts | 560 ++++++++++++++++ benchmarks/sandbox/dax.bench.ts | 59 +- benchmarks/sandbox/resource-options.ts | 63 ++ benchmarks/sandbox/types.ts | 15 + package.json | 1 + 8 files changed, 1345 insertions(+), 69 deletions(-) create mode 100644 benchmarks/sandbox/ci-capabilities-results.ts create mode 100644 benchmarks/sandbox/ci-capabilities.bench.ts create mode 100644 benchmarks/sandbox/ci-runtime.ts create mode 100644 benchmarks/sandbox/resource-options.ts diff --git a/.github/workflows/sandbox-capabilities.yml b/.github/workflows/sandbox-capabilities.yml index 4240d146..2676f3b8 100644 --- a/.github/workflows/sandbox-capabilities.yml +++ b/.github/workflows/sandbox-capabilities.yml @@ -1,4 +1,4 @@ -name: Sandbox Capabilities Matrix +name: Sandbox CI Capabilities Matrix # computesdk:secrets=NSC_TOKEN # computesdk:secrets-env=declared @@ -50,7 +50,7 @@ jobs: id-token: write runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe;permissions.additional_grant=vault/object:*:reveal # runs-on: self-hosted - timeout-minutes: 30 + timeout-minutes: 120 env: SHOULD_RUN: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.provider == '' || github.event.inputs.provider == matrix.provider }} strategy: @@ -185,7 +185,7 @@ jobs: . benchmarks/scripts/load-vault-secrets.sh "$KEYS" RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - pnpm exec -- tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/capabilities.bench.ts \ + pnpm exec -- tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/ci-capabilities.bench.ts \ --provider "$PROVIDER" \ --iterations "$ITERATIONS" \ --concurrency "$CONCURRENCY" \ @@ -194,7 +194,7 @@ jobs: if: always() && env.SHOULD_RUN == 'true' uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: results-capabilities-${{ matrix.provider }} + name: results-ci-capabilities-${{ matrix.provider }} path: results/ if-no-files-found: ignore retention-days: 7 @@ -246,7 +246,7 @@ jobs: uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: path: artifacts/ - pattern: results-capabilities-* + pattern: results-ci-capabilities-* - name: Merge results continue-on-error: true run: pnpm exec -- tsx benchmarks/src/merge-results.ts --input artifacts @@ -258,9 +258,9 @@ jobs: const fs = require('fs'); const path = require('path'); - const latestPath = path.join('results', 'sandbox-capabilities', 'latest.json'); + const latestPath = path.join('results', 'sandbox-ci-capabilities', 'latest.json'); if (!fs.existsSync(latestPath)) { - core.summary.addHeading('Sandbox Capabilities Matrix', 2); + core.summary.addHeading('Sandbox CI Capabilities Matrix', 2); core.summary.addRaw('No capabilities results were generated.'); await core.summary.write(); return; @@ -271,7 +271,7 @@ jobs: const featureNames = results.length > 0 ? Object.keys(results[0].features || {}) : []; const summary = core.summary; - summary.addHeading('Sandbox Capabilities Matrix', 2); + summary.addHeading('Sandbox CI Capabilities Matrix', 2); summary.addRaw(`

Timestamp: ${data.timestamp || 'unknown'}
Providers: ${results.length}

`); const header = ['Provider', ...featureNames]; @@ -308,10 +308,10 @@ jobs: run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - mkdir -p results/sandbox-capabilities - git add results/sandbox-capabilities/ + mkdir -p results/sandbox-ci-capabilities + git add results/sandbox-ci-capabilities/ git diff --cached --quiet && echo "No changes to commit" && exit 0 - git commit -m "chore: update sandbox capabilities results [skip ci]" + git commit -m "chore: update sandbox ci capabilities results [skip ci]" branch="${GITHUB_REF#refs/heads/}" for attempt in 1 2 3 4 5; do diff --git a/benchmarks/sandbox/ci-capabilities-results.ts b/benchmarks/sandbox/ci-capabilities-results.ts new file mode 100644 index 00000000..3189d7ec --- /dev/null +++ b/benchmarks/sandbox/ci-capabilities-results.ts @@ -0,0 +1,68 @@ +import { mkdirSync, writeFileSync, copyFileSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import type { ParticipantRecords, ResolvedRunConfig } from '@benchsdk/runner'; +import type { TaskResultRecord } from '@benchsdk/client'; +import type { ProviderCapabilityMatrix, CapabilityMatrixResult } from './types.js'; + +const RESULTS_VERSION = '1.0'; + +/** + * Prefer the task-level `features` matrix the capability task returns in + * `record.data`. As a fallback, reconstruct from per-step measurements when a + * step failed and the matrix wasn't emitted. + */ +function extractMatrix(record: TaskResultRecord): ProviderCapabilityMatrix { + const data = record.data ?? {}; + if (data && typeof data === 'object' && 'features' in data && data.features && typeof data.features === 'object' && !Array.isArray(data.features)) { + return data.features as unknown as ProviderCapabilityMatrix; + } + + const matrix: ProviderCapabilityMatrix = {}; + for (const step of record.steps ?? []) { + const stepData = step.data ?? {}; + if (stepData && typeof stepData === 'object' && 'passed' in stepData) { + const { passed, error } = stepData as { passed: boolean; error?: string }; + matrix[step.name] = passed ? { passed: true } : { passed: false, ...(error ? { error } : {}) }; + } + } + return matrix; +} + +/** Aggregate per-provider capability matrices and write dated + latest JSON. */ +export async function writeCiCapabilitiesResults( + participants: ParticipantRecords[], + opts: { resultsDir: string; runConfig: ResolvedRunConfig }, +): Promise { + const results: CapabilityMatrixResult[] = participants.map((participant) => { + const record = participant.records[0]; + const features = record ? extractMatrix(record) : {}; + return { provider: participant.participant, features }; + }); + + mkdirSync(opts.resultsDir, { recursive: true }); + + const timestamp = new Date().toISOString(); + const outPath = path.join(opts.resultsDir, `${timestamp.slice(0, 10)}.json`); + const latestPath = path.join(opts.resultsDir, 'latest.json'); + + const output = { + version: RESULTS_VERSION, + timestamp, + environment: { + node: process.version, + platform: os.platform(), + arch: os.arch(), + }, + config: { + benchmarkSlug: 'sandbox-ci-capabilities', + iterations: opts.runConfig.iterations, + concurrency: opts.runConfig.concurrency, + }, + results, + }; + + writeFileSync(outPath, JSON.stringify(output, null, 2)); + copyFileSync(outPath, latestPath); + console.log(`CI capabilities results written: ${outPath} -> ${latestPath}`); +} diff --git a/benchmarks/sandbox/ci-capabilities.bench.ts b/benchmarks/sandbox/ci-capabilities.bench.ts new file mode 100644 index 00000000..1975ccdd --- /dev/null +++ b/benchmarks/sandbox/ci-capabilities.bench.ts @@ -0,0 +1,626 @@ +/** + * CI capabilities benchmark — the Actions-product readiness gate. + * + * Runs the same in-sandbox path benchmarks-platform's CI executor uses, end + * to end, inside one provider sandbox per iteration: exec semantics, + * fresh-process re-attach, concurrent + detached exec, workspace and runtime, + * egress reachability, git clone, the resource floor, the real dockerd + * bring-up and act install scripts (./ci-runtime.ts, ported verbatim from + * lib/ci/executor/act-runtime.ts), the runner-image pull, and a detached + * `act push` that runs a real workflow to a green exit. A second sandbox is + * created with a short `timeout` to prove the instance deadline is honored — + * the auto-pause / hard-cap failure the namespace 1h bug shipped. + * + * A provider that passes every check can be placed into the Actions product; + * each failing check names the gap to fix. Checks never abort on first + * failure: each runs inside its own `step()` and records {passed, error} + * into the capability matrix (`record.data.features`), so a provider that + * fails dockerd still reports git/egress/detached. + * + * bench run benchmarks/sandbox/ci-capabilities.bench.ts + * bench run benchmarks/sandbox/ci-capabilities.bench.ts --provider e2b,modal + */ +import '../src/env.js'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { defineBenchmarkConfig, defineTask } from '@benchsdk/runner'; +import type { JsonObject } from '@benchsdk/client'; +import type { CommandResult, CreateSandboxOptions, SandboxInterface } from 'computesdk'; +import { withTimeout } from '../src/util/timeout.js'; +import { formatError } from '../src/util/error.js'; +import { sandboxId } from '../src/util/sandbox-id.js'; +import { providers } from './providers.js'; +import type { ProviderConfig, ProviderCapabilityMatrix } from './types.js'; +import { getSandboxOptionsWithResources } from './resource-options.js'; +import { + ACT_ARTIFACTS_DIR, + ACT_ARTIFACT_SERVER_PORT, + ACT_ACTION_CACHE_DIR, + BLAXEL_RUNNER_IMAGE, + DOCKER_BIN_DIR, + actBinaryPath, + actImageArgs, + buildActInstallScript, + buildBlaxelRunnerImageScript, + buildDockerBringUpScript, + buildRunnerImagePullScript, + dockerLocalBridgeGatewayScript, + dockerdClientEnv, + runnerImage, + sandboxPrimaryAddrScript, + shellQuote, +} from './ci-runtime.js'; +import { writeCiCapabilitiesResults } from './ci-capabilities-results.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + +const CREATE_TIMEOUT_MS = 300_000; +const EXEC_TIMEOUT_MS = 60_000; +const LONG_EXEC_TIMEOUT_MS = 600_000; // dockerd bring-up + act install +const PULL_TIMEOUT_MS = 1_200_000; // catthehacker/ubuntu:act-latest is ~1.4GB +const ACT_RUN_TIMEOUT_MS = 1_500_000; // detached act launch, polled +const DESTROY_TIMEOUT_MS = 15_000; +const SANDBOX_LEASE_MS = 3_600_000; // the battery can run ~45min on slow egress +const DEADLINE_MS = 240_000; // second sandbox's requested instance lifetime +const DEADLINE_GRACE_MS = 120_000; // how late a kill may land and still be honored +const DEADLINE_EARLY_MS = Math.floor(DEADLINE_MS * 0.75); // dying sooner fails +const DEADLINE_POLL_MS = 15_000; + +/** The CI floor the Actions product provisions: ~4 vCPU / 8 GiB / 16 GiB. */ +const FLOOR_VCPU = 4; +const FLOOR_MEM_MIB = 7_800; // 8 GiB minus what `free` hides behind the kernel +const FLOOR_DISK_AVAIL_MIB = 14_000; + +const WORK_DIR = '/tmp/ci-cap'; +const ACT_RUN_DIR = '/tmp/act/run'; + +const WORKFLOW_YML = `name: ci-capability-probe +on: push +jobs: + probe: + runs-on: ubuntu-latest + steps: + - name: Probe marker + run: echo "ci_capability_probe_ok" +`; + +const EVENT_JSON = '{"ref":"refs/heads/main"}\n'; + +const EGRESS_HOSTS: readonly { name: string; url: string }[] = [ + { name: 'github', url: 'https://github.com/' }, + { name: 'github_api', url: 'https://api.github.com/' }, + { name: 'codeload', url: 'https://codeload.github.com/octocat/Hello-World/tar.gz/refs/heads/master' }, + { name: 'npm', url: 'https://registry.npmjs.org/' }, + { name: 'docker_static', url: 'https://download.docker.com/' }, + { name: 'docker_hub', url: 'https://registry-1.docker.io/v2/' }, + { name: 'platform', url: 'https://platform.computesdk.com/' }, + { name: 'nodejs', url: 'https://nodejs.org/dist/' }, +]; + +export const config = defineBenchmarkConfig({ + benchmarkSlug: `sandbox-ci-capabilities${process.env.DAILY_BENCH_SLUG ? `-${process.env.DAILY_BENCH_SLUG}` : ''}`, + benchmarkName: `Sandbox CI Capabilities${process.env.DAILY_BENCH_NAME ? ` - ${process.env.DAILY_BENCH_NAME}` : ''}`, + iterations: 1, + concurrency: 1, + participants: providers, + display: { + metrics: [ + { key: 'checksPassedPct', label: 'Checks passed', unit: '%', direction: 'higher-better', decimals: 0 }, + { key: 'checksPassed', label: 'Checks passed', direction: 'higher-better', decimals: 0 }, + { key: 'checksTotal', label: 'Checks total', direction: 'higher-better', decimals: 0 }, + ], + steps: [ + { key: 'create', label: 'Create sandbox' }, + { key: 'exec', label: 'Exec semantics' }, + { key: 'reconnect', label: 'Re-attach (getById)' }, + { key: 'concurrent', label: 'Concurrent exec' }, + { key: 'detached', label: 'Detached process survives exec' }, + { key: 'workspace', label: 'Writable workspace' }, + { key: 'runtime', label: 'Node runtime' }, + { key: 'egress', label: 'Egress reachability' }, + { key: 'git', label: 'Git clone' }, + { key: 'sizing', label: 'Resource floor' }, + { key: 'deadline', label: 'Deadline honored' }, + { key: 'dockerd', label: 'Docker daemon bring-up' }, + { key: 'act_install', label: 'act install' }, + { key: 'runner_image', label: 'Runner image pull' }, + { key: 'act_run', label: 'act workflow run' }, + { key: 'destroy', label: 'Destroy sandbox' }, + ], + overview: { defaultMetric: 'checksPassedPct', defaultLayout: 'ranking' }, + }, + scoring: { + metrics: [ + { key: 'checksPassedPct', unit: '%', higherIsBetter: true, floor: 0, ceiling: 100, weights: { median: 0.6, p95: 0.25, p99: 0.15 } }, + ], + }, + onComplete: (outcome) => + writeCiCapabilitiesResults(outcome.participants, { + resultsDir: path.resolve(__dirname, '../../results/sandbox-ci-capabilities'), + runConfig: outcome.config, + }), +}); + +function b64(content: string): string { + return Buffer.from(content, 'utf8').toString('base64'); +} + +function tailLines(output: string | undefined, n: number): string { + return (output ?? '') + .split('\n') + .map((l) => l.trimEnd()) + .filter((l) => l !== '') + .slice(-n) + .join('\n'); +} + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +export const task = defineTask(async (ctx) => { + const { participant, step, measure, log } = ctx; + const provider = participant.name; + const compute = participant.createCompute() as { + sandbox: { + create(options?: CreateSandboxOptions): Promise; + getById(id: string): Promise; + destroy(id: string): Promise; + }; + }; + + const matrix: ProviderCapabilityMatrix = {}; + let sandbox: SandboxInterface | undefined; + let deadlineSandbox: SandboxInterface | undefined; + + const createOptions: CreateSandboxOptions = { + ...getSandboxOptionsWithResources(provider, participant.sandboxOptions), + // The battery outlives every TTI-oriented TTL in providers.ts (e2b's + // 10-minute sandboxOptions.timeout would kill the box mid-act-run). + timeout: SANDBOX_LEASE_MS, + }; + + async function exec( + command: string, + opts: { timeoutMs?: number; env?: Record } = {}, + ): Promise { + const timeoutMs = opts.timeoutMs ?? EXEC_TIMEOUT_MS; + return (await withTimeout( + sandbox!.runCommand(command, { timeout: timeoutMs, env: opts.env }), + timeoutMs + 15_000, + `exec timed out after ${timeoutMs}ms`, + )) as CommandResult; + } + + function mustExit(result: CommandResult, what: string): CommandResult { + if (result.exitCode !== 0) { + throw new Error(`${what} failed (exit ${result.exitCode}): ${tailLines(result.stderr || result.stdout, 6) || 'no output'}`); + } + return result; + } + + async function check(name: string, fn: () => Promise, opts: { needsSandbox?: boolean } = {}): Promise { + await step(name, async () => { + if (opts.needsSandbox !== false && !sandbox) { + const error = 'sandbox creation failed'; + measure({ passed: false, error }); + matrix[name] = { passed: false, error }; + return; + } + try { + const detail = (await fn()) ?? {}; + measure({ passed: true, ...detail }); + matrix[name] = { passed: true }; + } catch (err) { + const error = formatError(err); + measure({ passed: false, error }); + matrix[name] = { passed: false, error }; + log(`check ${name} failed`, { level: 'warn', meta: { error } }); + } + }); + } + + await step('create', async () => { + sandbox = await withTimeout( + compute.sandbox.create(createOptions), + participant.timeout ?? CREATE_TIMEOUT_MS, + 'Sandbox creation timed out', + ); + matrix['create'] = { passed: true }; + measure({ passed: true }); + }).catch((err: unknown) => { + const error = formatError(err); + measure({ passed: false, error }); + matrix['create'] = { passed: false, error }; + log('sandbox create failed', { level: 'error', meta: { provider, error } }); + }); + + try { + await check('exec', async () => { + const marker = 'ci-cap-marker-41a7'; + const echoResult = mustExit( + await exec('echo "$CI_CAP_MARKER"', { env: { CI_CAP_MARKER: marker } }), + 'marker echo', + ); + if (!echoResult.stdout.includes(marker)) { + throw new Error(`env injection missing: expected ${marker}, got "${tailLines(echoResult.stdout, 2)}"`); + } + const splitResult = await exec(`sh -c 'echo out-marker; echo err-marker >&2; exit 3'`); + if (splitResult.exitCode !== 3) { + throw new Error(`exit code not propagated: expected 3, got ${splitResult.exitCode}`); + } + if (!splitResult.stdout.includes('out-marker') || !splitResult.stderr.includes('err-marker')) { + throw new Error( + `stdout/stderr not separated: stdout="${tailLines(splitResult.stdout, 2)}" stderr="${tailLines(splitResult.stderr, 2)}"`, + ); + } + const killStart = Date.now(); + let killed = false; + try { + const r = await exec('sleep 30', { timeoutMs: 4_000 }); + killed = r.exitCode !== 0; + } catch { + killed = true; + } + const killMs = Date.now() - killStart; + if (!killed || killMs > 12_000) { + throw new Error(`per-command timeout ignored: sleep 30 returned after ${killMs}ms`); + } + return { timeoutKillMs: killMs }; + }); + + await check('reconnect', async () => { + const handle = await withTimeout( + compute.sandbox.getById(sandbox!.sandboxId), + EXEC_TIMEOUT_MS, + 'getById timed out', + ); + if (!handle) throw new Error('getById returned null for a live sandbox'); + mustExit(await withTimeout(handle.runCommand('echo reconnect-ok'), EXEC_TIMEOUT_MS, 'reconnect exec timed out'), 'reconnect exec'); + }); + + await check('concurrent', async () => { + const started = Date.now(); + const [a, b] = await Promise.all([ + exec('sleep 2; echo leg-a', { timeoutMs: 30_000 }), + exec('sleep 4; echo leg-b', { timeoutMs: 30_000 }), + ]); + const wallMs = Date.now() - started; + mustExit(a, 'concurrent exec leg a'); + mustExit(b, 'concurrent exec leg b'); + return { wallMs, overlapped: wallMs < 5_500 }; + }); + + await check('detached', async () => { + const marker = 'detached-ok-9f3c'; + const detachedFile = `${WORK_DIR}-detached`; + mustExit( + await exec( + `rm -f ${detachedFile}; nohup setsid sh -c 'sleep 3; echo ${marker} > ${detachedFile}' >/dev/null 2>&1 { + const result = mustExit( + await exec(`id -u; id -un 2>/dev/null || true; mkdir -p ${WORK_DIR}/work ${WORK_DIR}/act ${ACT_RUN_DIR} && touch ${WORK_DIR}/w && echo writable`), + 'workspace probe', + ); + const uid = result.stdout.split('\n')[0]?.trim() ?? 'unknown'; + return { execUid: uid }; + }); + + await check('runtime', async (): Promise => { + const result = mustExit( + await exec('(command -v node || command -v nodejs || true) | head -1'), + 'runtime probe', + ); + const nodeBin = result.stdout.trim(); + if (nodeBin) { + const version = mustExit(await exec(`${nodeBin} -v`), 'node -v').stdout.trim(); + return { node: version }; + } + // The product bootstraps a pinned Node itself — absent node is workable + // as long as the box can fetch it (musl arm64 remains a real gap). + const fetcher = mustExit( + await exec('for t in curl wget python3 busybox; do command -v $t && break; done'), + 'fetch-tool probe', + ).stdout.trim(); + if (!fetcher) throw new Error('no node runtime and no fetch tool for the pinned-node bootstrap'); + return { node: 'absent', note: `product bootstrap required (fetch via ${fetcher})` }; + }); + + await check('egress', async () => { + const script = [ + 'ci_probe() {', + ' ci_p_name="$1"; ci_p_url="$2"; ci_p_code=0; ci_p_ms=0', + ' if command -v curl >/dev/null 2>&1; then', + ' ci_p_out=$(curl -sSL -o /dev/null -w \'%{http_code} %{time_total}\' --max-time 30 "$ci_p_url" 2>/dev/null || true)', + ' ci_p_code=${ci_p_out%% *}', + ' ci_p_ms=$(printf \'%s\' "${ci_p_out##* }" | awk \'{printf "%d", $1*1000}\' 2>/dev/null || echo 0)', + ' elif command -v python3 >/dev/null 2>&1; then', + ' ci_p_out=$(python3 -c \'import sys,time,urllib.request,urllib.error', + 'u=sys.argv[1]; t=time.time()', + 'try:', + ' r=urllib.request.urlopen(u,timeout=30); r.read(64); print(r.status, int((time.time()-t)*1000))', + 'except urllib.error.HTTPError as e:', + ' print(e.code, int((time.time()-t)*1000))', + 'except Exception:', + ' print(0, int((time.time()-t)*1000))\' "$ci_p_url" 2>/dev/null || true)', + ' ci_p_code=${ci_p_out%% *}; ci_p_ms=${ci_p_out##* }', + ' elif command -v wget >/dev/null 2>&1; then', + ' ci_p_start=$(date +%s)', + ' ci_p_code=$(wget -O /dev/null --timeout=30 -S "$ci_p_url" 2>&1 | awk \'/ HTTP\\//{c=$2} END{print c+0}\')', + ' ci_p_ms=$(( ($(date +%s) - ci_p_start) * 1000 ))', + ' elif command -v busybox >/dev/null 2>&1; then', + ' busybox wget -q -O /dev/null "$ci_p_url" >/dev/null 2>&1 && ci_p_code=200', + ' fi', + ' echo "BENCH_EGRESS $ci_p_name ${ci_p_code:-0} ${ci_p_ms:-0}"', + '}', + ...EGRESS_HOSTS.map((h) => `ci_probe ${h.name} ${shellQuote(h.url)}`), + ].join('\n'); + const result = mustExit(await exec(script, { timeoutMs: 180_000 }), 'egress probe'); + const latencies: Record = {}; + const unreachable: string[] = []; + for (const line of result.stdout.split('\n')) { + const m = line.match(/^BENCH_EGRESS (\S+) (\d+) (\d+)$/); + if (!m) continue; + const code = Number(m[2]); + const ms = Number(m[3]); + if (code > 0) { + if (ms > 0) latencies[`egress_${m[1]}Ms`] = ms; + } else { + unreachable.push(m[1]); + } + } + const probed = Object.keys(latencies).length + unreachable.length; + if (unreachable.length > 0) { + throw new Error(`unreachable: ${unreachable.join(', ')} (${probed}/${EGRESS_HOSTS.length} probed)`); + } + if (probed === 0) throw new Error('egress probe produced no results'); + return latencies as JsonObject; + }); + + await check('git', async () => { + const script = [ + 'if ! command -v git >/dev/null 2>&1; then', + ' if command -v apt-get >/dev/null 2>&1; then apt-get update -qq >/dev/null 2>&1; apt-get install -y -qq git >/dev/null 2>&1;', + ' elif command -v dnf >/dev/null 2>&1; then dnf install -y -q git >/dev/null 2>&1;', + ' elif command -v yum >/dev/null 2>&1; then yum install -y -q git >/dev/null 2>&1;', + ' elif command -v apk >/dev/null 2>&1; then apk add --no-cache git >/dev/null 2>&1; fi', + 'fi', + 'command -v git >/dev/null 2>&1 || { echo "git is missing and could not be installed" >&2; exit 1; }', + `rm -rf ${WORK_DIR}/clone`, + `git clone -q --depth 1 https://github.com/octocat/Hello-World ${WORK_DIR}/clone`, + `test -d ${WORK_DIR}/clone/.git`, + ].join('\n'); + const result = await exec(script, { timeoutMs: 120_000 }); + if (result.exitCode !== 0) { + throw new Error(`git clone failed (exit ${result.exitCode}): ${tailLines(result.stderr || result.stdout, 6)}`); + } + }); + + await check('sizing', async () => { + const result = mustExit( + await exec('nproc; awk \'/MemTotal/{print int($2/1024)}\' /proc/meminfo; df -m / | awk \'NR==2{print $4}\''), + 'sizing probe', + ); + const [cpus, memMiB, diskAvailMiB] = result.stdout.split('\n').map((l) => Number(l.trim())); + const detail = { vcpus: cpus, memMiB, diskAvailMiB }; + const short: string[] = []; + if (!(cpus >= FLOOR_VCPU)) short.push(`${cpus} vCPU < ${FLOOR_VCPU}`); + if (!(memMiB >= FLOOR_MEM_MIB)) short.push(`${memMiB} MiB < ${FLOOR_MEM_MIB} MiB`); + if (!(diskAvailMiB >= FLOOR_DISK_AVAIL_MIB)) short.push(`${diskAvailMiB} MiB disk < ${FLOOR_DISK_AVAIL_MIB} MiB`); + if (short.length > 0) { + throw new Error(`below the Actions floor: ${short.join(', ')}`); + } + return detail; + }); + + // Runs its own short-lived sandbox, so it still executes when the main + // box failed to create. + await check('deadline', async () => { + // Second sandbox with a short requested lifetime: the instance must + // still exec until ~the deadline and be gone shortly after — auto-pause + // and silently ignored timeouts are both failures for the product. + let firstDeadMs: number | undefined; + try { + deadlineSandbox = await withTimeout( + compute.sandbox.create({ ...(participant.sandboxOptions ?? {}), timeout: DEADLINE_MS }), + participant.timeout ?? CREATE_TIMEOUT_MS, + 'deadline sandbox creation timed out', + ); + } catch (err) { + throw new Error(`deadline sandbox create failed: ${formatError(err)}`); + } + const t0 = Date.now(); + const probe = async (): Promise => { + try { + const handle = await compute.sandbox.getById(deadlineSandbox!.sandboxId); + if (!handle) return false; + const r = await withTimeout(handle.runCommand('true', { timeout: 15_000 }), 20_000, 'probe timed out'); + return r.exitCode === 0; + } catch { + return false; + } + }; + // Two consecutive failures mark the box dead: a single control-plane + // blip must not read as an early kill. + let misses = 0; + while (Date.now() - t0 < DEADLINE_MS + DEADLINE_GRACE_MS) { + if (await probe()) { + misses = 0; + } else { + misses += 1; + if (misses >= 2) { + firstDeadMs = Date.now() - t0; + break; + } + } + await sleep(DEADLINE_POLL_MS); + } + const elapsedS = Math.round((firstDeadMs ?? Date.now() - t0) / 1000); + if (firstDeadMs === undefined) { + throw new Error(`deadline not honored: still exec'ing ${Math.round((DEADLINE_MS + DEADLINE_GRACE_MS) / 1000)}s past a ${DEADLINE_MS / 1000}s timeout`); + } + if (firstDeadMs < DEADLINE_EARLY_MS) { + throw new Error(`box died ~${elapsedS}s in, before the requested ${DEADLINE_MS / 1000}s deadline`); + } + return { deadlineKillS: elapsedS }; + }, { needsSandbox: false }); + + await check('dockerd', async () => { + const prePull = provider === 'blaxel' ? [] : [runnerImage(provider)]; + const result = await exec( + buildDockerBringUpScript(process.env, { provider, prePullImages: prePull }), + { timeoutMs: LONG_EXEC_TIMEOUT_MS }, + ); + if (result.exitCode !== 0) { + log('dockerd bring-up output', { level: 'warn', meta: { tail: tailLines(result.stdout, 15), stderr: tailLines(result.stderr, 15) } }); + throw new Error(`dockerd bring-up failed (exit ${result.exitCode}): ${tailLines(result.stderr || result.stdout, 8)}`); + } + }); + + await check('act_install', async () => { + const result = await exec(buildActInstallScript(process.env), { timeoutMs: LONG_EXEC_TIMEOUT_MS }); + if (result.exitCode !== 0) { + throw new Error(`act install failed (exit ${result.exitCode}): ${tailLines(result.stderr || result.stdout, 8)}`); + } + }); + + await check('runner_image', async () => { + const image = runnerImage(provider); + const script = [ + `export PATH="${DOCKER_BIN_DIR}:$PATH"`, + ...dockerdClientEnv(provider), + provider === 'blaxel' && image === BLAXEL_RUNNER_IMAGE + ? buildBlaxelRunnerImageScript(image) + : buildRunnerImagePullScript([image]), + ].join('\n'); + const result = await exec(script, { timeoutMs: PULL_TIMEOUT_MS }); + if (result.exitCode !== 0) { + log('runner image output', { level: 'warn', meta: { tail: tailLines(result.stdout, 10), stderr: tailLines(result.stderr, 10) } }); + throw new Error(`runner image ${image} unavailable (exit ${result.exitCode}): ${tailLines(result.stderr || result.stdout, 8)}`); + } + return { image }; + }); + + await check('act_run', async () => { + const runScript = [ + `cd ${WORK_DIR}/work || exit 1`, + `export PATH="${DOCKER_BIN_DIR}:$PATH"`, + ...dockerdClientEnv(provider), + ...dockerLocalBridgeGatewayScript('artifact_server_addr'), + 'if [ -z "$artifact_server_addr" ]; then', + ...sandboxPrimaryAddrScript('artifact_server_addr'), + 'fi', + 'artifact_server_addr_arg=""', + 'cache_server_addr_arg=""', + 'if [ -n "$artifact_server_addr" ]; then', + ' artifact_server_addr_arg="--artifact-server-addr $artifact_server_addr"', + ' cache_server_addr_arg="--cache-server-addr $artifact_server_addr"', + 'fi', + 'set -- $artifact_server_addr_arg $cache_server_addr_arg', + [ + `${actBinaryPath()} "$@" push`, + `-W ${shellQuote(`${WORK_DIR}/work/workflow.yml`)}`, + '--job probe', + `--eventpath ${shellQuote(`${WORK_DIR}/act/event.json`)}`, + `--secret-file ${shellQuote(`${WORK_DIR}/act/secrets.env`)}`, + `--var-file ${shellQuote(`${WORK_DIR}/act/vars.env`)}`, + `--env-file ${shellQuote(`${WORK_DIR}/act/env.env`)}`, + '--json', + `--artifact-server-path ${shellQuote(ACT_ARTIFACTS_DIR)}`, + `--artifact-server-port ${ACT_ARTIFACT_SERVER_PORT}`, + `--action-cache-path ${shellQuote(ACT_ACTION_CACHE_DIR)}`, + '--action-offline-mode', + '--bind', + ...(provider === 'cloudflare' ? ['--network host'] : []), + ...(provider === 'blaxel' ? ['--pull=false'] : []), + ...actImageArgs(provider), + ].join(' '), + `rc=$?; printf '%s' "$rc" > ${ACT_RUN_DIR}/act.exit; exit $rc`, + ].join('\n'); + + const stage = [ + `mkdir -p ${WORK_DIR}/work ${WORK_DIR}/act ${ACT_RUN_DIR} ${ACT_ACTION_CACHE_DIR}`, + `cd ${WORK_DIR}/work && git init -q 2>/dev/null || true`, + `echo ${shellQuote(b64(WORKFLOW_YML))} | base64 -d > ${WORK_DIR}/work/workflow.yml`, + `echo ${shellQuote(b64(EVENT_JSON))} | base64 -d > ${WORK_DIR}/act/event.json`, + `: > ${WORK_DIR}/act/secrets.env`, + `: > ${WORK_DIR}/act/vars.env`, + `: > ${WORK_DIR}/act/env.env`, + `echo ${shellQuote(b64(runScript))} | base64 -d > ${WORK_DIR}/act-run.sh`, + `rm -f ${ACT_RUN_DIR}/act.exit`, + ].join(' && '); + mustExit(await exec(stage), 'act staging'); + + mustExit( + await exec( + `nohup setsid sh ${WORK_DIR}/act-run.sh >${ACT_RUN_DIR}/act.log 2>${ACT_RUN_DIR}/act.err ${ACT_RUN_DIR}/act.pid; echo launched`, + { timeoutMs: 15_000 }, + ), + 'act launch', + ); + + const t0 = Date.now(); + let exitCode: number | undefined; + while (Date.now() - t0 < ACT_RUN_TIMEOUT_MS) { + await sleep(15_000); + const poll = await exec(`cat ${ACT_RUN_DIR}/act.exit 2>/dev/null || true`, { timeoutMs: 15_000 }); + const text = poll.stdout.trim(); + if (text !== '') { + exitCode = Number(text); + break; + } + } + const logs = await exec(`tail -40 ${ACT_RUN_DIR}/act.log 2>/dev/null; tail -10 ${ACT_RUN_DIR}/act.err 2>/dev/null`, { timeoutMs: 15_000 }) + .catch(() => undefined); + if (exitCode === undefined) { + log('act run timed out', { level: 'warn', meta: { tail: tailLines(logs?.stdout, 30) } }); + throw new Error(`act run did not finish within ${Math.round(ACT_RUN_TIMEOUT_MS / 60000)}m: ${tailLines(logs?.stdout, 10)}`); + } + if (exitCode !== 0) { + log('act run failed', { level: 'warn', meta: { tail: tailLines(logs?.stdout, 30) } }); + throw new Error(`act exited ${exitCode}: ${tailLines(logs?.stdout, 12)}`); + } + return { actRunS: Math.round((Date.now() - t0) / 1000) }; + }); + } finally { + if (sandbox || deadlineSandbox) { + await step('destroy', async () => { + if (deadlineSandbox) { + await withTimeout(deadlineSandbox.destroy(), DESTROY_TIMEOUT_MS, 'deadline destroy timeout').catch(() => {}); + } + if (sandbox) { + await withTimeout(sandbox.destroy(), participant.destroyTimeoutMs ?? DESTROY_TIMEOUT_MS, 'Destroy timed out'); + } + }).then( + () => { + matrix['destroy'] = { passed: true }; + }, + (err: unknown) => { + const error = formatError(err); + log('destroy failed', { level: 'warn', meta: { error } }); + matrix['destroy'] = { passed: false, error }; + }, + ); + } + if (sandbox) { + log('sandbox', { level: 'info', meta: { provider, sandboxId: sandboxId(sandbox) } }); + } + } + + const checksTotal = Object.keys(matrix).length; + const checksPassed = Object.values(matrix).filter((f) => f.passed).length; + const checksPassedPct = checksTotal === 0 ? 0 : Math.round((checksPassed / checksTotal) * 100); + measure({ checksPassed, checksTotal, checksPassedPct }); + return { + data: { features: matrix, checksPassed, checksTotal, checksPassedPct } as unknown as JsonObject, + }; +}); diff --git a/benchmarks/sandbox/ci-runtime.ts b/benchmarks/sandbox/ci-runtime.ts new file mode 100644 index 00000000..af608fab --- /dev/null +++ b/benchmarks/sandbox/ci-runtime.ts @@ -0,0 +1,560 @@ +/** + * In-sandbox Actions runtime bring-up, ported from benchmarks-platform's + * `lib/ci/executor/act-runtime.ts` and `lib/ci/executor/shell.ts` — the exact + * scripts the CI executor runs inside a provider's sandbox. Keep them in + * sync: when the product's bring-up changes, this file must change the same + * way or the benchmark stops proving what the product does. + * + * The bench runs these verbatim over `runCommand`; only the platform-only + * legs are dropped (action-source seeding, cache/mirror/egress shims, + * sccache, supervisor) because they need control-plane infrastructure a + * benchmark does not have. + */ + +/** Single-quotes a value for `sh`, so no metacharacter in it is live. */ +export function shellQuote(value: string): string { + return `'${value.replaceAll("'", `'\\''`)}'`; +} + +export const STATIC_DOCKER_VERSION = '27.4.1'; +export const DEFAULT_ACT_VERSION = 'v0.2.89'; + +/** Under `/tmp` so the install survives a step but not the sandbox. */ +export const ACT_DIR = '/tmp/act'; +/** act's built-in artifact server writes here until the sandbox is destroyed. */ +export const ACT_ARTIFACTS_DIR = `${ACT_DIR}/artifacts`; +export const ACT_ARTIFACT_SERVER_PORT = 34_567; +/** + * act's action-source cache (`--action-cache-path`). Pinned instead of act's + * `~/.cache/act` default so the path does not depend on the exec user's HOME. + */ +export const ACT_ACTION_CACHE_DIR = '/tmp/ci-action-cache'; +/** Where a statically installed Docker lands, and so on every later `PATH`. */ +export const DOCKER_BIN_DIR = '/tmp/act/docker'; +/** The daemon's own tools — `iptables` above all — live here, not in `bin`. */ +const SBIN_DIRS = '/usr/local/sbin:/usr/sbin:/sbin'; + +/** + * The image act runs a job in for a given `runs-on:` label. + * `catthehacker/ubuntu` is what act's own documentation recommends for a + * workflow that expects a GitHub-hosted runner. + */ +export const RUNNER_IMAGES: Readonly> = { + 'ubuntu-latest': 'catthehacker/ubuntu:act-latest', + 'ubuntu-24.04': 'catthehacker/ubuntu:act-24.04', + 'ubuntu-22.04': 'catthehacker/ubuntu:act-22.04', + 'ubuntu-20.04': 'catthehacker/ubuntu:act-20.04', +}; + +/** + * The runner image every ubuntu label maps to on blaxel: its ~1GB rootfs + * overlay cannot hold catthehacker's 1.4GB+ toolchain image, so a slim + * node+bash+git image is built in-sandbox instead. + */ +export const BLAXEL_RUNNER_IMAGE = 'localhost/ci-blaxel-runner:latest'; + +/** Build the blaxel runner image inside the sandbox (node:22-alpine + bash + git). */ +export function buildBlaxelRunnerImageScript(image: string): string { + return [ + `if ! docker image inspect ${shellQuote(image)} >/dev/null 2>&1; then`, + ` echo "Building runner image ${image} (node:22-alpine + bash + git)"`, + ' docker system prune -af >/dev/null 2>&1 || true', + ` echo '--- host resolv.conf ---'; cat /etc/resolv.conf || true`, + ` docker network inspect bridge --format 'bridge ipv6={{.EnableIPv6}}' 2>/dev/null || true`, + ` printf 'FROM node:22-alpine\\nRUN apk add --no-cache bash git\\n' | docker build -t ${shellQuote(image)} - || printf 'FROM node:22-alpine\\nRUN apk add --no-cache bash git\\n' | docker build --network=host -t ${shellQuote(image)} -`, + ` if ! docker image inspect ${shellQuote(image)} >/dev/null 2>&1; then`, + ` echo "Runner image ${image} failed to build" >&2`, + ' exit 1', + ' fi', + ' docker builder prune -af >/dev/null 2>&1 || true', + ' df -h /var/lib/docker || true', + 'fi', + ].join('\n'); +} + +/** The label→image table act is given, with the env overrides the product honors. */ +export function runnerImages( + provider?: string, + env: NodeJS.ProcessEnv = process.env, +): Record { + const images = { ...RUNNER_IMAGES }; + if (provider === 'blaxel') { + for (const label of Object.keys(images)) images[label] = BLAXEL_RUNNER_IMAGE; + } + const providerOverride = + provider === undefined + ? undefined + : env[`CI_${provider.toUpperCase().replace(/-/g, '_')}_ACT_IMAGE`]; + const override = providerOverride ?? env.CI_ACT_IMAGE; + if (override) images['ubuntu-latest'] = override; + return images; +} + +/** + * `docker pull` for the images the job's container will run. act hides pull + * errors (their detail travels inside the streamed response body logged at + * debug level), so the pull runs in the open here instead. + */ +export function buildRunnerImagePullScript( + images: readonly string[], +): string { + return Array.from(new Set(images)) + .flatMap((image) => [ + `if docker image inspect ${shellQuote(image)} >/dev/null 2>&1; then`, + ` echo "Runner image ${image} is already present"`, + 'else', + ` echo "Pulling runner image ${image}"`, + ` if ! docker pull --quiet ${shellQuote(image)}; then`, + ` docker system prune -af >/dev/null 2>&1 || true`, + ` echo "Retrying pull of ${image} after docker system prune"`, + ` if ! docker pull --quiet ${shellQuote(image)}; then`, + ` echo "Failed to pull ${image}" >&2`, + ` docker system df >&2 || true`, + ` df -h >&2 || true`, + ' exit 1', + ' fi', + ' fi', + 'fi', + ]) + .join('\n'); +} + +/** + * One fetch verb for the downloads below. A minimal image may ship none of + * the usual tools, so the helper walks what is actually installed: curl, + * wget, busybox's wget applet, then python3's stdlib. + */ +export const FETCH_FN = [ + 'ci_fetch() {', + ' ci_fetch_url=$1', + ' ci_fetch_out=$2', + ' ci_fetch_timeout=${3:-}', + ' if command -v curl >/dev/null 2>&1; then', + ' curl -fsSL ${ci_fetch_timeout:+--max-time "$ci_fetch_timeout"} "$ci_fetch_url" -o "$ci_fetch_out"; return', + ' fi', + ' if command -v wget >/dev/null 2>&1; then', + ' wget -q ${ci_fetch_timeout:+--timeout="$ci_fetch_timeout"} -O "$ci_fetch_out" "$ci_fetch_url"; return', + ' fi', + ' if command -v busybox >/dev/null 2>&1; then', + ' ci_fetch_has_wget=', + ' for ci_fetch_applet in $(busybox --list 2>/dev/null); do', + ' [ "$ci_fetch_applet" = wget ] && ci_fetch_has_wget=1', + ' done', + ' if [ "$ci_fetch_has_wget" = 1 ]; then', + ' busybox wget -q ${ci_fetch_timeout:+-T "$ci_fetch_timeout"} -O "$ci_fetch_out" "$ci_fetch_url"; return', + ' fi', + ' fi', + ' if command -v python3 >/dev/null 2>&1; then', + " python3 -c 'import shutil, sys, urllib.request", + 'with urllib.request.urlopen(sys.argv[1], timeout=float(sys.argv[3]) if len(sys.argv) > 3 and sys.argv[3] else None) as r, open(sys.argv[2], "wb") as f:', + " shutil.copyfileobj(r, f)' \"$ci_fetch_url\" \"$ci_fetch_out\" \"$ci_fetch_timeout\"; return", + ' fi', + ' echo "Cannot fetch $ci_fetch_url: the image has none of curl, wget, busybox or python3" >&2', + ' return 1', + '}', +] as const; + +/** + * A SHA-256 hex digest of $1 through whichever hasher the image carries — + * sha256sum, shasum, or openssl — or nothing when it has none. + */ +export const SHA256_FN = [ + 'ci_sha256() {', + ' ci_sha_line=', + ' if command -v sha256sum >/dev/null 2>&1; then', + ' ci_sha_line=$(sha256sum "$1" 2>/dev/null)', + ' elif command -v shasum >/dev/null 2>&1; then', + ' ci_sha_line=$(shasum -a 256 "$1" 2>/dev/null)', + ' elif command -v openssl >/dev/null 2>&1; then', + ' ci_sha_line=$(openssl dgst -sha256 "$1" 2>/dev/null)', + ' fi', + ' [ -z "$ci_sha_line" ] && return 1', + ' case "$ci_sha_line" in', + ' *=*) printf "%s\\n" "${ci_sha_line##*= }" ;;', + ' *) printf "%s\\n" "${ci_sha_line%% *}" ;;', + ' esac', + '}', +] as const; + +/** Marker holding the digest of a successfully installed act binary. */ +export const ACT_SHA256_PATH = `${ACT_DIR}/.act.sha256`; + +export function actBinaryPath(): string { + return `${ACT_DIR}/act`; +} + +/** + * `-P label=image` for every label we map, so act never reaches its + * interactive "which image size?" prompt. + */ +export function actImageArgs( + provider?: string, + env: NodeJS.ProcessEnv = process.env, +): string[] { + return Object.entries(runnerImages(provider, env)).map( + ([label, image]) => `-P ${shellQuote(`${label}=${image}`)}`, + ); +} + +/** The image act pulls for the default label. */ +export function runnerImage(provider?: string, env: NodeJS.ProcessEnv = process.env): string { + return runnerImages(provider, env)['ubuntu-latest']; +} + +/** + * Socket and data-root for the executor's own dockerd, used on providers that + * boot a managed daemon of their own (blaxel's docker-in-sandbox image runs + * one at sandbox start — killing it severed the sandbox's exec channel + * entirely, so their daemon is left untouched). + */ +const EXECUTOR_DOCKER_SOCKET = 'unix:///var/run/docker-ce.sock'; +const EXECUTOR_DOCKER_DATA_ROOT = '/var/lib/docker/executor'; +const EXECUTOR_DOCKER_BRIDGE = 'ci-docker0'; +const EXECUTOR_DOCKER_CIDR = '172.31.0.1/24'; +const EXECUTOR_DOCKER_PIDFILE = '/var/run/docker-ce.pid'; +const EXECUTOR_DOCKER_EXEC_ROOT = '/var/run/docker-ce'; + +/** Shell lines pointing every docker client (and act, via DOCKER_HOST) at the executor's daemon on providers that need it. */ +export function dockerdClientEnv(provider: string | undefined): string[] { + return providerNeedsOwnDockerd(provider) + ? [`export DOCKER_HOST=${shellQuote(EXECUTOR_DOCKER_SOCKET)}`] + : []; +} + +/** + * Whether the provider needs the executor's own dockerd rather than whatever + * daemon (or none) the sandbox image ships. + */ +export function providerNeedsOwnDockerd(provider: string | undefined): boolean { + return provider === 'blaxel'; +} + +/** + * Extra dockerd flags for providers whose sandbox network needs more than + * defaults. Blaxel is IPv6-only; Cloudflare cannot touch iptables and has a + * read-only ip_forward sysctl. + */ +function dockerdExtraArgs( + provider: string | undefined, + env: NodeJS.ProcessEnv, +): string[] { + const dns = (env.CI_DOCKER_DNS ?? '') + .split(',') + .map((server) => server.trim()) + .filter((server) => server.length > 0); + return [ + ...dns.flatMap((server) => ['--dns', server]), + ...(provider === 'blaxel' + ? [ + '--ipv6', + '--fixed-cidr-v6', + env.CI_DOCKER_BRIDGE_IPV6 ?? 'fd00:c1:0:1::/64', + ] + : []), + ...(provider === 'cloudflare' + ? ['--iptables=false', '--ip6tables=false', '--ip-forward=false'] + : []), + ]; +} + +/** + * Docker, ready to use, or a non-zero exit. Ordered cheapest-first: a daemon + * already running costs one `docker info`, a daemon installed but stopped + * costs a start, and only a box with no Docker at all pays for the download. + */ +export function buildDockerBringUpScript( + env: NodeJS.ProcessEnv = process.env, + opts: { + provider?: string; + prePullImages?: readonly string[]; + } = {}, +): string { + const version = env.CI_ACT_DOCKER_VERSION ?? STATIC_DOCKER_VERSION; + const quotedVersion = shellQuote(version); + const ownDaemon = providerNeedsOwnDockerd(opts.provider); + const routeArgs = ownDaemon + ? ` -H ${shellQuote(EXECUTOR_DOCKER_SOCKET)} --data-root ${shellQuote(EXECUTOR_DOCKER_DATA_ROOT)} --bridge ${shellQuote(EXECUTOR_DOCKER_BRIDGE)} --fixed-cidr ${shellQuote(EXECUTOR_DOCKER_CIDR)} --pidfile ${shellQuote(EXECUTOR_DOCKER_PIDFILE)} --exec-root ${shellQuote(EXECUTOR_DOCKER_EXEC_ROOT)}` + : ''; + const dnsArgs = dockerdExtraArgs(opts.provider, env) + .map((arg) => ` ${shellQuote(arg)}`) + .join(''); + return [ + ...FETCH_FN, + `export PATH="${DOCKER_BIN_DIR}:$PATH:${SBIN_DIRS}"`, + `docker_version=${quotedVersion}`, + `dockerd_route_args='${routeArgs.trim()}'`, + `dockerd_extra_args='${dnsArgs.trim()}'`, + ...(ownDaemon + ? [ + `export DOCKER_HOST=${shellQuote(EXECUTOR_DOCKER_SOCKET)}`, + `mkdir -p ${shellQuote(EXECUTOR_DOCKER_DATA_ROOT)}`, + `ip link add name ${EXECUTOR_DOCKER_BRIDGE} type bridge 2>/dev/null || busybox ip link add name ${EXECUTOR_DOCKER_BRIDGE} type bridge 2>/dev/null || true`, + `ip link set ${EXECUTOR_DOCKER_BRIDGE} up 2>/dev/null || busybox ip link set ${EXECUTOR_DOCKER_BRIDGE} up 2>/dev/null || true`, + ] + : []), + 'if docker info >/dev/null 2>&1; then', + ' docker_server_version=$(docker version --format \'{{.Server.Version}}\' 2>/dev/null || true)', + ' echo "Docker daemon is already running (server ${docker_server_version:-unknown})"', + ' docker_server_normalized=${docker_server_version#v}', + ' docker_requested_normalized=${docker_version#v}', + ' if [ "$docker_server_normalized" != "$docker_requested_normalized" ]; then', + ' echo "Docker version drift: requested $docker_version, running server ${docker_server_version:-unknown}; reusing the running daemon"', + ' fi', + 'else', + ' docker_binary_version=', + ' if dockerd_path=$(command -v dockerd 2>/dev/null) && docker_path=$(command -v docker 2>/dev/null); then', + ' docker_binary_version=$(dockerd --version 2>/dev/null | sed -n \'s/.*Docker version \\([^, ]*\\).*/\\1/p\')', + ' docker_binary_normalized=${docker_binary_version#v}', + ' if [ "$docker_binary_normalized" = "${docker_version#v}" ]; then', + ' echo "Reusing Docker binary $docker_binary_version at $dockerd_path"', + ' else', + ' echo "Docker binary version drift: requested $docker_version, installed ${docker_binary_version:-unknown}; reusing the installed binary"', + ' fi', + ' else', + ' echo "Docker binary is not present; installing Docker $docker_version"', + ' fi', + ' if [ -z "$docker_binary_version" ]; then', + ' arch=$(uname -m)', + ' case "$arch" in', + ' x86_64) docker_arch=x86_64 ;;', + ' aarch64|arm64) docker_arch=aarch64 ;;', + ' *) echo "Docker is not available for $arch"; exit 1 ;;', + ' esac', + ` mkdir -p ${shellQuote(DOCKER_BIN_DIR)}`, + ' rm -rf /tmp/docker && mkdir -p /tmp/docker', + ` if ! ci_fetch "https://download.docker.com/linux/static/stable/$docker_arch/docker-$docker_version.tgz" /tmp/docker.tgz; then`, + ' echo "Failed to download Docker $docker_version"', + ' exit 1', + ' fi', + ' if ! tar -xzf /tmp/docker.tgz -C /tmp/docker --strip-components=1; then', + ' echo "Failed to extract Docker $docker_version"', + ' exit 1', + ' fi', + ` if ! cp -a /tmp/docker/. ${shellQuote(DOCKER_BIN_DIR)}/; then`, + ' echo "Failed to install Docker $docker_version"', + ' exit 1', + ' fi', + ' rm -rf /tmp/docker', + ' rm -f /tmp/docker.tgz', + ` if ! docker_binary_report=$(${shellQuote(`${DOCKER_BIN_DIR}/dockerd`)} --version 2>/dev/null); then`, + ' echo "Installed Docker binary could not report a version"', + ' exit 1', + ' fi', + ' docker_binary_version=$(printf \'%s\\n\' "$docker_binary_report" | sed -n \'s/.*Docker version \\([^, ]*\\).*/\\1/p\')', + ' docker_binary_normalized=${docker_binary_version#v}', + ' if [ -z "$docker_binary_version" ] || [ "$docker_binary_normalized" != "${docker_version#v}" ]; then', + ' echo "Installed Docker binary version mismatch: requested $docker_version, installed ${docker_binary_version:-unknown}"', + ' exit 1', + ' fi', + ' fi', + ...(opts.provider === 'cloudflare' + ? [] + : [ + ' if command -v iptables >/dev/null 2>&1; then', + ' echo "iptables is already present"', + ' else', + ' echo "Installing iptables"', + ' if [ "$(id -u)" = 0 ]; then as_root=""; else as_root="sudo -n"; fi', + ' if command -v apt-get >/dev/null 2>&1; then', + ' $as_root apt-get update -qq >/dev/null 2>&1', + ' $as_root apt-get install -y -qq iptables >/dev/null 2>&1 || true', + ' elif command -v dnf >/dev/null 2>&1; then', + ' $as_root dnf install -y -q iptables >/dev/null 2>&1 || true', + ' elif command -v yum >/dev/null 2>&1; then', + ' $as_root yum install -y -q iptables >/dev/null 2>&1 || true', + ' elif command -v apk >/dev/null 2>&1; then', + ' $as_root apk add --no-cache iptables >/dev/null 2>&1 || true', + ' fi', + ' if ! command -v iptables >/dev/null 2>&1; then', + ' echo "iptables is missing and could not be installed, so the Docker daemon cannot network a container"', + ' exit 1', + ' fi', + ' fi', + ]), + ' if [ "$(id -u)" = 0 ]; then', + ' dockerd_base="dockerd"', + ' elif sudo -n true >/dev/null 2>&1; then', + ' dockerd_base="sudo -n env PATH=$PATH dockerd --group $(id -g)"', + ' else', + ' echo "Docker needs root in this sandbox and sudo is not available"', + ' exit 1', + ' fi', + ' echo "Starting Docker daemon $docker_binary_version"', + ' dockerd_cmd="$dockerd_base $dockerd_route_args $dockerd_extra_args"', + ' eval "nohup setsid $dockerd_cmd >/tmp/dockerd.log 2>&1 /dev/null 2>&1; then break; fi', + ' if [ "$attempt" -ge 5 ] && [ -S /var/run/docker.sock ]; then', + ' sudo -n chgrp "$(id -g)" /var/run/docker.sock >/dev/null 2>&1 || true', + ' sudo -n chmod 660 /var/run/docker.sock >/dev/null 2>&1 || true', + ' fi', + ' sleep 1', + ' done', + ' if ! docker info >/dev/null 2>&1; then', + ' echo "The Docker daemon did not start:"', + ' tail -20 /tmp/dockerd.log', + ' if [ -n "$dockerd_extra_args" ]; then', + ' echo "Retrying without the provider flags"', + ' dockerd_cmd="$dockerd_base $dockerd_route_args"', + ' eval "nohup setsid $dockerd_cmd >/tmp/dockerd.log 2>&1 /dev/null 2>&1 && break', + ' sleep 1', + ' done', + ' fi', + ' fi', + ' if ! docker info >/dev/null 2>&1; then', + ' echo "The Docker daemon did not start:"', + ' tail -20 /tmp/dockerd.log', + ' exit 1', + ' fi', + 'fi', + ...prePullLines(opts.prePullImages ?? []), + ].join('\n'); +} + +function prePullLines(images: readonly string[]): string[] { + const unique = [...new Set(images)].filter((image) => image !== ''); + if (unique.length === 0) return []; + return [ + `for ci_pre_pull_image in ${unique.map((image) => shellQuote(image)).join(' ')}; do`, + ' if docker image inspect "$ci_pre_pull_image" >/dev/null 2>&1; then', + ' echo "Runner image $ci_pre_pull_image is already local"', + ' else', + ' echo "Pre-pulling $ci_pre_pull_image in the background"', + ' nohup setsid docker pull "$ci_pre_pull_image" >>/tmp/ci-pre-pull.log 2>&1 /dev/null || true)', + ' if [ -n "$act_candidate" ] && [ ! -x "$act_candidate" ]; then act_candidate=; fi', + 'fi', + `if [ -n "$act_candidate" ] && [ "$act_candidate" = ${shellQuote(`${ACT_DIR}/act`)} ] && [ -f ${shellQuote(ACT_SHA256_PATH)} ]; then`, + ` act_expected_sha=$(cat ${shellQuote(ACT_SHA256_PATH)} 2>/dev/null || true)`, + ` act_actual_sha=$(ci_sha256 ${shellQuote(`${ACT_DIR}/act`)} 2>/dev/null || true)`, + ' if [ -n "$act_expected_sha" ] && [ -n "$act_actual_sha" ] && [ "$act_expected_sha" != "$act_actual_sha" ]; then', + ' echo "Cached workflow runner checksum mismatch; downloading the workflow runner $act_version"', + ' act_candidate=', + ' fi', + 'fi', + 'if [ -n "$act_candidate" ]; then', + ' act_reported=$("$act_candidate" --version 2>/dev/null || true)', + ' act_found_version=$(printf \'%s\\n\' "$act_reported" | awk \'{ for (i = 1; i <= NF; i++) if ($i == "version") { print $(i + 1); exit } }\')', + ' act_found_version=${act_found_version:-unreported}', + ' act_found_normalized=${act_found_version#v}', + ' if [ "$act_found_normalized" = "$act_requested_normalized" ]; then', + ' echo "Reusing workflow runner $act_found_version"', + ` if [ "$act_candidate" != ${shellQuote(`${ACT_DIR}/act`)} ]; then`, + ` mkdir -p ${shellQuote(ACT_DIR)}`, + ` ln -sf "$act_candidate" ${shellQuote(`${ACT_DIR}/act`)}`, + ' fi', + ' else', + ' echo "Workflow runner version mismatch: requested $act_version, found $act_found_version; downloading the requested version"', + ' act_candidate=', + ' fi', + 'else', + ' echo "Downloading workflow runner $act_version"', + 'fi', + 'if [ -z "$act_candidate" ]; then', + ' arch=$(uname -m)', + ' case "$arch" in', + ' x86_64) asset=act_Linux_x86_64.tar.gz ;;', + ' aarch64|arm64) asset=act_Linux_arm64.tar.gz ;;', + ' *) echo "The workflow runner is not available for $arch"; exit 1 ;;', + ' esac', + ` mkdir -p ${shellQuote(ACT_DIR)}`, + ` mkdir -p ${shellQuote(ACT_ARTIFACTS_DIR)}`, + ` rm -f ${shellQuote(`${ACT_DIR}/act`)}`, + ' act_download_attempt=1', + ' while :; do', + ` if ci_fetch "https://github.com/nektos/act/releases/download/$act_version/$asset" ${shellQuote(`${ACT_DIR}/act.tar.gz`)}; then`, + ' break', + ' fi', + ' if [ "$act_download_attempt" -ge 3 ]; then', + ' echo "Failed to download the workflow runner $act_version"', + ' exit 1', + ' fi', + ' echo "Runner download attempt $act_download_attempt failed; retrying" >&2', + ' act_download_attempt=$((act_download_attempt + 1))', + ' sleep $((act_download_attempt * 3))', + ' done', + ` if ! tar -xzf ${shellQuote(`${ACT_DIR}/act.tar.gz`)} -C ${shellQuote(ACT_DIR)} act; then`, + ' echo "Failed to extract the workflow runner $act_version"', + ' exit 1', + ' fi', + ` rm -f ${shellQuote(`${ACT_DIR}/act.tar.gz`)}`, + ` if ! chmod +x ${shellQuote(`${ACT_DIR}/act`)}; then`, + ' echo "Failed to make the workflow runner $act_version executable"', + ' exit 1', + ' fi', + ` if ! act_reported=$(${shellQuote(`${ACT_DIR}/act`)} --version 2>/dev/null); then`, + ' echo "The downloaded workflow runner could not report a version"', + ' exit 1', + ' fi', + ' act_installed_version=$(printf \'%s\\n\' "$act_reported" | awk \'{ for (i = 1; i <= NF; i++) if ($i == "version") { print $(i + 1); exit } }\')', + ' act_installed_normalized=${act_installed_version#v}', + ' if [ -z "$act_installed_version" ] || [ "$act_installed_normalized" != "$act_requested_normalized" ]; then', + ' echo "Downloaded workflow runner version mismatch: requested $act_version, installed ${act_installed_version:-unknown}"', + ' exit 1', + ' fi', + ` ci_sha256 ${shellQuote(`${ACT_DIR}/act`)} > ${shellQuote(ACT_SHA256_PATH)} 2>/dev/null || rm -f ${shellQuote(ACT_SHA256_PATH)}`, + ' echo "Workflow runner $act_version ready"', + 'fi', + ].join('\n'); +} + +/** + * Shell lines that set `$varName` to the docker bridge gateway only when the + * gateway is one of this sandbox's own interface addresses, else empty. + * The gateway doubles as a *bind* address for act's artifact/cache servers, + * so it must be local. + */ +export function dockerLocalBridgeGatewayScript(varName: string): string[] { + return [ + `for attempt in $(seq 1 30); do`, + ` docker network inspect bridge >/dev/null 2>&1 && break`, + ` ip -o link show dev docker0 >/dev/null 2>&1 && break`, + ` sleep 1`, + `done`, + `${varName}="$(docker network inspect bridge --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || true)"`, + `case "$${varName}" in`, + ` *.*.*.*) ;;`, + ` *) ${varName}="$(ip -o -4 addr show dev docker0 2>/dev/null | awk '{split($4,a,"/");print a[1]}' | head -n 1)" ;;`, + `esac`, + `local_addrs="$(hostname -I 2>/dev/null || true)"`, + `if [ -z "$local_addrs" ]; then`, + ` local_addrs="$(awk '$1=="|--"||$1=="+--"{prev=$2} /32 host LOCAL/{print prev}' /proc/net/fib_trie 2>/dev/null | sort -u | tr '\\n' ' ')"`, + `fi`, + `case " $local_addrs " in`, + ` *" $${varName} "*) ;;`, + ` *) echo "docker bridge gateway '$${varName}' not among local addrs '$local_addrs'" >&2; ${varName}="" ;;`, + `esac`, + ]; +} + +/** + * Shell lines that set `$varName` to the sandbox's primary non-loopback + * IPv4 — the fallback bind address when the bridge gateway is remote. + */ +export function sandboxPrimaryAddrScript(varName: string): string[] { + return [ + `${varName}="$(printf '%s\\n' "$local_addrs" | awk '{for(i=1;i<=NF;i++) if ($i ~ /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/ && $i !~ /^127\\./) {print $i; exit}}')"`, + ]; +} diff --git a/benchmarks/sandbox/dax.bench.ts b/benchmarks/sandbox/dax.bench.ts index ba80a313..a5eb9cf8 100644 --- a/benchmarks/sandbox/dax.bench.ts +++ b/benchmarks/sandbox/dax.bench.ts @@ -22,7 +22,6 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { defineBenchmarkConfig, defineTask, TaskError } from '@benchsdk/runner'; import type { JsonObject, TaskStepRecord } from '@benchsdk/api'; -import { VMTier } from '@codesandbox/sdk'; import { withTimeout } from '../src/util/timeout.js'; import { formatError } from '../src/util/error.js'; import { sandboxId } from '../src/util/sandbox-id.js'; @@ -31,6 +30,7 @@ import type { ProviderConfig } from './types.js'; import { BENCH_SCRIPT_PATH } from './dax.js'; import type { DaxTimingResult } from './dax.js'; import { writeDaxLegacyResults } from './dax-legacy-results.js'; +import { getSandboxOptionsWithResources } from './resource-options.js'; /** Raw build output kept alongside the timing so failures can be diagnosed from the logs. */ interface DaxBuildOutput { @@ -109,63 +109,6 @@ export const config = defineBenchmarkConfig({ }), }); -// Standardized resource sizing for fair comparison across providers. -// Target: 8 vCPU, 16 GiB RAM. -// Each provider uses different parameter names and units, so we map per-provider. -// Providers not listed here don't support CPU/memory configuration at sandbox creation time. -// Note: E2B sets CPU/memory at template build time, not at sandbox creation. -// Note: lightning is sized via LIGHTNING_INSTANCE_TYPE=cpu-8 (8 vCPU / 16 GiB), applied on -// the provider factory in providers.ts — the SDK ignores an instanceType passed to create(). -const DAX_RESOURCE_OPTIONS: Record> = { - arker: { templateId: 'ubuntu-full-8' }, // 8 vCPU / 16 GiB golden - modal: { cpu: 4, cpuLimit: 4, memoryMiB: 16384, templateId: 'node:22' }, // Modal: 1 core = 2 vCPUs, so 4 cores = 8 vCPUs - tenki: { cpuCores: 8, memoryMb: 16384, diskSizeGb: 20 }, // default disk cannot hold the OpenCode install - tensorlake: { cpus: 8, memoryMb: 16384 }, - isorun: { vcpus: 8, memMiB: 16384 }, - runloop: { launch_parameters: { resource_size_request: 'CUSTOM_SIZE', custom_cpu_cores: 8, custom_gb_memory: 16 } }, - upstash: { size: 'large' }, // large = 8 cores, 16 GB - vercel: { resources: { vcpus: 8 } }, // no memory control - blaxel: { - memory: 16384, // CPU derived: cores = memory_MB / 2048 = 8 - timeout: 900_000, // Server-side TTL backstop if client cleanup cannot run - volumes: [ - { - name: 'dax-root', - type: 'ephemeral', - sizeMb: 16384, - mountPath: '/', // Keep the writable root off RAM - }, - ], - }, - beam: { cpu: 8, memory: 16384 }, // cpu = cores, memory = MiB - cloudflare: { vcpu: 8, memoryMib: 16384, diskMb: 20000, image: 'builder' }, - codesandbox: { vmTier: VMTier.Small }, // Small = 8 CPU, 16 GiB - northflank: { deploymentPlan: process.env.NORTHFLANK_DEPLOYMENT_PLAN || 'nf-compute-50', ephemeralStorageSize: 5120 }, // 5 GiB ephemeral storage - declaw: { templateId: 'node-large' }, // node-large template: 8 vCPU / 16 GiB RAM / 8 GiB disk - superserve: { templateId: 'node22-8cpu-16gb' }, // 8 vCPU / 16 GiB template built in the pre-step - createos: { shape: 's-8vcpu-16gb', ephemeralDiskMb: 61440 }, // 8 vCPU, 16 GiB RAM, 60 GiB disk - opencomputer: { cpuCount: 4, memoryMB: 16384, timeout: 600_000 }, - microsandbox: { cpus: 8, memoryMib: 16384, rootDiskMib: 20480 }, // 20 GiB writable disk - mosaic: { vcpus: 8, memoryMb: 16384 }, - buddy: { cpu: 8, memoryMb: 16384 }, // provider maps cpu/memory to its 8x16 preset (8 vCPU / 16 GB) - miosa: { vcpus: 8, memory: 16384 }, // maps to MIOSA size contract "large" (8 vCPU / 16 GiB) - // givemeanode has no 8 vCPU / 16 GiB named shape; asking for 8 vCPU and 16 GiB - // selects sandbox-lg (8 vCPU / 32 GiB). The actual memory is reported by the - // benchmark script, so the mismatch is visible in results. - givemeanode: { vcpus: 8, memoryMiB: 16384 }, - gravixlayer: { templateId: 'base-4xlarge' }, // built-in template requested by the Gravix team - // Sandbox0 exposes only memory. Override the 128 MiB TTI size; - // getSandboxOptionsWithResources preserves hardTtl from providers.ts. - sandbox0: { memory: 16384 }, - sail: { size: 'l' }, -}; - -function getSandboxOptionsWithResources(providerName: string, baseOptions?: Record): Record { - const resourceOpts = DAX_RESOURCE_OPTIONS[providerName]; - if (!resourceOpts) return baseOptions ?? {}; - return { ...baseOptions, ...resourceOpts }; -} - /** Runs the build script inside `sandbox` and parses its structured output. */ async function runDaxBuild( sandbox: any, diff --git a/benchmarks/sandbox/resource-options.ts b/benchmarks/sandbox/resource-options.ts new file mode 100644 index 00000000..f81e9306 --- /dev/null +++ b/benchmarks/sandbox/resource-options.ts @@ -0,0 +1,63 @@ +/** + * Standardized large resource sizing for sandbox benchmarks (8 vCPU / 16 GiB + * RAM) — at or above the Actions product's CI floor (4 vCPU / 8 GiB / 16 GiB). + * Shared by dax.bench.ts and ci-capabilities.bench.ts. + * + * Each provider uses different parameter names and units, so we map per-provider. + * Providers not listed here don't support CPU/memory configuration at sandbox + * creation time (or size via their providers.ts sandboxOptions / factory). + * Note: E2B sets CPU/memory at template build time, not at sandbox creation. + * Note: lightning is sized via LIGHTNING_INSTANCE_TYPE=cpu-8 (8 vCPU / 16 GiB), applied on + * the provider factory in providers.ts — the SDK ignores an instanceType passed to create(). + */ +import { VMTier } from '@codesandbox/sdk'; + +export const LARGE_RESOURCE_OPTIONS: Record> = { + arker: { templateId: 'ubuntu-full-8' }, // 8 vCPU / 16 GiB golden + modal: { cpu: 4, cpuLimit: 4, memoryMiB: 16384, templateId: 'node:22' }, // Modal: 1 core = 2 vCPUs, so 4 cores = 8 vCPUs + tenki: { cpuCores: 8, memoryMb: 16384, diskSizeGb: 20 }, // default disk cannot hold the OpenCode install + tensorlake: { cpus: 8, memoryMb: 16384 }, + isorun: { vcpus: 8, memMiB: 16384 }, + runloop: { launch_parameters: { resource_size_request: 'CUSTOM_SIZE', custom_cpu_cores: 8, custom_gb_memory: 16 } }, + upstash: { size: 'large' }, // large = 8 cores, 16 GB + vercel: { resources: { vcpus: 8 } }, // no memory control + blaxel: { + memory: 16384, // CPU derived: cores = memory_MB / 2048 = 8 + timeout: 900_000, // Server-side TTL backstop if client cleanup cannot run + volumes: [ + { + name: 'dax-root', + type: 'ephemeral', + sizeMb: 16384, + mountPath: '/', // Keep the writable root off RAM + }, + ], + }, + beam: { cpu: 8, memory: 16384 }, // cpu = cores, memory = MiB + cloudflare: { vcpu: 8, memoryMib: 16384, diskMb: 20000, image: 'builder' }, + codesandbox: { vmTier: VMTier.Small }, // Small = 8 CPU, 16 GiB + northflank: { deploymentPlan: process.env.NORTHFLANK_DEPLOYMENT_PLAN || 'nf-compute-50', ephemeralStorageSize: 5120 }, // 5 GiB ephemeral storage + declaw: { templateId: 'node-large' }, // node-large template: 8 vCPU / 16 GiB RAM / 8 GiB disk + superserve: { templateId: 'node22-8cpu-16gb' }, // 8 vCPU / 16 GiB template built in the pre-step + createos: { shape: 's-8vcpu-16gb', ephemeralDiskMb: 61440 }, // 8 vCPU, 16 GiB RAM, 60 GiB disk + opencomputer: { cpuCount: 4, memoryMB: 16384, timeout: 600_000 }, + microsandbox: { cpus: 8, memoryMib: 16384, rootDiskMib: 20480 }, // 20 GiB writable disk + mosaic: { vcpus: 8, memoryMb: 16384 }, + buddy: { cpu: 8, memoryMb: 16384 }, // provider maps cpu/memory to its 8x16 preset (8 vCPU / 16 GB) + miosa: { vcpus: 8, memory: 16384 }, // maps to MIOSA size contract "large" (8 vCPU / 16 GiB) + // givemeanode has no 8 vCPU / 16 GiB named shape; asking for 8 vCPU and 16 GiB + // selects sandbox-lg (8 vCPU / 32 GiB). The actual memory is reported by the + // benchmark script, so the mismatch is visible in results. + givemeanode: { vcpus: 8, memoryMiB: 16384 }, + gravixlayer: { templateId: 'base-4xlarge' }, // built-in template requested by the Gravix team + // Sandbox0 exposes only memory. Overrides the 128 MiB TTI size; + // getSandboxOptionsWithResources preserves hardTtl from providers.ts. + sandbox0: { memory: 16384 }, + sail: { size: 'l' }, +}; + +export function getSandboxOptionsWithResources(providerName: string, baseOptions?: Record): Record { + const resourceOpts = LARGE_RESOURCE_OPTIONS[providerName]; + if (!resourceOpts) return baseOptions ?? {}; + return { ...baseOptions, ...resourceOpts }; +} diff --git a/benchmarks/sandbox/types.ts b/benchmarks/sandbox/types.ts index 62cc0b2e..6f24f3b1 100644 --- a/benchmarks/sandbox/types.ts +++ b/benchmarks/sandbox/types.ts @@ -68,3 +68,18 @@ export interface StaggeredBenchmarkResult extends BenchmarkResult { /** Per-sandbox timing profile showing launch offset and TTI */ rampProfile: { launchedAt: number; readyAt: number; ttiMs: number }[]; } + +/** Pass/fail record for a single CI capability check in a provider run. */ +export interface CapabilityFeatureResult { + passed: boolean; + error?: string; +} + +/** Map of check name -> result for one provider iteration. */ +export type ProviderCapabilityMatrix = Record; + +/** One provider's row in the CI capabilities results JSON. */ +export interface CapabilityMatrixResult { + provider: string; + features: ProviderCapabilityMatrix; +} diff --git a/package.json b/package.json index 6901b176..d297bbf5 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "bench": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --iterations 3 --concurrency 3", "bench:burst": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --iterations 3 --concurrency 3", "bench:sandbox:dax": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/dax.bench.ts", + "bench:sandbox:ci-capabilities": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/ci-capabilities.bench.ts", "bench:blaxel": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --iterations 3 --concurrency 3 --provider blaxel", "bench:codesandbox": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --iterations 3 --concurrency 3 --provider codesandbox", "bench:daytona": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/sandbox/tti.bench.ts --iterations 3 --concurrency 3 --provider daytona",