diff --git a/.github/actions/cache-libkrunfw-kernel/action.yml b/.github/actions/cache-libkrunfw-kernel/action.yml index a36c384e0..32bc9b7b7 100644 --- a/.github/actions/cache-libkrunfw-kernel/action.yml +++ b/.github/actions/cache-libkrunfw-kernel/action.yml @@ -18,6 +18,10 @@ runs: # Published in kernel.org's signed v6.x sha256sums.asc index. kernel_sha256=194eef900ade82df74ed1d695daa45d03ee4bb415cae4f936a3dbaab2dbbb951 ;; + linux-6.12.108:'tarballs/$(KERNEL_VERSION).tar.gz') + # https://cdn.kernel.org/pub/linux/kernel/v6.x/sha256sums.asc + kernel_sha256=c4127aa9614a6a829c537cff96a58da634a5f8cfd1aed9d1ba076d3b3a80891a + ;; *) echo "::error::Add the checksum for ${kernel_version} (${kernel_tarball}) to ${GITHUB_ACTION_PATH}/action.yml" exit 1 diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index f8e156999..49ad364da 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -576,13 +576,13 @@ jobs: Set-MsvcEnvironment -Architecture ${{ matrix.vs_arch }} -HostArchitecture ${{ matrix.vs_host_arch }} cargo +stable test --no-default-features --features local,net -p microsandbox --lib --target ${{ matrix.rust_target }} sandbox::patch::tests::bind_patch_ - - name: Test Windows stdio inheritance cleanup + - name: Test Windows lifecycle handoff and stdio cleanup shell: pwsh run: | $ErrorActionPreference = "Stop" . "$env:GITHUB_WORKSPACE\vendor\libkrunfw\scripts\msvc-env.ps1" Set-MsvcEnvironment -Architecture ${{ matrix.vs_arch }} -HostArchitecture ${{ matrix.vs_host_arch }} - cargo +stable test --no-default-features --features local,net -p microsandbox --lib --target ${{ matrix.rust_target }} runtime::spawn::tests::windows_stdio_guard_ + cargo +stable test --no-default-features --features local,net -p microsandbox --lib --target ${{ matrix.rust_target }} runtime::spawn::tests::windows_ - name: Test Windows DNS resolver shell: pwsh @@ -1457,6 +1457,40 @@ jobs: scripts/smoke/cli/image-archive.sh scripts/smoke/cli/split-irqchip-bind-net.sh + - name: Snapshot smoke runner unit tests + run: python3 -m unittest discover -s scripts/smoke/cli -p test_snapshot_branch.py + + - name: Snapshot and branch live smoke + # The short operation deadline excludes image setup and bounded cleanup. + # Leave both layouts enough failure-path time to stop VMs and save evidence. + timeout-minutes: 15 + env: + MSB_LIBKRUNFW_PATH: ${{ github.workspace }}/build/libkrunfw.so.${{ env.LIBKRUNFW_VERSION }} + MSB_AGENTD_PATH: ${{ github.workspace }}/build/agentd + LD_LIBRARY_PATH: ${{ github.workspace }}/build + run: | + status=0 + # Short isolated homes avoid Unix socket limits and cross-layout reuse. + # Run both even if one fails so the artifact reports retain both outcomes. + for layout in managed flat; do + python3 scripts/smoke/cli/snapshot-branch.py \ + --binary "${{ github.workspace }}/build/msb" \ + --output "/tmp/msb-smoke-${{ github.run_id }}-${{ github.run_attempt }}-$layout" \ + --layout "$layout" || status=$? + done + exit "$status" + + - name: Upload snapshot smoke reports and logs + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: snapshot-branch-smoke-linux-x86_64 + # Keep text evidence, not the potentially large guest RAM/disk artifacts. + path: | + /tmp/msb-smoke-${{ github.run_id }}-${{ github.run_attempt }}-*/report.json + /tmp/msb-smoke-${{ github.run_id }}-${{ github.run_attempt }}-*/logs/*.log + if-no-files-found: warn + - name: Disk usage if: always() run: scripts/ci/clean-runner-disk.sh diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index 070672267..2f51e5d65 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -132,6 +132,12 @@ Sources: [`crates/runtime/lib/runner/control.rs`](crates/runtime/lib/runner/cont Add operations and optional fields rather than redefining existing ones. Capability-gate behavior whose absence cannot be interpreted safely by older clients. +Live disk-only snapshots use the distinct `disk_checkpoint_create` operation and capability. An absent capability is false: callers refuse before capture rather than silently capturing RAM or copying a writable disk. The runtime serializes the disk rollover with other control mutations and preserves a user's pause. This does not change the agent protocol or snapshot format; the result uses the existing file-state layer descriptor. Stopped disk capture retains its lifecycle lock and existing behavior. + +Resident pause/resume sends one authoritative mutation rather than first observing pause state and querying capabilities. The runtime checks support before mutation, including idempotent requests, and the client requires the expected state in the response; unknown operations and incomplete replies fail. Ordinary get/list pause projection remains unchanged. Guest freezing retains one `cgroup.events` descriptor and waits for notifications with a fixed deadline; each poll timeout is capped at 1 ms so rate-limited kernel notifications cannot delay the next authoritative state check. Clock correction still precedes workload thaw, and no control or agent wire format changes. + +The subsequent unreleased generation-9 transport repair routes internal freeze/thaw directly from the coordinator to the existing relay through a bounded in-process queue. Ordinary control/bulk input is gated at complete frames; admitted input remains guest-owned until consumed, while unadmitted input stays source-owned and ordered. The guest keeps stdin/TCP delivery nonblocking with respect to its control loop and preserves accepted input through restore. Private replies and cumulative credit updates never become SDK responses. The immutable frame header, released generation-8 schema, and public sockets are unchanged. Full capture requires an acknowledged bidirectional frame boundary; failure or timeout does not authorize a partial capture. + ## 5. Launcher-to-Runtime Process Protocol Starting a sandbox crosses a private process boundary. On Unix, launch JSON is passed through inherited descriptor 96, the parent watchdog uses descriptor 97, startup JSON uses descriptor 98, and the lifecycle lock uses descriptor 99. Windows uses a short-lived launch-config file and platform-specific startup plumbing. Detach acknowledgement bytes and graceful-shutdown signals are also part of this contract. @@ -140,7 +146,9 @@ Compatibility-sensitive elements include descriptor numbers, ownership and close Sources: [`crates/runtime/lib/client/launch.rs`](crates/runtime/lib/client/launch.rs), [`crates/runtime/lib/runner/vm.rs`](crates/runtime/lib/runner/vm.rs), [`sdk/rust/lib/runtime/spawn.rs`](sdk/rust/lib/runtime/spawn.rs), and [`crates/cli/lib/sandbox_cmd.rs`](crates/cli/lib/sandbox_cmd.rs). -This protocol has no explicit version envelope. Treat additions as optional and consider adding explicit version or capability negotiation before allowing independently versioned launchers and runtimes. +Launch JSON requires an explicit `execution` intent (`boot` or `restore`) and rejects unknown fields. Restores also pass the internal `msb sandbox --restore` argument: a runtime predating this contract rejects the unknown argument rather than ignoring a JSON restore source and cold-booting. The argument, intent, and complete strictly validated `checkpoint_restore` source must agree before VM construction. Unsupported restore behavior is an error, never a fresh-boot fallback. These #8 development contracts replace superseded unreleased forms without shims; they do not change portable snapshot bytes. + +The child database config retains `checkpoint_restore` while construction is incomplete. Only successful restore activation and creation finalization remove it. A failed or interrupted attempt retains its child-owned staging and rejects start, auto-start through exec, modification, compaction, and snapshot creation; remove and recreate it from the intact input snapshot. This replaces the earlier unreleased #8 behavior that discarded restore intent before success. Do not reopen these development rows with older #8 binaries that skip that field. Successful restores retain the ordinary later stop/start lifecycle; no portable snapshot format or schema version changes. ## 6. Database, Configuration, and Migration History @@ -180,10 +188,28 @@ Parsers and mutators must validate the complete supported feature set before the ## 9. Snapshots, Manifests, and Portable Archives -Snapshot descriptor bytes are identity-bearing: their canonical bytes determine the snapshot ID. Compatibility-sensitive elements include field order, required `null` values, map ordering, duplicate-key handling, tag spellings, schema and integrity identifiers, payload names, parent identities, state/scope/format variants, extension requirements, and translation-graph behavior. +Snapshot descriptors carry a stable random `snap_...` ID; their canonical bytes determine the descriptor digest, not that ID. Compatibility-sensitive elements include field order, required `null` values, map ordering, duplicate-key handling, tag spellings, schema and integrity identifiers, payload names, parent identities, state/scope/format variants, extension requirements, and translation-graph behavior. Archive compatibility includes compression detection, `archive.json`, canonical inventory order, transport digests, accepted path grammar, legacy paths, cache-closure entries, and rejection of duplicate, missing, or escaping paths. +Installed snapshots now live under `snapshots///`. `group.json` selects a head; `group-member.json` stores a local friendly name without changing descriptor identity. Bare selectors mean a group head, and `group:member` selects an exact member. Existing flat artifacts remain readable by explicit path; this change does not silently move their directories. The index keys local artifact paths rather than globally unique portable IDs/digests, so importing the same snapshot into two groups preserves both copies. Downgrade refuses grouped state before rewriting artifacts or rolling back the index. + +Capture records the actual source snapshot lineage in the existing descriptor `parent` field. Per-sandbox cursor publication serializes captures without holding a VM pause; group head publication is locked separately. Automatic head advancement requires known ancestry, not capture timestamps, export dependency bases, or import order. An explicit head selection may rewind or choose a sibling. Missing ancestry may prevent advancement but is not a missing payload dependency. Archives optionally carry friendly names in `msb-snapshot-member-names`; their snapshot IDs, payload paths and descriptor schema are unchanged. + +Capture publication and source removal/replacement share a stable lock in `run_dir/locks/.snapshot-lineage.lock`, outside the removable sandbox directory. A caller needing multiple ownership guards acquires transition, then lineage, then runtime lifecycle ownership. The cursor remains in the sandbox directory; its schema and portable snapshot identities are unchanged. This replaces the unreleased directory-local lock, not a shipped artifact format. + +`snapshot load` accepts multiple archive paths; the former positional destination is now `--dest DIR`. Single-archive SDK methods and their return types remain; batch methods return one handle per input archive head in input order. The batch resolves exact disk-layer and RAM-object dependencies from supplied archives, the explicitly selected destination group, and an optional external base. No archive encoding changes or global snapshot search are involved. Borrowed payloads belong to destination staging and use the existing integrity codecs before publication. A compatible source may contain more layers than the omitted prefix; dependency identities still must match. Direct archive restore retains its explicit-base contract. + +Batch head selection is independent of input order: one proven lineage tip uses existing fast-forward rules; ambiguous tips preserve an existing head or leave a new group headless. `--set-head` refuses an ambiguous batch. IDs, aliases, duplicate labels, and payloads are checked before member publication. An I/O failure during final publication can still leave complete additional members, as with single-archive publication, but never a head pointing at an incomplete member. + +Unreleased #8 incremental exports use `completeness: "dependent"` and the must-understand `msb-snapshot-dependencies-v1` extension. `--since` records omitted physical disk-prefix layers and reusable RAM-object identities; `--last-layers` only omits disk layers. The complete target memory manifest and CPU/device state remain included. Loading and direct archive restore resolve the explicitly supplied base into owned staging before opening the complete target. This replaces the unreleased disk-only dependency encoding without a compatibility shim or snapshot descriptor change. Readers that do not understand this requirement refuse it; ordinary standalone archives are unchanged. + +Full checkpoints and local branches now retain `transport_host_input`, `transport_input_credit`, and `transport_guest_bulk_bytes` in the existing `guest:agentd` resource binding. These are complete-frame cumulative positions and absolute grants, including credit still owned by pending captured input. Restore validates and seeds them before guest activation; resetting them would incorrectly grant capacity twice. Older unreleased development full snapshots missing this state are refused, and new full captures require their matching host/guest implementation. This is an approved replacement of unreleased state, not a snapshot schema bump or migration; released disk-only snapshots are unaffected. + +The finalized private transport-credit contract charges stdin, inline filesystem/TCP payloads, and ordered EOF to the existing logical data (`bulk_*`) counters on either physical port. Command/control counters remain available when captured input is still awaiting consumption. Ready advertises barrier contract `2`; the superseded development contract `1` is not translated or restored. The outer frame, generation-8 data format, snapshot descriptor schema, and public SDK requests are unchanged. The ordinary writer retains bounded admission permits until physical delivery, permits unrelated metadata to pass credit-blocked payloads, and preserves per-correlation and client-disconnect ordering. Guest input processing also yields to the runtime after bounded actual reads, including partial records; this does not shrink wire records or change snapshot boundaries. + +Routine host clock maintenance is independent of unrelated correlation input, but stays ordered with other clocks and true global lifecycle fences. Its timestamp is sampled at console admission, not when queued; disconnect cleanup signals fence their own session only. Maintenance remains subject to the pause gate. This bounds host-queue timestamp age, not subsequent aging of already-admitted bytes during arbitrary host suspension or the kernel-only pause fallback when the workload freezer is unavailable. + Evolution rules: - Do not make semantically harmless serialization changes to identity-bearing bytes without treating them as an identity format change. @@ -194,6 +220,10 @@ Evolution rules: Sources: [`crates/image/lib/snapshot/manifest.rs`](crates/image/lib/snapshot/manifest.rs), [`crates/image/lib/snapshot/migration.rs`](crates/image/lib/snapshot/migration.rs), and [`sdk/rust/lib/snapshot/archive.rs`](sdk/rust/lib/snapshot/archive.rs). +Runtime restore admits disk payloads before activation. Journal creation may reuse the verified root only for that same unchanged immutable file. Its in-process cache retains at most 32 file handles, preferring larger physical files; every layer still receives full admission, and uncached, copied or rewritten layers receive a fresh hash. Detected mutation of a retained admitted file fails. Candidates are opened once per lookup, with comparisons bounded by the cache size. This reuse is not a persistent "verified" flag or a path-only cache, and the cache bound does not reduce supported chain depth. + +Incremental capture retains immutable object receipts only within the owning runtime's store lifetime. Reopened stores and unadmitted objects still verify bytes. Receipts retain no file descriptors; active operations open, check and temporarily pin the exact file. Capture uses two writers and three recycled 32 MiB packs, then synchronizes new directory entries before the existing root-last publication. Eager restore and cold memory-cache construction use at most four reusable 32 MiB read/hash buffers. Errors join workers before cleanup. These changes preserve the snapshot format and restored bytes, dirty-baseline rollover, pause/publication ordering and durability barriers; summed worker timings must not be interpreted as additive wall time. + ## 10. OCI Cache and Materializer ABI The cache is rebuildable, but cache entries and closures can cross releases through `MSB_HOME` and snapshot archives. OCI semantics are externally defined: compressed descriptor digests, uncompressed diff IDs, ordered layers, whiteouts, opaque directories, hardlinks, extended attributes, non-UTF-8 paths, special files, and permissions must retain their meaning. @@ -271,6 +301,8 @@ Compatibility-sensitive ordering includes: Sources: [`crates/runtime/lib/client/ipc.rs`](crates/runtime/lib/client/ipc.rs), [`sdk/rust/lib/backend/local/mod.rs`](sdk/rust/lib/backend/local/mod.rs), [`sdk/rust/lib/runtime/handle.rs`](sdk/rust/lib/runtime/handle.rs), and artifact-specific migration and publication modules. +TCP completion follows both ordered half-closes; the first EOF alone keeps the opposite direction usable. In combined-port mode, validated guest-to-host TCP credit may pass queued host-to-guest raw data and its finish marker: it services the opposite direction without reordering input data or EOF. Opening, cancellation, ownership, and global lifecycle fences still constrain it. Raw TCP output may still be draining on the dedicated lane after its producer finishes. Decoded credit updates for a finished producer or absent TCP session are therefore no-ops, not cancellation: they cannot enable further output, and must not discard the queued tail or create a second terminal response. Active producers retain credit validation; data and finish messages retain their existing validation. + Review concurrency and crash points explicitly. A same-version happy-path test does not establish cross-version or crash compatibility. ## Review Triggers in Diffs diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 83ed90196..e7f462243 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -182,6 +182,28 @@ Run a specific test: cargo test -p microsandbox test_name ``` +### Snapshot and branch checks + +Run the focused logic suite without starting VMs: + +```bash +just test-snapshot +``` + +This covers snapshot archives/groups, dependency validation, checkpoint logic, snapshot CLI parsing, and the live-smoke runner's own unit tests. The Rust tests already run in the normal Linux workspace CI lane. Cached test execution is much shorter than a first build; Cargo compilation and dependency setup are additional costs, not snapshot-operation timings. + +For a compact end-to-end check, build a matching runtime bundle with `just build`, then run: + +```bash +just test-snapshot-live +just test-snapshot-live --layout flat +just test-snapshot-live --binary /path/to/msb --output /tmp/snapshot-smoke-new +``` + +The live check requires working virtualization and Python (`python3` on Linux/macOS, `python` on Windows). macOS binaries must be codesigned with `msb-entitlements.plist`; `just build` does this. It uses a new isolated `MSB_HOME`, stops its own VMs, verifies host-process exit, and retains a report and logs in the printed output directory. Successful runs remove their temporary RAM/disk artifacts; failed runs retain their home for investigation. An explicit `--output` directory must not exist; choose a short path under `/tmp` on Unix to stay within socket-path limits. Use `--help` for image and timeout options. + +The warm live target is under 60 seconds per layout, excluding compilation and image-pull setup; this is a target, not a guarantee or a performance benchmark. Per-command and suite deadlines bound failures separately. The existing Linux/KVM CLI smoke CI job runs managed and flat layouts and uploads reports/logs even on failure. This compact check complements, rather than replaces, the larger live invariant and benchmark matrices under `scripts/smoke/cli/`. + ## Benchmarking The benchmark suite lives in its own repository: diff --git a/README.md b/README.md index db4e14090..607740d6b 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ ## - **Hardware Isolation**: Hardware-level isolation with microVM technology. +- **Branch & Snapshot**: Save running sandbox state and restore later. Fork live sandboxes. - **Cross Platform**: Runs on Linux, macOS, and Windows. - **OCI Compatible**: Runs standard container images from Docker Hub, GHCR, or any OCI registry. - **Docker-Like Workflows**: Familiar image, command, shell, and volume workflows. @@ -43,34 +44,7 @@ ## rocket-darkrocket  Getting Started -####   Install the SDK -> ```sh -> npm i microsandbox # 🟦 TypeScript -> ``` -> -> ```sh -> cargo add microsandbox # 🦀 Rust -> ``` -> -> ```sh -> uv add microsandbox # 🐍 Python -> ``` -> -> ```sh -> go get github.com/superradcompany/microsandbox/sdk/go # 🐹 Go -> ``` ####   Install the CLI - -> Boot a microVM in a single command: -> -> ```sh -> npx microsandbox run debian -> ``` -> -> ## -> -> Or install the `msb` command globally: -> > ```sh > curl -fsSL https://install.microsandbox.dev | sh # 🍎 macOS / 🐧 Linux > ``` @@ -104,12 +78,30 @@ > > ## > -> Then you can run `msb` directly: +> Start creating sandboxes once installed: > > ```sh -> msb run debian +> msb run ubuntu > ``` +####   Install the SDK +> ```sh +> npm i microsandbox # 🟦 TypeScript +> ``` +> +> ```sh +> cargo add microsandbox # 🦀 Rust +> ``` +> +> ```sh +> uv add microsandbox # 🐍 Python +> ``` +> +> ```sh +> go get github.com/superradcompany/microsandbox/sdk/go # 🐹 Go +> ``` + + ## > **Requirements**: @@ -122,6 +114,114 @@
+## cli-darkcli  CLI + +The `msb` CLI provides a complete interface for managing sandboxes, snapshots, images, and volumes. + +####   Run a Command + +> ```sh +> msb run python -- python3 -c "print('Hello from a microVM!')" +> ``` + +####   Named Sandboxes + +> ```sh +> # Create and start a named sandbox +> msb create --name app python +> ``` +> +> ```sh +> # Execute commands +> msb exec app -- python -c "import this" +> msb exec app -- curl https://example.com +> ``` +> +> ```sh +> # Fork a running sandbox. +> msb branch app --name experiment +> msb exec experiment -- python -c "print('An independent copy!')" +> msb branch experiment --name another-experiment +> ``` +> +> ```sh +> # Save now, resume later +> msb snapshot create saved --from-sandbox app --full +> msb create --name restored --from-snapshot app:saved +> ``` +> +> ```sh +> # Lifecycle +> msb stop app +> msb start app +> msb rm app +> ``` + +####   Image Management + +> ```sh +> msb pull python # Pull an image +> msb image ls # List cached images +> msb image rm python # Remove an image +> ``` + +####   Configuration File + +> ```sh +> msb run --conf sandbox.yaml -- octocat +> ``` +> +> ```yaml +> # sandbox.yaml +> image: python:3.12 +> memory: 64M +> network: +> allow: +> - api.github.com +> scripts: +> octocat: | +> python - <<'PY' +> import urllib.request +> +> request = urllib.request.Request( +> "https://api.github.com/octocat", +> headers={"User-Agent": "microsandbox-example"}, +> ) +> with urllib.request.urlopen(request) as response: +> print(response.read().decode()) +> PY +> ``` + +####   Install & Uninstall Sandboxes + +> ```sh +> msb install ubuntu # Install ubuntu sandbox as 'ubuntu' command +> ubuntu # Opens Ubuntu in a microVM +> msb uninstall ubuntu # Uninstall the ubuntu sandbox +> ``` + +####   Status & Inspection + +> ```sh +> msb ls # List all sandboxes +> msb ps app # Show sandbox status +> msb inspect app # Detailed sandbox info +> msb metrics app # Live CPU/memory/network stats +> ``` + +> [!TIP] +> +> Run:
+> · `msb --help` for quick help menu.
+> · `msb --tree` for complete command hierarchy and descriptions.
+> · `msb --tree` for a specific command tree. + +
+ +CLI Docs + +
+ ## sdk-darksdk  SDK The SDK lets you create and control sandboxes directly from your application. `Sandbox.builder("...").create()` boots a microVM as a child process. No infrastructure required. @@ -282,100 +382,6 @@ The SDK lets you create and control sandboxes directly from your application. `S
-## cli-darkcli  CLI - -The `msb` CLI provides a complete interface for managing sandboxes, images, and volumes. - -####   Run a Command - -> ```sh -> msb run python -- python3 -c "print('Hello from a microVM!')" -> ``` - -####   Named Sandboxes - -> ```sh -> # Create and start a named sandbox -> msb create --name app python -> ``` -> -> ```sh -> # Execute commands -> msb exec app -- python -c "import this" -> msb exec app -- curl https://example.com -> ``` -> -> ```sh -> # Lifecycle -> msb stop app -> msb start app -> msb rm app -> ``` - -####   Image Management - -> ```sh -> msb pull python # Pull an image -> msb image ls # List cached images -> msb image rm python # Remove an image -> ``` - -####   Configuration File - -> ```sh -> msb run --conf sandbox.yaml -- octocat -> ``` -> -> ```yaml -> # sandbox.yaml -> image: python:3.12 -> network: -> allow: -> - api.github.com -> scripts: -> octocat: | -> python - <<'PY' -> import urllib.request -> -> request = urllib.request.Request( -> "https://api.github.com/octocat", -> headers={"User-Agent": "microsandbox-example"}, -> ) -> with urllib.request.urlopen(request) as response: -> print(response.read().decode()) -> PY -> ``` - -####   Install & Uninstall Sandboxes - -> ```sh -> msb install ubuntu # Install ubuntu sandbox as 'ubuntu' command -> ubuntu # Opens Ubuntu in a microVM -> msb uninstall ubuntu # Uninstall the ubuntu sandbox -> ``` - -####   Status & Inspection - -> ```sh -> msb ls # List all sandboxes -> msb ps app # Show sandbox status -> msb inspect app # Detailed sandbox info -> msb metrics app # Live CPU/memory/network stats -> ``` - -> [!TIP] -> -> Run:
-> · `msb --help` for quick help menu.
-> · `msb --tree` for complete command hierarchy and descriptions.
-> · `msb --tree` for a specific command tree. - -
- -CLI Docs - -
- ## beaker-darkbeaker  Examples Practical ways to put microsandbox to work: diff --git a/crates/agentd/lib/agent.rs b/crates/agentd/lib/agent.rs index ec4b9d05d..d3d5bdacf 100644 --- a/crates/agentd/lib/agent.rs +++ b/crates/agentd/lib/agent.rs @@ -6,6 +6,7 @@ use std::fs::{File, OpenOptions}; use std::os::fd::AsRawFd; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; +use std::task::{Context, Poll}; use std::time::Instant; use bytes::BytesMut; @@ -25,8 +26,11 @@ use microsandbox_protocol::bulk::{ use microsandbox_protocol::codec::{self, DecodedFrame, MAX_FRAME_SIZE}; use microsandbox_protocol::core::{ ClockSync, CoreError, CoreErrorKind, InitAck, InitResolved, Ping, Pong, Ready, - RelayClientDisconnected, ResolvedUser, Touch, Touched, WorkloadFreeze, WorkloadFrozen, - WorkloadThaw, WorkloadThawed, + RelayClientDisconnected, ResolvedUser, Touch, Touched, WORKLOAD_TRANSPORT_BARRIER_VERSION, + WORKLOAD_TRANSPORT_BULK_BYTES, WORKLOAD_TRANSPORT_BULK_FRAMES, + WORKLOAD_TRANSPORT_CONTROL_BYTES, WORKLOAD_TRANSPORT_CONTROL_FRAMES, WorkloadFailure, + WorkloadFailureDisposition, WorkloadFreeze, WorkloadFrozen, WorkloadThaw, WorkloadThawed, + WorkloadTransportCredit, WorkloadTransportPosition, }; use microsandbox_protocol::exec::{ ExecExited, ExecFailed, ExecFailureKind, ExecRequest, ExecResize, ExecSignal, ExecStarted, @@ -50,7 +54,7 @@ use crate::config::{AgentdConfig, scripts_path}; use crate::error::{AgentdError, AgentdResult}; use crate::fs::{FsReadSession, FsState, FsStreamSession, FsWriteSession}; use crate::process::ProcessManager; -use crate::serial::{AGENT_BULK_PORT_NAME, AGENT_PORT_NAME}; +use crate::serial::{AGENT_BULK_PORT_NAME, AGENT_PORT_NAME, InputCharge, InputLane, InputWindow}; use crate::session::{ BulkOutputCommand, ExecSession, RawActivity, RawSessionCompletion, RawSessionOutput, SessionOutput, SessionOutputEnvelope, SessionOutputSender, resolve_default_user, @@ -86,6 +90,11 @@ const MAX_INPUT_BUF_SIZE: usize = MAX_FRAME_SIZE as usize + 4; /// Dedicated records additionally carry the transport-level client incarnation. const MAX_BULK_INPUT_BUF_SIZE: usize = MAX_INPUT_BUF_SIZE + CLIENT_INCARNATION_SIZE; +/// Bound actual console work between runtime scheduling points, independently of wire records. +/// Partial records count too: a readable bulk port must not hide fresh primary/PTY readiness. +const AGENT_READ_QUANTUM_BYTES: usize = 256 * 1024; +const AGENT_READ_QUANTUM_CALLS: usize = 64; + /// Maximum time to wait for the host to acknowledge the init context. const INIT_ACK_TIMEOUT_SECS: u64 = 60; @@ -135,8 +144,16 @@ const BULK_FAILURE_FLUSH_TIMEOUT: Duration = Duration::from_secs(2); //-------------------------------------------------------------------------------------------------- struct AgentState { + input_window: InputWindow, + pending_freeze: Option, + aborted_transport_attempt: Option, + output_parked: bool, + resume_output_after_flush: bool, + frozen_host_input: Option, + frozen_guest_bulk_bytes: u64, // Retain stdin/PTY and process registrations until inherited output readers finish. detached_sessions: HashMap<(u64, u32), ExecSession>, + stdin_poll_offset: usize, restored_attempt: Option, client_incarnations: HashMap, bulk_input_budget: Arc, @@ -162,12 +179,20 @@ struct FsBulkWriteWorker { pub(crate) struct AdmittedBulkRecord { record: BulkRecord, _permit: OwnedSemaphorePermit, + _transport_charge: Option, +} + +/// Both budgets follow the payload all the way into asynchronous TCP/filesystem consumption. +pub(crate) struct BulkInputPermit { + _payload: OwnedSemaphorePermit, + _transport: Option, } /// Dedicated-lane input waiting for aggregate client capacity while the control actor stays live. struct PendingBulkInput { frame: IncarnatedBulkFrame, budget: Arc, + charge: InputCharge, } struct ActivityTracker { @@ -209,6 +234,13 @@ struct BulkInputState { input: BytesMut, } +/// Shared primary/bulk work retained across select turns, including incomplete wire records. +#[derive(Default)] +struct AgentReadBudget { + bytes: usize, + calls: usize, +} + /// One correlation's pending records in the guest-to-host DRR scheduler. struct BulkWriteFlow { queue: VecDeque, @@ -218,6 +250,10 @@ struct BulkWriteFlow { /// Deferred acknowledgement after the scheduler has discarded already-queued producer output. enum BulkOutputCleanup { + Park { + completion: tokio::sync::oneshot::Sender, + wire_bytes: u64, + }, Flow(tokio::sync::oneshot::Sender<()>), Incarnation { incarnation: ClientIncarnation, @@ -225,6 +261,12 @@ enum BulkOutputCleanup { }, } +#[derive(Default)] +struct BulkOutputPosition { + parked: bool, + wire_bytes: u64, +} + //-------------------------------------------------------------------------------------------------- // Methods //-------------------------------------------------------------------------------------------------- @@ -232,7 +274,15 @@ enum BulkOutputCleanup { impl Default for AgentState { fn default() -> Self { Self { + input_window: InputWindow::new(initial_input_credit()), + pending_freeze: None, + aborted_transport_attempt: None, + output_parked: false, + resume_output_after_flush: false, + frozen_host_input: None, + frozen_guest_bulk_bytes: 0, detached_sessions: HashMap::new(), + stdin_poll_offset: 0, restored_attempt: None, client_incarnations: HashMap::new(), bulk_input_budget: Arc::new(Semaphore::new(BULK_INPUT_BYTE_CAPACITY)), @@ -253,8 +303,14 @@ impl AdmittedBulkRecord { &self.record } - pub(crate) fn into_parts(self) -> (BulkRecord, OwnedSemaphorePermit) { - (self.record, self._permit) + pub(crate) fn into_parts(self) -> (BulkRecord, BulkInputPermit) { + ( + self.record, + BulkInputPermit { + _payload: self._permit, + _transport: self._transport_charge, + }, + ) } #[cfg(test)] @@ -267,6 +323,7 @@ impl AdmittedBulkRecord { Self { record, _permit: permit, + _transport_charge: None, } } } @@ -317,14 +374,19 @@ impl BulkInputState { } /// Read at most once, then return one bounded batch already available to the actor. - async fn read_turn(&mut self) -> AgentdResult> { + async fn read_turn( + &mut self, + read_budget: &mut AgentReadBudget, + ) -> AgentdResult> { let buffered = self.drain_turn()?; if !buffered.is_empty() { return Ok(buffered); } let mut guard = self.port.readable().await?; - match guard.try_io(|inner| read_from_fd(inner.get_ref().as_raw_fd(), &mut self.read_buf)) { + match guard + .try_io(|inner| read_budget.read_fd(inner.get_ref().as_raw_fd(), &mut self.read_buf)) + { Ok(Ok(0)) => { return Err(AgentdError::ExecSession( "dedicated bulk port closed".into(), @@ -372,6 +434,33 @@ impl BulkInputState { } } +impl AgentReadBudget { + fn read_fd(&mut self, fd: i32, buf: &mut [u8]) -> std::io::Result { + let result = read_from_fd(fd, buf); + self.record_read(result.as_ref().copied().unwrap_or(0)); + result + } + + fn record_read(&mut self, bytes: usize) { + self.calls = self.calls.saturating_add(1); + self.bytes = self.bytes.saturating_add(bytes); + } + + fn exhausted(&self) -> bool { + self.bytes >= AGENT_READ_QUANTUM_BYTES || self.calls >= AGENT_READ_QUANTUM_CALLS + } + + /// Call outside the competing select futures, after decoded records have an owning queue. + /// AsyncFd::readable can stay immediately ready without spending Tokio's cooperative budget; + /// returning to select alone does not let the driver discover a writable PTY or new control IO. + async fn yield_if_exhausted(&mut self) { + if self.exhausted() { + tokio::task::yield_now().await; + *self = Self::default(); + } + } +} + //-------------------------------------------------------------------------------------------------- // Functions //-------------------------------------------------------------------------------------------------- @@ -461,6 +550,7 @@ pub async fn run( // Shared arenas are negotiated only on the local SDK-to-runtime hop. Agentd speaks to // the runtime over the guest consoles, so the runtime injects this capability later. local_transport: None, + workload_transport_barrier_version: Some(WORKLOAD_TRANSPORT_BARRIER_VERSION), }, ) .map_err(|e| AgentdError::ExecSession(format!("encode ready: {e}")))?; @@ -485,9 +575,58 @@ pub async fn run( let mut pending_bulk_inputs = VecDeque::::new(); let mut bulk_input_bytes_since_snapshot = 0usize; let mut last_bulk_input_snapshot = Instant::now(); + let input_refunds = state.input_window.clone(); + let mut credit_deadline = None; + let mut last_input_credit = state.input_window.credit()?; + let mut read_budget = AgentReadBudget::default(); // Main loop. 'agent: loop { + // All consumed bytes are now in persistent input buffers or destination-owned records. + // Never yield after decoding inside read_turn: select cancellation could drop its frames. + read_budget.yield_if_exhausted().await; + if state.resume_output_after_flush { + // The private Thawed reply crosses the primary lane before either ordinary writer + // becomes eligible again. A partial old record is never abandoned by this release. + flush_write_buf(&async_port, &mut serial_out_buf).await?; + session_tx + .resume_bulk_output() + .await + .map_err(|error| AgentdError::ExecSession(error.into()))?; + state.resume_output_after_flush = false; + state.output_parked = false; + } + if state.pending_freeze.as_ref().is_some_and(|message| { + message.payload::().is_ok_and(|request| { + let position = state.input_window.position(); + position.bulk_bytes >= request.host_input.bulk_bytes + && position.bulk_frames >= request.host_input.bulk_frames + }) + }) { + let message = state.pending_freeze.take().expect("ready pending freeze"); + handle_message_with_charge( + message, + &mut state, + &mut activity, + &mut session_tx, + &mut serial_out_buf, + config, + &mut workload, + &heartbeat_control, + None, + ) + .await?; + flush_write_buf(&async_port, &mut serial_out_buf).await?; + } + if !state.output_parked { + let credit = state.input_window.credit()?; + if input_credit_update_due(&last_input_credit, &credit) { + encode_input_credit(credit, &mut serial_out_buf)?; + flush_write_buf(&async_port, &mut serial_out_buf).await?; + last_input_credit = credit; + credit_deadline = None; + } + } // A control-lane disconnect cancels the pending acquire future at the select boundary. // Remove its stale record before rebuilding that future against the global budget. pending_bulk_inputs.retain(|pending| { @@ -502,6 +641,29 @@ pub async fn run( }); let has_pending_bulk_admission = pending_bulk_admission.is_some(); tokio::select! { + _ = input_refunds.refunded(), if !state.output_parked && credit_deadline.is_none() => { + if state.input_window.credit()? != last_input_credit { + credit_deadline = Some(time::Instant::now() + Duration::from_millis(5)); + } + } + + _ = wait_input_credit_deadline(credit_deadline), if !state.output_parked => { + credit_deadline = None; + let credit = state.input_window.credit()?; + if credit != last_input_credit { + encode_input_credit(credit, &mut serial_out_buf)?; + flush_write_buf(&async_port, &mut serial_out_buf).await?; + last_input_credit = credit; + } + } + + (id, inherited, result) = std::future::poll_fn(|cx| poll_pending_stdin(&mut state, cx)), + if !state.output_parked && !workload.is_frozen() => { + if !inherited && let Err(error) = result { + encode_stdin_error(id, &AgentdError::Io(error), &mut serial_out_buf)?; + flush_write_buf(&async_port, &mut serial_out_buf).await?; + } + } failure = process_manager_failure.changed() => { let error = match failure { Ok(()) => process_manager_failure @@ -514,6 +676,11 @@ pub async fn run( } Some(error) = recv_optional(&mut bulk_failure_rx) => { + if state.output_parked { + return Err(AgentdError::ExecSession(format!( + "dedicated bulk transport failed while frozen: {error}" + ))); + } cancel_all_bulk_correlations( &mut state, &session_tx, @@ -567,6 +734,7 @@ pub async fn run( AdmittedBulkRecord { record: pending.frame.record, _permit: permit, + _transport_charge: Some(pending.charge), }, &mut state, &mut activity, @@ -587,7 +755,7 @@ pub async fn run( } } - Some(envelope) = recv_optional(&mut combined_bulk_rx) => { + Some(envelope) = recv_optional(&mut combined_bulk_rx), if !state.output_parked => { if discard_inherited_output(&mut state, &envelope, session_tx.generation()) { continue; } @@ -613,7 +781,7 @@ pub async fn run( bulk_input .as_mut() .expect("guarded dedicated bulk input") - .read_turn() + .read_turn(&mut read_budget) .await }, if bulk_input.is_some() && pending_bulk_inputs.is_empty() => { let frames = match turn { @@ -635,7 +803,13 @@ pub async fn run( } }; let mut capacity_deferred = false; + if state.output_parked && !frames.is_empty() { + return Err(AgentdError::ExecSession("bulk input crossed the frozen transport cut".into())); + } for frame in frames { + let charge = state.input_window.admit( + InputLane::Bulk, bulk_wire_bytes(&frame.record, true), + )?; if !validate_bulk_client_incarnation( &state, frame.record.id, @@ -647,7 +821,7 @@ pub async fn run( } let budget = Arc::clone(&state.bulk_input_budget); if capacity_deferred { - pending_bulk_inputs.push_back(PendingBulkInput { frame, budget }); + pending_bulk_inputs.push_back(PendingBulkInput { frame, budget, charge }); continue; } let payload_len = frame.record.payload.len(); @@ -658,7 +832,7 @@ pub async fn run( Ok(permit) => permit, Err(_) if !budget.is_closed() => { capacity_deferred = true; - pending_bulk_inputs.push_back(PendingBulkInput { frame, budget }); + pending_bulk_inputs.push_back(PendingBulkInput { frame, budget, charge }); continue; } Err(error) => { @@ -672,6 +846,7 @@ pub async fn run( AdmittedBulkRecord { record: frame.record, _permit: permit, + _transport_charge: Some(charge), }, &mut state, &mut activity, @@ -703,7 +878,9 @@ pub async fn run( let mut combined_turn_exhausted = false; loop { - match guard.try_io(|inner| read_from_fd(inner.get_ref().as_raw_fd(), &mut read_buf)) { + match guard.try_io(|inner| { + read_budget.read_fd(inner.get_ref().as_raw_fd(), &mut read_buf) + }) { Ok(Ok(0)) => { // EOF on serial — host disconnected. if !handoff::is_pid_1() { @@ -729,12 +906,19 @@ pub async fn run( // correlation ID with `core.error`; unrecoverable // frame-level failures still close the agent loop. loop { + let input_before = serial_in_buf.len(); if let Some(connected) = try_decode_relay_client_connected_from_bytes(&mut serial_in_buf) .map_err(|e| AgentdError::ExecSession(format!( "decode relay client lease: {e}" )))? { + if state.output_parked { + return Err(AgentdError::ExecSession("relay lease crossed the frozen transport cut".into())); + } + let _charge = state.input_window.admit( + InputLane::Control, input_before - serial_in_buf.len(), + )?; establish_relay_client(&mut state, connected)?; continue; } @@ -743,6 +927,7 @@ pub async fn run( else { break; }; + let wire_bytes = input_before - serial_in_buf.len(); let DecodedFrame::Control(msg) = frame else { let DecodedFrame::Bulk(record) = frame else { unreachable!(); @@ -753,10 +938,14 @@ pub async fn run( .into(), )); } + if state.output_parked { + return Err(AgentdError::ExecSession("raw input crossed the frozen transport cut".into())); + } let bulk_session_tx = session_tx.with_incarnation( client_incarnation_for_id(&state, record.id), ); let payload_len = record.payload.len(); + let charge = state.input_window.admit(InputLane::Bulk, wire_bytes)?; let budget = Arc::clone(&state.bulk_input_budget); let permit = acquire_bulk_input_permit(Some(( budget, @@ -766,6 +955,7 @@ pub async fn run( AdmittedBulkRecord { record, _permit: permit, + _transport_charge: Some(charge), }, &mut state, &mut activity, @@ -798,6 +988,19 @@ pub async fn run( } continue; }; + let charge = if private_lifecycle_message(&msg) { + None + } else { + if state.output_parked { + return Err(AgentdError::ExecSession("ordinary input crossed the frozen transport cut".into())); + } + let lane = if msg.t.uses_workload_data_credit() { + InputLane::Bulk + } else { + InputLane::Control + }; + Some(state.input_window.admit(lane, wire_bytes)?) + }; if msg.flags != msg.t.flags() { let out_before = serial_out_buf.len(); encode_core_error_if_supported( @@ -828,7 +1031,7 @@ pub async fn run( } let out_before = serial_out_buf.len(); - handle_message( + handle_message_with_charge( msg, &mut state, &mut activity, @@ -837,6 +1040,7 @@ pub async fn run( config, &mut workload, &heartbeat_control, + charge, ).await?; record_encoded_guest_messages( &serial_out_buf, @@ -850,11 +1054,22 @@ pub async fn run( if !serial_out_buf.is_empty() { flush_write_buf(&async_port, &mut serial_out_buf).await?; } - if combined_turn_exhausted { + // Thawed has now crossed the wire. A host can immediately resume + // ordinary input, so release our parked writers at the outer-loop + // boundary before draining another readable batch. + if combined_turn_exhausted + || state.resume_output_after_flush + || read_budget.exhausted() + { + break; + } + } + Ok(Err(e)) if e.kind() == std::io::ErrorKind::Interrupted => { + if read_budget.exhausted() { break; } + continue; } - Ok(Err(e)) if e.kind() == std::io::ErrorKind::Interrupted => continue, Ok(Err(_)) if !handoff::is_pid_1() => { guard.clear_ready(); drop(guard); @@ -868,7 +1083,7 @@ pub async fn run( } // Receive output events from session reader tasks. - Some(envelope) = session_rx.recv() => { + Some(envelope) = session_rx.recv(), if !state.output_parked => { if discard_inherited_output(&mut state, &envelope, session_tx.generation()) { continue; } @@ -1024,6 +1239,7 @@ async fn bulk_writer_task( let mut retired = HashMap::>::new(); let mut retiring_incarnations = HashSet::::new(); let mut pending_activity = RawActivity::default(); + let mut transport = BulkOutputPosition::default(); loop { let mut cleanups = Vec::new(); @@ -1037,9 +1253,10 @@ async fn bulk_writer_task( &mut active, &mut retired, &mut retiring_incarnations, + &mut transport, )?); } - Some(envelope) = output_rx.recv() => { + Some(envelope) = output_rx.recv(), if !transport.parked => { enqueue_bulk_output( envelope, generation, @@ -1059,6 +1276,7 @@ async fn bulk_writer_task( &mut active, &mut retired, &mut retiring_incarnations, + &mut transport, )?); } while let Ok(envelope) = output_rx.try_recv() { @@ -1073,7 +1291,7 @@ async fn bulk_writer_task( } complete_bulk_output_cleanups(cleanups, &mut retired, &mut retiring_incarnations); - while !active.is_empty() { + while !transport.parked && !active.is_empty() { let round_len = active.len(); let quantum = if round_len == 1 { BULK_SCHEDULER_MAX_BURST @@ -1081,6 +1299,9 @@ async fn bulk_writer_task( BULK_SCHEDULER_QUANTUM }; for _ in 0..round_len { + if transport.parked || active.is_empty() { + break; + } let key = active.pop_front().expect("active flow exists"); if let Some(flow) = flows.get_mut(&key) { flow.deficit = flow @@ -1124,6 +1345,12 @@ async fn bulk_writer_task( let output_activity = output.activity; write_incarnated_bulk_record_async_fd(&async_port, incarnation, &output.record) .await?; + transport.wire_bytes = transport + .wire_bytes + .checked_add(bulk_wire_bytes(&output.record, true) as u64) + .ok_or_else(|| { + AgentdError::ExecSession("bulk output counter exhausted".into()) + })?; pending_activity.guest_messages = pending_activity .guest_messages .saturating_add(output_activity.guest_messages); @@ -1141,11 +1368,43 @@ async fn bulk_writer_task( publish_bulk_activity(&activity_tx, &mut pending_activity)?; } burst = burst.saturating_add(next_len); + // Lifecycle commands cut only between complete wire records. In particular, + // never wait for the rest of this flow's burst before acknowledging Park. + let mut cleanups = Vec::new(); + while let Ok(command) = command_rx.try_recv() { + cleanups.push(apply_bulk_output_command( + command, + &mut generation, + &mut flows, + &mut active, + &mut retired, + &mut retiring_incarnations, + &mut transport, + )?); + } + while let Ok(envelope) = output_rx.try_recv() { + enqueue_bulk_output( + envelope, + generation, + &mut flows, + &mut active, + &retired, + &retiring_incarnations, + )?; + } + complete_bulk_output_cleanups( + cleanups, + &mut retired, + &mut retiring_incarnations, + ); + if transport.parked { + break; + } } if flows.get(&key).is_some_and(|flow| flow.queue.is_empty()) { flows.remove(&key); - } else { + } else if flows.contains_key(&key) && !active.contains(&key) { active.push_back(key); } } @@ -1159,6 +1418,7 @@ async fn bulk_writer_task( &mut active, &mut retired, &mut retiring_incarnations, + &mut transport, )?); } while let Ok(envelope) = output_rx.try_recv() { @@ -1206,8 +1466,20 @@ fn apply_bulk_output_command( active: &mut VecDeque<(ClientIncarnation, u32)>, retired: &mut HashMap>, retiring_incarnations: &mut HashSet, + transport: &mut BulkOutputPosition, ) -> AgentdResult { match command { + BulkOutputCommand::Park { completion } => { + transport.parked = true; + Ok(BulkOutputCleanup::Park { + completion, + wire_bytes: transport.wire_bytes, + }) + } + BulkOutputCommand::Resume { completion } => { + transport.parked = false; + Ok(BulkOutputCleanup::Flow(completion)) + } BulkOutputCommand::Restore { generation: next, completion, @@ -1253,6 +1525,12 @@ fn complete_bulk_output_cleanups( ) { for cleanup in cleanups { match cleanup { + BulkOutputCleanup::Park { + completion, + wire_bytes, + } => { + let _ = completion.send(wire_bytes); + } BulkOutputCleanup::Flow(completion) => { let _ = completion.send(()); } @@ -2095,11 +2373,16 @@ async fn restore_client_state( sender: &mut SessionOutputSender, ) -> AgentdResult<()> { let generation = sender.generation(); - state.detached_sessions.extend( - std::mem::take(&mut state.sessions) - .into_iter() - .map(|(id, session)| ((generation, id), session)), - ); + state + .detached_sessions + .extend( + std::mem::take(&mut state.sessions) + .into_iter() + .map(|(id, mut session)| { + session.detach_stdin(); + ((generation, id), session) + }), + ); state.client_incarnations.clear(); for (_, session) in state.read_sessions.drain() { session.abort(); @@ -2147,7 +2430,7 @@ fn discard_inherited_output( // Keep the loop-owned latch and output generation explicit at this dispatch boundary; merging // them into AgentState would obscure the ownership needed by restore and background producers. #[allow(clippy::too_many_arguments)] -async fn handle_message( +async fn handle_message_with_charge( msg: Message, state: &mut AgentState, activity: &mut ActivityTracker, @@ -2156,6 +2439,7 @@ async fn handle_message( config: &AgentdConfig, workload: &mut WorkloadLatch, heartbeat_control: &heartbeat::HeartbeatControl, + mut input_charge: Option, ) -> AgentdResult<()> { // Background producers retain the range owner that opened them. The main loop can then drop // queued output after a disconnect instead of relabelling it with a recycled correlation ID. @@ -2183,6 +2467,7 @@ async fn handle_message( kind: CoreErrorKind::CapabilityUnavailable, message, offending_type: Some(msg.t.as_str().into()), + workload_failure: None, }, ), } @@ -2224,6 +2509,61 @@ async fn handle_message( else { return Ok(()); }; + let position = state.input_window.position(); + if state.pending_freeze.as_ref().is_some_and(|pending| { + pending + .payload::() + .is_ok_and(|old| old != request) + }) { + encode_workload_error( + &msg, + &request.attempt_id, + WorkloadLatchError::Conflict("another transport cut is pending".into()), + out_buf, + )?; + return Ok(()); + } + if workload.is_frozen() { + if let Err(error) = workload.require_frozen_attempt(&request.attempt_id) { + encode_workload_error(&msg, &request.attempt_id, error, out_buf)?; + return Ok(()); + } + if state + .frozen_host_input + .is_some_and(|cut| cut != request.host_input) + { + encode_workload_error( + &msg, + &request.attempt_id, + WorkloadLatchError::Conflict("frozen transport cut cannot change".into()), + out_buf, + )?; + return Ok(()); + } + } + if request.host_input.control_bytes != position.control_bytes + || request.host_input.control_frames != position.control_frames + || request.host_input.bulk_bytes < position.bulk_bytes + || request.host_input.bulk_frames < position.bulk_frames + { + encode_workload_error( + &msg, + &request.attempt_id, + WorkloadLatchError::Conflict( + "transport input cut differs from received complete frames".into(), + ), + out_buf, + )?; + return Ok(()); + } + state.aborted_transport_attempt = None; + if request.host_input != position { + // Retain the private request while the independently ordered bulk prefix + // arrives. Decoding owns its bytes; application consumption is not required. + state.pending_freeze = Some(msg); + return Ok(()); + } + state.pending_freeze = None; let was_frozen = workload.is_frozen(); match workload.freeze(&request.attempt_id) { Ok(()) => { @@ -2241,15 +2581,24 @@ async fn handle_message( }; encode_workload_error( &msg, + &request.attempt_id, WorkloadLatchError::Io(std::io::Error::other(message)), out_buf, )?; } else { + state.output_parked = true; + state.frozen_host_input = Some(position); + state.frozen_guest_bulk_bytes = root_session_tx + .park_bulk_output() + .await + .map_err(|error| AgentdError::ExecSession(error.into()))?; let reply = Message::with_payload( MessageType::WorkloadFrozen, msg.id, &WorkloadFrozen { attempt_id: request.attempt_id, + guest_bulk_bytes_target: state.frozen_guest_bulk_bytes, + input_credit: state.input_window.credit()?, }, ) .map_err(|error| { @@ -2264,7 +2613,7 @@ async fn handle_message( })?; } } - Err(error) => encode_workload_error(&msg, error, out_buf)?, + Err(error) => encode_workload_error(&msg, &request.attempt_id, error, out_buf)?, } } @@ -2272,11 +2621,67 @@ async fn handle_message( let Some(request) = decode_payload_or_core_error::(&msg, out_buf)? else { return Ok(()); }; + if state.pending_freeze.as_ref().is_some_and(|pending| { + pending + .payload::() + .is_ok_and(|freeze| freeze.attempt_id != request.attempt_id) + }) { + encode_workload_error( + &msg, + &request.attempt_id, + WorkloadLatchError::Conflict("another transport cut is pending".into()), + out_buf, + )?; + return Ok(()); + } + if state.pending_freeze.as_ref().is_some_and(|pending| { + pending + .payload::() + .is_ok_and(|freeze| freeze.attempt_id == request.attempt_id) + }) && !workload.is_frozen() + || (!workload.is_frozen() + && state.aborted_transport_attempt.as_deref() == Some(&request.attempt_id)) + { + if request.mode != microsandbox_protocol::core::WorkloadThawMode::Continue { + encode_workload_error( + &msg, + &request.attempt_id, + WorkloadLatchError::Conflict( + "restore requires a completed transport cut".into(), + ), + out_buf, + )?; + return Ok(()); + } + if let Some(pending) = state.pending_freeze.take() { + // The abandoned Freeze RPC still owns a private host reply slot. Complete + // it explicitly before acknowledging the independent recovery Thaw RPC. + encode_workload_error( + &pending, + &request.attempt_id, + WorkloadLatchError::Conflict( + "transport cut aborted by source continuation".into(), + ), + out_buf, + )?; + } + state.aborted_transport_attempt = Some(request.attempt_id.clone()); + encode_input_credit(state.input_window.credit()?, out_buf)?; + let reply = Message::with_payload( + MessageType::WorkloadThawed, + msg.id, + &WorkloadThawed { + attempt_id: request.attempt_id, + }, + )?; + codec::encode_to_buf(&reply, out_buf)?; + return Ok(()); + } if request.mode == microsandbox_protocol::core::WorkloadThawMode::Restore && state.restored_attempt.as_deref() != Some(&request.attempt_id) { if let Err(error) = workload.require_frozen_attempt(&request.attempt_id) { - encode_workload_error(&msg, error, out_buf)?; + encode_workload_error(&msg, &request.attempt_id, error, out_buf)?; return Ok(()); } // Never call ordinary disconnect here: it kills the very processes we captured. @@ -2286,6 +2691,11 @@ async fn handle_message( match workload.thaw(&request.attempt_id) { Ok(()) => { heartbeat_control.resume(); + // Cumulative grants include refunds from detached transfer cleanup; accepted + // stdin remains charged until its inherited process consumes it after thaw. + encode_input_credit(state.input_window.credit()?, out_buf)?; + state.resume_output_after_flush = true; + state.frozen_host_input = None; let reply = Message::with_payload( MessageType::WorkloadThawed, msg.id, @@ -2302,7 +2712,7 @@ async fn handle_message( AgentdError::ExecSession(format!("encode workload-thawed frame: {error}")) })?; } - Err(error) => encode_workload_error(&msg, error, out_buf)?, + Err(error) => encode_workload_error(&msg, &request.attempt_id, error, out_buf)?, } } @@ -2395,22 +2805,10 @@ async fn handle_message( let Some(stdin) = decode_payload_or_core_error::(&msg, out_buf)? else { return Ok(()); }; - if let Some(session) = state.sessions.get_mut(&msg.id) { - if stdin.data.is_empty() { - // Empty data signals EOF — close stdin. - session.close_stdin(); - } else if let Err(e) = session.write_stdin(&stdin.data).await { - let payload = stdin_error_payload(&e); - eprintln!("stdin write error on session {}: {e}", msg.id); - let reply = - Message::with_payload(MessageType::ExecStdinError, msg.id, &payload) - .map_err(|e| { - AgentdError::ExecSession(format!("encode stdin error: {e}")) - })?; - codec::encode_to_buf(&reply, out_buf).map_err(|e| { - AgentdError::ExecSession(format!("encode stdin error frame: {e}")) - })?; - } + if let Some(session) = state.sessions.get_mut(&msg.id) + && let Err(error) = session.enqueue_stdin(stdin.data, input_charge.take()) + { + encode_stdin_error(msg.id, &AgentdError::Io(error), out_buf)?; } } @@ -2506,7 +2904,10 @@ async fn handle_message( BulkKind::Tcp => { let result = match state.tcp_sessions.get(&msg.id) { Some(session) => session.apply_credit(credit).await, - None => Err(format!("unknown TCP session: {}", msg.id)), + // A terminal may retire the producer before the other physical lane + // finishes delivering its output. Late credit has no recipient and must + // not cancel that tail or manufacture a second terminal response. + None => Ok(()), }; if let Err(error) = result { encode_bulk_tcp_failure(msg.id, error, out_buf)?; @@ -2589,7 +2990,10 @@ async fn handle_message( }; let len = data.data.len(); if let Some(session) = state.tcp_sessions.get(&msg.id) { - if let Err(e) = session.write_data(data.data).await { + if let Err(e) = session + .write_data_charged(data.data, input_charge.take()) + .await + { state.tcp_sessions.remove(&msg.id); clear_bulk_receive_state(state, msg.id); encode_tcp_failed(msg.id, e, out_buf)?; @@ -2606,7 +3010,7 @@ async fn handle_message( return Ok(()); }; if let Some(session) = state.tcp_sessions.get(&msg.id) - && let Err(e) = session.close_write().await + && let Err(e) = session.close_write_charged(input_charge.take()).await { state.tcp_sessions.remove(&msg.id); clear_bulk_receive_state(state, msg.id); @@ -2749,6 +3153,7 @@ fn guest_message_refreshes_idle_timer(t: &MessageType) -> bool { | MessageType::Touched | MessageType::WorkloadFrozen | MessageType::WorkloadThawed + | MessageType::WorkloadTransportCredit | MessageType::CoreError ) } @@ -2831,6 +3236,106 @@ fn heartbeat_snapshot(state: &AgentState, activity: &ActivityTracker) -> Heartbe } } +fn initial_input_credit() -> WorkloadTransportCredit { + WorkloadTransportCredit { + control_bytes: WORKLOAD_TRANSPORT_CONTROL_BYTES, + control_frames: WORKLOAD_TRANSPORT_CONTROL_FRAMES, + bulk_bytes: WORKLOAD_TRANSPORT_BULK_BYTES, + bulk_frames: WORKLOAD_TRANSPORT_BULK_FRAMES, + } +} + +fn bulk_wire_bytes(record: &BulkRecord, dedicated: bool) -> usize { + // The counter follows complete wire records, not destination payload consumption. Combined + // raw frames use the same bulk allowance without the dedicated lane's incarnation prefix. + 4 + FRAME_HEADER_SIZE + + BULK_HEADER_SIZE + + record.payload.len() + + if dedicated { + CLIENT_INCARNATION_SIZE + } else { + 0 + } +} + +fn private_lifecycle_message(message: &Message) -> bool { + message.id == u32::MAX + && matches!( + message.t, + MessageType::WorkloadFreeze | MessageType::WorkloadThaw + ) + && message.flags == message.t.flags() +} + +fn input_credit_update_due( + previous: &WorkloadTransportCredit, + current: &WorkloadTransportCredit, +) -> bool { + current.control_bytes.saturating_sub(previous.control_bytes) >= 64 * 1024 + || current.bulk_bytes.saturating_sub(previous.bulk_bytes) >= 64 * 1024 + || current + .control_frames + .saturating_sub(previous.control_frames) + >= 16 + || current.bulk_frames.saturating_sub(previous.bulk_frames) >= 16 +} + +async fn wait_input_credit_deadline(deadline: Option) { + match deadline { + Some(deadline) => time::sleep_until(deadline).await, + None => std::future::pending().await, + } +} + +fn encode_input_credit(credit: WorkloadTransportCredit, out_buf: &mut Vec) -> AgentdResult<()> { + // The host intercepts the reserved runtime correlation before SDK output admission, so a + // stalled client cannot stop refunds needed by unrelated stdin and bulk producers. + let message = Message::with_payload(MessageType::WorkloadTransportCredit, u32::MAX, &credit)?; + codec::encode_to_buf(&message, out_buf)?; + Ok(()) +} + +fn poll_pending_stdin( + state: &mut AgentState, + cx: &mut Context<'_>, +) -> Poll<(u32, bool, std::io::Result<()>)> { + // Rotate the first descriptor considered so a continuously writable active session cannot + // starve inherited input. The cursor needs no per-payload task or additional ownership queue. + let total = state.sessions.len() + state.detached_sessions.len(); + let start = state.stdin_poll_offset.checked_rem(total).unwrap_or(0); + for pass in 0..2 { + let sessions = state + .sessions + .iter_mut() + .map(|(id, session)| (*id, false, session)) + .chain( + state + .detached_sessions + .iter_mut() + .map(|((_, id), session)| (*id, true, session)), + ); + for (index, (id, inherited, session)) in sessions.enumerate() { + if (pass == 0 && index < start) || (pass == 1 && index >= start) { + continue; + } + if session.has_pending_stdin() + && let Poll::Ready(result) = session.poll_pending_stdin(cx) + { + state.stdin_poll_offset = index + 1; + return Poll::Ready((id, inherited, result)); + } + } + } + Poll::Pending +} + +fn encode_stdin_error(id: u32, error: &AgentdError, out_buf: &mut Vec) -> AgentdResult<()> { + let message = + Message::with_payload(MessageType::ExecStdinError, id, &stdin_error_payload(error))?; + codec::encode_to_buf(&message, out_buf)?; + Ok(()) +} + fn publish_heartbeat_snapshot( heartbeat_tx: &watch::Sender, state: &AgentState, @@ -3098,6 +3603,7 @@ fn encode_core_error( kind, message, offending_type, + workload_failure: None, }, ) .map_err(|e| AgentdError::ExecSession(format!("encode core error: {e}")))?; @@ -3108,6 +3614,7 @@ fn encode_core_error( fn encode_workload_error( source: &Message, + attempt_id: &str, error: WorkloadLatchError, out_buf: &mut Vec, ) -> AgentdResult<()> { @@ -3118,14 +3625,33 @@ fn encode_workload_error( WorkloadLatchError::InvalidAttempt(_) => CoreErrorKind::InvalidPayload, WorkloadLatchError::Conflict(_) => CoreErrorKind::InvalidSession, }; - encode_core_error_if_supported( - source, + // Keep the existing error category readable by older hosts. Only the additive, + // attempt-scoped detail proves that a basic-pause fallback is safe. + let disposition = match &error { + WorkloadLatchError::Unavailable(_) => WorkloadFailureDisposition::Unavailable, + _ => WorkloadFailureDisposition::RecoveryRequired, + }; + if !MessageType::CoreError.is_available_at(source.v) { + return Err(AgentdError::ExecSession( + "peer cannot receive workload errors".into(), + )); + } + let reply = Message::with_payload( + MessageType::CoreError, source.id, - kind, - error.to_string(), - Some(source.t.as_str().to_string()), - out_buf, + &CoreError { + kind, + message: error.to_string(), + offending_type: Some(source.t.as_str().to_string()), + workload_failure: Some(WorkloadFailure { + attempt_id: attempt_id.to_string(), + disposition, + }), + }, ) + .map_err(|error| AgentdError::ExecSession(format!("encode workload error: {error}")))?; + codec::encode_to_buf(&reply, out_buf) + .map_err(|error| AgentdError::ExecSession(format!("encode workload error frame: {error}"))) } fn encode_exec_failed(id: u32, payload: ExecFailed, out_buf: &mut Vec) -> AgentdResult<()> { @@ -3568,94 +4094,755 @@ mod tests { use bytes::Bytes; use microsandbox_protocol::message::PROTOCOL_VERSION; - #[test] - fn coalesced_bootstrap_and_init_ack_retain_the_second_frame() { - let bootstrap = GuestBootstrap::default(); - let bootstrap_message = - Message::with_payload(MessageType::Bootstrap, 0, &bootstrap).unwrap(); - let ack_message = Message::with_payload(MessageType::InitAck, 0, &InitAck {}).unwrap(); - let mut state = BootConsoleState::default(); - codec::encode_to_buf(&bootstrap_message, &mut state.input).unwrap(); - codec::encode_to_buf(&ack_message, &mut state.input).unwrap(); - - let decoded = read_boot_message( - -1, - &mut state, - Instant::now() + std::time::Duration::from_secs(1), - "guest bootstrap", - ) - .unwrap(); - assert_eq!(decode_bootstrap_message(decoded).unwrap(), bootstrap); - assert!( - !state.input.is_empty(), - "init ack frame should remain buffered" - ); - - wait_for_init_ack( - -1, - &mut state, - Instant::now() + std::time::Duration::from_secs(1), + #[allow(clippy::too_many_arguments)] + async fn handle_message( + msg: Message, + state: &mut AgentState, + activity: &mut ActivityTracker, + sender: &mut SessionOutputSender, + out_buf: &mut Vec, + config: &AgentdConfig, + workload: &mut WorkloadLatch, + heartbeat: &heartbeat::HeartbeatControl, + ) -> AgentdResult<()> { + handle_message_with_charge( + msg, state, activity, sender, out_buf, config, workload, heartbeat, None, ) - .unwrap(); - assert!(state.input.is_empty()); - } - - #[test] - fn bootstrap_rejects_non_control_correlation_fields() { - let mut message = - Message::with_payload(MessageType::Bootstrap, 1, &GuestBootstrap::default()).unwrap(); - message.flags = 1; - - let error = decode_bootstrap_message(message).unwrap_err(); - assert!(error.to_string().contains("requires id=0 and flags=0")); - } - - #[test] - fn bootstrap_rejects_wrong_first_message_type() { - let message = Message::with_payload(MessageType::Ping, 0, &Ping {}).unwrap(); - - let error = decode_bootstrap_message(message).unwrap_err(); - assert!(error.to_string().contains("expected core.bootstrap")); + .await } - #[test] - fn bootstrap_rejects_older_protocol_generation() { - let mut message = - Message::with_payload(MessageType::Bootstrap, 0, &GuestBootstrap::default()).unwrap(); - let min_version = MessageType::Bootstrap.min_protocol_version(); - assert!(min_version > 0); - message.v = min_version - 1; - - let error = decode_bootstrap_message(message).unwrap_err(); - assert!(error.to_string().contains("or newer")); + fn decode_reply_skipping_credit(bytes: &mut BytesMut) -> Message { + loop { + let Some(DecodedFrame::Control(reply)) = + codec::try_decode_frame_from_bytes(bytes).unwrap() + else { + panic!("missing control reply"); + }; + if reply.t != MessageType::WorkloadTransportCredit { + return reply; + } + assert_eq!(reply.id, u32::MAX); + } } #[test] - fn bootstrap_accepts_minimum_supported_protocol_generation() { - let mut message = - Message::with_payload(MessageType::Bootstrap, 0, &GuestBootstrap::default()).unwrap(); - message.v = MessageType::Bootstrap.min_protocol_version(); - - assert_eq!( - decode_bootstrap_message(message).unwrap(), - GuestBootstrap::default() + fn admitted_transport_window_fits_each_filesystem_input_queue() { + // Each accepted raw record occupies one queue slot until consumed. Even if one flow + // receives the whole allowance from both lanes, admission cannot stall the serial actor. + assert!( + WORKLOAD_TRANSPORT_CONTROL_FRAMES + WORKLOAD_TRANSPORT_BULK_FRAMES + <= FS_BULK_INPUT_ITEM_CAPACITY as u64 ); + assert!(WORKLOAD_TRANSPORT_CONTROL_BYTES <= BULK_INPUT_BYTE_CAPACITY as u64); + assert!(WORKLOAD_TRANSPORT_BULK_BYTES <= BULK_INPUT_BYTE_CAPACITY as u64); } #[test] - fn bootstrap_accepts_newer_additive_protocol_generation() { - let mut message = - Message::with_payload(MessageType::Bootstrap, 0, &GuestBootstrap::default()).unwrap(); - message.v = PROTOCOL_VERSION + 1; - + fn private_admission_and_credit_batching_use_the_complete_wire_contract() { + let mut request = Message::with_payload( + MessageType::WorkloadFreeze, + u32::MAX, + &WorkloadFreeze { + attempt_id: "cut".into(), + host_input: Default::default(), + }, + ) + .unwrap(); + assert!(private_lifecycle_message(&request)); + request.id = 0; + assert!(!private_lifecycle_message(&request)); + request.id = u32::MAX; + request.t = MessageType::ClockSync; + assert!(!private_lifecycle_message(&request)); + let record = BulkRecord { + id: 1, + kind: BulkKind::Tcp, + flow: BulkFlow::HostToGuest, + offset: 0, + payload: Bytes::from_static(b"payload"), + }; + let mut encoded = codec::encode_bulk_header(&record).unwrap().to_vec(); + encoded.extend_from_slice(&record.payload); + assert_eq!(bulk_wire_bytes(&record, false), encoded.len()); assert_eq!( - decode_bootstrap_message(message).unwrap(), - GuestBootstrap::default() + bulk_wire_bytes(&record, true), + encoded.len() + CLIENT_INCARNATION_SIZE ); + let previous = initial_input_credit(); + let mut current = previous; + current.control_frames += 15; + assert!(!input_credit_update_due(&previous, ¤t)); + current.control_frames += 1; + assert!(input_credit_update_due(&previous, ¤t)); + let mut credit = Vec::new(); + encode_input_credit(current, &mut credit).unwrap(); + let mut bytes = BytesMut::from(credit.as_slice()); + let Some(DecodedFrame::Control(reply)) = + codec::try_decode_frame_from_bytes(&mut bytes).unwrap() + else { + panic!() + }; + assert_eq!(reply.id, u32::MAX); + assert_eq!(reply.payload::().unwrap(), current); + assert!(!guest_message_refreshes_idle_timer(&reply.t)); } - #[test] - fn bootstrap_rejects_malformed_payload() { + #[tokio::test] + async fn transport_cut_waits_for_decoded_prefix_and_abort_is_owned_and_repeatable() { + use microsandbox_protocol::core::WorkloadThawMode::{Continue, Restore}; + let mut state = AgentState::default(); + let (mut sender, _output) = SessionOutputSender::channel(); + let mut activity = ActivityTracker::new(); + let config = AgentdConfig { + user: None, + security_profile: Default::default(), + default_cwd: None, + default_env: Vec::new(), + }; + let heartbeat = heartbeat::HeartbeatControl::default(); + let mut workload = crate::workload::tests::fake_latch(); + let target = WorkloadTransportPosition { + bulk_bytes: 80, + bulk_frames: 1, + ..Default::default() + }; + let freeze = || { + Message::with_payload( + MessageType::WorkloadFreeze, + u32::MAX, + &WorkloadFreeze { + attempt_id: "prefix".into(), + host_input: target, + }, + ) + .unwrap() + }; + for _ in 0..2 { + let mut out = Vec::new(); + handle_message( + freeze(), + &mut state, + &mut activity, + &mut sender, + &mut out, + &config, + &mut workload, + &heartbeat, + ) + .await + .unwrap(); + assert!(out.is_empty()); + assert!(!workload.is_frozen()); + assert!(state.pending_freeze.is_some()); + } + for (attempt, mode, expected) in [ + ("other", Continue, MessageType::CoreError), + ("prefix", Restore, MessageType::CoreError), + ("prefix", Continue, MessageType::WorkloadThawed), + ("prefix", Continue, MessageType::WorkloadThawed), + ] { + let thaw = Message::with_payload( + MessageType::WorkloadThaw, + u32::MAX, + &WorkloadThaw { + attempt_id: attempt.into(), + mode, + }, + ) + .unwrap(); + let aborts_pending = + expected == MessageType::WorkloadThawed && state.pending_freeze.is_some(); + let mut out = Vec::new(); + handle_message( + thaw, + &mut state, + &mut activity, + &mut sender, + &mut out, + &config, + &mut workload, + &heartbeat, + ) + .await + .unwrap(); + let mut bytes = BytesMut::from(out.as_slice()); + if aborts_pending { + let cancelled = decode_reply_skipping_credit(&mut bytes); + let error = cancelled.payload::().unwrap(); + assert_eq!(cancelled.id, u32::MAX); + assert_eq!( + error.offending_type.as_deref(), + Some(MessageType::WorkloadFreeze.as_str()) + ); + assert_eq!(error.workload_failure.unwrap().attempt_id, "prefix"); + } + assert_eq!(decode_reply_skipping_credit(&mut bytes).t, expected); + assert_eq!( + state.pending_freeze.is_some(), + expected == MessageType::CoreError + ); + } + let charge = state.input_window.admit(InputLane::Bulk, 80).unwrap(); + for _ in 0..2 { + let mut out = Vec::new(); + handle_message( + freeze(), + &mut state, + &mut activity, + &mut sender, + &mut out, + &config, + &mut workload, + &heartbeat, + ) + .await + .unwrap(); + let reply = decode_reply_skipping_credit(&mut BytesMut::from(out.as_slice())); + let frozen = reply.payload::().unwrap(); + assert_eq!(reply.t, MessageType::WorkloadFrozen); + assert!(workload.is_frozen()); + assert!(state.output_parked); + assert_eq!( + frozen.input_credit, + initial_input_credit(), + "decode is not consumption" + ); + assert_eq!(state.input_window.position(), target); + } + drop(charge); + } + + #[tokio::test] + async fn mixed_primary_data_cut_waits_for_the_complete_dedicated_prefix() { + let mut state = AgentState::default(); + let (mut sender, _output) = SessionOutputSender::channel(); + let mut activity = ActivityTracker::new(); + let config = AgentdConfig { + user: None, + security_profile: Default::default(), + default_cwd: None, + default_env: Vec::new(), + }; + let heartbeat = heartbeat::HeartbeatControl::default(); + let mut workload = crate::workload::tests::fake_latch(); + + // Primary stdin and its empty EOF share the logical data ledger with the dedicated + // port. Retain every charge to model a consumer that has not accepted any input yet. + let mut charges = Vec::new(); + for data in [vec![0x31; 1024], Vec::new()] { + let message = + Message::with_payload(MessageType::ExecStdin, 1, &ExecStdin { data }).unwrap(); + assert!(message.t.uses_workload_data_credit()); + let mut wire = Vec::new(); + codec::encode_to_buf(&message, &mut wire).unwrap(); + charges.push( + state + .input_window + .admit(InputLane::Bulk, wire.len()) + .unwrap(), + ); + } + let primary_position = state.input_window.position(); + assert_eq!(primary_position.control_bytes, 0); + assert_eq!(primary_position.control_frames, 0); + assert_eq!(primary_position.bulk_frames, 2); + + let incarnation = [0x43; CLIENT_INCARNATION_SIZE]; + let record = BulkRecord { + id: 2, + kind: BulkKind::Filesystem, + flow: BulkFlow::HostToGuest, + offset: 0, + payload: Bytes::from(vec![0x52; 512]), + }; + let mut dedicated_wire = incarnation.to_vec(); + codec::encode_bulk_to_buf(&record, &mut dedicated_wire).unwrap(); + let target = WorkloadTransportPosition { + bulk_bytes: primary_position.bulk_bytes + dedicated_wire.len() as u64, + bulk_frames: primary_position.bulk_frames + 1, + ..primary_position + }; + let freeze = Message::with_payload( + MessageType::WorkloadFreeze, + u32::MAX, + &WorkloadFreeze { + attempt_id: "mixed-prefix".into(), + host_input: target, + }, + ) + .unwrap(); + + let mut dedicated_input = BytesMut::new(); + for prefix in [ + &dedicated_wire[..0], + &dedicated_wire[..dedicated_wire.len() - 1], + ] { + dedicated_input.extend_from_slice(prefix); + assert!( + try_decode_incarnated_bulk_from_bytes(&mut dedicated_input) + .unwrap() + .is_none() + ); + let mut out = Vec::new(); + handle_message( + freeze.clone(), + &mut state, + &mut activity, + &mut sender, + &mut out, + &config, + &mut workload, + &heartbeat, + ) + .await + .unwrap(); + assert!( + out.is_empty(), + "primary data cannot cover missing dedicated bytes" + ); + assert!(!workload.is_frozen()); + assert!(!state.output_parked); + assert!(state.pending_freeze.is_some()); + assert_eq!(state.input_window.position(), primary_position); + } + + dedicated_input.extend_from_slice(&dedicated_wire[dedicated_wire.len() - 1..]); + let decoded = try_decode_incarnated_bulk_from_bytes(&mut dedicated_input) + .unwrap() + .unwrap(); + assert_eq!(decoded.incarnation, incarnation); + assert_eq!(decoded.record, record); + assert!(dedicated_input.is_empty()); + charges.push( + state + .input_window + .admit(InputLane::Bulk, bulk_wire_bytes(&decoded.record, true)) + .unwrap(), + ); + assert_eq!(state.input_window.position(), target); + + // Resume the retained request as the outer actor does once both cumulative counters + // reach the cut. Decoding suffices; none of the primary or dedicated data is consumed. + let pending = state.pending_freeze.take().unwrap(); + let mut out = Vec::new(); + handle_message( + pending, + &mut state, + &mut activity, + &mut sender, + &mut out, + &config, + &mut workload, + &heartbeat, + ) + .await + .unwrap(); + let mut bytes = BytesMut::from(out.as_slice()); + let reply = decode_reply_skipping_credit(&mut bytes); + let frozen = reply.payload::().unwrap(); + assert_eq!(reply.t, MessageType::WorkloadFrozen); + assert_eq!(reply.id, u32::MAX); + assert_eq!(frozen.attempt_id, "mixed-prefix"); + assert_eq!(frozen.input_credit, initial_input_credit()); + assert_eq!(state.frozen_host_input, Some(target)); + assert!(workload.is_frozen()); + assert!(state.output_parked); + assert!(state.pending_freeze.is_none()); + assert!(bytes.is_empty()); + drop(charges); + } + + #[tokio::test] + async fn late_tcp_credit_preserves_raw_tail_before_and_after_terminal_retirement() { + use std::collections::hash_map::DefaultHasher; + use std::hash::Hasher; + + use microsandbox_protocol::bulk::BulkOffer; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpListener; + + tokio::time::timeout(Duration::from_secs(10), async { + let mut state = AgentState::default(); + let incarnation = [0x57; CLIENT_INCARNATION_SIZE]; + establish_relay_client( + &mut state, + RelayClientConnected { + id_start: 1, + id_end_exclusive: microsandbox_protocol::AGENT_RELAY_ID_RANGE_STEP, + incarnation, + }, + ) + .unwrap(); + // Leave the dedicated scheduler's input queued: primary completion is allowed to + // overtake these bytes, but no late credit may send DropFlow to discard their tail. + let (mut sender, mut control, mut bulk, mut scheduler_commands) = + SessionOutputSender::split_channel(); + let producer = sender.with_incarnation(Some(incarnation)); + let listener = TcpListener::bind(("127.0.0.1", 0)).await.unwrap(); + state.tcp_sessions.insert( + 1, + TcpSession::open( + 1, + TcpConnect { + host: "127.0.0.1".into(), + port: listener.local_addr().unwrap().port(), + bulk: Some(BulkOffer::tcp()), + }, + &producer, + ), + ); + let (mut peer, _) = listener.accept().await.unwrap(); + for expected in [MessageType::TcpConnected, MessageType::BulkAccepted] { + let envelope = control.recv().await.unwrap(); + let SessionOutput::Raw(mut output) = envelope.output else { + panic!() + }; + assert_eq!( + codec::try_decode_from_buf(&mut output.frame) + .unwrap() + .unwrap() + .t, + expected + ); + } + state + .tcp_sessions + .get(&1) + .unwrap() + .finish_bulk(BulkFinish { + kind: BulkKind::Tcp, + flow: BulkFlow::HostToGuest, + final_offset: 0, + }) + .await + .unwrap(); + assert_eq!(peer.read(&mut [0]).await.unwrap(), 0); + let payload = Bytes::from( + (0..6 * 1024 * 1024) + .map(|index| (index % 251) as u8) + .collect::>(), + ); + let peer_payload = payload.clone(); + let peer_task = tokio::spawn(async move { + peer.write_all(&peer_payload).await.unwrap(); + peer.shutdown().await.unwrap(); + }); + let mut received = Vec::new(); + let mut consumed = 0; + while consumed < 4 * 1024 * 1024 { + let envelope = bulk.recv().await.unwrap(); + let SessionOutput::Bulk(output) = &envelope.output else { + panic!() + }; + consumed += output.record.payload.len(); + received.push(envelope); + } + peer_task.await.unwrap(); + while !state.tcp_sessions.get(&1).unwrap().is_finished() { + tokio::task::yield_now().await; + } + assert!(consumed < payload.len()); + assert!( + !bulk.is_empty(), + "the dedicated output tail must still be queued" + ); + let credit = BulkCredit { + kind: BulkKind::Tcp, + flow: BulkFlow::GuestToHost, + consumed_offset: consumed as u64, + credit_limit: consumed as u64 + DEFAULT_BULK_WINDOW, + }; + let mut activity = ActivityTracker::new(); + let config = AgentdConfig { + user: None, + security_profile: Default::default(), + default_cwd: None, + default_env: Vec::new(), + }; + let mut workload = crate::workload::tests::fake_latch(); + let heartbeat = heartbeat::HeartbeatControl::default(); + for retired in [false, true] { + if retired { + for expected in [MessageType::BulkFinish, MessageType::TcpClosed] { + let envelope = control.try_recv().unwrap(); + let SessionOutput::Raw(mut output) = envelope.output else { + panic!() + }; + let message = codec::try_decode_from_buf(&mut output.frame) + .unwrap() + .unwrap(); + assert_eq!(message.t, expected); + if expected == MessageType::BulkFinish { + assert_eq!( + message.payload::().unwrap().final_offset, + payload.len() as u64 + ); + } else { + assert_ne!( + message.flags & microsandbox_protocol::message::FLAG_TERMINAL, + 0 + ); + assert!(matches!(output.completion, Some(RawSessionCompletion::Tcp))); + complete_raw_session( + 1, + output.completion, + &mut state.read_sessions, + &mut state.tcp_sessions, + ); + clear_bulk_receive_state(&mut state, 1); + } + } + } + assert_eq!(state.tcp_sessions.contains_key(&1), !retired); + let mut out = Vec::new(); + handle_message( + Message::with_payload(MessageType::BulkCredit, 1, &credit).unwrap(), + &mut state, + &mut activity, + &mut sender, + &mut out, + &config, + &mut workload, + &heartbeat, + ) + .await + .unwrap(); + assert!( + out.is_empty(), + "late credit must not emit cancellation or another terminal" + ); + assert!( + matches!( + scheduler_commands.try_recv(), + Err(tokio::sync::mpsc::error::TryRecvError::Empty) + ), + "late credit must not purge raw output" + ); + assert!(!bulk.is_empty()); + } + assert!(matches!( + control.try_recv(), + Err(tokio::sync::mpsc::error::TryRecvError::Empty) + )); + while let Ok(envelope) = bulk.try_recv() { + received.push(envelope); + } + let mut offset = 0; + let mut actual_hash = DefaultHasher::new(); + let mut expected_hash = DefaultHasher::new(); + expected_hash.write(&payload); + for envelope in received { + assert_eq!(envelope.id, 1); + assert_eq!(envelope.incarnation, Some(incarnation)); + let SessionOutput::Bulk(output) = envelope.output else { + panic!() + }; + let record = output.record; + assert_eq!(record.offset, offset as u64); + let end = offset + record.payload.len(); + assert_eq!(record.payload.as_ref(), &payload[offset..end]); + actual_hash.write(&record.payload); + offset = end; + } + assert_eq!(offset, payload.len()); + assert_eq!(actual_hash.finish(), expected_hash.finish()); + }) + .await + .expect("late-credit TCP tail did not complete"); + } + + #[tokio::test] + async fn dedicated_bulk_park_finishes_one_record_and_retains_the_next() { + use std::os::fd::OwnedFd; + use tokio::io::AsyncReadExt; + + let (writer, reader) = std::os::unix::net::UnixStream::pair().unwrap(); + writer.set_nonblocking(true).unwrap(); + reader.set_nonblocking(true).unwrap(); + let send_buffer: libc::c_int = 4096; + assert_eq!( + unsafe { + libc::setsockopt( + writer.as_raw_fd(), + libc::SOL_SOCKET, + libc::SO_SNDBUF, + (&send_buffer as *const libc::c_int).cast(), + std::mem::size_of_val(&send_buffer) as libc::socklen_t, + ) + }, + 0 + ); + let file = File::from(OwnedFd::from(writer)); + let mut reader = tokio::net::UnixStream::from_std(reader).unwrap(); + let (sender, _control, bulk, commands) = SessionOutputSender::split_channel(); + let (activity, _activities) = tokio::sync::mpsc::channel(8); + let writer = tokio::spawn(bulk_writer_task(file, bulk, commands, activity)); + let sender = sender.with_incarnation(Some([0x62; CLIENT_INCARNATION_SIZE])); + let first = BulkRecord { + id: 1, + kind: BulkKind::Filesystem, + flow: BulkFlow::GuestToHost, + offset: 0, + payload: Bytes::from(vec![0x6a; 1024 * 1024]), + }; + let first_len = bulk_wire_bytes(&first, true); + let second = BulkRecord { + offset: first.payload.len() as u64, + payload: Bytes::from_static(b"next"), + ..first.clone() + }; + let second_len = bulk_wire_bytes(&second, true); + for record in [first, second] { + assert!( + sender + .send( + 1, + SessionOutput::Bulk(crate::session::BulkSessionOutput::new( + record, + RawActivity::default() + )) + ) + .await + ); + } + let mut bytes = vec![0; first_len]; + time::timeout(Duration::from_secs(5), reader.read_exact(&mut bytes[..37])) + .await + .unwrap() + .unwrap(); + let park_sender = sender.clone(); + let mut park = tokio::spawn(async move { park_sender.park_bulk_output().await }); + assert!( + time::timeout(Duration::from_millis(20), &mut park) + .await + .is_err(), + "park cut a partial record" + ); + time::timeout(Duration::from_secs(5), reader.read_exact(&mut bytes[37..])) + .await + .unwrap() + .unwrap(); + assert_eq!( + time::timeout(Duration::from_secs(5), park) + .await + .unwrap() + .unwrap() + .unwrap(), + first_len as u64 + ); + let mut decoded = BytesMut::from(bytes.as_slice()); + let frame = try_decode_incarnated_bulk_from_bytes(&mut decoded) + .unwrap() + .unwrap(); + assert_eq!(frame.record.payload, Bytes::from(vec![0x6a; 1024 * 1024])); + assert!(decoded.is_empty()); + assert!( + time::timeout(Duration::from_millis(20), reader.read(&mut [0; 1])) + .await + .is_err() + ); + sender.resume_bulk_output().await.unwrap(); + let mut bytes = vec![0; second_len]; + time::timeout(Duration::from_secs(5), reader.read_exact(&mut bytes)) + .await + .unwrap() + .unwrap(); + assert_eq!( + sender.park_bulk_output().await.unwrap(), + (first_len + second_len) as u64 + ); + writer.abort(); + let _ = writer.await; + } + + #[test] + fn coalesced_bootstrap_and_init_ack_retain_the_second_frame() { + let bootstrap = GuestBootstrap::default(); + let bootstrap_message = + Message::with_payload(MessageType::Bootstrap, 0, &bootstrap).unwrap(); + let ack_message = Message::with_payload(MessageType::InitAck, 0, &InitAck {}).unwrap(); + let mut state = BootConsoleState::default(); + codec::encode_to_buf(&bootstrap_message, &mut state.input).unwrap(); + codec::encode_to_buf(&ack_message, &mut state.input).unwrap(); + + let decoded = read_boot_message( + -1, + &mut state, + Instant::now() + std::time::Duration::from_secs(1), + "guest bootstrap", + ) + .unwrap(); + assert_eq!(decode_bootstrap_message(decoded).unwrap(), bootstrap); + assert!( + !state.input.is_empty(), + "init ack frame should remain buffered" + ); + + wait_for_init_ack( + -1, + &mut state, + Instant::now() + std::time::Duration::from_secs(1), + ) + .unwrap(); + assert!(state.input.is_empty()); + } + + #[test] + fn bootstrap_rejects_non_control_correlation_fields() { + let mut message = + Message::with_payload(MessageType::Bootstrap, 1, &GuestBootstrap::default()).unwrap(); + message.flags = 1; + + let error = decode_bootstrap_message(message).unwrap_err(); + assert!(error.to_string().contains("requires id=0 and flags=0")); + } + + #[test] + fn bootstrap_rejects_wrong_first_message_type() { + let message = Message::with_payload(MessageType::Ping, 0, &Ping {}).unwrap(); + + let error = decode_bootstrap_message(message).unwrap_err(); + assert!(error.to_string().contains("expected core.bootstrap")); + } + + #[test] + fn bootstrap_rejects_older_protocol_generation() { + let mut message = + Message::with_payload(MessageType::Bootstrap, 0, &GuestBootstrap::default()).unwrap(); + let min_version = MessageType::Bootstrap.min_protocol_version(); + assert!(min_version > 0); + message.v = min_version - 1; + + let error = decode_bootstrap_message(message).unwrap_err(); + assert!(error.to_string().contains("or newer")); + } + + #[test] + fn bootstrap_accepts_minimum_supported_protocol_generation() { + let mut message = + Message::with_payload(MessageType::Bootstrap, 0, &GuestBootstrap::default()).unwrap(); + message.v = MessageType::Bootstrap.min_protocol_version(); + + assert_eq!( + decode_bootstrap_message(message).unwrap(), + GuestBootstrap::default() + ); + } + + #[test] + fn bootstrap_accepts_newer_additive_protocol_generation() { + let mut message = + Message::with_payload(MessageType::Bootstrap, 0, &GuestBootstrap::default()).unwrap(); + message.v = PROTOCOL_VERSION + 1; + + assert_eq!( + decode_bootstrap_message(message).unwrap(), + GuestBootstrap::default() + ); + } + + #[test] + fn bootstrap_rejects_malformed_payload() { let message = Message::new(MessageType::Bootstrap, 0, vec![0xff]); let error = decode_bootstrap_message(message).unwrap_err(); @@ -3699,6 +4886,208 @@ mod tests { )); } + #[test] + fn agent_read_budget_counts_bytes_and_calls_independently() { + let mut bytes = AgentReadBudget::default(); + bytes.record_read(AGENT_READ_QUANTUM_BYTES - 1); + assert!(!bytes.exhausted()); + bytes.record_read(1); + assert!(bytes.exhausted()); + + let mut calls = AgentReadBudget::default(); + for _ in 1..AGENT_READ_QUANTUM_CALLS { + assert!(calls.read_fd(-1, &mut [0]).is_err()); + assert!(!calls.exhausted()); + } + assert!(calls.read_fd(-1, &mut [0]).is_err()); + assert!(calls.exhausted(), "failed reads also bound a retry loop"); + assert_eq!(calls.bytes, 0); + + let mut large = AgentReadBudget::default(); + large.record_read(BULK_SERIAL_READ_BUF_SIZE); + assert!( + large.exhausted(), + "a large read is not a smaller wire record" + ); + assert_eq!(large.bytes, BULK_SERIAL_READ_BUF_SIZE); + } + + #[tokio::test(flavor = "current_thread")] + async fn agent_read_budget_services_driver_during_partial_bulk_records() { + // The control demonstrates the failure without relying on wall-clock delays: cached + // readable bulk input never lets the runtime observe the newly readable/writable peers. + assert!(!observe_driver_during_partial_bulk(false).await); + assert!(observe_driver_during_partial_bulk(true).await); + } + + async fn observe_driver_during_partial_bulk(yield_at_boundary: bool) -> bool { + use std::io::Write; + use std::os::fd::OwnedFd; + use std::os::unix::net::UnixStream; + use std::sync::atomic::AtomicUsize; + + let incarnation = [0x42; CLIENT_INCARNATION_SIZE]; + let first = BulkRecord { + id: 1, + kind: BulkKind::Filesystem, + flow: BulkFlow::HostToGuest, + offset: 0, + payload: Bytes::from(vec![0xa5; 512]), + }; + let second = BulkRecord { + offset: first.payload.len() as u64, + payload: Bytes::from(vec![0x5a; 512]), + ..first.clone() + }; + let mut wire = Vec::new(); + for record in [&first, &second] { + wire.extend_from_slice(&incarnation); + codec::encode_bulk_to_buf(record, &mut wire).unwrap(); + } + let (mut source, input) = UnixStream::pair().unwrap(); + input.set_nonblocking(true).unwrap(); + source.write_all(&wire).unwrap(); + let mut input = BulkInputState::new(File::from(OwnedFd::from(input))).unwrap(); + // Model fragmented console reads while keeping the complete wire records unchanged. + input.read_buf.truncate(1); + + let (mut control_source, control) = UnixStream::pair().unwrap(); + control.set_nonblocking(true).unwrap(); + let control = AsyncFd::new(control).unwrap(); + let (pipe_reader, pipe_writer) = nix::unistd::pipe2(nix::fcntl::OFlag::O_NONBLOCK).unwrap(); + let pipe_writer = AsyncFd::new(pipe_writer).unwrap(); + // Prime the reactor, then clear the pipe's cached writable event with a real EAGAIN. + let mut writable = pipe_writer.writable().await.unwrap(); + loop { + match writable.try_io(|fd| write_to_fd(fd.get_ref().as_raw_fd(), &[0; 4096])) { + Ok(Ok(count)) => assert!(count > 0), + Ok(Err(error)) => panic!("fill stdin pipe: {error}"), + Err(_) => break, + } + } + drop(writable); + let mut drained = [0; 4096]; + loop { + match read_from_fd(pipe_reader.as_raw_fd(), &mut drained) { + Ok(count) => assert!(count > 0), + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => break, + Err(error) => panic!("drain stdin pipe: {error}"), + } + } + control_source.write_all(b"c").unwrap(); + let serviced = Arc::new(AtomicUsize::new(0)); + let control_serviced = Arc::clone(&serviced); + let control_task = tokio::spawn(async move { + loop { + let mut ready = control.readable().await.unwrap(); + let mut byte = [0]; + match ready.try_io(|fd| read_from_fd(fd.get_ref().as_raw_fd(), &mut byte)) { + Ok(Ok(1)) => { + assert_eq!(byte, *b"c"); + control_serviced.fetch_or(1, Ordering::Relaxed); + break; + } + Ok(result) => panic!("read control marker: {result:?}"), + Err(_) => continue, + } + } + }); + let stdin_serviced = Arc::clone(&serviced); + let stdin_task = tokio::spawn(async move { + std::future::poll_fn(|cx| { + loop { + let mut ready = std::task::ready!(pipe_writer.poll_write_ready(cx)).unwrap(); + match ready.try_io(|fd| write_to_fd(fd.get_ref().as_raw_fd(), b"i")) { + Ok(result) => return Poll::Ready(result), + Err(_) => continue, + } + } + }) + .await + .unwrap(); + stdin_serviced.fetch_or(2, Ordering::Relaxed); + }); + + let mut budget = AgentReadBudget::default(); + let mut received = Vec::new(); + let mut serviced_before_first_record = false; + while received.len() < 2 { + let frames = tokio::select! { + frames = input.read_turn(&mut budget) => frames.unwrap(), + _ = std::future::pending::<()>() => unreachable!(), + }; + for frame in frames { + assert_eq!(frame.incarnation, incarnation); + received.push(frame.record); + } + // This is the production cancellation-safe boundary: no decoded frames are local to + // a competing select future, and even partial-record reads have spent the budget. + if yield_at_boundary { + budget.yield_if_exhausted().await; + } + if received.is_empty() && serviced.load(Ordering::Relaxed) == 3 { + serviced_before_first_record = true; + } + } + assert_eq!( + received, + [first, second], + "yield preserves record bytes and FIFO" + ); + assert!(input.input.is_empty()); + control_task.await.unwrap(); + stdin_task.await.unwrap(); + let mut marker = [0]; + assert_eq!( + read_from_fd(pipe_reader.as_raw_fd(), &mut marker).unwrap(), + 1 + ); + assert_eq!(marker, *b"i"); + serviced_before_first_record + } + + #[tokio::test(flavor = "current_thread")] + async fn cancelled_bulk_read_keeps_partial_record_and_read_budget() { + use std::io::Write; + use std::os::fd::OwnedFd; + use std::os::unix::net::UnixStream; + + let incarnation = [0x29; CLIENT_INCARNATION_SIZE]; + let record = BulkRecord { + id: 1, + kind: BulkKind::Filesystem, + flow: BulkFlow::HostToGuest, + offset: 0, + payload: Bytes::from(vec![0x51; 512]), + }; + let mut wire = incarnation.to_vec(); + codec::encode_bulk_to_buf(&record, &mut wire).unwrap(); + let (mut source, input) = UnixStream::pair().unwrap(); + input.set_nonblocking(true).unwrap(); + let mut input = BulkInputState::new(File::from(OwnedFd::from(input))).unwrap(); + let mut budget = AgentReadBudget::default(); + source.write_all(&wire[..40]).unwrap(); + assert!(input.read_turn(&mut budget).await.unwrap().is_empty()); + // Consume the stale readable hint so the following select truly cancels a pending read. + assert!(input.read_turn(&mut budget).await.unwrap().is_empty()); + let calls = budget.calls; + tokio::select! { + biased; + result = input.read_turn(&mut budget) => panic!("unexpected read: {result:?}"), + _ = std::future::ready(()) => {}, + } + assert_eq!(input.input.as_ref(), &wire[..40]); + assert_eq!(budget.bytes, 40); + assert_eq!(budget.calls, calls); + source.write_all(&wire[40..]).unwrap(); + let frames = input.read_turn(&mut budget).await.unwrap(); + assert_eq!(frames.len(), 1); + assert_eq!(frames[0].record, record); + assert_eq!(frames[0].incarnation, incarnation); + assert!(input.input.is_empty()); + assert_eq!(budget.bytes, wire.len()); + } + #[test] fn disconnect_cleanup_keeps_other_clients_bulk_offsets() { let mut state = AgentState::default(); @@ -3890,6 +5279,7 @@ mod tests { &mut active, &mut retired, &mut retiring_incarnations, + &mut BulkOutputPosition::default(), ) .unwrap(); assert!(matches!( @@ -4126,11 +5516,7 @@ mod tests { .await .unwrap(); let mut bytes = BytesMut::from(encoded.as_slice()); - let Some(DecodedFrame::Control(reply)) = - codec::try_decode_frame_from_bytes(&mut bytes).unwrap() - else { - panic!("missing thaw response"); - }; + let reply = decode_reply_skipping_credit(&mut bytes); assert_eq!(reply.t, expected); assert_eq!(sender.generation(), 0); assert_eq!(client_incarnation_for_id(&state, 1), Some(owner)); @@ -4143,6 +5529,7 @@ mod tests { 0, &WorkloadFreeze { attempt_id: "capture".into(), + host_input: WorkloadTransportPosition::default(), }, ) .unwrap(); @@ -4182,11 +5569,7 @@ mod tests { .await .unwrap(); let mut bytes = BytesMut::from(encoded.as_slice()); - let Some(DecodedFrame::Control(reply)) = - codec::try_decode_frame_from_bytes(&mut bytes).unwrap() - else { - panic!("missing restored thaw response"); - }; + let reply = decode_reply_skipping_credit(&mut bytes); assert_eq!(reply.t, MessageType::WorkloadThawed); assert!(!workload.is_frozen()); assert_eq!(sender.generation(), generation); @@ -4199,6 +5582,231 @@ mod tests { } } + #[tokio::test] + async fn saturated_stdin_allows_thaw_and_preserves_input_and_eof() { + use microsandbox_protocol::core::WorkloadThawMode::{Continue, Restore}; + + // SIGSTOP provides a deterministic blocked pipe/PTY without a privileged cgroup mount. + // Thaw must complete before external consumer progress, then accepted data drains exactly + // once. Restore closes inherited pipe input after data even when no EOF was accepted. + struct StoppedProcessGuard(i32); + impl Drop for StoppedProcessGuard { + fn drop(&mut self) { + unsafe { + libc::kill(-self.0, libc::SIGCONT); + libc::kill(-self.0, libc::SIGKILL); + } + } + } + + for (tty, mode, accepted_eof) in [ + (false, Continue, false), + (false, Continue, true), + (false, Restore, false), + (false, Restore, true), + (true, Continue, true), + (true, Restore, true), + ] { + let mut state = AgentState::default(); + let (mut sender, mut output) = SessionOutputSender::channel(); + let mut activity = ActivityTracker::new(); + let heartbeat = heartbeat::HeartbeatControl::default(); + let config = AgentdConfig { + user: None, + security_profile: Default::default(), + default_cwd: None, + default_env: Vec::new(), + }; + let mut workload = crate::workload::tests::fake_latch(); + workload.freeze("stdin-cut").unwrap(); + let request = ExecRequest { + cmd: "/bin/sh".into(), + args: vec![ + "-c".into(), + if tty { + "stty raw -echo; kill -STOP $$; exec cat" + } else { + "kill -STOP $$; exec cat" + } + .into(), + ], + env: vec![], + cwd: None, + user: None, + tty, + rows: 24, + cols: 80, + rlimits: vec![], + }; + let session = ExecSession::spawn( + 1, + &request, + sender.clone(), + None, + crate::config::SecurityProfile::Default, + None, + ) + .unwrap(); + let pid = session.pid() as i32; + state.sessions.insert(1, session); + let _cleanup = StoppedProcessGuard(pid); + time::timeout(Duration::from_secs(5), async { + loop { + let status = std::fs::read_to_string(format!("/proc/{pid}/status")).unwrap(); + if status.lines().any(|line| line.starts_with("State:\tT")) { + break; + } + time::sleep(Duration::from_millis(1)).await; + } + }) + .await + .expect("workload should stop before stdin is admitted"); + + let mut encoded = Vec::new(); + let ledger = state.input_window.clone(); + let initial = ledger.credit().unwrap(); + let data = vec![0x61; 1024 * 1024]; + let charge = ledger.admit(InputLane::Bulk, data.len() + 32).unwrap(); + let mut stdin = + Message::with_payload(MessageType::ExecStdin, 1, &ExecStdin { data }).unwrap(); + // A generation-8 SDK still travels over the bundled private transport contract. + // Dispatch must not fall back to blocking writes based on the client message version. + stdin.v = 8; + time::timeout( + Duration::from_millis(100), + handle_message_with_charge( + stdin, + &mut state, + &mut activity, + &mut sender, + &mut encoded, + &config, + &mut workload, + &heartbeat, + Some(charge), + ), + ) + .await + .expect("stdin dispatch waited on its blocked consumer") + .unwrap(); + assert!(encoded.is_empty()); + assert!(state.sessions[&1].has_pending_stdin()); + if accepted_eof { + let charge = ledger.admit(InputLane::Bulk, 32).unwrap(); + let mut eof = Message::with_payload( + MessageType::ExecStdin, + 1, + &ExecStdin { data: Vec::new() }, + ) + .unwrap(); + eof.v = 8; + time::timeout( + Duration::from_millis(100), + handle_message_with_charge( + eof, + &mut state, + &mut activity, + &mut sender, + &mut encoded, + &config, + &mut workload, + &heartbeat, + Some(charge), + ), + ) + .await + .expect("EOF dispatch waited on preceding blocked data") + .unwrap(); + } + assert_eq!( + ledger.credit().unwrap(), + initial, + "blocked input refunded before consumption" + ); + let position = ledger.position(); + let thaw = Message::with_payload( + MessageType::WorkloadThaw, + u32::MAX, + &WorkloadThaw { + attempt_id: "stdin-cut".into(), + mode, + }, + ) + .unwrap(); + time::timeout( + Duration::from_millis(100), + handle_message( + thaw, + &mut state, + &mut activity, + &mut sender, + &mut encoded, + &config, + &mut workload, + &heartbeat, + ), + ) + .await + .expect("thaw waited on saturated stdin") + .unwrap(); + assert!(!workload.is_frozen()); + let mut bytes = BytesMut::from(encoded.as_slice()); + let reply = decode_reply_skipping_credit(&mut bytes); + assert_eq!(reply.t, MessageType::WorkloadThawed); + assert_eq!( + ledger.position(), + position, + "restore reset cumulative input position" + ); + assert_eq!( + ledger.credit().unwrap(), + initial, + "restore discarded accepted input" + ); + assert_eq!(unsafe { libc::kill(-pid, libc::SIGCONT) }, 0); + let mut received = Vec::new(); + let mut exited = false; + time::timeout(Duration::from_secs(5), async { + while received.len() < 1024 * 1024 + || state.sessions.values().chain(state.detached_sessions.values()).any(ExecSession::has_pending_stdin) + || (!tty && (accepted_eof || mode == Restore) && !exited) { + tokio::select! { + (_, _, result) = std::future::poll_fn(|cx| poll_pending_stdin(&mut state, cx)) => result.unwrap(), + envelope = output.recv() => match envelope.unwrap().output { + SessionOutput::Stdout(data) => received.extend(data), + SessionOutput::Exited(code) => { assert_eq!(code, 0); exited = true; }, + _ => {}, + }, + } + } + }) + .await + .expect("accepted stdin or ordered EOF did not drain"); + assert_eq!(received, vec![0x61; 1024 * 1024]); + assert_eq!( + ledger.credit().unwrap().bulk_bytes, + initial.bulk_bytes + position.bulk_bytes + ); + if !tty && mode == Continue && !accepted_eof { + // Source Continue must not synthesize EOF. The same owner can still send input. + assert!(!exited); + state + .sessions + .get_mut(&1) + .unwrap() + .enqueue_stdin(b"tail".to_vec(), None) + .unwrap(); + let envelope = time::timeout(Duration::from_secs(5), output.recv()) + .await + .unwrap() + .unwrap(); + assert!( + matches!(envelope.output, SessionOutput::Stdout(ref data) if data == b"tail") + ); + } + } + } + #[tokio::test] async fn restore_detaches_piped_and_pty_workloads_without_killing_or_reusing_output() { for tty in [false, true] { @@ -4478,6 +6086,7 @@ mod tests { &mut active, &mut retired, &mut retiring, + &mut BulkOutputPosition::default(), ) .unwrap(); enqueue_bulk_output( diff --git a/crates/agentd/lib/serial.rs b/crates/agentd/lib/serial.rs index 4e1325f81..5b258c27c 100644 --- a/crates/agentd/lib/serial.rs +++ b/crates/agentd/lib/serial.rs @@ -1,7 +1,11 @@ -//! Virtio serial port discovery. +//! Virtio serial port discovery and bounded workload input admission. +use std::sync::{Arc, Mutex}; use std::{fs, path::PathBuf}; +use microsandbox_protocol::core::{WorkloadTransportCredit, WorkloadTransportPosition}; +use tokio::sync::Notify; + use crate::error::{AgentdError, AgentdResult}; //-------------------------------------------------------------------------------------------------- @@ -14,6 +18,167 @@ const VIRTIO_PORTS_PATH: &str = "/sys/class/virtio-ports"; /// Re-export the canonical control and bulk port names from the protocol crate. pub use microsandbox_protocol::{AGENT_BULK_PORT_NAME, AGENT_PORT_NAME}; +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// Logical admission class, independent of the physical port. Raw records, stdin and inline +/// FS/TCP payloads use Bulk; command metadata and leases retain separate Control capacity. +#[derive(Clone, Copy, Debug)] +pub(crate) enum InputLane { + Control, + Bulk, +} + +/// Cumulative flow-control state stays in captured guest RAM across restore. A new host starts +/// from the descriptor's position; retained input refunds the same ledger as it is consumed. +#[derive(Clone, Debug)] +pub(crate) struct InputWindow(Arc); + +#[derive(Debug)] +struct InputState { + ledger: Mutex, + refunded: Notify, +} + +#[derive(Debug)] +struct InputLedger { + position: WorkloadTransportPosition, + credit: WorkloadTransportCredit, + exhausted: bool, +} + +/// One admitted allocation, retained until its bytes are consumed or deliberately discarded. +#[derive(Debug)] +pub(crate) struct InputCharge { + window: InputWindow, + lane: InputLane, + bytes: u64, +} + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl InputWindow { + pub(crate) fn new(credit: WorkloadTransportCredit) -> Self { + Self(Arc::new(InputState { + ledger: Mutex::new(InputLedger { + position: WorkloadTransportPosition::default(), + credit, + exhausted: false, + }), + refunded: Notify::new(), + })) + } + + pub(crate) fn position(&self) -> WorkloadTransportPosition { + self.0 + .ledger + .lock() + .expect("input ledger poisoned") + .position + } + + pub(crate) fn credit(&self) -> AgentdResult { + let ledger = self.0.ledger.lock().expect("input ledger poisoned"); + if ledger.exhausted { + return Err(AgentdError::ExecSession( + "transport input counter exhausted".into(), + )); + } + Ok(ledger.credit) + } + + pub(crate) fn admit(&self, lane: InputLane, bytes: usize) -> AgentdResult { + let bytes = u64::try_from(bytes) + .map_err(|_| AgentdError::ExecSession("transport frame length overflow".into()))?; + let mut ledger = self.0.ledger.lock().expect("input ledger poisoned"); + let (old_bytes, old_frames, byte_limit, frame_limit) = match lane { + InputLane::Control => ( + ledger.position.control_bytes, + ledger.position.control_frames, + ledger.credit.control_bytes, + ledger.credit.control_frames, + ), + InputLane::Bulk => ( + ledger.position.bulk_bytes, + ledger.position.bulk_frames, + ledger.credit.bulk_bytes, + ledger.credit.bulk_frames, + ), + }; + let next_bytes = old_bytes.checked_add(bytes); + let next_frames = old_frames.checked_add(1); + let (Some(next_bytes), Some(next_frames)) = (next_bytes, next_frames) else { + return Err(AgentdError::ExecSession( + "transport input counter exhausted".into(), + )); + }; + if ledger.exhausted || next_bytes > byte_limit || next_frames > frame_limit { + return Err(AgentdError::ExecSession( + "host exceeded its transport admission credit".into(), + )); + } + match lane { + InputLane::Control => { + ledger.position.control_bytes = next_bytes; + ledger.position.control_frames = next_frames; + } + InputLane::Bulk => { + ledger.position.bulk_bytes = next_bytes; + ledger.position.bulk_frames = next_frames; + } + } + drop(ledger); + Ok(InputCharge { + window: self.clone(), + lane, + bytes, + }) + } + + pub(crate) async fn refunded(&self) { + self.0.refunded.notified().await; + } +} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +impl Drop for InputCharge { + fn drop(&mut self) { + // This short ledger update never waits on guest I/O. Keeping the token with the buffer + // also refunds cancellation, errors, and EOF without a separate per-payload message. + let mut ledger = self.window.0.ledger.lock().expect("input ledger poisoned"); + let (bytes, frames) = match self.lane { + InputLane::Control => (ledger.credit.control_bytes, ledger.credit.control_frames), + InputLane::Bulk => (ledger.credit.bulk_bytes, ledger.credit.bulk_frames), + }; + let (Some(bytes), Some(frames)) = (bytes.checked_add(self.bytes), frames.checked_add(1)) + else { + ledger.exhausted = true; + self.window.0.refunded.notify_one(); + return; + }; + match self.lane { + InputLane::Control => { + ledger.credit.control_bytes = bytes; + ledger.credit.control_frames = frames; + } + InputLane::Bulk => { + ledger.credit.bulk_bytes = bytes; + ledger.credit.bulk_frames = frames; + } + } + drop(ledger); + // Notify coalesces repeated refunds while the actor batches a credit update. An idle + // guest has no periodic credit timer, and even one small refund wakes a blocked host. + self.window.0.refunded.notify_one(); + } +} + //-------------------------------------------------------------------------------------------------- // Functions //-------------------------------------------------------------------------------------------------- @@ -45,3 +210,117 @@ pub fn find_serial_port(name: &str) -> AgentdResult { "no virtio port with name '{name}' found" ))) } + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + fn window() -> InputWindow { + InputWindow::new(WorkloadTransportCredit { + control_bytes: 8, + control_frames: 2, + bulk_bytes: 16, + bulk_frames: 2, + }) + } + + #[test] + fn input_capacity_follows_consumption_not_decode_and_bounds_empty_messages() { + let window = window(); + let first = window.admit(InputLane::Control, 4).unwrap(); + let second = window.admit(InputLane::Control, 4).unwrap(); + let at_cut = window.position(); + assert_eq!(at_cut.control_bytes, 8); + assert_eq!(window.credit().unwrap().control_bytes, 8); + assert!(window.admit(InputLane::Control, 1).is_err()); + assert_eq!( + window.position(), + at_cut, + "rejection must not advance position" + ); + drop(first); + let next = window.admit(InputLane::Control, 1).unwrap(); + assert!( + window.admit(InputLane::Control, 1).is_err(), + "frame capacity is independent of bytes" + ); + drop((second, next)); + assert_eq!(window.credit().unwrap().control_bytes, 17); + assert_eq!(window.credit().unwrap().control_frames, 5); + } + + #[test] + fn inherited_input_refunds_the_same_cumulative_ledger_after_restore() { + let window = window(); + let inherited = window.admit(InputLane::Bulk, 15).unwrap(); + let source_position = window.position(); + let restored_view = window.clone(); + // Moving a retained session to the detached table must not grant a fresh window. + assert_eq!(restored_view.position(), source_position); + assert!(restored_view.admit(InputLane::Bulk, 2).is_err()); + // Captured input keeps its debt, but cannot prevent the fresh host from starting an exec. + let command = restored_view.admit(InputLane::Control, 8).unwrap(); + drop(command); + drop(inherited); + let fresh = restored_view.admit(InputLane::Bulk, 16).unwrap(); + assert_eq!(restored_view.position().bulk_bytes, 31); + drop(fresh); + } + + #[test] + fn logical_classes_are_independent_and_counter_overflow_fails_closed() { + let window = window(); + let control = window.admit(InputLane::Control, 8).unwrap(); + let bulk = window.admit(InputLane::Bulk, 16).unwrap(); + assert!(window.admit(InputLane::Bulk, 1).is_err()); + drop(control); + assert_eq!(window.credit().unwrap().bulk_bytes, 16); + drop(bulk); + { + let mut ledger = window.0.ledger.lock().unwrap(); + ledger.position.control_bytes = u64::MAX; + ledger.credit.control_bytes = u64::MAX; + } + assert!(window.admit(InputLane::Control, 1).is_err()); + } + + #[test] + fn dropping_a_cancelled_or_failed_input_refunds_exactly_one_frame() { + let window = window(); + let before = window.credit().unwrap(); + let cancelled = window.admit(InputLane::Control, 3).unwrap(); + drop(cancelled); + let after = window.credit().unwrap(); + assert_eq!(after.control_bytes, before.control_bytes + 3); + assert_eq!(after.control_frames, before.control_frames + 1); + assert_eq!(after.bulk_bytes, before.bulk_bytes); + assert_eq!(after.bulk_frames, before.bulk_frames); + assert_eq!(window.position().control_frames, 1); + } + + #[tokio::test] + async fn refund_notifications_are_idle_until_small_progress_and_coalesce() { + let window = window(); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(10), window.refunded()) + .await + .is_err() + ); + let first = window.admit(InputLane::Control, 1).unwrap(); + let second = window.admit(InputLane::Control, 1).unwrap(); + drop((first, second)); + tokio::time::timeout(std::time::Duration::from_millis(100), window.refunded()) + .await + .unwrap(); + assert_eq!(window.credit().unwrap().control_bytes, 10); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(10), window.refunded()) + .await + .is_err() + ); + } +} diff --git a/crates/agentd/lib/session.rs b/crates/agentd/lib/session.rs index e5ac0330c..7fad25182 100644 --- a/crates/agentd/lib/session.rs +++ b/crates/agentd/lib/session.rs @@ -1,16 +1,19 @@ //! Exec session management: spawning processes with PTY or pipe I/O. +use std::collections::VecDeque; use std::ffi::{CStr, CString}; use std::mem::MaybeUninit; use std::os::fd::{AsRawFd, FromRawFd, OwnedFd, RawFd}; use std::os::unix::process::CommandExt; use std::process::{Command, Stdio}; use std::sync::Arc; +use std::task::{Context, Poll}; use std::{iter, mem, ptr}; use nix::pty; use nix::sys::signal::Signal; use tokio::io::AsyncReadExt; +use tokio::io::unix::AsyncFd; use tokio::sync::{Semaphore, mpsc, oneshot}; use microsandbox_protocol::bulk::BulkRecord; @@ -21,6 +24,7 @@ use crate::config::SecurityProfile; use crate::error::{AgentdError, AgentdResult}; use crate::process::{ProcessExitWatcher, ProcessIdentity, ProcessManager}; use crate::rlimit; +use crate::serial::InputCharge; use crate::workload::WorkloadPlacement; //-------------------------------------------------------------------------------------------------- @@ -145,10 +149,20 @@ pub struct ExecSession { process_manager: Arc, /// The PTY master fd (only for PTY mode, used for writing and resize). - pty_master: Option, + pty_master: Option>, /// The child's stdin (only for pipe mode). - stdin: Option, + stdin: Option>, + + /// Accepted input stays ordered, including EOF, while a pipe or PTY backpressures. + pending_stdin: VecDeque, +} + +#[derive(Debug)] +struct PendingStdin { + data: Vec, + written: usize, + _charge: Option, } /// Output from a session that the agent loop should forward to the host. @@ -188,6 +202,16 @@ pub struct SessionOutputEnvelope { /// Lifecycle commands processed ahead of queued dedicated-lane output. pub enum BulkOutputCommand { + /// Park after the currently written complete record, retaining all queued source output. + Park { + /// Cumulative complete dedicated-lane wire bytes at the cut. + completion: oneshot::Sender, + }, + /// Release the parked source or restored output generation after its thaw reply. + Resume { + /// Resolves once ordinary output is eligible again. + completion: oneshot::Sender<()>, + }, /// Discard inherited transfer output before acknowledging restore activation. Restore { /// New attachment generation; late output from previous generations is discarded. @@ -432,6 +456,30 @@ impl SessionOutputSender { self.generation } + pub(crate) async fn park_bulk_output(&self) -> Result { + let Some(commands) = &self.bulk_command_tx else { + return Ok(0); + }; + let (completion, completed) = oneshot::channel(); + commands + .send(BulkOutputCommand::Park { completion }) + .await + .map_err(|_| "bulk scheduler closed while parking")?; + completed.await.map_err(|_| "bulk output park failed") + } + + pub(crate) async fn resume_bulk_output(&self) -> Result<(), &'static str> { + let Some(commands) = &self.bulk_command_tx else { + return Ok(()); + }; + let (completion, completed) = oneshot::channel(); + commands + .send(BulkOutputCommand::Resume { completion }) + .await + .map_err(|_| "bulk scheduler closed while resuming")?; + completed.await.map_err(|_| "bulk output resume failed") + } + /// Change only the root sender. Existing producers keep their old generation while they /// drain inherited pipes, so their output can never complete a new client's correlation. pub(crate) async fn restore_generation(&mut self) -> Result<(), &'static str> { @@ -650,12 +698,141 @@ impl ExecSession { /// Writes data to the process's stdin (or PTY master). pub async fn write_stdin(&self, data: &[u8]) -> AgentdResult<()> { - if let Some(ref master) = self.pty_master { - blocking_write_fd(master.as_raw_fd(), data).await - } else if let Some(ref stdin) = self.stdin { - blocking_write_fd(stdin.as_raw_fd(), data).await + let mut written = 0; + while written < data.len() { + let count = + std::future::poll_fn(|cx| self.poll_write_stdin(cx, &data[written..])).await?; + if count == 0 { + return Err(std::io::Error::from(std::io::ErrorKind::WriteZero).into()); + } + written += count; + } + Ok(()) + } + + /// Try the common writable-stdin path without a copy, task hop, or readiness registration. + pub(crate) fn try_write_stdin(&self, data: &[u8]) -> std::io::Result { + match self.pty_master.as_ref().or(self.stdin.as_ref()) { + Some(input) => write_nonblocking_fd(input.as_raw_fd(), data), + None => Ok(data.len()), + } + } + + /// Poll a previously blocked input without preventing the agent from reading lifecycle frames. + pub(crate) fn poll_write_stdin( + &self, + cx: &mut Context<'_>, + data: &[u8], + ) -> Poll> { + let Some(input) = self.pty_master.as_ref().or(self.stdin.as_ref()) else { + return Poll::Ready(Ok(data.len())); + }; + loop { + let mut ready = std::task::ready!(input.poll_write_ready(cx))?; + match ready.try_io(|inner| write_nonblocking_fd(inner.as_raw_fd(), data)) { + Ok(result) => return Poll::Ready(result), + Err(_would_block) => continue, + } + } + } + + /// Retain only an already-admitted frame. The caller's aggregate wire credit bounds both + /// this allocation and zero-length EOF cardinality across every active and detached session. + pub(crate) fn enqueue_stdin( + &mut self, + data: Vec, + charge: Option, + ) -> std::io::Result<()> { + let mut written = 0; + if self.pending_stdin.is_empty() { + if data.is_empty() { + self.close_stdin(); + return Ok(()); + } + match self.try_write_stdin(&data) { + Ok(count) if count == data.len() => return Ok(()), + Ok(0) => return Err(std::io::ErrorKind::WriteZero.into()), + Ok(count) => written = count, + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {} + Err(error) => return Err(error), + } + } + self.pending_stdin.push_back(PendingStdin { + data, + written, + _charge: charge, + }); + Ok(()) + } + + pub(crate) fn has_pending_stdin(&self) -> bool { + !self.pending_stdin.is_empty() + } + + /// The restored process no longer has a host-side stdin owner. Drain everything accepted + /// before the cut, then close pipe input. A PTY has no separate write half to close safely. + pub(crate) fn detach_stdin(&mut self) { + if self.stdin.is_none() { + return; + } + if self.pending_stdin.is_empty() { + self.close_stdin(); + } else if !self + .pending_stdin + .iter() + .any(|pending| pending.data.is_empty()) + { + // At most one marker per already-admitted nonempty queue: detachment cannot create + // an unbounded stream of uncharged empty messages. + self.pending_stdin.push_back(PendingStdin { + data: Vec::new(), + written: 0, + _charge: None, + }); + } + } + + /// Consume one bounded turn without losing a partial-write cursor when a lifecycle event + /// cancels this poll. Restored detached sessions use the same path for accepted input only. + pub(crate) fn poll_pending_stdin(&mut self, cx: &mut Context<'_>) -> Poll> { + let mut progressed = false; + for _ in 0..16 { + let Some(pending) = self.pending_stdin.front() else { + break; + }; + if pending.data.is_empty() { + self.close_stdin(); + self.pending_stdin.pop_front(); + progressed = true; + continue; + } + match self.poll_write_stdin(cx, &pending.data[pending.written..]) { + Poll::Ready(Ok(0)) => { + self.pending_stdin.pop_front(); + return Poll::Ready(Err(std::io::ErrorKind::WriteZero.into())); + } + Poll::Ready(Ok(count)) => { + let pending = self + .pending_stdin + .front_mut() + .expect("polled pending input"); + pending.written += count; + if pending.written == pending.data.len() { + self.pending_stdin.pop_front(); + } + progressed = true; + } + Poll::Ready(Err(error)) => { + self.pending_stdin.pop_front(); + return Poll::Ready(Err(error)); + } + Poll::Pending => break, + } + } + if progressed { + Poll::Ready(Ok(())) } else { - Ok(()) + Poll::Pending } } @@ -909,6 +1086,10 @@ impl ExecSession { return Err(std::io::Error::last_os_error().into()); } let reader_fd = unsafe { OwnedFd::from_raw_fd(reader_fd) }; + let pty_master = nonblocking_input(pty.master).inspect_err(|_| { + let _ = process_manager.signal_process_group(process_identity, Signal::SIGKILL as i32); + process_manager.release(process_identity); + })?; // Spawn background reader task. tokio::spawn(pty_reader_task(id, reader_fd, exit_watcher, tx)); @@ -916,8 +1097,9 @@ impl ExecSession { Ok(Self { process_identity, process_manager: Arc::clone(process_manager), - pty_master: Some(pty.master), + pty_master: Some(pty_master), stdin: None, + pending_stdin: VecDeque::new(), }) } @@ -988,6 +1170,14 @@ impl ExecSession { exit_watcher, } = spawn_piped_process(cmd, process_manager)?; let process_identity = exit_watcher.identity(); + let stdin = stdin + .map(|input| input.into_owned_fd().and_then(nonblocking_input)) + .transpose() + .inspect_err(|_| { + let _ = + process_manager.signal_process_group(process_identity, Signal::SIGKILL as i32); + process_manager.release(process_identity); + })?; // Spawn background reader task. tokio::spawn(pipe_reader_task(id, stdout, stderr, exit_watcher, tx)); @@ -997,6 +1187,7 @@ impl ExecSession { process_manager: Arc::clone(process_manager), pty_master: None, stdin, + pending_stdin: VecDeque::new(), }) } } @@ -1449,42 +1640,35 @@ fn agentd_to_io_error(err: AgentdError) -> std::io::Error { std::io::Error::other(err.to_string()) } -/// Writes data to a raw fd using a blocking task, handling short writes. -async fn blocking_write_fd(fd: RawFd, data: &[u8]) -> AgentdResult<()> { - let data = data.to_vec(); - tokio::task::spawn_blocking(move || { - let mut written = 0; - while written < data.len() { - let ptr = unsafe { data.as_ptr().add(written) as *const libc::c_void }; - let ret = unsafe { libc::write(fd, ptr, data.len() - written) }; - if ret < 0 { - let err = std::io::Error::last_os_error(); - let code = err.raw_os_error(); - if code == Some(libc::EAGAIN) || code == Some(libc::EWOULDBLOCK) { - wait_fd_writable(fd)?; - continue; - } - if code == Some(libc::EINTR) { - continue; - } - return Err(AgentdError::Io(err)); - } - if ret == 0 { - wait_fd_writable(fd)?; - continue; - } - written += ret as usize; +/// Keep one owned descriptor across readiness waits; cancellation cannot leave a blocking task +/// writing through a borrowed fd after its session has been removed or restored. +fn nonblocking_input(fd: OwnedFd) -> std::io::Result> { + let flags = unsafe { libc::fcntl(fd.as_raw_fd(), libc::F_GETFL) }; + if flags < 0 + || unsafe { libc::fcntl(fd.as_raw_fd(), libc::F_SETFL, flags | libc::O_NONBLOCK) } < 0 + { + return Err(std::io::Error::last_os_error()); + } + AsyncFd::new(fd) +} + +fn write_nonblocking_fd(fd: RawFd, data: &[u8]) -> std::io::Result { + loop { + let written = unsafe { libc::write(fd, data.as_ptr().cast(), data.len()) }; + if written >= 0 { + return Ok(written as usize); } - Ok(()) - }) - .await - .map_err(|e| AgentdError::ExecSession(format!("stdin write join error: {e}")))? + let error = std::io::Error::last_os_error(); + if error.kind() != std::io::ErrorKind::Interrupted { + return Err(error); + } + } } -fn wait_fd_writable(fd: RawFd) -> AgentdResult<()> { +fn wait_fd_readable(fd: RawFd) -> AgentdResult<()> { let mut pollfd = libc::pollfd { fd, - events: libc::POLLOUT, + events: libc::POLLIN, revents: 0, }; @@ -1500,10 +1684,7 @@ fn wait_fd_writable(fd: RawFd) -> AgentdResult<()> { if ret == 0 { continue; } - // Any positive return means the fd is actionable: POLLOUT lets the - // next write make progress, and POLLHUP/POLLERR/POLLNVAL will cause - // the next write to fail with a real errno (typically EPIPE) which - // is more meaningful than poll's revents. + // Always retry read on HUP/ERR too: a PTY may still contain final output before EIO. return Ok(()); } } @@ -1522,10 +1703,8 @@ async fn pty_reader_task( // edge-driven readiness. Fast writers followed by process exit can // strand the tail behind a missed wakeup/HUP transition. let raw = master_fd.as_raw_fd(); - let flags = unsafe { libc::fcntl(raw, libc::F_GETFL) }; - if flags >= 0 { - unsafe { libc::fcntl(raw, libc::F_SETFL, flags & !libc::O_NONBLOCK) }; - } + // The duplicated master shares O_NONBLOCK with stdin. Never clear that flag here: + // a blocked write would otherwise strand lifecycle handling on the agent actor. loop { let mut buf = [0u8; 4096]; @@ -1554,6 +1733,11 @@ async fn pty_reader_task( let err = std::io::Error::last_os_error(); match err.raw_os_error() { Some(libc::EINTR) => continue, + Some(libc::EAGAIN) => { + if wait_fd_readable(raw).is_err() { + break; + } + } Some(libc::EIO) => break, _ => break, } diff --git a/crates/agentd/lib/tcp.rs b/crates/agentd/lib/tcp.rs index dee6b5a1f..68278c961 100644 --- a/crates/agentd/lib/tcp.rs +++ b/crates/agentd/lib/tcp.rs @@ -8,7 +8,7 @@ use std::time::Duration; use bytes::Bytes; use tokio::io::{AsyncReadExt, AsyncWrite, AsyncWriteExt}; use tokio::net::TcpStream; -use tokio::sync::{OwnedSemaphorePermit, mpsc, watch}; +use tokio::sync::{mpsc, watch}; use tokio::task::JoinHandle; use microsandbox_protocol::bulk::{ @@ -20,7 +20,8 @@ use microsandbox_protocol::codec; use microsandbox_protocol::message::{Message, MessageType}; use microsandbox_protocol::tcp::{TcpClosed, TcpConnect, TcpConnected, TcpData, TcpEof, TcpFailed}; -use crate::agent::AdmittedBulkRecord; +use crate::agent::{AdmittedBulkRecord, BulkInputPermit}; +use crate::serial::InputCharge; #[cfg(test)] use crate::session::SessionOutputEnvelope; use crate::session::{ @@ -64,8 +65,8 @@ pub struct TcpSession { } enum TcpCommand { - Data(Vec), - Eof, + Data(Vec, Option), + Eof(Option), BulkRecord(AdmittedBulkRecord), } @@ -91,7 +92,8 @@ struct PendingTcpWrite { payload: Bytes, written: usize, bulk_end: Option, - _bulk_input_permit: Option, + _bulk_input_permit: Option, + _control_input_charge: Option, } //-------------------------------------------------------------------------------------------------- @@ -109,11 +111,19 @@ impl TcpSession { /// Awaits queue space when the per-session relay is behind, so a stalled /// destination backpressures the caller instead of growing memory. pub async fn write_data(&self, data: Vec) -> Result<(), String> { + self.write_data_charged(data, None).await + } + + pub(crate) async fn write_data_charged( + &self, + data: Vec, + charge: Option, + ) -> Result<(), String> { if self.bulk { return Err("CBOR TCP data is invalid after raw bulk acceptance".into()); } self.commands - .send(TcpCommand::Data(data)) + .send(TcpCommand::Data(data, charge)) .await .map_err(|_| "TCP session is closed".to_string()) } @@ -123,11 +133,18 @@ impl TcpSession { /// Ordered after any queued data, so the destination sees the write shutdown /// only once it has received everything sent before it. pub async fn close_write(&self) -> Result<(), String> { + self.close_write_charged(None).await + } + + pub(crate) async fn close_write_charged( + &self, + charge: Option, + ) -> Result<(), String> { if self.bulk { return Err("CBOR TCP EOF is invalid after raw bulk acceptance".into()); } self.commands - .send(TcpCommand::Eof) + .send(TcpCommand::Eof(charge)) .await .map_err(|_| "TCP session is closed".to_string()) } @@ -152,7 +169,9 @@ impl TcpSession { .as_ref() .ok_or_else(|| "TCP bulk control path is unavailable".to_string())?; if control.credit.is_closed() { - return Err("TCP session is closed".into()); + // Sink consumption can return credit after the producer queued its final output. + // There is no sender left to enable; failing here would cancel its queued raw tail. + return Ok(()); } control.credit.send_replace(Some(credit)); Ok(()) @@ -467,6 +486,12 @@ async fn relay_tcp_session( let mut read_eof = false; loop { + // One EOF leaves the opposite half usable. Once both halves finish, all ordered + // writes have completed and the peer's final output/EOF is already queued. Exit so + // the existing terminal frame releases the host route and guest session together. + if read_eof && write_shutdown { + break; + } let read_limit = bulk.as_ref().map_or(TCP_CHUNK_SIZE, |state| { state .send @@ -648,6 +673,7 @@ async fn relay_tcp_session( // The destination socket has consumed the full payload. Release aggregate // input capacity before an outbound credit waits on the opposite lane. drop(completed._bulk_input_permit); + drop(completed._control_input_charge); if let Some(end) = bulk_end { let Some(state) = bulk.as_mut() else { terminal_sent = send_tcp_failure( @@ -721,7 +747,13 @@ async fn relay_tcp_session( } command = commands.recv(), if pending_write.is_none() && !write_shutdown => { match command { - Some(TcpCommand::Data(data)) => { + Some(TcpCommand::Data(data, charge)) => { + if data.is_empty() { + // An empty data message is not EOF and owns no socket write. Its + // frame token still bounded admission until it reached this turn. + drop(charge); + continue; + } if bulk.is_some() { terminal_sent = send_tcp_failure( id, @@ -736,9 +768,10 @@ async fn relay_tcp_session( written: 0, bulk_end: None, _bulk_input_permit: None, + _control_input_charge: charge, }); } - Some(TcpCommand::Eof) => { + Some(TcpCommand::Eof(charge)) => { if bulk.is_some() { terminal_sent = send_tcp_failure( id, @@ -763,6 +796,7 @@ async fn relay_tcp_session( break; } write_shutdown = true; + drop(charge); } None => { break; @@ -795,6 +829,7 @@ async fn relay_tcp_session( written: 0, bulk_end: Some(end), _bulk_input_permit: Some(permit), + _control_input_charge: None, }); } } @@ -904,6 +939,100 @@ mod tests { use super::*; + #[test] + fn admitted_transport_window_fits_each_tcp_input_queue() { + use microsandbox_protocol::core::{ + WORKLOAD_TRANSPORT_BULK_FRAMES, WORKLOAD_TRANSPORT_CONTROL_FRAMES, + }; + + // Data and EOF retain their admission token until the socket consumes them. One input + // frame occupies at most one command slot, independent of its byte length. + assert!( + WORKLOAD_TRANSPORT_CONTROL_FRAMES + WORKLOAD_TRANSPORT_BULK_FRAMES + <= TCP_COMMAND_CAPACITY as u64 + ); + } + + #[tokio::test] + async fn blocked_tcp_retains_data_and_eof_credit_until_consumption_or_cancel() { + use crate::serial::{InputLane, InputWindow}; + use microsandbox_protocol::core::WorkloadTransportCredit; + use std::os::fd::AsRawFd; + + for cancel in [false, true] { + let listener = TcpListener::bind(("127.0.0.1", 0)).await.unwrap(); + // Bound receive buffering before accept so the peer cannot consume the entire + // admitted8MiB while the application intentionally has not started reading. + let receive_bytes: libc::c_int = 64 * 1024; + assert_eq!( + unsafe { + libc::setsockopt( + listener.as_raw_fd(), + libc::SOL_SOCKET, + libc::SO_RCVBUF, + (&receive_bytes as *const libc::c_int).cast(), + std::mem::size_of_val(&receive_bytes) as libc::socklen_t, + ) + }, + 0 + ); + let (sender, mut output) = SessionOutputSender::channel(); + let session = TcpSession::open( + 8, + TcpConnect { + host: "127.0.0.1".into(), + port: listener.local_addr().unwrap().port(), + bulk: None, + }, + &sender, + ); + let (mut peer, _) = listener.accept().await.unwrap(); + assert_eq!(recv_message(&mut output).await.t, MessageType::TcpConnected); + let initial = WorkloadTransportCredit { + control_bytes: 64, + control_frames: 2, + bulk_bytes: 8 * 1024 * 1024, + bulk_frames: 2, + }; + let ledger = InputWindow::new(initial); + let payload_len = initial.bulk_bytes as usize - 64; + let data_charge = ledger.admit(InputLane::Bulk, payload_len + 32).unwrap(); + let eof_charge = ledger.admit(InputLane::Bulk, 32).unwrap(); + tokio::time::timeout(Duration::from_millis(100), async { + session + .write_data_charged(vec![0x5c; payload_len], Some(data_charge)) + .await + .unwrap(); + session.close_write_charged(Some(eof_charge)).await.unwrap(); + }) + .await + .expect("admitted input waited for a blocked TCP consumer"); + tokio::time::sleep(Duration::from_millis(20)).await; + assert_eq!(ledger.credit().unwrap(), initial); + assert!(ledger.admit(InputLane::Bulk, 1).is_err()); + if cancel { + session.close(); + wait_finished(&session).await; + } else { + let mut bytes = Vec::new(); + tokio::time::timeout(Duration::from_secs(10), peer.read_to_end(&mut bytes)) + .await + .expect("ordered TCP EOF did not arrive") + .unwrap(); + assert_eq!(bytes.len(), payload_len); + assert!(bytes.iter().all(|byte| *byte == 0x5c)); + session.close(); + wait_finished(&session).await; + } + assert_eq!(ledger.credit().unwrap().bulk_bytes, initial.bulk_bytes * 2); + assert_eq!(ledger.credit().unwrap().bulk_frames, 4); + assert_eq!( + ledger.credit().unwrap().control_bytes, + initial.control_bytes + ); + } + } + #[tokio::test] async fn connect_failure_sends_terminal_failed() { let (session_tx, mut session_rx) = SessionOutputSender::channel(); @@ -1010,6 +1139,227 @@ mod tests { accept_task.await.unwrap(); } + #[tokio::test] + async fn active_raw_credit_validation_and_inline_negotiation_still_apply() { + for raw in [false, true] { + let listener = TcpListener::bind(("127.0.0.1", 0)).await.unwrap(); + let (tx, mut rx) = SessionOutputSender::channel(); + let session = TcpSession::open( + 41, + TcpConnect { + host: "127.0.0.1".into(), + port: listener.local_addr().unwrap().port(), + bulk: raw.then(BulkOffer::tcp), + }, + &tx, + ); + let (_peer, _) = listener.accept().await.unwrap(); + assert_eq!(recv_message(&mut rx).await.t, MessageType::TcpConnected); + if raw { + assert_eq!(recv_message(&mut rx).await.t, MessageType::BulkAccepted); + } + let result = session + .apply_credit(BulkCredit { + kind: BulkKind::Tcp, + flow: BulkFlow::GuestToHost, + consumed_offset: 1, + credit_limit: DEFAULT_BULK_WINDOW + 1, + }) + .await; + if raw { + result.unwrap(); + let failed = tokio::time::timeout(Duration::from_secs(1), recv_message(&mut rx)) + .await + .unwrap(); + assert_eq!(failed.t, MessageType::TcpFailed); + assert_eq!(failed.flags, FLAG_TERMINAL); + assert!( + failed + .payload::() + .unwrap() + .error + .contains("not admitted") + ); + wait_finished(&session).await; + } else { + assert!(result.unwrap_err().contains("generation-6")); + session.close(); + wait_finished(&session).await; + } + } + } + + #[tokio::test] + async fn both_half_close_orders_preserve_data_and_emit_one_terminal() { + for raw in [false, true] { + for peer_first in [false, true] { + tokio::time::timeout(Duration::from_secs(5), async { + let listener = TcpListener::bind(("127.0.0.1", 0)).await.unwrap(); + let (tx, mut rx) = SessionOutputSender::channel(); + let session = TcpSession::open( + 31, + TcpConnect { + host: "127.0.0.1".into(), + port: listener.local_addr().unwrap().port(), + bulk: raw.then(BulkOffer::tcp), + }, + &tx, + ); + let (mut peer, _) = listener.accept().await.unwrap(); + assert_eq!(recv_message(&mut rx).await.t, MessageType::TcpConnected); + if raw { + assert_eq!(recv_message(&mut rx).await.t, MessageType::BulkAccepted); + } + let host_data = b"host data survives the peer's first EOF"; + let peer_data = b"peer data survives the host's first EOF"; + if peer_first { + peer.write_all(peer_data).await.unwrap(); + peer.shutdown().await.unwrap(); + assert_tcp_output_through_eof(&mut rx, raw, peer_data).await; + assert!(!session.is_finished(), "one EOF must preserve host writes"); + send_test_input_and_eof(&session, raw, host_data).await; + } else { + send_test_input_and_eof(&session, raw, host_data).await; + } + + let mut received = Vec::new(); + peer.read_to_end(&mut received).await.unwrap(); + assert_eq!(received, host_data); + if !peer_first { + assert!(!session.is_finished(), "one EOF must preserve peer output"); + peer.write_all(peer_data).await.unwrap(); + peer.shutdown().await.unwrap(); + assert_tcp_output_through_eof(&mut rx, raw, peer_data).await; + } + assert_one_normal_terminal(&session, &mut rx).await; + }) + .await + .unwrap_or_else(|_| { + panic!("TCP completion timed out: raw={raw}, peer_first={peer_first}") + }); + } + } + } + + #[tokio::test] + async fn raw_finish_waits_for_delayed_record_and_pending_socket_write_before_terminal() { + use std::os::fd::AsRawFd; + + tokio::time::timeout(Duration::from_secs(5), async { + let listener = TcpListener::bind(("127.0.0.1", 0)).await.unwrap(); + let (stream, accepted) = tokio::join!( + TcpStream::connect(listener.local_addr().unwrap()), + listener.accept(), + ); + let stream = stream.unwrap(); + let (mut peer, _) = accepted.unwrap(); + // Make the last record larger than both fixed socket buffers. The test observes a + // delivered prefix before draining the rest, so EOF cannot be credited at enqueue. + for (fd, option, bytes) in [ + (stream.as_raw_fd(), libc::SO_SNDBUF, 4096 as libc::c_int), + (peer.as_raw_fd(), libc::SO_RCVBUF, 65536 as libc::c_int), + ] { + assert_eq!( + unsafe { + libc::setsockopt( + fd, + libc::SOL_SOCKET, + option, + (&bytes as *const libc::c_int).cast(), + std::mem::size_of_val(&bytes) as libc::socklen_t, + ) + }, + 0 + ); + } + let (tx, mut rx) = SessionOutputSender::channel(); + let (commands, commands_rx) = mpsc::channel(TCP_COMMAND_CAPACITY); + let (credit, credit_rx) = watch::channel(None); + let (finish, finish_rx) = mpsc::channel(1); + let task = tokio::spawn(relay_tcp_session( + 37, + stream, + commands_rx, + Some(TcpBulkControlReceivers { + credit: credit_rx, + finish: finish_rx, + }), + tx, + Some(TcpBulkState { + send: BulkSendState::new( + BulkKind::Tcp, + BulkFlow::GuestToHost, + DEFAULT_BULK_RECORD_PAYLOAD, + DEFAULT_BULK_WINDOW, + ) + .unwrap(), + receive: BulkReceiveState::new( + BulkKind::Tcp, + BulkFlow::HostToGuest, + DEFAULT_BULK_RECORD_PAYLOAD, + DEFAULT_BULK_WINDOW, + DEFAULT_BULK_WINDOW, + ) + .unwrap(), + }), + )); + let session = TcpSession { + owner_id: 37, + commands, + bulk_control: Some(TcpBulkControlSenders { credit, finish }), + task, + bulk: true, + }; + peer.shutdown().await.unwrap(); + assert_tcp_output_through_eof(&mut rx, true, b"").await; + let payload = Bytes::from(vec![0x6a; DEFAULT_BULK_RECORD_PAYLOAD as usize]); + session + .finish_bulk(BulkFinish { + kind: BulkKind::Tcp, + flow: BulkFlow::HostToGuest, + final_offset: payload.len() as u64, + }) + .await + .unwrap(); + while session.bulk_control.as_ref().unwrap().finish.capacity() == 0 { + tokio::task::yield_now().await; + } + assert!( + !session.is_finished(), + "finish cannot skip its missing final record" + ); + assert!(matches!( + rx.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + )); + session + .write_bulk(AdmittedBulkRecord::for_test(BulkRecord { + id: 37, + kind: BulkKind::Tcp, + flow: BulkFlow::HostToGuest, + offset: 0, + payload: payload.clone(), + })) + .await + .unwrap(); + let mut received = vec![0]; + peer.read_exact(&mut received).await.unwrap(); + assert!( + !session.is_finished(), + "finish cannot skip a partial socket write" + ); + assert!(matches!( + rx.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + )); + peer.read_to_end(&mut received).await.unwrap(); + assert_eq!(received, payload); + assert_one_normal_terminal(&session, &mut rx).await; + }) + .await + .expect("delayed raw record did not finish normally"); + } + #[tokio::test] async fn raw_bulk_tcp_relays_both_directions_and_exact_half_closes() { let listener = TcpListener::bind(("127.0.0.1", 0)).await.unwrap(); @@ -1209,6 +1559,88 @@ mod tests { session.close(); } + async fn send_test_input_and_eof(session: &TcpSession, raw: bool, data: &[u8]) { + if raw { + session + .write_bulk(AdmittedBulkRecord::for_test(BulkRecord { + id: session.owner_id(), + kind: BulkKind::Tcp, + flow: BulkFlow::HostToGuest, + offset: 0, + payload: Bytes::copy_from_slice(data), + })) + .await + .unwrap(); + session + .finish_bulk(BulkFinish { + kind: BulkKind::Tcp, + flow: BulkFlow::HostToGuest, + final_offset: data.len() as u64, + }) + .await + .unwrap(); + } else { + session.write_data(data.to_vec()).await.unwrap(); + session.close_write().await.unwrap(); + } + } + + async fn assert_tcp_output_through_eof( + rx: &mut mpsc::Receiver, + raw: bool, + expected: &[u8], + ) { + let mut received = Vec::new(); + loop { + let envelope = rx.recv().await.expect("TCP output ended before EOF"); + match envelope.output { + SessionOutput::Bulk(output) => { + assert!(raw); + assert_eq!(output.record.offset, received.len() as u64); + received.extend_from_slice(&output.record.payload); + } + SessionOutput::Raw(mut output) => { + let message = decode_one_message(&mut output.frame); + assert_eq!(message.flags & FLAG_TERMINAL, 0, "terminal preceded EOF"); + match message.t { + MessageType::TcpData => { + assert!(!raw); + received.extend(message.payload::().unwrap().data); + } + MessageType::TcpEof => { + assert!(!raw); + break; + } + MessageType::BulkFinish => { + assert!(raw); + let finish = message.payload::().unwrap(); + assert_eq!(finish.final_offset, received.len() as u64); + break; + } + _ => panic!("unexpected TCP output: {:?}", message.t), + } + } + _ => panic!("unexpected non-TCP output"), + } + } + assert_eq!(received, expected); + } + + async fn assert_one_normal_terminal( + session: &TcpSession, + rx: &mut mpsc::Receiver, + ) { + let closed = recv_message(rx).await; + assert_eq!(closed.t, MessageType::TcpClosed); + assert_eq!(closed.flags, FLAG_TERMINAL); + closed.payload::().unwrap(); + wait_finished(session).await; + assert!(matches!( + rx.try_recv(), + Err(mpsc::error::TryRecvError::Empty | mpsc::error::TryRecvError::Disconnected) + )); + } + async fn wait_finished(session: &TcpSession) { tokio::time::timeout(Duration::from_secs(1), async { while !session.is_finished() { diff --git a/crates/agentd/lib/workload.rs b/crates/agentd/lib/workload.rs index ff2ea6921..33e21e12d 100644 --- a/crates/agentd/lib/workload.rs +++ b/crates/agentd/lib/workload.rs @@ -1,8 +1,8 @@ //! Checkpoint-time execution latch for agentd-managed workloads. use std::fs::{File, OpenOptions}; -use std::io; -use std::os::fd::{AsRawFd, OwnedFd}; +use std::io::{self, Read, Seek, SeekFrom}; +use std::os::fd::{AsRawFd, OwnedFd, RawFd}; use std::path::{Path, PathBuf}; use std::time::{Duration, Instant}; @@ -12,7 +12,9 @@ use std::time::{Duration, Instant}; const CGROUP_ROOT: &str = "/sys/fs/cgroup/microsandbox-workload"; const FREEZE_TIMEOUT: Duration = Duration::from_secs(5); -const FREEZE_POLL_INTERVAL: Duration = Duration::from_millis(1); +const FREEZE_STATE_RECHECK_INTERVAL: Duration = Duration::from_millis(1); +const FREEZE_FAST_RECHECK_INTERVAL: Duration = Duration::from_micros(100); +const FREEZE_FAST_RECHECK_WINDOW: Duration = Duration::from_millis(1); const MAX_ATTEMPT_ID_BYTES: usize = 128; //-------------------------------------------------------------------------------------------------- @@ -33,6 +35,7 @@ pub(crate) struct WorkloadLatch { enum LatchState { Running { last_thawed: Option }, Frozen { attempt_id: String }, + RecoveryRequired { attempt_id: String }, } trait FreezerControl: Send { @@ -43,6 +46,7 @@ trait FreezerControl: Send { struct CgroupFreezer { root: PathBuf, cgroup_procs: File, + cgroup_events: File, } /// A child-owned cgroup placement handle prepared before `fork`. @@ -116,9 +120,9 @@ impl WorkloadLatch { .transpose() } - /// Whether a checkpoint attempt currently holds the workload frozen. + /// Whether an attempt blocks new work, including an uncertain freezer transition. pub(crate) fn is_frozen(&self) -> bool { - matches!(self.state, LatchState::Frozen { .. }) + !matches!(self.state, LatchState::Running { .. }) } /// Freeze every process in the agentd-managed workload cgroup. @@ -130,19 +134,27 @@ impl WorkloadLatch { } if current == attempt_id => return Ok(()), LatchState::Frozen { attempt_id: current, + } + | LatchState::RecoveryRequired { + attempt_id: current, } => { return Err(WorkloadLatchError::Conflict(format!( - "attempt {current:?} already owns the freeze" + "attempt {current:?} owns the latch; thaw it before another freeze" ))); } LatchState::Running { .. } => {} } + self.freezer()?; + // Record ownership before writing: an error may follow a successful cgroup write. + // Only a confirmed thaw can release an uncertain transition. + self.state = LatchState::RecoveryRequired { + attempt_id: attempt_id.to_string(), + }; self.freezer()?.set_frozen(true)?; - // Agentd itself remains outside the workload cgroup, so it can flush every mounted - // filesystem after user processes stop mutating them and before the host pauses the VM. - // `sync(2)` has no error return; completion is the durability boundary exposed by Linux. - unsafe { libc::sync() }; + // This latch stops execution, not guest writeback. Full captures preserve dirty guest + // cache pages in RAM alongside the matching device/disk cut; disk-only extraction is + // crash-consistent. Host block draining and durable publication remain separate gates. self.state = LatchState::Frozen { attempt_id: attempt_id.to_string(), }; @@ -179,14 +191,21 @@ impl WorkloadLatch { } LatchState::Frozen { attempt_id: current, + } + | LatchState::RecoveryRequired { + attempt_id: current, } if current != attempt_id => { return Err(WorkloadLatchError::Conflict(format!( "attempt {current:?} owns the freeze" ))); } - LatchState::Frozen { .. } => {} + LatchState::Frozen { .. } | LatchState::RecoveryRequired { .. } => {} } + // A failed thaw must not leave a state that freeze retries can acknowledge as frozen. + self.state = LatchState::RecoveryRequired { + attempt_id: attempt_id.to_string(), + }; self.freezer()?.set_frozen(false)?; self.state = LatchState::Running { last_thawed: Some(attempt_id.to_string()), @@ -230,27 +249,36 @@ impl CgroupFreezer { Ok(Self { root: root.to_path_buf(), cgroup_procs, + cgroup_events: File::open(events)?, }) } fn wait_for_state(&self, expected: bool) -> io::Result<()> { - let deadline = Instant::now() + FREEZE_TIMEOUT; - loop { - let events = std::fs::read_to_string(self.root.join("cgroup.events"))?; - if parse_frozen_event(&events) == Some(expected) { - return Ok(()); - } - if Instant::now() >= deadline { - return Err(io::Error::new( - io::ErrorKind::TimedOut, - format!( - "cgroup did not report frozen={} within {FREEZE_TIMEOUT:?}", - expected as u8 - ), - )); - } - std::thread::sleep(FREEZE_POLL_INTERVAL); - } + let mut events = &self.cgroup_events; + let fd = events.as_raw_fd(); + let mut contents = String::with_capacity(128); + // cgroup.events sends POLLPRI/POLLERR when frozen changes. Read on the same open + // descriptor before each wait: an early completion is observed immediately, and a + // change between read and poll remains pending on this descriptor's kernfs counter. + // cgroup_file_notify rate-limits notifications, however, so bounded poll timeouts + // also recheck authoritative state instead of waiting for a delayed notification. + wait_for_frozen_event( + expected, + Instant::now() + FREEZE_TIMEOUT, + || { + events.seek(SeekFrom::Start(0))?; + contents.clear(); + events.read_to_string(&mut contents)?; + parse_frozen_event(&contents).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "cgroup.events omitted frozen state", + ) + }) + }, + |remaining| wait_for_cgroup_event(fd, remaining), + Instant::now, + ) } } @@ -308,6 +336,94 @@ impl FreezerControl for CgroupFreezer { // Functions //-------------------------------------------------------------------------------------------------- +fn wait_for_frozen_event( + expected: bool, + deadline: Instant, + mut read_state: impl FnMut() -> io::Result, + mut wait: impl FnMut(Duration) -> io::Result<()>, + mut now: impl FnMut() -> Instant, +) -> io::Result<()> { + let fast_until = now() + FREEZE_FAST_RECHECK_WINDOW; + loop { + let interrupted = match read_state() { + Ok(state) if state == expected => return Ok(()), + Ok(_) => false, + Err(error) if error.kind() == io::ErrorKind::Interrupted => true, + Err(error) => return Err(error), + }; + let observed_at = now(); + let remaining = deadline.saturating_duration_since(observed_at); + if remaining.is_zero() { + return Err(io::Error::new( + io::ErrorKind::TimedOut, + format!( + "cgroup did not report frozen={} within {FREEZE_TIMEOUT:?}", + expected as u8 + ), + )); + } + if interrupted { + continue; + } + // Cgroup notifications can be delayed even after frozen=1. Brief sleeping rechecks + // avoid a whole millisecond of observation lag without spinning for the deadline. + let interval = if observed_at < fast_until { + FREEZE_FAST_RECHECK_INTERVAL + } else { + FREEZE_STATE_RECHECK_INTERVAL + }; + match wait(remaining.min(interval)) { + Ok(()) => {} + // Recheck state and the original deadline after interruptions or spurious events. + Err(error) if error.kind() == io::ErrorKind::Interrupted => {} + Err(error) => return Err(error), + } + } +} + +fn wait_for_cgroup_event(fd: RawFd, remaining: Duration) -> io::Result<()> { + let mut event = libc::pollfd { + fd, + events: libc::POLLPRI | libc::POLLERR, + revents: 0, + }; + // Linux guests use ppoll so sub-millisecond waits are not rounded back up to 1 ms. + // Non-Linux builds only exercise the portable unit-test fallback, never a guest freezer. + #[cfg(target_os = "linux")] + let result = { + let timeout = libc::timespec { + // Infer the platform's field type: naming libc::time_t is deprecated on musl. + tv_sec: remaining.as_secs().try_into().map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidInput, + "cgroup wait duration is too large", + ) + })?, + tv_nsec: remaining.subsec_nanos().into(), + }; + unsafe { libc::ppoll(&mut event, 1, &timeout, std::ptr::null()) } + }; + #[cfg(not(target_os = "linux"))] + let result = { + let timeout = remaining + .as_nanos() + .div_ceil(1_000_000) + .min(i32::MAX as u128) as i32; + unsafe { libc::poll(&mut event, 1, timeout) } + }; + if result < 0 { + return Err(io::Error::last_os_error()); + } + if event.revents & (libc::POLLNVAL | libc::POLLHUP) != 0 { + return Err(io::Error::other( + "cgroup.events descriptor became unavailable", + )); + } + // POLLERR accompanies normal kernfs notifications; it is not by itself an I/O failure. + // Timeout and other wakeups both lead to a fresh state read and deadline check. + Ok(()) +} + fn validate_attempt_id(attempt_id: &str) -> Result<(), WorkloadLatchError> { if attempt_id.is_empty() || attempt_id.len() > MAX_ATTEMPT_ID_BYTES @@ -340,14 +456,287 @@ fn parse_frozen_event(events: &str) -> Option { #[cfg(test)] pub(crate) mod tests { + use std::cell::Cell; + use std::collections::VecDeque; use std::sync::{Arc, Mutex}; use super::*; + #[test] + fn completed_freezer_transition_does_not_wait() { + let start = Instant::now(); + wait_for_frozen_event( + true, + start + FREEZE_TIMEOUT, + || Ok(true), + |_| panic!("already frozen"), + || start, + ) + .unwrap(); + } + + #[test] + fn freezer_transition_between_read_and_wait_is_not_lost() { + let start = Instant::now(); + let frozen = Cell::new(false); + let waits = Cell::new(0); + wait_for_frozen_event( + true, + start + FREEZE_TIMEOUT, + || Ok(frozen.get()), + |_| { + // Model a pending kernfs notification from a transition after the state read. + frozen.set(true); + waits.set(waits.get() + 1); + Ok(()) + }, + || start, + ) + .unwrap(); + assert_eq!(waits.get(), 1); + } + + #[test] + fn freezer_spurious_notifications_and_eintr_recheck_state() { + let start = Instant::now(); + let waits = Cell::new(0); + wait_for_frozen_event( + false, + start + FREEZE_TIMEOUT, + || Ok(waits.get() < 3), + |_| { + waits.set(waits.get() + 1); + if waits.get() == 2 { + Err(io::ErrorKind::Interrupted.into()) + } else { + Ok(()) + } + }, + || start, + ) + .unwrap(); + assert_eq!(waits.get(), 3); + } + + #[test] + fn freezer_delayed_notification_does_not_delay_the_authoritative_state_read() { + let start = Instant::now(); + let elapsed = Cell::new(Duration::ZERO); + let frozen = Cell::new(false); + let waits = Cell::new(0); + wait_for_frozen_event( + true, + start + FREEZE_TIMEOUT, + || Ok(frozen.get()), + |remaining| { + assert_eq!(remaining, FREEZE_FAST_RECHECK_INTERVAL); + // The state is ready but cgroup_file_notify defers its notification by + // roughly 10 ms. A bounded timeout observes readiness without that event. + frozen.set(true); + elapsed.set(elapsed.get() + remaining); + waits.set(waits.get() + 1); + Ok(()) + }, + || start + elapsed.get(), + ) + .unwrap(); + assert_eq!(waits.get(), 1); + assert_eq!(elapsed.get(), FREEZE_FAST_RECHECK_INTERVAL); + } + + #[test] + fn freezer_interruptions_do_not_extend_original_deadline() { + let start = Instant::now(); + let elapsed = Cell::new(Duration::ZERO); + let error = wait_for_frozen_event( + true, + start + Duration::from_millis(3), + || Ok(false), + |remaining| { + assert_eq!( + remaining, + (Duration::from_millis(3) - elapsed.get()).min( + if elapsed.get() < FREEZE_FAST_RECHECK_WINDOW { + FREEZE_FAST_RECHECK_INTERVAL + } else { + FREEZE_STATE_RECHECK_INTERVAL + } + ) + ); + elapsed.set(elapsed.get() + Duration::from_millis(1)); + Err(io::ErrorKind::Interrupted.into()) + }, + || start + elapsed.get(), + ) + .unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::TimedOut); + assert_eq!(elapsed.get(), Duration::from_millis(3)); + } + + #[test] + fn freezer_fast_rechecks_back_off_and_respect_short_final_wait() { + let start = Instant::now(); + let elapsed = Cell::new(Duration::ZERO); + let waits = Cell::new(0); + let timeout = Duration::from_micros(2_050); + let error = wait_for_frozen_event( + true, + start + timeout, + || Ok(false), + |duration| { + let expected = if elapsed.get() < FREEZE_FAST_RECHECK_WINDOW { + FREEZE_FAST_RECHECK_INTERVAL + } else { + FREEZE_STATE_RECHECK_INTERVAL + }; + assert_eq!(duration, expected.min(timeout - elapsed.get())); + elapsed.set(elapsed.get() + duration); + waits.set(waits.get() + 1); + Ok(()) + }, + || start + elapsed.get(), + ) + .unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::TimedOut); + assert_eq!(waits.get(), 12); + assert_eq!(elapsed.get(), timeout); + } + + #[test] + fn freezer_read_interruptions_retry_with_a_bounded_deadline() { + let start = Instant::now(); + let reads = Cell::new(0); + let error = wait_for_frozen_event( + true, + start + Duration::from_millis(3), + || { + reads.set(reads.get() + 1); + Err(io::ErrorKind::Interrupted.into()) + }, + |_| panic!("an interrupted read must be retried before waiting"), + || start + Duration::from_millis(reads.get()), + ) + .unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::TimedOut); + assert_eq!(reads.get(), 3); + } + + #[test] + fn freezer_read_and_notification_errors_are_not_success() { + let start = Instant::now(); + let error = wait_for_frozen_event( + true, + start + FREEZE_TIMEOUT, + || Err(io::ErrorKind::InvalidData.into()), + |_| unreachable!(), + || start, + ) + .unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + let error = wait_for_frozen_event( + true, + start + FREEZE_TIMEOUT, + || Ok(false), + |_| Err(io::ErrorKind::BrokenPipe.into()), + || start, + ) + .unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::BrokenPipe); + } + + #[test] + fn freezer_reads_final_state_before_reporting_timeout() { + let start = Instant::now(); + let completed = Cell::new(false); + wait_for_frozen_event( + true, + start + FREEZE_TIMEOUT, + || Ok(completed.get()), + |_| { + completed.set(true); + Ok(()) + }, + || { + if completed.get() { + start + FREEZE_TIMEOUT + } else { + start + } + }, + ) + .unwrap(); + } + + #[test] + fn invalid_cgroup_poll_descriptor_is_an_error() { + assert!(wait_for_cgroup_event(i32::MAX, Duration::from_millis(1)).is_err()); + } + struct FakeFreezer { states: Arc>>, } + struct FailingFreezer { + outcomes: Mutex>, + } + + impl FreezerControl for FailingFreezer { + fn placement(&self) -> io::Result { + unreachable!() + } + + fn set_frozen(&self, _frozen: bool) -> io::Result<()> { + // Model either a failed write or a write that succeeded before acknowledgement failed. + if self.outcomes.lock().unwrap().pop_front().unwrap() { + Ok(()) + } else { + Err(io::Error::other("injected freezer transition failure")) + } + } + } + + #[test] + fn failed_freeze_retains_ownership_until_confirmed_thaw() { + let mut latch = WorkloadLatch::with_freezer(Box::new(FailingFreezer { + outcomes: Mutex::new(VecDeque::from([false, false, true])), + })); + assert!(latch.freeze("a").is_err()); + assert!(latch.is_frozen()); + assert!(latch.freeze("a").is_err()); + assert!(latch.freeze("b").is_err()); + assert!(latch.thaw("b").is_err()); + assert!(latch.thaw("a").is_err()); + assert!(latch.is_frozen()); + latch.thaw("a").unwrap(); + assert!(!latch.is_frozen()); + latch.thaw("a").unwrap(); + } + + #[test] + fn failed_thaw_never_acknowledges_a_freeze_retry() { + let mut latch = WorkloadLatch::with_freezer(Box::new(FailingFreezer { + outcomes: Mutex::new(VecDeque::from([true, false, true])), + })); + latch.freeze("a").unwrap(); + assert!(latch.thaw("a").is_err()); + assert!(latch.is_frozen()); + assert!(latch.freeze("a").is_err()); + latch.thaw("a").unwrap(); + assert!(!latch.is_frozen()); + } + + #[test] + fn known_unavailable_never_takes_ownership() { + let mut latch = WorkloadLatch::unavailable("no cgroup freezer"); + for attempt in ["a", "b"] { + assert!(matches!( + latch.freeze(attempt), + Err(WorkloadLatchError::Unavailable(_)) + )); + assert!(!latch.is_frozen()); + } + } + impl FreezerControl for FakeFreezer { fn placement(&self) -> io::Result { Err(io::Error::new(io::ErrorKind::Unsupported, "not needed")) diff --git a/crates/cli/bin/main.rs b/crates/cli/bin/main.rs index 9770b7843..77d750d5e 100644 --- a/crates/cli/bin/main.rs +++ b/crates/cli/bin/main.rs @@ -22,8 +22,9 @@ const TOP_LEVEL_COMMAND_GROUPS: &[CommandGroup] = &[ CommandGroup { heading: "Sandboxes", commands: &[ - "run", "create", "modify", "start", "stop", "restart", "ping", "touch", "list", - "status", "metrics", "remove", "exec", "copy", "logs", "ssh", "inspect", + "run", "create", "modify", "start", "stop", "pause", "resume", "restart", "ping", + "touch", "list", "status", "metrics", "remove", "exec", "copy", "logs", "ssh", + "inspect", ], }, CommandGroup { @@ -107,6 +108,12 @@ enum Commands { /// Stop one or more running sandboxes. Stop(stop::StopArgs), + /// Suspend a resident sandbox without creating a snapshot. + Pause(microsandbox_cli::commands::pause::PauseArgs), + /// Branch running execution into a new local CoW child without a durable full snapshot. + Branch(microsandbox_cli::commands::branch::BranchArgs), + /// Resume a user-paused resident sandbox. + Resume(microsandbox_cli::commands::pause::PauseArgs), /// Restart one or more sandboxes. Restart(restart::RestartArgs), @@ -290,7 +297,9 @@ fn main() { // Handle --tree before Cli::parse() so it works even when // required arguments (e.g. `msb run --tree`) are missing. - if let Some(tree) = microsandbox_cli::tree::try_show_tree(&Cli::command()) { + if std::env::args_os().any(|arg| arg == "--tree") + && let Some(tree) = microsandbox_cli::tree::try_show_tree(&Cli::command()) + { println!("{tree}"); return; } @@ -637,13 +646,19 @@ fn run_async_command_anyhow( // Pull and create can overlap network I/O, decompression, and progress UI. // Use a small-but-not-tiny worker pool so foreground UI tasks still get // scheduled while multiple layers are downloading and materializing. - let worker_threads = std::thread::available_parallelism() - .map(|count| count.get().clamp(4, 8)) - .unwrap_or(4); - let runtime = tokio::runtime::Builder::new_multi_thread() - .worker_threads(worker_threads) - .enable_all() - .build()?; + // Resident control performs one IPC exchange. It needs I/O and timers, not the image + // pipeline's worker pool. Blocking filesystem/SQLite work keeps its normal executor. + let mut builder = if matches!(command, Commands::Pause(_) | Commands::Resume(_)) { + tokio::runtime::Builder::new_current_thread() + } else { + let worker_threads = std::thread::available_parallelism() + .map(|count| count.get().clamp(4, 8)) + .unwrap_or(4); + let mut builder = tokio::runtime::Builder::new_multi_thread(); + builder.worker_threads(worker_threads); + builder + }; + let runtime = builder.enable_all().build()?; runtime.block_on(async move { // Stale-sandbox reaping and ephemeral cleanup are owned by host @@ -670,6 +685,9 @@ fn run_async_command_anyhow( Commands::Modify(args) => modify::run(args).await, Commands::Start(args) => start::run(args).await, Commands::Stop(args) => stop::run(args).await, + Commands::Pause(args) => microsandbox_cli::commands::pause::run(args, false).await, + Commands::Branch(args) => microsandbox_cli::commands::branch::run(args).await, + Commands::Resume(args) => microsandbox_cli::commands::pause::run(args, true).await, Commands::Restart(args) => restart::run(args).await, Commands::Ping(args) => ping::run(args).await, Commands::Touch(args) => touch::run(args).await, diff --git a/crates/cli/lib/commands/branch.rs b/crates/cli/lib/commands/branch.rs new file mode 100644 index 000000000..6c2b096ed --- /dev/null +++ b/crates/cli/lib/commands/branch.rs @@ -0,0 +1,37 @@ +//! Direct local execution branching without a durable full snapshot. + +use clap::Args; +use microsandbox::Sandbox; + +use crate::ui; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// Create an independent child from a running or user-paused local sandbox. +#[derive(Args)] +pub struct BranchArgs { + /// Source sandbox name. + pub source: String, + /// Name of the new child sandbox. + #[arg(long)] + pub name: String, + /// Suppress progress output. + #[arg(short, long)] + pub quiet: bool, +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +/// Branch source execution. The child's CoW memory is inherent to this operation. +pub async fn run(args: BranchArgs) -> anyhow::Result<()> { + let source = Sandbox::get(&args.source).await?; + let child = source.branch(&args.name).await?; + if !args.quiet { + ui::success("Branched", child.name()); + } + Ok(()) +} diff --git a/crates/cli/lib/commands/common.rs b/crates/cli/lib/commands/common.rs index aec558061..45f5e8c34 100644 --- a/crates/cli/lib/commands/common.rs +++ b/crates/cli/lib/commands/common.rs @@ -127,6 +127,10 @@ pub struct SandboxOpts { #[arg(long, value_name = "POLICY", value_parser = ["always", "madvise", "never"])] pub thp: Option, + /// Restore a full snapshot with private copy-on-write memory. + #[arg(long, requires = "from_snapshot", conflicts_with = "disk_only")] + pub forked: bool, + /// Mount a host path or named volume into the sandbox (`SOURCE:DEST[:OPTIONS]`). /// OPTIONS may include paired `uid=,gid=` for directory-backed mounts. #[arg(short, long)] @@ -984,6 +988,7 @@ impl SandboxOpts { || self.memory.is_some() || self.max_memory.is_some() || self.thp.is_some() + || self.forked || !self.volume.is_empty() || !self.mount_dir.is_empty() || !self.mount_file.is_empty() @@ -1259,6 +1264,9 @@ fn apply_sandbox_opts_inner( .map_err(anyhow::Error::msg)?; builder = builder.thp(policy); } + if opts.forked { + builder = builder.forked(); + } if let Some(ref workdir) = opts.workdir { builder = builder.workdir(workdir); } diff --git a/crates/cli/lib/commands/inspect.rs b/crates/cli/lib/commands/inspect.rs index 5da068159..24359c95d 100644 --- a/crates/cli/lib/commands/inspect.rs +++ b/crates/cli/lib/commands/inspect.rs @@ -86,6 +86,17 @@ pub async fn run(args: InspectArgs) -> anyhow::Result<()> { let handle = Sandbox::get(&args.name).await?; let desired_config = handle.config().ok(); let active_config = handle.active_config().ok().flatten(); + let pause_state = if matches!( + handle.status_snapshot(), + SandboxStatus::Running | SandboxStatus::Paused + ) { + tokio::time::timeout(std::time::Duration::from_millis(250), handle.pause_state()) + .await + .ok() + .and_then(Result::ok) + } else { + None + }; let pending_changes = pending_config_changes( handle.status_snapshot(), desired_config.as_ref(), @@ -98,6 +109,7 @@ pub async fn run(args: InspectArgs) -> anyhow::Result<()> { let mut json = serde_json::json!({ "name": handle.name(), "status": format!("{:?}", handle.status_snapshot()), + "pause": pause_state, "config": config, "created_at": handle.created_at().map(|dt| ui::format_json_datetime(&dt)), "updated_at": handle.updated_at().map(|dt| ui::format_json_datetime(&dt)), @@ -116,6 +128,14 @@ pub async fn run(args: InspectArgs) -> anyhow::Result<()> { ui::detail_kv("Name", handle.name()); ui::detail_kv("Status", &ui::format_status(&status)); + if let Some(state) = &pause_state { + if state.recovery_required { + ui::detail_kv("Recovery", "Required; ordinary resume is fenced"); + } + if let Some(reason) = &state.capture_unavailable { + ui::detail_kv("Full snapshot", reason); + } + } if let Some(dt) = handle.created_at() { ui::detail_kv("Created", &ui::format_datetime(&dt)); diff --git a/crates/cli/lib/commands/mod.rs b/crates/cli/lib/commands/mod.rs index d57878ed7..034bc9335 100644 --- a/crates/cli/lib/commands/mod.rs +++ b/crates/cli/lib/commands/mod.rs @@ -8,6 +8,7 @@ use crate::ui; // Exports //-------------------------------------------------------------------------------------------------- +pub mod branch; pub mod common; pub mod completion; pub mod context; @@ -21,6 +22,7 @@ pub mod list; pub mod logs; pub mod metrics; pub mod modify; +pub mod pause; pub mod ping; pub mod ps; pub mod pull; diff --git a/crates/cli/lib/commands/pause.rs b/crates/cli/lib/commands/pause.rs new file mode 100644 index 000000000..28fca5343 --- /dev/null +++ b/crates/cli/lib/commands/pause.rs @@ -0,0 +1,38 @@ +//! Explicit resident pause and resume. + +use clap::Args; +use microsandbox::Sandbox; + +use crate::ui; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// Arguments shared by resident pause and resume. +#[derive(Args)] +pub struct PauseArgs { + /// Sandbox name. + pub name: String, + /// Suppress progress output. + #[arg(short, long)] + pub quiet: bool, +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +/// Change resident execution state through host control, without opening the guest agent. +pub async fn run(args: PauseArgs, resume: bool) -> anyhow::Result<()> { + let sandbox = Sandbox::get_for_control(&args.name).await?; + if resume { + sandbox.resume().await?; + } else { + sandbox.pause().await?; + } + if !args.quiet { + ui::success(if resume { "Resumed" } else { "Paused" }, &args.name); + } + Ok(()) +} diff --git a/crates/cli/lib/commands/self_cmd.rs b/crates/cli/lib/commands/self_cmd.rs index d3019716d..83529b5f3 100644 --- a/crates/cli/lib/commands/self_cmd.rs +++ b/crates/cli/lib/commands/self_cmd.rs @@ -1108,6 +1108,15 @@ async fn run_downgrade_with_db( } if ctx.operation.phase() < DowngradePhase::DatabaseReverted { + if fresh_plan + .rollback + .iter() + .any(|migration| migration.id == schema_metadata::SNAPSHOT_GROUPS_MIGRATION_ID) + { + // Refuse before any artifact rewrite. A grouped tree cannot be represented + // by the target's flat namespace, even if its rebuildable index is missing. + refuse_snapshot_group_downgrade(ctx.db.inner(), ctx.snapshots_dir).await?; + } let reverses_legacy_snapshots = fresh_plan.rollback.iter().any(|migration| { migration.id == schema_metadata::SNAPSHOT_ARTIFACT_TRANSITION_MIGRATION_ID }); @@ -2528,6 +2537,34 @@ async fn applied_migrations(db: &DatabaseConnection) -> anyhow::Result anyhow::Result<()> { + let grouped = optional_count( + db, + "SELECT COUNT(*) FROM snapshot_index WHERE group_path IS NOT NULL OR group_name IS NOT NULL", + ).await?; + let mut grouped_on_disk = false; + if snapshots_dir.exists() { + for entry in fs::read_dir(snapshots_dir)? { + let path = entry?.path(); + // The metadata file is the on-disk namespace marker. Refuse even an empty or + // unindexed group rather than making it disappear from the older CLI. + if path.join("group.json").exists() { + grouped_on_disk = true; + break; + } + } + } + if grouped > 0 || grouped_on_disk { + anyhow::bail!( + "snapshot groups prevent downgrade: retain this version or export and remove snapshot groups before retrying" + ); + } + Ok(()) +} + async fn user_data_warnings(db: &DatabaseConnection) -> anyhow::Result> { let snapshot_count = optional_count(db, "SELECT COUNT(*) FROM snapshot_index").await?; let disk_volume_count = optional_count( @@ -3513,6 +3550,27 @@ mod tests { assert_eq!(row.try_get_by_index::(0).unwrap(), "wal-value"); } + #[tokio::test] + async fn downgrade_refuses_unindexed_group_before_artifact_mutation() { + let dir = tempfile::tempdir().unwrap(); + let db = sea_orm::Database::connect("sqlite::memory:").await.unwrap(); + Migrator::up(&db, None).await.unwrap(); + let snapshots = dir.path().join("snapshots"); + let group = snapshots.join("unindexed"); + fs::create_dir_all(&group).unwrap(); + let state = br#"{"schema":"microsandbox.snapshot-group/1","head":null}"#; + fs::write(group.join("group.json"), state).unwrap(); + let error = refuse_snapshot_group_downgrade(&db, &snapshots) + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("snapshot groups prevent downgrade") + ); + assert_eq!(fs::read(group.join("group.json")).unwrap(), state); + } + #[tokio::test] async fn rollback_schema_steps_through_latest_migrations() { let dir = tempfile::tempdir().unwrap(); @@ -3526,7 +3584,10 @@ mod tests { .unwrap(); Migrator::up(db.inner(), None).await.unwrap(); - // Stable snapshot identity is the newest migration. With no snapshot + // Empty databases can drop grouped addressing without discarding any instances. + rollback_schema(db.inner(), 1).await.unwrap(); + + // With no snapshot // artifacts to translate, rollback removes its two rebuildable index // projections before touching any migration from the released prefix. rollback_schema(db.inner(), 1).await.unwrap(); diff --git a/crates/cli/lib/commands/snapshot.rs b/crates/cli/lib/commands/snapshot.rs index c444cd546..1b931fd29 100644 --- a/crates/cli/lib/commands/snapshot.rs +++ b/crates/cli/lib/commands/snapshot.rs @@ -20,7 +20,7 @@ pub struct SnapshotArgs { /// Snapshot subcommands. #[derive(Debug, Subcommand)] pub enum SnapshotCommands { - /// Create a disk snapshot from a stopped sandbox or a full snapshot from a running one. + /// Create a disk snapshot, or include memory and execution state with --full. Create(SnapshotCreateArgs), /// List indexed snapshots. @@ -40,26 +40,32 @@ pub enum SnapshotCommands { /// Rebuild the local index from artifacts on disk. Reindex(SnapshotReindexArgs), - /// Save a snapshot into a `.tar.zst` archive. + /// Save a snapshot into a `.msb` archive (tar + zstd). Save(SnapshotSaveArgs), /// Load a snapshot archive into the snapshots directory. Load(SnapshotLoadArgs), + + /// Read a group's head, or select a member as its head. + Head(SnapshotHeadArgs), } /// Arguments for `msb snapshot create`. #[derive(Debug, Args)] pub struct SnapshotCreateArgs { - /// Snapshot name, resolved under `~/.microsandbox/snapshots//` - /// (or under `--dest-dir` when given). - pub name: String, + /// Snapshot member name (generated when omitted). + pub name: Option, + + /// Snapshot group to create or add to (defaults to the source sandbox name). + #[arg(long, value_name = "GROUP")] + pub group: Option, - /// Source sandbox name. + /// Source sandbox name. Disk capture also supports running and user-paused sources. #[arg(long, value_name = "SANDBOX")] pub from_sandbox: String, /// Parent directory to create the artifact in, instead of the - /// default snapshots directory. The artifact lands at `DIR/`. + /// default snapshots directory. The group is created under this root. #[arg(long = "dest-dir", value_name = "DIR")] pub dest_dir: Option, @@ -75,7 +81,7 @@ pub struct SnapshotCreateArgs { #[arg(long = "label", value_name = "K=V")] pub labels: Vec, - /// Overwrite an existing artifact at the destination. + /// Overwrite an existing archive file; installed group members are immutable. #[arg(short = 'f', long)] pub force: bool, @@ -151,7 +157,7 @@ pub struct SnapshotSaveArgs { /// Snapshot to save (path, name, or digest). pub snapshot: String, - /// Output archive path (`.tar.zst` recommended). + /// Output archive path (`.msb` recommended; explicit filenames are preserved). pub out: std::path::PathBuf, /// Walk the parent chain and include each ancestor in the archive. @@ -163,11 +169,11 @@ pub struct SnapshotSaveArgs { #[arg(long)] pub with_image: bool, - /// Write a plain `.tar` instead of `.tar.zst`. Tradeoff: smaller + /// Write plain tar instead of zstd-compressed tar. Tradeoff: smaller /// CPU but much larger file for sparse uppers. #[arg(long)] pub plain_tar: bool, - /// Export disk layers after an exact base snapshot or standalone base archive. + /// Omit disk layers and RAM objects supplied by an exact base snapshot or standalone archive. #[arg(long, conflicts_with_all = ["last_layers", "with_parents"])] pub since: Option, /// Export only the newest N sealed disk layers (load requires the omitted base). @@ -178,14 +184,35 @@ pub struct SnapshotSaveArgs { /// Arguments for `msb snapshot load`. #[derive(Debug, Args)] pub struct SnapshotLoadArgs { - /// Archive to unpack. - pub archive: std::path::PathBuf, + /// Archives to import together; dependencies are resolved regardless of argument order. + #[arg(required = true, num_args = 1.., value_name = "ARCHIVE")] + pub archives: Vec, /// Destination directory (defaults to `~/.microsandbox/snapshots/`). + #[arg(long, value_name = "DIR")] pub dest: Option, - /// Exact base snapshot or standalone base archive for a dependent archive. + /// External base snapshot or standalone archive if batch/group members cannot supply dependencies. #[arg(long)] pub base: Option, + + /// Import into this group (generated when omitted). + #[arg(long, value_name = "GROUP")] + pub group: Option, + + /// Select the batch's unique tip as head even if it is not a fast-forward. + #[arg(long)] + pub set_head: bool, +} + +/// Arguments for `msb snapshot head`. +#[derive(Debug, Args)] +pub struct SnapshotHeadArgs { + /// Group to read, or GROUP:MEMBER to select a new head. + pub selector: String, + + /// Output format (json). + #[arg(long, value_name = "FORMAT", value_parser = ["json"])] + pub format: Option, } //-------------------------------------------------------------------------------------------------- @@ -203,11 +230,16 @@ pub async fn run(args: SnapshotArgs) -> anyhow::Result<()> { SnapshotCommands::Reindex(args) => reindex(args).await, SnapshotCommands::Save(args) => save(args).await, SnapshotCommands::Load(args) => load(args).await, + SnapshotCommands::Head(args) => head(args).await, } } async fn create(args: SnapshotCreateArgs) -> anyhow::Result<()> { - let mut builder = Snapshot::builder(&args.name).from_sandbox(&args.from_sandbox); + let mut builder = + Snapshot::builder(args.name.unwrap_or_default()).from_sandbox(&args.from_sandbox); + if let Some(group) = args.group { + builder = builder.group(group); + } if let Some(ref dest_dir) = args.dest_dir { builder = builder.dest_dir(dest_dir); } @@ -254,6 +286,9 @@ async fn create(args: SnapshotCreateArgs) -> anyhow::Result<()> { Ok(snap) => { spinner.finish_success("Snapshotted"); if !args.quiet { + if let Some(update) = snap.head_update() { + report_head_update(update); + } println!("{}", snap.id()); println!("{}", snap.path().display()); } @@ -274,8 +309,10 @@ async fn list(args: SnapshotListArgs) -> anyhow::Result<()> { .iter() .map(|s| { serde_json::json!({ + "snapshot_id": s.id(), "digest": s.digest(), "name": s.name(), + "group": s.group(), "parent_digest": s.parent_digest(), "scope": format_scope(s.scope()), "state_kind": s.state_kind(), @@ -319,7 +356,7 @@ async fn list(args: SnapshotListArgs) -> anyhow::Result<()> { "DIGEST", ]); for s in &snapshots { - let name = s.name().unwrap_or("-").to_string(); + let name = format_member_selector(s.group(), s.name(), s.id()); let size = s .size_bytes() .map(format_size) @@ -476,13 +513,48 @@ async fn save(args: SnapshotSaveArgs) -> anyhow::Result<()> { } async fn load(args: SnapshotLoadArgs) -> anyhow::Result<()> { - let handle = if let Some(base) = args.base.as_deref() { - Snapshot::load_with_base(&args.archive, args.dest.as_deref(), base).await? + let handles = Snapshot::load_many( + &args.archives, + microsandbox::snapshot::LoadOpts { + dest: args.dest, + base: args.base, + group: args.group, + set_head: args.set_head, + }, + ) + .await?; + // Every imported member belongs to one batch; report its single head decision once. + if let Some(update) = handles.iter().find_map(|handle| handle.head_update()) { + report_head_update(update); + } else if let Some(group) = handles.first().and_then(|handle| handle.group()) { + eprintln!( + "group {group}: imported members without selecting a head; choose a member explicitly" + ); + } + for (index, handle) in handles.iter().enumerate() { + if handles.len() > 1 { + if index > 0 { + println!(); + } + println!("Snapshot: {}", handle.id()); + } + println!("{}", handle.digest()); + // Preserve the single-archive digest/path output consumed by shell scripts. + println!("{}", handle.path().display()); + } + Ok(()) +} + +async fn head(args: SnapshotHeadArgs) -> anyhow::Result<()> { + let update = Snapshot::group_head(&args.selector).await?; + if args.format.as_deref() == Some("json") { + println!("{}", serde_json::to_string_pretty(&update)?); } else { - Snapshot::load(&args.archive, args.dest.as_deref()).await? - }; - println!("{}", handle.digest()); - println!("{}", handle.path().display()); + ui::detail_kv("Group", &update.group); + ui::detail_kv("Previous head", update.previous.as_deref().unwrap_or("-")); + ui::detail_kv("Head", &update.head); + ui::detail_kv("Reason", &format!("{:?}", update.reason)); + } Ok(()) } @@ -490,6 +562,22 @@ async fn load(args: SnapshotLoadArgs) -> anyhow::Result<()> { // Functions: Helpers //-------------------------------------------------------------------------------------------------- +fn format_member_selector(group: Option<&str>, name: Option<&str>, id: &str) -> String { + let member = name.unwrap_or(id); + // Friendly member names are scoped to one group; qualify them so rows stay distinct. + match group { + Some(group) => format!("{group}:{member}"), + None => member.to_string(), + } +} + +fn report_head_update(update: µsandbox::snapshot::HeadUpdate) { + eprintln!( + "group {}: head {} ({:?})", + update.group, update.head, update.reason + ); +} + fn format_str(f: microsandbox::SnapshotFormat) -> &'static str { match f { microsandbox::SnapshotFormat::Raw => "raw", @@ -592,7 +680,7 @@ mod tests { let SnapshotCommands::Create(args) = args.command else { panic!("expected create command"); }; - assert_eq!(args.name, "clean"); + assert_eq!(args.name.as_deref(), Some("clean")); assert_eq!(args.from_sandbox, "box"); assert!(args.full); } @@ -663,14 +751,104 @@ mod tests { #[test] fn load_parses_args() { - let parsed = parse_snapshot_args(&["load", "bundle.tar", "/tmp/snaps"]); + let parsed = parse_snapshot_args(&["load", "bundle.tar", "--dest", "/tmp/snaps"]); let SnapshotCommands::Load(args) = parsed.command else { panic!("expected load command"); }; - assert_eq!(args.archive, std::path::PathBuf::from("bundle.tar")); + assert_eq!(args.archives, vec![std::path::PathBuf::from("bundle.tar")]); assert_eq!( args.dest.as_deref(), Some(std::path::Path::new("/tmp/snaps")) ); } + + #[test] + fn load_parses_multiple_archives_and_a_named_destination() { + let parsed = parse_snapshot_args(&[ + "load", + "changes.msb", + "base.msb", + "--dest", + "/tmp/snaps", + "--group", + "received", + ]); + let SnapshotCommands::Load(args) = parsed.command else { + panic!("expected load command"); + }; + assert_eq!( + args.archives, + vec![ + std::path::PathBuf::from("changes.msb"), + std::path::PathBuf::from("base.msb"), + ] + ); + assert_eq!( + args.dest.as_deref(), + Some(std::path::Path::new("/tmp/snaps")) + ); + assert_eq!(args.group.as_deref(), Some("received")); + } + + #[test] + fn load_requires_at_least_one_archive() { + assert!(TestCli::try_parse_from(["msb", "load", "--group", "received"]).is_err()); + } + + #[test] + fn create_accepts_generated_member_in_explicit_group() { + let parsed = parse_snapshot_args(&["create", "--from-sandbox", "box", "--group", "work"]); + let SnapshotCommands::Create(args) = parsed.command else { + panic!("expected create command"); + }; + assert!(args.name.is_none()); + assert_eq!(args.group.as_deref(), Some("work")); + assert_eq!(args.from_sandbox, "box"); + } + + #[test] + fn load_accepts_group_and_explicit_head_selection() { + let parsed = parse_snapshot_args(&[ + "load", + "changes.msb", + "--base", + "work:base", + "--group", + "work", + "--set-head", + ]); + let SnapshotCommands::Load(args) = parsed.command else { + panic!("expected load command"); + }; + assert_eq!(args.base.as_deref(), Some("work:base")); + assert_eq!(args.group.as_deref(), Some("work")); + assert!(args.set_head); + } + + #[test] + fn head_accepts_member_selector_and_json_format() { + let parsed = parse_snapshot_args(&["head", "work:baseline", "--format", "json"]); + let SnapshotCommands::Head(args) = parsed.command else { + panic!("expected head command"); + }; + assert_eq!(args.selector, "work:baseline"); + assert_eq!(args.format.as_deref(), Some("json")); + } + + #[test] + fn list_disambiguates_aliases_and_unnamed_members_by_group() { + assert_eq!( + format_member_selector(Some("work"), Some("base"), "snap_1"), + "work:base" + ); + assert_eq!( + format_member_selector(Some("copy"), Some("base"), "snap_1"), + "copy:base" + ); + assert_eq!( + format_member_selector(Some("copy"), None, "snap_1"), + "copy:snap_1" + ); + assert_eq!(format_member_selector(None, None, "snap_1"), "snap_1"); + } } diff --git a/crates/cli/lib/sandbox_cmd.rs b/crates/cli/lib/sandbox_cmd.rs index 2cefa3cfd..bbe3d8171 100644 --- a/crates/cli/lib/sandbox_cmd.rs +++ b/crates/cli/lib/sandbox_cmd.rs @@ -36,6 +36,9 @@ use microsandbox_runtime::{ /// `--config-file` for manual invocation). See issue #997. #[derive(Debug, Args)] pub struct SandboxArgs { + /// Require captured execution; runtimes without this protocol reject the invocation. + #[arg(long, hide = true)] + pub restore: bool, /// Override automatic internal host/guest agent transport selection. #[arg( long = "agent-transport", @@ -292,6 +295,7 @@ pub fn run(args: SandboxArgs) -> ! { let vm_config = VmConfig { libkrunfw_path: launch.libkrunfw_path, thp: launch.thp, + memory_cache_dir: launch.memory_cache_dir, vcpus: args.vcpus, memory_mib: args.memory_mib, max_cpus: args.max_vcpus.unwrap_or(args.vcpus).max(args.vcpus), @@ -426,7 +430,12 @@ fn load_launch_config(args: &SandboxArgs) -> Result { .map_err(|e| format!("failed to read --config-file {}: {e}", path.display()))?, None => return Err("missing --config-file for `msb sandbox`".to_string()), }; - serde_json::from_slice(&bytes).map_err(|e| format!("invalid launch config: {e}")) + let config = LaunchConfig::decode(&bytes)?; + if args.restore != (config.execution == microsandbox_runtime::launch::ExecutionIntent::Restore) + { + return Err("--restore and launch execution intent disagree".into()); + } + Ok(config) } /// Read the full contents of the inherited config fd, taking ownership so it @@ -724,6 +733,7 @@ mod tests { let _ = config_fd; SandboxArgs { + restore: false, agent_transport: AgentTransportProfile::Auto, sandbox_name: "test".to_string(), sandbox_id: 1, @@ -854,6 +864,21 @@ mod tests { ); } + #[test] + fn restore_argument_cannot_select_a_fresh_boot() { + use std::io::Write; + let mut file = tempfile::NamedTempFile::new().unwrap(); + file.write_all(&serde_json::to_vec(&LaunchConfig::default()).unwrap()) + .unwrap(); + let mut args = args_with(None, Some(file.path().to_path_buf())); + args.restore = true; + assert!( + load_launch_config(&args) + .unwrap_err() + .contains("intent disagree") + ); + } + #[test] fn test_old_launch_config_without_run_dir_remains_readable() { use std::io::Write; diff --git a/crates/db/lib/connection.rs b/crates/db/lib/connection.rs index cd540a1df..650c9daee 100644 --- a/crates/db/lib/connection.rs +++ b/crates/db/lib/connection.rs @@ -71,6 +71,30 @@ impl DbReadConnection { Ok(Self(conn)) } + /// Open an existing catalog without creating it or changing its journal mode. + /// + /// Intended for short control lookups after the caller coordinates with migrations. + /// This is a normal WAL-aware reader, never an immutable-file shortcut. + pub async fn open_read_only( + db_path: &Path, + connect_timeout: Duration, + busy_timeout: Duration, + ) -> Result { + let options = sqlx::sqlite::SqliteConnectOptions::new() + .filename(db_path) + .read_only(true) + .create_if_missing(false) + .busy_timeout(busy_timeout); + let pool = sqlx::sqlite::SqlitePoolOptions::new() + .max_connections(1) + .acquire_timeout(connect_timeout) + .connect_with(options) + .await?; + Ok(Self(sea_orm::SqlxSqliteConnector::from_sqlx_sqlite_pool( + pool, + ))) + } + /// Borrow the underlying sea-orm connection. pub fn inner(&self) -> &DatabaseConnection { &self.0 @@ -217,6 +241,55 @@ mod tests { const TIMEOUT: Duration = Duration::from_secs(5); + #[tokio::test] + async fn strict_reader_sees_wal_commits_but_cannot_write() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("catalog.db"); + let writer = DbWriteConnection::open(&path, TIMEOUT, TIMEOUT) + .await + .unwrap(); + writer + .execute_unprepared("CREATE TABLE control_test (value INTEGER)") + .await + .unwrap(); + let reader = DbReadConnection::open_read_only(&path, TIMEOUT, TIMEOUT) + .await + .unwrap(); + // Keep the writer alive: control reads must see WAL commits, not an immutable + // view of only the main database file. + writer + .execute_unprepared("INSERT INTO control_test VALUES (42)") + .await + .unwrap(); + let row = reader + .query_one_raw(Statement::from_string( + DbBackend::Sqlite, + "SELECT value FROM control_test", + )) + .await + .unwrap() + .unwrap(); + assert_eq!(row.try_get_by_index::(0).unwrap(), 42); + assert!( + reader + .execute_unprepared("INSERT INTO control_test VALUES (43)") + .await + .is_err() + ); + } + + #[tokio::test] + async fn strict_reader_never_creates_a_catalog() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("missing.db"); + assert!( + DbReadConnection::open_read_only(&path, TIMEOUT, TIMEOUT) + .await + .is_err() + ); + assert!(!path.exists()); + } + #[tokio::test] async fn read_open_does_not_create_db() { // Existing directory, missing DB file. diff --git a/crates/db/lib/entity/snapshot.rs b/crates/db/lib/entity/snapshot.rs index f17f7b849..0155a7021 100644 --- a/crates/db/lib/entity/snapshot.rs +++ b/crates/db/lib/entity/snapshot.rs @@ -14,16 +14,19 @@ use sea_orm::entity::prelude::*; #[derive(Clone, Debug, PartialEq, DeriveEntityModel)] #[sea_orm(table_name = "snapshot_index")] pub struct Model { - /// Released descriptor-digest primary key retained for database compatibility. - #[sea_orm(primary_key, auto_increment = false)] + /// Descriptor digest, shared by identical artifacts in different local groups. pub digest: String, /// Stable opaque snapshot identity. pub snapshot_id: Option, /// SHA-256 of canonical descriptor bytes. pub descriptor_digest: Option, - /// Convenience name (unique when present). NULL for digest-only entries. + /// Convenience name, unique within its group directory. pub name: Option, - /// Manifest digest of the parent snapshot, or NULL for a root. + /// Local group label, absent for explicitly opened ungrouped artifacts. + pub group_name: Option, + /// Absolute group directory, which scopes member aliases across storage roots. + pub group_path: Option, + /// Stable identity of the parent snapshot, or NULL for a root. pub parent_digest: Option, /// Snapshot payload scope (`disk` or `full`). pub scope: String, @@ -40,6 +43,7 @@ pub struct Model { /// Checkpoint-manifest digest for checkpoint state. pub checkpoint_manifest_digest: Option, /// Absolute path to the artifact directory on this host. + #[sea_orm(primary_key, auto_increment = false)] pub artifact_path: String, /// Apparent size of the upper file in bytes. pub size_bytes: Option, @@ -57,7 +61,7 @@ pub struct Model { pub created_at: DateTime, /// When this row was inserted/refreshed. pub indexed_at: DateTime, - /// Number of indexed snapshots whose `parent_digest == self.digest`. + /// Number of distinct child identities whose parent is this snapshot's stable identity. pub child_count: i32, } diff --git a/crates/image/lib/checkpoint/admitted_disk.rs b/crates/image/lib/checkpoint/admitted_disk.rs new file mode 100644 index 000000000..6e561aa63 --- /dev/null +++ b/crates/image/lib/checkpoint/admitted_disk.rs @@ -0,0 +1,284 @@ +//! In-process reuse of a verified immutable disk file, never a path-only verification cache. + +use std::fs::{File, Metadata, OpenOptions}; +use std::io; +use std::path::Path; +use std::sync::Arc; +use std::time::SystemTime; + +use super::sparse_file_integrity; +use crate::error::{ImageError, ImageResult}; + +//-------------------------------------------------------------------------------------------------- +// Constants +//-------------------------------------------------------------------------------------------------- + +/// Admission still checks every layer; only this many file handles are kept for hash reuse. +const MAX_ADMITTED_DISK_FILES: usize = 32; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// A bounded optimization cache, not a limit on the number of admitted disk layers. +#[derive(Clone, Debug, Default)] +pub(super) struct AdmittedDiskLayers { + layers: Vec, +} + +/// A file handle keeps the admitted inode alive even if its original name is removed. +#[derive(Clone, Debug)] +struct AdmittedDiskLayer { + file: Arc, + stamp: FileStamp, + root: String, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct FileStamp { + identity: (u64, u64), + length: u64, + modified: SystemTime, +} + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl AdmittedDiskLayers { + pub(super) fn admit(&mut self, path: &Path, expected: &str) -> ImageResult<()> { + // Always verify, even when the resulting receipt will not fit in the cache. Retaining + // the largest physical files usually keeps the expensive base disk rather than tiny + // overlays; uncached layers simply take the caller's normal fresh-hash path. + let layer = AdmittedDiskLayer::admit(path, expected)?; + if self.layers.len() < MAX_ADMITTED_DISK_FILES { + self.layers.push(layer); + } else if let Some((index, smallest)) = self + .layers + .iter() + .enumerate() + .min_by_key(|(_, candidate)| candidate.stamp.length) + && layer.stamp.length > smallest.stamp.length + { + self.layers[index] = layer; + } + Ok(()) + } + + pub(super) fn reuse_for(&self, path: &Path) -> ImageResult> { + if self.layers.is_empty() { + return Ok(None); + } + // Open once per candidate, not once per admitted ancestor. The remaining checks are + // bounded by the cache size, including detection of a changed retained inode. + let candidate = FileStamp::read(&open_regular(path)?)?; + for layer in &self.layers { + if let Some(root) = layer.reuse_for(&candidate)? { + return Ok(Some(root)); + } + } + Ok(None) + } +} + +impl AdmittedDiskLayer { + fn admit(path: &Path, expected: &str) -> ImageResult { + let file = open_regular(path)?; + let stamp = FileStamp::read(&file)?; + let integrity = sparse_file_integrity(path)?; + if integrity.root != expected { + return Err(ImageError::DigestMismatch { + digest: path.display().to_string(), + expected: expected.into(), + actual: integrity.root, + }); + } + // Cooperative writers never mutate sealed files. Detect accidental replacement or a + // concurrent writer before binding the computed root to this exact owned file. + if FileStamp::read(&file)? != stamp || FileStamp::read(&open_regular(path)?)? != stamp { + return Err(io::Error::other("disk layer changed during admission").into()); + } + Ok(Self { + file: Arc::new(file), + stamp, + root: expected.into(), + }) + } + + fn reuse_for(&self, candidate: &FileStamp) -> ImageResult> { + // A copied or header-relocated layer needs a fresh identity, but an admitted sealed + // inode changing is corruption. Never bless its replacement contents as a new root. + if FileStamp::read(&self.file)? != self.stamp + || (candidate.identity == self.stamp.identity && *candidate != self.stamp) + { + return Err(io::Error::other("admitted disk layer was modified").into()); + } + if *candidate == self.stamp { + Ok(Some(&self.root)) + } else { + Ok(None) + } + } +} + +impl FileStamp { + fn read(file: &File) -> io::Result { + let metadata = file.metadata()?; + Ok(Self { + identity: file_identity(file, &metadata)?, + length: metadata.len(), + modified: metadata.modified()?, + }) + } +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +fn open_regular(path: &Path) -> io::Result { + if !std::fs::symlink_metadata(path)?.is_file() { + return Err(io::Error::other("disk layer is not a regular file")); + } + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK); + } + #[cfg(windows)] + { + use std::os::windows::fs::OpenOptionsExt; + use windows_sys::Win32::Storage::FileSystem::{FILE_SHARE_DELETE, FILE_SHARE_READ}; + options.share_mode(FILE_SHARE_READ | FILE_SHARE_DELETE); + } + let file = options.open(path)?; + if !file.metadata()?.is_file() { + return Err(io::Error::other("disk layer is not a regular file")); + } + Ok(file) +} + +#[cfg(unix)] +fn file_identity(_file: &File, metadata: &Metadata) -> io::Result<(u64, u64)> { + use std::os::unix::fs::MetadataExt; + Ok((metadata.dev(), metadata.ino())) +} + +#[cfg(windows)] +fn file_identity(file: &File, _metadata: &Metadata) -> io::Result<(u64, u64)> { + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::Storage::FileSystem::{ + BY_HANDLE_FILE_INFORMATION, GetFileInformationByHandle, + }; + let mut info: BY_HANDLE_FILE_INFORMATION = unsafe { std::mem::zeroed() }; + // The live File owns this handle; the API fills the complete fixed-size output structure. + if unsafe { GetFileInformationByHandle(file.as_raw_handle(), &mut info) } == 0 { + return Err(io::Error::last_os_error()); + } + Ok(( + u64::from(info.dwVolumeSerialNumber), + u64::from(info.nFileIndexHigh) << 32 | u64::from(info.nFileIndexLow), + )) +} + +#[cfg(not(any(unix, windows)))] +fn file_identity(_file: &File, _metadata: &Metadata) -> io::Result<(u64, u64)> { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "disk file identity is unavailable", + )) +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn admission_reuses_a_hardlink_but_not_a_copy_or_replacement() { + let dir = tempfile::tempdir().unwrap(); + let source = dir.path().join("source"); + let linked = dir.path().join("linked"); + let copied = dir.path().join("copied"); + std::fs::write(&source, b"sealed disk").unwrap(); + let root = sparse_file_integrity(&source).unwrap().root; + let mut admitted = AdmittedDiskLayers::default(); + admitted.admit(&source, &root).unwrap(); + std::fs::hard_link(&source, &linked).unwrap(); + std::fs::copy(&source, &copied).unwrap(); + assert_eq!(admitted.reuse_for(&linked).unwrap(), Some(root.as_str())); + assert_eq!(admitted.reuse_for(&copied).unwrap(), None); + std::fs::remove_file(&source).unwrap(); + std::fs::write(&source, b"sealed disk").unwrap(); + assert_eq!(admitted.reuse_for(&source).unwrap(), None); + assert_eq!(admitted.reuse_for(&linked).unwrap(), Some(root.as_str())); + } + + #[test] + fn admission_rejects_corruption() { + let dir = tempfile::tempdir().unwrap(); + let source = dir.path().join("source"); + std::fs::write(&source, b"original").unwrap(); + let root = sparse_file_integrity(&source).unwrap().root; + std::fs::write(&source, b"modified").unwrap(); + assert!(AdmittedDiskLayers::default().admit(&source, &root).is_err()); + } + + #[cfg(unix)] + #[test] + fn detected_mutation_is_not_treated_as_an_unrelated_copy() { + use std::fs::FileTimes; + use std::time::Duration; + + let dir = tempfile::tempdir().unwrap(); + let source = dir.path().join("source"); + let copy = dir.path().join("copy"); + std::fs::write(&source, b"original").unwrap(); + std::fs::copy(&source, ©).unwrap(); + let root = sparse_file_integrity(&source).unwrap().root; + let mut admitted = AdmittedDiskLayers::default(); + admitted.admit(&source, &root).unwrap(); + std::fs::write(&source, b"modified").unwrap(); + let writer = OpenOptions::new().write(true).open(&source).unwrap(); + writer + .set_times( + FileTimes::new() + .set_modified(admitted.layers[0].stamp.modified + Duration::from_secs(1)), + ) + .unwrap(); + assert!(admitted.reuse_for(&source).is_err()); + assert!(admitted.reuse_for(©).is_err()); + } + + #[test] + fn receipt_cache_retains_largest_files_and_still_admits_uncached_layers() { + let dir = tempfile::tempdir().unwrap(); + let mut admitted = AdmittedDiskLayers::default(); + let mut paths = Vec::new(); + for length in 1..=MAX_ADMITTED_DISK_FILES + 3 { + let path = dir.path().join(format!("layer-{length}")); + std::fs::write(&path, vec![0x55; length]).unwrap(); + let root = sparse_file_integrity(&path).unwrap().root; + admitted.admit(&path, &root).unwrap(); + assert!(admitted.layers.len() <= MAX_ADMITTED_DISK_FILES); + paths.push((path, root)); + } + assert_eq!(admitted.layers.len(), MAX_ADMITTED_DISK_FILES); + for (index, (path, root)) in paths.iter().enumerate() { + let expected = (index >= 3).then_some(root.as_str()); + assert_eq!(admitted.reuse_for(path).unwrap(), expected); + } + + // The smallest layer will not receive a retained receipt, but its bytes must still + // pass full admission. A full cache is never permission to skip verification. + let (smallest, root) = &paths[0]; + std::fs::write(smallest, b"X").unwrap(); + assert!(admitted.admit(smallest, root).is_err()); + assert_eq!(admitted.layers.len(), MAX_ADMITTED_DISK_FILES); + } +} diff --git a/crates/image/lib/checkpoint/mod.rs b/crates/image/lib/checkpoint/mod.rs index af2b79e47..bf80e6593 100644 --- a/crates/image/lib/checkpoint/mod.rs +++ b/crates/image/lib/checkpoint/mod.rs @@ -3,6 +3,7 @@ //! Checkpoint artifacts keep guest-visible state in canonical, content-addressed objects. Mutable //! operation progress, runtime ownership, and provider locations deliberately live elsewhere. +mod admitted_disk; mod compact; mod layer_selection; mod manifest; @@ -26,5 +27,8 @@ pub use manifest::{ ResourceDescriptor, ResourceTreatment, }; pub use qcow::{create_qcow2_overlay, relocate_qcow2_backing, relocated_qcow2_header}; -pub use resolver::CheckpointClosure; -pub use store::{LocalObjectStore, ObjectId, SparseFileIntegrity, sparse_file_integrity}; +pub use resolver::{CheckpointClosure, CheckpointObjectReadTiming}; +pub use store::{ + AdmittedObject, CaptureObjectBatch, CaptureObjectBatchStats, LocalObjectStore, ObjectId, + SparseFileIntegrity, sparse_file_integrity, +}; diff --git a/crates/image/lib/checkpoint/resolver.rs b/crates/image/lib/checkpoint/resolver.rs index 3958b91f5..6fc00578e 100644 --- a/crates/image/lib/checkpoint/resolver.rs +++ b/crates/image/lib/checkpoint/resolver.rs @@ -4,12 +4,14 @@ use std::collections::BTreeSet; use std::fs::File; use std::io::Read; use std::path::{Path, PathBuf}; +use std::time::Instant; use sha2::{Digest as _, Sha256}; +use super::admitted_disk::AdmittedDiskLayers; use super::{ CheckpointManifest, DiskGenerationManifest, DiskLayerRef, MemoryExtentContent, MemoryManifest, - ObjectId, sparse_file_integrity, + ObjectId, }; use crate::error::{ImageError, ImageResult}; @@ -38,6 +40,16 @@ pub struct CheckpointClosure { checkpoint: CheckpointManifest, memory: MemoryManifest, disks: Vec, + admitted_disks: AdmittedDiskLayers, +} + +/// Separate wall times for loading and verifying one checkpoint object. +#[derive(Clone, Copy, Debug, Default)] +pub struct CheckpointObjectReadTiming { + /// File open, allocation or buffer growth, and read time in microseconds. + pub read_us: u128, + /// Identity verification time in microseconds. + pub hash_us: u128, } //-------------------------------------------------------------------------------------------------- @@ -45,6 +57,15 @@ pub struct CheckpointClosure { //-------------------------------------------------------------------------------------------------- impl CheckpointClosure { + /// Inspect the bounded, identity-verified root for construction planning, not payload admission. + /// The child closure must still be fully opened before its contents are consumed. + pub fn inspect_manifest( + root: &Path, + expected_root: Option<&ObjectId>, + ) -> ImageResult { + read_checkpoint_root(root, expected_root).map(|(_, manifest)| manifest) + } + /// Open and validate a checkpoint closure for restore on this host architecture. pub fn open(root: impl Into, expected_root: Option<&ObjectId>) -> ImageResult { Self::open_inner(root.into(), expected_root, true) @@ -66,22 +87,7 @@ impl CheckpointClosure { expected_root: Option<&ObjectId>, require_host_architecture: bool, ) -> ImageResult { - let metadata = std::fs::symlink_metadata(&root)?; - if !metadata.file_type().is_dir() { - return checkpoint_error("checkpoint root is not a directory"); - } - - let root_bytes = - read_regular_bounded(&root.join(CHECKPOINT_ROOT_FILE), MAX_MANIFEST_BYTES)?; - let root_id = ObjectId::from_bytes(&root_bytes)?; - if expected_root.is_some_and(|expected| expected != &root_id) { - return Err(ImageError::DigestMismatch { - digest: root_id.to_string(), - expected: expected_root.expect("checked Some").to_string(), - actual: root_id.to_string(), - }); - } - let checkpoint = CheckpointManifest::from_bytes(&root_bytes)?; + let (root_id, checkpoint) = read_checkpoint_root(&root, expected_root)?; if require_host_architecture && checkpoint.architecture != std::env::consts::ARCH { return checkpoint_error(format!( "checkpoint architecture {} cannot restore on {}", @@ -111,6 +117,7 @@ impl CheckpointClosure { } let mut disks = Vec::with_capacity(checkpoint.disks.len()); + let mut admitted_disks = AdmittedDiskLayers::default(); let mut volumes = BTreeSet::new(); for disk_id in &checkpoint.disks { let bytes = read_object_verified(&root, disk_id, MAX_MANIFEST_BYTES)?; @@ -121,7 +128,11 @@ impl CheckpointClosure { if !volumes.insert(disk.volume_id.clone()) { return checkpoint_error("checkpoint repeats a logical disk volume"); } - validate_disk_layers(&root, &disk)?; + for layer in &disk.layers { + let path = disk_layer_path(&root, layer); + open_regular(&path)?; + admitted_disks.admit(&path, &layer.integrity_root)?; + } disks.push(disk); } @@ -131,6 +142,7 @@ impl CheckpointClosure { checkpoint, memory, disks, + admitted_disks, }) } @@ -159,11 +171,29 @@ impl CheckpointClosure { read_object_verified(&self.root, id, max_len) } + /// Load and verify one object into reusable storage, without changing its identity contract. + /// The buffer must not be consumed when this method fails. + pub fn read_object_into( + &self, + id: &ObjectId, + max_len: u64, + bytes: &mut Vec, + ) -> ImageResult { + read_object_verified_into(&self.root, id, max_len, bytes) + } + /// Return the confined path of a validated disk layer. pub fn disk_layer_path(&self, layer: &DiskLayerRef) -> PathBuf { - self.root - .join("layers") - .join(format!("{}.{}", layer.layer_id, layer.format)) + disk_layer_path(&self.root, layer) + } + + /// Reuse a disk root only while the candidate is the exact unchanged admitted file. + /// Copies, rewritten qcow headers, replaced names, and uncached layers require a new integrity + /// computation. Retained file handles are bounded independently of admitted chain depth. + pub fn reused_disk_integrity(&self, path: &Path) -> ImageResult> { + self.admitted_disks + .reuse_for(path) + .map(|root| root.map(str::to_owned)) } /// Stream and verify every immutable memory payload referenced by the logical generation. @@ -188,6 +218,25 @@ impl CheckpointClosure { // Functions //-------------------------------------------------------------------------------------------------- +fn read_checkpoint_root( + root: &Path, + expected_root: Option<&ObjectId>, +) -> ImageResult<(ObjectId, CheckpointManifest)> { + if !std::fs::symlink_metadata(root)?.file_type().is_dir() { + return checkpoint_error("checkpoint root is not a directory"); + } + let bytes = read_regular_bounded(&root.join(CHECKPOINT_ROOT_FILE), MAX_MANIFEST_BYTES)?; + let id = ObjectId::from_bytes(&bytes)?; + if let Some(expected) = expected_root.filter(|expected| *expected != &id) { + return Err(ImageError::DigestMismatch { + digest: id.to_string(), + expected: expected.to_string(), + actual: id.to_string(), + }); + } + Ok((id, CheckpointManifest::from_bytes(&bytes)?)) +} + fn validate_memory_objects(root: &Path, memory: &MemoryManifest) -> ImageResult<()> { let mut verified = BTreeSet::new(); for extent in &memory.extents { @@ -212,28 +261,24 @@ fn validate_memory_objects(root: &Path, memory: &MemoryManifest) -> ImageResult< Ok(()) } -fn validate_disk_layers(root: &Path, disk: &DiskGenerationManifest) -> ImageResult<()> { - for layer in &disk.layers { - let path = root - .join("layers") - .join(format!("{}.{}", layer.layer_id, layer.format)); - let metadata = std::fs::symlink_metadata(&path)?; - if !metadata.file_type().is_file() { - return checkpoint_error("checkpoint disk layer is not a regular file"); - } - let integrity = sparse_file_integrity(&path)?; - if integrity.root != layer.integrity_root { - return Err(ImageError::DigestMismatch { - digest: layer.layer_id.clone(), - expected: layer.integrity_root.clone(), - actual: integrity.root, - }); - } - } - Ok(()) +fn disk_layer_path(root: &Path, layer: &DiskLayerRef) -> PathBuf { + root.join("layers") + .join(format!("{}.{}", layer.layer_id, layer.format)) } fn read_object_verified(root: &Path, id: &ObjectId, max_len: u64) -> ImageResult> { + let mut bytes = Vec::new(); + read_object_verified_into(root, id, max_len, &mut bytes)?; + Ok(bytes) +} + +fn read_object_verified_into( + root: &Path, + id: &ObjectId, + max_len: u64, + bytes: &mut Vec, +) -> ImageResult { + let started = Instant::now(); let path = object_path(root, id); let mut file = open_regular(&path)?; let length = file.metadata()?.len(); @@ -242,9 +287,17 @@ fn read_object_verified(root: &Path, id: &ObjectId, max_len: u64) -> ImageResult } let length = usize::try_from(length) .map_err(|_| checkpoint_error_value("checkpoint object exceeds host limits"))?; - let mut bytes = Vec::with_capacity(length); - file.read_to_end(&mut bytes)?; - let actual = ObjectId::from_bytes(&bytes)?; + // Keep the initialized buffer between packs. Unlike clear + resize this does not zero + // an entire reused pack before the file read overwrites it. A growing file cannot make + // read_to_end allocate beyond the admitted object bound. + bytes.resize(length, 0); + file.read_exact(bytes)?; + if file.read(&mut [0u8; 1])? != 0 { + return checkpoint_error("checkpoint object changed length during read"); + } + let read_us = started.elapsed().as_micros(); + let hash_started = Instant::now(); + let actual = ObjectId::from_bytes(bytes)?; if &actual != id { return Err(ImageError::DigestMismatch { digest: id.to_string(), @@ -252,7 +305,10 @@ fn read_object_verified(root: &Path, id: &ObjectId, max_len: u64) -> ImageResult actual: actual.to_string(), }); } - Ok(bytes) + Ok(CheckpointObjectReadTiming { + read_us, + hash_us: hash_started.elapsed().as_micros(), + }) } fn verify_object_streaming(root: &Path, id: &ObjectId) -> ImageResult<()> { @@ -332,6 +388,27 @@ mod tests { ResourceDescriptor, ResourceTreatment, }; + #[test] + fn reusable_object_reader_checks_each_identity_and_reuses_allocation() { + let directory = tempfile::tempdir().unwrap(); + let store = super::super::LocalObjectStore::open(directory.path()).unwrap(); + let first = store.put_bytes(b"first payload").unwrap(); + let second = store.put_bytes(b"next payload!").unwrap(); + let mut buffer = Vec::with_capacity(64); + let allocation = buffer.as_ptr(); + read_object_verified_into(directory.path(), &first, 64, &mut buffer).unwrap(); + assert_eq!(buffer, b"first payload"); + read_object_verified_into(directory.path(), &second, 64, &mut buffer).unwrap(); + assert_eq!(buffer, b"next payload!"); + assert_eq!(buffer.as_ptr(), allocation); + assert!(read_object_verified_into(directory.path(), &first, 4, &mut buffer).is_err()); + std::fs::write(store.object_path(&second), b"bad payload!!").unwrap(); + assert!(matches!( + read_object_verified_into(directory.path(), &second, 64, &mut buffer), + Err(ImageError::DigestMismatch { .. }) + )); + } + fn fixture() -> (tempfile::TempDir, ObjectId) { let directory = tempfile::tempdir().unwrap(); let store = super::super::LocalObjectStore::open(directory.path()).unwrap(); @@ -387,6 +464,21 @@ mod tests { (directory, root_id) } + #[test] + fn manifest_inspection_does_not_substitute_for_payload_admission() { + let (directory, root) = fixture(); + let manifest = CheckpointClosure::inspect_manifest(directory.path(), Some(&root)).unwrap(); + std::fs::remove_file(super::object_path( + directory.path(), + &manifest.execution_state, + )) + .unwrap(); + assert!(CheckpointClosure::inspect_manifest(directory.path(), Some(&root)).is_ok()); + assert!(CheckpointClosure::open(directory.path(), Some(&root)).is_err()); + let wrong = ObjectId::from_bytes(b"wrong root").unwrap(); + assert!(CheckpointClosure::inspect_manifest(directory.path(), Some(&wrong)).is_err()); + } + #[test] fn opens_complete_valid_closure() { let (directory, expected) = fixture(); @@ -397,6 +489,115 @@ mod tests { assert_eq!(closure.memory().pause_generation, 7); } + #[test] + fn deep_closure_admission_keeps_file_handles_bounded() { + #[cfg(unix)] + if std::env::var_os("MSB_TEST_DEEP_ADMISSION_LOW_FD").is_none() { + use std::os::unix::process::CommandExt; + + // Isolate the process-wide limit from concurrently running tests. The old one-FD- + // per-layer implementation cannot admit these 512 files with only 64 descriptors. + let mut child = std::process::Command::new(std::env::current_exe().unwrap()); + child + .args([ + "--exact", + "checkpoint::resolver::tests::deep_closure_admission_keeps_file_handles_bounded", + "--nocapture", + ]) + .env("MSB_TEST_DEEP_ADMISSION_LOW_FD", "1"); + // SAFETY: the pre-exec callback only invokes async-signal-safe libc resource-limit + // operations; it does not allocate or acquire locks in the forked child. + unsafe { + child.pre_exec(|| { + let mut limit = std::mem::zeroed::(); + if libc::getrlimit(libc::RLIMIT_NOFILE, &mut limit) != 0 { + return Err(std::io::Error::last_os_error()); + } + limit.rlim_cur = limit.rlim_max.min(64); + if libc::setrlimit(libc::RLIMIT_NOFILE, &limit) != 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + let output = child.output().unwrap(); + assert!( + output.status.success(), + "low-FD admission failed: {}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!(String::from_utf8_lossy(&output.stdout).contains("1 passed")); + return; + } + + let (directory, _) = fixture(); + let store = super::super::LocalObjectStore::open(directory.path()).unwrap(); + let root_path = directory.path().join(CHECKPOINT_ROOT_FILE); + let mut checkpoint = + CheckpointManifest::from_bytes(&std::fs::read(&root_path).unwrap()).unwrap(); + std::fs::create_dir_all(directory.path().join("layers")).unwrap(); + let mut paths = Vec::new(); + // Each volume stays inside the existing 256-layer manifest limit. Disk header codecs + // are runtime concerns: this resolver fixture exercises byte admission and membership. + for volume in 0..2 { + let mut layers = Vec::new(); + for index in 0..256 { + let layer_id = format!("volume_{volume}_layer_{index}"); + let format = if index == 0 { "raw" } else { "qcow2" }; + let path = directory + .path() + .join("layers") + .join(format!("{layer_id}.{format}")); + std::fs::write(&path, [0x55]).unwrap(); + let integrity_root = super::super::sparse_file_integrity(&path).unwrap().root; + layers.push(DiskLayerRef { + layer_id, + format: format.into(), + virtual_size: 4096, + predecessor: (index > 0) + .then(|| format!("volume_{volume}_layer_{}", index - 1)), + integrity_root, + }); + paths.push(path); + } + let disk = DiskGenerationManifest { + schema: "microsandbox.disk-generation/1".into(), + volume_id: format!("volume_{volume}"), + device_id: format!("device_{volume}"), + generation: 1, + head: layers.last().unwrap().layer_id.clone(), + layers, + pause_generation: checkpoint.pause_generation, + }; + checkpoint.disks.push( + store + .put_bytes(&disk.to_canonical_bytes().unwrap()) + .unwrap(), + ); + } + let bytes = checkpoint.to_canonical_bytes().unwrap(); + let root = ObjectId::from_bytes(&bytes).unwrap(); + std::fs::write(&root_path, bytes).unwrap(); + + let closure = CheckpointClosure::open(directory.path(), Some(&root)).unwrap(); + assert_eq!(closure.disks().len(), 2); + assert!(closure.disks().iter().all(|disk| disk.layers.len() == 256)); + let reusable = paths + .iter() + .filter(|path| closure.reused_disk_integrity(path).unwrap().is_some()) + .count(); + assert_eq!(reusable, 32); + drop(closure); + + // A layer outside the retained receipt set must still be verified during admission. + std::fs::write(paths.last().unwrap(), [0xAA]).unwrap(); + assert!(matches!( + CheckpointClosure::open(directory.path(), Some(&root)), + Err(ImageError::DigestMismatch { .. }) + )); + } + #[test] fn portable_open_separates_integrity_from_restore_architecture() { let (directory, _expected) = fixture(); diff --git a/crates/image/lib/checkpoint/store.rs b/crates/image/lib/checkpoint/store.rs index f56e74544..d9e033c45 100644 --- a/crates/image/lib/checkpoint/store.rs +++ b/crates/image/lib/checkpoint/store.rs @@ -1,9 +1,13 @@ //! Crash-safe local immutable-object storage. +use std::collections::{BTreeMap, BTreeSet}; use std::fmt; use std::fs::{File, OpenOptions}; use std::io::{Read, Seek, SeekFrom, Write}; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::SystemTime; use serde::{Deserialize, Serialize}; use sha2::{Digest as _, Sha256}; @@ -32,6 +36,61 @@ pub struct ObjectId(String); #[derive(Clone, Debug)] pub struct LocalObjectStore { root: PathBuf, + ownership: Arc, +} + +#[derive(Debug, Default)] +struct StoreOwnership { + publication: Mutex<()>, +} + +/// A verified immutable inode identity in a live runtime's owned object store. +/// +/// This receipt is neither serializable nor constructible from a path, and is scoped to its store +/// instance. The owning runtime must retain its published object names and never mutate their bytes. +/// Reuse opens and pins the exact inode only for the active operation, checking identity and stamp; +/// missing, replaced or modified members fail closed. Retaining generations therefore costs no FD +/// per object. Unadmitted stores/imports still verify payloads instead of trusting these receipts. +#[derive(Clone, Debug)] +pub struct AdmittedObject { + id: ObjectId, + path: PathBuf, + stamp: ObjectStamp, + ownership: Arc, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct ObjectStamp { + identity: (u64, u64), + length: u64, + modified: SystemTime, +} + +/// Capture-local object publication with deferred directory durability, but durable file data. +/// +/// Finish this batch before publishing any manifest root. Existing `LocalObjectStore::put_bytes` +/// keeps its immediate durability contract; only this explicitly scoped API batches directory sync. +pub struct CaptureObjectBatch { + store: LocalObjectStore, + admitted: Mutex>, + directories: Mutex>, + hashed_bytes: AtomicU64, + linked_bytes: AtomicU64, + copied_bytes: AtomicU64, + directory_syncs: AtomicU64, +} + +/// Actual work performed by a capture object batch, independent of its logical RAM size. +#[derive(Clone, Copy, Debug, Default)] +pub struct CaptureObjectBatchStats { + /// Bytes hashed to create or admit immutable objects. + pub hashed_bytes: u64, + /// Bytes referenced by newly installed closure links, including copy fallbacks. + pub linked_bytes: u64, + /// Bytes physically copied when hardlinks were unavailable. + pub copied_bytes: u64, + /// Directory durability barriers issued by this batch. + pub directory_syncs: u64, } /// Sparse-aware immutable identity of one physical layer file. @@ -94,7 +153,10 @@ impl LocalObjectStore { pub fn open(root: impl Into) -> ImageResult { let root = root.into(); std::fs::create_dir_all(root.join("objects").join("sha256"))?; - Ok(Self { root }) + Ok(Self { + root, + ownership: Arc::new(StoreOwnership::default()), + }) } /// Store exact bytes durably and return their immutable identity. @@ -103,6 +165,7 @@ impl LocalObjectStore { let path = self.object_path(&id); if path.exists() { self.verify_existing(&id, &path)?; + sync_directories_through(path.parent().expect("object parent"), &self.root)?; return Ok(id); } let parent = path.parent().expect("object path has a parent"); @@ -115,17 +178,10 @@ impl LocalObjectStore { file.write_all(bytes)?; file.sync_all()?; drop(file); - match std::fs::rename(&temporary, &path) { - Ok(()) => {} - Err(_error) if path.exists() => { - let _ = std::fs::remove_file(&temporary); - self.verify_existing(&id, &path)?; - return Ok(id); - } - Err(error) => { - let _ = std::fs::remove_file(&temporary); - return Err(error.into()); - } + let published = publish_object_file(&temporary, &path, &self.ownership); + let _ = std::fs::remove_file(&temporary); + if !published? { + self.verify_existing(&id, &path)?; } sync_directories_through(parent, &self.root)?; Ok(id) @@ -201,6 +257,303 @@ impl LocalObjectStore { } } +impl CaptureObjectBatch { + /// Start a new batch, retaining only explicitly supplied previous-generation capabilities. + pub fn new(store: LocalObjectStore, previous: &[AdmittedObject]) -> Self { + let admitted = previous + .iter() + .filter(|object| Arc::ptr_eq(&object.ownership, &store.ownership)) + .map(|object| (object.id.clone(), object.clone())) + .collect(); + Self { + store, + admitted: Mutex::new(admitted), + directories: Mutex::new(BTreeSet::new()), + hashed_bytes: AtomicU64::new(0), + linked_bytes: AtomicU64::new(0), + copied_bytes: AtomicU64::new(0), + directory_syncs: AtomicU64::new(0), + } + } + + /// Hash new bytes once and store durable file data. Directory entries commit at `finish`. + pub fn put_bytes(&self, bytes: &[u8]) -> ImageResult { + let id = ObjectId::from_bytes(bytes)?; + self.hashed_bytes + .fetch_add(bytes.len() as u64, Ordering::Relaxed); + if let Some(object) = self.admitted.lock().unwrap().get(&id).cloned() { + object.validate()?; + return Ok(id); + } + let path = self.store.object_path(&id); + if path.exists() { + self.admit(&id)?; + // Also sync the path of an object left by a previously interrupted batch. + self.record_directories(path.parent().unwrap(), &self.store.root); + return Ok(id); + } + let parent = path.parent().expect("object parent"); + std::fs::create_dir_all(parent)?; + let temporary = parent.join(format!(".{}.{}.tmp", id.hex(), rand::random::())); + let result = (|| -> ImageResult { + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(&temporary)?; + file.write_all(bytes)?; + file.sync_all()?; + drop(file); + if publish_object_file(&temporary, &path, &self.store.ownership)? { + self.open_admitted(id.clone(), path.clone()) + } else { + self.admit(&id) + } + })(); + let _ = std::fs::remove_file(&temporary); + let object = result?; + self.admitted.lock().unwrap().insert(id.clone(), object); + self.record_directories(parent, &self.store.root); + Ok(id) + } + + /// Link admitted bytes without rehashing the generation's complete inherited RAM payload. + pub fn link_into(&self, id: &ObjectId, closure_root: &Path) -> ImageResult { + // A receipt itself is not sufficient authority to read bytes. Pin/check it once below; + // avoid the redundant open that admitting an already-owned receipt would otherwise do. + let known = self.admitted.lock().unwrap().get(id).cloned(); + let object = match known { + Some(object) => object, + None => self.admit(id)?, + }; + let pinned = object.pin()?; + let encoded = id.hex(); + let target = closure_root + .join("objects") + .join("sha256") + .join(&encoded[..2]) + .join(encoded); + let parent = target.parent().expect("closure object parent"); + std::fs::create_dir_all(parent)?; + if target.exists() { + // Existing targets are not automatically part of this batch's ownership. Retain + // the checked public behavior, except for the exact already-admitted inode. + let file = File::open(&target)?; + if ObjectStamp::read(&file)? != object.stamp { + self.verify_and_sync_target(id, &target)?; + } + } else { + match std::fs::hard_link(&object.path, &target) { + Ok(()) => { + if ObjectStamp::read(&File::open(&target)?)? != object.stamp { + let _ = std::fs::remove_file(&target); + return Err( + std::io::Error::other("admitted object path was replaced").into() + ); + } + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + self.verify_and_sync_target(id, &target)?; + } + Err(_) => { + // Copy the retained inode, not a potentially replaced path. Positioned + // reads avoid shared cursor races when several closures reuse one object. + copy_admitted_object(&object, &pinned, &target)?; + self.copied_bytes + .fetch_add(object.stamp.length, Ordering::Relaxed); + } + } + object.validate_pin(&pinned)?; + self.linked_bytes + .fetch_add(object.stamp.length, Ordering::Relaxed); + } + self.record_directories(parent, closure_root); + Ok(target) + } + + fn verify_and_sync_target(&self, id: &ObjectId, path: &Path) -> ImageResult<()> { + // A pre-existing independent copy may have been written without a durability barrier. + // Verify and sync the very same open file, rather than hashing one path then flushing a + // replacement. Windows requires write access for FlushFileBuffers. + #[cfg(unix)] + let file = File::open(path)?; + #[cfg(windows)] + let file = OpenOptions::new().read(true).write(true).open(path)?; + let stamp = ObjectStamp::read(&file)?; + let mut hasher = Sha256::new(); + let mut buffer = vec![0; 1024 * 1024]; + let mut offset = 0; + loop { + let count = read_object_at(&file, &mut buffer, offset)?; + if count == 0 { + break; + } + hasher.update(&buffer[..count]); + offset += count as u64; + } + self.hashed_bytes.fetch_add(offset, Ordering::Relaxed); + let actual = format!("sha256:{}", hex::encode(hasher.finalize())); + if actual != id.as_str() { + return Err(ImageError::DigestMismatch { + digest: id.to_string(), + expected: id.to_string(), + actual, + }); + } + file.sync_all()?; + if ObjectStamp::read(&file)? != stamp || ObjectStamp::read(&File::open(path)?)? != stamp { + return Err(std::io::Error::other("closure object changed during verification").into()); + } + Ok(()) + } + + /// Make every new directory entry durable before its caller publishes a root descriptor. + /// Call only after all batch writers have joined. A failed sync leaves the set available for retry. + pub fn finish(&self) -> ImageResult { + let mut directories = self.directories.lock().unwrap(); + let mut ordered = directories.iter().collect::>(); + ordered.sort_by_key(|path| std::cmp::Reverse(path.components().count())); + for path in ordered { + #[cfg(unix)] + { + File::open(path)?.sync_all()?; + self.directory_syncs.fetch_add(1, Ordering::Relaxed); + } + #[cfg(not(unix))] + let _ = path; + } + directories.clear(); + Ok(self.stats()) + } + + /// Retain exactly the next generation's referenced receipts without keeping object FDs open. + pub fn retained_objects(&self, ids: &[ObjectId]) -> ImageResult> { + ids.iter().map(|id| self.admit(id)).collect() + } + + /// Read counters without adding timing or content-verification work. + pub fn stats(&self) -> CaptureObjectBatchStats { + CaptureObjectBatchStats { + hashed_bytes: self.hashed_bytes.load(Ordering::Relaxed), + linked_bytes: self.linked_bytes.load(Ordering::Relaxed), + copied_bytes: self.copied_bytes.load(Ordering::Relaxed), + directory_syncs: self.directory_syncs.load(Ordering::Relaxed), + } + } + + fn admit(&self, id: &ObjectId) -> ImageResult { + if let Some(object) = self.admitted.lock().unwrap().get(id).cloned() { + object.validate()?; + return Ok(object); + } + let object = self.open_admitted(id.clone(), self.store.object_path(id))?; + let pinned = object.pin()?; + let mut hasher = Sha256::new(); + let mut buffer = vec![0; 1024 * 1024]; + let mut offset = 0; + loop { + let count = read_object_at(&pinned, &mut buffer, offset)?; + if count == 0 { + break; + } + hasher.update(&buffer[..count]); + offset += count as u64; + } + self.hashed_bytes.fetch_add(offset, Ordering::Relaxed); + let actual = format!("sha256:{}", hex::encode(hasher.finalize())); + if actual != id.as_str() { + return Err(ImageError::DigestMismatch { + digest: id.to_string(), + expected: id.to_string(), + actual, + }); + } + object.validate_pin(&pinned)?; + self.admitted + .lock() + .unwrap() + .insert(id.clone(), object.clone()); + Ok(object) + } + + fn open_admitted(&self, id: ObjectId, path: PathBuf) -> ImageResult { + let file = File::open(&path)?; + let stamp = ObjectStamp::read(&file)?; + Ok(AdmittedObject { + id, + path, + stamp, + ownership: Arc::clone(&self.store.ownership), + }) + } + + fn record_directories(&self, path: &Path, stop: &Path) { + let mut directories = self.directories.lock().unwrap(); + for directory in path.ancestors() { + directories.insert(directory.to_path_buf()); + if directory == stop { + break; + } + } + } +} + +impl AdmittedObject { + fn validate(&self) -> ImageResult<()> { + self.pin().map(|_| ()) + } + + fn pin(&self) -> ImageResult { + let file = File::open(&self.path)?; + self.validate_pin(&file)?; + Ok(file) + } + + fn validate_pin(&self, file: &File) -> ImageResult<()> { + if ObjectStamp::read(file)? != self.stamp { + return Err(std::io::Error::other("admitted immutable object was modified").into()); + } + Ok(()) + } +} + +impl ObjectStamp { + fn read(file: &File) -> std::io::Result { + let metadata = file.metadata()?; + if !metadata.is_file() { + return Err(std::io::Error::other( + "immutable object is not a regular file", + )); + } + #[cfg(unix)] + let identity = { + use std::os::unix::fs::MetadataExt; + (metadata.dev(), metadata.ino()) + }; + #[cfg(windows)] + let identity = { + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::Storage::FileSystem::{ + BY_HANDLE_FILE_INFORMATION, GetFileInformationByHandle, + }; + let mut info = std::mem::MaybeUninit::::uninit(); + // SAFETY: the file owns a valid handle and the API initializes the output on success. + if unsafe { GetFileInformationByHandle(file.as_raw_handle(), info.as_mut_ptr()) } == 0 { + return Err(std::io::Error::last_os_error()); + } + let info = unsafe { info.assume_init() }; + ( + u64::from(info.dwVolumeSerialNumber), + (u64::from(info.nFileIndexHigh) << 32) | u64::from(info.nFileIndexLow), + ) + }; + Ok(Self { + identity, + length: metadata.len(), + modified: metadata.modified()?, + }) + } +} + impl MerkleAccumulator { fn new(height: u32) -> Self { Self { @@ -259,6 +612,73 @@ impl From for String { // Functions: Helpers //-------------------------------------------------------------------------------------------------- +fn publish_object_file( + temporary: &Path, + path: &Path, + ownership: &StoreOwnership, +) -> std::io::Result { + // Atomic no-replace publication keeps admitted inode bindings stable for concurrent writers. + match std::fs::hard_link(temporary, path) { + Ok(()) => Ok(true), + Err(_) if path.exists() => Ok(false), + Err(_) => { + // Some filesystems do not support hardlinks. All writers in this runtime's store + // share the fallback namespace lock; it covers only the final check and rename. + let _publication = ownership.publication.lock().unwrap(); + if path.exists() { + return Ok(false); + } + std::fs::rename(temporary, path)?; + Ok(true) + } + } +} + +fn read_object_at(file: &File, bytes: &mut [u8], offset: u64) -> std::io::Result { + #[cfg(unix)] + { + use std::os::unix::fs::FileExt; + file.read_at(bytes, offset) + } + #[cfg(windows)] + { + use std::os::windows::fs::FileExt; + file.seek_read(bytes, offset) + } +} + +fn copy_admitted_object(object: &AdmittedObject, source: &File, target: &Path) -> ImageResult<()> { + let mut destination = OpenOptions::new() + .write(true) + .create_new(true) + .open(target)?; + let result = (|| -> ImageResult<()> { + let mut buffer = vec![0; 1024 * 1024]; + let mut offset = 0; + while offset < object.stamp.length { + let length = buffer.len().min((object.stamp.length - offset) as usize); + let count = read_object_at(source, &mut buffer[..length], offset)?; + if count == 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "immutable object was truncated", + ) + .into()); + } + destination.write_all(&buffer[..count])?; + offset += count as u64; + } + object.validate_pin(source)?; + destination.sync_all()?; + Ok(()) + })(); + drop(destination); + if result.is_err() { + let _ = std::fs::remove_file(target); + } + result +} + fn sync_directories_through(path: &Path, stop: &Path) -> ImageResult<()> { #[cfg(unix)] { @@ -278,6 +698,7 @@ fn sync_directories_through(path: &Path, stop: &Path) -> ImageResult<()> { /// Compute a sparse-aware fixed-leaf Merkle root without reading unallocated holes. pub fn sparse_file_integrity(path: &Path) -> ImageResult { + let started = std::time::Instant::now(); let mut file = File::open(path)?; let logical_size = file.metadata()?.len(); let logical_leaves = logical_size.div_ceil(FILE_MERKLE_LEAF_SIZE as u64).max(1); @@ -289,6 +710,8 @@ pub fn sparse_file_integrity(path: &Path) -> ImageResult { let mut accumulator = MerkleAccumulator::new(tree_height); let mut cursor = 0u64; let mut buffer = vec![0u8; FILE_MERKLE_LEAF_SIZE]; + let mut read_bytes = 0u64; + let mut read_leaves = 0u64; for (start, end) in ranges { push_zero_range(&mut accumulator, &zero_roots, cursor, start); @@ -300,6 +723,8 @@ pub fn sparse_file_integrity(path: &Path) -> ImageResult { buffer.fill(0); file.seek(SeekFrom::Start(offset))?; file.read_exact(&mut buffer[..readable])?; + read_bytes += readable as u64; + read_leaves += 1; accumulator.push_subtree(0, hash_leaf(&buffer)); } cursor = end; @@ -312,6 +737,7 @@ pub fn sparse_file_integrity(path: &Path) -> ImageResult { root.update(&(FILE_MERKLE_LEAF_SIZE as u32).to_le_bytes()); root.update(&tree_height.to_le_bytes()); root.update(&accumulator.finish(tree_height)); + tracing::info!(target: "microsandbox_checkpoint_timing", operation = "disk_hash", logical_bytes = logical_size, read_bytes, read_leaves, hash_us = started.elapsed().as_micros(), "sealed disk integrity timing"); Ok(SparseFileIntegrity { root: format!("blake3:{}", root.finalize().to_hex()), logical_size, @@ -399,6 +825,337 @@ fn hash_parent(left: &[u8; 32], right: &[u8; 32]) -> [u8; 32] { mod tests { use super::*; + #[test] + fn capture_batch_reuses_owned_objects_and_syncs_each_directory_once() { + let directory = tempfile::tempdir().unwrap(); + let store = LocalObjectStore::open(directory.path().join("store")).unwrap(); + let first = CaptureObjectBatch::new(store.clone(), &[]); + let id = first.put_bytes(b"captured immutable RAM").unwrap(); + first + .link_into(&id, &directory.path().join("first")) + .unwrap(); + let stats = first.finish().unwrap(); + assert_eq!( + stats.hashed_bytes, 22, + "new objects must not be rehashed when linked" + ); + let retained = first.retained_objects(std::slice::from_ref(&id)).unwrap(); + let second = CaptureObjectBatch::new(store, &retained); + second + .link_into(&id, &directory.path().join("second")) + .unwrap(); + second + .link_into(&id, &directory.path().join("second")) + .unwrap(); + let stats = second.finish().unwrap(); + assert_eq!(stats.hashed_bytes, 0); + assert_eq!(stats.linked_bytes, 22); + #[cfg(unix)] + assert_eq!( + stats.directory_syncs, 4, + "prefix, algorithm, objects and closure directories" + ); + assert_eq!( + second.finish().unwrap().directory_syncs, + stats.directory_syncs + ); + } + + #[test] + fn capture_batch_checks_unadmitted_data_and_pinned_copy_keeps_exact_inode() { + let directory = tempfile::tempdir().unwrap(); + let store = LocalObjectStore::open(directory.path().join("store")).unwrap(); + let id = store.put_bytes(b"original").unwrap(); + let batch = CaptureObjectBatch::new(store.clone(), &[]); + batch + .link_into(&id, &directory.path().join("first")) + .unwrap(); + assert_eq!(batch.stats().hashed_bytes, 8); + let admitted = batch + .retained_objects(std::slice::from_ref(&id)) + .unwrap() + .remove(0); + let pinned = admitted.pin().unwrap(); + // An active operation's pin is sufficient for a copy even if its original entry is + // unlinked. A later operation must refuse: the runtime no longer owns that name. + std::fs::remove_file(store.object_path(&id)).unwrap(); + let target = directory.path().join("pinned-copy"); + copy_admitted_object(&admitted, &pinned, &target).unwrap(); + assert_eq!(std::fs::read(target).unwrap(), b"original"); + assert!( + batch + .link_into(&id, &directory.path().join("second")) + .is_err() + ); + + let corrupt_id = store.put_bytes(b"must be checked").unwrap(); + std::fs::write(store.object_path(&corrupt_id), b"corrupted").unwrap(); + let fresh = CaptureObjectBatch::new(store, &[]); + assert!( + fresh + .link_into(&corrupt_id, &directory.path().join("bad")) + .is_err() + ); + } + + #[test] + fn capture_batch_rejects_replaced_or_modified_admitted_inodes() { + let directory = tempfile::tempdir().unwrap(); + let store = LocalObjectStore::open(directory.path().join("store")).unwrap(); + let batch = CaptureObjectBatch::new(store.clone(), &[]); + let id = batch.put_bytes(b"original").unwrap(); + let path = store.object_path(&id); + std::fs::remove_file(&path).unwrap(); + std::fs::write(&path, b"replaced").unwrap(); + assert!( + batch + .link_into(&id, &directory.path().join("replaced")) + .is_err() + ); + + let other = batch.put_bytes(b"another object").unwrap(); + // Length change is portable and reliably visible even on coarse timestamp filesystems. + std::fs::write(store.object_path(&other), b"short").unwrap(); + assert!( + batch + .link_into(&other, &directory.path().join("mutated")) + .is_err() + ); + assert!(batch.put_bytes(b"another object").is_err()); + } + + #[test] + fn capture_batch_directory_failure_does_not_report_durable_completion() { + let directory = tempfile::tempdir().unwrap(); + let store = LocalObjectStore::open(directory.path().join("store")).unwrap(); + let batch = CaptureObjectBatch::new(store.clone(), &[]); + let id = batch + .put_bytes(b"durable data pending publication") + .unwrap(); + let closure = directory.path().join("closure"); + batch.link_into(&id, &closure).unwrap(); + #[cfg(unix)] + { + std::fs::remove_dir_all(&closure).unwrap(); + assert!(batch.finish().is_err()); + } + assert!(!closure.join("checkpoint.json").exists()); + assert!(store.object_path(&id).is_file()); + } + + #[test] + fn existing_independent_closure_copies_are_verified_before_reuse() { + let directory = tempfile::tempdir().unwrap(); + let store = LocalObjectStore::open(directory.path().join("store")).unwrap(); + let batch = CaptureObjectBatch::new(store.clone(), &[]); + let id = batch.put_bytes(b"original").unwrap(); + let closure = directory.path().join("closure"); + let target = LocalObjectStore::open(&closure).unwrap().object_path(&id); + std::fs::create_dir_all(target.parent().unwrap()).unwrap(); + // A separate inode, initially written without sync_all, must not inherit source admission. + std::fs::write(&target, b"original").unwrap(); + batch.link_into(&id, &closure).unwrap(); + assert_eq!(batch.stats().hashed_bytes, 16); + batch.finish().unwrap(); + std::fs::write(&target, b"modified").unwrap(); + assert!(batch.link_into(&id, &closure).is_err()); + assert_eq!( + std::fs::read(&target).unwrap(), + b"modified", + "never delete a pre-existing target on verification failure" + ); + assert_eq!(std::fs::read(store.object_path(&id)).unwrap(), b"original"); + } + + #[test] + fn concurrent_checked_and_batched_writers_keep_the_winning_inode() { + let directory = tempfile::tempdir().unwrap(); + let store = LocalObjectStore::open(directory.path().join("store")).unwrap(); + let batch = Arc::new(CaptureObjectBatch::new(store.clone(), &[])); + let gate = std::sync::Barrier::new(8); + let payload = vec![7; 65536]; + std::thread::scope(|scope| { + let handles = (0..8) + .map(|index| { + let gate = &gate; + let payload = &payload; + let batch = &batch; + let store = &store; + scope.spawn(move || { + gate.wait(); + if index % 2 == 0 { + store.put_bytes(payload) + } else { + batch.put_bytes(payload) + } + .unwrap() + }) + }) + .collect::>(); + let expected = ObjectId::from_bytes(&payload).unwrap(); + for handle in handles { + assert_eq!(handle.join().unwrap(), expected); + } + }); + let id = ObjectId::from_bytes(&payload).unwrap(); + batch + .link_into(&id, &directory.path().join("closure")) + .unwrap(); + batch.finish().unwrap(); + let stamp = ObjectStamp::read(&File::open(store.object_path(&id)).unwrap()).unwrap(); + assert_eq!(batch.retained_objects(&[id]).unwrap()[0].stamp, stamp); + } + + #[test] + #[ignore = "opt-in old/new object-store experiment; prints measured times, not a CI latency threshold"] + fn capture_store_full_incremental_experiment() { + const COUNT: usize = 16; + const SIZE: usize = 1024 * 1024; + const DELTA: usize = 4096; + let directory = tempfile::tempdir().unwrap(); + let payloads = (0..COUNT) + .map(|index| vec![index as u8 + 1; SIZE]) + .collect::>(); + let changed = vec![255; DELTA]; + for round in 0..3 { + let old = + LocalObjectStore::open(directory.path().join(format!("old-{round}"))).unwrap(); + let started = std::time::Instant::now(); + let mut old_ids = Vec::new(); + for bytes in &payloads { + let id = old.put_bytes(bytes).unwrap(); + old.link_into(&id, &directory.path().join(format!("old-full-{round}"))) + .unwrap(); + old_ids.push(id); + } + let old_full_us = started.elapsed().as_micros(); + let started = std::time::Instant::now(); + old_ids.push(old.put_bytes(&changed).unwrap()); + for id in &old_ids { + old.link_into(id, &directory.path().join(format!("old-delta-{round}"))) + .unwrap(); + } + let old_delta_us = started.elapsed().as_micros(); + + let new = + LocalObjectStore::open(directory.path().join(format!("new-{round}"))).unwrap(); + let full = CaptureObjectBatch::new(new.clone(), &[]); + let started = std::time::Instant::now(); + let mut new_ids = Vec::new(); + for bytes in &payloads { + let id = full.put_bytes(bytes).unwrap(); + full.link_into(&id, &directory.path().join(format!("new-full-{round}"))) + .unwrap(); + new_ids.push(id); + } + let full_stats = full.finish().unwrap(); + let receipts = full.retained_objects(&new_ids).unwrap(); + let new_full_us = started.elapsed().as_micros(); + let delta = CaptureObjectBatch::new(new, &receipts); + let started = std::time::Instant::now(); + new_ids.push(delta.put_bytes(&changed).unwrap()); + for id in &new_ids { + delta + .link_into(id, &directory.path().join(format!("new-delta-{round}"))) + .unwrap(); + } + let delta_stats = delta.finish().unwrap(); + let new_delta_us = started.elapsed().as_micros(); + assert_eq!( + old_ids, new_ids, + "the optimized publication preserves content identities" + ); + assert_eq!(full_stats.hashed_bytes, (COUNT * SIZE) as u64); + assert_eq!( + delta_stats.hashed_bytes, DELTA as u64, + "inherited payload must not be read again" + ); + #[cfg(unix)] + { + assert!(full_stats.directory_syncs <= (2 * (COUNT + 3)) as u64); + assert!(delta_stats.directory_syncs <= (COUNT + 8) as u64); + } + // Old byte/sync counts follow the unchanged checked public path's exact loop; new + // counts come from runtime counters. Timings are measured; no speed ratio is asserted. + println!( + "{}", + serde_json::json!({ + "experiment": "capture_object_store", "round": round, + "baseline_bytes": COUNT * SIZE, "changed_bytes": DELTA, + "old_full_us": old_full_us, "new_full_us": new_full_us, + "old_incremental_us": old_delta_us, "new_incremental_us": new_delta_us, + "old_full_expected_hashed_bytes": 2 * COUNT * SIZE, + "old_incremental_expected_hashed_bytes": COUNT * SIZE + 2 * DELTA, + "new_full_hashed_bytes": full_stats.hashed_bytes, + "new_incremental_hashed_bytes": delta_stats.hashed_bytes, + "new_full_directory_syncs": full_stats.directory_syncs, + "new_incremental_directory_syncs": delta_stats.directory_syncs + }) + ); + } + } + + #[test] + fn admission_receipts_do_not_escape_their_store_lifetime() { + let directory = tempfile::tempdir().unwrap(); + let store = LocalObjectStore::open(directory.path()).unwrap(); + let first = CaptureObjectBatch::new(store.clone(), &[]); + let id = first.put_bytes(b"payload").unwrap(); + let receipts = first.retained_objects(std::slice::from_ref(&id)).unwrap(); + first.finish().unwrap(); + // Opening the same path does not confer the old runtime's ownership. Re-admit bytes. + let reopened = + CaptureObjectBatch::new(LocalObjectStore::open(directory.path()).unwrap(), &receipts); + reopened + .retained_objects(std::slice::from_ref(&id)) + .unwrap(); + assert_eq!(reopened.stats().hashed_bytes, 7); + } + + #[cfg(unix)] + #[test] + fn retained_receipts_fit_low_fd_budget() { + const CHILD: &str = "MSB_STORE_LOW_FD_TEST_CHILD"; + if std::env::var_os(CHILD).is_none() { + let result = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "checkpoint::store::tests::retained_receipts_fit_low_fd_budget", + "--nocapture", + ]) + .env(CHILD, "1") + .status() + .unwrap(); + assert!(result.success()); + return; + } + // Set a low limit only in this isolated test process, never in the parallel test runner. + let mut limit = std::mem::MaybeUninit::::uninit(); + assert_eq!( + unsafe { libc::getrlimit(libc::RLIMIT_NOFILE, limit.as_mut_ptr()) }, + 0 + ); + let mut limit = unsafe { limit.assume_init() }; + limit.rlim_cur = limit.rlim_cur.min(64); + assert_eq!(unsafe { libc::setrlimit(libc::RLIMIT_NOFILE, &limit) }, 0); + let directory = tempfile::tempdir().unwrap(); + let store = LocalObjectStore::open(directory.path().join("store")).unwrap(); + let first = CaptureObjectBatch::new(store.clone(), &[]); + let ids = (0_u32..512) + .map(|index| first.put_bytes(&index.to_le_bytes()).unwrap()) + .collect::>(); + first.finish().unwrap(); + let receipts = first.retained_objects(&ids).unwrap(); + drop(first); + let second = CaptureObjectBatch::new(store, &receipts); + for id in &ids { + second + .link_into(id, &directory.path().join("closure")) + .unwrap(); + } + assert_eq!(second.finish().unwrap().hashed_bytes, 0); + } + #[test] fn identical_objects_are_reused_and_linked_into_a_closure() { let directory = tempfile::tempdir().unwrap(); diff --git a/crates/image/lib/registry/client.rs b/crates/image/lib/registry/client.rs index 133591530..41cc6e01f 100644 --- a/crates/image/lib/registry/client.rs +++ b/crates/image/lib/registry/client.rs @@ -167,12 +167,93 @@ impl Registry { manifest_digest: &Digest, ) -> ImageResult> { Ok( - resolve_cached_pull_result_by_manifest_digest_async(cache, manifest_digest) + resolve_cached_pull_result_by_manifest_digest_async(cache, manifest_digest, false) .await? .map(|cached| (cached.result, cached.metadata)), ) } + /// Resolve snapshot image defaults by immutable digest. Flat snapshots own + /// their complete disk, so they need metadata but no materialized OCI layers. + pub async fn pull_snapshot_cached( + cache: &GlobalCache, + references: &[oci_client::Reference], + manifest_digest: &Digest, + materialization: RootfsMaterialization, + ) -> ImageResult> { + let metadata_only = materialization == RootfsMaterialization::Flat; + let expected = manifest_digest.to_string(); + // Most snapshots retain either their original tag key or a pinned key. + // Avoid scanning every unrelated image on this common path. + for reference in references { + if let Some(metadata) = cache.read_image_metadata_async(reference).await? + && metadata.manifest_digest == expected + && let Some(cached) = + resolve_snapshot_metadata(cache, metadata, metadata_only).await? + { + return Ok(Some((cached.result, cached.metadata))); + } + } + Ok(resolve_cached_pull_result_by_manifest_digest_async( + cache, + manifest_digest, + metadata_only, + ) + .await? + .map(|cached| (cached.result, cached.metadata))) + } + + /// Fetch only the immutable manifest and config needed by a flat snapshot. + /// Normal pulls still independently require their filesystem artifacts. + pub async fn pull_snapshot_metadata( + &self, + reference: &oci_client::Reference, + ) -> ImageResult { + let expected = reference.digest().ok_or_else(|| { + ImageError::ManifestParse("snapshot metadata requires a digest-pinned reference".into()) + })?; + let (manifest_bytes, digest, config_bytes) = + self.fetch_manifest_and_config(reference).await?; + if digest != expected { + return Err(ImageError::ManifestParse( + "snapshot manifest digest differs from pinned reference".into(), + )); + } + let (manifest, config_bytes, resolved) = self + .parse_and_resolve_manifest(&manifest_bytes, config_bytes, reference) + .await?; + let (config, diff_ids) = ImageConfig::parse(&config_bytes)?; + let layers = self.extract_layer_digests(&manifest)?; + if layers.len() != diff_ids.len() { + return Err(ImageError::ManifestParse( + "snapshot manifest/config layer count mismatch".into(), + )); + } + let metadata = CachedImageMetadata { + manifest_digest: digest, + config_digest: manifest.config_digest().unwrap_or_default(), + raw_manifest_json: json_bytes_to_string(&resolved, "resolved manifest")?, + raw_config_json: json_bytes_to_string(&config_bytes, "image config")?, + config, + layers: layers + .iter() + .zip(diff_ids) + .map(|(layer, diff_id)| CachedLayerMetadata { + digest: layer.digest.to_string(), + media_type: layer.media_type.clone(), + size_bytes: layer.size, + diff_id, + }) + .collect(), + }; + let mut result = cached_pull_result(&metadata)?; + self.cache + .write_image_metadata_async(reference, &metadata) + .await?; + result.cached = false; + Ok(result) + } + /// Pull an image. Downloads blobs and materializes EROFS layers concurrently. pub async fn pull( &self, @@ -1652,9 +1733,9 @@ async fn resolve_cached_pull_result_async( async fn resolve_cached_pull_result_by_manifest_digest_async( cache: &GlobalCache, manifest_digest: &Digest, + metadata_only: bool, ) -> ImageResult> { let expected = manifest_digest.to_string(); - let platform = Platform::host_linux(); let mut entries = tokio::fs::read_dir(cache.manifests_dir()) .await .map_err(|e| ImageError::Cache { @@ -1683,14 +1764,7 @@ async fn resolve_cached_pull_result_by_manifest_digest_async( continue; } - if let Some(cached) = resolve_cached_metadata_pull_result_async( - cache, - metadata, - RootfsMaterialization::Layered, - &platform, - ) - .await? - { + if let Some(cached) = resolve_snapshot_metadata(cache, metadata, metadata_only).await? { return Ok(Some(cached)); } } @@ -1698,6 +1772,25 @@ async fn resolve_cached_pull_result_by_manifest_digest_async( Ok(None) } +async fn resolve_snapshot_metadata( + cache: &GlobalCache, + metadata: CachedImageMetadata, + metadata_only: bool, +) -> ImageResult> { + if metadata_only { + return Ok(cached_pull_result(&metadata) + .ok() + .map(|result| CachedPullInfo { result, metadata })); + } + resolve_cached_metadata_pull_result_async( + cache, + metadata, + RootfsMaterialization::Layered, + &Platform::host_linux(), + ) + .await +} + async fn resolve_cached_metadata_pull_result_async( cache: &GlobalCache, metadata: CachedImageMetadata, @@ -1961,6 +2054,51 @@ mod tests { assert_eq!(cached.1.manifest_digest, metadata.manifest_digest); } + #[tokio::test] + async fn snapshot_flat_metadata_does_not_require_disks_or_accept_moved_tags() { + let temp = tempdir().unwrap(); + let cache = GlobalCache::new(temp.path()).unwrap(); + let reference: oci_client::Reference = "docker.io/library/alpine:latest".parse().unwrap(); + let metadata = write_cached_image_fixture(&cache, &reference, &[false, false]); + let digest = parse_digest(&metadata.manifest_digest); + for refs in [vec![reference.clone()], vec![]] { + let found = super::Registry::pull_snapshot_cached( + &cache, + &refs, + &digest, + RootfsMaterialization::Flat, + ) + .await + .unwrap() + .unwrap(); + assert_eq!(found.0.manifest_digest, digest); + assert_eq!(found.0.config.env, metadata.config.env); + assert!( + super::Registry::pull_snapshot_cached( + &cache, + &refs, + &digest, + RootfsMaterialization::Layered + ) + .await + .unwrap() + .is_none() + ); + } + let other = parse_digest(&format!("sha256:{}", "f".repeat(64))); + assert!( + super::Registry::pull_snapshot_cached( + &cache, + &[reference], + &other, + RootfsMaterialization::Flat + ) + .await + .unwrap() + .is_none() + ); + } + #[tokio::test] async fn test_pull_cached_by_manifest_digest_requires_complete_artifacts() { let temp = tempdir().unwrap(); diff --git a/crates/migration/lib/lib.rs b/crates/migration/lib/lib.rs index b048be3ec..0648f0801 100644 --- a/crates/migration/lib/lib.rs +++ b/crates/migration/lib/lib.rs @@ -26,6 +26,7 @@ mod m20260813_000001_share_cpu_allocations; mod m20260818_000001_sandbox_network_slot; mod m20260824_000001_mount_owner_config; mod m20260829_000001_split_snapshot_identity; +mod m20260910_000001_snapshot_groups; pub mod schema_metadata; use sea_orm_migration::prelude::*; @@ -81,6 +82,7 @@ impl MigratorTrait for Migrator { Box::new(m20260818_000001_sandbox_network_slot::Migration), // Unreleased snapshot-stack migrations follow the complete released prefix. Box::new(m20260829_000001_split_snapshot_identity::Migration), + Box::new(m20260910_000001_snapshot_groups::Migration), ] } } diff --git a/crates/migration/lib/m20260910_000001_snapshot_groups.rs b/crates/migration/lib/m20260910_000001_snapshot_groups.rs new file mode 100644 index 000000000..91f72bccc --- /dev/null +++ b/crates/migration/lib/m20260910_000001_snapshot_groups.rs @@ -0,0 +1,185 @@ +//! Index local snapshot instances separately from portable identities. + +use sea_orm_migration::{ + prelude::*, + sea_orm::{DatabaseBackend, Statement}, +}; + +//-------------------------------------------------------------------------------------------------- +// Constants +//-------------------------------------------------------------------------------------------------- + +const SHARED_COLUMNS: &str = "digest, snapshot_id, descriptor_digest, name, parent_digest, scope, state_kind, image_ref, image_manifest_digest, format, fstype, checkpoint_manifest_digest, artifact_path, size_bytes, locality, storage_binding_id, availability, migration_state, migration_error_code, created_at, indexed_at, child_count"; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +#[derive(DeriveMigrationName)] +pub struct Migration; + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + rebuild_index(manager, true).await + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + let connection = manager.get_connection(); + // Check before rebuilding: older binaries cannot address groups or retain several + // copies of one portable identity. Never silently discard rows to satisfy old keys. + let incompatible = connection + .query_one_raw(Statement::from_string( + DatabaseBackend::Sqlite, + "SELECT EXISTS(SELECT 1 FROM snapshot_index WHERE group_path IS NOT NULL OR group_name IS NOT NULL) OR EXISTS(SELECT 1 FROM snapshot_index GROUP BY digest HAVING COUNT(*) > 1) OR EXISTS(SELECT 1 FROM snapshot_index WHERE snapshot_id IS NOT NULL GROUP BY snapshot_id HAVING COUNT(*) > 1) OR EXISTS(SELECT 1 FROM snapshot_index WHERE name IS NOT NULL GROUP BY name HAVING COUNT(*) > 1) AS incompatible", + )) + .await? + .ok_or_else(|| DbErr::Custom("snapshot group downgrade preflight returned no row".into()))? + .try_get::("", "incompatible")?; + if incompatible != 0 { + return Err(DbErr::Custom( + "snapshot groups prevent downgrade: retain this version or export and remove grouped/duplicate snapshot instances before retrying".into(), + )); + } + rebuild_index(manager, false).await + } +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +async fn rebuild_index(manager: &SchemaManager<'_>, grouped: bool) -> Result<(), DbErr> { + let connection = manager.get_connection(); + connection + .execute_unprepared("ALTER TABLE snapshot_index RENAME TO snapshot_index_group_transition") + .await?; + let digest_key = if grouped { "" } else { " PRIMARY KEY" }; + let path_key = if grouped { " PRIMARY KEY" } else { "" }; + let group_columns = if grouped { + "group_name TEXT, group_path TEXT," + } else { + "" + }; + connection + .execute_unprepared(&format!( + "CREATE TABLE snapshot_index (digest TEXT NOT NULL{digest_key}, snapshot_id TEXT, descriptor_digest TEXT, name TEXT, {group_columns} parent_digest TEXT, scope TEXT NOT NULL, state_kind TEXT NOT NULL, image_ref TEXT NOT NULL, image_manifest_digest TEXT NOT NULL, format TEXT, fstype TEXT, checkpoint_manifest_digest TEXT, artifact_path TEXT NOT NULL{path_key}, size_bytes BIGINT, locality TEXT NOT NULL DEFAULT 'embedded', storage_binding_id TEXT, availability TEXT NOT NULL DEFAULT 'ready', migration_state TEXT NOT NULL DEFAULT 'canonical', migration_error_code TEXT, created_at DATETIME NOT NULL, indexed_at DATETIME NOT NULL, child_count INTEGER NOT NULL DEFAULT 0)" + )) + .await?; + connection + .execute_unprepared(&format!( + "INSERT INTO snapshot_index ({SHARED_COLUMNS}) SELECT {SHARED_COLUMNS} FROM snapshot_index_group_transition" + )) + .await?; + // Dropping the old table also releases its index names before creating replacements. + connection + .execute_unprepared("DROP TABLE snapshot_index_group_transition") + .await?; + let name_index = if grouped { + "CREATE UNIQUE INDEX idx_snapshot_index_name ON snapshot_index (group_path, name) WHERE group_path IS NOT NULL AND name IS NOT NULL" + } else { + "CREATE UNIQUE INDEX idx_snapshot_index_name ON snapshot_index (name) WHERE name IS NOT NULL" + }; + connection.execute_unprepared(name_index).await?; + let identity_unique = if grouped { "" } else { "UNIQUE " }; + connection.execute_unprepared(&format!("CREATE {identity_unique}INDEX idx_snapshot_index_snapshot_id ON snapshot_index (snapshot_id)")).await?; + for (name, column) in [ + ("idx_snapshot_index_digest", "digest"), + ("idx_snapshot_index_descriptor_digest", "descriptor_digest"), + ("idx_snapshot_index_parent", "parent_digest"), + ("idx_snapshot_index_image", "image_manifest_digest"), + ] { + connection + .execute_unprepared(&format!("CREATE INDEX {name} ON snapshot_index ({column})")) + .await?; + } + Ok(()) +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use sea_orm_migration::sea_orm::{Database, DatabaseConnection}; + + use super::*; + use crate::{Migrator, MigratorTrait}; + + async fn prior_database() -> DatabaseConnection { + let db = Database::connect("sqlite::memory:").await.unwrap(); + Migrator::up(&db, Some((Migrator::migrations().len() - 1) as u32)) + .await + .unwrap(); + db.execute_unprepared("INSERT INTO snapshot_index (digest, snapshot_id, descriptor_digest, name, scope, state_kind, image_ref, image_manifest_digest, artifact_path, created_at, indexed_at) VALUES ('sha256:original', 'snap_original', 'sha256:original', 'baseline', 'disk', 'file', 'example', 'sha256:image', '/old/baseline', '2026-09-10 00:00:00', '2026-09-10 00:00:00')").await.unwrap(); + db + } + + #[tokio::test] + async fn preserves_old_rows_and_round_trips_ungrouped_database() { + let db = prior_database().await; + Migrator::up(&db, None).await.unwrap(); + let row = db + .query_one_raw(Statement::from_string( + DatabaseBackend::Sqlite, + "SELECT digest, artifact_path, group_name FROM snapshot_index", + )) + .await + .unwrap() + .unwrap(); + assert_eq!( + row.try_get::("", "digest").unwrap(), + "sha256:original" + ); + assert_eq!( + row.try_get::("", "artifact_path").unwrap(), + "/old/baseline" + ); + assert_eq!( + row.try_get::>("", "group_name").unwrap(), + None + ); + Migrator::down(&db, Some(1)).await.unwrap(); + let row = db + .query_one_raw(Statement::from_string( + DatabaseBackend::Sqlite, + "SELECT digest FROM snapshot_index", + )) + .await + .unwrap() + .unwrap(); + assert_eq!( + row.try_get::("", "digest").unwrap(), + "sha256:original" + ); + } + + #[tokio::test] + async fn permits_duplicate_imports_and_refuses_lossy_downgrade() { + let db = prior_database().await; + Migrator::up(&db, None).await.unwrap(); + for group in ["first", "second"] { + db.execute_unprepared(&format!("INSERT INTO snapshot_index ({SHARED_COLUMNS}, group_name, group_path) SELECT digest, snapshot_id, descriptor_digest, name, parent_digest, scope, state_kind, image_ref, image_manifest_digest, format, fstype, checkpoint_manifest_digest, '/snapshots/{group}/baseline', size_bytes, locality, storage_binding_id, availability, migration_state, migration_error_code, created_at, indexed_at, child_count, '{group}', '/snapshots/{group}' FROM snapshot_index WHERE artifact_path = '/old/baseline'")).await.unwrap(); + } + let error = Migrator::down(&db, Some(1)).await.unwrap_err(); + assert!( + error + .to_string() + .contains("snapshot groups prevent downgrade") + ); + let row = db + .query_one_raw(Statement::from_string( + DatabaseBackend::Sqlite, + "SELECT COUNT(*) AS n FROM snapshot_index", + )) + .await + .unwrap() + .unwrap(); + assert_eq!(row.try_get::("", "n").unwrap(), 3); + } +} diff --git a/crates/migration/lib/schema_metadata.rs b/crates/migration/lib/schema_metadata.rs index d27d56f79..e4a599081 100644 --- a/crates/migration/lib/schema_metadata.rs +++ b/crates/migration/lib/schema_metadata.rs @@ -54,6 +54,9 @@ pub const MOUNT_OWNER_CONFIG_MIGRATION_ID: &str = "m20260824_000001_mount_owner_ /// Migration that separates stable snapshot identity from descriptor integrity. pub const SNAPSHOT_IDENTITY_MIGRATION_ID: &str = "m20260829_000001_split_snapshot_identity"; +/// Migration that separates local group membership from portable snapshot identity. +pub const SNAPSHOT_GROUPS_MIGRATION_ID: &str = "m20260910_000001_snapshot_groups"; + /// Frozen migration baseline for the transitional 0.6.0 release. /// /// The released 0.6.0 binary predates `msb __schema-baseline --json`, so @@ -265,6 +268,13 @@ pub const MIGRATION_METADATA: &[MigrationMetadata] = &[ affects_user_data: true, summary: "reverse final snapshot descriptors before dropping identity projections", }, + MigrationMetadata { + id: SNAPSHOT_GROUPS_MIGRATION_ID, + reversible: true, + affects_cache: false, + affects_user_data: true, + summary: "restore the flat snapshot index only when no groups or duplicate identities remain", + }, ]; //-------------------------------------------------------------------------------------------------- @@ -351,6 +361,7 @@ mod tests { #[test] fn canonical_applied_prefix_uses_metadata_order() { let applied = [ + SNAPSHOT_GROUPS_MIGRATION_ID, SNAPSHOT_IDENTITY_MIGRATION_ID, MOUNT_OWNER_CONFIG_MIGRATION_ID, SANDBOX_NETWORK_SLOT_MIGRATION_ID, diff --git a/crates/protocol/VERSIONING.md b/crates/protocol/VERSIONING.md index 9a265c37f..764c31f11 100644 --- a/crates/protocol/VERSIONING.md +++ b/crates/protocol/VERSIONING.md @@ -196,6 +196,10 @@ body = CBOR { v, t, p } <- ordinary control envelope Generation 9 adds attempt-scoped workload freeze/thaw for full checkpoint capture and activation. Hosts reject those operations against generation-8 agents before sending. Generation 8's released bulk-transfer contract remains unchanged; the discarded, unreleased freeze/thaw assignment to generation 8 has no compatibility shim. +The unreleased generation-9 handshake includes complete-frame transport boundaries and `core.workload.transport.credit`. The bundled host and guest use the optional Ready capability `workload_transport_barrier_version: 2`; an absent or unsupported value refuses full capture/pause before mutation. The superseded development contract `1` charged stdin against command capacity and is refused on full restore, not translated. This internal contract does not change SDK framing or add a socket. Older SDK requests still use their negotiated generation; their payloads are not reinterpreted to implement the barrier. + +The host gates ordinary input, finishes any admitted frame, and sends its cumulative control/data wire-byte and frame positions through a bounded private lifecycle queue. The existing `bulk_*` fields count logical data: raw bulk, stdin, inline filesystem/TCP payloads, and ordered EOF, regardless of physical port. Command metadata uses separate `control_*` capacity. The guest retains accepted input independently of blocked consumers, freezes workloads, and parks output at complete frames. Frozen reports the dedicated bulk output cut and absolute input grants; combined transport orders its output on the primary stream instead. Continue releases source-owned queued input only after Thawed. Restore carries the existing cumulative counters and retained input debt forward instead of granting a fresh window. Unrelated admitted metadata may bypass credit-blocked data, but per-correlation and client-disconnect ordering are retained. Incomplete boundaries time out without authorizing capture. These required fields finalize unreleased generation 9 in place; superseded development full snapshots are refused, not translated. + Generation 8 adds one negotiated data-body alternative without changing the header: ``` diff --git a/crates/protocol/lib/core.rs b/crates/protocol/lib/core.rs index 3dce49f7b..46e86bdfd 100644 --- a/crates/protocol/lib/core.rs +++ b/crates/protocol/lib/core.rs @@ -4,6 +4,27 @@ use serde::{Deserialize, Serialize}; use crate::transport::{BulkTransportReady, LocalTransportReady, RelayLeaseReady}; +//-------------------------------------------------------------------------------------------------- +// Constants +//-------------------------------------------------------------------------------------------------- + +/// Complete-frame workload barrier with logical control/data admission classes. +/// +/// Version 1 was an unreleased development contract that charged stdin to control. Its captured +/// debt cannot be reinterpreted by this contract; full restore rejects that development state. +pub const WORKLOAD_TRANSPORT_BARRIER_VERSION: u8 = 2; +/// Maximum outstanding command/control wire bytes, including frame headers. +pub const WORKLOAD_TRANSPORT_CONTROL_BYTES: u64 = 8 * 1024 * 1024; +/// Maximum outstanding command/control frames, excluding retained workload payloads. +pub const WORKLOAD_TRANSPORT_CONTROL_FRAMES: u64 = 256; +/// Maximum outstanding data wire bytes, including raw bulk, stdin and inline FS/TCP payloads. +pub const WORKLOAD_TRANSPORT_BULK_BYTES: u64 = 32 * 1024 * 1024; +/// Maximum outstanding data records/messages, including ordered empty EOF messages. +/// +/// Together with control frames, this fits the existing 512-entry guest input +/// queues even when all admitted traffic targets one stalled consumer. +pub const WORKLOAD_TRANSPORT_BULK_FRAMES: u64 = 256; + //-------------------------------------------------------------------------------------------------- // Types //-------------------------------------------------------------------------------------------------- @@ -50,6 +71,13 @@ pub struct Ready { /// Agentd leaves this absent because local shared memory is below the guest protocol. #[serde(default, skip_serializing_if = "Option::is_none")] pub local_transport: Option, + + /// Internal host-to-guest complete-frame barriers and aggregate input credit. + /// + /// Absence does not change ordinary generation-8 clients. Full capture and + /// pause require the supported contract instead of assuming frame safety. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workload_transport_barrier_version: Option, } /// Payload for `core.clock.sync` messages. @@ -99,6 +127,8 @@ pub struct Touched { pub struct WorkloadFreeze { /// Stable checkpoint attempt identity selected by the host. pub attempt_id: String, + /// Complete ordinary frames admitted by the host before gating user input. + pub host_input: WorkloadTransportPosition, } /// Payload for `core.workload.frozen` messages. @@ -106,6 +136,48 @@ pub struct WorkloadFreeze { pub struct WorkloadFrozen { /// Attempt identity whose workload boundary is now frozen. pub attempt_id: String, + /// Complete dedicated bulk wire bytes emitted before the guest writer parked. + /// + /// Zero for combined transport, whose primary stream already orders output + /// before this acknowledgement. The host drains to this cut before pausing. + pub guest_bulk_bytes_target: u64, + /// Absolute input limits captured with this boundary, not a fresh window. + pub input_credit: WorkloadTransportCredit, +} + +/// Cumulative ordinary input admitted at complete frame or record boundaries. +/// +/// These counters survive restore. Guest-accepted input is captured guest state; +/// host-queued input that has not been admitted remains source-owned. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct WorkloadTransportPosition { + /// Command/control wire bytes admitted, including length/header bytes. + /// Payload-bearing messages and raw bulk use `bulk_bytes` on either physical port. + pub control_bytes: u64, + /// Command/control frames admitted, excluding payload messages and raw bulk. + pub control_frames: u64, + /// Data wire bytes admitted, including stdin, inline payloads, and complete raw bulk headers. + pub bulk_bytes: u64, + /// Data records/messages admitted, including ordered EOF. + pub bulk_frames: u64, +} + +/// Absolute aggregate input grants in `core.workload.transport.credit`. +/// +/// Grants advance only as guest consumers release admitted input. Updates may be +/// coalesced; applying one twice never grants additional capacity. Both byte and +/// frame limits bound retained data without making lifecycle progress depend on +/// a workload consuming stdin or a network socket becoming writable. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct WorkloadTransportCredit { + /// Cumulative command/control wire-byte limit. + pub control_bytes: u64, + /// Cumulative command/control frame limit. + pub control_frames: u64, + /// Cumulative data wire-byte limit across both physical ports. + pub bulk_bytes: u64, + /// Cumulative data record/message limit across both physical ports. + pub bulk_frames: u64, } /// Payload for `core.workload.thaw` messages. @@ -167,6 +239,32 @@ pub struct CoreError { /// Wire message type involved in the error, when it could be determined. #[serde(default, skip_serializing_if = "Option::is_none")] pub offending_type: Option, + + /// Attempt-scoped freezer disposition. Absence is ambiguous, not proof that no work froze. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workload_failure: Option, +} + +/// Additional recovery information for a workload control error. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WorkloadFailure { + /// Attempt whose request failed. + pub attempt_id: String, + /// Whether a freeze was rejected before any freezer operation or needs recovery. + pub disposition: WorkloadFailureDisposition, +} + +/// Freezer failure dispositions; unknown future values never authorize a fallback. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WorkloadFailureDisposition { + /// No freezer exists and no freeze was attempted. + Unavailable, + /// The caller must obtain a confirmed thaw before treating the workload as running. + RecoveryRequired, + /// Unrecognized additional information from a newer agent. + #[serde(other)] + Unknown, } /// Machine-readable `core.error` categories. @@ -282,6 +380,7 @@ mod tests { bulk_transport: None, relay_lease: None, local_transport: None, + workload_transport_barrier_version: None, }; let mut legacy_bytes = Vec::new(); ciborium::into_writer(&legacy, &mut legacy_bytes).unwrap(); @@ -293,6 +392,7 @@ mod tests { assert!(decoded.bulk_transport.is_none()); assert!(decoded.relay_lease.is_none()); assert!(decoded.local_transport.is_none()); + assert!(decoded.workload_transport_barrier_version.is_none()); } #[test] @@ -319,3 +419,107 @@ mod tests { assert_eq!(decoded.incarnation, None); } } + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod workload_tests { + use super::*; + + #[test] + fn workload_barrier_payloads_roundtrip_without_resetting_counters() { + let position = WorkloadTransportPosition { + control_bytes: 73 * WORKLOAD_TRANSPORT_CONTROL_BYTES, + control_frames: 20_000, + bulk_bytes: 91 * WORKLOAD_TRANSPORT_BULK_BYTES, + bulk_frames: 30_000, + }; + let freeze = WorkloadFreeze { + attempt_id: "captured-generation".into(), + host_input: position, + }; + let mut bytes = Vec::new(); + ciborium::into_writer(&freeze, &mut bytes).unwrap(); + let decoded: WorkloadFreeze = ciborium::from_reader(bytes.as_slice()).unwrap(); + assert_eq!(decoded, freeze); + + // A restored guest may still own most of the window as pending stdin. + // Carry absolute grants, not a reset that would admit that much again. + let frozen = WorkloadFrozen { + attempt_id: freeze.attempt_id, + guest_bulk_bytes_target: 987_654_321, + input_credit: WorkloadTransportCredit { + control_bytes: position.control_bytes + 100, + control_frames: position.control_frames + 2, + bulk_bytes: position.bulk_bytes + 200, + bulk_frames: position.bulk_frames + 3, + }, + }; + bytes.clear(); + ciborium::into_writer(&frozen, &mut bytes).unwrap(); + let decoded: WorkloadFrozen = ciborium::from_reader(bytes.as_slice()).unwrap(); + assert_eq!(decoded, frozen); + } + + #[test] + fn superseded_development_freeze_payloads_do_not_imply_safe_boundaries() { + let old = serde_json::json!({"attempt_id":"old-development-capture"}); + assert!(serde_json::from_value::(old.clone()).is_err()); + assert!(serde_json::from_value::(old).is_err()); + } + + #[test] + fn unknown_barrier_capability_is_preserved_for_explicit_negotiation() { + let ready = Ready { + workload_transport_barrier_version: Some(99), + ..Ready::default() + }; + let mut bytes = Vec::new(); + ciborium::into_writer(&ready, &mut bytes).unwrap(); + let decoded: Ready = ciborium::from_reader(bytes.as_slice()).unwrap(); + assert_eq!(decoded.workload_transport_barrier_version, Some(99)); + assert_ne!( + decoded.workload_transport_barrier_version, + Some(WORKLOAD_TRANSPORT_BARRIER_VERSION) + ); + } + + #[test] + fn input_window_fits_a_maximum_primary_frame() { + let credit = WorkloadTransportCredit { + control_bytes: WORKLOAD_TRANSPORT_CONTROL_BYTES, + control_frames: WORKLOAD_TRANSPORT_CONTROL_FRAMES, + bulk_bytes: WORKLOAD_TRANSPORT_BULK_BYTES, + bulk_frames: WORKLOAD_TRANSPORT_BULK_FRAMES, + }; + assert!(credit.control_bytes >= crate::codec::MAX_FRAME_SIZE as u64 + 4); + assert!(credit.control_frames > 0); + assert!(credit.bulk_frames > 0); + } + + #[test] + fn freezer_error_details_are_additive_and_unknown_details_are_not_unavailable() { + let old = serde_json::json!({"kind":"capability_unavailable", "message":"freezer failed"}); + let decoded: CoreError = serde_json::from_value(old.clone()).unwrap(); + assert!(decoded.workload_failure.is_none()); + let mut new = old; + new["workload_failure"] = + serde_json::json!({"attempt_id":"a", "disposition":"future_state"}); + let decoded: CoreError = serde_json::from_value(new.clone()).unwrap(); + assert_eq!( + decoded.workload_failure.unwrap().disposition, + WorkloadFailureDisposition::Unknown + ); + + #[derive(Deserialize)] + struct OldCoreError { + kind: CoreErrorKind, + message: String, + } + let old_reader: OldCoreError = serde_json::from_value(new).unwrap(); + assert_eq!(old_reader.kind, CoreErrorKind::CapabilityUnavailable); + assert_eq!(old_reader.message, "freezer failed"); + } +} diff --git a/crates/protocol/lib/message.rs b/crates/protocol/lib/message.rs index f2b867bc1..d4ade57cb 100644 --- a/crates/protocol/lib/message.rs +++ b/crates/protocol/lib/message.rs @@ -153,6 +153,10 @@ pub enum MessageType { #[strum(serialize = "core.workload.thawed")] WorkloadThawed, + /// Guest grants cumulative ordinary input capacity to its host relay. + #[strum(serialize = "core.workload.transport.credit")] + WorkloadTransportCredit, + /// Host checks mounted root-filesystem growth before changing block capacity. #[strum(serialize = "core.root_disk.prepare")] RootDiskPrepare, @@ -317,6 +321,18 @@ impl Message { } impl MessageType { + /// Whether host-to-guest delivery can retain payload credit behind a workload consumer. + /// + /// The bundled workload barrier uses logical classes, not physical console ports. Payload + /// messages (including ordered EOF) share data credit with raw bulk records, leaving control + /// capacity available for fresh commands when restored stdin has not yet been consumed. + pub fn uses_workload_data_credit(self) -> bool { + matches!( + self, + Self::ExecStdin | Self::FsData | Self::TcpData | Self::TcpEof + ) + } + /// Computes the frame flags byte for this message type. pub fn flags(&self) -> u8 { match self { @@ -382,7 +398,8 @@ impl MessageType { Self::WorkloadFreeze | Self::WorkloadFrozen | Self::WorkloadThaw - | Self::WorkloadThawed => 9, + | Self::WorkloadThawed + | Self::WorkloadTransportCredit => 9, Self::RootDiskPrepare | Self::RootDiskGrow | Self::RootDiskState => 9, Self::BulkAccepted | Self::BulkCredit | Self::BulkFinish | Self::BulkCancel => 8, Self::TcpConnect @@ -454,6 +471,35 @@ impl<'de> Deserialize<'de> for MessageType { mod tests { use super::*; + #[test] + fn retained_payload_and_eof_use_data_credit_without_changing_frame_flags() { + for message in [ + MessageType::ExecStdin, + MessageType::FsData, + MessageType::TcpData, + MessageType::TcpEof, + ] { + assert!(message.uses_workload_data_credit()); + assert_eq!( + message.flags(), + 0, + "logical admission must not change the wire header" + ); + } + for message in [ + MessageType::ExecRequest, + MessageType::Ping, + MessageType::FsRequest, + MessageType::TcpConnect, + MessageType::ExecSignal, + MessageType::BulkFinish, + MessageType::BulkCancel, + MessageType::RelayClientDisconnected, + ] { + assert!(!message.uses_workload_data_credit()); + } + } + #[test] fn test_message_type_roundtrip() { let types = [ @@ -475,6 +521,10 @@ mod tests { (MessageType::WorkloadFrozen, "core.workload.frozen"), (MessageType::WorkloadThaw, "core.workload.thaw"), (MessageType::WorkloadThawed, "core.workload.thawed"), + ( + MessageType::WorkloadTransportCredit, + "core.workload.transport.credit", + ), (MessageType::CoreError, "core.error"), (MessageType::BulkAccepted, "core.bulk.accepted"), (MessageType::BulkCredit, "core.bulk.credit"), @@ -526,6 +576,7 @@ mod tests { MessageType::WorkloadFrozen, MessageType::WorkloadThaw, MessageType::WorkloadThawed, + MessageType::WorkloadTransportCredit, MessageType::CoreError, MessageType::BulkAccepted, MessageType::BulkCredit, @@ -743,6 +794,7 @@ mod tests { MessageType::WorkloadFrozen, MessageType::WorkloadThaw, MessageType::WorkloadThawed, + MessageType::WorkloadTransportCredit, ] { assert_eq!(mt.min_protocol_version(), 9, "{mt:?} should require gen 9"); } diff --git a/crates/protocol/schema/gen-9.json b/crates/protocol/schema/gen-9.json index ff254753d..c08c0e469 100644 --- a/crates/protocol/schema/gen-9.json +++ b/crates/protocol/schema/gen-9.json @@ -74,6 +74,10 @@ "introduced_in": 9, "wire": "core.workload.thawed" }, + { + "introduced_in": 9, + "wire": "core.workload.transport.credit" + }, { "introduced_in": 9, "wire": "core.root_disk.prepare" diff --git a/crates/runtime/Cargo.toml b/crates/runtime/Cargo.toml index 84a0b8bc2..be76c3855 100644 --- a/crates/runtime/Cargo.toml +++ b/crates/runtime/Cargo.toml @@ -68,7 +68,7 @@ tracing.workspace = true zeroize.workspace = true [target.'cfg(windows)'.dependencies] -windows-sys = { version = "0.61.2", features = ["Win32_Foundation", "Win32_Storage_FileSystem", "Win32_System_Kernel", "Win32_System_SystemInformation", "Win32_System_Threading"] } +windows-sys = { version = "0.61.2", features = ["Win32_Foundation", "Win32_Security", "Win32_Security_Authorization", "Win32_Storage_FileSystem", "Win32_System_Kernel", "Win32_System_SystemInformation", "Win32_System_Threading"] } [target.'cfg(unix)'.dependencies] microsandbox-agent-client = { workspace = true, features = ["uds"], optional = true } diff --git a/crates/runtime/lib/checkpoint/capture_pipeline.rs b/crates/runtime/lib/checkpoint/capture_pipeline.rs new file mode 100644 index 000000000..5df9700a5 --- /dev/null +++ b/crates/runtime/lib/checkpoint/capture_pipeline.rs @@ -0,0 +1,428 @@ +//! Bounded ownership transfer from the paused RAM reader to immutable-object writers. + +use std::io; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, mpsc}; +use std::thread::JoinHandle; +use std::time::Instant; + +use microsandbox_image::checkpoint::{ + CaptureObjectBatch, ContentRef, MemoryExtent, MemoryExtentContent, ObjectId, +}; +use msb_krun::{GuestMemoryRange, MemoryCaptureSink}; + +//-------------------------------------------------------------------------------------------------- +// Constants +//-------------------------------------------------------------------------------------------------- + +pub(super) const MEMORY_OBJECT_PACK_SIZE: usize = 32 * 1024 * 1024; +const WRITERS: usize = 2; +const BUFFER_COUNT: usize = WRITERS + 1; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +pub(super) struct MemoryObjectSink { + sender: Option>, + completed: mpsc::Receiver, + workers: Vec>, + cancelled: Arc, + pending: Pack, + free: Vec>, + updates: Vec, + in_flight: usize, + stats: MemoryPipelineStats, +} + +type PackWriter = dyn Fn(&[u8]) -> Result + Send + Sync; + +#[derive(Default)] +struct Pack { + bytes: Vec, + extents: Vec<(u64, u64, u64)>, +} + +struct CompletedPack { + pack: Pack, + object: Result, + persist_us: u128, +} + +#[derive(Clone, Copy, Debug, Default)] +pub(super) struct MemoryPipelineStats { + pub(super) wait_us: u128, + pub(super) persist_us: u128, + pub(super) packs: u64, + pub(super) peak_in_flight_bytes: usize, +} + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl MemoryObjectSink { + pub(super) fn new(batch: Arc) -> io::Result { + Self::with_writer(Arc::new(move |bytes| { + batch.put_bytes(bytes).map_err(|error| error.to_string()) + })) + } + + fn with_writer(write: Arc) -> io::Result { + let (sender, receiver) = mpsc::sync_channel::(WRITERS); + let receiver = Arc::new(Mutex::new(receiver)); + let (completed_sender, completed) = mpsc::channel(); + let cancelled = Arc::new(AtomicBool::new(false)); + let mut sink = Self { + sender: Some(sender), + completed, + workers: Vec::with_capacity(WRITERS), + cancelled, + pending: Pack { + bytes: Vec::with_capacity(MEMORY_OBJECT_PACK_SIZE), + extents: Vec::new(), + }, + free: (1..BUFFER_COUNT) + .map(|_| Vec::with_capacity(MEMORY_OBJECT_PACK_SIZE)) + .collect(), + updates: Vec::new(), + in_flight: 0, + stats: MemoryPipelineStats::default(), + }; + for index in 0..WRITERS { + let receiver = Arc::clone(&receiver); + let completed_sender = completed_sender.clone(); + let cancelled = Arc::clone(&sink.cancelled); + let write = Arc::clone(&write); + let worker = std::thread::Builder::new() + .name(format!("capture-pack-{index}")) + .spawn(move || { + loop { + // The queue mutex protects receive only; never hold it during hashing or I/O. + let Ok(pack) = receiver.lock().unwrap_or_else(|e| e.into_inner()).recv() + else { + break; + }; + let started = Instant::now(); + let object = if cancelled.load(Ordering::Acquire) { + Err("memory capture cancelled".to_string()) + } else { + // Always return a buffer/completion even on a panicking storage worker, + // so the producer cannot wait forever for an in-flight pack. + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + write(&pack.bytes) + })) + .map_err(|_| "memory object writer panicked".to_string()) + .and_then(|result| result) + }; + if object.is_err() { + cancelled.store(true, Ordering::Release); + } + if completed_sender + .send(CompletedPack { + pack, + object, + persist_us: started.elapsed().as_micros(), + }) + .is_err() + { + break; + } + } + })?; + sink.workers.push(worker); + } + Ok(sink) + } + + pub(super) fn finish(mut self) -> io::Result<(Vec, MemoryPipelineStats)> { + self.flush_pending()?; + self.sender.take(); + while self.in_flight != 0 { + self.receive()?; + } + self.join()?; + Ok((std::mem::take(&mut self.updates), self.stats)) + } + + fn flush_pending(&mut self) -> io::Result<()> { + if self.pending.bytes.is_empty() { + return Ok(()); + } + if self.cancelled.load(Ordering::Acquire) { + return Err(io::Error::other("memory object writer failed")); + } + // No borrowed guest-memory slice leaves write_bytes. At most three owned packs exist, + // including this producer's pack; a slow disk applies backpressure instead of allocating. + let pack = std::mem::take(&mut self.pending); + let started = Instant::now(); + self.sender + .as_ref() + .expect("capture is open") + .send(pack) + .map_err(|_| io::Error::other("memory object writers disconnected"))?; + self.stats.wait_us += started.elapsed().as_micros(); + self.in_flight += 1; + self.stats.packs += 1; + self.stats.peak_in_flight_bytes = self + .stats + .peak_in_flight_bytes + .max(self.in_flight * MEMORY_OBJECT_PACK_SIZE); + while self.free.is_empty() { + self.receive()?; + } + self.pending.bytes = self.free.pop().expect("received reusable buffer"); + Ok(()) + } + + fn receive(&mut self) -> io::Result<()> { + let started = Instant::now(); + let completed = self + .completed + .recv() + .map_err(|_| io::Error::other("memory object writers disconnected"))?; + self.stats.wait_us += started.elapsed().as_micros(); + self.in_flight -= 1; + self.stats.persist_us += completed.persist_us; + let object = completed.object.map_err(io::Error::other)?; + let mut pack = completed.pack; + self.updates.extend( + pack.extents + .drain(..) + .map(|(start, length, object_offset)| MemoryExtent { + start, + length, + content: MemoryExtentContent::Object(ContentRef { + object: object.clone(), + object_offset, + }), + }), + ); + pack.bytes.clear(); + self.free.push(pack.bytes); + Ok(()) + } + + fn join(&mut self) -> io::Result<()> { + let mut failed = false; + for worker in self.workers.drain(..) { + failed |= worker.join().is_err(); + } + if failed { + return Err(io::Error::other("memory object writer panicked")); + } + Ok(()) + } +} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +impl Drop for MemoryObjectSink { + fn drop(&mut self) { + self.cancelled.store(true, Ordering::Release); + self.sender.take(); + // Finish/drop cannot let a writer recreate staging files after failure cleanup starts. + let _ = self.join(); + } +} + +impl MemoryCaptureSink for MemoryObjectSink { + fn write_bytes(&mut self, range: GuestMemoryRange, bytes: &[u8]) -> io::Result<()> { + if bytes.len() as u64 != range.length() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "memory sink range length does not match bytes", + )); + } + // libkrun currently supplies <=2MiB ranges. Splitting also keeps the bound valid if a + // future caller supplies a larger range, without changing that range's guest projection. + let mut consumed = 0; + while consumed < bytes.len() { + if self.pending.bytes.len() == MEMORY_OBJECT_PACK_SIZE { + self.flush_pending()?; + } + let count = + (MEMORY_OBJECT_PACK_SIZE - self.pending.bytes.len()).min(bytes.len() - consumed); + let offset = self.pending.bytes.len() as u64; + self.pending + .bytes + .extend_from_slice(&bytes[consumed..consumed + count]); + self.pending + .extents + .push((range.start() + consumed as u64, count as u64, offset)); + consumed += count; + } + Ok(()) + } + + fn write_zero(&mut self, range: GuestMemoryRange) -> io::Result<()> { + self.updates.push(MemoryExtent { + start: range.start(), + length: range.length(), + content: MemoryExtentContent::Zero, + }); + Ok(()) + } +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use microsandbox_image::checkpoint::LocalObjectStore; + + #[test] + fn sparse_ranges_keep_exact_object_offsets() { + let dir = tempfile::tempdir().unwrap(); + let store = LocalObjectStore::open(dir.path()).unwrap(); + let batch = Arc::new(CaptureObjectBatch::new(store.clone(), &[])); + let mut sink = MemoryObjectSink::new(Arc::clone(&batch)).unwrap(); + sink.write_bytes(GuestMemoryRange::new(4096, 3).unwrap(), b"abc") + .unwrap(); + sink.write_zero(GuestMemoryRange::new(8192, 4).unwrap()) + .unwrap(); + sink.write_bytes(GuestMemoryRange::new(12288, 2).unwrap(), b"de") + .unwrap(); + let (mut extents, stats) = sink.finish().unwrap(); + batch.finish().unwrap(); + extents.sort_by_key(|extent| extent.start); + assert_eq!(stats.packs, 1); + assert!(matches!(extents[1].content, MemoryExtentContent::Zero)); + let MemoryExtentContent::Object(first) = &extents[0].content else { + panic!() + }; + let MemoryExtentContent::Object(last) = &extents[2].content else { + panic!() + }; + assert_eq!(first.object, last.object); + assert_eq!(last.object_offset, 3); + assert_eq!( + std::fs::read(store.object_path(&first.object)).unwrap(), + b"abcde" + ); + } + + #[test] + fn oversized_input_remains_bounded_and_storage_failure_joins_workers() { + let dir = tempfile::tempdir().unwrap(); + let store = LocalObjectStore::open(dir.path()).unwrap(); + let batch = Arc::new(CaptureObjectBatch::new(store, &[])); + let mut sink = MemoryObjectSink::new(Arc::clone(&batch)).unwrap(); + let bytes = vec![7; MEMORY_OBJECT_PACK_SIZE * 4 + 1]; + sink.write_bytes( + GuestMemoryRange::new(0, bytes.len() as u64).unwrap(), + &bytes, + ) + .unwrap(); + let (extents, stats) = sink.finish().unwrap(); + assert_eq!(stats.packs, 5); + assert_eq!( + extents.iter().map(|extent| extent.length).sum::(), + bytes.len() as u64 + ); + assert!(stats.peak_in_flight_bytes <= BUFFER_COUNT * MEMORY_OBJECT_PACK_SIZE); + + let bad = LocalObjectStore::open(dir.path().join("bad")).unwrap(); + std::fs::remove_dir_all(dir.path().join("bad/objects")).unwrap(); + std::fs::write(dir.path().join("bad/objects"), b"not a directory").unwrap(); + let batch = Arc::new(CaptureObjectBatch::new(bad, &[])); + let mut sink = MemoryObjectSink::new(Arc::clone(&batch)).unwrap(); + sink.write_bytes(GuestMemoryRange::new(0, 1).unwrap(), b"x") + .unwrap(); + assert!(sink.finish().is_err()); + assert_eq!( + Arc::strong_count(&batch), + 1, + "all writer references must be joined" + ); + } + + #[test] + fn panicking_writer_returns_an_error_instead_of_stranding_a_pack() { + let mut sink = + MemoryObjectSink::with_writer(Arc::new(|_| panic!("injected pack writer panic"))) + .unwrap(); + sink.write_bytes(GuestMemoryRange::new(0, 1).unwrap(), b"x") + .unwrap(); + assert!(sink.finish().unwrap_err().to_string().contains("panicked")); + } + + #[test] + fn dropping_capture_waits_until_active_writes_have_finished() { + use std::sync::atomic::AtomicUsize; + let active = Arc::new(AtomicUsize::new(0)); + let barrier = Arc::new(std::sync::Barrier::new(2)); + let (started_sender, started) = mpsc::channel(); + let writer_active = Arc::clone(&active); + let writer_barrier = Arc::clone(&barrier); + let mut sink = MemoryObjectSink::with_writer(Arc::new(move |bytes| { + writer_active.fetch_add(1, Ordering::SeqCst); + started_sender.send(()).unwrap(); + writer_barrier.wait(); + let id = ObjectId::from_bytes(bytes).map_err(|error| error.to_string()); + writer_active.fetch_sub(1, Ordering::SeqCst); + id + })) + .unwrap(); + sink.write_bytes(GuestMemoryRange::new(0, 1).unwrap(), b"x") + .unwrap(); + sink.flush_pending().unwrap(); + started.recv().unwrap(); + let cancelled = Arc::clone(&sink.cancelled); + let dropping = std::thread::spawn(move || drop(sink)); + while !cancelled.load(Ordering::Acquire) { + std::thread::yield_now(); + } + assert_eq!(active.load(Ordering::SeqCst), 1); + barrier.wait(); + dropping.join().unwrap(); + assert_eq!(active.load(Ordering::SeqCst), 0); + } + + #[test] + fn out_of_order_writers_preserve_each_packs_guest_projection() { + let (release_first, wait_first) = mpsc::channel(); + let wait_first = Mutex::new(wait_first); + let mut sink = MemoryObjectSink::with_writer(Arc::new(move |bytes| { + if bytes == b"a" { + wait_first.lock().unwrap().recv().unwrap(); + } + ObjectId::from_bytes(bytes).map_err(|error| error.to_string()) + })) + .unwrap(); + sink.write_bytes(GuestMemoryRange::new(4096, 1).unwrap(), b"a") + .unwrap(); + sink.flush_pending().unwrap(); + sink.write_bytes(GuestMemoryRange::new(8192, 1).unwrap(), b"b") + .unwrap(); + sink.flush_pending().unwrap(); + sink.receive().unwrap(); + assert_eq!( + sink.updates[0].start, 8192, + "the later pack must finish first in this test" + ); + release_first.send(()).unwrap(); + let (extents, _) = sink.finish().unwrap(); + assert_eq!( + extents + .iter() + .map(|extent| extent.start) + .collect::>(), + vec![8192, 4096] + ); + for (extent, bytes) in extents.iter().zip([b"b", b"a"]) { + let MemoryExtentContent::Object(content) = &extent.content else { + panic!() + }; + assert_eq!(content.object, ObjectId::from_bytes(bytes).unwrap()); + assert_eq!(content.object_offset, 0); + } + // The coordinator's overlay_extents sorts and validates these projections before any + // canonical manifest is encoded. Worker completion ordering is never artifact ordering. + } +} diff --git a/crates/runtime/lib/checkpoint/coordinator.rs b/crates/runtime/lib/checkpoint/coordinator.rs index ea58e49bb..8674d1fbe 100644 --- a/crates/runtime/lib/checkpoint/coordinator.rs +++ b/crates/runtime/lib/checkpoint/coordinator.rs @@ -2,27 +2,29 @@ use std::collections::{BTreeMap, BTreeSet}; use std::fmt; -use std::io; +use std::io::{self, Read}; use std::path::{Path, PathBuf}; +use std::sync::Arc; use std::time::{Duration, Instant}; use microsandbox_agent_client::AgentClient; use microsandbox_image::checkpoint::{ - CaptureIntent, CheckpointManifest, ContentRef, DeviceStateRef, LocalObjectStore, - MemoryCaptureMode, MemoryExtent, MemoryExtentContent, MemoryManifest, ObjectId, - ResourceDescriptor, ResourceTreatment, + AdmittedObject, CaptureIntent, CaptureObjectBatch, CheckpointManifest, ContentRef, + DeviceStateRef, LocalObjectStore, MemoryCaptureMode, MemoryExtent, MemoryExtentContent, + MemoryManifest, ObjectId, ResourceDescriptor, ResourceTreatment, }; use microsandbox_protocol::bootstrap::GuestBootstrap; use microsandbox_protocol::core::{ - CoreError, Ready, WorkloadFreeze, WorkloadFrozen, WorkloadThaw, WorkloadThawed, + CoreError, CoreErrorKind, Ready, WorkloadFailureDisposition, WorkloadFreeze, WorkloadFrozen, + WorkloadThaw, WorkloadThawed, WorkloadTransportCredit, WorkloadTransportPosition, }; use microsandbox_protocol::message::{Message, MessageType}; -use msb_krun::{ - GuestMemoryRange, IncrementalCaptureDecision, MemoryCaptureOptions, MemoryCapturePlan, - MemoryCaptureSink, -}; +use msb_krun::{IncrementalCaptureDecision, MemoryCaptureOptions, MemoryCapturePlan}; +use super::capture_pipeline::{MEMORY_OBJECT_PACK_SIZE, MemoryObjectSink}; use super::disk::RuntimeOwnedRootDisk; +use super::local_memory::{LocalMemoryCapture, LocalMemoryPin}; +use crate::runner::workload_control::{InputGate, WorkloadControl}; use crate::vm::VmConfig; //-------------------------------------------------------------------------------------------------- @@ -38,7 +40,6 @@ pub(super) const TYPE_FS: u32 = 26; // object store. Independently pack non-zero ranges into larger immutable objects to amortize // hashing, fsync, directory publication, and restore-time object opens. const MEMORY_SCAN_CHUNK_SIZE: usize = 2 * 1024 * 1024; -const MEMORY_OBJECT_PACK_SIZE: usize = 32 * 1024 * 1024; const WORKLOAD_CONTROL_TIMEOUT: Duration = Duration::from_secs(10); //-------------------------------------------------------------------------------------------------- @@ -51,10 +52,17 @@ pub(crate) struct CheckpointCoordinator { store: LocalObjectStore, runtime: tokio::runtime::Handle, agent_sock: PathBuf, + workload_control: Arc, root_disk: Option, fs_resource_bindings: BTreeMap>, network_resource_binding: Option, previous_memory: Option, + previous_memory_objects: Vec, + memory_cache: Option, + cached_baseline: Option<(MemoryManifest, super::CachedMemory)>, + local_cache_root: Option, + local_baseline: Option, + boot_geometry: (u8, u8, u32, u32), } /// Published checkpoint identity returned to the control executor. @@ -72,6 +80,7 @@ pub(crate) struct CheckpointResult { #[derive(Debug)] pub(crate) struct CheckpointFailure { message: String, + freezer_unavailable: bool, pub(crate) keep_paused: bool, pub(crate) published: Option>, } @@ -84,7 +93,9 @@ struct AdmittedResources { struct PausedCapture { result: CheckpointResult, memory_plan: MemoryCapturePlan, - memory_manifest: MemoryManifest, + memory_manifest: Option, + memory_objects: Vec, + local_memory: Option, timings: PausedCaptureTimings, } @@ -98,26 +109,33 @@ struct PausedCaptureTimings { extent_overlay_us: u128, memory_manifest_us: u128, checkpoint_publish_us: u128, + pipeline_wait_us: u128, + object_persist_worker_us: u128, + object_packs: u64, + peak_in_flight_bytes: usize, + object_hashed_bytes: u64, + object_linked_bytes: u64, + object_copied_bytes: u64, + object_directory_syncs: u64, } struct FrozenWorkload { - client: AgentClient, + gate: InputGate, attempt_id: String, protocol_generation: u8, ready: Ready, + host_input: WorkloadTransportPosition, + input_credit: WorkloadTransportCredit, + guest_bulk_bytes: u64, } -struct MemoryObjectSink<'a> { - store: &'a LocalObjectStore, - updates: Vec, - pending_bytes: Vec, - pending_extents: Vec, -} - -struct PendingMemoryExtent { - start: u64, - length: u64, - object_offset: u64, +/// Executor-owned resident pause. A recovery pause never acquires this public resume authority. +pub(crate) struct UserPause { + generation: msb_krun::VmPauseGeneration, + workload: Option, + // A kernel-only resident pause still keeps unadmitted host input source-owned. + input_gate: Option, + pub(crate) capture_unavailable: Option, } struct PendingDeviceState { @@ -131,6 +149,89 @@ struct PendingDeviceState { //-------------------------------------------------------------------------------------------------- impl CheckpointCoordinator { + /// Establish a resident, user-owned pause without requiring snapshot resource admission. + pub(crate) fn pause_user( + &self, + vm: &msb_krun::VmControl, + attempt_id: &str, + ) -> Result { + if !vm.clock_sync_supported() { + return Err(CheckpointFailure::before_pause( + "guest kernel lacks clock-only resume support", + )); + } + let (workload, capture_unavailable) = match self.freeze_workload(vm, attempt_id) { + Ok(workload) => (Some(workload), None), + Err(error) if error.freezer_unavailable => (None, Some(error.to_string())), + Err(error) => return Err(error), + }; + let input_gate = if workload.is_none() { + Some( + self.gate_input(Instant::now() + WORKLOAD_CONTROL_TIMEOUT)? + .0, + ) + } else { + None + }; + match vm.pause() { + Ok(generation) => Ok(UserPause { + generation, + workload, + input_gate, + capture_unavailable, + }), + Err(error) => { + if let Some(workload) = workload { + return Err(recover_failed_freeze( + attempt_id, + error.to_string(), + || self.thaw_workload(&workload), + || vm.pause().map(|_| ()).map_err(|error| error.to_string()), + )); + } + if let Some(gate) = input_gate { + gate.release(); + } + Err(CheckpointFailure::before_pause(error)) + } + } + } + + /// Resume this exact resident VM, processing clock correction before releasing workloads. + pub(crate) fn resume_user( + &self, + vm: &msb_krun::VmControl, + paused: &UserPause, + ) -> Result<(), CheckpointFailure> { + paused.validate(vm).map_err(CheckpointFailure::paused)?; + let request = vm + .request_clock_sync() + .ok_or_else(|| CheckpointFailure::paused("clock-only resume request unavailable"))?; + vm.resume(paused.generation) + .map_err(CheckpointFailure::paused)?; + let result = if vm.wait_vm_generation_processed(request, WORKLOAD_CONTROL_TIMEOUT) + == Some(msb_krun::VmGenerationWaitOutcome::Processed) + { + match &paused.workload { + Some(workload) => self.thaw_workload(workload), + None => { + if let Some(gate) = &paused.input_gate { + gate.release(); + } + Ok(()) + } + } + } else { + Err("guest did not acknowledge resident resume clock correction".into()) + }; + result.map_err(|error| { + let pause_error = vm.pause().err(); + CheckpointFailure::paused(format!( + "resume recovery required: {error}; pause error: {pause_error:?}" + )) + }) + } + pub(crate) fn compact( &mut self, vm: &msb_krun::VmControl, @@ -229,6 +330,7 @@ impl CheckpointCoordinator { guest_bootstrap: &GuestBootstrap, runtime: tokio::runtime::Handle, agent_sock: &Path, + workload_control: Arc, ) -> Result { let root = runtime_dir.join("checkpoints"); std::fs::create_dir_all(&root).map_err(|error| error.to_string())?; @@ -249,20 +351,188 @@ impl CheckpointCoordinator { store, runtime, agent_sock: agent_sock.to_path_buf(), + workload_control, root_disk, fs_resource_bindings, network_resource_binding, previous_memory: None, + previous_memory_objects: Vec::new(), + memory_cache: if vm + .checkpoint_restore + .as_ref() + .is_some_and(|restore| restore.forked) + { + Some( + super::MemoryCache::open(vm.memory_cache_dir.as_ref().ok_or_else(|| { + "CoW memory requires its backend-resolved cache directory".to_string() + })?) + .map_err(|error| error.to_string())?, + ) + } else { + None + }, + cached_baseline: None, + local_cache_root: vm.memory_cache_dir.clone(), + local_baseline: None, + boot_geometry: (vm.vcpus, vm.max_cpus, vm.memory_mib, vm.max_memory_mib), }) } - /// Capture and publish one complete same-epoch checkpoint, then restore source execution. + /// Seal the owned disk at a crash-consistent cut without capturing RAM or guest execution. + pub(crate) fn capture_disk( + &mut self, + vm: &msb_krun::VmControl, + checkpoint_id: &str, + user_pause: Option<&UserPause>, + ) -> Result + { + use super::disk::RootDiskRolloverError as Failure; + let started = Instant::now(); + validate_checkpoint_id(checkpoint_id).map_err(Failure::pre_rebind)?; + if let Some(paused) = user_pause { + paused.validate(vm).map_err(Failure::pre_rebind)?; + } + let disk = self.root_disk.as_mut().ok_or_else(|| { + Failure::pre_rebind("disk-only capture requires an owned managed or flat root disk") + })?; + if disk.growth_pending() { + return Err(Failure::pre_rebind( + "complete pending root-disk growth before snapshotting", + )); + } + let path = self.root.join(checkpoint_id); + std::fs::create_dir(&path).map_err(Failure::pre_rebind)?; + let paused_at = Instant::now(); + let pause = match user_pause + .map(|p| Ok(p.generation)) + .unwrap_or_else(|| vm.pause()) + { + Ok(pause) => pause, + Err(error) => { + let _ = std::fs::remove_dir_all(&path); + return Err(Failure::pre_rebind(error)); + } + }; + // Only the root block worker is drained and switched. Rollover inspects its state, + // but no full CPU/device payload, RAM scan, guest handshake, or dirty-baseline update + // is needed. The result is a crash-consistent disk cut, not an execution checkpoint. + let result = disk.rollover(vm, &self.runtime, &path, pause.get()); + if user_pause.is_none() && !result.as_ref().is_err_and(|e| e.keep_paused) { + vm.resume(pause).map_err(Failure::post_journal)?; + } + let pause_us = paused_at.elapsed().as_micros(); + match result { + Ok(captured) => { + tracing::info!(target: "microsandbox_checkpoint_timing", operation = "capture_disk", + checkpoint_id, source_already_paused = user_pause.is_some(), pause_us, + total_us = started.elapsed().as_micros(), "disk-only checkpoint timing"); + Ok(crate::control::DiskCheckpointControlState { + checkpoint_id: checkpoint_id.into(), + path, + disk: captured.manifest, + }) + } + Err(error) => { + // The runtime's forward journal owns any committed new head. Only discard the + // unreturned immutable closure, never source layers or its recovery journal. + let _ = std::fs::remove_dir_all(&path); + Err(error) + } + } + } + + /// Capture a same-epoch full checkpoint while preserving prior execution state. pub(crate) fn capture( &mut self, vm: &msb_krun::VmControl, checkpoint_id: &str, intent: CaptureIntent, + user_pause: Option<&UserPause>, + ) -> Result { + self.capture_to(vm, checkpoint_id, intent, user_pause, None) + } + + /// Capture a local handoff directly, without publishing a portable RAM closure. + pub(crate) fn branch( + &mut self, + vm: &msb_krun::VmControl, + id: &str, + child_name: &str, + reserved_cache: &Path, + user_pause: Option<&UserPause>, ) -> Result { + let cache = self.local_cache_root.as_ref().ok_or_else(|| { + CheckpointFailure::before_pause("runtime has no backend-resolved memory cache") + })?; + // Reject unsupported hosts before freezing or rolling over the source disk. + super::MemoryCache::open_namespace(cache.clone(), "branches") + .map_err(CheckpointFailure::before_pause)?; + if std::fs::canonicalize(cache).map_err(CheckpointFailure::before_pause)? + != std::fs::canonicalize(reserved_cache).map_err(CheckpointFailure::before_pause)? + { + return Err(CheckpointFailure::before_pause( + "branch handoff cache differs from the source runtime; use the source's original backend cache configuration", + )); + } + validate_checkpoint_id(id).map_err(CheckpointFailure::before_pause)?; + microsandbox_types::validate_sandbox_name(child_name) + .map_err(CheckpointFailure::before_pause)?; + // The SDK reserves a fresh child directory under this same backend. Never accept + // caller-selected host paths, symlinked children, or an existing handoff destination. + let source = self.root.parent().and_then(Path::parent).ok_or_else(|| { + CheckpointFailure::before_pause("source storage has no sandbox parent") + })?; + let parent = source + .parent() + .ok_or_else(|| CheckpointFailure::before_pause("missing sandbox storage root"))?; + let child = parent.join(child_name); + if child == source + || !std::fs::symlink_metadata(&child).is_ok_and(|m| m.file_type().is_dir()) + { + return Err(CheckpointFailure::before_pause( + "branch requires a reserved child directory", + )); + } + let reservation = child.join(".branch-reservation"); + if !std::fs::symlink_metadata(&reservation) + .is_ok_and(|m| m.file_type().is_file() && m.len() <= 128) + || std::fs::read_to_string(&reservation).map_err(CheckpointFailure::before_pause)? != id + { + return Err(CheckpointFailure::before_pause( + "child reservation does not match branch attempt", + )); + } + let destination = child.join(".branch-restore"); + if std::fs::symlink_metadata(&destination).is_ok() { + return Err(CheckpointFailure::before_pause( + "child already has a branch handoff", + )); + } + self.capture_to( + vm, + id, + CaptureIntent::FullSnapshot, + user_pause, + Some(&destination), + ) + } + + fn capture_to( + &mut self, + vm: &msb_krun::VmControl, + checkpoint_id: &str, + intent: CaptureIntent, + user_pause: Option<&UserPause>, + local_destination: Option<&Path>, + ) -> Result { + if let Some(paused) = user_pause { + paused + .validate(vm) + .map_err(CheckpointFailure::before_pause)?; + if let Some(reason) = &paused.capture_unavailable { + return Err(CheckpointFailure::before_pause(reason)); + } + } if self .root_disk .as_ref() @@ -285,29 +555,43 @@ impl CheckpointCoordinator { .map_err(CheckpointFailure::before_pause)?; let admission_us = admission_started.elapsed().as_micros(); let staging_started = Instant::now(); - let final_path = self.root.join(checkpoint_id); + let final_path = local_destination + .map(Path::to_path_buf) + .unwrap_or_else(|| self.root.join(checkpoint_id)); if final_path.exists() { return Err(CheckpointFailure::before_pause( "checkpoint identity is already published", )); } - let staging = self.root.join(format!( - ".{checkpoint_id}.{}.staging", - rand::random::() - )); + let staging = final_path + .parent() + .ok_or_else(|| CheckpointFailure::before_pause("capture destination has no parent"))? + .join(format!( + ".{checkpoint_id}.{}.staging", + rand::random::() + )); std::fs::create_dir(&staging).map_err(CheckpointFailure::before_pause)?; let staging_us = staging_started.elapsed().as_micros(); // The guest latch is acquired while vCPUs can still service agentd. // It remains held in captured guest memory so a restored child cannot // run application code before VM Generation ID activation completes. + // An already-paused source borrows its original latch and token: even a brief resume + // here would invalidate the user's paused boundary and require another guest handshake. let workload_unavailable_started = Instant::now(); let freeze_started = Instant::now(); - let workload = match self.freeze_workload(checkpoint_id) { - Ok(workload) => workload, - Err(error) => { - let _ = std::fs::remove_dir_all(&staging); - return Err(CheckpointFailure::before_pause(error)); + let acquired_workload; + let workload = match user_pause { + Some(paused) => paused.workload.as_ref().expect("validated workload latch"), + None => { + acquired_workload = match self.freeze_workload(vm, checkpoint_id) { + Ok(workload) => workload, + Err(error) => { + let _ = std::fs::remove_dir_all(&staging); + return Err(error); + } + }; + &acquired_workload } }; let freeze_us = freeze_started.elapsed().as_micros(); @@ -315,11 +599,14 @@ impl CheckpointCoordinator { let vm_pause_window_started = Instant::now(); let pause_started = Instant::now(); - let pause = match vm.pause() { + let pause = match user_pause + .map(|paused| Ok(paused.generation)) + .unwrap_or_else(|| vm.pause()) + { Ok(pause) => pause, Err(error) => { let _ = std::fs::remove_dir_all(&staging); - return match self.thaw_workload(&workload) { + return match self.thaw_workload(workload) { Ok(()) => Err(CheckpointFailure::before_pause(error)), Err(thaw_error) => Err(CheckpointFailure::paused(format!( "VM pause failed: {error}; workload thaw failed: {thaw_error}" @@ -338,18 +625,21 @@ impl CheckpointCoordinator { pause.get(), &staging, &final_path, + local_destination.is_some(), ); let paused_capture_us = paused_capture_started.elapsed().as_micros(); let captured = match paused { Ok(captured) => captured, Err(mut failure) => { - if !failure.keep_paused + if user_pause.is_none() + && !failure.keep_paused && let Err(error) = vm.resume(pause) { failure.keep_paused = true; failure.message = format!("{}; source resume failed: {error}", failure.message); - } else if !failure.keep_paused - && let Err(error) = self.thaw_workload(&workload) + } else if user_pause.is_none() + && !failure.keep_paused + && let Err(error) = self.thaw_workload(workload) { failure.keep_paused = true; failure.message = format!("{}; workload thaw failed: {error}", failure.message); @@ -378,8 +668,11 @@ impl CheckpointCoordinator { }; let baseline_publish_us = baseline_started.elapsed().as_micros(); let resume_started = Instant::now(); - if let Err(error) = vm.resume(pause) { + if user_pause.is_none() + && let Err(error) = vm.resume(pause) + { return Err(CheckpointFailure { + freezer_unavailable: false, message: format!("checkpoint published but source resume failed: {error}"), keep_paused: true, published: Some(Box::new(captured.result)), @@ -388,7 +681,9 @@ impl CheckpointCoordinator { let resume_us = resume_started.elapsed().as_micros(); let vm_pause_window_us = vm_pause_window_started.elapsed().as_micros(); let thaw_started = Instant::now(); - if let Err(error) = self.thaw_workload(&workload) { + if user_pause.is_none() + && let Err(error) = self.thaw_workload(workload) + { let repause = vm.pause().err(); let message = match repause { Some(pause_error) => format!( @@ -397,6 +692,7 @@ impl CheckpointCoordinator { None => format!("checkpoint published but workload thaw failed: {error}"), }; return Err(CheckpointFailure { + freezer_unavailable: false, message, keep_paused: true, published: Some(Box::new(captured.result)), @@ -404,14 +700,65 @@ impl CheckpointCoordinator { } let thaw_us = thaw_started.elapsed().as_micros(); let workload_unavailable_us = workload_unavailable_started.elapsed().as_micros(); + if let (Some(cache), Some(memory_manifest)) = + (&self.memory_cache, &captured.memory_manifest) + { + // Source execution has resumed (unless explicitly user-paused). Read only the + // completed immutable capture, never live RAM, while preparing child acceleration. + let prepared = (|| -> Result { + let bytes = memory_manifest + .to_canonical_bytes() + .map_err(|e| e.to_string())?; + let identity = ObjectId::from_bytes(&bytes).map_err(|e| e.to_string())?; + cache + .materialize_with_baseline( + memory_manifest, + &identity, + self.cached_baseline + .as_ref() + .map(|(manifest, cached)| (manifest, cached)), + |id| { + let mut bytes = Vec::new(); + std::fs::File::open(self.store.object_path(id))? + .take(MEMORY_OBJECT_PACK_SIZE as u64 + 1) + .read_to_end(&mut bytes)?; + if bytes.len() > MEMORY_OBJECT_PACK_SIZE + || ObjectId::from_bytes(&bytes).map_err(io::Error::other)? != *id + { + return Err(io::Error::other( + "memory object failed size/identity validation", + )); + } + Ok(bytes) + }, + ) + .map_err(|e| e.to_string()) + })(); + match prepared { + Ok(cached) => { + tracing::info!(target: "microsandbox_checkpoint_timing", operation = "memory_cache", prepare_us = cached.prepare_us, cache_hit = cached.cache_hit, reflink = cached.reflink, "prepared immutable capture cache"); + self.cached_baseline = Some((memory_manifest.clone(), cached)); + } + Err(error) => { + // Publication already succeeded. Losing optional acceleration does not + // erase the artifact or turn its successful capture into a false failure. + tracing::warn!(%error, "checkpoint published without memory cache acceleration"); + } + } + } if baseline_published { - self.previous_memory = Some(captured.memory_manifest); + self.previous_memory = captured.memory_manifest; + self.previous_memory_objects = captured.memory_objects; + self.local_baseline = captured.local_memory; } else { self.previous_memory = None; + self.previous_memory_objects.clear(); + self.local_baseline = None; } tracing::info!( target: "microsandbox_checkpoint_timing", operation = "capture", + source_already_paused = user_pause.is_some(), checkpoint_id, memory_mode = ?captured.result.memory_mode, memory_logical_bytes = captured.result.memory_logical_bytes, @@ -430,6 +777,14 @@ impl CheckpointCoordinator { extent_overlay_us = captured.timings.extent_overlay_us, memory_manifest_us = captured.timings.memory_manifest_us, checkpoint_publish_us = captured.timings.checkpoint_publish_us, + pipeline_wait_us = captured.timings.pipeline_wait_us, + object_persist_worker_us = captured.timings.object_persist_worker_us, + object_packs = captured.timings.object_packs, + peak_in_flight_bytes = captured.timings.peak_in_flight_bytes, + object_hashed_bytes = captured.timings.object_hashed_bytes, + object_linked_bytes = captured.timings.object_linked_bytes, + object_copied_bytes = captured.timings.object_copied_bytes, + object_directory_syncs = captured.timings.object_directory_syncs, baseline_publish_us, resume_us, thaw_us, @@ -440,44 +795,121 @@ impl CheckpointCoordinator { Ok(captured.result) } - fn freeze_workload(&self, attempt_id: &str) -> Result { - let client = self - .runtime - .block_on(AgentClient::connect_with_timeout( - &self.agent_sock, - WORKLOAD_CONTROL_TIMEOUT, - )) - .map_err(|error| format!("connect workload latch: {error}"))?; + fn freeze_workload( + &self, + vm: &msb_krun::VmControl, + attempt_id: &str, + ) -> Result { + // These are the bundled guest's capabilities, not a newly connected SDK client's + // generation. Internal lifecycle work must not join the FIFO it is about to gate. + let (protocol_generation, ready) = self + .workload_control + .ready() + .map_err(CheckpointFailure::before_pause)?; + if !MessageType::WorkloadFreeze.is_available_at(protocol_generation) { + return Err(CheckpointFailure::before_pause( + "guest protocol does not support workload freeze", + )); + } + let deadline = Instant::now() + WORKLOAD_CONTROL_TIMEOUT; + let (gate, host_input) = self.gate_input(deadline)?; + let mut workload = FrozenWorkload { + gate, + attempt_id: attempt_id.to_string(), + protocol_generation, + ready, + host_input, + input_credit: WorkloadTransportCredit::default(), + guest_bulk_bytes: 0, + }; let request = WorkloadFreeze { attempt_id: attempt_id.to_string(), + host_input, + }; + let message = match Message::with_payload(MessageType::WorkloadFreeze, 0, &request) { + Ok(message) => message, + Err(error) => { + // No lifecycle request has been admitted, so ordinary input can safely resume. + workload.gate.release(); + return Err(CheckpointFailure::before_pause(error)); + } }; let reply = self .runtime .block_on(async { - tokio::time::timeout( - WORKLOAD_CONTROL_TIMEOUT, - client.request(MessageType::WorkloadFreeze, &request), + tokio::time::timeout_at( + deadline.into(), + self.workload_control.request(message, attempt_id), ) .await }) - .map_err(|_| "workload freeze timed out".to_string())? - .map_err(|error| format!("request workload freeze: {error}"))?; - validate_workload_reply::( - reply, - MessageType::WorkloadFrozen, - attempt_id, - |payload| &payload.attempt_id, - )?; - let protocol_generation = client.negotiated_version(); - let ready = client - .ready() - .map_err(|error| format!("read workload-agent identity: {error}"))?; - Ok(FrozenWorkload { - client, - attempt_id: attempt_id.to_string(), - protocol_generation, - ready, - }) + .map_err(|_| "workload freeze timed out".to_string()) + .and_then(|reply| reply.map_err(|error| format!("request workload freeze: {error}"))); + if let Ok(reply) = &reply + && let Some(reason) = unavailable_freezer_reason(reply, attempt_id) + { + // Explicit guest evidence says the freezer was never attempted. + workload.gate.release(); + let mut error = CheckpointFailure::before_pause(reason); + error.freezer_unavailable = true; + return Err(error); + } + let result = reply.and_then(|reply| { + let frozen = validate_workload_reply::( + reply, + MessageType::WorkloadFrozen, + attempt_id, + |payload| &payload.attempt_id, + )?; + self.workload_control.update_credit(frozen.input_credit)?; + self.runtime + .block_on(async { + tokio::time::timeout_at( + deadline.into(), + self.workload_control + .wait_bulk_cut(frozen.guest_bulk_bytes_target), + ) + .await + }) + .map_err(|_| { + "guest output did not reach the frozen transport boundary".to_string() + })??; + Ok(frozen) + }); + let frozen = result.map_err(|error| { + recover_failed_freeze( + attempt_id, + error, + || self.thaw_workload(&workload), + || vm.pause().map(|_| ()).map_err(|error| error.to_string()), + ) + })?; + workload.input_credit = frozen.input_credit; + workload.guest_bulk_bytes = frozen.guest_bulk_bytes_target; + Ok(workload) + } + + /// Park both ordinary writers at complete records before taking their cumulative cut. + fn gate_input( + &self, + deadline: Instant, + ) -> Result<(InputGate, WorkloadTransportPosition), CheckpointFailure> { + let gate = self.workload_control.gate(); + let result = self + .runtime + .block_on(async { + tokio::time::timeout_at(deadline.into(), self.workload_control.parked_position()) + .await + }) + .map_err(|_| "host input did not reach a complete transport boundary".to_string()) + .and_then(|result| result); + match result { + Ok(position) => Ok((gate, position)), + Err(error) => { + gate.release(); + Err(CheckpointFailure::before_pause(error)) + } + } } fn thaw_workload(&self, workload: &FrozenWorkload) -> Result<(), String> { @@ -485,12 +917,14 @@ impl CheckpointCoordinator { attempt_id: workload.attempt_id.clone(), mode: microsandbox_protocol::core::WorkloadThawMode::Continue, }; + let message = Message::with_payload(MessageType::WorkloadThaw, 0, &request) + .map_err(|error| error.to_string())?; let reply = self .runtime .block_on(async { tokio::time::timeout( WORKLOAD_CONTROL_TIMEOUT, - workload.client.request(MessageType::WorkloadThaw, &request), + self.workload_control.request(message, &workload.attempt_id), ) .await }) @@ -501,7 +935,11 @@ impl CheckpointCoordinator { MessageType::WorkloadThawed, &workload.attempt_id, |payload| &payload.attempt_id, - ) + )?; + // Only acknowledged thaw releases the source-owned FIFO. Dropping a failed capture + // without reaching here leaves the helper fenced instead of implicitly flushing input. + workload.gate.release(); + Ok(()) } #[allow(clippy::too_many_arguments)] @@ -515,30 +953,21 @@ impl CheckpointCoordinator { pause_generation: u64, staging: &Path, final_path: &Path, + local: bool, ) -> Result { let mut timings = PausedCaptureTimings::default(); - let execution_started = Instant::now(); - let execution = vm - .capture_execution_state() - .map_err(CheckpointFailure::resumable)?; - if execution.pause_generation() != pause_generation { - return Err(CheckpointFailure::resumable( - "execution state belongs to another pause generation", - )); - } - let execution_bytes = execution.encode().map_err(CheckpointFailure::resumable)?; - let execution_id = self - .store - .put_bytes(&execution_bytes) - .map_err(CheckpointFailure::resumable)?; - self.store - .link_into(&execution_id, staging) - .map_err(CheckpointFailure::resumable)?; - timings.execution_us = execution_started.elapsed().as_micros(); - + let batch = Arc::new(CaptureObjectBatch::new( + self.store.clone(), + if local { + &[] + } else { + &self.previous_memory_objects + }, + )); let devices_started = Instant::now(); let mut pending_devices = Vec::with_capacity(inventory.len()); let mut disk_roots = Vec::new(); + let mut local_disks = Vec::new(); for (device_type, device_id) in inventory { let runtime_owned_root = self .root_disk @@ -554,23 +983,26 @@ impl CheckpointCoordinator { let rollover = disk .rollover(vm, &self.runtime, staging, pause_generation) .map_err(|error| CheckpointFailure { + freezer_unavailable: false, message: error.to_string(), keep_paused: error.keep_paused, published: None, })?; timings.managed_disk_us += disk_started.elapsed().as_micros(); - let manifest_bytes = rollover - .manifest - .to_canonical_bytes() - .map_err(CheckpointFailure::resumable)?; - let manifest_id = self - .store - .put_bytes(&manifest_bytes) - .map_err(CheckpointFailure::resumable)?; - self.store - .link_into(&manifest_id, staging) - .map_err(CheckpointFailure::resumable)?; - disk_roots.push(manifest_id); + if !local { + let manifest_bytes = rollover + .manifest + .to_canonical_bytes() + .map_err(CheckpointFailure::resumable)?; + let manifest_id = batch + .put_bytes(&manifest_bytes) + .map_err(CheckpointFailure::resumable)?; + batch + .link_into(&manifest_id, staging) + .map_err(CheckpointFailure::resumable)?; + disk_roots.push(manifest_id); + } + local_disks.push(rollover.manifest); rollover.device_state } else if *device_type == TYPE_BLOCK { vm.capture_block_device_state(device_id) @@ -612,19 +1044,143 @@ impl CheckpointCoordinator { bytes, }); } - let device_refs = persist_device_states(&self.store, staging, &pending_devices) - .map_err(CheckpointFailure::resumable)?; + let device_refs = if local { + pending_devices + .iter() + .map(|device| { + Ok(DeviceStateRef { + device_type: device.device_type, + device_id: device.device_id.clone(), + state: put_local_object(staging, &device.bytes)?, + }) + }) + .collect::, String>>() + } else { + persist_device_states(&batch, staging, &pending_devices) + } + .map_err(CheckpointFailure::resumable)?; timings.devices_us = devices_started.elapsed().as_micros(); + // Device capture parks each worker. Capture interrupt-controller state + // only after their final completions have been published; otherwise a + // used queue could survive in RAM without its corresponding interrupt. + // Execution capture must also precede RAM capture: KVM flushes its LPI + // pending tables into guest RAM as part of this operation. + let execution_started = Instant::now(); + let execution = vm + .capture_execution_state() + .map_err(CheckpointFailure::resumable)?; + if execution.pause_generation() != pause_generation { + return Err(CheckpointFailure::resumable( + "execution state belongs to another pause generation", + )); + } + let execution_bytes = execution.encode().map_err(CheckpointFailure::resumable)?; + let execution_id = if local { + put_local_object(staging, &execution_bytes).map_err(CheckpointFailure::resumable)? + } else { + let id = batch + .put_bytes(&execution_bytes) + .map_err(CheckpointFailure::resumable)?; + batch + .link_into(&id, staging) + .map_err(CheckpointFailure::resumable)?; + id + }; + timings.execution_us = execution_started.elapsed().as_micros(); + + if local { + let (memory_plan, incremental) = self + .plan_local_memory(vm) + .map_err(CheckpointFailure::resumable)?; + let captured = (|| { + let started = Instant::now(); + let mut sink = LocalMemoryCapture::new( + self.local_cache_root + .as_ref() + .expect("validated local cache"), + checkpoint_id, + if incremental { + self.local_baseline.as_ref() + } else { + None + }, + ) + .map_err(CheckpointFailure::resumable)?; + let reflink = sink.reflink; + let stats = vm + .capture_memory( + &memory_plan, + MemoryCaptureOptions::new(MEMORY_SCAN_CHUNK_SIZE, true) + .map_err(CheckpointFailure::resumable)?, + &mut sink, + ) + .map_err(CheckpointFailure::resumable)?; + let memory = sink + .finish(memory_plan.generation().get(), memory_plan.topology().get()) + .map_err(CheckpointFailure::resumable)?; + timings.memory_capture_us = started.elapsed().as_micros(); + let state = super::LocalBranchState { + id: checkpoint_id.into(), + architecture: std::env::consts::ARCH.into(), + pause_generation, + execution_state: execution_id, + devices: device_refs, + resources, + disks: local_disks, + memory: memory.memory.clone(), + vcpus: self.boot_geometry.0, + max_cpus: self.boot_geometry.1, + memory_mib: self.boot_geometry.2, + max_memory_mib: self.boot_geometry.3, + }; + let bytes = serde_json::to_vec(&state).map_err(CheckpointFailure::resumable)?; + // This handoff has no snapshot root or RAM object manifest. Child-owned disk + // links and bounded metadata are installed before acknowledging the capture. + std::fs::write(staging.join("branch.json"), bytes) + .map_err(CheckpointFailure::resumable)?; + std::fs::rename(staging, final_path).map_err(CheckpointFailure::resumable)?; + tracing::info!(target: "microsandbox_checkpoint_timing", operation = "local_memory_capture", incremental, reflink, capture_us = timings.memory_capture_us); + Ok((memory, stats)) + })(); + let (memory, stats) = match captured { + Ok(captured) => captured, + Err(error) => { + let _ = vm.abandon_memory_capture(&memory_plan); + return Err(error); + } + }; + return Ok(PausedCapture { + result: CheckpointResult { + checkpoint_id: checkpoint_id.into(), + checkpoint_root: String::new(), + path: final_path.into(), + memory_mode: if incremental { + MemoryCaptureMode::Incremental + } else { + MemoryCaptureMode::Full + }, + memory_logical_bytes: stats.logical_bytes, + memory_emitted_bytes: stats.emitted_bytes, + }, + memory_plan, + memory_manifest: None, + memory_objects: Vec::new(), + local_memory: Some(memory), + timings, + }); + } + let memory_plan_started = Instant::now(); let (memory_plan, memory_mode, base_extents) = self.plan_memory(vm).map_err(CheckpointFailure::resumable)?; timings.memory_plan_us = memory_plan_started.elapsed().as_micros(); - let mut sink = MemoryObjectSink { - store: &self.store, - updates: Vec::new(), - pending_bytes: Vec::with_capacity(MEMORY_OBJECT_PACK_SIZE), - pending_extents: Vec::new(), + let mut sink = match MemoryObjectSink::new(Arc::clone(&batch)) { + Ok(sink) => sink, + Err(error) => { + let _ = vm.abandon_memory_capture(&memory_plan); + return Err(CheckpointFailure::resumable(error)); + } }; let memory_capture_started = Instant::now(); let stats = match vm.capture_memory( @@ -635,18 +1191,24 @@ impl CheckpointCoordinator { ) { Ok(stats) => stats, Err(error) => { + // Stop/join queued writers before the caller can remove this capture's staging. + drop(sink); let _ = vm.abandon_memory_capture(&memory_plan); return Err(CheckpointFailure::resumable(error)); } }; - let updates = match sink.finish() { - Ok(updates) => updates, + let (updates, pipeline_stats) = match sink.finish() { + Ok(result) => result, Err(error) => { let _ = vm.abandon_memory_capture(&memory_plan); return Err(CheckpointFailure::resumable(error)); } }; timings.memory_capture_us = memory_capture_started.elapsed().as_micros(); + timings.pipeline_wait_us = pipeline_stats.wait_us; + timings.object_persist_worker_us = pipeline_stats.persist_us; + timings.object_packs = pipeline_stats.packs; + timings.peak_in_flight_bytes = pipeline_stats.peak_in_flight_bytes; let extent_overlay_started = Instant::now(); let extents = match overlay_extents(base_extents, updates) { Ok(extents) => extents, @@ -682,22 +1244,19 @@ impl CheckpointCoordinator { linked_memory_objects.insert(content.object.clone()); } } - if let Err(error) = parallel_link_objects( - &self.store, - staging, - &linked_memory_objects.into_iter().collect::>(), - ) { + let linked_memory_objects = linked_memory_objects.into_iter().collect::>(); + if let Err(error) = parallel_link_objects(&batch, staging, &linked_memory_objects) { let _ = vm.abandon_memory_capture(&memory_plan); return Err(CheckpointFailure::resumable(error)); } - let memory_id = match self.store.put_bytes(&memory_bytes) { + let memory_id = match batch.put_bytes(&memory_bytes) { Ok(id) => id, Err(error) => { let _ = vm.abandon_memory_capture(&memory_plan); return Err(CheckpointFailure::resumable(error)); } }; - if let Err(error) = self.store.link_into(&memory_id, staging) { + if let Err(error) = batch.link_into(&memory_id, staging) { let _ = vm.abandon_memory_capture(&memory_plan); return Err(CheckpointFailure::resumable(error)); } @@ -724,17 +1283,32 @@ impl CheckpointCoordinator { return Err(CheckpointFailure::resumable(error)); } }; - let checkpoint_root = match self.store.put_bytes(&checkpoint_bytes) { + let checkpoint_root = match batch.put_bytes(&checkpoint_bytes) { Ok(id) => id, Err(error) => { let _ = vm.abandon_memory_capture(&memory_plan); return Err(CheckpointFailure::resumable(error)); } }; - if let Err(error) = self.store.link_into(&checkpoint_root, staging) { + if let Err(error) = batch.link_into(&checkpoint_root, staging) { let _ = vm.abandon_memory_capture(&memory_plan); return Err(CheckpointFailure::resumable(error)); } + let memory_objects = match batch + .retained_objects(&linked_memory_objects) + .and_then(|objects| batch.finish().map(|_| objects)) + { + Ok(objects) => objects, + Err(error) => { + let _ = vm.abandon_memory_capture(&memory_plan); + return Err(CheckpointFailure::resumable(error)); + } + }; + let object_stats = batch.stats(); + timings.object_hashed_bytes = object_stats.hashed_bytes; + timings.object_linked_bytes = object_stats.linked_bytes; + timings.object_copied_bytes = object_stats.copied_bytes; + timings.object_directory_syncs = object_stats.directory_syncs; if let Err(error) = publish_root_last(staging, final_path, &checkpoint_bytes) { let _ = vm.abandon_memory_capture(&memory_plan); return Err(CheckpointFailure::resumable(error)); @@ -751,11 +1325,38 @@ impl CheckpointCoordinator { memory_emitted_bytes: stats.emitted_bytes, }, memory_plan, - memory_manifest, + memory_manifest: Some(memory_manifest), + memory_objects, + local_memory: None, timings, }) } + fn plan_local_memory( + &self, + vm: &msb_krun::VmControl, + ) -> Result<(MemoryCapturePlan, bool), String> { + if let (Some(baseline), Some(previous)) = + (vm.retained_memory_baseline(), self.local_baseline.as_ref()) + && previous.memory.generation == baseline.generation().get() + && previous.memory.topology == baseline.topology().get() + { + match vm + .plan_incremental_memory_capture(baseline) + .map_err(|e| e.to_string())? + { + IncrementalCaptureDecision::Incremental(plan) => return Ok((plan, true)), + IncrementalCaptureDecision::Complete { capture, .. } => { + return Ok((capture, false)); + } + IncrementalCaptureDecision::FullRequired(_) => {} + } + } + vm.plan_full_memory_capture() + .map(|plan| (plan, false)) + .map_err(|e| e.to_string()) + } + fn plan_memory( &self, vm: &msb_krun::VmControl, @@ -795,10 +1396,23 @@ impl CheckpointCoordinator { } } +impl UserPause { + fn validate(&self, vm: &msb_krun::VmControl) -> Result<(), String> { + if vm.execution_state() != Some(msb_krun::VmExecutionState::Paused(self.generation)) { + return Err("user pause no longer owns the current VM execution boundary".into()); + } + if self.workload.is_none() && self.capture_unavailable.is_none() { + return Err("user pause has no prepared workload latch for full capture".into()); + } + Ok(()) + } +} + impl CheckpointFailure { fn before_pause(error: impl fmt::Display) -> Self { Self { message: error.to_string(), + freezer_unavailable: false, keep_paused: false, published: None, } @@ -807,6 +1421,7 @@ impl CheckpointFailure { fn paused(error: impl fmt::Display) -> Self { Self { message: error.to_string(), + freezer_unavailable: false, keep_paused: true, published: None, } @@ -824,6 +1439,7 @@ impl FrozenWorkload { kind: "agent".into(), treatment: ResourceTreatment::Serialize, binding: BTreeMap::from([ + ("attempt_id".into(), self.attempt_id.clone()), ( "protocol_generation".into(), self.protocol_generation.to_string(), @@ -837,42 +1453,25 @@ impl FrozenWorkload { "ready".into(), serde_json::to_string(&self.ready).expect("Ready is serializable"), ), + ( + "transport_host_input".into(), + serde_json::to_string(&self.host_input) + .expect("input position is serializable"), + ), + ( + "transport_input_credit".into(), + serde_json::to_string(&self.input_credit) + .expect("input credit is serializable"), + ), + ( + "transport_guest_bulk_bytes".into(), + self.guest_bulk_bytes.to_string(), + ), ]), } } } -impl MemoryObjectSink<'_> { - /// Publish the final partial content pack and return its exact guest-address projection. - fn finish(mut self) -> io::Result> { - self.flush_pending()?; - Ok(self.updates) - } - - /// Store up to one bounded chunk containing bytes from multiple sparse guest ranges. - fn flush_pending(&mut self) -> io::Result<()> { - if self.pending_bytes.is_empty() { - return Ok(()); - } - let bytes = std::mem::take(&mut self.pending_bytes); - let object = self - .store - .put_bytes(&bytes) - .map_err(|error| io::Error::other(error.to_string()))?; - self.updates - .extend(self.pending_extents.drain(..).map(|extent| MemoryExtent { - start: extent.start, - length: extent.length, - content: MemoryExtentContent::Object(ContentRef { - object: object.clone(), - object_offset: extent.object_offset, - }), - })); - self.pending_bytes = Vec::with_capacity(MEMORY_OBJECT_PACK_SIZE); - Ok(()) - } -} - //-------------------------------------------------------------------------------------------------- // Trait Implementations //-------------------------------------------------------------------------------------------------- @@ -885,56 +1484,6 @@ impl fmt::Display for CheckpointFailure { impl std::error::Error for CheckpointFailure {} -impl MemoryCaptureSink for MemoryObjectSink<'_> { - fn write_bytes(&mut self, range: GuestMemoryRange, bytes: &[u8]) -> io::Result<()> { - if bytes.len() as u64 != range.length() { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "memory sink range length does not match bytes", - )); - } - - if !self.pending_bytes.is_empty() - && self.pending_bytes.len().saturating_add(bytes.len()) > MEMORY_OBJECT_PACK_SIZE - { - self.flush_pending()?; - } - if bytes.len() > MEMORY_OBJECT_PACK_SIZE { - let object = self - .store - .put_bytes(bytes) - .map_err(|error| io::Error::other(error.to_string()))?; - self.updates.push(MemoryExtent { - start: range.start(), - length: range.length(), - content: MemoryExtentContent::Object(ContentRef { - object, - object_offset: 0, - }), - }); - return Ok(()); - } - - let object_offset = self.pending_bytes.len() as u64; - self.pending_bytes.extend_from_slice(bytes); - self.pending_extents.push(PendingMemoryExtent { - start: range.start(), - length: range.length(), - object_offset, - }); - Ok(()) - } - - fn write_zero(&mut self, range: GuestMemoryRange) -> io::Result<()> { - self.updates.push(MemoryExtent { - start: range.start(), - length: range.length(), - content: MemoryExtentContent::Zero, - }); - Ok(()) - } -} - //-------------------------------------------------------------------------------------------------- // Functions //-------------------------------------------------------------------------------------------------- @@ -963,12 +1512,48 @@ async fn root_growth_request( reply.payload().map_err(|e| e.to_string()) } +/// Only new, scoped evidence of no attempted freeze permits a capability fallback. +fn unavailable_freezer_reason(reply: &Message, attempt_id: &str) -> Option { + if reply.t != MessageType::CoreError { + return None; + } + let error = reply.payload::().ok()?; + let detail = error.workload_failure?; + (error.kind == CoreErrorKind::CapabilityUnavailable + && error.offending_type.as_deref() == Some(MessageType::WorkloadFreeze.as_str()) + && detail.attempt_id == attempt_id + && detail.disposition == WorkloadFailureDisposition::Unavailable) + .then_some(error.message) +} + +fn recover_failed_freeze( + attempt_id: &str, + error: String, + thaw: impl FnOnce() -> Result<(), String>, + pause: impl FnOnce() -> Result<(), String>, +) -> CheckpointFailure { + match thaw() { + Ok(()) => CheckpointFailure::before_pause(error), + Err(thaw_error) => { + // Stop further guest progress if possible, and fence host mutations even if the + // hypervisor pause itself fails. Never turn uncertainty into a running disposition. + let pause_status = match pause() { + Ok(()) => "VM paused".to_string(), + Err(error) => format!("VM pause also failed: {error}"), + }; + CheckpointFailure::paused(format!( + "attempt {attempt_id}: {error}; workload recovery required: {thaw_error}; {pause_status}" + )) + } + } +} + fn validate_workload_reply( reply: Message, expected_type: MessageType, expected_attempt: &str, attempt_id: impl for<'a> Fn(&'a T) -> &'a str, -) -> Result<(), String> +) -> Result where T: serde::de::DeserializeOwned, { @@ -991,7 +1576,7 @@ where if attempt_id(&payload) != expected_attempt { return Err("workload control reply belongs to another checkpoint attempt".into()); } - Ok(()) + Ok(payload) } fn admit_resources( @@ -1111,40 +1696,43 @@ fn overlay_extents( mut base: Vec, mut updates: Vec, ) -> Result, String> { + base.sort_by_key(|extent| extent.start); updates.sort_by_key(|extent| extent.start); + validate_non_overlapping(&base)?; validate_non_overlapping(&updates)?; + // Consume each old range once. A suffix split by an update remains at the + // front for the next update; object offsets are retained by slice_extent. + let mut pending = std::collections::VecDeque::from(base); + let mut output = Vec::with_capacity(pending.len() + updates.len()); for update in updates { - let update_end = update - .start - .checked_add(update.length) - .ok_or_else(|| "memory update overflows".to_string())?; - let mut next = Vec::with_capacity(base.len() + 1); - for extent in base { - let extent_end = extent - .start - .checked_add(extent.length) - .ok_or_else(|| "memory base extent overflows".to_string())?; - if extent_end <= update.start || extent.start >= update_end { - next.push(extent); + let update_end = update.start + update.length; + while let Some(extent) = pending.front() { + if extent.start >= update_end { + break; + } + let extent = pending.pop_front().expect("front was present"); + let extent_end = extent.start + extent.length; + if extent_end <= update.start { + output.push(extent); continue; } if extent.start < update.start { - next.push(slice_extent( + output.push(slice_extent( &extent, extent.start, update.start - extent.start, )); } if extent_end > update_end { - next.push(slice_extent(&extent, update_end, extent_end - update_end)); + pending.push_front(slice_extent(&extent, update_end, extent_end - update_end)); + break; } } - next.push(update); - next.sort_by_key(|extent| extent.start); - base = next; + output.push(update); } - validate_non_overlapping(&base)?; - Ok(coalesce_extents(base)) + output.extend(pending); + validate_non_overlapping(&output)?; + Ok(coalesce_extents(output)) } //-------------------------------------------------------------------------------------------------- @@ -1255,8 +1843,20 @@ fn resource_kind(device_type: u32) -> &'static str { /// Persist independent device envelopes concurrently after every device has reached the same /// paused epoch. Immutable-object publication is thread-safe, and the returned vector retains the /// inventory order required by the checkpoint manifest. +/// Local handoffs reuse the state codecs and object paths, but make no crash-recovery promise. +/// Only bounded CPU/device state reaches this helper; RAM goes straight to its mmap backing. +fn put_local_object(staging: &Path, bytes: &[u8]) -> Result { + let id = ObjectId::from_bytes(bytes).map_err(|e| e.to_string())?; + let store = LocalObjectStore::open(staging).map_err(|e| e.to_string())?; + let path = store.object_path(&id); + std::fs::create_dir_all(path.parent().expect("confined object parent")) + .map_err(|e| e.to_string())?; + std::fs::write(path, bytes).map_err(|e| e.to_string())?; + Ok(id) +} + fn persist_device_states( - store: &LocalObjectStore, + store: &CaptureObjectBatch, staging: &Path, pending: &[PendingDeviceState], ) -> Result, String> { @@ -1301,10 +1901,9 @@ fn persist_device_states( }) } -/// Link independent immutable memory objects concurrently. Each object remains fully verified by -/// `LocalObjectStore::link_into`; this only overlaps hashing and filesystem durability waits. +/// Link independent immutable memory objects concurrently, reusing this batch's inode ownership. fn parallel_link_objects( - store: &LocalObjectStore, + store: &CaptureObjectBatch, staging: &Path, objects: &[ObjectId], ) -> Result<(), String> { @@ -1359,17 +1958,88 @@ fn sync_directory(path: &Path) -> io::Result<()> { #[cfg(test)] mod tests { use super::{ - MemoryObjectSink, PendingDeviceState, overlay_extents, persist_device_states, - publish_root_last, runtime_owned_fs_bindings, validate_vm_generation_state, - validate_workload_reply, + FrozenWorkload, MemoryObjectSink, PendingDeviceState, WorkloadControl, overlay_extents, + persist_device_states, publish_root_last, runtime_owned_fs_bindings, + validate_vm_generation_state, validate_workload_reply, }; use microsandbox_image::checkpoint::{ - ContentRef, LocalObjectStore, MemoryExtent, MemoryExtentContent, ObjectId, + CaptureObjectBatch, ContentRef, LocalObjectStore, MemoryExtent, MemoryExtentContent, + ObjectId, + }; + use microsandbox_protocol::core::{ + CoreError, CoreErrorKind, Ready, WorkloadFrozen, WorkloadTransportCredit, + WorkloadTransportPosition, }; - use microsandbox_protocol::core::{CoreError, CoreErrorKind, WorkloadFrozen}; use microsandbox_protocol::message::{Message, MessageType}; use msb_krun::{GuestMemoryRange, MemoryCaptureSink}; + use std::sync::Arc; + + #[test] + fn unavailable_freezer_requires_explicit_matching_evidence() { + use microsandbox_protocol::core::{WorkloadFailure, WorkloadFailureDisposition}; + let mut error = CoreError { + kind: CoreErrorKind::CapabilityUnavailable, + message: "missing freezer".into(), + offending_type: Some(MessageType::WorkloadFreeze.as_str().into()), + workload_failure: None, + }; + let check = |error: &CoreError| { + let reply = Message::with_payload(MessageType::CoreError, 7, error).unwrap(); + super::unavailable_freezer_reason(&reply, "a").is_some() + }; + assert!(!check(&error), "older agent errors are ambiguous"); + for disposition in [ + WorkloadFailureDisposition::RecoveryRequired, + WorkloadFailureDisposition::Unknown, + ] { + error.workload_failure = Some(WorkloadFailure { + attempt_id: "a".into(), + disposition, + }); + assert!(!check(&error)); + } + error.workload_failure.as_mut().unwrap().disposition = + WorkloadFailureDisposition::Unavailable; + assert!(check(&error)); + error.workload_failure.as_mut().unwrap().attempt_id = "b".into(); + assert!(!check(&error)); + error.workload_failure.as_mut().unwrap().attempt_id = "a".into(); + error.offending_type = Some(MessageType::WorkloadThaw.as_str().into()); + assert!(!check(&error)); + error.offending_type = Some(MessageType::WorkloadFreeze.as_str().into()); + error.kind = CoreErrorKind::InvalidSession; + assert!(!check(&error)); + } + + #[test] + fn failed_freeze_returns_running_only_after_confirmed_recovery() { + let failure = super::recover_failed_freeze( + "a", + "lost reply".into(), + || Ok(()), + || panic!("must not pause after thaw"), + ); + assert!(!failure.keep_paused); + for pause_fails in [false, true] { + let failure = super::recover_failed_freeze( + "a", + "lost reply".into(), + || Err("thaw failed".into()), + || { + if pause_fails { + Err("pause failed".into()) + } else { + Ok(()) + } + }, + ); + assert!(failure.keep_paused); + assert!(failure.message.contains("attempt a")); + assert!(failure.message.contains("recovery required")); + assert_eq!(failure.message.contains("pause also failed"), pause_fails); + } + } #[test] fn incremental_updates_split_and_reuse_unchanged_object_ranges() { @@ -1408,6 +2078,74 @@ mod tests { )); } + #[test] + fn incremental_merge_matches_byte_oracle_for_fragmented_ranges() { + let original = ObjectId::from_bytes(b"base").unwrap(); + let changed = ObjectId::from_bytes(b"update").unwrap(); + // Independent per-byte oracle includes holes, zero ranges, nonzero + // object offsets, unsorted input, and updates spanning multiple ranges. + let expand = |extents: &[MemoryExtent]| { + let mut bytes = vec![None; 256]; + for extent in extents { + for delta in 0..extent.length { + bytes[(extent.start + delta) as usize] = Some(match &extent.content { + MemoryExtentContent::Zero => (None, 0), + MemoryExtentContent::Object(content) => { + (Some(content.object.clone()), content.object_offset + delta) + } + }); + } + } + bytes + }; + let mut seed = 7u64; + for _ in 0..1000 { + let mut make = |object: &ObjectId| { + let mut ranges = Vec::new(); + let mut start = 0; + while start < 256 { + seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1); + let length = (1 + (seed >> 32) % 17).min(256 - start); + if !seed.is_multiple_of(5) { + ranges.push(MemoryExtent { + start, + length, + content: if seed.is_multiple_of(3) { + MemoryExtentContent::Zero + } else { + MemoryExtentContent::Object(ContentRef { + object: object.clone(), + object_offset: 1024 + start, + }) + }, + }); + } + start += length; + } + ranges.reverse(); + ranges + }; + let base = make(&original); + let updates = make(&changed); + let mut expected = expand(&base); + for (slot, update) in expected.iter_mut().zip(expand(&updates)) { + if update.is_some() { + *slot = update; + } + } + assert_eq!(expand(&overlay_extents(base, updates).unwrap()), expected); + } + let zero = |start, length| MemoryExtent { + start, + length, + content: MemoryExtentContent::Zero, + }; + assert!(overlay_extents(vec![zero(0, 8), zero(4, 8)], vec![]).is_err()); + assert!(overlay_extents(vec![], vec![zero(u64::MAX, 2)]).is_err()); + assert!(overlay_extents(vec![], vec![zero(0, 0)]).is_err()); + assert!(overlay_extents(vec![], vec![zero(0, 8), zero(4, 8)]).is_err()); + } + #[test] fn managed_root_admits_only_runtime_owned_filesystems() { let bindings = runtime_owned_fs_bindings(true); @@ -1494,12 +2232,8 @@ mod tests { fn sparse_memory_ranges_share_one_bounded_content_object() { let temp = tempfile::tempdir().unwrap(); let store = LocalObjectStore::open(temp.path()).unwrap(); - let mut sink = MemoryObjectSink { - store: &store, - updates: Vec::new(), - pending_bytes: Vec::new(), - pending_extents: Vec::new(), - }; + let batch = Arc::new(CaptureObjectBatch::new(store.clone(), &[])); + let mut sink = MemoryObjectSink::new(Arc::clone(&batch)).unwrap(); sink.write_bytes(GuestMemoryRange::new(0x1000, 3).unwrap(), b"abc") .unwrap(); @@ -1507,7 +2241,8 @@ mod tests { .unwrap(); sink.write_bytes(GuestMemoryRange::new(0x3000, 2).unwrap(), b"de") .unwrap(); - let mut extents = sink.finish().unwrap(); + let (mut extents, _) = sink.finish().unwrap(); + batch.finish().unwrap(); extents.sort_by_key(|extent| extent.start); assert_eq!(extents.len(), 3); @@ -1541,7 +2276,9 @@ mod tests { }) .collect::>(); - let persisted = persist_device_states(&store, &staging, &pending).unwrap(); + let batch = CaptureObjectBatch::new(store.clone(), &[]); + let persisted = persist_device_states(&batch, &staging, &pending).unwrap(); + batch.finish().unwrap(); assert_eq!(persisted.len(), pending.len()); for (index, state) in persisted.iter().enumerate() { @@ -1554,6 +2291,49 @@ mod tests { } } + #[test] + fn captured_agent_descriptor_retains_transport_debt() { + let control = WorkloadControl::new(); + let workload = FrozenWorkload { + gate: control.gate(), + attempt_id: "checkpoint-42".into(), + protocol_generation: 9, + ready: Ready { + workload_transport_barrier_version: Some( + microsandbox_protocol::core::WORKLOAD_TRANSPORT_BARRIER_VERSION, + ), + ..Ready::default() + }, + host_input: WorkloadTransportPosition { + control_bytes: 90_000_000, + control_frames: 4_000, + bulk_bytes: 100_000_000, + bulk_frames: 5_000, + }, + input_credit: WorkloadTransportCredit { + control_bytes: 90_000_128, + control_frames: 4_002, + bulk_bytes: 100_000_256, + bulk_frames: 5_003, + }, + guest_bulk_bytes: 123_456_789, + }; + let descriptor = workload.resource_descriptor(); + let position: WorkloadTransportPosition = + serde_json::from_str(&descriptor.binding["transport_host_input"]).unwrap(); + let credit: WorkloadTransportCredit = + serde_json::from_str(&descriptor.binding["transport_input_credit"]).unwrap(); + assert_eq!(position, workload.host_input); + assert_eq!(credit, workload.input_credit); + assert_eq!( + descriptor.binding["transport_guest_bulk_bytes"], + "123456789" + ); + // Publication does not release queued source input. Only confirmed thaw does. + assert!(control.gated()); + workload.gate.release(); + } + #[test] fn workload_reply_must_confirm_the_exact_attempt() { let reply = Message::with_payload( @@ -1561,6 +2341,8 @@ mod tests { 7, &WorkloadFrozen { attempt_id: "checkpoint-42".into(), + guest_bulk_bytes_target: 0, + input_credit: WorkloadTransportCredit::default(), }, ) .unwrap(); @@ -1578,6 +2360,8 @@ mod tests { 7, &WorkloadFrozen { attempt_id: "checkpoint-41".into(), + guest_bulk_bytes_target: 0, + input_credit: WorkloadTransportCredit::default(), }, ) .unwrap(); @@ -1601,6 +2385,7 @@ mod tests { kind: CoreErrorKind::CapabilityUnavailable, message: "freezer unavailable".into(), offending_type: Some(MessageType::WorkloadFreeze.as_str().into()), + workload_failure: None, }, ) .unwrap(); diff --git a/crates/runtime/lib/checkpoint/disk.rs b/crates/runtime/lib/checkpoint/disk.rs index 5df994e5b..e9afc1fdb 100644 --- a/crates/runtime/lib/checkpoint/disk.rs +++ b/crates/runtime/lib/checkpoint/disk.rs @@ -11,11 +11,11 @@ use std::time::Instant; use std::fs::File; use microsandbox_image::checkpoint::sparse_file_integrity; +#[cfg(feature = "runner")] +use microsandbox_image::checkpoint::{CheckpointClosure, DiskGenerationManifest, DiskLayerRef}; use microsandbox_image::checkpoint::{ CompactLayer, DiskCompactionPlan, compact_layer_capacity, materialize_compact_prefix, }; -#[cfg(feature = "runner")] -use microsandbox_image::checkpoint::{DiskGenerationManifest, DiskLayerRef}; pub use microsandbox_types::DiskCompactionResult; use serde::{Deserialize, Serialize}; @@ -129,6 +129,7 @@ impl RuntimeOwnedRootDisk { self.state.growth_target.is_some() } + #[cfg(feature = "runner")] pub(crate) fn begin_growth(&mut self, target: u64) -> Result<(), String> { let capacities = microsandbox_image::checkpoint::layer_capacities( self.state @@ -171,6 +172,7 @@ impl RuntimeOwnedRootDisk { Ok(()) } + #[cfg(feature = "runner")] pub(crate) fn finish_growth(&mut self) -> Result<(), String> { let mut next = self.state.clone(); next.growth_target = None; @@ -182,6 +184,15 @@ impl RuntimeOwnedRootDisk { /// Open the authoritative chain journal or initialize it from a sandbox-owned root disk. #[cfg(feature = "runner")] pub(crate) fn open(runtime_dir: &Path, vm: &VmConfig) -> Result, String> { + Self::open_with_admitted(runtime_dir, vm, None) + } + + #[cfg(feature = "runner")] + fn open_with_admitted( + runtime_dir: &Path, + vm: &VmConfig, + admitted: Option<&CheckpointClosure>, + ) -> Result, String> { let Some(layout) = configured_layout(vm) else { return Ok(None); }; @@ -213,13 +224,28 @@ impl RuntimeOwnedRootDisk { .collect(), }; let last = state.layers.len() - 1; + let mut reused_layers = 0_u64; + let mut hashed_layers = 0_u64; + let started = Instant::now(); for layer in state.layers.iter_mut().take(last) { - layer.integrity_root = Some( + let reused = admitted + .map(|closure| closure.reused_disk_integrity(&layer.path)) + .transpose() + .map_err(|error| format!("reuse admitted root ancestor: {error}"))? + .flatten(); + layer.integrity_root = Some(if let Some(root) = reused { + reused_layers += 1; + root + } else { + // Copies and relocated qcow headers are different physical artifacts. + // Never reuse their predecessor's root merely because sizes match. + hashed_layers += 1; sparse_file_integrity(&layer.path) .map_err(|error| format!("hash sealed root ancestor: {error}"))? - .root, - ); + .root + }); } + tracing::info!(target: "microsandbox_checkpoint_timing", operation = "root_journal_admission", reused_layers, hashed_layers, total_us = started.elapsed().as_micros(), "root journal admission timing"); write_state(&state_path, &state)?; state }; @@ -419,14 +445,10 @@ impl RuntimeOwnedRootDisk { .encode() .map_err(RootDiskRolloverError::pre_rebind)?; - for layer in &mut self.state.layers { - if layer.integrity_root.is_none() { - let integrity = sparse_file_integrity(&layer.path) - .map_err(RootDiskRolloverError::pre_rebind)?; - layer.integrity_root = Some(integrity.root); - } - } - let published_integrities = publish_layer_closure(checkpoint_root, &self.state.layers) + // Hash only a tentative generation: preparation may fail and resume this same writable + // head. Its captured root becomes reusable only after the forward journal commits. + let mut next_state = self.state.sealed_generation()?; + let published_integrities = publish_layer_closure(checkpoint_root, &next_state.layers) .map_err(RootDiskRolloverError::pre_rebind)?; let generation = self @@ -492,7 +514,6 @@ impl RuntimeOwnedRootDisk { )) .map_err(RootDiskRolloverError::pre_rebind)?; - let mut next_state = self.state.clone(); next_state.published_generation = generation; next_state.layers.push(RootDiskLayer { layer_id: new_id("layer"), @@ -505,7 +526,7 @@ impl RuntimeOwnedRootDisk { // This durable forward record is written before touching the running backend. Once it // exists, process restart always opens the new head whether the following rebind completed // or returned an uncertain error. - write_state(&self.state_path, &next_state).map_err(RootDiskRolloverError::pre_rebind)?; + write_state_with_sync(&self.state_path, &next_state, sync_directory)?; self.state = next_state; vm.replace_block_backend(&self.state.device_id, backend) .map_err(RootDiskRolloverError::post_journal)?; @@ -518,6 +539,21 @@ impl RuntimeOwnedRootDisk { } impl RootDiskState { + #[cfg(feature = "runner")] + fn sealed_generation(&self) -> Result { + let mut next = self.clone(); + for layer in &mut next.layers { + if layer.integrity_root.is_none() { + layer.integrity_root = Some( + sparse_file_integrity(&layer.path) + .map_err(RootDiskRolloverError::pre_rebind)? + .root, + ); + } + } + Ok(next) + } + fn validate(&self) -> Result<(), String> { if self.schema != ROOT_DISK_STATE_SCHEMA || !valid_id(&self.volume_id, "vol") @@ -645,6 +681,19 @@ impl std::error::Error for RootDiskRolloverError {} // Functions //-------------------------------------------------------------------------------------------------- +/// Seed a new child's journal from disk admission already completed in this runtime process. +/// The existing journal, when present, remains authoritative; transformed or copied files are +/// hashed instead of inheriting an identity belonging to their source representation. +#[cfg(feature = "runner")] +pub(crate) fn seed_restored_root_disk( + runtime_dir: &Path, + vm: &VmConfig, + admitted: &CheckpointClosure, +) -> Result<(), String> { + RuntimeOwnedRootDisk::open_with_admitted(runtime_dir, vm, Some(admitted))?; + Ok(()) +} + /// Apply the durable forward chain before VM construction after a runtime restart. #[cfg(feature = "runner")] pub(crate) fn recover_runtime_owned_root( @@ -987,29 +1036,42 @@ fn read_state(path: &Path) -> Result { } fn write_state(path: &Path, state: &RootDiskState) -> Result<(), String> { - state.validate()?; + write_state_with_sync(path, state, sync_directory).map_err(|error| error.to_string()) +} + +fn write_state_with_sync( + path: &Path, + state: &RootDiskState, + sync_parent: impl FnOnce(&Path) -> std::io::Result<()>, +) -> Result<(), RootDiskRolloverError> { + state + .validate() + .map_err(RootDiskRolloverError::pre_rebind)?; let parent = path .parent() - .ok_or_else(|| "root-disk state path has no parent".to_string())?; - std::fs::create_dir_all(parent).map_err(|error| error.to_string())?; + .ok_or_else(|| RootDiskRolloverError::pre_rebind("root-disk state path has no parent"))?; + std::fs::create_dir_all(parent).map_err(RootDiskRolloverError::pre_rebind)?; let temporary = parent.join(format!( ".{ROOT_DISK_STATE_FILE}.{}.tmp", rand::random::() )); - let bytes = serde_json::to_vec(state).map_err(|error| error.to_string())?; + let bytes = serde_json::to_vec(state).map_err(RootDiskRolloverError::pre_rebind)?; let mut file = OpenOptions::new() .write(true) .create_new(true) .open(&temporary) - .map_err(|error| error.to_string())?; - file.write_all(&bytes).map_err(|error| error.to_string())?; - file.sync_all().map_err(|error| error.to_string())?; + .map_err(RootDiskRolloverError::pre_rebind)?; + file.write_all(&bytes) + .map_err(RootDiskRolloverError::pre_rebind)?; + file.sync_all().map_err(RootDiskRolloverError::pre_rebind)?; drop(file); if let Err(error) = super::replace_file(&temporary, path) { let _ = std::fs::remove_file(&temporary); - return Err(error.to_string()); + // Treat replacement failures conservatively across platforms: the visible journal may + // already have changed even when the platform reports uncertain durable completion. + return Err(RootDiskRolloverError::post_journal(error)); } - sync_directory(parent).map_err(|error| error.to_string()) + sync_parent(parent).map_err(RootDiskRolloverError::post_journal) } #[cfg(feature = "runner")] @@ -1050,6 +1112,428 @@ fn sync_directory(path: &Path) -> std::io::Result<()> { #[cfg(test)] mod tests { + #[cfg(feature = "runner")] + #[tokio::test] + async fn failed_preparation_does_not_retain_a_writable_head_integrity() { + use std::io::{Seek, SeekFrom}; + + use super::*; + + for qcow2 in [false, true] { + let directory = tempfile::tempdir().unwrap(); + let base = directory.path().join("base.raw"); + std::fs::write(&base, vec![17u8; 131072]).unwrap(); + let mut layers = vec![RootDiskLayer { + layer_id: new_id("layer"), + path: base.clone(), + format: RootDiskFormat::Raw, + integrity_root: None, + }]; + if qcow2 { + let head = directory.path().join("head.qcow2"); + microsandbox_image::checkpoint::create_qcow2_overlay(&head, 131072, &base, "raw") + .await + .unwrap(); + layers[0].integrity_root = Some(sparse_file_integrity(&base).unwrap().root); + layers.push(RootDiskLayer { + layer_id: new_id("layer"), + path: head, + format: RootDiskFormat::Qcow2, + integrity_root: None, + }); + } + let state = RootDiskState { + schema: ROOT_DISK_STATE_SCHEMA.into(), + volume_id: new_id("vol"), + device_id: FLAT_ROOT_DEVICE_ID.into(), + layout: RootDiskLayout::FlatRoot, + published_generation: 0, + launch_base: None, + growth_target: None, + layers, + }; + let tentative = state.sealed_generation().unwrap(); + let blocked = directory.path().join("blocked"); + std::fs::write(&blocked, b"not a directory").unwrap(); + assert!(publish_layer_closure(&blocked, &tentative.layers).is_err()); + assert!(state.layers.last().unwrap().integrity_root.is_none()); + + // Model a resumed guest changing the same head before a retry. The abandoned cut + // must not supply a reusable root to the next attempt, for raw or qcow2 heads. + let head = &state.layers.last().unwrap().path; + let mut writer = OpenOptions::new().write(true).open(head).unwrap(); + writer.seek(SeekFrom::End(0)).unwrap(); + writer.write_all(b"resumed write").unwrap(); + writer.sync_all().unwrap(); + drop(writer); + let retry = state.sealed_generation().unwrap(); + assert_ne!( + retry.layers.last().unwrap().integrity_root, + tentative.layers.last().unwrap().integrity_root, + ); + assert_eq!( + retry + .layers + .last() + .unwrap() + .integrity_root + .as_ref() + .unwrap(), + &sparse_file_integrity(head).unwrap().root, + ); + } + } + + #[cfg(feature = "runner")] + #[tokio::test] + async fn journal_sync_failure_is_fenced_after_forward_publication() { + use super::*; + + let directory = tempfile::tempdir().unwrap(); + let base = directory.path().join("base.raw"); + std::fs::write(&base, vec![17u8; 4096]).unwrap(); + let mut state = RootDiskState { + schema: ROOT_DISK_STATE_SCHEMA.into(), + volume_id: new_id("vol"), + device_id: FLAT_ROOT_DEVICE_ID.into(), + layout: RootDiskLayout::FlatRoot, + published_generation: 0, + launch_base: None, + growth_target: None, + layers: vec![RootDiskLayer { + layer_id: new_id("layer"), + path: base.clone(), + format: RootDiskFormat::Raw, + integrity_root: None, + }], + }; + let path = directory.path().join(ROOT_DISK_STATE_FILE); + write_state(&path, &state).unwrap(); + let successor = directory.path().join("successor.qcow2"); + microsandbox_image::checkpoint::create_qcow2_overlay(&successor, 4096, &base, "raw") + .await + .unwrap(); + state = state.sealed_generation().unwrap(); + state.layers.push(RootDiskLayer { + layer_id: new_id("layer"), + path: successor.clone(), + format: RootDiskFormat::Qcow2, + integrity_root: None, + }); + state.published_generation = 1; + let error = write_state_with_sync(&path, &state, |_| { + Err(std::io::Error::other("injected directory sync failure")) + }) + .unwrap_err(); + assert!(error.keep_paused); + assert_eq!(read_state(&path).unwrap().published_generation, 1); + let recovered = load_runtime_owned_root_chain(directory.path()) + .unwrap() + .unwrap(); + assert_eq!(recovered.layers.len(), 2); + assert_eq!(recovered.layers[1].path, successor); + assert_eq!(recovered.virtual_size, 4096); + + let published = std::fs::read(&path).unwrap(); + state.schema = "invalid".into(); + let error = write_state_with_sync(&path, &state, |_| { + panic!("invalid state must fail before journal publication") + }) + .unwrap_err(); + assert!(!error.keep_paused); + assert_eq!(std::fs::read(&path).unwrap(), published); + } + + #[cfg(feature = "runner")] + fn admitted_fixture( + root: &std::path::Path, + sources: &[super::UpperLayerSpec], + ) -> microsandbox_image::checkpoint::CheckpointClosure { + use microsandbox_image::checkpoint::{ + CaptureIntent, CheckpointClosure, CheckpointManifest, DiskGenerationManifest, + DiskLayerRef, LocalObjectStore, MemoryCaptureMode, MemoryExtent, MemoryExtentContent, + MemoryManifest, ObjectId, sparse_file_integrity, + }; + + let store = LocalObjectStore::open(root).unwrap(); + std::fs::create_dir(root.join("layers")).unwrap(); + let memory = MemoryManifest { + schema: "microsandbox.memory/1".into(), + architecture: std::env::consts::ARCH.into(), + guest_page_size: 4096, + topology_generation: 1, + generation: 1, + capture_mode: MemoryCaptureMode::Full, + pause_generation: 7, + extents: vec![MemoryExtent { + start: 0, + length: 4096, + content: MemoryExtentContent::Zero, + }], + }; + let layers: Vec<_> = sources + .iter() + .enumerate() + .map(|(index, source)| { + let layer_id = format!("sealed_{index}"); + let format = match source.format { + msb_krun::DiskImageFormat::Raw => "raw", + msb_krun::DiskImageFormat::Qcow2 => "qcow2", + _ => panic!("unsupported fixture format"), + }; + let target = root.join("layers").join(format!("{layer_id}.{format}")); + std::fs::hard_link(&source.path, &target).unwrap(); + DiskLayerRef { + layer_id, + format: format.into(), + virtual_size: 131072, + predecessor: index + .checked_sub(1) + .map(|previous| format!("sealed_{previous}")), + integrity_root: sparse_file_integrity(&target).unwrap().root, + } + }) + .collect(); + let disk = DiskGenerationManifest { + schema: "microsandbox.disk-generation/1".into(), + volume_id: "root".into(), + device_id: "vda".into(), + generation: 1, + head: layers.last().unwrap().layer_id.clone(), + layers, + pause_generation: 7, + }; + // The closure checks opaque execution bytes; only live restore decodes their codec. + let checkpoint = CheckpointManifest { + schema: "microsandbox.checkpoint/1".into(), + checkpoint_id: "journal-fixture".into(), + capture_intent: CaptureIntent::FullSnapshot, + architecture: std::env::consts::ARCH.into(), + pause_generation: 7, + execution_state: store.put_bytes(b"execution fixture").unwrap(), + memory: store + .put_bytes(&memory.to_canonical_bytes().unwrap()) + .unwrap(), + disks: vec![ + store + .put_bytes(&disk.to_canonical_bytes().unwrap()) + .unwrap(), + ], + devices: Vec::new(), + resources: Vec::new(), + requires: Vec::new(), + }; + let bytes = checkpoint.to_canonical_bytes().unwrap(); + let id = ObjectId::from_bytes(&bytes).unwrap(); + std::fs::write(root.join("checkpoint.json"), bytes).unwrap(); + CheckpointClosure::open(root, Some(&id)).unwrap() + } + + #[cfg(feature = "runner")] + fn root_vm( + layout: super::RootDiskLayout, + layers: Vec, + ) -> super::VmConfig { + let spec = super::UpperSpec { + layers, + read_only: false, + }; + let mut vm = super::VmConfig { + libkrunfw_path: Default::default(), + thp: Default::default(), + memory_cache_dir: None, + vcpus: 1, + memory_mib: 256, + max_cpus: 1, + max_memory_mib: 256, + cpu_placement: Default::default(), + placement_profile_name: None, + placement_profile: None, + block_writeback_limit_bytes: None, + rootfs_path: None, + rootfs_follow_root_symlinks: false, + rootfs_disk: None, + rootfs_disk_format: None, + rootfs_disk_readonly: false, + rootfs_disk_spec: None, + rootfs_disk_runtime_owned: false, + rootfs_vmdk: None, + rootfs_upper: None, + rootfs_upper_spec: None, + mounts: Vec::new(), + file_mounts: Vec::new(), + disks: Vec::new(), + vsock: Vec::new(), + #[cfg(unix)] + backends: Vec::new(), + init_path: None, + bootstrap: Default::default(), + exec_path: None, + exec_args: Vec::new(), + #[cfg(feature = "net")] + network: Default::default(), + #[cfg(feature = "net")] + deployment_profile: Default::default(), + #[cfg(feature = "net")] + sandbox_slot: 1, + checkpoint_restore: None, + }; + match layout { + super::RootDiskLayout::ManagedUpper => { + vm.rootfs_vmdk = Some("fixture.vmdk".into()); + vm.rootfs_upper_spec = Some(spec); + } + super::RootDiskLayout::FlatRoot => { + vm.rootfs_disk_runtime_owned = true; + vm.rootfs_disk_spec = Some(spec); + } + } + vm + } + + #[tokio::test] + #[cfg(feature = "runner")] + async fn admitted_raw_hardlink_seeds_once_and_reopens_without_the_snapshot() { + use super::*; + for layout in [RootDiskLayout::ManagedUpper, RootDiskLayout::FlatRoot] { + let directory = tempfile::tempdir().unwrap(); + let source = directory.path().join("source.raw"); + std::fs::write(&source, vec![17; 131072]).unwrap(); + let admitted = admitted_fixture( + &directory.path().join("snapshot"), + &[UpperLayerSpec { + path: source.clone(), + format: msb_krun::DiskImageFormat::Raw, + }], + ); + let child_base = directory.path().join("child.raw"); + std::fs::hard_link(&source, &child_base).unwrap(); + let expected = admitted.disks()[0].layers[0].integrity_root.clone(); + assert_eq!( + admitted.reused_disk_integrity(&child_base).unwrap(), + Some(expected.clone()) + ); + let runtime = directory.path().join("runtime"); + std::fs::create_dir(&runtime).unwrap(); + let head = runtime.join("head.qcow2"); + microsandbox_image::checkpoint::create_qcow2_overlay(&head, 131072, &child_base, "raw") + .await + .unwrap(); + let vm = root_vm( + layout, + vec![ + UpperLayerSpec { + path: child_base.clone(), + format: msb_krun::DiskImageFormat::Raw, + }, + UpperLayerSpec { + path: head, + format: msb_krun::DiskImageFormat::Qcow2, + }, + ], + ); + seed_restored_root_disk(&runtime, &vm, &admitted).unwrap(); + let journal = runtime.join(ROOT_DISK_STATE_FILE); + let first = std::fs::read(&journal).unwrap(); + let state = read_state(&journal).unwrap(); + assert_eq!(state.layers[0].integrity_root.as_ref(), Some(&expected)); + assert!( + state.layers[1].integrity_root.is_none(), + "writable head must not be sealed" + ); + assert_eq!(state.layout, layout); + seed_restored_root_disk(&runtime, &vm, &admitted).unwrap(); + assert_eq!(std::fs::read(&journal).unwrap(), first); + let snapshot_layer = admitted.disk_layer_path(&admitted.disks()[0].layers[0]); + drop(admitted); + std::fs::remove_file(source).unwrap(); + std::fs::remove_file(snapshot_layer).unwrap(); + RuntimeOwnedRootDisk::open(&runtime, &vm).unwrap().unwrap(); + assert_eq!(std::fs::read(&journal).unwrap(), first); + assert_eq!(std::fs::read(child_base).unwrap(), vec![17; 131072]); + } + } + + #[tokio::test] + #[cfg(feature = "runner")] + async fn copied_raw_and_relocated_qcow_seed_their_own_physical_integrities() { + use super::*; + use microsandbox_image::checkpoint::{create_qcow2_overlay, relocate_qcow2_backing}; + let directory = tempfile::tempdir().unwrap(); + let source = directory.path().join("source.raw"); + let overlay = directory.path().join("source.qcow2"); + std::fs::write(&source, vec![31; 131072]).unwrap(); + create_qcow2_overlay(&overlay, 131072, &source, "raw") + .await + .unwrap(); + let original_overlay = std::fs::read(&overlay).unwrap(); + let admitted = admitted_fixture( + &directory.path().join("snapshot"), + &[ + UpperLayerSpec { + path: source.clone(), + format: msb_krun::DiskImageFormat::Raw, + }, + UpperLayerSpec { + path: overlay.clone(), + format: msb_krun::DiskImageFormat::Qcow2, + }, + ], + ); + let base_copy = directory.path().join("copied-base.raw"); + let overlay_copy = directory.path().join("copied-overlay.qcow2"); + std::fs::copy(&source, &base_copy).unwrap(); + std::fs::copy(&overlay, &overlay_copy).unwrap(); + relocate_qcow2_backing(&overlay_copy, &base_copy).unwrap(); + assert!( + admitted + .reused_disk_integrity(&base_copy) + .unwrap() + .is_none() + ); + assert!( + admitted + .reused_disk_integrity(&overlay_copy) + .unwrap() + .is_none() + ); + let expected_raw = sparse_file_integrity(&base_copy).unwrap().root; + let expected_qcow = sparse_file_integrity(&overlay_copy).unwrap().root; + assert_ne!(expected_qcow, admitted.disks()[0].layers[1].integrity_root); + let runtime = directory.path().join("runtime"); + std::fs::create_dir(&runtime).unwrap(); + let head = runtime.join("head.qcow2"); + create_qcow2_overlay(&head, 131072, &overlay_copy, "qcow2") + .await + .unwrap(); + let vm = root_vm( + RootDiskLayout::FlatRoot, + vec![ + UpperLayerSpec { + path: base_copy, + format: msb_krun::DiskImageFormat::Raw, + }, + UpperLayerSpec { + path: overlay_copy, + format: msb_krun::DiskImageFormat::Qcow2, + }, + UpperLayerSpec { + path: head, + format: msb_krun::DiskImageFormat::Qcow2, + }, + ], + ); + seed_restored_root_disk(&runtime, &vm, &admitted).unwrap(); + let state = read_state(&runtime.join(ROOT_DISK_STATE_FILE)).unwrap(); + assert_eq!(state.layers[0].integrity_root.as_ref(), Some(&expected_raw)); + assert_eq!( + state.layers[1].integrity_root.as_ref(), + Some(&expected_qcow) + ); + assert!(state.layers[2].integrity_root.is_none()); + assert_eq!(std::fs::read(overlay).unwrap(), original_overlay); + } + #[test] fn stopped_growth_preserves_ancestors_and_recovers_pending_target() { use super::*; diff --git a/crates/runtime/lib/checkpoint/local.rs b/crates/runtime/lib/checkpoint/local.rs new file mode 100644 index 000000000..84cd6c8c8 --- /dev/null +++ b/crates/runtime/lib/checkpoint/local.rs @@ -0,0 +1,85 @@ +//! Bounded process-local branch handoff, deliberately distinct from a full snapshot. + +use std::fs::File; +use std::io::{self, Read}; +use std::path::Path; + +use microsandbox_image::checkpoint::{ + DeviceStateRef, DiskGenerationManifest, LocalObjectStore, ObjectId, ResourceDescriptor, +}; +use serde::{Deserialize, Serialize}; + +use super::local_memory::LocalMemory; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// Same-epoch local execution handoff. RAM has no portable object representation here. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LocalBranchState { + /// Unique capture attempt, shared with the workload freeze latch. + pub id: String, + /// Host architecture required by the captured execution. + pub architecture: String, + /// Shared CPU/device/RAM pause boundary. + pub pause_generation: u64, + /// Encoded CPU and interrupt-controller state. + pub execution_state: ObjectId, + /// Existing device state encodings. + pub devices: Vec, + /// Existing resource bindings, including the captured agent identity. + pub resources: Vec, + /// Complete sealed disk generations. + pub disks: Vec, + /// Complete, immutable, mmap-ready RAM; never a partial memory manifest. + pub memory: LocalMemory, + /// Boot CPU count and configured capacity, not a mutable guest online count. + pub vcpus: u8, + /// Maximum CPU count used for device construction. + pub max_cpus: u8, + /// Boot RAM geometry in MiB. + pub memory_mib: u32, + /// Configured hotplug capacity in MiB. + pub max_memory_mib: u32, +} + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl LocalBranchState { + /// Read bounded handoff metadata; ordinary snapshot readers never recognize this file. + pub fn open(root: &Path) -> io::Result { + let bytes = read_bounded(&root.join("branch.json"), 16 * 1024 * 1024)?; + let state: Self = serde_json::from_slice(&bytes).map_err(io::Error::other)?; + if state.architecture != std::env::consts::ARCH { + return Err(io::Error::other("branch architecture differs")); + } + Ok(state) + } + + /// Read an existing bounded state object, checking its recorded identity. + pub fn read_object(root: &Path, id: &ObjectId, limit: u64) -> io::Result> { + let store = LocalObjectStore::open(root).map_err(io::Error::other)?; + let bytes = read_bounded(&store.object_path(id), limit)?; + if ObjectId::from_bytes(&bytes).map_err(io::Error::other)? != *id { + return Err(io::Error::other("branch state object differs")); + } + Ok(bytes) + } +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +fn read_bounded(path: &Path, limit: u64) -> io::Result> { + let mut bytes = Vec::new(); + File::open(path)?.take(limit + 1).read_to_end(&mut bytes)?; + if bytes.len() as u64 > limit { + return Err(io::Error::other("local state exceeds size bound")); + } + Ok(bytes) +} diff --git a/crates/runtime/lib/checkpoint/local_memory.rs b/crates/runtime/lib/checkpoint/local_memory.rs new file mode 100644 index 000000000..b1a856334 --- /dev/null +++ b/crates/runtime/lib/checkpoint/local_memory.rs @@ -0,0 +1,386 @@ +//! Direct local RAM generations. The source's live mappings are never replaced. + +use std::fs::File; +use std::io; +use std::path::{Path, PathBuf}; + +#[cfg(feature = "runner")] +use std::{ + fs::OpenOptions, + io::{Seek, SeekFrom, Write}, +}; + +#[cfg(feature = "runner")] +use msb_krun::{GuestMemoryRange, MemoryCaptureSink}; +use serde::{Deserialize, Serialize}; + +use super::memory_cache::open_pinned; +use super::{CachedMemoryRegion, MemoryCache}; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// Complete local memory geometry; this is not a portable memory manifest. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LocalMemory { + /// Immutable backend-owned file, independent of the source sandbox directory. + pub path: PathBuf, + /// Native mapping geometry in guest-physical order. + pub regions: Vec, + /// Retained memory generation used only for incremental capture continuity. + pub generation: u64, + /// Memory topology to which the generation belongs. + pub topology: u64, +} + +#[cfg(feature = "runner")] +pub(crate) struct LocalMemoryPin { + pub(crate) memory: LocalMemory, + pub(crate) _file: File, +} + +#[cfg(feature = "runner")] +pub(super) struct LocalMemoryCapture { + staging: tempfile::TempDir, + file: File, + path: PathBuf, + regions: Vec, + length: u64, + incremental: bool, + page_size: u64, + pub(super) reflink: bool, +} + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl LocalMemory { + /// Reserve publication-to-pin ownership before asking the source to capture RAM. + /// Stable lock inodes are never unlinked, so eviction cannot race a replacement lock. + pub fn reserve(root: &Path, id: &str) -> io::Result { + if id.is_empty() + || id.len() > 128 + || !id + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'-')) + { + return Err(io::Error::other("invalid local generation identity")); + } + let cache = MemoryCache::open_namespace(root.into(), "branches")?; + let path = cache + .root + .join(format!("{id}-{}.handoff-lock", cache.page_size)); + let file = microsandbox_utils::process_lock::open_lock_file(&path)?; + microsandbox_utils::process_lock::lock_exclusive(&file)?; + Ok(file) + } + + /// Reclaim only after pending handoff, retained-baseline and VM pins have been released. + pub fn evict(&self) -> io::Result { + let handoff = microsandbox_utils::process_lock::open_lock_file( + &self.path.with_extension("handoff-lock"), + )?; + if !microsandbox_utils::process_lock::try_lock_exclusive(&handoff)? { + return Ok(false); + } + super::memory_cache::evict_unpinned(&self.path) + } + + /// Acquire independent backing ownership before launching or mapping a child. + pub fn pin(&self) -> io::Result { + let mut file_end = 0; + let mut guest_end = 0; + for region in &self.regions { + if region.length == 0 + || region.file_offset != file_end + || region.guest_address < guest_end + { + return Err(io::Error::other("invalid local memory geometry")); + } + file_end = region + .file_offset + .checked_add(region.length) + .ok_or_else(|| io::Error::other("memory size overflow"))?; + guest_end = region + .guest_address + .checked_add(region.length) + .ok_or_else(|| io::Error::other("guest range overflow"))?; + } + if file_end == 0 { + return Err(io::Error::other("empty local memory")); + } + open_pinned(&self.path, file_end)? + .ok_or_else(|| io::Error::other("local memory backing is missing")) + } +} + +#[cfg(feature = "runner")] +impl LocalMemoryCapture { + pub(super) fn new( + root: &Path, + id: &str, + baseline: Option<&LocalMemoryPin>, + ) -> io::Result { + let cache = MemoryCache::open_namespace(root.into(), "branches")?; + let staging = tempfile::Builder::new() + .prefix(".capture-") + .tempdir_in(&cache.root)?; + let temporary = staging.path().join("memory"); + let mut reflink = false; + if let Some(base) = baseline { + let (_, strategy) = + microsandbox_utils::copy::fast_copy_with_strategy(&base.memory.path, &temporary)?; + reflink = strategy == microsandbox_utils::copy::FastCopyStrategy::Reflink; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&temporary, std::fs::Permissions::from_mode(0o600))?; + } + } + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(&temporary)?; + let length = file.metadata()?.len(); + Ok(Self { + staging, + file, + length, + reflink, + path: cache.root.join(format!("{id}-{}.ram", cache.page_size)), + regions: baseline + .map(|base| base.memory.regions.clone()) + .unwrap_or_default(), + incremental: baseline.is_some(), + page_size: cache.page_size, + }) + } + + fn offset(&mut self, range: GuestMemoryRange) -> io::Result { + let end = range + .start() + .checked_add(range.length()) + .ok_or_else(|| io::Error::other("range overflow"))?; + if self.incremental { + let region = self + .regions + .iter() + .find(|region| { + range.start() >= region.guest_address + && end <= region.guest_address + region.length + }) + .ok_or_else(|| io::Error::other("delta falls outside retained memory topology"))?; + return Ok(region.file_offset + range.start() - region.guest_address); + } + let offset = self.length; + if let Some(last) = self.regions.last_mut() { + let previous_end = last.guest_address + last.length; + if range.start() < previous_end { + return Err(io::Error::other("unordered full memory capture")); + } + if range.start() == previous_end { + last.length += range.length(); + } else { + self.regions.push(CachedMemoryRegion { + guest_address: range.start(), + length: range.length(), + file_offset: offset, + }); + } + } else { + self.regions.push(CachedMemoryRegion { + guest_address: range.start(), + length: range.length(), + file_offset: offset, + }); + } + self.length = self + .length + .checked_add(range.length()) + .ok_or_else(|| io::Error::other("memory file overflow"))?; + Ok(offset) + } + + pub(super) fn finish(self, generation: u64, topology: u64) -> io::Result { + for region in &self.regions { + if !region.guest_address.is_multiple_of(self.page_size) + || !region.length.is_multiple_of(self.page_size) + || !region.file_offset.is_multiple_of(self.page_size) + { + return Err(io::Error::other( + "local memory geometry is not native-page aligned", + )); + } + } + self.file.set_len(self.length)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + self.file + .set_permissions(std::fs::Permissions::from_mode(0o400))?; + } + // Local branching promises process-independent ownership, not crash recovery. Closing + // the writer and publishing the completed inode suffices; no RAM-sized fsync/hash pass. + drop(self.file); + let memory = LocalMemory { + path: self.path, + regions: self.regions, + generation, + topology, + }; + let file = open_pinned(&self.staging.path().join("memory"), self.length)? + .ok_or_else(|| io::Error::other("capture staging disappeared"))?; + std::fs::hard_link(self.staging.path().join("memory"), &memory.path)?; + Ok(LocalMemoryPin { + memory, + _file: file, + }) + } +} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +#[cfg(feature = "runner")] +impl MemoryCaptureSink for LocalMemoryCapture { + fn write_bytes(&mut self, range: GuestMemoryRange, bytes: &[u8]) -> io::Result<()> { + if range.length() != bytes.len() as u64 { + return Err(io::Error::other("capture length mismatch")); + } + let offset = self.offset(range)?; + self.file.seek(SeekFrom::Start(offset))?; + self.file.write_all(bytes) + } + + fn write_zero(&mut self, range: GuestMemoryRange) -> io::Result<()> { + let offset = self.offset(range)?; + if self.incremental { + // A zero/discard delta must replace old bytes, not leave stale private content. + self.file.seek(SeekFrom::Start(offset))?; + let zeroes = [0u8; 65536]; + let mut remaining = range.length(); + while remaining != 0 { + let count = remaining.min(zeroes.len() as u64) as usize; + self.file.write_all(&zeroes[..count])?; + remaining -= count as u64; + } + } + Ok(()) + } +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(all(test, feature = "runner"))] +mod tests { + use std::io::Read; + #[cfg(unix)] + use std::os::unix::fs::PermissionsExt; + + use super::*; + + fn range(start: u64, length: u64) -> GuestMemoryRange { + GuestMemoryRange::new(start, length).unwrap() + } + + #[test] + fn direct_generation_is_sparse_complete_and_independently_pinned() { + let dir = tempfile::tempdir().unwrap(); + let page = MemoryCache::open(dir.path()).unwrap().page_size; + let mut sink = LocalMemoryCapture::new(dir.path(), "first", None).unwrap(); + sink.write_bytes(range(0, page), &vec![7; page as usize]) + .unwrap(); + sink.write_zero(range(page, page)).unwrap(); + sink.write_bytes(range(4 * page, page), &vec![9; page as usize]) + .unwrap(); + let captured = sink.finish(1, 1).unwrap(); + assert_eq!(captured.memory.regions.len(), 2); + assert_eq!(captured.memory.regions[1].file_offset, 2 * page); + #[cfg(unix)] + assert_eq!( + captured._file.metadata().unwrap().permissions().mode() & 0o777, + 0o400 + ); + let mut child = captured.memory.pin().unwrap(); + std::fs::remove_file(&captured.memory.path).unwrap(); + drop(captured); + let mut bytes = Vec::new(); + child.read_to_end(&mut bytes).unwrap(); + assert_eq!(bytes.len(), (3 * page) as usize); + assert!( + bytes[page as usize..(2 * page) as usize] + .iter() + .all(|byte| *byte == 0) + ); + } + + #[test] + fn incremental_zero_and_write_do_not_mutate_the_baseline() { + let dir = tempfile::tempdir().unwrap(); + let page = MemoryCache::open(dir.path()).unwrap().page_size; + let mut full = LocalMemoryCapture::new(dir.path(), "base", None).unwrap(); + full.write_bytes(range(0, 2 * page), &vec![7; (2 * page) as usize]) + .unwrap(); + let base = full.finish(1, 1).unwrap(); + let mut delta = LocalMemoryCapture::new(dir.path(), "delta", Some(&base)).unwrap(); + delta.write_zero(range(0, page)).unwrap(); + delta + .write_bytes(range(page, page), &vec![9; page as usize]) + .unwrap(); + let child = delta.finish(2, 1).unwrap(); + assert!( + std::fs::read(&base.memory.path) + .unwrap() + .iter() + .all(|b| *b == 7) + ); + let bytes = std::fs::read(&child.memory.path).unwrap(); + assert!(bytes[..page as usize].iter().all(|b| *b == 0)); + assert!(bytes[page as usize..].iter().all(|b| *b == 9)); + } + + #[test] + fn capture_rejects_overlap_unaligned_geometry_and_identity_collision() { + let dir = tempfile::tempdir().unwrap(); + let page = MemoryCache::open(dir.path()).unwrap().page_size; + let mut sink = LocalMemoryCapture::new(dir.path(), "same", None).unwrap(); + sink.write_zero(range(0, page)).unwrap(); + assert!(sink.write_zero(range(0, page)).is_err()); + let _pin = sink.finish(1, 1).unwrap(); + let mut collision = LocalMemoryCapture::new(dir.path(), "same", None).unwrap(); + collision.write_zero(range(0, page)).unwrap(); + assert!(collision.finish(2, 1).is_err()); + let mut unaligned = LocalMemoryCapture::new(dir.path(), "unaligned", None).unwrap(); + unaligned.write_zero(range(0, 4096)).unwrap(); + if page > 4096 { + assert!(unaligned.finish(3, 1).is_err()); + } + } + + #[test] + fn pending_handoff_survives_source_pin_loss_and_eviction() { + let dir = tempfile::tempdir().unwrap(); + let page = MemoryCache::open(dir.path()).unwrap().page_size; + let reservation = LocalMemory::reserve(dir.path(), "handoff").unwrap(); + let mut sink = LocalMemoryCapture::new(dir.path(), "handoff", None).unwrap(); + sink.write_zero(range(0, page)).unwrap(); + let source = sink.finish(1, 1).unwrap(); + let memory = source.memory.clone(); + drop(source); // source exits before the SDK receives the response + assert!(!memory.evict().unwrap()); + let child = memory.pin().unwrap(); + drop(reservation); + assert!(!memory.evict().unwrap()); + drop(child); + assert!(memory.evict().unwrap()); + assert!(memory.pin().is_err()); + } +} diff --git a/crates/runtime/lib/checkpoint/memory_cache.rs b/crates/runtime/lib/checkpoint/memory_cache.rs new file mode 100644 index 000000000..5e0afdf1f --- /dev/null +++ b/crates/runtime/lib/checkpoint/memory_cache.rs @@ -0,0 +1,976 @@ +//! Immutable local realizations of complete portable memory manifests. +//! +//! The cache is trusted host storage, not a second portable snapshot format. Publication is +//! atomic; readers retain read-only handles, so removing a snapshot never invalidates live RAM. + +use std::collections::BTreeMap; +use std::fs::{File, OpenOptions}; +use std::io::{self, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; +use std::time::Instant; + +use microsandbox_image::checkpoint::{ + CheckpointObjectReadTiming, MemoryExtentContent, MemoryManifest, ObjectId, +}; + +use super::object_pipeline::{ObjectPipelineTiming, consume_verified_objects}; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// One native-aligned, contiguous guest address span in a flat cache file. +#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CachedMemoryRegion { + /// Start of the guest physical span. + pub guest_address: u64, + /// Length of the span in bytes. + pub length: u64, + /// Byte offset in the immutable cache file. + pub file_offset: u64, +} + +/// An opened realization pinned against cooperative eviction until the handle is dropped. +pub struct CachedMemory { + path: PathBuf, + identity: ObjectId, + /// Read-only backing ownership. Transfer this handle to the VMM, not merely its pathname. + pub file: File, + /// Exact guest coverage, with address holes omitted from physical storage. + pub regions: Vec, + /// Whether existing verified bytes were reused without rereading portable objects. + pub cache_hit: bool, + /// Whether this construction cloned its baseline using a filesystem reflink. + pub reflink: bool, + /// Time spent resolving or constructing this backing, in microseconds. + pub prepare_us: u128, +} + +/// Host-local, immutable memory cache. No entry is ever modified in place. +pub struct MemoryCache { + pub(super) root: PathBuf, + pub(super) page_size: u64, +} + +type ObjectSlices = BTreeMap>; + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl MemoryCache { + /// Open a dedicated cache directory using this host's native mapping alignment. + pub fn open(root: impl Into) -> io::Result { + Self::open_namespace(root.into(), "snapshots") + } + + pub(super) fn open_namespace(root: PathBuf, namespace: &str) -> io::Result { + #[cfg(unix)] + { + let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + if page_size <= 0 { + return Err(io::Error::last_os_error()); + } + std::fs::create_dir_all(&root)?; + // Cache contents are guest RAM, not public image data. Restrict traversal even + // when the caller's umask permits other local users to read ordinary cache files. + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700))?; + let root = root.join(namespace); + std::fs::create_dir_all(&root)?; + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700))?; + Ok(Self { + root, + page_size: page_size as u64, + }) + } + #[cfg(windows)] + { + use windows_sys::Win32::System::SystemInformation::{GetSystemInfo, SYSTEM_INFO}; + let mut info: SYSTEM_INFO = unsafe { std::mem::zeroed() }; + unsafe { + GetSystemInfo(&mut info); + } + std::fs::create_dir_all(&root)?; + restrict_cache_directory(&root)?; + let root = root.join(namespace); + std::fs::create_dir_all(&root)?; + restrict_cache_directory(&root)?; + Ok(Self { + root, + page_size: u64::from(info.dwPageSize), + }) + } + #[cfg(not(any(unix, windows)))] + { + let _ = (root, namespace); + Err(io::Error::new( + io::ErrorKind::Unsupported, + "private memory cache is not qualified on this backend", + )) + } + } + + /// Resolve a complete memory image, verifying portable objects only on a cache miss. + /// + /// `read_object` must return identity-verified bytes. It is called once per distinct packed + /// object, not once per extent. The complete canonical manifest identity names the cache; + /// neither an unverified partial delta nor a mutable file may be published under that name. + pub fn materialize( + &self, + manifest: &MemoryManifest, + identity: &ObjectId, + read_object: impl FnMut(&ObjectId) -> io::Result>, + ) -> io::Result { + self.materialize_with_baseline(manifest, identity, None, read_object) + } + + /// Prepare a durable restore backing with bounded parallel verification and reusable buffers. + /// Readers must enforce the 32 MiB portable memory-object bound and return verified bytes. + pub fn materialize_parallel( + &self, + manifest: &MemoryManifest, + identity: &ObjectId, + read_object: impl Fn(&ObjectId, &mut Vec) -> io::Result + Sync, + ) -> io::Result { + self.materialize_with_baseline_inner(manifest, identity, None, |objects, staging| { + let timings = consume_verified_objects(objects, read_object, |slices, bytes| { + write_object_slices(staging, slices, bytes) + })?; + tracing::info!( + target: "microsandbox_checkpoint_timing", + operation = "memory_cache_objects", + object_io_worker_us = timings.read_us, + object_hash_worker_us = timings.hash_us, + object_write_us = timings.consume_us, + object_pipeline_us = timings.elapsed_us, + object_bytes = timings.object_bytes, + "parallel memory cache object timing" + ); + Ok(timings) + }) + } + + /// Reuse a pinned complete baseline before overlaying immutable changed object slices. + /// The source VM is never read or remapped here; both inputs are completed captures. + pub fn materialize_with_baseline( + &self, + manifest: &MemoryManifest, + identity: &ObjectId, + baseline: Option<(&MemoryManifest, &CachedMemory)>, + mut read_object: impl FnMut(&ObjectId) -> io::Result>, + ) -> io::Result { + self.materialize_with_baseline_inner(manifest, identity, baseline, |objects, staging| { + let started = Instant::now(); + let mut timings = ObjectPipelineTiming::default(); + for (id, slices) in objects { + let reading = Instant::now(); + let bytes = read_object(&id)?; + timings.read_us += reading.elapsed().as_micros(); + timings.object_bytes += bytes.len() as u64; + let writing = Instant::now(); + write_object_slices(staging, slices, &bytes)?; + timings.consume_us += writing.elapsed().as_micros(); + } + timings.elapsed_us = started.elapsed().as_micros(); + Ok(timings) + }) + } + + fn materialize_with_baseline_inner( + &self, + manifest: &MemoryManifest, + identity: &ObjectId, + baseline: Option<(&MemoryManifest, &CachedMemory)>, + consume_objects: impl FnOnce(ObjectSlices, &mut File) -> io::Result, + ) -> io::Result { + let started = Instant::now(); + let canonical = manifest.to_canonical_bytes().map_err(io::Error::other)?; + if ObjectId::from_bytes(&canonical).map_err(io::Error::other)? != *identity { + return Err(invalid( + "memory cache identity does not match its complete manifest", + )); + } + let regions = memory_regions(manifest, self.page_size)?; + let length = regions + .last() + .and_then(|r| r.file_offset.checked_add(r.length)) + .ok_or_else(|| invalid("empty or overflowing memory topology"))?; + let path = self.entry_path(identity); + if let Some(file) = open_pinned(&path, length)? { + return Ok(CachedMemory { + path, + identity: identity.clone(), + file, + regions, + cache_hit: true, + reflink: false, + prepare_us: started.elapsed().as_micros(), + }); + } + + // Stable per-identity lock inodes serialize cache misses across processes, without + // placing warm hits or unrelated snapshots behind a global cache lock. Never unlink a + // build lock: waiters must not acquire different inodes for the same identity. + let build_lock = + microsandbox_utils::process_lock::open_lock_file(&path.with_extension("build-lock"))?; + microsandbox_utils::process_lock::lock_exclusive(&build_lock)?; + if let Some(file) = open_pinned(&path, length)? { + return Ok(CachedMemory { + path, + identity: identity.clone(), + file, + regions, + cache_hit: true, + reflink: false, + prepare_us: started.elapsed().as_micros(), + }); + } + + let staging_dir = tempfile::Builder::new() + .prefix(".memory-") + .tempdir_in(&self.root)?; + let staging_path = staging_dir.path().join("memory"); + let baseline = baseline.filter(|(_, cached)| cached.regions == regions); + if let Some((previous, cached)) = baseline { + let bytes = previous.to_canonical_bytes().map_err(io::Error::other)?; + if ObjectId::from_bytes(&bytes).map_err(io::Error::other)? != cached.identity { + return Err(invalid("cache baseline does not match its pinned manifest")); + } + } + let mut reflink = false; + if let Some((_, cached)) = baseline { + let (_, strategy) = + microsandbox_utils::copy::fast_copy_with_strategy(&cached.path, &staging_path)?; + reflink = strategy == microsandbox_utils::copy::FastCopyStrategy::Reflink; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&staging_path, std::fs::Permissions::from_mode(0o600))?; + } + } + let mut staging = OpenOptions::new() + .write(true) + .create(true) + .truncate(false) + .open(&staging_path)?; + // A fresh sparse file supplies all zero extents without allocating or writing RAM-sized + // buffers. Only immutable nonzero object slices are copied into it. + staging.set_len(length)?; + let previous = baseline.map(|(manifest, _)| { + manifest + .extents + .iter() + .map(|extent| (extent.start, extent)) + .collect::>() + }); + let mut objects = BTreeMap::>::new(); + let mut region_index = 0; + for extent in &manifest.extents { + while extent.start >= regions[region_index].guest_address + regions[region_index].length + { + region_index += 1; + } + if previous.as_ref().and_then(|map| map.get(&extent.start)) == Some(&extent) { + continue; + } + let region = ®ions[region_index]; + let offset = region.file_offset + (extent.start - region.guest_address); + if let MemoryExtentContent::Object(content) = &extent.content { + objects.entry(content.object.clone()).or_default().push(( + offset, + content.object_offset, + extent.length, + )); + } else if baseline.is_some() { + // A newly zero range must overwrite the cloned bytes, never resurrect them. + // Bound the temporary allocation independently of guest RAM size. + staging.seek(SeekFrom::Start(offset))?; + let zeros = [0u8; 64 * 1024]; + let mut remaining = extent.length; + while remaining > 0 { + let count = remaining.min(zeros.len() as u64) as usize; + staging.write_all(&zeros[..count])?; + remaining -= count as u64; + } + } + } + let objects = consume_objects(objects, &mut staging)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + staging.set_permissions(std::fs::Permissions::from_mode(0o400))?; + } + let syncing = Instant::now(); + staging.sync_all()?; + let file_sync_us = syncing.elapsed().as_micros(); + // Windows readers deliberately deny write sharing. Close the completed writer before + // publishing/opening its immutable view; keeping it open would cause a sharing violation. + drop(staging); + let mut directory_sync_us = 0; + let file = publish_pinned_entry(&staging_path, &path, length, || { + let syncing = Instant::now(); + #[cfg(unix)] + File::open(&self.root)?.sync_all()?; + directory_sync_us = syncing.elapsed().as_micros(); + Ok(()) + })?; + tracing::info!( + target: "microsandbox_checkpoint_timing", + operation = "memory_cache_materialize", + total_us = started.elapsed().as_micros(), + object_pipeline_us = objects.elapsed_us, + object_write_us = objects.consume_us, + object_bytes = objects.object_bytes, + file_sync_us, + directory_sync_us, + "memory cache construction timing" + ); + Ok(CachedMemory { + path, + identity: identity.clone(), + file, + regions, + cache_hit: false, + reflink, + prepare_us: started.elapsed().as_micros(), + }) + } + + /// Remove an unpinned immutable entry. `false` means absent or still owned by a VM. + /// + /// Never truncate or hole-punch a live entry. POSIX open-handle lifetime also protects a + /// reader that opened the inode immediately before an eviction acquired its exclusive lock. + pub fn evict(&self, identity: &ObjectId) -> io::Result { + let path = self.entry_path(identity); + evict_unpinned(&path) + } + + fn entry_path(&self, identity: &ObjectId) -> PathBuf { + // Geometry lives in the identity-bearing manifest. Native alignment is local realization + // policy, so a cache prepared on a different page-size host must not collide with it. + self.root.join(format!( + "{}-{}.ram", + identity.as_str().replace(':', "-"), + self.page_size + )) + } +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +/// Pin the completed inode before exposing its name to cooperative eviction. An older builder +/// may win publication without taking our build lock; pin its winner or retry if it was evicted. +fn publish_pinned_entry( + staging: &Path, + path: &Path, + length: u64, + after_publication: impl FnOnce() -> io::Result<()>, +) -> io::Result { + let staged = open_pinned(staging, length)? + .ok_or_else(|| io::Error::other("completed memory staging disappeared"))?; + let file = loop { + match std::fs::hard_link(staging, path) { + Ok(()) => break staged, + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => { + if let Some(winner) = open_pinned(path, length)? { + break winner; + } + } + Err(error) => return Err(error), + } + }; + // The pin also spans the directory durability barrier, which can take arbitrarily long. + after_publication()?; + Ok(file) +} + +fn write_object_slices( + staging: &mut File, + slices: Vec<(u64, u64, u64)>, + bytes: &[u8], +) -> io::Result<()> { + for (target, offset, count) in slices { + let start = + usize::try_from(offset).map_err(|_| invalid("memory object offset overflows"))?; + let count = + usize::try_from(count).map_err(|_| invalid("memory object length overflows"))?; + let end = start + .checked_add(count) + .ok_or_else(|| invalid("memory object slice overflows"))?; + let bytes = bytes + .get(start..end) + .ok_or_else(|| invalid("memory object slice exceeds verified bytes"))?; + staging.seek(SeekFrom::Start(target))?; + staging.write_all(bytes)?; + } + Ok(()) +} + +fn memory_regions( + manifest: &MemoryManifest, + page_size: u64, +) -> io::Result> { + let mut regions: Vec = Vec::new(); + let mut file_length = 0u64; + for extent in &manifest.extents { + if let Some(last) = regions.last_mut() + && last.guest_address.checked_add(last.length) == Some(extent.start) + { + last.length = last + .length + .checked_add(extent.length) + .ok_or_else(|| invalid("memory topology overflows"))?; + } else { + regions.push(CachedMemoryRegion { + guest_address: extent.start, + length: extent.length, + file_offset: file_length, + }); + } + file_length = file_length + .checked_add(extent.length) + .ok_or_else(|| invalid("memory cache size overflows"))?; + } + for region in ®ions { + if !region.guest_address.is_multiple_of(page_size) + || !region.length.is_multiple_of(page_size) + || !region.file_offset.is_multiple_of(page_size) + { + return Err(invalid( + "guest memory topology is not aligned for private mappings on this host", + )); + } + } + Ok(regions) +} + +/// Both cache namespaces use the same inode/lock checks and never mutate mapped RAM. +pub(super) fn evict_unpinned(path: &Path) -> io::Result { + let file = match open_readonly(path) { + Ok(file) => file, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error), + }; + if !microsandbox_utils::process_lock::try_lock_exclusive(&file)? { + return Ok(false); + } + // A competing evictor can have removed this same inode while we waited to acquire it. + // Do not unlink a new realization published at the old name in the meantime. + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let opened = file.metadata()?; + let current = match std::fs::symlink_metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error), + }; + if (opened.dev(), opened.ino()) != (current.dev(), current.ino()) { + return Ok(false); + } + } + #[cfg(windows)] + { + let current = match open_readonly(path) { + Ok(file) => file, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error), + }; + if windows_file_identity(&file)? != windows_file_identity(¤t)? { + return Ok(false); + } + } + std::fs::remove_file(path)?; + Ok(true) +} + +fn open_readonly(path: &Path) -> io::Result { + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK); + } + #[cfg(windows)] + { + use std::os::windows::fs::OpenOptionsExt; + use windows_sys::Win32::Storage::FileSystem::{FILE_SHARE_DELETE, FILE_SHARE_READ}; + options.share_mode(FILE_SHARE_READ | FILE_SHARE_DELETE); + } + options.open(path) +} + +pub(super) fn open_pinned(path: &Path, length: u64) -> io::Result> { + let file = match open_readonly(path) { + Ok(file) => file, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + let metadata = file.metadata()?; + if !metadata.is_file() || metadata.len() != length { + return Err(invalid( + "memory cache entry has invalid type or length; evict and rebuild it", + )); + } + microsandbox_utils::process_lock::lock_shared(&file)?; + Ok(Some(file)) +} + +#[cfg(windows)] +fn windows_file_identity(file: &File) -> io::Result<(u32, u32, u32)> { + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::Storage::FileSystem::{ + BY_HANDLE_FILE_INFORMATION, GetFileInformationByHandle, + }; + let mut info: BY_HANDLE_FILE_INFORMATION = unsafe { std::mem::zeroed() }; + if unsafe { GetFileInformationByHandle(file.as_raw_handle(), &mut info) } == 0 { + return Err(io::Error::last_os_error()); + } + Ok(( + info.dwVolumeSerialNumber, + info.nFileIndexHigh, + info.nFileIndexLow, + )) +} + +/// Guest RAM must not inherit broad read permissions from a custom cache parent. +#[cfg(windows)] +fn restrict_cache_directory(path: &Path) -> io::Result<()> { + use std::os::windows::ffi::OsStrExt; + use windows_sys::Win32::Foundation::LocalFree; + use windows_sys::Win32::Security::Authorization::ConvertStringSecurityDescriptorToSecurityDescriptorW; + use windows_sys::Win32::Security::{ + DACL_SECURITY_INFORMATION, PROTECTED_DACL_SECURITY_INFORMATION, SetFileSecurityW, + }; + // OWNER RIGHTS follows the actual owner; SYSTEM is retained for OS maintenance. Children + // inherit these ACEs. This is local-user confidentiality, not an adversarial-host boundary. + let sddl: Vec = "D:P(A;OICI;FA;;;OW)(A;OICI;FA;;;SY)\0" + .encode_utf16() + .collect(); + let mut descriptor = std::ptr::null_mut(); + if unsafe { + ConvertStringSecurityDescriptorToSecurityDescriptorW( + sddl.as_ptr(), + 1, + &mut descriptor, + std::ptr::null_mut(), + ) + } == 0 + { + return Err(io::Error::last_os_error()); + } + let path: Vec = path.as_os_str().encode_wide().chain(Some(0)).collect(); + let success = unsafe { + SetFileSecurityW( + path.as_ptr(), + DACL_SECURITY_INFORMATION | PROTECTED_DACL_SECURITY_INFORMATION, + descriptor, + ) + }; + let result = if success == 0 { + Err(io::Error::last_os_error()) + } else { + Ok(()) + }; + unsafe { + LocalFree(descriptor); + } + result +} + +fn invalid(message: &str) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, message) +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use microsandbox_image::checkpoint::{ContentRef, MemoryCaptureMode, MemoryExtent}; + #[cfg(unix)] + use std::os::unix::fs::FileExt; + #[cfg(windows)] + trait ReadAt { + fn read_exact_at(&self, bytes: &mut [u8], offset: u64) -> io::Result<()>; + } + #[cfg(windows)] + impl ReadAt for File { + fn read_exact_at(&self, bytes: &mut [u8], offset: u64) -> io::Result<()> { + use std::io::Read; + let mut file = self.try_clone()?; + file.seek(SeekFrom::Start(offset))?; + file.read_exact(bytes) + } + } + + fn fixture(page: u64) -> (MemoryManifest, ObjectId, Vec) { + let bytes = vec![0x5a; page as usize]; + let object = ObjectId::from_bytes(&bytes).unwrap(); + let manifest = MemoryManifest { + schema: "microsandbox.memory/1".into(), + architecture: std::env::consts::ARCH.into(), + guest_page_size: 4096, + topology_generation: 1, + generation: 1, + capture_mode: MemoryCaptureMode::Full, + pause_generation: 1, + extents: vec![ + MemoryExtent { + start: 0, + length: page, + content: MemoryExtentContent::Object(ContentRef { + object: object.clone(), + object_offset: 0, + }), + }, + MemoryExtent { + start: page, + length: page, + content: MemoryExtentContent::Zero, + }, + MemoryExtent { + start: page * 4, + length: page, + content: MemoryExtentContent::Object(ContentRef { + object, + object_offset: 0, + }), + }, + ], + }; + let id = ObjectId::from_bytes(&manifest.to_canonical_bytes().unwrap()).unwrap(); + (manifest, id, bytes) + } + + #[test] + fn publication_already_owns_a_pin_before_the_directory_barrier() { + let directory = tempfile::tempdir().unwrap(); + let staging = directory.path().join("staged"); + let published = directory.path().join("published"); + std::fs::write(&staging, b"complete memory").unwrap(); + let file = publish_pinned_entry(&staging, &published, 15, || { + // This is the old publication-to-pin window. Run eviction on another thread so the + // test exercises independent lock ownership even on process-oriented platforms. + assert!(!std::thread::scope(|scope| { + scope + .spawn(|| evict_unpinned(&published).unwrap()) + .join() + .unwrap() + })); + Ok(()) + }) + .unwrap(); + assert!(!evict_unpinned(&published).unwrap()); + drop(file); + assert!(evict_unpinned(&published).unwrap()); + } + + #[test] + fn publication_pins_an_existing_winner_without_replacing_its_inode() { + let directory = tempfile::tempdir().unwrap(); + let staging = directory.path().join("staged"); + let published = directory.path().join("published"); + std::fs::write(&staging, b"candidate").unwrap(); + std::fs::write(&published, b"thewinner").unwrap(); + let file = publish_pinned_entry(&staging, &published, 9, || { + assert!(!evict_unpinned(&published).unwrap()); + Ok(()) + }) + .unwrap(); + assert_eq!(std::fs::read(&published).unwrap(), b"thewinner"); + drop(file); + assert!(evict_unpinned(&published).unwrap()); + } + + #[test] + fn materialize_once_reuse_pinned_bytes_and_evict_after_last_owner() { + let directory = tempfile::tempdir().unwrap(); + let cache = MemoryCache::open(directory.path()).unwrap(); + let (manifest, id, bytes) = fixture(cache.page_size); + let mut reads = 0; + let first = cache + .materialize(&manifest, &id, |_| { + reads += 1; + Ok(bytes.clone()) + }) + .unwrap(); + assert_eq!(reads, 1); + assert!(!first.cache_hit); + assert_eq!(first.regions.len(), 2); + assert_eq!(first.file.metadata().unwrap().len(), cache.page_size * 3); + let mut zero = vec![1; cache.page_size as usize]; + first + .file + .read_exact_at(&mut zero, cache.page_size) + .unwrap(); + assert!(zero.iter().all(|byte| *byte == 0)); + let second = cache + .materialize(&manifest, &id, |_| { + panic!("warm cache reread a portable object") + }) + .unwrap(); + assert!(second.cache_hit); + assert!(!cache.evict(&id).unwrap()); + drop(first); + assert!(!cache.evict(&id).unwrap()); + drop(second); + assert!(cache.evict(&id).unwrap()); + assert!(!cache.evict(&id).unwrap()); + } + + #[test] + fn parallel_materialization_preserves_holes_zeroes_and_warm_pins() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + let directory = tempfile::tempdir().unwrap(); + let cache = MemoryCache::open(directory.path()).unwrap(); + let (manifest, id, bytes) = fixture(cache.page_size); + let reads = AtomicUsize::new(0); + let first = cache + .materialize_parallel(&manifest, &id, |_, buffer| { + reads.fetch_add(1, Ordering::Relaxed); + buffer.resize(bytes.len(), 0); + buffer.copy_from_slice(&bytes); + Ok(CheckpointObjectReadTiming::default()) + }) + .unwrap(); + assert_eq!(reads.load(Ordering::Relaxed), 1); + let mut actual = vec![0xff; cache.page_size as usize * 3]; + first.file.read_exact_at(&mut actual, 0).unwrap(); + assert_eq!(&actual[..bytes.len()], bytes); + assert!( + actual[bytes.len()..bytes.len() * 2] + .iter() + .all(|byte| *byte == 0) + ); + assert_eq!(&actual[bytes.len() * 2..], bytes); + let second = cache + .materialize_parallel(&manifest, &id, |_, _| panic!("warm restore reread objects")) + .unwrap(); + assert!(second.cache_hit); + assert!(!cache.evict(&id).unwrap()); + drop(first); + assert!(!cache.evict(&id).unwrap()); + drop(second); + assert!(cache.evict(&id).unwrap()); + } + + #[test] + fn failed_parallel_read_or_slice_never_publishes_a_cache_entry() { + let directory = tempfile::tempdir().unwrap(); + let cache = MemoryCache::open(directory.path()).unwrap(); + let (manifest, id, _) = fixture(cache.page_size); + assert!( + cache + .materialize_parallel(&manifest, &id, |_, _| { + Err(io::Error::other("injected verification failure")) + }) + .is_err() + ); + assert_eq!(payload_count(directory.path()), 0); + assert!( + cache + .materialize_parallel(&manifest, &id, |_, buffer| { + buffer.resize(1, 0); + Ok(CheckpointObjectReadTiming::default()) + }) + .is_err() + ); + assert_eq!(payload_count(directory.path()), 0); + } + + #[test] + fn descendant_reuses_unchanged_objects_and_clears_new_zero_ranges() { + let directory = tempfile::tempdir().unwrap(); + let cache = MemoryCache::open(directory.path()).unwrap(); + let (manifest, id, bytes) = fixture(cache.page_size); + let baseline = cache + .materialize(&manifest, &id, |_| Ok(bytes.clone())) + .unwrap(); + let mut descendant = manifest.clone(); + descendant.generation += 1; + descendant.extents[0].content = MemoryExtentContent::Zero; + let changed = vec![0x7c; cache.page_size as usize]; + let changed_id = ObjectId::from_bytes(&changed).unwrap(); + descendant.extents[1].content = MemoryExtentContent::Object(ContentRef { + object: changed_id.clone(), + object_offset: 0, + }); + let descendant_id = + ObjectId::from_bytes(&descendant.to_canonical_bytes().unwrap()).unwrap(); + let mut reads = 0; + let child = cache + .materialize_with_baseline( + &descendant, + &descendant_id, + Some((&manifest, &baseline)), + |id| { + assert_eq!(id, &changed_id, "unchanged object was reread"); + reads += 1; + Ok(changed.clone()) + }, + ) + .unwrap(); + assert_eq!(reads, 1); + let mut result = vec![0; cache.page_size as usize * 3]; + child.file.read_exact_at(&mut result, 0).unwrap(); + assert!( + result[..cache.page_size as usize] + .iter() + .all(|byte| *byte == 0) + ); + assert_eq!( + &result[cache.page_size as usize..cache.page_size as usize * 2], + &changed + ); + assert_eq!(&result[cache.page_size as usize * 2..], &bytes); + baseline + .file + .read_exact_at(&mut result[..cache.page_size as usize], 0) + .unwrap(); + assert_eq!( + &result[..cache.page_size as usize], + &bytes, + "baseline was mutated" + ); + assert!(!cache.evict(&id).unwrap()); + } + + #[test] + fn reject_a_manifest_paired_with_the_wrong_baseline() { + let directory = tempfile::tempdir().unwrap(); + let cache = MemoryCache::open(directory.path()).unwrap(); + let (manifest, id, bytes) = fixture(cache.page_size); + let baseline = cache + .materialize(&manifest, &id, |_| Ok(bytes.clone())) + .unwrap(); + let mut wrong = manifest.clone(); + wrong.generation += 1; + let target = ObjectId::from_bytes(&wrong.to_canonical_bytes().unwrap()).unwrap(); + assert!( + cache + .materialize_with_baseline(&wrong, &target, Some((&wrong, &baseline)), |_| panic!( + "must reject before reads" + )) + .is_err() + ); + assert_eq!(payload_count(directory.path()), 1); + } + + #[test] + fn failed_materialization_does_not_publish_or_leave_staging() { + let directory = tempfile::tempdir().unwrap(); + let cache = MemoryCache::open(directory.path()).unwrap(); + let (manifest, id, _) = fixture(cache.page_size); + let failure = cache.materialize(&manifest, &id, |_| { + Err(io::Error::other("injected object read failure")) + }); + assert!(failure.is_err()); + assert_eq!(payload_count(directory.path()), 0); + assert!(cache.materialize(&manifest, &id, |_| Ok(vec![])).is_err()); + assert_eq!(payload_count(directory.path()), 0); + } + + #[test] + fn reject_wrong_manifest_identity_and_host_alignment() { + let directory = tempfile::tempdir().unwrap(); + let cache = MemoryCache::open(directory.path()).unwrap(); + let (mut manifest, id, _) = fixture(cache.page_size); + manifest.generation += 1; + assert!( + cache + .materialize(&manifest, &id, |_| panic!( + "identity rejection must precede reads" + )) + .is_err() + ); + let (mut manifest, _, _) = fixture(4096); + manifest.extents.truncate(1); + assert!(memory_regions(&manifest, 16384).is_err()); + } + + fn payload_count(root: &Path) -> usize { + // Build-lock inodes intentionally survive failed builders. Only RAM or staging entries + // count as payloads; removing lock files would permit two independent flock owners. + std::fs::read_dir(root.join("snapshots")) + .unwrap() + .filter(|entry| { + entry + .as_ref() + .unwrap() + .path() + .extension() + .and_then(|ext| ext.to_str()) + != Some("build-lock") + }) + .count() + } + + #[test] + fn concurrent_builders_publish_one_immutable_inode() { + #[cfg(unix)] + use std::os::unix::fs::MetadataExt; + let directory = tempfile::tempdir().unwrap(); + let cache = MemoryCache::open(directory.path()).unwrap(); + let (manifest, id, bytes) = fixture(cache.page_size); + let barrier = std::sync::Barrier::new(2); + let reads = std::sync::atomic::AtomicUsize::new(0); + std::thread::scope(|scope| { + let run = || { + barrier.wait(); + cache + .materialize(&manifest, &id, |_| { + reads.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + Ok(bytes.clone()) + }) + .unwrap() + }; + let first = scope.spawn(run); + let second = scope.spawn(run); + let first = first.join().unwrap(); + let second = second.join().unwrap(); + #[cfg(unix)] + assert_eq!( + first.file.metadata().unwrap().ino(), + second.file.metadata().unwrap().ino() + ); + #[cfg(windows)] + assert_eq!( + windows_file_identity(&first.file).unwrap(), + windows_file_identity(&second.file).unwrap() + ); + }); + assert_eq!(reads.load(std::sync::atomic::Ordering::Relaxed), 1); + assert_eq!( + std::fs::read_dir(directory.path().join("snapshots")) + .unwrap() + .count(), + 2 + ); + } + + #[test] + fn unlinked_backing_survives_without_snapshot_paths() { + let directory = tempfile::tempdir().unwrap(); + let cache = MemoryCache::open(directory.path()).unwrap(); + let (manifest, id, bytes) = fixture(cache.page_size); + let memory = cache + .materialize(&manifest, &id, |_| Ok(bytes.clone())) + .unwrap(); + std::fs::remove_file(cache.entry_path(&id)).unwrap(); + let mut actual = vec![0; bytes.len()]; + memory + .file + .read_exact_at(&mut actual, cache.page_size * 2) + .unwrap(); + assert_eq!(actual, bytes); + } +} diff --git a/crates/runtime/lib/checkpoint/mod.rs b/crates/runtime/lib/checkpoint/mod.rs index e6a61f2b2..c9789b342 100644 --- a/crates/runtime/lib/checkpoint/mod.rs +++ b/crates/runtime/lib/checkpoint/mod.rs @@ -1,8 +1,14 @@ //! Runtime-owned composite checkpoint production. +#[cfg(feature = "runner")] +mod capture_pipeline; #[cfg(feature = "runner")] mod coordinator; mod disk; +mod local; +mod local_memory; +mod memory_cache; +mod object_pipeline; #[cfg(feature = "runner")] mod restore; @@ -11,14 +17,17 @@ mod restore; //-------------------------------------------------------------------------------------------------- #[cfg(feature = "runner")] -pub(crate) use coordinator::{CheckpointCoordinator, CheckpointResult}; -#[cfg(feature = "runner")] -pub(crate) use disk::recover_runtime_owned_root; +pub(crate) use coordinator::{CheckpointCoordinator, CheckpointResult, UserPause}; pub use disk::{ DiskCompactionResult, RuntimeOwnedRootChain, RuntimeOwnedRootLayer, compact_stopped_root, grow_stopped_root, load_runtime_owned_root_chain, recover_stopped_root_growth, }; #[cfg(feature = "runner")] +pub(crate) use disk::{recover_runtime_owned_root, seed_restored_root_disk}; +pub use local::LocalBranchState; +pub use local_memory::LocalMemory; +pub use memory_cache::{CachedMemory, CachedMemoryRegion, MemoryCache}; +#[cfg(feature = "runner")] pub(crate) use restore::{PreparedCheckpointRestore, RestoredAgentState}; //-------------------------------------------------------------------------------------------------- diff --git a/crates/runtime/lib/checkpoint/object_pipeline.rs b/crates/runtime/lib/checkpoint/object_pipeline.rs new file mode 100644 index 000000000..f8e97d496 --- /dev/null +++ b/crates/runtime/lib/checkpoint/object_pipeline.rs @@ -0,0 +1,248 @@ +//! Bounded object verification with construction-thread-only consumption. + +use std::io; +use std::sync::mpsc; +use std::time::Instant; + +use microsandbox_image::checkpoint::{CheckpointObjectReadTiming, ObjectId}; + +//-------------------------------------------------------------------------------------------------- +// Constants +//-------------------------------------------------------------------------------------------------- + +const MAX_READERS: usize = 4; +const MAX_OBJECT_BYTES: usize = 32 * 1024 * 1024; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +#[derive(Debug, Default)] +pub(super) struct ObjectPipelineTiming { + /// Sum of worker read times; parallel worker times are not pipeline wall time. + pub read_us: u128, + pub hash_us: u128, + pub consume_us: u128, + pub elapsed_us: u128, + pub object_bytes: u64, +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +/// Verify at most four objects ahead, returning each buffer to its reader after consumption. +/// +/// Only the invoking thread consumes bytes. Reader completion order may differ from object-ID +/// order, so callers must supply disjoint destination slices. On any error, dropping the work +/// channels cancels queued work and all active readers are joined before their inputs disappear. +pub(super) fn consume_verified_objects( + objects: impl IntoIterator, + read: impl Fn(&ObjectId, &mut Vec) -> io::Result + Sync, + mut consume: impl FnMut(T, &[u8]) -> io::Result<()>, +) -> io::Result { + let objects: Vec<_> = objects.into_iter().collect(); + let readers = std::thread::available_parallelism() + .map(usize::from) + .unwrap_or(1) + .min(MAX_READERS) + .min(objects.len()); + run_pipeline(objects, readers, &read, &mut consume) +} + +fn run_pipeline( + objects: Vec<(ObjectId, T)>, + readers: usize, + read: &(impl Fn(&ObjectId, &mut Vec) -> io::Result + Sync), + consume: &mut impl FnMut(T, &[u8]) -> io::Result<()>, +) -> io::Result { + let started = Instant::now(); + if objects.is_empty() { + return Ok(ObjectPipelineTiming::default()); + } + let readers = readers.clamp(1, MAX_READERS).min(objects.len()); + std::thread::scope(|scope| { + let (ready_tx, ready_rx) = mpsc::sync_channel(readers); + let mut senders = Vec::with_capacity(readers); + let mut handles = Vec::with_capacity(readers); + for worker in 0..readers { + let (work_tx, work_rx) = mpsc::sync_channel::<(ObjectId, T, Vec)>(1); + let ready_tx = ready_tx.clone(); + let handle = std::thread::Builder::new() + .name("checkpoint-reader".into()) + .spawn_scoped(scope, move || { + while let Ok((id, item, mut bytes)) = work_rx.recv() { + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + read(&id, &mut bytes) + })) + .unwrap_or_else(|_| Err(io::Error::other("checkpoint reader panicked"))) + .and_then(|timing| { + if bytes.len() > MAX_OBJECT_BYTES { + Err(io::Error::new( + io::ErrorKind::InvalidData, + "checkpoint reader exceeded the memory object limit", + )) + } else { + Ok(timing) + } + }); + if ready_tx.send((worker, item, bytes, result)).is_err() { + break; + } + } + }); + match handle { + Ok(handle) => { + senders.push(work_tx); + handles.push(handle); + } + Err(error) => { + drop(senders); + drop(ready_rx); + for handle in handles { + let _ = handle.join(); + } + return Err(error); + } + } + } + drop(ready_tx); + let result = (|| { + let total = objects.len(); + let mut pending = objects.into_iter(); + for sender in &senders { + let (id, item) = pending.next().expect("one initial item per reader"); + sender + .send((id, item, Vec::with_capacity(MAX_OBJECT_BYTES))) + .map_err(|_| io::Error::other("checkpoint reader stopped before reading"))?; + } + let mut timings = ObjectPipelineTiming::default(); + for _ in 0..total { + let (worker, item, bytes, timing) = ready_rx + .recv() + .map_err(|_| io::Error::other("checkpoint reader stopped before completion"))?; + let timing = timing?; + timings.read_us += timing.read_us; + timings.hash_us += timing.hash_us; + timings.object_bytes += bytes.len() as u64; + let consuming = Instant::now(); + consume(item, &bytes)?; + timings.consume_us += consuming.elapsed().as_micros(); + if let Some((id, item)) = pending.next() { + senders[worker].send((id, item, bytes)).map_err(|_| { + io::Error::other("checkpoint reader stopped before its next object") + })?; + } + } + timings.elapsed_us = started.elapsed().as_micros(); + Ok(timings) + })(); + // Break both directions before joining: an errored consumer must not leave workers + // blocked on a full completion queue or waiting for their next returned buffer. + drop(senders); + drop(ready_rx); + let mut panicked = false; + for handle in handles { + panicked |= handle.join().is_err(); + } + if panicked { + return Err(io::Error::other("checkpoint object reader panicked")); + } + result + }) +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + use std::sync::Mutex; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use super::*; + + fn jobs(count: usize) -> Vec<(ObjectId, usize)> { + (0..count) + .map(|index| (ObjectId::from_bytes(&index.to_le_bytes()).unwrap(), index)) + .collect() + } + + #[test] + fn buffers_are_bounded_reused_and_consumed_on_the_caller_thread() { + let pointers = Mutex::new(BTreeSet::new()); + let owner = std::thread::current().id(); + let mut observed = BTreeSet::new(); + let timings = run_pipeline( + jobs(20), + 2, + &|_, bytes| { + pointers.lock().unwrap().insert(bytes.as_ptr() as usize); + assert_eq!(bytes.capacity(), MAX_OBJECT_BYTES); + bytes.resize(100, 7); + Ok(CheckpointObjectReadTiming { + read_us: 2, + hash_us: 3, + }) + }, + &mut |index, bytes| { + assert_eq!(std::thread::current().id(), owner); + assert_eq!(bytes, &[7; 100]); + observed.insert(index); + Ok(()) + }, + ) + .unwrap(); + assert_eq!(pointers.lock().unwrap().len(), 2); + assert_eq!(observed.len(), 20); + assert_eq!(timings.object_bytes, 2_000); + assert_eq!((timings.read_us, timings.hash_us), (40, 60)); + } + + #[test] + fn consumption_failure_joins_readers_and_does_not_start_remaining_jobs() { + let reads = AtomicUsize::new(0); + let active = AtomicUsize::new(0); + let result = run_pipeline( + jobs(20), + 2, + &|_, _| { + active.fetch_add(1, Ordering::SeqCst); + reads.fetch_add(1, Ordering::SeqCst); + active.fetch_sub(1, Ordering::SeqCst); + Ok(CheckpointObjectReadTiming::default()) + }, + &mut |_, _| Err(io::Error::other("injected guest write failure")), + ); + assert!(result.is_err()); + assert_eq!(active.load(Ordering::SeqCst), 0); + assert!(reads.load(Ordering::SeqCst) <= 2); + } + + #[test] + fn unverified_bytes_never_reach_the_consumer() { + let result = run_pipeline( + jobs(1), + 1, + &|_, bytes| { + bytes.extend_from_slice(b"corrupt"); + Err(io::Error::new(io::ErrorKind::InvalidData, "bad digest")) + }, + &mut |_, _| panic!("unverified object was consumed"), + ); + assert_eq!(result.unwrap_err().kind(), io::ErrorKind::InvalidData); + } + + #[test] + fn reader_panic_does_not_leave_other_readers_waiting_forever() { + let result = run_pipeline( + jobs(4), + 2, + &|_, _| panic!("injected reader panic"), + &mut |_, _| panic!("panicked reader was consumed"), + ); + assert!(result.unwrap_err().to_string().contains("panicked")); + } +} diff --git a/crates/runtime/lib/checkpoint/restore.rs b/crates/runtime/lib/checkpoint/restore.rs index f8381d19a..b56c824b0 100644 --- a/crates/runtime/lib/checkpoint/restore.rs +++ b/crates/runtime/lib/checkpoint/restore.rs @@ -8,7 +8,9 @@ use std::time::Instant; use microsandbox_image::checkpoint::{ CheckpointClosure, MemoryExtentContent, ObjectId, ResourceDescriptor, ResourceTreatment, }; -use microsandbox_protocol::core::Ready; +use microsandbox_protocol::core::{ + Ready, WORKLOAD_TRANSPORT_BARRIER_VERSION, WorkloadTransportCredit, WorkloadTransportPosition, +}; use microsandbox_protocol::message::{MessageType, PROTOCOL_VERSION}; use super::coordinator::TYPE_FS; @@ -30,7 +32,8 @@ const MAX_MEMORY_OBJECT_BYTES: u64 = 32 * 1024 * 1024; pub(crate) struct PreparedCheckpointRestore { execution: msb_krun::ExecutionState, devices: Vec, - memory: CheckpointMemoryRestore, + memory: Option, + local_memory: Option, agent: RestoredAgentState, } @@ -42,6 +45,12 @@ pub(crate) struct RestoredAgentState { pub(crate) ready: Ready, /// Checkpoint attempt that owns the captured workload freeze. pub(crate) attempt_id: String, + /// Complete host input admitted before the captured freeze acknowledgement. + pub(crate) host_input: WorkloadTransportPosition, + /// Absolute guest grants, including debt retained by inherited stdin. + pub(crate) input_credit: WorkloadTransportCredit, + /// Complete dedicated guest bulk output observed before capture. + pub(crate) guest_bulk_bytes_target: u64, } enum PreparedDeviceRestore { @@ -61,6 +70,57 @@ struct CheckpointMemoryRestore { //-------------------------------------------------------------------------------------------------- impl PreparedCheckpointRestore { + /// Borrow disk admission while the prepared durable restore owns its validated closure. + pub(crate) fn disk_closure(&self) -> Option<&CheckpointClosure> { + self.memory.as_ref().map(|memory| &memory.closure) + } + + /// Decode a local handoff and pin its RAM before constructing any guest mappings. + pub(crate) fn open_local(root: PathBuf, expected_id: &str) -> Result { + let state = super::LocalBranchState::open(&root).map_err(|e| e.to_string())?; + if state.id != expected_id { + return Err("local branch identity differs".into()); + } + let read = |id: &ObjectId, limit| { + super::LocalBranchState::read_object(&root, id, limit).map_err(|e| e.to_string()) + }; + let execution = msb_krun::ExecutionState::decode(&read( + &state.execution_state, + MAX_EXECUTION_STATE_BYTES, + )?) + .map_err(|e| e.to_string())?; + if execution.pause_generation() != state.pause_generation { + return Err("branch execution epoch differs".into()); + } + let devices = decode_devices(&state.devices, state.pause_generation, read)?; + let resource = state + .resources + .iter() + .find(|r| r.id == "guest:agentd") + .ok_or("branch has no captured agent identity")?; + let agent = parse_restored_agent_resource(resource, &state.id)?; + let file = state.memory.pin().map_err(|e| e.to_string())?; + let regions = state + .memory + .regions + .into_iter() + .map(|region| msb_krun::PrivateMemoryRegion { + guest_address: region.guest_address, + length: region.length, + file_offset: region.file_offset, + }) + .collect(); + let backing = + msb_krun::PrivateMemoryBacking::new(file, regions).map_err(|e| e.to_string())?; + Ok(Self { + execution, + devices, + memory: None, + local_memory: Some(backing), + agent, + }) + } + /// Resolve and decode every construction-time state envelope before building the VM. pub(crate) fn open(root: PathBuf, expected_root: &str) -> Result { let total_started = Instant::now(); @@ -89,50 +149,11 @@ impl PreparedCheckpointRestore { let execution_us = execution_started.elapsed().as_micros(); let devices_started = Instant::now(); - let mut devices = Vec::with_capacity(closure.checkpoint().devices.len()); - for device in &closure.checkpoint().devices { - let max_state_bytes = if device.device_type == TYPE_FS { - MAX_FS_DEVICE_STATE_BYTES - } else { - MAX_DEVICE_STATE_BYTES - }; - let bytes = closure - .read_object(&device.state, max_state_bytes) - .map_err(|error| format!("read checkpoint device {}: {error}", device.device_id))?; - if device.device_type == 2 { - let state = msb_krun::BlockDeviceState::decode(&bytes).map_err(|error| { - format!( - "decode checkpoint block device {}: {error}", - device.device_id - ) - })?; - if state.pause_generation != pause_generation { - return Err(format!( - "block device {} does not belong to the checkpoint epoch", - device.device_id - )); - } - devices.push(PreparedDeviceRestore::Block { - device_id: device.device_id.clone(), - state, - }); - } else { - let state = msb_krun::VirtioDeviceState::decode(&bytes).map_err(|error| { - format!( - "decode checkpoint virtio device {}: {error}", - device.device_id - ) - })?; - if state.pause_generation != pause_generation || state.device_id != device.device_id - { - return Err(format!( - "virtio device {} does not belong to the checkpoint binding/epoch", - device.device_id - )); - } - devices.push(PreparedDeviceRestore::Virtio(state)); - } - } + let devices = decode_devices( + &closure.checkpoint().devices, + pause_generation, + |id, limit| closure.read_object(id, limit).map_err(|e| e.to_string()), + )?; let devices_us = devices_started.elapsed().as_micros(); tracing::info!( target: "microsandbox_checkpoint_timing", @@ -151,15 +172,59 @@ impl PreparedCheckpointRestore { Ok(Self { execution, devices, - memory: CheckpointMemoryRestore { closure }, + memory: Some(CheckpointMemoryRestore { closure }), + local_memory: None, agent, }) } /// Install all restore sources and leave the VM at an explicit activation gate. - pub(crate) fn install(self, vm: &mut msb_krun::Vm) -> RestoredAgentState { + pub(crate) fn install( + self, + vm: &mut msb_krun::Vm, + cache_root: Option, + ) -> Result { vm.set_execution_restore(self.execution); - vm.set_memory_restore(self.memory); + if let Some(backing) = self.local_memory { + vm.set_private_memory_backing(backing); + } else if let Some(root) = cache_root { + let closure = &self + .memory + .as_ref() + .expect("durable restore memory") + .closure; + let cache = super::MemoryCache::open(root).map_err(|e| e.to_string())?; + let cached = cache + .materialize_parallel( + closure.memory(), + &closure.checkpoint().memory, + |id, bytes| { + closure + .read_object_into(id, MAX_MEMORY_OBJECT_BYTES, bytes) + .map_err(io::Error::other) + }, + ) + .map_err(|e| e.to_string())?; + tracing::info!( + cache_hit = cached.cache_hit, + prepare_us = cached.prepare_us, + "prepared private memory backing" + ); + let regions = cached + .regions + .into_iter() + .map(|region| msb_krun::PrivateMemoryRegion { + guest_address: region.guest_address, + length: region.length, + file_offset: region.file_offset, + }) + .collect(); + let backing = msb_krun::PrivateMemoryBacking::new(cached.file, regions) + .map_err(|e| e.to_string())?; + vm.set_private_memory_backing(backing); + } else { + vm.set_memory_restore(self.memory.expect("durable restore memory")); + } for device in self.devices { match device { PreparedDeviceRestore::Block { device_id, state } => { @@ -169,7 +234,7 @@ impl PreparedCheckpointRestore { } } vm.set_start_paused(true); - self.agent + Ok(self.agent) } } @@ -182,9 +247,7 @@ impl msb_krun::VmMemoryRestoreSource for CheckpointMemoryRestore { let total_started = Instant::now(); let mut zero_write_us = 0u128; let mut zero_bytes = 0u64; - let mut object_read_us = 0u128; let mut guest_write_us = 0u128; - let mut object_bytes = 0u64; let mut guest_object_bytes = 0u64; let mut object_extent_count = 0usize; let mut objects: BTreeMap> = @@ -209,53 +272,58 @@ impl msb_krun::VmMemoryRestoreSource for CheckpointMemoryRestore { } } - // Read and identity-check each packed object exactly once, write all of its referenced - // guest ranges, then release the small object buffer. This fuses integrity with the - // unavoidable restore pass without retaining a RAM-sized cache. + // Read and identity-check each packed object exactly once with bounded read-ahead. + // Guest ranges are disjoint and only this construction thread writes them; workers + // never obtain guest-memory access or permit activation before verification completes. let object_count = objects.len(); - for (id, extents) in objects { - let read_started = Instant::now(); - let bytes = self - .closure - .read_object(&id, MAX_MEMORY_OBJECT_BYTES) - .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error.to_string()))?; - object_read_us += read_started.elapsed().as_micros(); - object_bytes = object_bytes.saturating_add(bytes.len() as u64); - for (range, offset) in extents { - let start = usize::try_from(offset).map_err(|_| { - io::Error::new( - io::ErrorKind::InvalidData, - "memory object offset is too large", - ) - })?; - let length = usize::try_from(range.length()).map_err(|_| { - io::Error::new(io::ErrorKind::InvalidData, "memory extent is too large") - })?; - let end = start.checked_add(length).ok_or_else(|| { - io::Error::new(io::ErrorKind::InvalidData, "memory object slice overflows") - })?; - let slice = bytes.get(start..end).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "memory object slice exceeds verified bytes", - ) - })?; - let write_started = Instant::now(); - target.write_bytes(range, slice)?; - guest_write_us += write_started.elapsed().as_micros(); - guest_object_bytes = guest_object_bytes.saturating_add(range.length()); - } - } + let pipeline = super::object_pipeline::consume_verified_objects( + objects, + |id, bytes| { + self.closure + .read_object_into(id, MAX_MEMORY_OBJECT_BYTES, bytes) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error.to_string())) + }, + |extents, bytes| { + for (range, offset) in extents { + let start = usize::try_from(offset).map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidData, + "memory object offset is too large", + ) + })?; + let length = usize::try_from(range.length()).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidData, "memory extent is too large") + })?; + let end = start.checked_add(length).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "memory object slice overflows") + })?; + let slice = bytes.get(start..end).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "memory object slice exceeds verified bytes", + ) + })?; + let write_started = Instant::now(); + target.write_bytes(range, slice)?; + guest_write_us += write_started.elapsed().as_micros(); + guest_object_bytes = guest_object_bytes.saturating_add(range.length()); + } + Ok(()) + }, + )?; tracing::info!( target: "microsandbox_checkpoint_timing", operation = "restore_memory", total_us = total_started.elapsed().as_micros(), - object_read_us, + object_read_us = pipeline.read_us + pipeline.hash_us, + object_io_worker_us = pipeline.read_us, + object_hash_worker_us = pipeline.hash_us, + object_pipeline_us = pipeline.elapsed_us, guest_write_us, zero_write_us, object_count, object_extent_count, - object_bytes, + object_bytes = pipeline.object_bytes, guest_object_bytes, zero_bytes, "checkpoint memory restore timing" @@ -268,6 +336,56 @@ impl msb_krun::VmMemoryRestoreSource for CheckpointMemoryRestore { // Functions //-------------------------------------------------------------------------------------------------- +fn decode_devices( + references: &[microsandbox_image::checkpoint::DeviceStateRef], + pause_generation: u64, + mut read: impl FnMut(&ObjectId, u64) -> Result, String>, +) -> Result, String> { + let mut devices = Vec::with_capacity(references.len()); + for device in references { + let max_state_bytes = if device.device_type == TYPE_FS { + MAX_FS_DEVICE_STATE_BYTES + } else { + MAX_DEVICE_STATE_BYTES + }; + let bytes = read(&device.state, max_state_bytes) + .map_err(|error| format!("read checkpoint device {}: {error}", device.device_id))?; + if device.device_type == 2 { + let state = msb_krun::BlockDeviceState::decode(&bytes).map_err(|error| { + format!( + "decode checkpoint block device {}: {error}", + device.device_id + ) + })?; + if state.pause_generation != pause_generation { + return Err(format!( + "block device {} does not belong to the checkpoint epoch", + device.device_id + )); + } + devices.push(PreparedDeviceRestore::Block { + device_id: device.device_id.clone(), + state, + }); + } else { + let state = msb_krun::VirtioDeviceState::decode(&bytes).map_err(|error| { + format!( + "decode checkpoint virtio device {}: {error}", + device.device_id + ) + })?; + if state.pause_generation != pause_generation || state.device_id != device.device_id { + return Err(format!( + "virtio device {} does not belong to the checkpoint binding/epoch", + device.device_id + )); + } + devices.push(PreparedDeviceRestore::Virtio(state)); + } + } + Ok(devices) +} + fn parse_restored_agent(closure: &CheckpointClosure) -> Result { let resource = closure .checkpoint() @@ -304,11 +422,41 @@ fn parse_restored_agent_resource( )); } + let ready: Ready = serde_json::from_str(value("ready")?) + .map_err(|error| format!("checkpoint guest readiness is invalid: {error}"))?; + if ready.workload_transport_barrier_version != Some(WORKLOAD_TRANSPORT_BARRIER_VERSION) { + return Err("checkpoint guest has an unsupported development transport-credit contract; recreate the full snapshot with a matching build".into()); + } + let host_input: WorkloadTransportPosition = + serde_json::from_str(value("transport_host_input")?) + .map_err(|error| format!("checkpoint host input position is invalid: {error}"))?; + let input_credit: WorkloadTransportCredit = + serde_json::from_str(value("transport_input_credit")?) + .map_err(|error| format!("checkpoint input credit is invalid: {error}"))?; + if host_input.control_bytes > input_credit.control_bytes + || host_input.control_frames > input_credit.control_frames + || host_input.bulk_bytes > input_credit.bulk_bytes + || host_input.bulk_frames > input_credit.bulk_frames + { + return Err("checkpoint transport input exceeds captured credit".into()); + } + let guest_bulk_bytes_target = value("transport_guest_bulk_bytes")? + .parse::() + .map_err(|error| format!("checkpoint guest bulk position is invalid: {error}"))?; + if ready.bulk_transport.is_none() && guest_bulk_bytes_target != 0 { + return Err("combined checkpoint has a dedicated guest bulk counter".into()); + } Ok(RestoredAgentState { protocol_generation, - ready: serde_json::from_str(value("ready")?) - .map_err(|error| format!("checkpoint guest readiness is invalid: {error}"))?, - attempt_id: checkpoint_id.into(), + ready, + host_input, + input_credit, + guest_bulk_bytes_target, + attempt_id: resource + .binding + .get("attempt_id") + .cloned() + .unwrap_or_else(|| checkpoint_id.into()), }) } @@ -334,6 +482,15 @@ mod tests { ("boot_time_ns".into(), "10".into()), ("init_time_ns".into(), "20".into()), ("ready_time_ns".into(), "30".into()), + ( + "transport_host_input".into(), + serde_json::to_string(&WorkloadTransportPosition::default()).unwrap(), + ), + ( + "transport_input_credit".into(), + serde_json::to_string(&WorkloadTransportCredit::default()).unwrap(), + ), + ("transport_guest_bulk_bytes".into(), "0".into()), ( "ready".into(), serde_json::to_string(&Ready { @@ -341,6 +498,9 @@ mod tests { boot_time_ns: 10, init_time_ns: 20, ready_time_ns: 30, + workload_transport_barrier_version: Some( + WORKLOAD_TRANSPORT_BARRIER_VERSION, + ), ..Default::default() }) .unwrap(), @@ -372,6 +532,20 @@ mod tests { assert!(error.contains("protocol generation 8 is unsupported")); } + #[test] + fn rejects_development_snapshot_with_stdin_charged_to_control() { + let mut resource = agent_resource(PROTOCOL_VERSION); + let mut ready: Ready = serde_json::from_str(&resource.binding["ready"]).unwrap(); + ready.workload_transport_barrier_version = Some(1); + resource + .binding + .insert("ready".into(), serde_json::to_string(&ready).unwrap()); + let error = parse_restored_agent_resource(&resource, "old-development-cut") + .err() + .unwrap(); + assert!(error.contains("unsupported development transport-credit contract")); + } + #[test] fn rejects_reconstructed_agent_resource() { let mut resource = agent_resource(PROTOCOL_VERSION); @@ -383,4 +557,74 @@ mod tests { assert!(error.contains("incompatible resource treatment")); } + + #[test] + fn rejects_development_capture_without_proven_transport_position() { + let mut resource = agent_resource(PROTOCOL_VERSION); + resource.binding.remove("transport_host_input"); + assert!( + parse_restored_agent_resource(&resource, "attempt") + .err() + .unwrap() + .contains("transport_host_input") + ); + } + + #[test] + fn rejects_transport_debt_beyond_captured_grants() { + let mut resource = agent_resource(PROTOCOL_VERSION); + resource.binding.insert( + "transport_host_input".into(), + serde_json::to_string(&WorkloadTransportPosition { + control_bytes: 1, + ..Default::default() + }) + .unwrap(), + ); + assert!( + parse_restored_agent_resource(&resource, "attempt") + .err() + .unwrap() + .contains("exceeds captured credit") + ); + } + + #[test] + fn combined_transport_accepts_input_bulk_counter_but_not_dedicated_output_cut() { + let mut resource = agent_resource(PROTOCOL_VERSION); + resource.binding.insert( + "transport_host_input".into(), + serde_json::to_string(&WorkloadTransportPosition { + bulk_bytes: 32, + bulk_frames: 1, + ..Default::default() + }) + .unwrap(), + ); + resource.binding.insert( + "transport_input_credit".into(), + serde_json::to_string(&WorkloadTransportCredit { + bulk_bytes: 64, + bulk_frames: 2, + ..Default::default() + }) + .unwrap(), + ); + assert_eq!( + parse_restored_agent_resource(&resource, "attempt") + .unwrap() + .host_input + .bulk_bytes, + 32 + ); + resource + .binding + .insert("transport_guest_bulk_bytes".into(), "1".into()); + assert!( + parse_restored_agent_resource(&resource, "attempt") + .err() + .unwrap() + .contains("dedicated guest bulk counter") + ); + } } diff --git a/crates/runtime/lib/client/control.rs b/crates/runtime/lib/client/control.rs index adddb48b6..1bc5ff2d5 100644 --- a/crates/runtime/lib/client/control.rs +++ b/crates/runtime/lib/client/control.rs @@ -35,6 +35,26 @@ pub const CONTROL_PROTOCOL_VERSION: u16 = 1; #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(tag = "op", rename_all = "snake_case")] pub enum ControlRequest { + /// Seal only the owned root disk; never capture guest RAM or execution state. + DiskCheckpointCreate { + /// Caller-selected safe capture identity. + checkpoint_id: String, + }, + /// Capture directly into a reserved child-owned local handoff directory. + BranchCreate { + /// Unique capture identity matching the child's reservation. + branch_id: String, + /// Reserved sandbox name in this runtime's backend, never a host path. + child_name: String, + /// Cache in which the caller holds its handoff lock; must match the source runtime. + memory_cache_dir: PathBuf, + }, + /// Retain a resident pause until an explicit resume or stop. + Pause, + /// Resume a user-owned resident pause. + Resume, + /// Inspect user pause and full-capture availability without entering the guest. + PauseState, /// Grow the owned root disk and mounted ext4 filesystem without rebooting. RootDiskGrow { /// Target capacity in bytes. @@ -138,6 +158,12 @@ pub struct SecretValue(pub String); /// The reply to any control request. #[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct ControlResponse { + /// Completed local handoff, deliberately not a portable checkpoint identity. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub branch: Option, + /// Resident pause status for lifecycle operations. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pause: Option, /// Guest-observed root capacities after successful filesystem expansion. #[serde(default, skip_serializing_if = "Option::is_none")] pub root_disk: Option, @@ -171,6 +197,9 @@ pub struct ControlResponse { /// failure such as an unsuccessful source resume. #[serde(default, skip_serializing_if = "Option::is_none")] pub checkpoint: Option, + /// Sealed disk-only capture, with no RAM or execution-state closure. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub disk_checkpoint: Option, } /// Published checkpoint information returned by the runtime control executor. @@ -190,6 +219,17 @@ pub struct CheckpointControlState { pub memory_emitted_bytes: u64, } +/// Immutable disk closure returned after a live root-head rollover. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct DiskCheckpointControlState { + /// Capture identity echoed from the request. + pub checkpoint_id: String, + /// Runtime-owned closure, independent of the source's new writable head. + pub path: PathBuf, + /// Complete base-to-head disk generation; contains no memory or device payloads. + pub disk: microsandbox_image::checkpoint::DiskGenerationManifest, +} + /// Verified capacity and measured phases of a completed online root growth. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RootDiskGrowthResult { @@ -211,6 +251,12 @@ pub struct RootDiskGrowthResult { /// resize-capable and secrets-incapable. #[derive(Debug, Default, Clone, Copy, Serialize, Deserialize)] pub struct ControlCapabilities { + /// Direct local branch capture is supported on this host. + #[serde(default)] + pub branch_create: bool, + /// Resident pause/resume with identity-preserving clock correction. + #[serde(default)] + pub pause_resume: bool, /// Host control supports root growth; guest capability is checked before mutation. #[serde(default)] pub root_disk_grow: bool, @@ -229,6 +275,20 @@ pub struct ControlCapabilities { /// Same-epoch composite checkpoint capture is available. #[serde(default)] pub checkpoint_create: bool, + /// Disk-only live capture is available without full-state admission. + #[serde(default)] + pub disk_checkpoint_create: bool, +} + +/// Host-confirmed resident suspension state. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PauseControlState { + /// Whether a user pause is currently held. + pub paused: bool, + /// Whether a failed operation has fenced ordinary resume and mutations. + pub recovery_required: bool, + /// Why full capture cannot use this pause, if guest preparation is unavailable. + pub capture_unavailable: Option, } /// Memory sizing carried in [`ControlResponse`], all in MiB. @@ -402,6 +462,9 @@ mod tests { memory_resize: false, secrets_update: true, checkpoint_create: true, + disk_checkpoint_create: true, + branch_create: true, + pause_resume: true, disk_compact: true, }), ..Default::default() diff --git a/crates/runtime/lib/client/ipc.rs b/crates/runtime/lib/client/ipc.rs index b7e703aa9..c42f0cccb 100644 --- a/crates/runtime/lib/client/ipc.rs +++ b/crates/runtime/lib/client/ipc.rs @@ -133,6 +133,46 @@ pub fn acquire_lifecycle_guard( } } +/// Stable namespace shared by launchers and read-time recovery. +pub fn sandbox_transition_lock_path(run_dir: &Path, name: &str) -> PathBuf { + let digest = Sha256::digest(name.as_bytes()); + run_dir + .join("creation-locks") + .join(format!("{}.lock", hex::encode(&digest[..16]))) +} + +/// Claim a name transition without waiting; a live creator must never be reaped as abandoned. +pub fn try_acquire_transition_guard(run_dir: &Path, name: &str) -> std::io::Result> { + let path = sandbox_transition_lock_path(run_dir, name); + std::fs::create_dir_all(path.parent().expect("transition path has parent"))?; + let file = microsandbox_utils::process_lock::open_lock_file(&path)?; + if microsandbox_utils::process_lock::try_lock_exclusive(&file)? { + Ok(Some(file)) + } else { + Ok(None) + } +} + +/// Stable capture-publication ownership, outside the removable source directory. +pub fn snapshot_lineage_lock_path(run_dir: &Path, name: &str) -> PathBuf { + lifecycle_lock_path(run_dir, name).with_extension("snapshot-lineage.lock") +} + +/// Claim source publication ownership without waiting, including from a runtime exit observer. +pub fn try_acquire_snapshot_lineage_guard( + run_dir: &Path, + name: &str, +) -> std::io::Result> { + let path = snapshot_lineage_lock_path(run_dir, name); + std::fs::create_dir_all(path.parent().expect("lineage lock path has parent"))?; + let file = microsandbox_utils::process_lock::open_lock_file(&path)?; + if microsandbox_utils::process_lock::try_lock_exclusive(&file)? { + Ok(Some(file)) + } else { + Ok(None) + } +} + /// Try to acquire exclusive lifecycle ownership without blocking. pub fn try_acquire_lifecycle_guard( run_dir: &Path, diff --git a/crates/runtime/lib/client/launch.rs b/crates/runtime/lib/client/launch.rs index 3935fb512..6b27a6429 100644 --- a/crates/runtime/lib/client/launch.rs +++ b/crates/runtime/lib/client/launch.rs @@ -77,7 +77,11 @@ pub struct StartupCommand { /// The bulk `msb sandbox` configuration delivered over the config fd. #[derive(Debug, Default, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct LaunchConfig { + /// Required execution intent. Restore intent must never be inferred from optional hints. + pub execution: ExecutionIntent, + /// Path to the sandbox database file. pub db_path: PathBuf, @@ -124,6 +128,10 @@ pub struct LaunchConfig { #[serde(default)] pub thp: TransparentHugePagePolicy, + /// Backend-resolved protected cache for explicit memory captures and restores. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub memory_cache_dir: Option, + /// Per-writable-raw-disk hard budget for buffered host dirty data. #[serde(default, skip_serializing_if = "Option::is_none")] pub block_writeback_limit_bytes: Option, @@ -192,6 +200,11 @@ pub struct LaunchConfig { #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct CheckpointRestoreConfig { + /// Restore a local branch handoff instead of a durable checkpoint closure. + pub local_branch: bool, + /// Require private CoW memory rather than eager restoration. + /// Kept inside the strict restore contract: an unsupported mode must not become a boot. + pub forked: bool, /// Path to the complete eager checkpoint closure. pub closure: PathBuf, /// Expected algorithm-qualified composite checkpoint root. @@ -200,6 +213,17 @@ pub struct CheckpointRestoreConfig { pub checkpoint_id: String, } +/// Required process-construction intent, independent of any guest startup command. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ExecutionIntent { + /// Construct a fresh guest from its disk/image state. + #[default] + Boot, + /// Continue captured execution; a complete recognized restore source is mandatory. + Restore, +} + /// Lifetime bounds for the sandbox. #[derive(Debug, Default, Clone, Serialize, Deserialize)] pub struct Lifecycle { @@ -295,13 +319,130 @@ pub struct FileMountConfig { pub filename: String, } +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl LaunchConfig { + /// Decode and validate execution intent before allocating or starting a VM. + pub fn decode(bytes: &[u8]) -> Result { + let config: Self = serde_json::from_slice(bytes) + .map_err(|error| format!("invalid launch config: {error}"))?; + match (config.execution, config.checkpoint_restore.as_ref()) { + (ExecutionIntent::Boot, None) => {} + (ExecutionIntent::Restore, Some(restore)) => { + if restore.closure.as_os_str().is_empty() + || (!restore.local_branch && restore.checkpoint_root.is_empty()) + || restore.checkpoint_id.is_empty() + { + return Err("restore requires a complete checkpoint source".into()); + } + if restore.local_branch && (!restore.forked || !restore.checkpoint_root.is_empty()) + { + return Err( + "local branch requires private memory and no durable checkpoint root" + .into(), + ); + } + if config.startup.is_some() { + return Err("restore cannot execute a fresh startup command".into()); + } + } + _ => return Err("execution intent and checkpoint restore source disagree".into()), + } + Ok(config) + } +} + //-------------------------------------------------------------------------------------------------- // Tests //-------------------------------------------------------------------------------------------------- #[cfg(test)] mod tests { - use super::{FileMountConfig, LaunchConfig}; + use super::*; + + fn restore_request() -> serde_json::Value { + serde_json::to_value(LaunchConfig { + execution: ExecutionIntent::Restore, + checkpoint_restore: Some(CheckpointRestoreConfig { + local_branch: false, + forked: true, + closure: "/owned/child/restore".into(), + checkpoint_root: "blake3:captured-root".into(), + checkpoint_id: "captured".into(), + }), + ..Default::default() + }) + .unwrap() + } + + fn decode(value: serde_json::Value) -> Result { + LaunchConfig::decode(&serde_json::to_vec(&value).unwrap()) + } + + #[test] + fn matching_boot_and_restore_intents_are_accepted() { + assert!(decode(serde_json::to_value(LaunchConfig::default()).unwrap()).is_ok()); + let restored = decode(restore_request()).unwrap(); + assert!(restored.checkpoint_restore.unwrap().forked); + } + + #[test] + fn unsupported_or_missing_restore_never_becomes_boot() { + for mutation in [ + "missing", + "null", + "unknown_outer", + "unknown_nested", + "unknown_intent", + "boot", + ] { + let mut request = restore_request(); + match mutation { + "missing" => { + request + .as_object_mut() + .unwrap() + .remove("checkpoint_restore"); + } + "null" => request["checkpoint_restore"] = serde_json::Value::Null, + "unknown_outer" => request["branch_restore"] = serde_json::json!({}), + "unknown_nested" => request["checkpoint_restore"]["unsupported"] = true.into(), + "unknown_intent" => request["execution"] = "future_restore".into(), + "boot" => request["execution"] = "boot".into(), + _ => unreachable!(), + } + assert!(decode(request).is_err(), "{mutation}"); + } + } + + #[test] + fn launch_requires_explicit_intent_and_memory_policy() { + let mut request = restore_request(); + request.as_object_mut().unwrap().remove("execution"); + assert!(decode(request).is_err()); + let mut request = restore_request(); + request["checkpoint_restore"] + .as_object_mut() + .unwrap() + .remove("forked"); + assert!(decode(request).is_err()); + } + + #[test] + fn local_branch_requires_restore_and_private_memory_without_a_fake_root() { + let mut request = restore_request(); + request["checkpoint_restore"]["local_branch"] = true.into(); + assert!(decode(request.clone()).is_err()); + request["checkpoint_restore"]["checkpoint_root"] = "".into(); + assert!(decode(request.clone()).is_ok()); + request["checkpoint_restore"]["forked"] = false.into(); + assert!(decode(request.clone()).is_err()); + request["checkpoint_restore"]["forked"] = true.into(); + request["execution"] = "boot".into(); + assert!(decode(request).is_err()); + } #[test] fn isolated_file_mount_survives_the_client_runner_handoff() { diff --git a/crates/runtime/lib/client/maintenance.rs b/crates/runtime/lib/client/maintenance.rs index 4863d9043..af788f7e1 100644 --- a/crates/runtime/lib/client/maintenance.rs +++ b/crates/runtime/lib/client/maintenance.rs @@ -27,7 +27,8 @@ use microsandbox_db::entity::{ }; use sea_orm::sea_query::{Expr, OnConflict}; use sea_orm::{ - ColumnTrait, Condition, DbErr, EntityTrait, QueryFilter, QueryOrder, QuerySelect, Set, + ColumnTrait, Condition, ConnectionTrait, DbErr, EntityTrait, QueryFilter, QueryOrder, + QuerySelect, Set, }; use crate::{RuntimeError, RuntimeResult}; @@ -87,7 +88,7 @@ pub enum CleanupOutcome { /// The sandbox is persistent, so it is intentionally left in place. SkippedPersistent, - /// The sandbox is not in a terminal status yet. + /// The sandbox is not terminal yet or a source snapshot is still being published. SkippedActive, /// The sandbox still has a run with a live PID. @@ -363,6 +364,14 @@ async fn cleanup_terminal_ephemeral_sandbox_inner( return Ok(CleanupOutcome::SkippedActive); } + // Snapshot publication may outlive its runtime. Never remove its source cursor or storage + // while that capture owns lineage. Exit observers already own lifecycle, so this must be + // try-only: waiting here would invert the lineage -> lifecycle order used by SDK removal. + let Some(_lineage) = crate::ipc::try_acquire_snapshot_lineage_guard(run_dir, &sandbox.name)? + else { + return Ok(CleanupOutcome::SkippedActive); + }; + let _guard = if owner_holds_guard { None } else { @@ -657,7 +666,7 @@ pub async fn clear_install_exclusive_lease_idempotent( /// migration yet. In that case startup continues so normal migrations can /// create it. Once the table exists, the install-exclusive row becomes a hard /// refusal while unexpired. -pub async fn refuse_if_install_exclusive_held(db: &DbWriteConnection) -> RuntimeResult<()> { +pub async fn refuse_if_install_exclusive_held(db: &C) -> RuntimeResult<()> { let now = chrono::Utc::now().naive_utc(); let lease = match lease_entity::Entity::find_by_id(lease_entity::INSTALL_EXCLUSIVE) .one(db) @@ -717,6 +726,12 @@ async fn reconcile_stale_active( run_dir: &Path, sandbox: &sandbox_entity::Model, ) -> RuntimeResult { + // A creator may have persisted Starting but not spawned its child yet. On Windows it also + // briefly releases the runtime lock for handoff; transition ownership closes both gaps. + let Some(_transition) = crate::ipc::try_acquire_transition_guard(run_dir, &sandbox.name)? + else { + return Ok(false); + }; let Some(_guard) = crate::ipc::try_acquire_lifecycle_guard(run_dir, &sandbox.name)? else { return Ok(false); }; @@ -742,11 +757,13 @@ async fn reconcile_stale_active( .one(db) .await?; - // No active run yet while Starting means the runtime has not inserted a run row. Draining with no active run - // means the stop request already reached a terminal run state, so repair - // the sandbox status instead of leaving future stop callers polling. + // With neither creator nor runtime ownership, Starting without an active run is abandoned. + // Preserve the config (including pending restore intent) while publishing its terminal state. let Some(run) = run else { - if sandbox.status == sandbox_entity::SandboxStatus::Draining { + if matches!( + sandbox.status, + sandbox_entity::SandboxStatus::Starting | sandbox_entity::SandboxStatus::Draining + ) { remove_runtime_socket_artifacts(run_dir, sandboxes_dir, &sandbox.name)?; let now = chrono::Utc::now().naive_utc(); let (terminal_status, _) = stale_runtime_terminal_state(sandbox.status); @@ -762,7 +779,7 @@ async fn reconcile_stale_active( ) .col_expr(sandbox_entity::Column::UpdatedAt, Expr::value(now)) .filter(sandbox_entity::Column::Id.eq(sandbox.id)) - .filter(sandbox_entity::Column::Status.eq(sandbox_entity::SandboxStatus::Draining)) + .filter(sandbox_entity::Column::Status.eq(sandbox.status)) .exec(db) .await?; return Ok(result.rows_affected > 0); @@ -1374,4 +1391,109 @@ mod tests { assert_socket_artifacts_absent(dir.path(), &draining_no_run_sockets, "draining-no-run"); } } + + #[tokio::test] + async fn starting_without_run_is_reaped_only_after_creator_releases_transition() { + let (dir, db) = test_db().await; + let run_dir = dir.path().join("run"); + let id = insert_sandbox( + &db, + "abandoned", + sandbox_entity::SandboxStatus::Starting, + false, + ) + .await; + let model = sandbox_entity::Entity::find_by_id(id) + .one(&db) + .await + .unwrap() + .unwrap(); + let creator = crate::ipc::try_acquire_transition_guard(&run_dir, "abandoned") + .unwrap() + .unwrap(); + assert!( + !reconcile_stale_active(&db, dir.path(), &run_dir, &model) + .await + .unwrap() + ); + assert_eq!( + status_of(&db, id).await, + Some(sandbox_entity::SandboxStatus::Starting) + ); + drop(creator); + assert!( + reconcile_stale_active(&db, dir.path(), &run_dir, &model) + .await + .unwrap() + ); + assert_eq!( + status_of(&db, id).await, + Some(sandbox_entity::SandboxStatus::Crashed) + ); + } + + #[tokio::test] + async fn cleanup_defers_for_lineage_publication_then_retries() { + for owner_holds_guard in [false, true] { + let (dir, db) = test_db().await; + let run_dir = dir.path().join("run"); + let name = "publishing"; + let id = insert_sandbox(&db, name, sandbox_entity::SandboxStatus::Stopped, true).await; + let sandbox_dir = dir.path().join(name); + std::fs::create_dir_all(&sandbox_dir).unwrap(); + let marker = sandbox_dir.join("snapshot-cursor"); + std::fs::write(&marker, b"publication in progress").unwrap(); + #[cfg(unix)] + let agent_socket = { + let path = crate::ipc::canonical_agent_endpoint(&run_dir, name); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(&path, b"runtime endpoint").unwrap(); + path + }; + let lineage = crate::ipc::try_acquire_snapshot_lineage_guard(&run_dir, name) + .unwrap() + .unwrap(); + // Model the synchronous exit observer, which cannot release lifecycle ownership + // just to wait for the SDK publisher. The ordinary maintenance path owns no guard. + let _runtime = owner_holds_guard + .then(|| crate::ipc::acquire_lifecycle_guard(&run_dir, name).unwrap()); + let outcome = tokio::time::timeout( + std::time::Duration::from_secs(1), + cleanup_terminal_ephemeral_sandbox_inner( + &db, + dir.path(), + &run_dir, + id, + owner_holds_guard, + ), + ) + .await + .expect("cleanup must not wait for lineage while owning lifecycle") + .unwrap(); + assert_eq!(outcome, CleanupOutcome::SkippedActive); + assert_eq!(std::fs::read(&marker).unwrap(), b"publication in progress"); + assert_eq!( + status_of(&db, id).await, + Some(sandbox_entity::SandboxStatus::Stopped) + ); + #[cfg(unix)] + assert!(agent_socket.exists()); + + drop(lineage); + let outcome = cleanup_terminal_ephemeral_sandbox_inner( + &db, + dir.path(), + &run_dir, + id, + owner_holds_guard, + ) + .await + .unwrap(); + assert_eq!(outcome, CleanupOutcome::Removed); + assert!(!sandbox_dir.exists()); + assert_eq!(status_of(&db, id).await, None); + #[cfg(unix)] + assert!(!agent_socket.exists()); + } + } } diff --git a/crates/runtime/lib/runner/clock.rs b/crates/runtime/lib/runner/clock.rs index cd695657e..cf86bba29 100644 --- a/crates/runtime/lib/runner/clock.rs +++ b/crates/runtime/lib/runner/clock.rs @@ -6,10 +6,9 @@ use bytes::Bytes; use microsandbox_protocol::codec; use microsandbox_protocol::core::ClockSync; use microsandbox_protocol::message::{Message, MessageType}; -use tokio::sync::mpsc; use tokio::task::JoinHandle; -use crate::relay::ControlWrite; +use crate::relay::{ControlWrite, ControlWriter}; use crate::{RuntimeError, RuntimeResult}; //-------------------------------------------------------------------------------------------------- @@ -31,13 +30,13 @@ const CLOCK_SYNC_WAKE_THRESHOLD: Duration = Duration::from_secs(6); /// Spawns a background task that keeps the guest wall clock aligned with the host. pub(crate) fn spawn_clock_sync_task( - agent_tx: mpsc::Sender, + agent_tx: ControlWriter, already_synchronized: bool, ) -> JoinHandle<()> { tokio::spawn(clock_sync_task(agent_tx, already_synchronized)) } -async fn clock_sync_task(agent_tx: mpsc::Sender, already_synchronized: bool) { +async fn clock_sync_task(agent_tx: ControlWriter, already_synchronized: bool) { let mut last_wall = SystemTime::now(); // Full restore completed the kernel clock barrier before workload thaw. Do not immediately // overwrite it with a queued userspace timestamp. Ordinary boot keeps its existing sync. @@ -79,14 +78,28 @@ async fn clock_sync_task(agent_tx: mpsc::Sender, already_synchroni } } -async fn send_clock_sync(agent_tx: &mpsc::Sender) -> RuntimeResult { +async fn send_clock_sync(agent_tx: &ControlWriter) -> RuntimeResult { let now = SystemTime::now(); - let elapsed = now + agent_tx + .send(ControlWrite::clock_sync()?) + .await + .map_err(|_| RuntimeError::Custom("agent relay ring writer channel closed".into()))?; + Ok(now) +} + +/// Sample only when the ordinary writer can admit the maintenance frame to the console queue. +pub(crate) fn current_clock_sync_frame() -> RuntimeResult { + let elapsed = SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .map_err(|e| RuntimeError::Custom(format!("clock sync before Unix epoch: {e}")))?; let unix_time_nanos = u64::try_from(elapsed.as_nanos()).map_err(|_| { RuntimeError::Custom("clock sync timestamp does not fit in u64 nanoseconds".into()) })?; + encode_clock_sync_frame(unix_time_nanos) +} + +/// The queue initially reserves the maximum encoded timestamp size, then sends the actual value. +pub(crate) fn encode_clock_sync_frame(unix_time_nanos: u64) -> RuntimeResult { let sync = ClockSync { unix_time_nanos }; let msg = Message::with_payload(MessageType::ClockSync, 0, &sync) .map_err(|e| RuntimeError::Custom(format!("encode clock sync: {e}")))?; @@ -94,10 +107,5 @@ async fn send_clock_sync(agent_tx: &mpsc::Sender) -> RuntimeResult let mut buf = Vec::new(); codec::encode_to_buf(&msg, &mut buf) .map_err(|e| RuntimeError::Custom(format!("encode clock sync frame: {e}")))?; - agent_tx - .send(Bytes::from(buf).into()) - .await - .map_err(|_| RuntimeError::Custom("agent relay ring writer channel closed".into()))?; - - Ok(now) + Ok(Bytes::from(buf)) } diff --git a/crates/runtime/lib/runner/console.rs b/crates/runtime/lib/runner/console.rs index d87a80610..db2304e64 100644 --- a/crates/runtime/lib/runner/console.rs +++ b/crates/runtime/lib/runner/console.rs @@ -98,6 +98,10 @@ pub struct ByteQueueSnapshot { /// transmitted by the guest agent", `rx_ring` = "bytes received by the guest /// agent". pub struct ConsoleSharedState { + /// Trusted lifecycle control shares no capacity with SDK input. + pub(crate) workload_control: Arc, + /// User pause or recovery fence for new guest operations and idle policy. + pub resident_paused: Arc, /// Guest → Host: console TX thread pushes byte chunks, relay pops them. pub tx_ring: ByteQueue, @@ -159,6 +163,8 @@ impl ConsoleSharedState { /// Create shared state with a specific byte capacity in each direction. pub fn with_capacity(byte_capacity: usize) -> Self { Self { + workload_control: super::workload_control::WorkloadControl::new(), + resident_paused: Arc::new(std::sync::atomic::AtomicBool::new(false)), tx_ring: ByteQueue::new(byte_capacity), rx_ring: ByteQueue::new(byte_capacity), tx_wake: WakePipe::new(), @@ -171,6 +177,7 @@ impl ConsoleSharedState { /// Unblock console producers because the runtime is shutting down. pub fn close(&self) { + self.workload_control.close(); self.closed.store(true, Ordering::Release); self.tx_capacity_wake.wake(); self.rx_capacity_wake.wake(); diff --git a/crates/runtime/lib/runner/control.rs b/crates/runtime/lib/runner/control.rs index 5015b88db..2db85ea4c 100644 --- a/crates/runtime/lib/runner/control.rs +++ b/crates/runtime/lib/runner/control.rs @@ -318,17 +318,38 @@ mod tests { )); } + #[test] + fn disk_only_capture_has_a_distinct_wire_operation() { + let request = ControlRequest::DiskCheckpointCreate { + checkpoint_id: "disk_test".into(), + }; + let json = serde_json::to_string(&request).unwrap(); + assert_eq!( + serde_json::from_str::(&json).unwrap()["op"], + "disk_checkpoint_create" + ); + assert!( + matches!(serde_json::from_str::(&json).unwrap(), ControlRequest::DiskCheckpointCreate { checkpoint_id } if checkpoint_id == "disk_test") + ); + // An older runtime's capability response cannot accidentally opt into this operation. + let old: ControlCapabilities = serde_json::from_str(r#"{"cpu_resize":false,"memory_resize":false,"secrets_update":false,"checkpoint_create":true}"#).unwrap(); + assert!(!old.disk_checkpoint_create); + } + #[test] fn capabilities_response_serializes_flags() { let response = ControlResponse { ok: true, capabilities: Some(ControlCapabilities { + branch_create: true, + pause_resume: true, root_disk_grow: true, disk_compact: true, cpu_resize: true, memory_resize: false, secrets_update: true, checkpoint_create: true, + disk_checkpoint_create: true, }), ..Default::default() }; diff --git a/crates/runtime/lib/runner/control/executor.rs b/crates/runtime/lib/runner/control/executor.rs index b1fb71df9..6ed520f12 100644 --- a/crates/runtime/lib/runner/control/executor.rs +++ b/crates/runtime/lib/runner/control/executor.rs @@ -6,13 +6,10 @@ use std::io::Write; use std::path::{Path, PathBuf}; use std::sync::Mutex; -#[cfg(unix)] -use std::fs::File; - use rand::Rng as _; use sha2::{Digest as _, Sha256}; -use crate::checkpoint::{CheckpointCoordinator, CheckpointResult}; +use crate::checkpoint::{CheckpointCoordinator, CheckpointResult, UserPause}; use crate::control::*; use crate::vm::VmConfig; use microsandbox_protocol::bootstrap::GuestBootstrap; @@ -31,6 +28,8 @@ const RUNTIME_BOOT_ID_FILE: &str = "runtime-boot-id"; /// One in-process authority for all host-owned runtime mutations. pub struct RuntimeControlExecutor { + pause_observation: std::sync::RwLock, + resident_paused: std::sync::Arc, vm: msb_krun::VmControl, #[cfg(feature = "net")] secrets: Option, @@ -44,6 +43,7 @@ struct ExecutorState { dedup: BTreeMap, dedup_order: VecDeque, checkpoint: CheckpointCoordinator, + user_pause: Option, } #[derive(Clone)] @@ -57,8 +57,10 @@ struct DedupEntry { //-------------------------------------------------------------------------------------------------- impl RuntimeControlExecutor { - /// Construct an executor and durably publish a fresh runtime boot identity. - pub fn new( + /// Construct an executor and atomically publish a fresh runtime boot identity. + // Construction binds the VM, guest channel, and host lifecycle resources once. + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( vm: msb_krun::VmControl, #[cfg(feature = "net")] secrets: Option< microsandbox_network::secrets::handle::SecretsHandle, @@ -68,6 +70,8 @@ impl RuntimeControlExecutor { guest_bootstrap: &GuestBootstrap, runtime: tokio::runtime::Handle, agent_sock: &Path, + workload_control: std::sync::Arc, + resident_paused: std::sync::Arc, ) -> Result { let runtime_boot_id = new_runtime_boot_id(); persist_runtime_boot_id(runtime_dir, &runtime_boot_id) @@ -78,8 +82,19 @@ impl RuntimeControlExecutor { guest_bootstrap, runtime, agent_sock, + workload_control, )?; Ok(Self { + pause_observation: std::sync::RwLock::new(ControlResponse { + ok: true, + pause: Some(super::PauseControlState { + paused: false, + recovery_required: false, + capture_unavailable: None, + }), + ..Default::default() + }), + resident_paused, vm, #[cfg(feature = "net")] secrets, @@ -90,14 +105,22 @@ impl RuntimeControlExecutor { dedup: BTreeMap::new(), dedup_order: VecDeque::new(), checkpoint, + user_pause: None, }), }) } /// Execute a legacy command through the same exclusive mutation path. pub fn execute_legacy(&self, command: ControlRequest) -> ControlResponse { + // Observation remains available during a long capture. It describes the last completed + // lifecycle transition; it neither borrows nor releases mutation/pause authority. + if matches!(command, ControlRequest::PauseState) { + return self.pause_observation.read().unwrap().clone(); + } let mut state = self.state.lock().unwrap(); - self.execute_locked(&mut state, command) + let response = self.execute_locked(&mut state, command); + *self.pause_observation.write().unwrap() = pause_response(&state); + response } /// Execute a fenced, idempotent control request. @@ -161,6 +184,7 @@ impl RuntimeControlExecutor { let request_id = envelope.request_id; let response = self.execute_locked(&mut state, envelope.command); + *self.pause_observation.write().unwrap() = pause_response(&state); let response = ControlEnvelopeResponse { request_id: request_id.clone(), runtime: snapshot_state(&state), @@ -180,6 +204,14 @@ impl RuntimeControlExecutor { state: &mut ExecutorState, request: ControlRequest, ) -> ControlResponse { + // Gate the authoritative operation, including idempotent Resume on a running VM. + // Clients need no separate capability exchange, and refusal never changes ownership. + if matches!(request, ControlRequest::Pause | ControlRequest::Resume) + && let Some(response) = + unsupported_lifecycle_request(&request, self.vm.clock_sync_supported()) + { + return response; + } let mutation = matches!( request, ControlRequest::MemoryTarget { .. } @@ -187,9 +219,22 @@ impl RuntimeControlExecutor { | ControlRequest::CpuTarget { .. } | ControlRequest::SecretsUpdate { .. } | ControlRequest::CheckpointCreate { .. } + | ControlRequest::DiskCheckpointCreate { .. } + | ControlRequest::BranchCreate { .. } | ControlRequest::DiskCompact { dry_run: false, .. } + | ControlRequest::Pause + | ControlRequest::Resume ); - if mutation && state.lifecycle != RuntimeLifecycle::Running { + let resident_operation = state.user_pause.is_some() + && matches!( + request, + ControlRequest::Pause + | ControlRequest::Resume + | ControlRequest::CheckpointCreate { .. } + | ControlRequest::DiskCheckpointCreate { .. } + | ControlRequest::BranchCreate { .. } + ); + if mutation && state.lifecycle != RuntimeLifecycle::Running && !resident_operation { return control_error( "runtime_busy", "runtime lifecycle does not currently admit mutations", @@ -197,6 +242,77 @@ impl RuntimeControlExecutor { } let response = match request { + ControlRequest::DiskCheckpointCreate { checkpoint_id } => { + state.lifecycle = RuntimeLifecycle::Quiescing; + match state.checkpoint.capture_disk( + &self.vm, + &checkpoint_id, + state.user_pause.as_ref(), + ) { + Ok(result) => { + state.lifecycle = if state.user_pause.is_some() { + RuntimeLifecycle::Quiesced + } else { + RuntimeLifecycle::Running + }; + ControlResponse { + ok: true, + disk_checkpoint: Some(result), + ..Default::default() + } + } + Err(error) => { + if error.keep_paused { + state.user_pause = None; + } + state.lifecycle = if error.keep_paused || state.user_pause.is_some() { + RuntimeLifecycle::Quiesced + } else { + RuntimeLifecycle::Running + }; + control_error("disk_checkpoint_failed", error.to_string()) + } + } + } + ControlRequest::Pause => { + if state.user_pause.is_none() { + self.resident_paused + .store(true, std::sync::atomic::Ordering::Release); + let attempt = format!("pause-{}-{}", state.runtime_boot_id, state.revision); + state.lifecycle = RuntimeLifecycle::Quiescing; + match state.checkpoint.pause_user(&self.vm, &attempt) { + Ok(paused) => { + state.user_pause = Some(paused); + state.lifecycle = RuntimeLifecycle::Quiesced; + } + Err(error) => { + state.lifecycle = if error.keep_paused { + RuntimeLifecycle::Quiesced + } else { + RuntimeLifecycle::Running + }; + self.resident_paused + .store(error.keep_paused, std::sync::atomic::Ordering::Release); + return control_error("pause_failed", error.to_string()); + } + } + } + pause_response(state) + } + ControlRequest::Resume => { + if let Some(paused) = state.user_pause.take() { + if let Err(error) = state.checkpoint.resume_user(&self.vm, &paused) { + // A failed resume becomes recovery-owned, never a public resume token. + state.lifecycle = RuntimeLifecycle::Quiesced; + return control_error("resume_recovery_required", error.to_string()); + } + state.lifecycle = RuntimeLifecycle::Running; + self.resident_paused + .store(false, std::sync::atomic::Ordering::Release); + } + pause_response(state) + } + ControlRequest::PauseState => pause_response(state), ControlRequest::RootDiskGrow { size_bytes } => { match state.checkpoint.grow_root(&self.vm, size_bytes) { Ok(root_disk) => ControlResponse { @@ -234,6 +350,44 @@ impl RuntimeControlExecutor { } } } + ControlRequest::BranchCreate { + branch_id, + child_name, + memory_cache_dir, + } => { + state.lifecycle = RuntimeLifecycle::Quiescing; + match state.checkpoint.branch( + &self.vm, + &branch_id, + &child_name, + &memory_cache_dir, + state.user_pause.as_ref(), + ) { + Ok(result) => { + state.lifecycle = if state.user_pause.is_some() { + RuntimeLifecycle::Quiesced + } else { + RuntimeLifecycle::Running + }; + ControlResponse { + ok: true, + branch: Some(result.path), + ..Default::default() + } + } + Err(error) => { + if error.keep_paused { + state.user_pause = None; + } + state.lifecycle = if error.keep_paused || state.user_pause.is_some() { + RuntimeLifecycle::Quiesced + } else { + RuntimeLifecycle::Running + }; + control_error("branch_failed", error.to_string()) + } + } + } ControlRequest::CheckpointCreate { checkpoint_id, intent, @@ -253,13 +407,23 @@ impl RuntimeControlExecutor { microsandbox_image::checkpoint::CaptureIntent::TransparentTransfer } }, + state.user_pause.as_ref(), ) { Ok(result) => { - state.lifecycle = RuntimeLifecycle::Running; + state.lifecycle = if state.user_pause.is_some() { + RuntimeLifecycle::Quiesced + } else { + RuntimeLifecycle::Running + }; checkpoint_response(Some(result), true, None, None) } Err(error) => { - state.lifecycle = if error.keep_paused { + // A failed rebind can invalidate the user's original pause authority. + // Recovery-owned suspension must never be released by ordinary resume. + if error.keep_paused { + state.user_pause = None; + } + state.lifecycle = if error.keep_paused || state.user_pause.is_some() { RuntimeLifecycle::Quiesced } else { RuntimeLifecycle::Running @@ -275,6 +439,10 @@ impl RuntimeControlExecutor { } request => self.handle_request(request), }; + self.resident_paused.store( + state.lifecycle != RuntimeLifecycle::Running, + std::sync::atomic::Ordering::Release, + ); if mutation && response.ok { match state.revision.checked_add(1) { Some(revision) => state.revision = revision, @@ -332,8 +500,11 @@ impl RuntimeControlExecutor { memory_resize: self.vm.memory_resize_supported(), secrets_update: self.secrets_update_supported(), checkpoint_create: true, + disk_checkpoint_create: true, + branch_create: cfg!(any(unix, windows)), disk_compact: true, root_disk_grow: true, + pause_resume: self.vm.clock_sync_supported(), }), ..Default::default() }, @@ -353,6 +524,11 @@ impl RuntimeControlExecutor { ControlRequest::CpuState => cpu(self.vm.cpu_state()), ControlRequest::SecretsUpdate { changes } => self.handle_secrets_update(changes), ControlRequest::CheckpointCreate { .. } + | ControlRequest::DiskCheckpointCreate { .. } + | ControlRequest::BranchCreate { .. } + | ControlRequest::Pause + | ControlRequest::Resume + | ControlRequest::PauseState | ControlRequest::DiskCompact { .. } | ControlRequest::RootDiskGrow { .. } => { unreachable!("checkpoint requests are handled by the executor lifecycle path") @@ -415,12 +591,40 @@ impl RuntimeControlExecutor { // Functions //-------------------------------------------------------------------------------------------------- +fn unsupported_lifecycle_request( + request: &ControlRequest, + clock_sync: bool, +) -> Option { + (!clock_sync && matches!(request, ControlRequest::Pause | ControlRequest::Resume)).then(|| { + control_error( + "pause_resume_unavailable", + "resident pause/resume requires a runtime and guest kernel with clock-only resume support", + ) + }) +} + fn new_runtime_boot_id() -> String { let mut bytes = [0u8; 16]; rand::rng().fill_bytes(&mut bytes); format!("boot_{}", hex::encode(bytes)) } +fn pause_response(state: &ExecutorState) -> ControlResponse { + ControlResponse { + ok: true, + pause: Some(super::PauseControlState { + paused: state.user_pause.is_some(), + recovery_required: state.lifecycle == RuntimeLifecycle::Quiesced + && state.user_pause.is_none(), + capture_unavailable: state + .user_pause + .as_ref() + .and_then(|paused| paused.capture_unavailable.clone()), + }), + ..Default::default() + } +} + fn checkpoint_response( result: Option, ok: bool, @@ -458,11 +662,10 @@ fn persist_runtime_boot_id(runtime_dir: &Path, boot_id: &str) -> std::io::Result .open(&temporary)?; file.write_all(boot_id.as_bytes())?; file.write_all(b"\n")?; - file.sync_all()?; + // This file is diagnostic/live discovery, not a restart journal. Fencing uses the new + // in-memory identity on every boot; atomic visibility is sufficient here. drop(file); crate::checkpoint::replace_file(&temporary, &target)?; - #[cfg(unix)] - File::open(runtime_dir)?.sync_all()?; Ok(()) } @@ -544,6 +747,23 @@ fn control_error(code: &str, message: impl Into) -> ControlResponse { mod tests { use super::*; + #[test] + fn unsupported_pause_and_resume_refuse_before_idempotent_mutation() { + for request in [ControlRequest::Pause, ControlRequest::Resume] { + let refused = unsupported_lifecycle_request(&request, false).unwrap(); + assert!(!refused.ok); + assert_eq!( + refused.error_code.as_deref(), + Some("pause_resume_unavailable") + ); + assert!(refused.pause.is_none()); + assert!(unsupported_lifecycle_request(&request, true).is_none()); + } + // Observation remains safe without a kernel clock callback. + assert!(unsupported_lifecycle_request(&ControlRequest::PauseState, false).is_none()); + assert!(unsupported_lifecycle_request(&ControlRequest::Capabilities, false).is_none()); + } + #[test] fn control_ids_are_bounded_and_printable() { assert!(valid_control_id("request_42")); diff --git a/crates/runtime/lib/runner/mod.rs b/crates/runtime/lib/runner/mod.rs index b1a82f7b5..48fc978d8 100644 --- a/crates/runtime/lib/runner/mod.rs +++ b/crates/runtime/lib/runner/mod.rs @@ -18,4 +18,5 @@ pub mod policy; pub mod relay; pub(crate) mod startup; pub mod vm; +pub(crate) mod workload_control; pub(crate) mod writeback; diff --git a/crates/runtime/lib/runner/relay.rs b/crates/runtime/lib/runner/relay.rs index 1446172c2..45dd92178 100644 --- a/crates/runtime/lib/runner/relay.rs +++ b/crates/runtime/lib/runner/relay.rs @@ -32,9 +32,11 @@ use microsandbox_protocol::AGENT_RELAY_MAX_CLIENTS; use microsandbox_protocol::bulk::BulkRecord; use microsandbox_protocol::bulk::{ BULK_FLOW_MASK_GUEST_TO_HOST, BULK_HEADER_SIZE, BulkAccepted, BulkCancel, BulkCancelReason, - BulkFinish, BulkFlow, BulkKind, MAX_BULK_RECORD_PAYLOAD, + BulkCredit, BulkFinish, BulkFlow, BulkKind, MAX_BULK_RECORD_PAYLOAD, MAX_BULK_WINDOW, }; use microsandbox_protocol::codec::{self, MAX_FRAME_SIZE, MAX_WIRE_FRAME}; +#[cfg(test)] +use microsandbox_protocol::core::WORKLOAD_TRANSPORT_BARRIER_VERSION; use microsandbox_protocol::core::{ CoreError, InitAck, InitResolved, Ready, RelayClientDisconnected, WorkloadThaw, WorkloadThawed, }; @@ -63,6 +65,7 @@ use tokio::net::UnixListener; use tokio::net::windows::named_pipe::{NamedPipeServer, PipeMode, ServerOptions}; use tokio::sync::{Mutex, Semaphore, mpsc, oneshot, watch}; +use super::workload_control::{WORKLOAD_CONTROL_ID, WorkloadControl}; use crate::checkpoint::RestoredAgentState; use crate::clock::spawn_clock_sync_task; use crate::console::ConsoleSharedState; @@ -132,9 +135,12 @@ const CLIENT_WRITE_BATCH_BYTES: usize = 256 * 1024; /// Maximum frame slices opportunistically coalesced in one client socket batch. const CLIENT_WRITE_BATCH_FRAMES: usize = 64; -/// At most eight generation-6 frames may wait between clients and the console. -/// Since a frame is capped at 4 MiB, this bounds the channel at 32 MiB. -const AGENT_WRITE_CHANNEL_CAPACITY: usize = 8; +/// Separate admission reserves share one FIFO. Permits survive dequeue and physical writes, so +/// credit-starved payload cannot consume the space needed by another client's metadata. +const AGENT_WRITE_CLASS_FRAMES: usize = 8; +const AGENT_WRITE_CHANNEL_CAPACITY: usize = 2 * AGENT_WRITE_CLASS_FRAMES; +const AGENT_WRITE_DATA_BYTES: usize = 32 * 1024 * 1024; +const AGENT_WRITE_CONTROL_BYTES: usize = 8 * 1024 * 1024; /// Aggregate client-to-bulk-lane bytes waiting outside the console backend. const BULK_WRITE_BYTE_CAPACITY: usize = 32 * 1024 * 1024; @@ -187,12 +193,47 @@ struct ClientState { local_outbound: Option, } -/// One ordered control-lane write, optionally acknowledged after physical ring admission. +/// One primary-lane write, optionally acknowledged after physical ring admission. pub(crate) struct ControlWrite { data: Bytes, completion: Option>, + uses_data_credit: bool, + order: ControlOrder, + admission: Option, +} + +/// A client lease transition fences its range; shutdown and unattributed internal traffic fence +/// the whole FIFO. Ordinary correlations may bypass only unrelated blocked payload. +#[derive(Clone, Copy)] +enum ControlOrder { + Correlation(u32), + MaintenanceClock, + TcpInputData(u32), + TcpInputFinish(u32), + TcpOutputCredit(u32), + ClientFence { start: u32, end: u32 }, + GlobalFence, +} + +struct ControlAdmission { + _bytes: tokio::sync::OwnedSemaphorePermit, + _frame: tokio::sync::OwnedSemaphorePermit, +} + +/// Bounded, class-reserved admission into the existing ordinary transport queue. No payload is +/// copied, and cancellation releases reservations without dropping any already accepted frame. +#[derive(Clone)] +pub(crate) struct ControlWriter { + tx: mpsc::Sender, + data_bytes: Arc, + data_frames: Arc, + control_bytes: Arc, + control_frames: Arc, } +/// Every exit, including task abortion while waiting for ring capacity, wakes lifecycle waiters. +struct WorkloadWriterGuard(Arc); + /// A disconnected leased owner whose untagged control output is still being drained. struct PendingClientDisconnect { id_start: u32, @@ -460,6 +501,211 @@ struct RestoreActivationRecord<'a> { // Methods //-------------------------------------------------------------------------------------------------- +impl ControlWrite { + pub(crate) fn clock_sync() -> RuntimeResult { + // Reserve the largest CBOR integer representation. The scheduler replaces this sentinel + // before charging transport bytes, so queued time is never replayed after a long pause. + Ok(Self { + order: ControlOrder::MaintenanceClock, + ..crate::clock::encode_clock_sync_frame(u64::MAX)?.into() + }) + } + + fn ordinary(data: Bytes, id: u32, uses_data_credit: bool) -> Self { + let order = if id == 0 + || data + .get(LEN_PREFIX_SIZE + 4) + .is_some_and(|flags| flags & FLAG_SHUTDOWN != 0) + { + ControlOrder::GlobalFence + } else { + ControlOrder::Correlation(id) + }; + Self { + data, + completion: None, + uses_data_credit, + order, + admission: None, + } + } + + fn client_fence(data: Bytes, start: u32, end: u32) -> Self { + Self { + order: ControlOrder::ClientFence { start, end }, + ..data.into() + } + } + + /// Only the independent TCP return-credit flow may cross ordered input. Raw metadata was + /// already validated by the client reader; parse only the two small control payloads here. + fn classify_tcp_order( + &mut self, + id: u32, + raw: Option<(BulkKind, BulkFlow, u64, usize)>, + message: Option<&Message>, + ) { + if matches!(self.order, ControlOrder::GlobalFence) { + return; + } + if matches!(raw, Some((BulkKind::Tcp, BulkFlow::HostToGuest, _, _))) { + self.order = ControlOrder::TcpInputData(id); + return; + } + let Some(message) = message else { + return; + }; + match message.t { + MessageType::BulkFinish + if message.payload::().is_ok_and(|finish| { + finish.kind == BulkKind::Tcp && finish.flow == BulkFlow::HostToGuest + }) => + { + self.order = ControlOrder::TcpInputFinish(id); + } + MessageType::BulkCredit + if message.payload::().is_ok_and(|credit| { + credit.kind == BulkKind::Tcp + && credit.flow == BulkFlow::GuestToHost + && credit.credit_limit >= credit.consumed_offset + && credit.credit_limit - credit.consumed_offset <= MAX_BULK_WINDOW + }) => + { + self.order = ControlOrder::TcpOutputCredit(id); + } + _ => {} + } + } +} + +impl ControlOrder { + fn conflicts(self, other: Self) -> bool { + match (self, other) { + (Self::GlobalFence, _) | (_, Self::GlobalFence) => true, + (Self::MaintenanceClock, Self::MaintenanceClock) => true, + (Self::MaintenanceClock, _) | (_, Self::MaintenanceClock) => false, + (Self::ClientFence { start: a, end: b }, Self::ClientFence { start: c, end: d }) => { + a < d && c < b + } + (Self::ClientFence { start, end }, correlation) + | (correlation, Self::ClientFence { start, end }) => { + (start..end).contains(&correlation.id()) + } + // Credit advances the opposite (guest-to-host) producer, not these input bytes or + // their end marker. Reordering only the credit breaks a full-duplex credit cycle; + // input data and input finish still conflict with each other in their original FIFO. + (Self::TcpInputData(_) | Self::TcpInputFinish(_), Self::TcpOutputCredit(_)) => false, + (a, b) => a.id() == b.id(), + } + } + + fn id(self) -> u32 { + match self { + Self::Correlation(id) + | Self::TcpInputData(id) + | Self::TcpInputFinish(id) + | Self::TcpOutputCredit(id) => id, + Self::ClientFence { .. } | Self::GlobalFence | Self::MaintenanceClock => { + unreachable!("fence or maintenance order handled first") + } + } + } +} + +impl ControlWriter { + fn new() -> (Self, mpsc::Receiver) { + let (tx, rx) = mpsc::channel(AGENT_WRITE_CHANNEL_CAPACITY); + (Self::from_sender(tx), rx) + } + + fn from_sender(tx: mpsc::Sender) -> Self { + Self { + tx, + data_bytes: Arc::new(Semaphore::new(AGENT_WRITE_DATA_BYTES)), + data_frames: Arc::new(Semaphore::new(AGENT_WRITE_CLASS_FRAMES)), + control_bytes: Arc::new(Semaphore::new(AGENT_WRITE_CONTROL_BYTES)), + control_frames: Arc::new(Semaphore::new(AGENT_WRITE_CLASS_FRAMES)), + } + } + + fn budgets(&self, write: &ControlWrite) -> (&Arc, &Arc) { + if write.uses_data_credit { + (&self.data_bytes, &self.data_frames) + } else { + (&self.control_bytes, &self.control_frames) + } + } + + pub(crate) async fn send( + &self, + mut write: ControlWrite, + ) -> Result<(), mpsc::error::SendError> { + let Ok(bytes) = u32::try_from(write.data.len()) else { + return Err(mpsc::error::SendError(write)); + }; + let (byte_budget, frame_budget) = self.budgets(&write); + // Closing the receiver must also wake senders waiting for a class reservation. A canceled + // send drops its partial permits; frames already in the canonical queue retain theirs. + let reservation = async { + let frame = Arc::clone(frame_budget).acquire_owned().await.ok()?; + let bytes = Arc::clone(byte_budget) + .acquire_many_owned(bytes) + .await + .ok()?; + Some(ControlAdmission { + _bytes: bytes, + _frame: frame, + }) + }; + let admission = tokio::select! { + biased; + _ = self.tx.closed() => None, + admission = reservation => admission, + }; + let Some(admission) = admission else { + return Err(mpsc::error::SendError(write)); + }; + write.admission = Some(admission); + self.tx.send(write).await.map_err(|mut error| { + error.0.admission.take(); + error + }) + } + + fn try_send( + &self, + mut write: ControlWrite, + ) -> Result<(), mpsc::error::TrySendError> { + if self.tx.is_closed() { + return Err(mpsc::error::TrySendError::Closed(write)); + } + let Ok(bytes) = u32::try_from(write.data.len()) else { + return Err(mpsc::error::TrySendError::Full(write)); + }; + let (byte_budget, frame_budget) = self.budgets(&write); + let Ok(frame) = Arc::clone(frame_budget).try_acquire_owned() else { + return Err(mpsc::error::TrySendError::Full(write)); + }; + let Ok(bytes) = Arc::clone(byte_budget).try_acquire_many_owned(bytes) else { + return Err(mpsc::error::TrySendError::Full(write)); + }; + write.admission = Some(ControlAdmission { + _bytes: bytes, + _frame: frame, + }); + self.tx.try_send(write).map_err(|error| match error { + mpsc::error::TrySendError::Full(mut write) => { + write.admission.take(); + mpsc::error::TrySendError::Full(write) + } + mpsc::error::TrySendError::Closed(mut write) => { + write.admission.take(); + mpsc::error::TrySendError::Closed(write) + } + }) + } +} + impl GuestFrameMerger { /// Register an opening operation before its request can reach agentd. fn register(&mut self, incarnation: ClientIncarnation, id: u32) -> RuntimeResult<()> { @@ -1053,6 +1299,13 @@ impl AgentRelay { ready.local_transport = Some(LocalTransportReady::shared_arena_v1()); ready }; + // Capture the complete client-facing capability set, including the existing + // host-local shared-arena offer, for post-restore handshakes. + self.shared.workload_control.install_ready( + msg.v, + ready.clone(), + self.dual_port_active, + ); let mut client_ready = Message::with_payload(MessageType::Ready, msg.id, &ready).map_err( |error| { @@ -1308,6 +1561,19 @@ impl AgentRelay { } fn thaw_restored_workload(&mut self, restored: &RestoredAgentState) -> RuntimeResult<()> { + if !self.restored_input.control.is_empty() || !self.restored_input.bulk.is_empty() { + return Err(RuntimeError::Custom( + "restored transport did not start at a complete-frame boundary".into(), + )); + } + self.shared + .workload_control + .restore( + restored.host_input, + restored.input_credit, + restored.guest_bulk_bytes_target, + ) + .map_err(RuntimeError::Custom)?; let mut request = Message::with_payload( MessageType::WorkloadThaw, RESTORE_CONTROL_ID, @@ -1330,7 +1596,11 @@ impl AgentRelay { // Drain old bulk records while the guest waits for its scheduler cut. Preserve any // partial final record for the ordinary reader; never restart decoding mid-frame. if let Some(shared) = &self.bulk_shared { - drain_restored_bulk(shared, &mut self.restored_input.bulk)?; + let bytes = drain_restored_bulk(shared, &mut self.restored_input.bulk)?; + self.shared + .workload_control + .observed_bulk(bytes, self.restored_input.bulk.len()) + .map_err(RuntimeError::Custom)?; } if let Some(frame) = pending_request.take() { match self.shared.rx_ring.push(frame) { @@ -1380,6 +1650,13 @@ impl AgentRelay { error.message ))); } + if message.t == MessageType::WorkloadTransportCredit { + self.shared + .workload_control + .reply(message) + .map_err(RuntimeError::Custom)?; + continue; + } if message.t != MessageType::WorkloadThawed { return Err(RuntimeError::Custom(format!( "unexpected restored workload reply {}", @@ -1426,6 +1703,11 @@ impl AgentRelay { .as_ref() .map(|ready| ready.connection_id); self.select_ready_transport(&restored.ready)?; + self.shared.workload_control.install_ready( + restored.protocol_generation, + restored.ready.clone(), + self.dual_port_active, + ); let mut ready = Message::with_payload(MessageType::Ready, 0, &restored.ready) .map_err(|error| RuntimeError::Custom(format!("encode restored ready: {error}")))?; ready.v = restored.protocol_generation; @@ -1467,7 +1749,10 @@ impl AgentRelay { // Bounded channel for client reader tasks to send frames to the ring writer. // Backpressure prevents unbounded memory growth from client floods. - let (agent_tx, agent_rx) = mpsc::channel::(AGENT_WRITE_CHANNEL_CAPACITY); + let (agent_tx, agent_rx) = ControlWriter::new(); + self.shared + .workload_control + .register_ordinary_writer(agent_tx.clone()); // Track which client slots are in use. let used_slots: Arc>> = Arc::new(Mutex::new(HashSet::new())); @@ -1495,9 +1780,10 @@ impl AgentRelay { .clone(); let (tx, rx) = mpsc::channel::(256); let failure_tx = bulk_failure_tx.clone(); + let workload = Arc::clone(&self.shared.workload_control); let handle = tokio::spawn(async move { let _ = failure_tx - .send(bulk_ring_writer_task(shared, rx).await) + .send(bulk_ring_writer_task(shared, rx, workload).await) .await; }); (Some(tx), Some(handle)) @@ -1744,6 +2030,7 @@ impl AgentRelay { write_tx, write_budget, disconnect_rx, + Arc::clone(&self.shared.resident_paused), #[cfg(unix)] local_write_tx, )); @@ -1877,13 +2164,23 @@ impl Drop for AgentRelay { impl From for ControlWrite { fn from(data: Bytes) -> Self { + let uses_data_credit = data.get(LEN_PREFIX_SIZE + 4) == Some(&FLAG_BULK); Self { data, completion: None, + uses_data_credit, + order: ControlOrder::GlobalFence, + admission: None, } } } +impl Drop for WorkloadWriterGuard { + fn drop(&mut self) { + self.0.close(); + } +} + //-------------------------------------------------------------------------------------------------- // Functions //-------------------------------------------------------------------------------------------------- @@ -1896,18 +2193,20 @@ pub(crate) fn push_guest_frame_blocking( } /// Discard complete pre-activation records, retaining a possible fragmented tail. -fn drain_restored_bulk(shared: &ConsoleSharedState, input: &mut BytesMut) -> RuntimeResult<()> { +fn drain_restored_bulk(shared: &ConsoleSharedState, input: &mut BytesMut) -> RuntimeResult { + let mut decoded_bytes = 0; shared.tx_wake.drain(); while let Some(chunk) = shared.tx_ring.pop() { input.extend_from_slice(&chunk); drop(chunk); shared.tx_capacity_wake.wake(); - while try_decode_incarnated_bulk_from_bytes(input) + while let Some(decoded) = try_decode_incarnated_bulk_from_bytes(input) .map_err(|error| RuntimeError::Custom(format!("restored bulk framing: {error}")))? - .is_some() - {} + { + decoded_bytes += CLIENT_INCARNATION_SIZE + decoded.frame.len(); + } } - Ok(()) + Ok(decoded_bytes) } fn persist_restore_activation( @@ -1934,11 +2233,9 @@ fn persist_restore_activation( ) .map_err(|error| RuntimeError::Custom(format!("encode restore activation: {error}")))?; file.write_all(b"\n")?; - file.sync_all()?; + // Diagnostic only: the live activation barrier, not this record, controls readiness. drop(file); crate::checkpoint::replace_file(&temporary, &target)?; - #[cfg(unix)] - std::fs::File::open(runtime_dir)?.sync_all()?; Ok(()) } @@ -1947,6 +2244,13 @@ pub(crate) fn push_guest_frame_until( frame: Vec, timeout: std::time::Duration, ) -> RuntimeResult<()> { + if let Some(writer) = shared + .workload_control + .ordinary_writer() + .map_err(RuntimeError::Custom)? + { + return push_ordered_guest_frame_until(shared, &writer, Bytes::from(frame), timeout); + } let deadline = std::time::Instant::now() + timeout; let mut frame = Bytes::from(frame); @@ -1976,6 +2280,53 @@ pub(crate) fn push_guest_frame_until( } } +/// The VMM shutdown observer is synchronous. Reuse the ordinary queue and its admission receipt +/// without blocking Tokio's channel APIs or bypassing a frozen/credit-starved FIFO head. +fn push_ordered_guest_frame_until( + shared: &ConsoleSharedState, + writer: &ControlWriter, + data: Bytes, + timeout: std::time::Duration, +) -> RuntimeResult<()> { + let deadline = Instant::now() + timeout; + let (completion, mut completed) = oneshot::channel(); + let mut pending = Some(ControlWrite { + completion: Some(completion), + ..data.into() + }); + loop { + if let Some(write) = pending.take() { + match writer.try_send(write) { + Ok(()) => {} + Err(mpsc::error::TrySendError::Full(write)) => pending = Some(write), + Err(mpsc::error::TrySendError::Closed(_)) => { + return Err(RuntimeError::Custom("agent control writer stopped".into())); + } + } + } + match completed.try_recv() { + Ok(()) => return Ok(()), + Err(oneshot::error::TryRecvError::Closed) => { + return Err(RuntimeError::Custom( + "agent control writer dropped admission receipt".into(), + )); + } + Err(oneshot::error::TryRecvError::Empty) => {} + } + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() || shared.is_closed() { + return Err(RuntimeError::Custom( + "timed out sending ordered frame to agentd".into(), + )); + } + // This rare synchronous shutdown path polls its receipt at a bounded interval using the + // existing capacity wake. No new socket, queue or background waiter is introduced. + let _ = shared + .rx_capacity_wake + .wait_timeout(remaining.min(std::time::Duration::from_millis(10))); + } +} + /// Try to extract a complete frame from a byte buffer. /// /// Returns `None` if the buffer doesn't contain a full frame yet. On @@ -2322,6 +2673,7 @@ async fn ring_writer_task( shared: Arc, mut rx: mpsc::Receiver, ) -> RuntimeResult<()> { + let _lifetime = WorkloadWriterGuard(Arc::clone(&shared.workload_control)); #[cfg(unix)] let capacity_fd = match AsyncFd::new(shared.rx_capacity_wake.as_raw_fd()) { Ok(fd) => fd, @@ -2332,75 +2684,198 @@ async fn ring_writer_task( } }; - while let Some(write) = rx.recv().await { - let ControlWrite { - mut data, - completion, - } = write; - let mut attempts = 0u64; - loop { - match shared.rx_ring.push(data) { - Ok(()) => { - shared.rx_wake.wake(); - if let Some(completion) = completion { - let _ = completion.send(()); - } - break; + let workload = &shared.workload_control; + let mut private = workload.start(); + let mut pending = VecDeque::with_capacity(AGENT_WRITE_CHANNEL_CAPACITY); + let mut ordinary_closed = false; + loop { + let changed = workload.changed.notified(); + tokio::pin!(changed); + changed.as_mut().enable(); + if shared.is_closed() { + break; + } + if workload.gated() { + workload.park(false); + } + // Moving frames into the bounded scheduler does not release their class admission. A full + // data class therefore cannot hide another client's metadata in the canonical mailbox. + while pending.len() < AGENT_WRITE_CHANNEL_CAPACITY && !ordinary_closed { + match rx.try_recv() { + Ok(write) => pending.push_back(write), + Err(mpsc::error::TryRecvError::Empty) => break, + Err(mpsc::error::TryRecvError::Disconnected) => ordinary_closed = true, + } + } + let mut wait_clock_capacity = false; + let write = if let Ok(write) = private.try_recv() { + Some(ControlWrite::from(write.0)) + } else { + let (write, wait_capacity) = + select_control_write(&mut pending, workload, Some(&shared)) + .map_err(RuntimeError::Custom)?; + wait_clock_capacity = wait_capacity; + write + }; + if let Some(write) = write { + let ControlWrite { + data, + completion, + admission, + .. + } = write; + if !push_bulk_fragment( + &shared, + data, + #[cfg(unix)] + &capacity_fd, + ) + .await + { + workload.close(); + return Err(RuntimeError::Custom("agent console writer closed".into())); + } + if let Some(completion) = completion { + let _ = completion.send(()); + shared.rx_capacity_wake.wake(); + } + drop(admission); + continue; + } + if ordinary_closed && pending.is_empty() { + break; + } + tokio::select! { + biased; + write = private.recv() => { + let Some(write) = write else { break; }; + if !push_bulk_fragment(&shared, write.0, #[cfg(unix)] &capacity_fd).await { + workload.close(); + return Err(RuntimeError::Custom("private agent console writer closed".into())); } - Err(returned) => { - attempts = attempts.saturating_add(1); - if attempts == 50 || attempts.is_multiple_of(500) { - tracing::warn!( - attempts, - "agent relay: rx_ring full, waiting to deliver frame" - ); - } - data = returned; - if shared.is_closed() { - return Ok(()); - } - - shared.rx_capacity_wake.drain(); - if shared.rx_ring.can_fit(data.len()) { - continue; - } - - #[cfg(unix)] - { - let mut guard = match capacity_fd.readable().await { - Ok(guard) => guard, - Err(error) => { - return Err(RuntimeError::Custom(format!( - "agent relay: console capacity wait failed: {error}" - ))); - } - }; - guard.clear_ready(); - } - - #[cfg(windows)] - { - let shared_for_wait = Arc::clone(&shared); - let _ = tokio::task::spawn_blocking(move || { - shared_for_wait - .rx_capacity_wake - .wait_timeout(std::time::Duration::from_secs(60)) - }) - .await; - } + } + _ = &mut changed => {} + available = wait_console_capacity(&shared, #[cfg(unix)] &capacity_fd), if wait_clock_capacity => { + if !available { + return Err(RuntimeError::Custom("agent console capacity watcher closed".into())); + } + } + write = rx.recv(), if pending.len() < AGENT_WRITE_CHANNEL_CAPACITY && !ordinary_closed => { + if let Some(write) = write { + pending.push_back(write); + } else { + ordinary_closed = true; } } } } + workload.close(); tracing::debug!("agent relay: ring writer task exiting"); Ok(()) } +/// Keep the common FIFO path constant-time. Only a credit-blocked payload head enables a bounded +/// scan for unrelated metadata or independent TCP return credit. Input/finish order and all +/// cancellation, opening, lease and global fences remain intact. +fn select_control_write( + pending: &mut VecDeque, + workload: &WorkloadControl, + shared: Option<&ConsoleSharedState>, +) -> Result<(Option, bool), String> { + let mut wait_capacity = false; + let Some(head) = pending.front_mut() else { + return Ok((None, false)); + }; + if admit_control_write( + head, + workload, + shared, + &mut wait_capacity, + crate::clock::current_clock_sync_frame, + )? { + return Ok((pending.pop_front(), wait_capacity)); + } + if !head.uses_data_credit || workload.gated() { + return Ok((None, wait_capacity)); + } + for index in 1..pending.len() { + let candidate = &pending[index]; + if candidate.uses_data_credit + || pending + .iter() + .take(index) + .any(|earlier| earlier.order.conflicts(candidate.order)) + { + continue; + } + if admit_control_write( + &mut pending[index], + workload, + shared, + &mut wait_capacity, + crate::clock::current_clock_sync_frame, + )? { + return Ok((pending.remove(index), wait_capacity)); + } + } + Ok((None, wait_capacity)) +} + +fn admit_control_write( + write: &mut ControlWrite, + workload: &WorkloadControl, + shared: Option<&ConsoleSharedState>, + wait_capacity: &mut bool, + clock_frame: impl FnOnce() -> RuntimeResult, +) -> Result { + if matches!(write.order, ControlOrder::MaintenanceClock) { + // No transport credit has been charged yet. Refresh before each capacity attempt, including + // retries after pause, and charge the actual CBOR length rather than the reserved maximum. + write.data = clock_frame().map_err(|error| error.to_string())?; + if let Some(shared) = shared { + shared.rx_capacity_wake.drain(); + if !shared.rx_ring.can_fit(write.data.len()) { + *wait_capacity = !workload.gated(); + return Ok(false); + } + // This is the sole post-Ready producer. No await separates this successful capacity + // check, transport admission and the atomic whole-frame queue push, so a clock cannot + // acquire a timestamp and then sleep waiting for physical queue capacity. + } + } + workload.admit(write.uses_data_credit, write.data.len()) +} + +async fn wait_console_capacity( + shared: &Arc, + #[cfg(unix)] capacity_fd: &AsyncFd, +) -> bool { + #[cfg(unix)] + { + let _ = shared; + let Ok(mut ready) = capacity_fd.readable().await else { + return false; + }; + ready.clear_ready(); + true + } + #[cfg(windows)] + { + // This select branch is cancelable. A blocking wake waiter would survive cancellation + // and accumulate across other traffic; only a pending, ring-blocked maintenance clock + // needs this bounded retry on platforms without the Unix readiness adapter. + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + !shared.is_closed() + } +} + /// Apply deficit round robin before admitting client raw records to the bulk console ring. async fn bulk_ring_writer_task( shared: Arc, mut rx: mpsc::Receiver, + workload: Arc, ) -> RuntimeResult<()> { + let _lifetime = WorkloadWriterGuard(Arc::clone(&workload)); #[cfg(unix)] let capacity_fd = match AsyncFd::new(shared.rx_capacity_wake.as_raw_fd()) { Ok(fd) => fd, @@ -2414,13 +2889,24 @@ async fn bulk_ring_writer_task( let mut active = VecDeque::<(ClientIncarnation, u32)>::new(); let mut retired = HashMap::>::new(); - while let Some(command) = rx.recv().await { - apply_bulk_writer_command(command, &mut flows, &mut active, &mut retired)?; + loop { + let changed = workload.changed.notified(); + tokio::pin!(changed); + changed.as_mut().enable(); + if shared.is_closed() { + break; + } + if workload.gated() { + workload.park(true); + changed.await; + continue; + } while let Ok(command) = rx.try_recv() { apply_bulk_writer_command(command, &mut flows, &mut active, &mut retired)?; } - - while !active.is_empty() { + let mut progressed = false; + let mut needs_deficit_round = false; + if !active.is_empty() { let round_len = active.len(); // DRR fairness is irrelevant when only one flow is runnable. Grant the full bounded // burst in that case so a 256 KiB default record does not force one executor yield per @@ -2452,6 +2938,19 @@ async fn bulk_ring_writer_task( && burst.saturating_add(next_len) <= BULK_WRITE_MAX_BURST }); if !can_send { + needs_deficit_round |= + flows.get(&key).is_some_and(|flow| next_len > flow.deficit); + break; + } + let wire_len = CLIENT_INCARNATION_SIZE + + LEN_PREFIX_SIZE + + FRAME_HEADER_SIZE + + BULK_HEADER_SIZE + + next_len; + if !workload + .admit(true, wire_len) + .map_err(RuntimeError::Custom)? + { break; } @@ -2475,6 +2974,7 @@ async fn bulk_ring_writer_task( )); } burst = burst.saturating_add(next_len); + progressed = true; } if flows.get(&key).is_some_and(|flow| flow.queue.is_empty()) { @@ -2483,10 +2983,17 @@ async fn bulk_ring_writer_task( active.push_back(key); } } - while let Ok(command) = rx.try_recv() { - apply_bulk_writer_command(command, &mut flows, &mut active, &mut retired)?; - } + } + if progressed || needs_deficit_round { tokio::task::yield_now().await; + } else { + tokio::select! { + _ = &mut changed => {} + command = rx.recv() => { + let Some(command) = command else { break; }; + apply_bulk_writer_command(command, &mut flows, &mut active, &mut retired)?; + } + } } } Ok(()) @@ -2755,6 +3262,8 @@ async fn ring_reader_task( .await; } let dual_port = bulk_shared.is_some(); + let workload = Arc::clone(&shared.workload_control); + let control_workload = Arc::clone(&workload); let control_lane_budget = Arc::new(Semaphore::new(if dual_port { CONTROL_LANE_OUTPUT_BYTE_CAPACITY } else { @@ -2774,6 +3283,7 @@ async fn ring_reader_task( range_lease_active, control_lane_tx, control_lane_budget, + control_workload, ) .await; let _ = control_failure_tx.send((GuestLane::Control, result)).await; @@ -2788,6 +3298,7 @@ async fn ring_reader_task( range_lease_active, bulk_lane_tx, bulk_lane_budget, + workload, ) .await; let _ = lane_failure_tx.send((GuestLane::Bulk, result)).await; @@ -3106,6 +3617,25 @@ async fn route_guest_lane_frame( Ok(()) } +/// Private lifecycle replies bypass SDK output budgets, but never skip a framing boundary. +fn handle_workload_frame( + workload: &WorkloadControl, + frame: &RawFrame, + remaining: usize, +) -> RuntimeResult<()> { + let message = decode_frame(&frame.data)?; + if remaining != 0 + && workload + .requires_frozen_boundary(&message) + .map_err(RuntimeError::Custom)? + { + return Err(RuntimeError::Custom( + "frozen primary transport has a trailing frame or partial prefix".into(), + )); + } + workload.reply(message).map_err(RuntimeError::Custom) +} + /// Read and route the single ordered guest stream without dual-port actor hops. async fn combined_ring_reader_task( mut buf: BytesMut, @@ -3171,6 +3701,10 @@ async fn combined_ring_reader_task( let Some(frame) = try_extract_frame(&mut buf)? else { break; }; + if frame.id == WORKLOAD_CONTROL_ID { + handle_workload_frame(&shared.workload_control, &frame, buf.len())?; + continue; + } let charged = frame.data.len().saturating_add(OUTPUT_BUDGET_GRANULE - 1) / OUTPUT_BUDGET_GRANULE * OUTPUT_BUDGET_GRANULE; @@ -3198,6 +3732,7 @@ async fn combined_ring_reader_task( } /// Read and frame one physical guest console lane without interpreting control payloads. +#[allow(clippy::too_many_arguments)] async fn lane_reader_task( mut buf: BytesMut, shared: Arc, @@ -3206,6 +3741,7 @@ async fn lane_reader_task( range_lease_active: bool, event_tx: mpsc::Sender, budget: Arc, + workload: Arc, ) -> RuntimeResult<()> { #[cfg(unix)] let async_fd = AsyncFd::new(shared.tx_wake.as_raw_fd()).map_err(RuntimeError::Io)?; @@ -3241,6 +3777,12 @@ async fn lane_reader_task( shared.tx_capacity_wake.wake(); } + if lane == GuestLane::Bulk { + workload + .observed_bulk(0, buf.len()) + .map_err(RuntimeError::Custom)?; + } + loop { if lane == GuestLane::Control && range_lease_active { match try_decode_relay_client_disconnected_ack_from_bytes(&mut buf) { @@ -3298,6 +3840,14 @@ async fn lane_reader_task( frame.id, frame.flags ))); } + if lane == GuestLane::Bulk { + workload + .observed_bulk(CLIENT_INCARNATION_SIZE + frame.data.len(), buf.len()) + .map_err(RuntimeError::Custom)?; + } else if frame.id == WORKLOAD_CONTROL_ID { + handle_workload_frame(&workload, &frame, buf.len())?; + continue; + } let charged = frame.data.len().saturating_add(OUTPUT_BUDGET_GRANULE - 1) / OUTPUT_BUDGET_GRANULE * OUTPUT_BUDGET_GRANULE; @@ -3413,8 +3963,17 @@ fn queue_bulk_open_rejection( .map_err(|error| RuntimeError::Custom(format!("encode bulk admission rejection: {error}")))?; // Match the initiating request so an older compatible SDK can decode the terminal response. message.v = version; + queue_client_rejection(write_tx, write_budget, &message) +} + +/// Host-side rejection uses the same bounded mailbox as guest responses. +fn queue_client_rejection( + write_tx: &mpsc::UnboundedSender, + write_budget: &Arc, + message: &Message, +) -> RuntimeResult<()> { let mut wire = Vec::new(); - codec::encode_to_buf(&message, &mut wire).map_err(|error| { + codec::encode_to_buf(message, &mut wire).map_err(|error| { RuntimeError::Custom(format!("encode bulk admission rejection frame: {error}")) })?; let charged = wire @@ -3450,7 +4009,7 @@ fn queue_bulk_open_rejection( async fn client_reader_task( slot: u32, mut reader: impl AsyncRead + Unpin + Send + 'static, - agent_tx: mpsc::Sender, + agent_tx: ControlWriter, clients: Arc>>, used_slots: Arc>>, drain_tx: mpsc::Sender<()>, @@ -3467,6 +4026,7 @@ async fn client_reader_task( write_tx: mpsc::UnboundedSender, write_budget: Arc, mut disconnect_rx: watch::Receiver, + resident_paused: Arc, #[cfg(unix)] local_write_tx: mpsc::UnboundedSender, ) { #[cfg(unix)] @@ -3652,6 +4212,28 @@ async fn client_reader_task( .then(|| decode_frame(frame.data.as_ref()).ok()) .flatten(); let message_type = decoded_message.as_ref().map(|message| message.t); + // A suspended guest cannot reject new work itself. Existing stream data keeps the + // bounded transport path; this does not touch guest slot ownership or bulk state. + if is_session_start && resident_paused.load(Ordering::Acquire) { + let Some(request) = decoded_message.as_ref() else { + break; + }; + let error = CoreError { + kind: microsandbox_protocol::core::CoreErrorKind::InvalidSession, + message: "sandbox is paused; resume it before starting guest work".into(), + offending_type: None, + workload_failure: None, + }; + let Ok(mut response) = Message::with_payload(MessageType::CoreError, frame.id, &error) + else { + break; + }; + response.v = request.v; + if queue_client_rejection(&write_tx, &write_budget, &response).is_err() { + break; + } + continue; + } let opened_bulk_kind = decoded_message .as_ref() .and_then(|message| match message.t { @@ -3924,9 +4506,37 @@ async fn client_reader_task( tracing::error!("agent relay: bulk ring writer channel closed"); break; } - } else if agent_tx.send(frame.data.into()).await.is_err() { - tracing::error!("agent relay: control ring writer channel closed"); - break; + } else { + #[cfg(unix)] + let data = if let Some(record) = shared_bulk.take() { + // Combined transport has one owned, ordered frame queue rather than the dual + // port's split header/payload writer. Materialize a bounded standard raw frame + // before releasing its arena slot; the queue then retains this copy through the + // complete physical write. The dual-port zero-copy path above is unchanged. + let mut encoded = Vec::with_capacity( + LEN_PREFIX_SIZE + FRAME_HEADER_SIZE + BULK_HEADER_SIZE + record.payload.len(), + ); + if let Err(error) = codec::encode_bulk_to_buf(&record, &mut encoded) { + tracing::error!(%error, "agent relay: encode combined shared bulk frame failed"); + break; + } + Bytes::from(encoded) + } else { + frame.data + }; + #[cfg(not(unix))] + let data = frame.data; + let mut write = ControlWrite::ordinary( + data, + frame.id, + frame.flags == FLAG_BULK + || message_type.is_some_and(MessageType::uses_workload_data_credit), + ); + write.classify_tcp_order(frame.id, bulk_metadata, decoded_message.as_ref()); + if agent_tx.send(write).await.is_err() { + tracing::error!("agent relay: control ring writer channel closed"); + break; + } } } @@ -4012,7 +4622,11 @@ async fn client_reader_task( continue; } - if agent_tx.send(Bytes::from(buf).into()).await.is_err() { + if agent_tx + .send(ControlWrite::ordinary(Bytes::from(buf), session_id, false)) + .await + .is_err() + { tracing::error!("agent relay: ring writer channel closed during cleanup"); break; } @@ -4082,7 +4696,7 @@ async fn random_unused_client_incarnation( /// Establish one dual-port range owner before its SDK connection becomes usable. async fn send_relay_client_connected( - agent_tx: &mpsc::Sender, + agent_tx: &ControlWriter, id_start: u32, id_end_exclusive: u32, incarnation: ClientIncarnation, @@ -4093,14 +4707,18 @@ async fn send_relay_client_connected( incarnation, }); agent_tx - .send(Bytes::copy_from_slice(&frame).into()) + .send(ControlWrite::client_fence( + Bytes::copy_from_slice(&frame), + id_start, + id_end_exclusive, + )) .await .map_err(|_| RuntimeError::Custom("agent control writer stopped".into())) } /// Send cleanup and, in dual-port mode, register the reverse-lane drain acknowledgement first. async fn begin_relay_client_disconnect( - agent_tx: &mpsc::Sender, + agent_tx: &ControlWriter, pending_disconnects: &Arc>>, id_start: u32, id_end_exclusive: u32, @@ -4162,42 +4780,37 @@ async fn complete_relay_client_disconnect( /// Remove exactly the range owner that disconnected, preserving combined-mode compatibility. async fn send_relay_client_disconnected( - agent_tx: &mpsc::Sender, + agent_tx: &ControlWriter, id_start: u32, id_end_exclusive: u32, incarnation: Option, ) -> RuntimeResult<()> { - send_relay_lifecycle( - agent_tx, + let message = Message::with_payload( MessageType::RelayClientDisconnected, + 0, &RelayClientDisconnected { id_start, id_end_exclusive, incarnation, }, ) - .await -} - -async fn send_relay_lifecycle( - agent_tx: &mpsc::Sender, - message_type: MessageType, - payload: &T, -) -> RuntimeResult<()> { - let message = Message::with_payload(message_type, 0, payload) - .map_err(|error| RuntimeError::Custom(format!("encode relay lifecycle: {error}")))?; + .map_err(|error| RuntimeError::Custom(format!("encode relay lifecycle: {error}")))?; let mut frame = Vec::new(); codec::encode_to_buf(&message, &mut frame) .map_err(|error| RuntimeError::Custom(format!("encode relay lifecycle frame: {error}")))?; agent_tx - .send(Bytes::from(frame).into()) + .send(ControlWrite::client_fence( + Bytes::from(frame), + id_start, + id_end_exclusive, + )) .await .map_err(|_| RuntimeError::Custom("agent control writer stopped".into())) } /// Publish typed cancellation for every active raw-bulk operation while control is still usable. async fn handle_relay_transport_failure( - agent_tx: &mpsc::Sender, + agent_tx: &ControlWriter, merge_command_tx: &mpsc::Sender, clients: &Arc>>, wait_for_terminals: bool, @@ -4255,8 +4868,8 @@ async fn handle_relay_transport_failure( let (completion, completed) = oneshot::channel(); agent_tx .send(ControlWrite { - data: Bytes::from(frame), completion: Some(completion), + ..ControlWrite::ordinary(Bytes::from(frame), id, false) }) .await .map_err(|_| RuntimeError::Custom("agent control writer stopped".into()))?; @@ -4389,6 +5002,13 @@ mod tests { use super::*; + fn next_control_write( + pending: &mut VecDeque, + workload: &WorkloadControl, + ) -> Result, String> { + select_control_write(pending, workload, None).map(|(write, _)| write) + } + #[cfg(unix)] use microsandbox_agent_client::local_shm::{ LocalShmClient, LocalShmUpgrade, local_upgrade_request_frame, receive_local_shm_upgrade, @@ -4600,6 +5220,17 @@ mod tests { #[cfg(unix)] #[tokio::test] async fn client_shared_descriptor_reaches_bulk_scheduler_without_socket_payload() { + exercise_shared_descriptor_input(true).await; + } + + #[cfg(unix)] + #[tokio::test] + async fn client_shared_descriptor_combined_preserves_frame_credit_and_arena_release() { + exercise_shared_descriptor_input(false).await; + } + + #[cfg(unix)] + async fn exercise_shared_descriptor_input(dual_port: bool) { let (mut client_socket, server_socket) = tokio::net::UnixStream::pair().unwrap(); let ancillary_fd = server_socket.as_fd().try_clone_to_owned().unwrap(); let (server_reader, server_writer) = tokio::io::split(server_socket); @@ -4631,7 +5262,23 @@ mod tests { local_write_rx, ancillary_fd, )); - let (agent_tx, _agent_rx) = mpsc::channel(1); + let (agent_tx, agent_rx) = ControlWriter::new(); + let queue_budget = agent_tx.clone(); + let expected_len = LEN_PREFIX_SIZE + + FRAME_HEADER_SIZE + + BULK_HEADER_SIZE + + MAX_BULK_RECORD_PAYLOAD as usize; + let shared = workload_test_shared(expected_len, false); + if !dual_port { + // Queue entries are whole owned frames. Occupy some capacity so the next full-size + // frame must wait, without configuring a queue too small to ever admit that frame. + shared + .rx_ring + .push(Bytes::from_static(b"occupied")) + .unwrap(); + } + let ring_writer = + (!dual_port).then(|| tokio::spawn(ring_writer_task(Arc::clone(&shared), agent_rx))); let used_slots = Arc::new(Mutex::new(HashSet::from([0]))); let (drain_tx, _drain_rx) = mpsc::channel(1); let (bulk_tx, mut bulk_rx) = mpsc::channel(1); @@ -4646,8 +5293,8 @@ mod tests { drain_tx, Arc::new(std::sync::Mutex::new(HashMap::new())), Arc::new(AtomicU64::new(1)), - Some(bulk_tx), - Some(Arc::new(Semaphore::new(BULK_WRITE_BYTE_CAPACITY))), + dual_port.then_some(bulk_tx), + dual_port.then(|| Arc::new(Semaphore::new(BULK_WRITE_BYTE_CAPACITY))), merge_tx, pending_disconnects, 1, @@ -4657,6 +5304,7 @@ mod tests { write_tx, write_budget, disconnect_rx, + Arc::new(std::sync::atomic::AtomicBool::new(false)), local_write_tx, )); @@ -4674,27 +5322,117 @@ mod tests { kind: BulkKind::Filesystem, flow: BulkFlow::HostToGuest, offset: 17, - payload: Bytes::from_static(b"arena payload"), + payload: if dual_port { + Bytes::from_static(b"arena payload") + } else { + Bytes::from(vec![0x53; MAX_BULK_RECORD_PAYLOAD as usize]) + }, }; let mut prepared = local.outbound.try_prepare(&record).unwrap(); - let wire = encode_local_bulk_ref(prepared.descriptor()).unwrap(); + let descriptor = prepared.descriptor(); + let wire = encode_local_bulk_ref(descriptor).unwrap(); client_socket.write_all(&wire).await.unwrap(); prepared.commit(); - let command = tokio::time::timeout(Duration::from_secs(1), bulk_rx.recv()) + if dual_port { + let command = tokio::time::timeout(Duration::from_secs(1), bulk_rx.recv()) + .await + .unwrap() + .unwrap(); + let BulkWriterCommand::Write(write) = command else { + panic!("shared record did not enter the bulk scheduler"); + }; + let BulkWriteData::Shared { payload, .. } = write.data else { + panic!("dual-port runtime rebuilt shared input as an in-band socket frame"); + }; + assert_eq!(payload, record.payload); + } else { + // The arena can be released once the fallback owns its copy, even while the console + // queue cannot admit that frame. Reusing the slot must not change the owned copy. + let release = tokio::time::timeout( + Duration::from_secs(1), + codec::read_raw_frame(&mut client_socket), + ) + .await + .unwrap() + .unwrap(); + let LocalShmFrame::BulkRelease(release) = decode_local_body(&release.body).unwrap() + else { + panic!("copied combined input did not release its arena slot"); + }; + assert_eq!(release.slot, descriptor.slot); + assert_eq!(release.generation, descriptor.generation); + local.outbound.release(release).unwrap(); + let replacement = BulkRecord { + payload: Bytes::from(vec![0xa7; record.payload.len()]), + ..record.clone() + }; + let _replacement = local.outbound.try_prepare(&replacement).unwrap(); + + tokio::time::timeout(Duration::from_secs(1), async { + while shared.rx_ring.snapshot().full_events == 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!( + queue_budget.data_bytes.available_permits(), + AGENT_WRITE_DATA_BYTES - expected_len + ); + assert_eq!( + queue_budget.data_frames.available_permits(), + AGENT_WRITE_CLASS_FRAMES - 1 + ); + assert_eq!( + queue_budget.control_bytes.available_permits(), + AGENT_WRITE_CONTROL_BYTES + ); + assert_eq!( + queue_budget.control_frames.available_permits(), + AGENT_WRITE_CLASS_FRAMES + ); + let gate = shared.workload_control.gate(); + assert_eq!(next_host_fragment(&shared).await.as_ref(), b"occupied"); + let mut wire = BytesMut::from(next_host_fragment(&shared).await.as_ref()); + assert_eq!(wire.len(), expected_len); + let position = tokio::time::timeout( + Duration::from_secs(1), + shared.workload_control.parked_position(), + ) .await .unwrap() .unwrap(); - let BulkWriterCommand::Write(write) = command else { - panic!("shared record did not enter the bulk scheduler"); - }; - let BulkWriteData::Shared { payload, .. } = write.data else { - panic!("runtime rebuilt shared input as an in-band socket frame"); - }; - assert_eq!(payload, record.payload); + assert_eq!(position.bulk_bytes, expected_len as u64); + assert_eq!(position.bulk_frames, 1); + assert_eq!(position.control_frames, 0); + assert_eq!( + queue_budget.data_bytes.available_permits(), + AGENT_WRITE_DATA_BYTES + ); + assert_eq!( + queue_budget.data_frames.available_permits(), + AGENT_WRITE_CLASS_FRAMES + ); + let Some(codec::DecodedFrame::Bulk(received)) = + codec::try_decode_frame_from_bytes(&mut wire).unwrap() + else { + panic!("combined descriptor did not produce one complete raw frame"); + }; + assert_eq!(received, record); + assert!(wire.is_empty()); + assert!(shared.rx_ring.pop().is_none()); + gate.release(); + } reader.abort(); writer.abort(); + if let Some(ring_writer) = ring_writer { + ring_writer.abort(); + let _ = ring_writer.await; + } + let _ = reader.await; + let _ = writer.await; } fn lane_frame(bytes: Vec, budget: &Arc) -> LaneFrame { @@ -4782,8 +5520,8 @@ mod tests { let task = tokio::spawn(ring_writer_task(Arc::clone(&shared), rx)); let (completion, completed) = oneshot::channel(); tx.send(ControlWrite { - data: Bytes::from_static(b"control frame"), completion: Some(completion), + ..Bytes::from_static(b"control frame").into() }) .await .unwrap(); @@ -4800,6 +5538,7 @@ mod tests { let id_start = 1; let id_end_exclusive = AGENT_RELAY_ID_RANGE_STEP; let (agent_tx, mut agent_rx) = mpsc::channel(1); + let agent_tx = ControlWriter::from_sender(agent_tx); let pending = Arc::new(Mutex::new(HashMap::new())); let mut completion = begin_relay_client_disconnect( @@ -4837,6 +5576,124 @@ mod tests { assert!(!pending.lock().await.contains_key(&incarnation)); } + #[tokio::test] + async fn paused_client_rejects_work_without_registering_or_forwarding_sessions() { + let slot = 0; + let incarnation = [0x82; CLIENT_INCARNATION_SIZE]; + let (id_start, id_end_exclusive) = relay_client_id_range(slot).unwrap(); + let (reader, mut peer) = tokio::io::duplex(4096); + let (agent_tx, mut agent_rx) = mpsc::channel(4); + let agent_tx = ControlWriter::from_sender(agent_tx); + let (write_tx, mut write_rx) = mpsc::unbounded_channel(); + #[cfg(unix)] + let (local_write_tx, _local_write_rx) = mpsc::unbounded_channel(); + let (disconnect_tx, disconnect_rx) = watch::channel(false); + let write_budget = Arc::new(Semaphore::new(CLIENT_OUTPUT_PER_CLIENT_BYTE_CAPACITY)); + let active_bulk = Arc::new(std::sync::Mutex::new(HashMap::new())); + let clients = Arc::new(Mutex::new(HashMap::from([( + slot, + ClientState { + incarnation: Some(incarnation), + active_sessions: HashSet::new(), + active_bulk: Arc::clone(&active_bulk), + write_tx: write_tx.clone(), + write_budget: Arc::clone(&write_budget), + disconnect_tx, + #[cfg(unix)] + local_outbound: None, + }, + )]))); + let used_slots = Arc::new(Mutex::new(HashSet::from([slot]))); + let (drain_tx, _drain_rx) = mpsc::channel(1); + let (merge_command_tx, _merge_command_rx) = mpsc::channel(1); + let pending_disconnects = Arc::new(Mutex::new(HashMap::new())); + + let task = tokio::spawn(client_reader_task( + slot, + reader, + agent_tx, + Arc::clone(&clients), + Arc::clone(&used_slots), + drain_tx, + Arc::new(std::sync::Mutex::new(HashMap::new())), + Arc::new(AtomicU64::new(1)), + None, + None, + merge_command_tx, + Arc::clone(&pending_disconnects), + id_start, + id_end_exclusive, + Some(incarnation), + Arc::clone(&active_bulk), + write_tx, + Arc::clone(&write_budget), + disconnect_rx, + Arc::new(std::sync::atomic::AtomicBool::new(true)), + #[cfg(unix)] + local_write_tx, + )); + + let initial_budget = write_budget.available_permits(); + for kind in [MessageType::FsRequest, MessageType::ExecRequest] { + let wire = encoded_message_id(kind, id_start, &serde_json::json!({})); + peer.write_all(&wire).await.unwrap(); + let rejected = tokio::time::timeout(Duration::from_secs(1), write_rx.recv()) + .await + .unwrap() + .unwrap(); + let ClientWriteData::Inline(bytes) = &rejected.data else { + panic!("pause rejection must use the bounded control lane"); + }; + let response = decode_frame(bytes).unwrap(); + assert_eq!(response.id, id_start); + assert_eq!(response.t, MessageType::CoreError); + assert_eq!(response.v, decode_frame(&wire).unwrap().v); + let error: CoreError = response.payload().unwrap(); + assert!(error.message.contains("sandbox is paused")); + assert!(agent_rx.try_recv().is_err()); + assert!(active_bulk.lock().unwrap().is_empty()); + assert!(clients.lock().await[&slot].active_sessions.is_empty()); + assert!(write_budget.available_permits() < initial_budget); + drop(rejected); + assert_eq!(write_budget.available_permits(), initial_budget); + } + // Existing streams still enter bounded source-owned admission while paused. Classification + // reuses this reader's already decoded envelope, including empty stdin/TCP EOF payloads. + for (kind, uses_data_credit) in [ + (MessageType::ExecStdin, true), + (MessageType::FsData, true), + (MessageType::TcpData, true), + (MessageType::TcpEof, true), + (MessageType::BulkFinish, false), + (MessageType::BulkCredit, false), + (MessageType::Ping, false), + ] { + let wire = encoded_message_id(kind, id_start, &serde_json::json!({})); + peer.write_all(&wire).await.unwrap(); + let admitted = tokio::time::timeout(Duration::from_secs(1), agent_rx.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(admitted.data.as_ref(), wire); + assert_eq!(admitted.uses_data_credit, uses_data_credit, "{kind:?}"); + assert!(matches!(admitted.order, ControlOrder::Correlation(id) if id == id_start)); + } + active_bulk + .lock() + .unwrap() + .insert(id_start, BulkKind::Filesystem); + let raw = encoded_host_raw(id_start, 0, b"raw input"); + peer.write_all(&raw).await.unwrap(); + let admitted = tokio::time::timeout(Duration::from_secs(1), agent_rx.recv()) + .await + .unwrap() + .unwrap(); + assert!(admitted.uses_data_credit); + assert_eq!(admitted.data.as_ref(), raw); + task.abort(); + let _ = task.await; + } + #[tokio::test] async fn combined_leased_disconnect_releases_slot_without_a_merger() { let slot = 0; @@ -4845,6 +5702,7 @@ mod tests { let (reader, peer) = tokio::io::duplex(64); drop(peer); let (agent_tx, mut agent_rx) = mpsc::channel(4); + let agent_tx = ControlWriter::from_sender(agent_tx); let (write_tx, _write_rx) = mpsc::unbounded_channel(); #[cfg(unix)] let (local_write_tx, _local_write_rx) = mpsc::unbounded_channel(); @@ -4889,6 +5747,7 @@ mod tests { write_tx, write_budget, disconnect_rx, + Arc::new(std::sync::atomic::AtomicBool::new(false)), #[cfg(unix)] local_write_tx, )); @@ -5341,6 +6200,7 @@ mod tests { true, frame_tx, budget, + Arc::clone(&shared.workload_control), ), ) .await @@ -5397,7 +6257,11 @@ mod tests { .unwrap(); drop(tx); - let task = tokio::spawn(bulk_ring_writer_task(Arc::clone(&shared), rx)); + let task = tokio::spawn(bulk_ring_writer_task( + Arc::clone(&shared), + rx, + Arc::clone(&shared.workload_control), + )); tokio::time::timeout(std::time::Duration::from_secs(2), task) .await .expect("bulk scheduler stalled") @@ -5455,7 +6319,11 @@ mod tests { tokio::time::timeout( std::time::Duration::from_secs(2), - bulk_ring_writer_task(Arc::clone(&shared), rx), + bulk_ring_writer_task( + Arc::clone(&shared), + rx, + Arc::clone(&shared.workload_control), + ), ) .await .expect("maximum filesystem record stalled") @@ -5703,6 +6571,7 @@ mod tests { boot_time_ns: 0, init_time_ns: 0, ready_time_ns: 0, + workload_transport_barrier_version: Some(WORKLOAD_TRANSPORT_BARRIER_VERSION), ..Default::default() }, ); @@ -5711,6 +6580,16 @@ mod tests { relay.wait_ready().unwrap(); + let _private = shared.workload_control.start(); + let (_, captured_ready) = shared.workload_control.ready().unwrap(); + #[cfg(unix)] + assert_eq!( + captured_ready.local_transport, + Some(LocalTransportReady::shared_arena_v1()) + ); + #[cfg(not(unix))] + assert!(captured_ready.local_transport.is_none()); + let cached = relay.ready_frame.as_ref().expect("SDK-facing ready frame"); let cached_ready: Ready = decode_frame(cached).unwrap().payload().unwrap(); #[cfg(unix)] @@ -5905,9 +6784,13 @@ mod tests { init_time_ns: 22, ready_time_ns: 33, agent_version: "test-agent".into(), + workload_transport_barrier_version: Some(WORKLOAD_TRANSPORT_BARRIER_VERSION), ..Default::default() }, attempt_id: attempt_id.into(), + host_input: Default::default(), + input_credit: Default::default(), + guest_bulk_bytes_target: 0, } } @@ -5920,6 +6803,7 @@ mod tests { .await .unwrap(); let restored = restored_agent("checkpoint-attempt"); + relay.install_restored_ready(&restored).unwrap(); let guest_shared = Arc::clone(&shared); let guest = std::thread::spawn(move || { @@ -5958,6 +6842,20 @@ mod tests { .unwrap(); response.v = request.v; let mut frame = Vec::new(); + codec::encode_to_buf( + &Message::with_payload( + MessageType::WorkloadTransportCredit, + RESTORE_CONTROL_ID, + µsandbox_protocol::core::WorkloadTransportCredit { + control_bytes: 100, + control_frames: 1, + ..Default::default() + }, + ) + .unwrap(), + &mut frame, + ) + .unwrap(); codec::encode_to_buf(&response, &mut frame).unwrap(); guest_shared.tx_ring.push(frame).unwrap(); guest_shared.tx_wake.wake(); @@ -5965,6 +6863,8 @@ mod tests { relay.thaw_restored_workload(&restored).unwrap(); guest.join().unwrap(); + assert!(shared.workload_control.admit(false, 100).unwrap()); + assert!(!shared.workload_control.admit(false, 1).unwrap()); } #[tokio::test] @@ -6025,6 +6925,83 @@ mod tests { } } + #[cfg(unix)] + #[test] + fn restored_bulk_suffix_requires_the_source_decoder_prefix() { + use msb_krun::ConsolePortBackend; + + let record = microsandbox_protocol::bulk::BulkRecord { + id: 1, + kind: BulkKind::Filesystem, + flow: BulkFlow::GuestToHost, + offset: 0, + payload: Bytes::from(vec![0x61; 128]), + }; + let mut wire = vec![0x51; CLIENT_INCARNATION_SIZE]; + codec::encode_bulk_to_buf(&record, &mut wire).unwrap(); + let split = wire.len() - 64; + let source = Arc::new(ConsoleSharedState::with_capacity(4096)); + let source_backend = crate::runner::console::AgentConsoleBackend::new(source.clone()); + let mut source_input = BytesMut::new(); + assert_eq!(source_backend.write(&wire[..split]).unwrap(), split); + drain_restored_bulk(&source, &mut source_input).unwrap(); + assert_eq!(source_input.as_ref(), &wire[..split]); + + // Model a cut after the source host consumed this prefix. The VMM console state does + // not serialize the backend queue or this reader buffer, and the resumed guest writer + // can still have the suffix pending. A fresh destination therefore starts mid-record. + let destination = Arc::new(ConsoleSharedState::with_capacity(4096)); + let destination_backend = + crate::runner::console::AgentConsoleBackend::new(destination.clone()); + destination_backend.write(&wire[split..]).unwrap(); + let error = drain_restored_bulk(&destination, &mut BytesMut::new()).unwrap_err(); + assert!(error.to_string().contains("restored bulk framing")); + + // This is missing state, not malformed producer bytes: keeping the exact prefix makes + // the same suffix decode normally. A VM-level cut test must establish which boundary + // the freeze handshake guarantees before treating a fresh decoder as safe. + source_backend.write(&wire[split..]).unwrap(); + drain_restored_bulk(&source, &mut source_input).unwrap(); + assert!(source_input.is_empty()); + } + + #[cfg(unix)] + #[test] + fn restored_control_suffix_requires_the_source_decoder_prefix() { + use msb_krun::ConsolePortBackend; + + let wire = encoded_message_id( + MessageType::ExecStdout, + 1, + µsandbox_protocol::exec::ExecStdout { + data: vec![0x61; 128], + }, + ); + let split = wire.len() - 64; + let source = Arc::new(ConsoleSharedState::with_capacity(4096)); + let backend = crate::runner::console::AgentConsoleBackend::new(source.clone()); + backend.write(&wire[..split]).unwrap(); + let mut source_input = BytesMut::from(source.tx_ring.pop().unwrap().as_ref()); + assert!( + codec::try_decode_frame_from_bytes(&mut source_input) + .unwrap() + .is_none() + ); + + let destination = Arc::new(ConsoleSharedState::with_capacity(4096)); + let backend = crate::runner::console::AgentConsoleBackend::new(destination.clone()); + backend.write(&wire[split..]).unwrap(); + let suffix = destination.tx_ring.pop().unwrap(); + assert!(codec::try_decode_frame_from_bytes(&mut BytesMut::from(suffix.as_ref())).is_err()); + source_input.extend_from_slice(suffix.as_ref()); + assert!( + codec::try_decode_frame_from_bytes(&mut source_input) + .unwrap() + .is_some() + ); + assert!(source_input.is_empty()); + } + #[tokio::test] async fn restore_thaw_drains_backpressured_bulk_before_acknowledgement() { let shared = Arc::new(ConsoleSharedState::with_capacity(4096)); @@ -6106,12 +7083,13 @@ mod tests { let frame = encoded_message_id(MessageType::Pong, 1, µsandbox_protocol::core::Pong {}); let reader = tokio::spawn(lane_reader_task( BytesMut::from(frame.as_slice()), - shared, + Arc::clone(&shared), GuestLane::Control, false, false, sender, Arc::new(Semaphore::new(4096)), + Arc::clone(&shared.workload_control), )); let event = tokio::time::timeout(std::time::Duration::from_secs(1), receiver.recv()) .await @@ -6147,4 +7125,1204 @@ mod tests { "atomic publication must not leave temporary activation records" ); } + + fn workload_test_shared(capacity: usize, dual_port: bool) -> Arc { + let shared = Arc::new(ConsoleSharedState::with_capacity(capacity)); + shared.workload_control.install_ready( + 9, + Ready { + workload_transport_barrier_version: Some(WORKLOAD_TRANSPORT_BARRIER_VERSION), + ..Default::default() + }, + dual_port, + ); + shared + } + + async fn next_host_fragment(shared: &ConsoleSharedState) -> Bytes { + tokio::time::timeout(std::time::Duration::from_secs(1), async { + loop { + if let Some(bytes) = shared.rx_ring.pop() { + let bytes = Bytes::copy_from_slice(&bytes); + shared.rx_capacity_wake.wake(); + return bytes; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("host writer made no progress") + } + + #[tokio::test] + async fn workload_restored_payload_debt_allows_fresh_lease_and_exec() { + use microsandbox_protocol::core::{ + WORKLOAD_TRANSPORT_BULK_BYTES, WORKLOAD_TRANSPORT_BULK_FRAMES, + WORKLOAD_TRANSPORT_CONTROL_BYTES, WORKLOAD_TRANSPORT_CONTROL_FRAMES, + WorkloadTransportCredit, WorkloadTransportPosition, + }; + let shared = workload_test_shared(4096, false); + let control = Arc::clone(&shared.workload_control); + // The inherited stdin remains unconsumed in guest RAM. Its cumulative debt must not be + // forgiven just to make a new lease/exec usable on the restored host's empty queue. + let position = WorkloadTransportPosition { + bulk_bytes: WORKLOAD_TRANSPORT_BULK_BYTES, + bulk_frames: WORKLOAD_TRANSPORT_BULK_FRAMES, + ..Default::default() + }; + let mut credit = WorkloadTransportCredit { + control_bytes: WORKLOAD_TRANSPORT_CONTROL_BYTES, + control_frames: WORKLOAD_TRANSPORT_CONTROL_FRAMES, + bulk_bytes: position.bulk_bytes, + bulk_frames: position.bulk_frames, + }; + control.restore(position, credit, 0).unwrap(); + let (tx, rx) = ControlWriter::new(); + let stdin = Bytes::from(encoded_message_id( + MessageType::ExecStdin, + 1, + µsandbox_protocol::exec::ExecStdin { + data: b"retained".to_vec(), + }, + )); + let eof = Bytes::from(encoded_message_id( + MessageType::ExecStdin, + 1, + µsandbox_protocol::exec::ExecStdin { data: Vec::new() }, + )); + tx.send(ControlWrite::ordinary(stdin.clone(), 1, true)) + .await + .unwrap(); + tx.send(ControlWrite::ordinary(eof.clone(), 1, true)) + .await + .unwrap(); + let finish = Bytes::from(encoded_message_id(MessageType::BulkFinish, 1, &())); + tx.send(ControlWrite::ordinary(finish.clone(), 1, false)) + .await + .unwrap(); + send_relay_client_connected(&tx, 100, 200, TEST_INCARNATION) + .await + .unwrap(); + let exec = Bytes::from(encoded_message_id(MessageType::ExecRequest, 100, &())); + tx.send(ControlWrite::ordinary(exec.clone(), 100, false)) + .await + .unwrap(); + let writer = tokio::spawn(ring_writer_task(Arc::clone(&shared), rx)); + assert_eq!( + next_host_fragment(&shared).await.as_ref(), + encode_relay_client_connected(RelayClientConnected { + id_start: 100, + id_end_exclusive: 200, + incarnation: TEST_INCARNATION, + }) + ); + assert_eq!(next_host_fragment(&shared).await, exec); + assert!(shared.rx_ring.pop().is_none()); + let gate = control.gate(); + let parked = control.parked_position().await.unwrap(); + assert_eq!(parked.bulk_bytes, position.bulk_bytes); + assert_eq!(parked.bulk_frames, position.bulk_frames); + credit.bulk_bytes += (stdin.len() + eof.len()) as u64; + credit.bulk_frames += 2; + control.update_credit(credit).unwrap(); + assert!(shared.rx_ring.pop().is_none()); + gate.release(); + // Same-correlation metadata remains behind both bytes and EOF despite spare control credit. + assert_eq!(next_host_fragment(&shared).await, stdin); + assert_eq!(next_host_fragment(&shared).await, eof); + assert_eq!(next_host_fragment(&shared).await, finish); + drop(tx); + writer.await.unwrap().unwrap(); + } + + #[test] + fn workload_metadata_scheduler_preserves_client_and_global_fences() { + use microsandbox_protocol::core::{WorkloadTransportCredit, WorkloadTransportPosition}; + let shared = workload_test_shared(4096, false); + let control = &shared.workload_control; + control + .restore( + WorkloadTransportPosition::default(), + WorkloadTransportCredit { + control_bytes: 4096, + control_frames: 16, + ..Default::default() + }, + 0, + ) + .unwrap(); + let frame = Bytes::from_static(b"fence"); + let mut pending = VecDeque::from([ + ControlWrite::ordinary(frame.clone(), 101, true), + ControlWrite::client_fence(frame.clone(), 100, 200), + ControlWrite::ordinary(frame.clone(), 102, false), + ControlWrite::ordinary(frame.clone(), 201, false), + ControlWrite::from(frame.clone()), + ControlWrite::ordinary(frame, 301, false), + ]); + assert!(matches!( + next_control_write(&mut pending, control) + .unwrap() + .unwrap() + .order, + ControlOrder::Correlation(201) + )); + assert!(next_control_write(&mut pending, control).unwrap().is_none()); + assert_eq!(pending.len(), 5); + } + + #[tokio::test] + async fn workload_maintenance_clock_and_cleanup_do_not_fence_unrelated_clients() { + use microsandbox_protocol::core::{ + ClockSync, WorkloadTransportCredit, WorkloadTransportPosition, + }; + let shared = workload_test_shared(4096, false); + let control = &shared.workload_control; + let mut credit = WorkloadTransportCredit { + control_bytes: 4096, + control_frames: 16, + ..Default::default() + }; + control + .restore(WorkloadTransportPosition::default(), credit, 0) + .unwrap(); + let (tx, rx) = ControlWriter::new(); + let stdin = Bytes::from(encoded_message_id(MessageType::ExecStdin, 1, &())); + let kill = Bytes::from(encoded_message_id( + MessageType::ExecSignal, + 101, + &ExecSignal { signal: 9 }, + )); + let same_flow_kill = Bytes::from(encoded_message_id( + MessageType::ExecSignal, + 1, + &ExecSignal { signal: 9 }, + )); + let exec = Bytes::from(encoded_message_id(MessageType::ExecRequest, 201, &())); + tx.send(ControlWrite::ordinary(stdin.clone(), 1, true)) + .await + .unwrap(); + tx.send(ControlWrite::clock_sync().unwrap()).await.unwrap(); + tx.send(ControlWrite::ordinary(kill.clone(), 101, false)) + .await + .unwrap(); + tx.send(ControlWrite::ordinary(same_flow_kill.clone(), 1, false)) + .await + .unwrap(); + send_relay_client_connected(&tx, 200, 300, TEST_INCARNATION) + .await + .unwrap(); + tx.send(ControlWrite::ordinary(exec.clone(), 201, false)) + .await + .unwrap(); + tx.send(ControlWrite::clock_sync().unwrap()).await.unwrap(); + let writer = tokio::spawn(ring_writer_task(Arc::clone(&shared), rx)); + + let first_clock = decode_frame(&next_host_fragment(&shared).await).unwrap(); + assert_eq!(first_clock.t, MessageType::ClockSync); + assert_eq!(next_host_fragment(&shared).await, kill); + assert_eq!( + next_host_fragment(&shared).await.as_ref(), + encode_relay_client_connected(RelayClientConnected { + id_start: 200, + id_end_exclusive: 300, + incarnation: TEST_INCARNATION, + }) + ); + assert_eq!(next_host_fragment(&shared).await, exec); + let second_clock = decode_frame(&next_host_fragment(&shared).await).unwrap(); + assert_eq!(second_clock.t, MessageType::ClockSync); + assert!( + second_clock.payload::().unwrap().unix_time_nanos + >= first_clock.payload::().unwrap().unix_time_nanos + ); + let gate = control.gate(); + let position = control.parked_position().await.unwrap(); + assert_eq!( + position.bulk_frames, 0, + "no blocked data was discarded or admitted" + ); + assert!(shared.rx_ring.pop().is_none()); + credit.bulk_bytes = stdin.len() as u64; + credit.bulk_frames = 1; + control.update_credit(credit).unwrap(); + gate.release(); + assert_eq!(next_host_fragment(&shared).await, stdin); + assert_eq!(next_host_fragment(&shared).await, same_flow_kill); + drop(tx); + writer.await.unwrap().unwrap(); + } + + #[tokio::test] + async fn workload_maintenance_clock_stays_behind_global_lifecycle_fence() { + use microsandbox_protocol::core::{WorkloadTransportCredit, WorkloadTransportPosition}; + let shared = workload_test_shared(4096, false); + let control = &shared.workload_control; + let mut credit = WorkloadTransportCredit { + control_bytes: 4096, + control_frames: 16, + ..Default::default() + }; + control + .restore(WorkloadTransportPosition::default(), credit, 0) + .unwrap(); + let (tx, rx) = ControlWriter::new(); + let stdin = Bytes::from(encoded_message_id(MessageType::ExecStdin, 1, &())); + let shutdown = Bytes::from(encoded_message_id(MessageType::Shutdown, 0, &())); + tx.send(ControlWrite::ordinary(stdin.clone(), 1, true)) + .await + .unwrap(); + tx.send(shutdown.clone().into()).await.unwrap(); + tx.send(ControlWrite::clock_sync().unwrap()).await.unwrap(); + let writer = tokio::spawn(ring_writer_task(Arc::clone(&shared), rx)); + tokio::time::sleep(Duration::from_millis(10)).await; + assert!(shared.rx_ring.pop().is_none()); + let gate = control.gate(); + assert_eq!( + control.parked_position().await.unwrap(), + WorkloadTransportPosition::default() + ); + credit.bulk_bytes = stdin.len() as u64; + credit.bulk_frames = 1; + control.update_credit(credit).unwrap(); + assert!( + shared.rx_ring.pop().is_none(), + "pause still gates maintenance" + ); + gate.release(); + assert_eq!(next_host_fragment(&shared).await, stdin); + assert_eq!(next_host_fragment(&shared).await, shutdown); + assert_eq!( + decode_frame(&next_host_fragment(&shared).await).unwrap().t, + MessageType::ClockSync + ); + drop(tx); + writer.await.unwrap().unwrap(); + } + + #[tokio::test] + async fn workload_clock_refreshes_after_full_ring_and_pause_without_charging_stale_bytes() { + use microsandbox_protocol::core::{ClockSync, WorkloadTransportPosition}; + let shared = workload_test_shared(4096, false); + shared.rx_ring.push(Bytes::from(vec![0; 4096])).unwrap(); + let (tx, rx) = ControlWriter::new(); + tx.send(ControlWrite::clock_sync().unwrap()).await.unwrap(); + let writer = tokio::spawn(ring_writer_task(Arc::clone(&shared), rx)); + tokio::time::sleep(Duration::from_millis(20)).await; + let gate = shared.workload_control.gate(); + assert_eq!( + shared.workload_control.parked_position().await.unwrap(), + WorkloadTransportPosition::default() + ); + assert_eq!(next_host_fragment(&shared).await.len(), 4096); + tokio::time::sleep(Duration::from_millis(20)).await; + assert!(shared.rx_ring.pop().is_none()); + let not_before = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() as u64; + gate.release(); + let wire = next_host_fragment(&shared).await; + let clock = decode_frame(&wire).unwrap().payload::().unwrap(); + assert!( + clock.unix_time_nanos >= not_before, + "queued clock age must not cross pause" + ); + let gate = shared.workload_control.gate(); + let position = shared.workload_control.parked_position().await.unwrap(); + assert_eq!(position.control_bytes, wire.len() as u64); + assert_eq!(position.control_frames, 1); + assert_eq!( + tx.control_bytes.available_permits(), + AGENT_WRITE_CONTROL_BYTES + ); + gate.release(); + drop(tx); + writer.await.unwrap().unwrap(); + } + + #[tokio::test] + async fn workload_clock_credit_uses_actual_cbor_width_not_reserved_maximum() { + use microsandbox_protocol::core::{ + ClockSync, WorkloadTransportCredit, WorkloadTransportPosition, + }; + for timestamp in [0, u32::MAX as u64, u64::MAX] { + let shared = workload_test_shared(4096, false); + let frame = crate::clock::encode_clock_sync_frame(timestamp).unwrap(); + shared + .workload_control + .restore( + WorkloadTransportPosition::default(), + WorkloadTransportCredit { + control_bytes: frame.len() as u64, + control_frames: 1, + ..Default::default() + }, + 0, + ) + .unwrap(); + let (tx, mut rx) = ControlWriter::new(); + let queued = ControlWrite::clock_sync().unwrap(); + let reservation = queued.data.len(); + assert!(reservation >= frame.len()); + tx.send(queued).await.unwrap(); + let mut write = rx.recv().await.unwrap(); + assert_eq!( + tx.control_bytes.available_permits(), + AGENT_WRITE_CONTROL_BYTES - reservation + ); + assert!( + admit_control_write( + &mut write, + &shared.workload_control, + Some(&shared), + &mut false, + || crate::clock::encode_clock_sync_frame(timestamp) + ) + .unwrap() + ); + assert_eq!(write.data, frame); + assert_eq!( + decode_frame(&write.data) + .unwrap() + .payload::() + .unwrap() + .unix_time_nanos, + timestamp + ); + assert!(!shared.workload_control.admit(false, 1).unwrap()); + drop(write); + assert_eq!( + tx.control_bytes.available_permits(), + AGENT_WRITE_CONTROL_BYTES + ); + } + } + + fn ordered_tcp_metadata(kind: MessageType, payload: &T) -> ControlWrite { + let message = Message::with_payload(kind, 17, payload).unwrap(); + let mut encoded = Vec::new(); + codec::encode_to_buf(&message, &mut encoded).unwrap(); + let mut write = ControlWrite::ordinary(Bytes::from(encoded), 17, false); + write.classify_tcp_order(17, None, Some(&message)); + write + } + + fn ordered_tcp_input() -> ControlWrite { + let record = microsandbox_protocol::bulk::BulkRecord { + id: 17, + kind: BulkKind::Tcp, + flow: BulkFlow::HostToGuest, + offset: 0, + payload: Bytes::from_static(b"request tail"), + }; + let mut encoded = Vec::new(); + codec::encode_bulk_to_buf(&record, &mut encoded).unwrap(); + let mut write = ControlWrite::ordinary(Bytes::from(encoded), 17, true); + write.classify_tcp_order(17, Some(bulk_wire_metadata(&write.data).unwrap()), None); + write + } + + fn tcp_input_finish() -> ControlWrite { + ordered_tcp_metadata( + MessageType::BulkFinish, + &BulkFinish { + kind: BulkKind::Tcp, + flow: BulkFlow::HostToGuest, + final_offset: b"request tail".len() as u64, + }, + ) + } + + fn tcp_output_credit() -> ControlWrite { + ordered_tcp_metadata( + MessageType::BulkCredit, + &BulkCredit { + kind: BulkKind::Tcp, + flow: BulkFlow::GuestToHost, + consumed_offset: 0, + credit_limit: microsandbox_protocol::bulk::DEFAULT_BULK_WINDOW, + }, + ) + } + + #[test] + fn workload_combined_tcp_credit_passes_blocked_input_and_finish_only() { + use microsandbox_protocol::core::{WorkloadTransportCredit, WorkloadTransportPosition}; + let shared = workload_test_shared(4096, false); + let control = &shared.workload_control; + let mut credit = WorkloadTransportCredit { + control_bytes: 4096, + control_frames: 16, + ..Default::default() + }; + control + .restore(WorkloadTransportPosition::default(), credit, 0) + .unwrap(); + let input = ordered_tcp_input(); + let input_bytes = input.data.clone(); + let finish = tcp_input_finish(); + let finish_bytes = finish.data.clone(); + let mut pending = VecDeque::from([input, finish, tcp_output_credit()]); + let gate = control.gate(); + assert!(next_control_write(&mut pending, control).unwrap().is_none()); + gate.release(); + let returned = next_control_write(&mut pending, control).unwrap().unwrap(); + assert!(matches!(returned.order, ControlOrder::TcpOutputCredit(17))); + assert_eq!(pending.len(), 2); + assert!(next_control_write(&mut pending, control).unwrap().is_none()); + credit.bulk_bytes = input_bytes.len() as u64; + credit.bulk_frames = 1; + control.update_credit(credit).unwrap(); + assert_eq!( + next_control_write(&mut pending, control) + .unwrap() + .unwrap() + .data, + input_bytes + ); + assert_eq!( + next_control_write(&mut pending, control) + .unwrap() + .unwrap() + .data, + finish_bytes + ); + assert!(pending.is_empty()); + + // With input capacity available, the new exception does not turn credit into priority. + let shared = workload_test_shared(4096, false); + let mut pending = + VecDeque::from([ordered_tcp_input(), tcp_input_finish(), tcp_output_credit()]); + for expected in [ + ControlOrder::TcpInputData(17), + ControlOrder::TcpInputFinish(17), + ControlOrder::TcpOutputCredit(17), + ] { + let actual = next_control_write(&mut pending, &shared.workload_control) + .unwrap() + .unwrap() + .order; + assert_eq!( + std::mem::discriminant(&actual), + std::mem::discriminant(&expected) + ); + } + } + + #[test] + fn workload_combined_tcp_credit_never_crosses_control_or_owner_fences() { + use microsandbox_protocol::core::{WorkloadTransportCredit, WorkloadTransportPosition}; + let shared = workload_test_shared(4096, false); + let control = &shared.workload_control; + control + .restore( + WorkloadTransportPosition::default(), + WorkloadTransportCredit { + control_bytes: 4096, + control_frames: 16, + ..Default::default() + }, + 0, + ) + .unwrap(); + for fence in [ + ordered_tcp_metadata(MessageType::BulkCancel, &()), + ordered_tcp_metadata(MessageType::TcpConnect, &()), + ordered_tcp_metadata(MessageType::Ping, &()), + ControlWrite::client_fence(Bytes::new(), 1, 100), + ControlWrite::from(Bytes::new()), + ] { + let mut pending = VecDeque::from([ordered_tcp_input(), fence, tcp_output_credit()]); + assert!(next_control_write(&mut pending, control).unwrap().is_none()); + assert_eq!(pending.len(), 3); + } + let valid = BulkCredit { + kind: BulkKind::Tcp, + flow: BulkFlow::GuestToHost, + consumed_offset: 0, + credit_limit: microsandbox_protocol::bulk::DEFAULT_BULK_WINDOW, + }; + for payload in [ + BulkCredit { + kind: BulkKind::Filesystem, + ..valid + }, + BulkCredit { + flow: BulkFlow::HostToGuest, + ..valid + }, + BulkCredit { + consumed_offset: 2, + credit_limit: 1, + ..valid + }, + BulkCredit { + credit_limit: MAX_BULK_WINDOW + 1, + ..valid + }, + ] { + let mut pending = VecDeque::from([ + ordered_tcp_input(), + ordered_tcp_metadata(MessageType::BulkCredit, &payload), + ]); + assert!(next_control_write(&mut pending, control).unwrap().is_none()); + } + let mut pending = VecDeque::from([ + ordered_tcp_input(), + ordered_tcp_metadata(MessageType::BulkCredit, &()), + ]); + assert!(next_control_write(&mut pending, control).unwrap().is_none()); + // The exclusive raw flag is not sufficient: only previously validated TCP input gets + // the exception. Filesystem or reverse-direction raw metadata retains strict ordering. + for (kind, flow) in [ + (BulkKind::Filesystem, BulkFlow::HostToGuest), + (BulkKind::Tcp, BulkFlow::GuestToHost), + ] { + let mut input = ControlWrite::ordinary(ordered_tcp_input().data, 17, true); + input.classify_tcp_order(17, Some((kind, flow, 0, 12)), None); + let mut pending = VecDeque::from([input, tcp_output_credit()]); + assert!(next_control_write(&mut pending, control).unwrap().is_none()); + } + // Only the correctly directed TCP finish commutes; unknown/malformed metadata remains a fence. + for finish in [ + BulkFinish { + kind: BulkKind::Tcp, + flow: BulkFlow::GuestToHost, + final_offset: 0, + }, + BulkFinish { + kind: BulkKind::Filesystem, + flow: BulkFlow::HostToGuest, + final_offset: 0, + }, + ] { + let mut pending = VecDeque::from([ + ordered_tcp_input(), + ordered_tcp_metadata(MessageType::BulkFinish, &finish), + tcp_output_credit(), + ]); + assert!(next_control_write(&mut pending, control).unwrap().is_none()); + } + } + + #[tokio::test] + async fn workload_class_reservations_bound_pending_bytes_and_frames() { + let (tx, mut rx) = ControlWriter::new(); + let data = Bytes::from(vec![0; AGENT_WRITE_DATA_BYTES / AGENT_WRITE_CLASS_FRAMES]); + let metadata = Bytes::from(vec![ + 0; + AGENT_WRITE_CONTROL_BYTES / AGENT_WRITE_CLASS_FRAMES + ]); + let mut pending = Vec::new(); + for id in 1..=AGENT_WRITE_CLASS_FRAMES as u32 { + tx.try_send(ControlWrite::ordinary(data.clone(), id, true)) + .unwrap(); + pending.push(rx.recv().await.unwrap()); + } + assert_eq!(tx.data_bytes.available_permits(), 0); + assert_eq!(tx.data_frames.available_permits(), 0); + assert!(matches!( + tx.try_send(ControlWrite::ordinary(Bytes::new(), 99, true)), + Err(mpsc::error::TrySendError::Full(_)) + )); + for id in 100..100 + AGENT_WRITE_CLASS_FRAMES as u32 { + tx.try_send(ControlWrite::ordinary(metadata.clone(), id, false)) + .unwrap(); + pending.push(rx.recv().await.unwrap()); + } + assert_eq!(pending.len(), AGENT_WRITE_CHANNEL_CAPACITY); + assert_eq!(tx.control_bytes.available_permits(), 0); + assert_eq!(tx.control_frames.available_permits(), 0); + assert!(matches!( + tx.try_send(ControlWrite::ordinary(Bytes::new(), 999, false)), + Err(mpsc::error::TrySendError::Full(_)) + )); + drop(pending); + assert_eq!(tx.data_bytes.available_permits(), AGENT_WRITE_DATA_BYTES); + assert_eq!( + tx.control_bytes.available_permits(), + AGENT_WRITE_CONTROL_BYTES + ); + assert_eq!(tx.data_frames.available_permits(), AGENT_WRITE_CLASS_FRAMES); + assert_eq!( + tx.control_frames.available_permits(), + AGENT_WRITE_CLASS_FRAMES + ); + } + + #[tokio::test] + async fn workload_reservation_waiters_cancel_and_wake_on_receiver_close() { + let (tx, mut rx) = ControlWriter::new(); + let mut retained = Vec::new(); + for id in 1..=AGENT_WRITE_CLASS_FRAMES as u32 { + tx.send(ControlWrite::ordinary(Bytes::new(), id, true)) + .await + .unwrap(); + retained.push(rx.recv().await.unwrap()); + } + assert!( + tokio::time::timeout( + Duration::from_millis(10), + tx.send(ControlWrite::ordinary(Bytes::new(), 99, true)) + ) + .await + .is_err() + ); + assert_eq!(tx.data_frames.available_permits(), 0); + let sender = tx.clone(); + let waiting = tokio::spawn(async move { + sender + .send(ControlWrite::ordinary(Bytes::new(), 100, true)) + .await + }); + drop(rx); + assert!( + tokio::time::timeout(Duration::from_secs(1), waiting) + .await + .unwrap() + .unwrap() + .is_err() + ); + drop(retained); + assert_eq!(tx.data_frames.available_permits(), AGENT_WRITE_CLASS_FRAMES); + } + + #[tokio::test] + async fn workload_private_freeze_bypasses_both_full_admission_classes() { + use microsandbox_protocol::core::{ + WorkloadFreeze, WorkloadFrozen, WorkloadTransportCredit, WorkloadTransportPosition, + }; + let shared = workload_test_shared(4096, false); + let control = Arc::clone(&shared.workload_control); + control + .restore( + WorkloadTransportPosition::default(), + WorkloadTransportCredit::default(), + 0, + ) + .unwrap(); + let gate = control.gate(); + let (tx, rx) = ControlWriter::new(); + let mut expected = Vec::new(); + for uses_data_credit in [true, false] { + for id in 1..=AGENT_WRITE_CLASS_FRAMES as u32 { + let kind = if uses_data_credit { + MessageType::ExecStdin + } else { + MessageType::Ping + }; + let bytes = Bytes::from(encoded_message_id(kind, id, &())); + tx.send(ControlWrite::ordinary(bytes.clone(), id, uses_data_credit)) + .await + .unwrap(); + expected.push(bytes); + } + } + assert_eq!(tx.data_frames.available_permits(), 0); + assert_eq!(tx.control_frames.available_permits(), 0); + let writer = tokio::spawn(ring_writer_task(Arc::clone(&shared), rx)); + let position = tokio::time::timeout(Duration::from_secs(1), control.parked_position()) + .await + .unwrap() + .unwrap(); + assert_eq!(position, WorkloadTransportPosition::default()); + let requester = Arc::clone(&control); + let request = tokio::spawn(async move { + requester + .request( + Message::with_payload( + MessageType::WorkloadFreeze, + 0, + &WorkloadFreeze { + attempt_id: "full-classes".into(), + host_input: position, + }, + ) + .unwrap(), + "full-classes", + ) + .await + }); + assert_eq!( + decode_frame(&next_host_fragment(&shared).await).unwrap().t, + MessageType::WorkloadFreeze + ); + assert!(shared.rx_ring.pop().is_none()); + control + .reply( + Message::with_payload( + MessageType::WorkloadFrozen, + WORKLOAD_CONTROL_ID, + &WorkloadFrozen { + attempt_id: "full-classes".into(), + guest_bulk_bytes_target: 0, + input_credit: WorkloadTransportCredit::default(), + }, + ) + .unwrap(), + ) + .unwrap(); + request.await.unwrap().unwrap(); + control + .update_credit(WorkloadTransportCredit { + control_bytes: 4096, + control_frames: AGENT_WRITE_CLASS_FRAMES as u64, + bulk_bytes: 4096, + bulk_frames: AGENT_WRITE_CLASS_FRAMES as u64, + }) + .unwrap(); + gate.release(); + for frame in expected { + assert_eq!(next_host_fragment(&shared).await, frame); + } + assert_eq!(tx.data_frames.available_permits(), AGENT_WRITE_CLASS_FRAMES); + assert_eq!( + tx.control_frames.available_permits(), + AGENT_WRITE_CLASS_FRAMES + ); + drop(tx); + writer.await.unwrap().unwrap(); + } + + #[tokio::test] + async fn workload_gate_keeps_source_fifo_until_confirmed_continue() { + use microsandbox_protocol::core::{ + WorkloadFreeze, WorkloadFrozen, WorkloadTransportCredit, WorkloadTransportPosition, + }; + let shared = workload_test_shared(4096, false); + let control = Arc::clone(&shared.workload_control); + // Model a live guest whose previously admitted input still occupies the whole window. + control + .restore( + WorkloadTransportPosition::default(), + WorkloadTransportCredit::default(), + 0, + ) + .unwrap(); + let (tx, rx) = mpsc::channel(2); + let writer = tokio::spawn(ring_writer_task(Arc::clone(&shared), rx)); + let first = Bytes::from(encoded_message_id( + MessageType::Ping, + 1, + µsandbox_protocol::core::Ping {}, + )); + let second = Bytes::from(encoded_message_id( + MessageType::Ping, + 2, + µsandbox_protocol::core::Ping {}, + )); + let (done, mut completed) = oneshot::channel(); + tx.send(ControlWrite { + completion: Some(done), + ..first.clone().into() + }) + .await + .unwrap(); + tx.send(second.clone().into()).await.unwrap(); + let gate = control.gate(); + let position = + tokio::time::timeout(std::time::Duration::from_secs(1), control.parked_position()) + .await + .unwrap() + .unwrap(); + assert!(shared.rx_ring.pop().is_none()); + assert!(completed.try_recv().is_err()); + let request = Message::with_payload( + MessageType::WorkloadFreeze, + 0, + &WorkloadFreeze { + attempt_id: "fifo".into(), + host_input: position, + }, + ) + .unwrap(); + let requester = Arc::clone(&control); + let freeze = tokio::spawn(async move { requester.request(request, "fifo").await }); + assert_eq!( + decode_frame(&next_host_fragment(&shared).await).unwrap().t, + MessageType::WorkloadFreeze + ); + control + .reply( + Message::with_payload( + MessageType::WorkloadFrozen, + WORKLOAD_CONTROL_ID, + &WorkloadFrozen { + attempt_id: "fifo".into(), + guest_bulk_bytes_target: 0, + input_credit: Default::default(), + }, + ) + .unwrap(), + ) + .unwrap(); + freeze.await.unwrap().unwrap(); + assert!( + shared.rx_ring.pop().is_none(), + "Frozen must not release source input" + ); + let request = Message::with_payload( + MessageType::WorkloadThaw, + 0, + &WorkloadThaw { + attempt_id: "fifo".into(), + mode: microsandbox_protocol::core::WorkloadThawMode::Continue, + }, + ) + .unwrap(); + let requester = Arc::clone(&control); + let thaw = tokio::spawn(async move { requester.request(request, "fifo").await }); + assert_eq!( + decode_frame(&next_host_fragment(&shared).await).unwrap().t, + MessageType::WorkloadThaw + ); + control + .reply( + Message::with_payload( + MessageType::WorkloadThawed, + WORKLOAD_CONTROL_ID, + &WorkloadThawed { + attempt_id: "fifo".into(), + }, + ) + .unwrap(), + ) + .unwrap(); + thaw.await.unwrap().unwrap(); + control + .update_credit(WorkloadTransportCredit { + control_bytes: 4096, + control_frames: 2, + ..Default::default() + }) + .unwrap(); + assert!( + shared.rx_ring.pop().is_none(), + "credit alone cannot release the gate" + ); + gate.release(); + assert_eq!(next_host_fragment(&shared).await, first); + assert_eq!(next_host_fragment(&shared).await, second); + completed.await.unwrap(); + writer.abort(); + let _ = writer.await; + } + + #[tokio::test] + async fn workload_canceled_frozen_and_recovery_thawed_share_one_input_batch() { + use microsandbox_protocol::core::{WorkloadFreeze, WorkloadFrozen}; + let shared = workload_test_shared(4096, false); + let control = Arc::clone(&shared.workload_control); + let mut writes = control.start(); + let requester = Arc::clone(&control); + let freeze = tokio::spawn(async move { + requester + .request( + Message::with_payload( + MessageType::WorkloadFreeze, + 0, + &WorkloadFreeze { + attempt_id: "cancel".into(), + host_input: Default::default(), + }, + ) + .unwrap(), + "cancel", + ) + .await + }); + writes.recv().await.unwrap(); + freeze.abort(); + let _ = freeze.await; + let requester = Arc::clone(&control); + let thaw = tokio::spawn(async move { + requester + .request( + Message::with_payload( + MessageType::WorkloadThaw, + 0, + &WorkloadThaw { + attempt_id: "cancel".into(), + mode: microsandbox_protocol::core::WorkloadThawMode::Continue, + }, + ) + .unwrap(), + "cancel", + ) + .await + }); + writes.recv().await.unwrap(); + let mut wire = encoded_message_id( + MessageType::WorkloadFrozen, + WORKLOAD_CONTROL_ID, + &WorkloadFrozen { + attempt_id: "cancel".into(), + guest_bulk_bytes_target: 0, + input_credit: Default::default(), + }, + ); + wire.extend_from_slice(&encoded_message_id( + MessageType::WorkloadThawed, + WORKLOAD_CONTROL_ID, + &WorkloadThawed { + attempt_id: "cancel".into(), + }, + )); + let reader = tokio::spawn(combined_ring_reader_task( + BytesMut::from(wire.as_slice()), + shared, + false, + Arc::new(Mutex::new(HashMap::new())), + None, + Arc::new(SessionRegistry::default()), + Arc::new(Mutex::new(HashMap::new())), + )); + tokio::time::timeout(std::time::Duration::from_secs(1), thaw) + .await + .unwrap() + .unwrap() + .unwrap(); + reader.abort(); + let _ = reader.await; + } + + #[tokio::test] + async fn workload_private_reply_bypasses_sdk_output_admission() { + use microsandbox_protocol::core::{WorkloadFreeze, WorkloadFrozen}; + let shared = workload_test_shared(4096, false); + let control = Arc::clone(&shared.workload_control); + let mut writes = control.start(); + let requester = Arc::clone(&control); + let request = tokio::spawn(async move { + requester + .request( + Message::with_payload( + MessageType::WorkloadFreeze, + 0, + &WorkloadFreeze { + attempt_id: "private".into(), + host_input: Default::default(), + }, + ) + .unwrap(), + "private", + ) + .await + }); + writes.recv().await.unwrap(); + let wire = encoded_message_id( + MessageType::WorkloadFrozen, + WORKLOAD_CONTROL_ID, + &WorkloadFrozen { + attempt_id: "private".into(), + guest_bulk_bytes_target: 0, + input_credit: Default::default(), + }, + ); + let (events, mut received) = mpsc::channel(1); + let reader = tokio::spawn(lane_reader_task( + BytesMut::from(wire.as_slice()), + shared, + GuestLane::Control, + true, + false, + events, + Arc::new(Semaphore::new(0)), + control, + )); + tokio::time::timeout(std::time::Duration::from_secs(1), request) + .await + .unwrap() + .unwrap() + .unwrap(); + assert!(received.try_recv().is_err()); + reader.abort(); + let _ = reader.await; + } + + #[tokio::test] + async fn workload_bulk_gate_finishes_current_fragmented_record() { + let shared = workload_test_shared(100, true); + let control = Arc::clone(&shared.workload_control); + let (tx, rx) = mpsc::channel(2); + let budget = Arc::new(Semaphore::new(1024)); + let data = Bytes::from(encoded_host_raw(1, 0, &[0x5a; 64])); + tx.send(BulkWriterCommand::Write(BulkWrite { + id: 1, + incarnation: TEST_INCARNATION, + flow: BulkFlow::HostToGuest, + payload_len: 64, + _permit: Arc::clone(&budget) + .acquire_many_owned(data.len() as u32) + .await + .unwrap(), + data: BulkWriteData::Inline(data.clone()), + })) + .await + .unwrap(); + let writer = tokio::spawn(bulk_ring_writer_task( + Arc::clone(&shared), + rx, + Arc::clone(&control), + )); + let prefix = next_host_fragment(&shared).await; + assert_eq!(prefix.as_ref(), TEST_INCARNATION); + let gate = control.gate(); + control.park(false); // This fixture has no ordinary primary writer. + let position = + tokio::time::timeout(std::time::Duration::from_secs(1), control.parked_position()) + .await + .unwrap() + .unwrap(); + assert_eq!( + position.bulk_bytes, + (CLIENT_INCARNATION_SIZE + data.len()) as u64 + ); + assert_eq!(position.bulk_frames, 1); + assert_eq!(next_host_fragment(&shared).await, data); + gate.release(); + drop(tx); + tokio::time::timeout(std::time::Duration::from_secs(1), writer) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(budget.available_permits(), 1024); + } + + #[tokio::test] + async fn workload_post_ready_shutdown_uses_counted_fifo() { + let shared = workload_test_shared(4096, false); + let control = Arc::clone(&shared.workload_control); + let (tx, rx) = ControlWriter::new(); + control.register_ordinary_writer(tx.clone()); + let first = Bytes::from(encoded_message_id( + MessageType::Ping, + 1, + µsandbox_protocol::core::Ping {}, + )); + tx.send(first.clone().into()).await.unwrap(); + let writer = tokio::spawn(ring_writer_task(Arc::clone(&shared), rx)); + assert_eq!(next_host_fragment(&shared).await, first); + let shutdown = encoded_message_id(MessageType::Shutdown, 0, &()); + let shutdown_len = shutdown.len(); + let sender = Arc::clone(&shared); + tokio::task::spawn_blocking(move || { + push_guest_frame_until(&sender, shutdown, std::time::Duration::from_secs(1)) + }) + .await + .unwrap() + .unwrap(); + assert_eq!( + decode_frame(&next_host_fragment(&shared).await).unwrap().t, + MessageType::Shutdown + ); + let gate = control.gate(); + let position = + tokio::time::timeout(std::time::Duration::from_secs(1), control.parked_position()) + .await + .unwrap() + .unwrap(); + assert_eq!(position.control_bytes, (first.len() + shutdown_len) as u64); + assert_eq!(position.control_frames, 2); + gate.release(); + writer.abort(); + let _ = writer.await; + } + + #[tokio::test] + async fn workload_post_ready_shutdown_respects_gate_and_deadline() { + let shared = workload_test_shared(4096, false); + let control = Arc::clone(&shared.workload_control); + let (tx, rx) = ControlWriter::new(); + control.register_ordinary_writer(tx); + let writer = tokio::spawn(ring_writer_task(Arc::clone(&shared), rx)); + let gate = control.gate(); + tokio::time::timeout(std::time::Duration::from_secs(1), control.parked_position()) + .await + .unwrap() + .unwrap(); + let sender = Arc::clone(&shared); + let result = tokio::task::spawn_blocking(move || { + push_guest_frame_until( + &sender, + encoded_message_id(MessageType::Shutdown, 0, &()), + std::time::Duration::from_millis(20), + ) + }) + .await + .unwrap(); + assert!(result.unwrap_err().to_string().contains("timed out")); + assert!(shared.rx_ring.pop().is_none()); + writer.abort(); + let _ = writer.await; + gate.release(); + } + + #[test] + fn workload_ready_without_writer_never_falls_back_to_direct_input() { + let shared = workload_test_shared(4096, false); + assert!( + push_guest_frame_until( + &shared, + encoded_message_id(MessageType::Shutdown, 0, &()), + std::time::Duration::ZERO + ) + .unwrap_err() + .to_string() + .contains("not running") + ); + assert!(shared.rx_ring.pop().is_none()); + let pre_ready = ConsoleSharedState::with_capacity(4096); + push_guest_frame_until(&pre_ready, vec![1, 2, 3], std::time::Duration::ZERO).unwrap(); + assert_eq!(pre_ready.rx_ring.pop().unwrap().as_ref(), &[1, 2, 3]); + } + + #[tokio::test] + async fn workload_writer_abort_wakes_pending_lifecycle_request() { + use microsandbox_protocol::core::WorkloadFreeze; + let shared = workload_test_shared(4096, false); + let (tx, rx) = mpsc::channel(1); + let writer = tokio::spawn(ring_writer_task(Arc::clone(&shared), rx)); + tx.send( + Bytes::from(encoded_message_id( + MessageType::Ping, + 1, + µsandbox_protocol::core::Ping {}, + )) + .into(), + ) + .await + .unwrap(); + next_host_fragment(&shared).await; + let control = Arc::clone(&shared.workload_control); + let request = tokio::spawn(async move { + control + .request( + Message::with_payload( + MessageType::WorkloadFreeze, + 0, + &WorkloadFreeze { + attempt_id: "abort".into(), + host_input: Default::default(), + }, + ) + .unwrap(), + "abort", + ) + .await + }); + next_host_fragment(&shared).await; + writer.abort(); + let _ = writer.await; + assert!( + tokio::time::timeout(std::time::Duration::from_secs(1), request) + .await + .unwrap() + .unwrap() + .unwrap_err() + .contains("closed") + ); + } } diff --git a/crates/runtime/lib/runner/vm.rs b/crates/runtime/lib/runner/vm.rs index b43ebb73f..731a449c5 100644 --- a/crates/runtime/lib/runner/vm.rs +++ b/crates/runtime/lib/runner/vm.rs @@ -272,6 +272,9 @@ pub struct VmConfig { /// Guest transparent huge-page policy selected at boot. pub thp: microsandbox_types::TransparentHugePagePolicy, + /// Protected memory cache resolved by the sandbox's owning local backend. + pub memory_cache_dir: Option, + /// Number of virtual CPUs online at boot. pub vcpus: u8, @@ -1048,6 +1051,8 @@ fn run(mut config: Config) -> RuntimeResult { &resolved_bootstrap, tokio_rt.handle().clone(), &config.agent_sock_path, + Arc::clone(&shared.workload_control), + Arc::clone(&shared.resident_paused), ); let context = super::control::ControlContext { executor: match executor { @@ -1292,6 +1297,7 @@ fn run(mut config: Config) -> RuntimeResult { { let shutdown_exit_handle = exit_handle.clone(); let shutdown_reason = Arc::clone(&exit_reason); + let shutdown_paused = Arc::clone(&shared.resident_paused); tokio_rt.spawn(async move { if relay_drain_rx.recv().await.is_some() { shutdown_reason.store( @@ -1301,7 +1307,9 @@ fn run(mut config: Config) -> RuntimeResult { tracing::info!( "core.shutdown forwarded to agentd, allowing flush window before host fallback" ); - tokio::time::sleep(shutdown_flush_timeout).await; + if !shutdown_paused.load(std::sync::atomic::Ordering::Acquire) { + tokio::time::sleep(shutdown_flush_timeout).await; + } tracing::info!("flush window elapsed, triggering host exit"); shutdown_exit_handle.trigger(); } @@ -1353,6 +1361,13 @@ fn run(mut config: Config) -> RuntimeResult { } } + if startup_shared + .resident_paused + .load(std::sync::atomic::Ordering::Acquire) + { + startup_exit_handle.trigger(); + return; + } match request_guest_shutdown(&startup_shared) { Ok(()) => { tokio::time::sleep(startup_shutdown_flush_timeout).await; @@ -1384,6 +1399,12 @@ fn run(mut config: Config) -> RuntimeResult { let mut interval = tokio::time::interval(Duration::from_secs(1)); loop { interval.tick().await; + if heartbeat_shared + .resident_paused + .load(std::sync::atomic::Ordering::Acquire) + { + continue; + } let decision = heartbeat_reader.check(idle_timeout); match decision { @@ -2192,12 +2213,39 @@ fn build_vm( .build() .map_err(|e| RuntimeError::Custom(format!("build VM: {e}")))?; let restored_agent = if let Some(restore) = &config.vm.checkpoint_restore { - let prepared = crate::checkpoint::PreparedCheckpointRestore::open( - restore.closure.clone(), - &restore.checkpoint_root, - ) + let prepared = if restore.local_branch { + crate::checkpoint::PreparedCheckpointRestore::open_local( + restore.closure.clone(), + &restore.checkpoint_id, + ) + } else { + crate::checkpoint::PreparedCheckpointRestore::open( + restore.closure.clone(), + &restore.checkpoint_root, + ) + } .map_err(|error| RuntimeError::Custom(format!("prepare checkpoint restore: {error}")))?; - Some(prepared.install(&mut vm)) + if let Some(admitted) = prepared.disk_closure() { + // Reuse this process's exact admitted file bindings before the closure is moved + // into RAM restoration. The later coordinator opens the completed journal. + crate::checkpoint::seed_restored_root_disk(&config.runtime_dir, &config.vm, admitted) + .map_err(RuntimeError::Custom)?; + } + let cache_root = restore + .forked + .then(|| { + config.vm.memory_cache_dir.clone().ok_or_else(|| { + RuntimeError::Custom( + "CoW memory requires its backend-resolved cache directory".into(), + ) + }) + }) + .transpose()?; + Some( + prepared + .install(&mut vm, cache_root) + .map_err(RuntimeError::Custom)?, + ) } else { None }; @@ -2240,9 +2288,10 @@ fn publish_control_endpoint( run_dir: &Path, sandbox_name: &str, ) -> RuntimeResult<()> { - // Only Unix publishes the legacy socket symlink; Windows uses the named pipe directly. + // Windows uses named pipes and has no legacy Unix socket link to publish. #[cfg(not(unix))] let _ = (run_dir, sandbox_name); + match super::control::spawn_control_listener(control_sock_path.clone(), context) { Ok(()) => { #[cfg(unix)] @@ -2755,6 +2804,15 @@ fn spawn_parent_watchdog( Ok(ParentWatchdogSignal::ParentExited) => { tracing::info!("creator process exited; stopping attached sandbox"); exit_reason.store(EXIT_REASON_PARENT_EXIT, std::sync::atomic::Ordering::SeqCst); + // A suspended guest cannot process shutdown. Release the resident VM + // directly without thawing user workloads merely to stop them. + if shared + .resident_paused + .load(std::sync::atomic::Ordering::Acquire) + { + exit_handle.trigger(); + return; + } if let Err(err) = request_guest_shutdown(&shared) { tracing::warn!(error = %err, "parent-watch shutdown request failed"); } else { diff --git a/crates/runtime/lib/runner/workload_control.rs b/crates/runtime/lib/runner/workload_control.rs new file mode 100644 index 000000000..e210b02a5 --- /dev/null +++ b/crates/runtime/lib/runner/workload_control.rs @@ -0,0 +1,699 @@ +//! Private lifecycle control and bounded input admission for a bundled guest transport. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +use bytes::Bytes; +use microsandbox_protocol::codec; +use microsandbox_protocol::core::{ + CoreError, Ready, WORKLOAD_TRANSPORT_BARRIER_VERSION, WORKLOAD_TRANSPORT_BULK_BYTES, + WORKLOAD_TRANSPORT_BULK_FRAMES, WORKLOAD_TRANSPORT_CONTROL_BYTES, + WORKLOAD_TRANSPORT_CONTROL_FRAMES, WorkloadFrozen, WorkloadThawed, WorkloadTransportCredit, + WorkloadTransportPosition, +}; +use microsandbox_protocol::message::{Message, MessageType}; +use tokio::sync::{Notify, mpsc, oneshot}; + +//-------------------------------------------------------------------------------------------------- +// Constants +//-------------------------------------------------------------------------------------------------- + +/// Outside every leased SDK correlation range; restore activation uses it before the relay runs. +pub(crate) const WORKLOAD_CONTROL_ID: u32 = u32::MAX; +const PRIVATE_QUEUE_CAPACITY: usize = 2; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// A complete trusted frame. Ordinary client data can never enter this mailbox. +pub(crate) struct LifecycleWrite(pub(crate) Bytes); + +struct PendingReply { + request: MessageType, + attempt: String, + reply: oneshot::Sender>, +} + +struct State { + ready: Option<(u8, Ready)>, + active: bool, + closed: bool, + gates: usize, + fenced: bool, + primary_parked: bool, + bulk_parked: bool, + dual_port: bool, + position: WorkloadTransportPosition, + credit: WorkloadTransportCredit, + guest_bulk_bytes: u64, + bulk_tail: usize, + pending: Vec, + ordinary_writer: Option, +} + +/// Shared by the trusted coordinator and relay, never exposed through the SDK socket. +pub(crate) struct WorkloadControl { + state: Mutex, + tx: mpsc::Sender, + rx: Mutex>>, + pub(crate) changed: Notify, +} + +/// A transient capture or resident pause keeps ordinary input source-owned until thaw succeeds. +pub(crate) struct InputGate { + control: Arc, + released: AtomicBool, +} + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl WorkloadControl { + pub(crate) fn new() -> Arc { + let (tx, rx) = mpsc::channel(PRIVATE_QUEUE_CAPACITY); + Arc::new(Self { + state: Mutex::new(State { + ready: None, + active: false, + closed: false, + gates: 0, + fenced: false, + primary_parked: false, + bulk_parked: true, + dual_port: false, + position: WorkloadTransportPosition::default(), + credit: WorkloadTransportCredit::default(), + guest_bulk_bytes: 0, + bulk_tail: 0, + pending: Vec::new(), + ordinary_writer: None, + }), + tx, + rx: Mutex::new(Some(rx)), + changed: Notify::new(), + }) + } + + pub(crate) fn install_ready(&self, version: u8, ready: Ready, dual_port: bool) { + let mut state = self.state.lock().unwrap(); + if ready.workload_transport_barrier_version == Some(WORKLOAD_TRANSPORT_BARRIER_VERSION) + && state.ready.is_none() + { + state.credit = WorkloadTransportCredit { + control_bytes: WORKLOAD_TRANSPORT_CONTROL_BYTES, + control_frames: WORKLOAD_TRANSPORT_CONTROL_FRAMES, + bulk_bytes: WORKLOAD_TRANSPORT_BULK_BYTES, + bulk_frames: WORKLOAD_TRANSPORT_BULK_FRAMES, + }; + } + state.ready = Some((version, ready)); + state.dual_port = dual_port; + state.bulk_parked = !dual_port; + } + + pub(crate) fn start(&self) -> mpsc::Receiver { + self.state.lock().unwrap().active = true; + self.rx + .lock() + .unwrap() + .take() + .expect("one lifecycle writer") + } + + pub(crate) fn register_ordinary_writer(&self, writer: super::relay::ControlWriter) { + self.state.lock().unwrap().ordinary_writer = Some(writer); + } + + /// Bootstrap may write directly before Ready. Every later ordinary write joins the same FIFO. + pub(crate) fn ordinary_writer(&self) -> Result, String> { + let state = self.state.lock().unwrap(); + if state.closed { + return Err("workload transport closed".into()); + } + if state.ready.is_none() { + return Ok(None); + } + if !state.active { + return Err("ready workload transport writer is not running".into()); + } + state + .ordinary_writer + .clone() + .map(Some) + .ok_or_else(|| "ready workload transport writer is unavailable".into()) + } + + pub(crate) fn ready(&self) -> Result<(u8, Ready), String> { + let state = self.state.lock().unwrap(); + if !state.active || state.closed { + return Err("workload control transport is not running".into()); + } + let (version, ready) = state.ready.clone().ok_or("guest readiness unavailable")?; + if ready.workload_transport_barrier_version != Some(WORKLOAD_TRANSPORT_BARRIER_VERSION) { + return Err(format!( + "guest lacks workload transport barrier version {WORKLOAD_TRANSPORT_BARRIER_VERSION}" + )); + } + Ok((version, ready)) + } + + pub(crate) fn gate(self: &Arc) -> InputGate { + let mut state = self.state.lock().unwrap(); + if state.gates == 0 { + state.primary_parked = false; + state.bulk_parked = !state.dual_port; + } + state.gates += 1; + drop(state); + self.changed.notify_waiters(); + InputGate { + control: Arc::clone(self), + released: AtomicBool::new(false), + } + } + + pub(crate) fn fence(&self) { + self.state.lock().unwrap().fenced = true; + self.changed.notify_waiters(); + } + + pub(crate) fn gated(&self) -> bool { + let state = self.state.lock().unwrap(); + state.gates != 0 || state.fenced + } + + pub(crate) fn park(&self, bulk: bool) { + let mut state = self.state.lock().unwrap(); + let mut changed = false; + if state.gates != 0 || state.fenced { + if bulk { + changed = !state.bulk_parked; + state.bulk_parked = true; + } else { + changed = !state.primary_parked; + state.primary_parked = true; + } + } + drop(state); + if changed { + self.changed.notify_waiters(); + } + } + + pub(crate) async fn parked_position(&self) -> Result { + loop { + let changed = self.changed.notified(); + tokio::pin!(changed); + changed.as_mut().enable(); + { + let state = self.state.lock().unwrap(); + if state.closed { + return Err("workload transport closed".into()); + } + if state.primary_parked && state.bulk_parked { + return Ok(state.position); + } + } + changed.await; + } + } + + /// Reserve the whole record before its first fragment. The writer must finish it before parking. + pub(crate) fn admit(&self, bulk: bool, bytes: usize) -> Result { + let mut state = self.state.lock().unwrap(); + if state.closed { + return Err("workload transport closed".into()); + } + if state.gates != 0 || state.fenced { + return Ok(false); + } + match state + .ready + .as_ref() + .and_then(|(_, ready)| ready.workload_transport_barrier_version) + { + None => return Ok(true), + Some(WORKLOAD_TRANSPORT_BARRIER_VERSION) => {} + Some(version) => { + return Err(format!( + "unsupported workload transport barrier version {version}" + )); + } + } + let (sent_bytes, sent_frames, byte_limit, frame_limit) = if bulk { + ( + state.position.bulk_bytes, + state.position.bulk_frames, + state.credit.bulk_bytes, + state.credit.bulk_frames, + ) + } else { + ( + state.position.control_bytes, + state.position.control_frames, + state.credit.control_bytes, + state.credit.control_frames, + ) + }; + let next_bytes = sent_bytes + .checked_add(bytes as u64) + .ok_or("transport byte counter overflow")?; + let next_frames = sent_frames + .checked_add(1) + .ok_or("transport frame counter overflow")?; + if next_bytes > byte_limit || next_frames > frame_limit { + return Ok(false); + } + if bulk { + state.position.bulk_bytes = next_bytes; + state.position.bulk_frames = next_frames; + } else { + state.position.control_bytes = next_bytes; + state.position.control_frames = next_frames; + } + Ok(true) + } + + pub(crate) fn update_credit(&self, credit: WorkloadTransportCredit) -> Result<(), String> { + let mut state = self.state.lock().unwrap(); + let old = &state.credit; + if credit.control_bytes < old.control_bytes + || credit.control_frames < old.control_frames + || credit.bulk_bytes < old.bulk_bytes + || credit.bulk_frames < old.bulk_frames + { + return Err("workload transport credit regressed".into()); + } + state.credit = credit; + drop(state); + self.changed.notify_waiters(); + Ok(()) + } + + pub(crate) fn restore( + &self, + position: WorkloadTransportPosition, + credit: WorkloadTransportCredit, + guest_bulk_bytes: u64, + ) -> Result<(), String> { + if position.control_bytes > credit.control_bytes + || position.control_frames > credit.control_frames + || position.bulk_bytes > credit.bulk_bytes + || position.bulk_frames > credit.bulk_frames + { + return Err("captured workload input exceeds its credit limits".into()); + } + let mut state = self.state.lock().unwrap(); + if state.active { + return Err("cannot reseed an active workload transport".into()); + } + state.position = position; + state.credit = credit; + state.guest_bulk_bytes = guest_bulk_bytes; + Ok(()) + } + + pub(crate) fn observed_bulk(&self, wire_bytes: usize, tail: usize) -> Result<(), String> { + let mut state = self.state.lock().unwrap(); + state.guest_bulk_bytes = state + .guest_bulk_bytes + .checked_add(wire_bytes as u64) + .ok_or("guest bulk counter overflow")?; + state.bulk_tail = tail; + let needs_cut = state.gates != 0; + drop(state); + if needs_cut { + self.changed.notify_waiters(); + } + Ok(()) + } + + pub(crate) async fn wait_bulk_cut(&self, target: u64) -> Result<(), String> { + loop { + let changed = self.changed.notified(); + tokio::pin!(changed); + changed.as_mut().enable(); + { + let state = self.state.lock().unwrap(); + if state.closed { + return Err("workload transport closed".into()); + } + if state.guest_bulk_bytes > target { + return Err("guest bulk crossed frozen transport cut".into()); + } + if state.guest_bulk_bytes == target { + if state.bulk_tail != 0 { + return Err("guest bulk cut has an incomplete frame tail".into()); + } + return Ok(()); + } + } + changed.await; + } + } + + pub(crate) async fn request( + &self, + mut message: Message, + attempt: &str, + ) -> Result { + if !matches!( + message.t, + MessageType::WorkloadFreeze | MessageType::WorkloadThaw + ) { + return Err("non-lifecycle request on private workload channel".into()); + } + message.id = WORKLOAD_CONTROL_ID; + let mut data = Vec::new(); + codec::encode_to_buf(&message, &mut data).map_err(|error| error.to_string())?; + let (tx, rx) = oneshot::channel(); + { + let mut state = self.state.lock().unwrap(); + if !state.active || state.closed { + return Err("workload transport closed".into()); + } + // Do not reuse an abandoned operation until its actual reply has drained. Otherwise a + // late acknowledgement could incorrectly complete a later attempt with the same ID. + if state.pending.len() == PRIVATE_QUEUE_CAPACITY + || state + .pending + .iter() + .any(|pending| pending.request == message.t && pending.attempt == attempt) + { + return Err("previous workload request has not drained".into()); + } + self.tx + .try_send(LifecycleWrite(Bytes::from(data))) + .map_err(|error| error.to_string())?; + state.pending.push(PendingReply { + request: message.t, + attempt: attempt.into(), + reply: tx, + }); + } + rx.await + .map_err(|_| "workload reply channel closed".to_string())? + } + + /// Called before SDK routing. An SDK client cannot allocate this reserved correlation ID. + pub(crate) fn requires_frozen_boundary(&self, message: &Message) -> Result { + if message.t != MessageType::WorkloadFrozen { + return Ok(false); + } + let frozen: WorkloadFrozen = message.payload().map_err(|error| error.to_string())?; + Ok(self.state.lock().unwrap().pending.iter().any(|pending| { + pending.request == MessageType::WorkloadFreeze + && pending.attempt == frozen.attempt_id + && !pending.reply.is_closed() + })) + } + + /// Consume only trusted reserved replies. A canceled operation still drains its acknowledgement. + pub(crate) fn reply(&self, message: Message) -> Result<(), String> { + if message.id != WORKLOAD_CONTROL_ID || message.flags != message.t.flags() { + return Err("invalid private workload reply envelope".into()); + } + if message.t == MessageType::WorkloadTransportCredit { + return self.update_credit(message.payload().map_err(|error| error.to_string())?); + } + let (request, attempt) = match message.t { + MessageType::WorkloadFrozen => ( + MessageType::WorkloadFreeze, + message + .payload::() + .map_err(|error| error.to_string())? + .attempt_id, + ), + MessageType::WorkloadThawed => ( + MessageType::WorkloadThaw, + message + .payload::() + .map_err(|error| error.to_string())? + .attempt_id, + ), + MessageType::CoreError => { + let error = message + .payload::() + .map_err(|error| error.to_string())?; + let Some(request) = error + .offending_type + .as_deref() + .and_then(MessageType::from_wire_str) + else { + return Err("private workload error omitted its operation".into()); + }; + let Some(failure) = error.workload_failure else { + return Err("private workload error omitted its attempt".into()); + }; + (request, failure.attempt_id) + } + _ => return Err("unexpected reserved workload reply".into()), + }; + let mut state = self.state.lock().unwrap(); + if let Some(index) = state + .pending + .iter() + .position(|pending| pending.request == request && pending.attempt == attempt) + { + let pending = state.pending.remove(index); + let _ = pending.reply.send(Ok(message)); + } + Ok(()) + } + + pub(crate) fn close(&self) { + let mut state = self.state.lock().unwrap(); + state.closed = true; + state.ordinary_writer = None; + for pending in state.pending.drain(..) { + let _ = pending.reply.send(Err("workload transport closed".into())); + } + drop(state); + self.changed.notify_waiters(); + } +} + +impl InputGate { + /// Release only after confirmed thaw, or before any lifecycle request was admitted. + pub(crate) fn release(&self) { + if !self.released.swap(true, Ordering::AcqRel) { + self.control.state.lock().unwrap().gates -= 1; + self.control.changed.notify_waiters(); + } + } +} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +impl Drop for InputGate { + fn drop(&mut self) { + if !self.released.load(Ordering::Acquire) { + // A dropped capture token without a confirmed thaw is an ambiguous guest state. + // Never let cancellation turn that into permission to flush queued SDK input. + self.control.fence(); + self.release(); + } + } +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use microsandbox_protocol::core::{WorkloadFreeze, WorkloadThaw, WorkloadThawMode}; + + fn transport() -> Arc { + let control = WorkloadControl::new(); + control.install_ready( + 9, + Ready { + workload_transport_barrier_version: Some(WORKLOAD_TRANSPORT_BARRIER_VERSION), + ..Ready::default() + }, + false, + ); + control + } + + #[test] + fn input_gate_releases_explicitly_but_drop_fences() { + let control = transport(); + let gate = control.gate(); + assert!(!control.admit(false, 1).unwrap()); + gate.release(); + gate.release(); + drop(gate); + assert!(control.admit(false, 1).unwrap()); + drop(control.gate()); + assert!(!control.admit(false, 1).unwrap()); + } + + #[test] + fn unsupported_private_contract_fails_instead_of_disabling_admission() { + let control = WorkloadControl::new(); + control.install_ready(8, Ready::default(), false); + assert!(control.admit(false, usize::MAX).unwrap()); + control.install_ready( + 9, + Ready { + workload_transport_barrier_version: Some(WORKLOAD_TRANSPORT_BARRIER_VERSION - 1), + ..Default::default() + }, + false, + ); + assert!(control.admit(false, 1).unwrap_err().contains("unsupported")); + assert!(control.admit(true, 1).unwrap_err().contains("unsupported")); + } + + #[tokio::test] + async fn stable_park_does_not_wake_its_own_waiter() { + let control = transport(); + let gate = control.gate(); + control.park(false); + let changed = control.changed.notified(); + tokio::pin!(changed); + changed.as_mut().enable(); + control.park(false); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(10), changed) + .await + .is_err() + ); + gate.release(); + } + + #[test] + fn aggregate_credit_bounds_bytes_and_empty_frame_count() { + let control = transport(); + assert!( + control + .admit(false, WORKLOAD_TRANSPORT_CONTROL_BYTES as usize) + .unwrap() + ); + assert!(!control.admit(false, 1).unwrap()); + let control = transport(); + for _ in 0..WORKLOAD_TRANSPORT_CONTROL_FRAMES { + assert!(control.admit(false, 0).unwrap()); + } + assert!(!control.admit(false, 0).unwrap()); + } + + #[tokio::test] + async fn guest_bulk_cut_requires_complete_decoder_boundary() { + let control = transport(); + control.observed_bulk(32, 1).unwrap(); + assert!( + control + .wait_bulk_cut(32) + .await + .unwrap_err() + .contains("incomplete") + ); + control.observed_bulk(0, 0).unwrap(); + control.wait_bulk_cut(32).await.unwrap(); + assert!( + control + .wait_bulk_cut(31) + .await + .unwrap_err() + .contains("crossed") + ); + } + + #[tokio::test] + async fn canceled_reply_drains_before_same_attempt_can_be_reused() { + let control = transport(); + let mut writes = control.start(); + let freeze = Message::with_payload( + MessageType::WorkloadFreeze, + 0, + &WorkloadFreeze { + attempt_id: "first".into(), + host_input: WorkloadTransportPosition::default(), + }, + ) + .unwrap(); + let task_control = Arc::clone(&control); + let task_request = freeze.clone(); + let task = tokio::spawn(async move { task_control.request(task_request, "first").await }); + writes.recv().await.unwrap(); + task.abort(); + let _ = task.await; + assert!( + control + .request(freeze.clone(), "first") + .await + .unwrap_err() + .contains("not drained") + ); + let stale = Message::with_payload( + MessageType::WorkloadFrozen, + WORKLOAD_CONTROL_ID, + &WorkloadFrozen { + attempt_id: "first".into(), + guest_bulk_bytes_target: 0, + input_credit: WorkloadTransportCredit::default(), + }, + ) + .unwrap(); + control.reply(stale).unwrap(); + let task_control = Arc::clone(&control); + let thaw = Message::with_payload( + MessageType::WorkloadThaw, + 0, + &WorkloadThaw { + attempt_id: "second".into(), + mode: WorkloadThawMode::Continue, + }, + ) + .unwrap(); + let task = tokio::spawn(async move { task_control.request(thaw, "second").await }); + writes.recv().await.unwrap(); + control + .reply( + Message::with_payload( + MessageType::WorkloadThawed, + WORKLOAD_CONTROL_ID, + &WorkloadThawed { + attempt_id: "first".into(), + }, + ) + .unwrap(), + ) + .unwrap(); + assert!(!task.is_finished()); + control.close(); + assert!(task.await.unwrap().unwrap_err().contains("closed")); + } + + #[test] + fn restored_position_retains_outstanding_debt_and_credit_is_idempotent() { + let control = transport(); + let position = WorkloadTransportPosition { + control_bytes: 100, + control_frames: 2, + ..Default::default() + }; + let credit = WorkloadTransportCredit { + control_bytes: 110, + control_frames: 3, + ..Default::default() + }; + control.restore(position, credit, 0).unwrap(); + assert!(control.admit(false, 10).unwrap()); + control.update_credit(credit).unwrap(); + assert!(!control.admit(false, 1).unwrap()); + assert!( + control + .update_credit(WorkloadTransportCredit::default()) + .is_err() + ); + } +} diff --git a/crates/utils/lib/process_lock.rs b/crates/utils/lib/process_lock.rs index 04a43d07f..fd6a6cc31 100644 --- a/crates/utils/lib/process_lock.rs +++ b/crates/utils/lib/process_lock.rs @@ -43,6 +43,38 @@ pub fn lock_exclusive(file: &File) -> io::Result<()> { lock_exclusive_inner(file, false).map(|_| ()) } +/// Pins immutable data against cooperative exclusive eviction until the file closes. +pub fn lock_shared(file: &File) -> io::Result<()> { + #[cfg(unix)] + loop { + if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_SH) } == 0 { + return Ok(()); + } + let error = io::Error::last_os_error(); + if error.kind() != io::ErrorKind::Interrupted { + return Err(error); + } + } + #[cfg(windows)] + { + let mut overlapped: OVERLAPPED = unsafe { std::mem::zeroed() }; + let result = unsafe { + LockFileEx( + file.as_raw_handle() as HANDLE, + 0, + 0, + u32::MAX, + u32::MAX, + &mut overlapped, + ) + }; + if result == 0 { + return Err(io::Error::last_os_error()); + } + Ok(()) + } +} + /// Attempts to acquire an exclusive process-held lock without blocking. /// /// Returns `Ok(false)` only when another process currently owns the lock. diff --git a/docs/sandboxes/lifecycle.mdx b/docs/sandboxes/lifecycle.mdx index 134cd66ad..7f4bb32d4 100644 --- a/docs/sandboxes/lifecycle.mdx +++ b/docs/sandboxes/lifecycle.mdx @@ -52,6 +52,45 @@ msb create python --name worker An attached local SDK handle normally stops its sandbox when the client process exits. See [Keep a sandbox running](#keep-a-sandbox-running) when the sandbox should outlive that process. +## Pause and resume + +Pause keeps the local VM and its RAM allocated without creating a snapshot. Resume continues the same processes and corrects guest wall clock before releasing prepared workloads. New guest commands fail while paused; host inspection and stop remain available. Network peers may time out during a long pause. + + +```typescript TypeScript +const worker = await Sandbox.get("worker"); +await worker.pause(); +await worker.resume(); +``` + +```rust Rust +let worker = Sandbox::get("worker").await?; +worker.pause().await?; +worker.resume().await?; +``` + +```python Python +worker = await Sandbox.get("worker") +await worker.pause() +await worker.resume() +``` + +```go Go +worker, err := m.GetSandbox(ctx, "worker") +if err != nil { return err } +if err := worker.Pause(ctx); err != nil { return err } +if err := worker.Resume(ctx); err != nil { return err } +``` + +```bash CLI +msb pause worker +msb snapshot create paused-state --from-sandbox worker --full +msb resume worker +``` + + +Pausing an already-paused VM and resuming an already-running VM are no-ops. Full snapshots taken while paused leave it paused. A stopped VM has no resident execution state: use `start` to boot it again. Pause/resume requires a matching runtime and guest kernel; cloud sandboxes are not supported. + ## Stop and start again Stopping gracefully terminates guest processes and shuts down the VM. The sandbox moves to `Stopped`, but its configuration and filesystem remain available for the next start. @@ -536,11 +575,12 @@ Most applications only need to distinguish between `Running` and `Stopped`. The | **Created** | Configuration has been saved, but the sandbox has not started yet. | | **Starting** | The VM and guest agent are starting. Commands are not ready yet. | | **Running** | The sandbox is ready for `exec`, `shell`, and filesystem operations. | +| **Paused** | The local VM and its RAM remain resident. Resume continues the same processes. | | **Draining** | Existing commands may finish, but new commands are rejected. The sandbox stops when the drain completes. | | **Stopped** | The VM is off. Configuration and sandbox state are preserved for a later start. | | **Crashed** | The VM exited unexpectedly and can be started again. | -Some backends can also report `Paused`. Resume is not currently exposed through the SDKs, so start and connect operations do not treat a paused sandbox as stopped. +A paused sandbox is not stopped: use `resume`, not `start`, to continue it. Pause and resume are available through the local CLI and SDKs. ## Names, handles, and concurrent callers diff --git a/docs/sandboxes/snapshots.mdx b/docs/sandboxes/snapshots.mdx index c4e1b67b3..0c6baa689 100644 --- a/docs/sandboxes/snapshots.mdx +++ b/docs/sandboxes/snapshots.mdx @@ -6,21 +6,54 @@ icon: "code-branch" Local-only -A snapshot is a portable artifact that can hold either a sandbox's writable disk state or a full checkpoint of a running sandbox. Managed and flat OCI roots are supported; the descriptor preserves the root layout so a flat `rootfs.raw` is never mistaken for a managed OverlayFS upper. Move it with `scp`, archive it as `.tar.zst`, or create a child sandbox from it. +A snapshot is a portable artifact that can hold either a sandbox's writable disk state or a full checkpoint of a running sandbox. Managed and flat OCI roots are supported; the descriptor preserves the root layout so a flat `rootfs.raw` is never mistaken for a managed OverlayFS upper. Move it with `scp`, archive it as `.msb`, or create a child sandbox from it. -Disk snapshots capture stopped or crashed sandboxes. Full snapshots use `--full` and capture a running sandbox without requiring the user to pause it first. +Disk snapshots work with running, paused, stopped, or crashed sandboxes. A running disk capture briefly pauses the VM, seals its disk, and resumes it without copying RAM. Full snapshots use `--full` to include memory and execution state. A user-paused source stays paused in either mode. ## What gets captured +`.msb` is the recommended snapshot archive extension; compression defaults to tar + zstd. Archives are recognized by their contents, so existing `.tar.zst`, `.tar`, and other filenames still work. Explicit output filenames are kept as given. Disk-only, full, and incremental exports use the same extension; the archive records what it contains. + | Mode | Source | Captured state | Restore behavior | | ---- | ------ | -------------- | ---------------- | -| Disk (default) | Stopped or crashed sandbox | Writable disk closure and pinned image | Cold-boots a fresh VM | -| Full | Running sandbox | Disk, memory, vCPU, device, and admitted resource state | Eagerly resumes the captured execution in a child VM | +| Disk (default) | Running, paused, stopped, or crashed sandbox | Writable disk closure and pinned image | Cold-boots a fresh VM | +| Full | Running or user-paused sandbox | Disk, memory, vCPU, device, and admitted resource state | Resumes the captured execution in a child VM | Both modes produce the same schema-1 `snapshot.json` descriptor. Its closed `state.kind` is either `file` or `checkpoint`, and `root_disk.layout` records `managed`, `flat`, or `tmpfs`. Checkpoint payloads live in a content-addressed closure rather than a standalone disk file. A tmpfs root has no stopped disk snapshot because its writable state exists only in memory, but it is included in a running full snapshot; such a snapshot must be resumed and cannot use `--disk-only`. +## Share restored memory with CoW + +On supported Linux, macOS, and Windows hosts, add `--forked` when restoring a full snapshot to share clean memory pages between children. Writes stay private to each child. Without the flag, restore eagerly copies memory. The source needs no special creation option. + +```bash +msb create alpine --name baseline --root-disk flat:1G +msb snapshot create ready --from-sandbox baseline --full +msb create --name worker-a --from-snapshot baseline:ready --forked +msb create --name worker-b --from-snapshot baseline:ready --forked +``` + +The SDK creation options are Rust `.forked()`, Python `forked=True`, TypeScript `.forked()`, and Go `WithForked()`. The option requires a full snapshot and cannot be combined with `--disk-only` or an image cold boot. + +Snapshots are still manual. CoW uses a protected local memory cache; the first uncached restore must build it, while later children reuse it. Captures from forked children can prepare later restore backing with filesystem reflinks when available. Removing the input archive does not invalidate a running child's backing. Explicit NUMA placement with CoW is not supported yet; requests fail instead of silently switching modes. + +If a restore fails after creating a child record, remove that child and create a new one from the snapshot. An incomplete restore cannot be started as a fresh VM or modified; the original snapshot remains reusable. + +## Branch a running sandbox + +Use `branch` for an independent local child without first saving a full snapshot: + +```bash +msb branch baseline --name worker +``` + +The child resumes the captured processes, memory, and disk state. Its writes do not affect the source. CoW memory is built in—no `--forked` flag is needed. A running source resumes after capture; a user-paused source stays paused. Branches work on supported Linux, macOS, and Windows hosts, require a new child name, and cannot inherit published host ports. + +The SDK methods are Rust `source.branch("worker").await?`, Python `await source.branch("worker")`, TypeScript `await source.branch("worker")`, and Go `source.Branch(ctx, "worker")`. They also work on sandbox handles returned by `get`. + +Local branching still writes a consistent memory backing file, but skips portable RAM packaging and snapshot registration. It is not a saved recovery point. Use `snapshot create --full` when you need a durable, exportable snapshot. + ## Quick start You'll usually reach for the CLI first: @@ -35,17 +68,67 @@ msb stop baseline msb snapshot create after-pip-install --from-sandbox baseline # 3. Boot a fresh sandbox from the snapshot -msb run --name worker --from-snapshot after-pip-install \ +msb run --name worker --from-snapshot baseline:after-pip-install \ -- python -c "import requests; print(requests.__version__)" ``` -By default, the snapshot lives at `~/.microsandbox/snapshots/after-pip-install/`. That whole directory is the snapshot. +The snapshot belongs to the source's group, `baseline`, and lives at `~/.microsandbox/snapshots/baseline/snap_/`. Its friendly name is `baseline:after-pip-install`. ## Snapshot a sandbox -Snapshot under a bare name, resolved to `~/.microsandbox/snapshots//` by default. The name is a local alias; the descriptor carries a stable `snap_...` identity. Pass a destination directory to create the artifact on a different volume (`DIR/`). Either way the directory is the whole artifact; move it with save/load (or plain `mv`): +Each capture is an immutable member of a snapshot group. The group defaults to the source sandbox's name; use `--group` to choose another. Member names are local aliases, while the descriptor carries a stable `snap_...` identity. Omitting the member name generates one. A destination directory selects another group-store root (`DIR//`). + +## Groups and head selection + +Use a bare group to restore its selected head, or `group:member` for an exact checkpoint: + +```bash +msb snapshot create cp01 --from-sandbox baseline --group work +msb snapshot create cp02 --from-sandbox baseline --group work +msb create --name latest --from-snapshot work +msb create --name earlier --from-snapshot work:cp01 + +msb snapshot head work # Show the selected snapshot ID +msb snapshot head work:cp01 # Explicitly select an earlier checkpoint +``` + +The first snapshot initializes the head. Later captures or imports advance it only when known ancestry proves they descend from the current head. Importing an older checkpoint, a sibling, or one with missing history keeps the head unchanged. Both members remain available. Concurrent siblings use the first successful head update; the other capture still succeeds. There is no special main branch and timestamps do not decide the winner. + +```text +work:cp01 ---- work:cp02 ---- work:cp03 <- head + \ + +------ work:experiment +``` + +`msb snapshot head work:experiment` selects the other branch. Import with `--set-head` to explicitly select the imported archive's head. Missing historical checkpoints are allowed if the snapshot's disk and RAM dependencies are complete. Imports resolve payload dependencies from the supplied archives and an explicitly named destination group; `--base` supplies an external fallback. + +```bash +msb snapshot load checkpoint.msb --group work +msb snapshot load experiment.msb --group work --set-head +``` + +Loading without `--group` creates a fresh generated group. The same archive can be imported into different groups without overwriting either copy. Within a group, an identical ID/descriptor is reusable; conflicting IDs or names fail. A current head cannot be removed while other members remain—select another first. Existing flat snapshot directories can still be opened by explicit path. + +An interrupted call may already have published its snapshot. Inspect the group before retrying; member publication never overwrites an existing snapshot. + +## Load a set of archives + +Import a baseline and its dependent archives together. The shell expands the wildcard; archive order does not matter: + +```bash +msb snapshot load checkpoints/*.msb --group received +msb snapshot load changes.msb base.msb --group received --dest /mnt/snapshots +``` + +`load` accepts one or more archive paths. Use `--dest DIR` for a different group-store root; a trailing positional path is another archive, not a destination. A batch without `--group` creates one generated group for all its archives. Existing single-archive calls still print the digest followed by the installed path. Multi-archive calls print each result and report the batch's head decision once. + +Dependencies are resolved from the batch and exact matching snapshots already installed in the explicitly named destination group. For example, after importing `base.msb` into `received`, `msb snapshot load changes.msb --group received` can reuse that base automatically. Use `--base SOURCE` when a dependency must come from another installed snapshot or standalone archive. Imports do not search unrelated groups or nearby archive files. These operations accept the existing archive format; no format conversion or schema bump is required. + +All requested members are validated before publication. A batch with one tip follows the normal head-update rules. If a batch contains divergent tips, the existing group head stays selected; a new group has no head until you choose a member with `msb snapshot head GROUP:MEMBER`. `--set-head` requires a unique batch tip and rejects an ambiguous batch before publishing members. The final printed result is not necessarily the selected head. + +## Capture from the SDK ```typescript TypeScript @@ -53,7 +136,7 @@ import { Sandbox } from "microsandbox"; const h = await Sandbox.get("baseline"); -// Resolves under ~/.microsandbox/snapshots// +// Installs in baseline's group; snap.path is the exact artifact directory. const snap = await h.snapshot("after-pip-install"); console.log(snap.digest); // sha256:... @@ -64,7 +147,7 @@ use microsandbox::Sandbox; let h = Sandbox::get("baseline").await?; -// Resolves under ~/.microsandbox/snapshots// +// Installs in baseline's group; snap.path() is the exact artifact directory. let snap = h.snapshot("after-pip-install").await?; println!("{}", snap.digest()); // sha256:... @@ -75,7 +158,7 @@ from microsandbox import Sandbox h = await Sandbox.get("baseline") -# Resolves under ~/.microsandbox/snapshots// +# Installs in baseline's group; snap.path is the exact artifact directory. snap = await h.snapshot("after-pip-install") print(snap.digest) # sha256:... @@ -87,7 +170,7 @@ if err != nil { return err } -// Resolves under ~/.microsandbox/snapshots// +// Installs in baseline's group; snap.Path() is the exact artifact directory. snap, err := h.Snapshot(ctx, "after-pip-install") fmt.Println(snap.Digest()) // sha256:... @@ -95,15 +178,14 @@ fmt.Println(snap.Digest()) // sha256:... ```bash CLI msb snapshot create after-pip-install --from-sandbox baseline -msb snapshot create after-pip-install --from-sandbox baseline --label stage=ready +msb snapshot create ready-with-label --from-sandbox baseline --label stage=ready -# Create the artifact on another volume: lands at /mnt/big/after-pip-install +# Use another group-store root: /mnt/big/baseline/snap_/ msb snapshot create after-pip-install --from-sandbox baseline --dest-dir /mnt/big ``` - -The default disk mode requires a stopped or crashed sandbox; running sandboxes are rejected. If the sandbox has previously produced a full snapshot, its writable state may already be a raw/qcow2 chain. Disk snapshotting preserves that complete chain, including writes made after the earlier full snapshot, and every restored child receives a fresh private writable head. +The default disk mode captures only the owned managed or flat root disk. Live captures are crash-consistent: unsaved application buffers and tmpfs contents are not included. Use `--full` when you need memory too. Disk snapshotting preserves the complete raw/qcow2 chain, and every restored child receives a fresh private writable head. Direct `--archive ./saved.msb` capture works in either mode without installing a snapshot directory. ## Capture a running sandbox @@ -160,21 +242,23 @@ Repeated full checkpoints add disk layers. You choose when to export changes and ```bash msb snapshot create checkpoint-b --from-sandbox worker --full -msb snapshot save checkpoint-b changes.tar.zst --since checkpoint-a +msb snapshot save worker:checkpoint-b changes.msb --since worker:checkpoint-a msb modify worker --compact --layers 3 --dry-run msb modify worker --compact --layers 3 ``` `--layers 3` merges the oldest three physical layers, **including the base**. It never includes the writable head, even when stopped. Omit `--layers` to merge all sealed layers; a chain with fewer than two sealed layers is unchanged. Managed and flat roots support compaction, running or stopped. Existing snapshots remain valid and retain their storage until you remove them separately. -`--since` requires an exact physical-prefix base. Alternatively, `--last-layers 2` includes the newest two sealed checkpoint layers. A full checkpoint export still includes all required memory and device state. Unlike ordinary standalone exports, these smaller archives require the omitted base when loading or restoring: +`--since` omits disk layers and, for full checkpoints, RAM objects already supplied by the base. The disk base must be an exact physical prefix. Each archive still includes its complete memory map and CPU/device state; it does not replay earlier memory images. Alternatively, `--last-layers 2` selects only disk layers and keeps all required RAM objects. Import these smaller archives with their bases in the same batch, into a named group containing their dependencies, or with an explicit external base. Direct restore still accepts an explicit base: ```bash -msb snapshot load changes.tar.zst --base checkpoint-a -msb create --name child --from-snapshot changes.tar.zst --snapshot-base checkpoint-a +msb snapshot load changes.msb --base worker:checkpoint-a --group imported +msb create --name child --from-snapshot changes.msb --snapshot-base worker:checkpoint-a ``` -The base can also be a standalone snapshot archive. Restore copies the required closure into child-owned storage, without installing an intermediate snapshot. Add `--disk-only` to cold-boot only disk state. After compaction, export a new standalone baseline before resuming incremental exports: the old physical prefix no longer matches. Do not combine compaction with unrelated `modify` options, or incremental export with `--with-parents`. +An explicit external base can also be a standalone snapshot archive. Dependent archives can supply one another within a batch; they need not be loaded in dependency order. Loads resolve disk and RAM dependencies without starting a VM; only the final sandbox creation resumes execution. Missing or incorrect dependencies fail before publication or execution. Loaded snapshots and restored children own their required files, so removing the base later does not break them. Direct restore skips installing an intermediate snapshot. Add `--disk-only` to cold-boot only disk state. After compaction, export a new standalone baseline before resuming incremental exports: the old physical prefix no longer matches. Do not combine compaction with unrelated `modify` options, or incremental export with `--with-parents`. + +Current development limitation: successive disk-only captures reassign layer IDs, so their `--since` export can reject the base even when physical layers were unchanged. Use standalone disk-only exports for now. Full-checkpoint incremental exports and imports are unaffected by this capture issue. ## Capture directly to an archive @@ -194,7 +278,7 @@ use microsandbox::Snapshot; let archive = Snapshot::builder("after-pip-install") .from_sandbox("baseline") - .create_archive("/tmp/after-pip-install.tar.zst", false) + .create_archive("/tmp/after-pip-install.msb", false) .await?; ``` @@ -203,7 +287,7 @@ from microsandbox import Snapshot archive = await Snapshot.create_archive( "after-pip-install", - "/tmp/after-pip-install.tar.zst", + "/tmp/after-pip-install.msb", from_sandbox="baseline", ) ``` @@ -214,14 +298,14 @@ archive, err := m.Snapshot.CreateArchive(ctx, m.SnapshotArchiveOptions{ Name: "after-pip-install", FromSandbox: "baseline", }, - ArchivePath: "/tmp/after-pip-install.tar.zst", + ArchivePath: "/tmp/after-pip-install.msb", }) ``` ```bash CLI msb snapshot create after-pip-install \ --from-sandbox baseline \ - --archive /tmp/after-pip-install.tar.zst + --archive /tmp/after-pip-install.msb ``` @@ -230,13 +314,13 @@ Direct capture publishes only the archive and returns its snapshot ID and path. An explicit archive path can also be used directly as the source of a new sandbox: ```bash -msb run --name worker --from-snapshot ./after-pip-install.tar.zst -- python -V +msb run --name worker --from-snapshot ./after-pip-install.msb -- python -V ``` The same archive path works with the SDK restore methods shown below. The archive is unpacked into child-owned staging, so no intermediate installed snapshot is loaded into `~/.microsandbox/snapshots`. -Direct capture records the pinned image identity, but does not bundle the cached OCI image. The target must already have that image or be able to pull it. For an offline target, create an installed snapshot and use `msb snapshot save --with-image` instead. +Direct capture records the pinned image identity, but does not bundle the cached OCI image. A flat snapshot already contains the complete disk, so restore only needs the pinned image's configuration from cache or the registry; it does not download the original disk layers again. Layered snapshots still need their base image artifacts. For an offline target, create an installed snapshot and use `msb snapshot save --with-image` instead. ## Boot from a snapshot @@ -248,7 +332,7 @@ A snapshot already pins its image, so booting from one is mutually exclusive wit import { Sandbox } from "microsandbox"; const sb = await Sandbox.builder("worker") - .fromSnapshot("after-pip-install") + .fromSnapshot("baseline:after-pip-install") .create(); ``` @@ -256,7 +340,7 @@ const sb = await Sandbox.builder("worker") use microsandbox::Sandbox; let sb = Sandbox::builder("worker") - .from_snapshot("after-pip-install") + .from_snapshot("baseline:after-pip-install") .create() .await?; ``` @@ -265,19 +349,18 @@ let sb = Sandbox::builder("worker") from microsandbox import Sandbox # `from_snapshot=` is a peer of `image=` and mutually exclusive with it -sb = await Sandbox.create("worker", from_snapshot="after-pip-install") +sb = await Sandbox.create("worker", from_snapshot="baseline:after-pip-install") ``` ```go Go sb, err := m.CreateSandbox(ctx, "worker", - m.WithFromSnapshot("after-pip-install"), + m.WithFromSnapshot("baseline:after-pip-install"), ) ``` ```bash CLI -msb run --name worker --from-snapshot after-pip-install -- python -V +msb run --name worker --from-snapshot baseline:after-pip-install -- python -V ``` - For disk state, creation cold-boots from a child-owned writable copy. For checkpoint state, creation eagerly restores the captured execution into child-owned state. @@ -289,14 +372,14 @@ To discard the captured execution and cold-boot only its filesystem state, selec ```typescript TypeScript const sb = await Sandbox.builder("worker") - .fromSnapshot("worker-checkpoint") + .fromSnapshot("worker:worker-checkpoint") .diskOnly() .create(); ``` ```rust Rust let sb = Sandbox::builder("worker") - .from_snapshot("worker-checkpoint") + .from_snapshot("worker:worker-checkpoint") .disk_only() .create() .await?; @@ -305,24 +388,24 @@ let sb = Sandbox::builder("worker") ```python Python sb = await Sandbox.create( "worker", - from_snapshot="worker-checkpoint", + from_snapshot="worker:worker-checkpoint", disk_only=True, ) ``` ```go Go sb, err := m.CreateSandbox(ctx, "worker", - m.WithFromSnapshot("worker-checkpoint"), + m.WithFromSnapshot("worker:worker-checkpoint"), m.WithSnapshotDiskOnly(), ) ``` ```bash CLI -msb run --name worker --from-snapshot worker-checkpoint --disk-only +msb run --name worker --from-snapshot worker:worker-checkpoint --disk-only ``` -Disk-only restore copies only the checkpoint's disk chain, creates a fresh writable qcow2 head, and performs an ordinary boot. It does not require the destination to support the checkpoint's memory or execution codec. The full capture freezes workloads and syncs guest filesystems before pausing the VM, so this cold-boot view is filesystem-clean. +Disk-only restore copies only the checkpoint's disk chain, creates a fresh writable qcow2 head, and performs an ordinary boot. It does not require the destination to support the checkpoint's memory or execution codec. Like a live disk snapshot, this disk-only view is crash-consistent: recent writes still buffered in guest RAM may be absent, and filesystem journal recovery may run during boot. Full restore preserves those buffers in captured RAM. Pause and full capture do not automatically sync guest filesystems; host disk flushes and durable snapshot publication are still enforced. ## List, inspect, and remove @@ -331,10 +414,10 @@ Disk-only restore copies only the checkpoint's disk chain, creates a fresh writa import { Snapshot } from "microsandbox"; const all = await Snapshot.list(); // Indexed snapshots -const h = await Snapshot.get("after-pip-install"); // By name, digest, or path +const h = await Snapshot.get("baseline:after-pip-install"); // By group/member selector or explicit path console.log(`${h.name ?? "-"} (${h.digest})`); -await Snapshot.remove("after-pip-install"); +await Snapshot.remove("baseline:after-pip-install"); await Snapshot.reindex(); // Default snapshots directory ``` @@ -342,10 +425,10 @@ await Snapshot.reindex(); // Default snapshots dir use microsandbox::Snapshot; let all = Snapshot::list().await?; // Indexed snapshots -let h = Snapshot::get("after-pip-install").await?; // By name, digest, or path +let h = Snapshot::get("baseline:after-pip-install").await?; // By group/member selector or explicit path println!("{} ({})", h.name().unwrap_or("-"), h.digest()); -Snapshot::remove("after-pip-install", false).await?; +Snapshot::remove("baseline:after-pip-install", false).await?; Snapshot::reindex("/data/snapshots").await?; ``` @@ -353,42 +436,41 @@ Snapshot::reindex("/data/snapshots").await?; from microsandbox import Snapshot all = await Snapshot.list() # Indexed snapshots -h = await Snapshot.get("after-pip-install") # By name, digest, or path +h = await Snapshot.get("baseline:after-pip-install") # By group/member selector or explicit path print(f"{h.name or '-'} ({h.digest})") -await Snapshot.remove("after-pip-install") +await Snapshot.remove("baseline:after-pip-install") await Snapshot.reindex() # Default snapshots directory ``` ```go Go all, err := m.Snapshot.List(ctx) // Indexed snapshots fmt.Printf("%d snapshots\n", len(all)) -h, err := m.Snapshot.Get(ctx, "after-pip-install") // By name, digest, or path +h, err := m.Snapshot.Get(ctx, "baseline:after-pip-install") // By group/member selector or explicit path name := "-" if h.Name() != nil { name = *h.Name() } fmt.Printf("%s (%s)\n", name, h.Digest()) -err = m.Snapshot.Remove(ctx, "after-pip-install", false) +err = m.Snapshot.Remove(ctx, "baseline:after-pip-install", false) _, err = m.Snapshot.Reindex(ctx, "/data/snapshots") ``` ```bash CLI msb snapshots # Also: msb snaps, msb snapshot ls -msb snapshot inspect after-pip-install -msb snapshot rm after-pip-install +msb snapshot inspect baseline:after-pip-install +msb snapshot rm baseline:after-pip-install # Also if it has indexed children -msb snapshot rm after-pip-install --force +msb snapshot rm baseline:after-pip-install --force # Rebuild the index from artifacts on disk msb snapshot reindex ``` - -`list` and `get` use a local index for fast lookup. If the index gets out of sync, `reindex` rebuilds it from the snapshot artifacts on disk. +`list` uses a rebuildable local index; group/head selection comes from the artifacts on disk. If the index gets out of sync, `reindex` rebuilds it. Global ID or digest lookup refuses ambiguity when several local copies exist—use a group selector or explicit path. `--force` does not bypass the current-head removal guard. ## Move snapshots between machines @@ -396,31 +478,35 @@ The snapshot directory is the whole artifact; there is no hidden daemon state. C ```bash # Copy the directory directly with scp (image must be cached or pullable on the target) -scp -r ~/.microsandbox/snapshots/after-pip-install \ - other-host:~/.microsandbox/snapshots/ +scp -r ~/.microsandbox/snapshots/baseline/snap_ other-host:/tmp/saved-snapshot +# Open that copied artifact by its explicit path; use archive load to add it to a group. -# Bundle into a .tar.zst, transport, then load -msb snapshot save after-pip-install /tmp/snap.tar.zst -scp /tmp/snap.tar.zst other-host: -ssh other-host msb snapshot load /tmp/snap.tar.zst +# Bundle into a .msb, transport, then load +msb snapshot save baseline:after-pip-install /tmp/snap.msb +scp /tmp/snap.msb other-host: +ssh other-host msb snapshot load /tmp/snap.msb # Fully offline: include the OCI image cache so the target needs no network -msb snapshot save after-pip-install /tmp/snap.tar.zst --with-image -ssh other-host msb snapshot load /tmp/snap.tar.zst +msb snapshot save baseline:after-pip-install /tmp/snap.msb --with-image +ssh other-host msb snapshot load /tmp/snap.msb ``` -Archives default to `.tar.zst`. Pass `--plain-tar` for a plain `.tar`. SDKs expose the same save and load operations as the CLI. +Archives use tar + zstd by default; `.msb` is the recommended filename extension. Pass `--plain-tar` for uncompressed tar. SDKs expose the same save and load operations as the CLI. ## Artifact identity and layout Current file snapshots use a stable opaque snapshot ID, a separate SHA-256 descriptor digest, and one ordered physical layer closure. A normal raw ext4 snapshot is still one layer, whether those bytes represent a managed upper or a complete flat root: ```text -after-pip-install/ -├── snapshot.json -├── metadata.json # present when local labels exist -└── layers/ - └── layer_....raw +snapshots/ +└── baseline/ + ├── group.json # selected head ID + └── snap_/ + ├── snapshot.json # stable ID + parent ID + disk/state references + ├── group-member.json # local name: after-pip-install + ├── metadata.json # optional labels + └── layers/ + └── layer_....raw ``` `metadata.json` contains mutable local labels and does not participate in the descriptor digest or snapshot identity. The descriptor names each layer by `DiskLayerId`. Once a full snapshot has rolled the sandbox onto qcow2, a later disk snapshot carries the complete oldest-first raw/qcow2 chain in this same artifact family; archive save/load and direct archive restore preserve every member. @@ -491,10 +577,9 @@ report, err := snap.Verify(ctx) msb snapshot create after-pip-install --from-sandbox baseline --integrity # Verify a snapshot's recorded integrity on demand -msb snapshot verify after-pip-install -msb snapshot inspect after-pip-install --verify +msb snapshot verify baseline:after-pip-install +msb snapshot inspect baseline:after-pip-install --verify ``` - `msb snapshot save` and `msb snapshot load` preserve recorded integrity but do not silently execute it. They still enforce the archive grammar, path confinement, entry sizes, descriptor identities, and ordinary archive-entry hashes. Run `msb snapshot verify` explicitly after receiving a snapshot when your workflow requires an independent payload scan. Released `msb-sparse-sha256-v1` descriptors remain readable and verifiable, but ordinary open, boot, save, load, and upgrade paths do not pay their full logical-size SHA cost. diff --git a/docs/sdk/go/snapshots.mdx b/docs/sdk/go/snapshots.mdx index 1976d27e2..6c2fbb1a0 100644 --- a/docs/sdk/go/snapshots.mdx +++ b/docs/sdk/go/snapshots.mdx @@ -6,22 +6,58 @@ keywords: ["Go SDK", "Go snapshots", "microsandbox snapshots"] Local-only -Create disk snapshots of stopped sandboxes and full checkpoints of running sandboxes. See [Snapshots](/sandboxes/snapshots) for usage and lifecycle concepts. +Create disk snapshots of running, paused, stopped, or crashed sandboxes, or full checkpoints of running and paused sandboxes. See [Snapshots](/sandboxes/snapshots) for usage and lifecycle concepts. + +## Snapshot groups + +Installed snapshots belong to a group. A bare group selects its head; `group:member` selects a member by name or stable snapshot ID. Artifact paths remain valid selectors. Creation defaults to the source sandbox's group and generates a member name when empty; imports without a group create a new generated group. `DestDir` and `SnapshotLoadOptions.Dest` select the parent directory containing groups. + +```go +snap, err := m.Snapshot.Create(ctx, m.SnapshotCreateOptions{ + Name: "baseline", FromSandbox: "box", Group: "work", +}) +loaded, err := m.Snapshot.LoadWithOptions(ctx, "changes.msb", m.SnapshotLoadOptions{ + Base: "work:baseline", Group: "work", +}) +head, err := m.Snapshot.GroupHead(ctx, "work") +selected, err := m.Snapshot.GroupHead(ctx, "work:baseline") +loaded, err = m.Snapshot.LoadWithOptions(ctx, "other.msb", m.SnapshotLoadOptions{ + Group: "work", SetHead: true, +}) +``` + +`SnapshotLoadOptions` contains `Dest`, `Base`, `Group`, and `SetHead`. `Load` and `LoadWithBase` use generated groups. `GroupHead` returns `SnapshotHeadUpdate` with `Group`, `Previous`, `Head`, `Reason`, and `Changed`; `Previous` and `Head` are stable snapshot IDs. Automatic head updates initialize an empty group or advance to a proven descendant; divergent imports and imports with unknown ancestry retain the current head. Reasons are `initialized`, `fast_forwarded`, `selected`, `unchanged`, `diverged`, `unknown_ancestry`, and `ambiguous_candidates`. Direct archive capture creates no group and rejects a nonempty `Group`. + +## Load multiple archives + +```go +handles, err := m.Snapshot.LoadMany(ctx, + []string{"changes.msb", "base.msb"}, + m.SnapshotLoadOptions{Group: "received"}, +) +if err != nil { return err } +``` + +`LoadMany(ctx, archives []string, opts SnapshotLoadOptions)` returns `([]*SnapshotHandle, error)`, with one handle per supplied archive in input order. Dependencies are resolved regardless of argument order from the batch and exact matching snapshots already installed in an explicitly named destination group. `Base` supplies an external snapshot or standalone archive when needed. Single-file `LoadWithOptions` also reuses dependencies from its named destination group. An empty `Group` creates one generated group for the batch. + +All batch members are validated before publication. `SetHead: true` requires a unique tip. With divergent tips and the default `false`, an existing head is retained (`ambiguous_candidates`); a new group has no head and imported handles return `nil` from `HeadUpdate()`. Select a member explicitly to give that group a head. The archive format is unchanged. The CLI equivalent is `msb snapshot load checkpoints/*.msb --group received`, with `--dest DIR` for another group-store root. ## Disk maintenance and incremental export +Exporting since a base omits its reusable disk layers and RAM objects. Full checkpoints keep the complete memory map and CPU/device state. The last-layers option only selects disk layers and leaves RAM payloads complete. Imports resolve dependencies from the batch or an explicitly named destination group. Use an external base for missing dependencies; direct restore still accepts an explicit base. + ```go worker, err := m.GetSandbox(ctx, "worker") if err != nil { return err } layers := uint32(3) result, err := worker.Compact(ctx, m.DiskCompactionOptions{Layers: &layers}) if err != nil { return err } -err = m.Snapshot.Save(ctx, "checkpoint-b", "changes.tar.zst", m.SnapshotSaveOptions{Since: "checkpoint-a"}) +err = m.Snapshot.Save(ctx, "worker:checkpoint-b", "changes.msb", m.SnapshotSaveOptions{Since: "worker:checkpoint-a"}) if err != nil { return err } -snapshot, err := m.Snapshot.LoadWithBase(ctx, "changes.tar.zst", "", "checkpoint-a") +snapshot, err := m.Snapshot.LoadWithBase(ctx, "changes.msb", "", "worker:checkpoint-a") ``` -The count includes the oldest base but excludes the writable head. A nil `Layers` selects all sealed layers; `DryRun: true` only resolves the plan. `LastLayers` is an alternative to `Since` for export. For direct restore combine `WithFromSnapshot("changes.tar.zst")` with `WithSnapshotBase("checkpoint-a")`. See [disk-chain maintenance](/sandboxes/snapshots#export-changes-and-compact-a-disk-chain) for dependency and retention rules. +The count includes the oldest base but excludes the writable head. A nil `Layers` selects all sealed layers; `DryRun: true` only resolves the plan. `LastLayers` is an alternative to `Since` for export. For direct restore combine `WithFromSnapshot("changes.msb")` with `WithSnapshotBase("worker:checkpoint-a")`. See [disk-chain maintenance](/sandboxes/snapshots#export-changes-and-compact-a-disk-chain) for dependency and retention rules. ## Snapshot @@ -33,7 +69,7 @@ Package-level helpers for snapshot artifacts. Access them through the exported ` func (snapshotFactory) Create(ctx context.Context, opts SnapshotCreateOptions) (*SnapshotArtifact, error) ``` -Create a disk snapshot from a stopped or crashed sandbox, or set `Full` to checkpoint a running sandbox. [`SnapshotCreateOptions.Name`](#snapshotcreateoptionsstruct) and [`SnapshotCreateOptions.FromSandbox`](#snapshotcreateoptionsstruct) are both required. +Create a disk snapshot, or set `Full` to include memory and execution state. [`SnapshotCreateOptions.FromSandbox`](#snapshotcreateoptionsstruct) is required. An empty `Name` is generated; an empty `Group` uses the source sandbox's name.

Parameters

@@ -86,7 +122,7 @@ archive, err := m.Snapshot.CreateArchive(ctx, m.SnapshotArchiveOptions{ Name: "after-pip-install", FromSandbox: "baseline", }, - ArchivePath: "/tmp/after-pip-install.tar.zst", + ArchivePath: "/tmp/after-pip-install.msb", }) ``` @@ -101,12 +137,12 @@ func (snapshotFactory) Open(ctx context.Context, pathOrName string) (*SnapshotAr ```go -snap, err := m.Snapshot.Open(ctx, "after-pip-install") +snap, err := m.Snapshot.Open(ctx, "baseline:after-pip-install") ``` -Open an existing artifact by bare name or filesystem path. This validates metadata only; call [`s.Verify()`](#s-verify) for content checks. +Open an existing artifact by group head, `group:member`, or filesystem path. This validates metadata only; call [`s.Verify()`](#s-verify) for content checks.

Parameters

@@ -117,7 +153,7 @@ Open an existing artifact by bare name or filesystem path. This validates metada
pathOrNamestring
-
Bare name (resolved under the default snapshots directory) or artifact directory path.
+
Group head or group:member selector or artifact directory path.
@@ -139,12 +175,12 @@ func (snapshotFactory) Get(ctx context.Context, nameOrDigest string) (*SnapshotH ```go -h, err := m.Snapshot.Get(ctx, "after-pip-install") +h, err := m.Snapshot.Get(ctx, "baseline:after-pip-install") ``` -Look up a lightweight handle in the local index by name, digest, or path. +Look up a lightweight handle by group head, `group:member`, stable snapshot ID, descriptor digest, or artifact path. Global IDs and digests must resolve unambiguously.

Parameters

@@ -155,7 +191,7 @@ Look up a lightweight handle in the local index by name, digest, or path.
nameOrDigeststring
-
Bare name, manifest digest, or artifact path.
+
Group head, group:member, stable snapshot ID, digest, or artifact path.
@@ -235,7 +271,7 @@ func (snapshotFactory) Remove(ctx context.Context, pathOrName string, force bool ```go -err := m.Snapshot.Remove(ctx, "after-pip-install", false) +err := m.Snapshot.Remove(ctx, "baseline:after-pip-install", false) ``` @@ -251,7 +287,7 @@ Remove a snapshot artifact and its index row. Refuses to delete a snapshot with
pathOrNamestring
-
Bare name or artifact path.
+
Group head, group:member, or artifact path.
forcebool
@@ -307,7 +343,7 @@ Walk `dir` and rebuild the local index from the artifacts it finds. func (snapshotFactory) Save(ctx context.Context, nameOrPath, outPath string, opts SnapshotSaveOptions) error ``` -Bundle a snapshot into a `.tar.zst` archive at `outPath`. Set [`SnapshotSaveOptions.PlainTar`](#snapshotsaveoptionsstruct) to skip compression. +Bundle a snapshot into a `.msb` archive at `outPath`. Set [`SnapshotSaveOptions.PlainTar`](#snapshotsaveoptionsstruct) to skip compression.

Parameters

@@ -318,7 +354,7 @@ Bundle a snapshot into a `.tar.zst` archive at `outPath`. Set [`SnapshotSaveOpti
nameOrPathstring
-
Bare name or artifact path to save.
+
Group head, group:member, or artifact path to save.
outPathstring
@@ -333,7 +369,7 @@ Bundle a snapshot into a `.tar.zst` archive at `outPath`. Set [`SnapshotSaveOpti ```go -err := m.Snapshot.Save(ctx, "after-pip-install", "/tmp/snap.tar.zst", +err := m.Snapshot.Save(ctx, "baseline:after-pip-install", "/tmp/snap.msb", m.SnapshotSaveOptions{WithParents: true}, ) ``` @@ -380,14 +416,14 @@ Unpack a snapshot archive into the snapshots directory or an explicit `dest` dir ```go -h, err := m.Snapshot.Load(ctx, "/tmp/snap.tar.zst", "") +h, err := m.Snapshot.Load(ctx, "/tmp/snap.msb", "") ``` ## SandboxHandle -Snapshots are taken from a metadata handle, so stop the sandbox first and then call [`GetSandbox`](/sdk/go/sandbox#m-getsandbox). +Snapshots are taken from a metadata handle returned by [`GetSandbox`](/sdk/go/sandbox#m-getsandbox). Disk capture also supports a running or paused sandbox. ```go _ = sb.Stop(ctx) @@ -406,7 +442,7 @@ snap, err := h.Snapshot(ctx, "after-pip-install") func (h *SandboxHandle) Snapshot(ctx context.Context, name string) (*SnapshotArtifact, error) ``` -Snapshot this sandbox under a bare name in the default snapshots directory. The sandbox must be stopped or crashed. To place the artifact elsewhere, use [`Snapshot.Save`](#snapshot-save) / [`Snapshot.Load`](#snapshot-load) or move the self-contained artifact directory. +Snapshot this sandbox into its default group with the given member name. Live disk captures preserve the source's running or paused state. Use the returned artifact path or `sandbox:member` to open it later.

Parameters

@@ -417,7 +453,7 @@ Snapshot this sandbox under a bare name in the default snapshots directory. The
namestring
-
Bare name for the artifact.
+
Member name within the source sandbox's group.
@@ -637,7 +673,7 @@ err := h.Remove(ctx, false) -Remove this snapshot. Equivalent to [`Snapshot.Remove`](#snapshot-remove) on this handle's digest. +Remove this installed snapshot copy by its stored artifact path. Other groups containing the same snapshot ID or digest remain unchanged.

Parameters

@@ -666,7 +702,7 @@ Manifest digest. func (h *SnapshotHandle) Name() *string ``` -Bare-name alias, if the snapshot was indexed with one; otherwise `nil`. Returns a defensive copy. +Member name within its group, or `nil` when no alias is recorded. Returns a defensive copy. #### h.ParentDigest() @@ -748,15 +784,16 @@ Scope of what a snapshot captures: disk-only or disk plus complete VM state.

Accepted by Snapshot.Create()

-Configures [`Snapshot.Create`](#snapshot-create). `Name` and `FromSandbox` are both required. +Configures [`Snapshot.Create`](#snapshot-create). `FromSandbox` is required. An empty `Name` is generated; an empty `Group` uses the source sandbox's name. | Field | Type | Description | |-------|------|-------------| -| Name | `string` | Bare name; always the artifact directory's basename | +| Name | `string` | Member name within its group; generated when empty | +| Group | `string` | Destination snapshot group; defaults to the source sandbox's name | | FromSandbox | `string` | Name of the stopped or crashed sandbox to capture | -| DestDir | `string` | Parent directory for the artifact (`DestDir/`); empty = the default snapshots directory | +| DestDir | `string` | Parent directory containing snapshot groups; empty = the default snapshots directory | | Labels | `map[string]string` | Arbitrary user labels recorded in the manifest | -| Force | `bool` | Overwrite an existing artifact with the same name | +| Force | `bool` | Overwrite a direct archive output; rejected for installed group members | | RecordIntegrity | `bool` | Record content hashes so [`Verify`](#s-verify) can recompute them later | | Full | `bool` | Capture a running sandbox's full checkpoint instead of a stopped-sandbox disk snapshot | @@ -770,7 +807,7 @@ Configures [`Snapshot.Save`](#snapshot-save). |-------|------|-------------| | WithParents | `bool` | Include the snapshot's parent chain in the archive | | WithImage | `bool` | Include the base OCI image in the archive | -| PlainTar | `bool` | Write an uncompressed `.tar` instead of `.tar.zst` | +| PlainTar | `bool` | Write an uncompressed `.tar` instead of `.msb` | ### SnapshotVerifyReportstruct diff --git a/docs/sdk/python/snapshots.mdx b/docs/sdk/python/snapshots.mdx index c1e51eaa7..008929814 100644 --- a/docs/sdk/python/snapshots.mdx +++ b/docs/sdk/python/snapshots.mdx @@ -6,7 +6,37 @@ keywords: ["Python SDK", "Python snapshots", "microsandbox snapshots"] Local-only -Create disk snapshots of stopped sandboxes and full checkpoints of running sandboxes. See [Snapshots](/sandboxes/snapshots) for usage and lifecycle concepts. +Create disk snapshots and full checkpoints in local snapshot groups. See [Snapshots](/sandboxes/snapshots) for usage and lifecycle concepts. + +## Snapshot groups + +Installed snapshots belong to a group. A bare group selects its head; `group:member` selects a member by name or stable snapshot ID. Artifact paths remain valid selectors. Creation defaults to the source sandbox's group and generates a member name when omitted; imports without `group=` create a new generated group. `dest_dir` and `dest` select the parent directory containing groups. + +```python +snap = await Snapshot.create("baseline", from_sandbox="box", group="work") +loaded = await Snapshot.load("changes.msb", base="work:baseline", group="work") +head = await Snapshot.group_head("work") +selected = await Snapshot.group_head("work:baseline") +# Explicitly select an imported member even when it is not a descendant. +loaded = await Snapshot.load("other.msb", group="work", set_head=True) +``` + +`group_head(selector)` returns a dictionary with `group`, `previous`, `head`, `reason`, and `changed`. `previous` and `head` are stable snapshot IDs. Automatic head updates initialize an empty group or advance to a proven descendant; divergent imports and imports with unknown ancestry retain the current head. Reasons are `initialized`, `fast_forwarded`, `selected`, `unchanged`, `diverged`, `unknown_ancestry`, and `ambiguous_candidates`. Direct archive capture creates no group and rejects `group=`. + +## Load multiple archives + +```python +from pathlib import Path + +handles = await Snapshot.load_many( + list(Path("checkpoints").glob("*.msb")), + group="received", +) +``` + +`load_many(archives, *, dest=None, base=None, group=None, set_head=False)` returns one `SnapshotHandle` for each supplied archive, in input order. The batch resolves dependencies regardless of argument order, using its archives and exact matching members already installed in an explicitly named destination group. `base` provides an external snapshot or standalone archive when needed. Single-file `load` also reuses dependencies from its named destination group. Omitting `group` creates one generated group for the whole batch. + +The importer validates the complete batch before publishing members. `set_head=True` requires a unique tip. With divergent tips and the default `False`, an existing head is retained (`ambiguous_candidates`); a new group has no head and imported handles report `head_update=None`. Select a member explicitly to give that group a head. Existing archive formats are unchanged. The CLI equivalent is `msb snapshot load checkpoints/*.msb --group received`, with `--dest DIR` for another group-store root. ## SandboxHandle @@ -16,14 +46,14 @@ Create disk snapshots of stopped sandboxes and full checkpoints of running sandb async def snapshot(self, name: str) -> Snapshot ``` -Snapshot this sandbox under a bare name in the default snapshots directory (`~/.microsandbox/snapshots//`). The sandbox must be stopped or crashed. To place the artifact elsewhere, use [`Snapshot.save()`](#snapshot-save) / [`Snapshot.load()`](#snapshot-load) or move the self-contained artifact directory. Called on a [`SandboxHandle`](/sdk/python/sandbox#sandboxhandle), obtained from [`Sandbox.get()`](/sdk/python/sandbox#sandbox-get). +Snapshot this sandbox into its default group with the given member name. Live disk captures preserve the source's running or paused state. Use the returned artifact path or `sandbox:member` to open it later. Called on a [`SandboxHandle`](/sdk/python/sandbox#sandboxhandle), obtained from [`Sandbox.get()`](/sdk/python/sandbox#sandbox-get).

Parameters

namestr
-
Snapshot name; resolved under the default snapshots directory.
+
Member name within the source sandbox's group.
@@ -69,7 +99,7 @@ Create a sandbox from a snapshot artifact by passing `from_snapshot=` as a peer
from_snapshotstr | os.PathLike | None
-
Snapshot bare name, artifact directory, or archive file to boot from instead of image=.
+
Group head or group:member, artifact directory, or archive file to boot from instead of image=.
disk_onlybool
@@ -90,7 +120,7 @@ Create a sandbox from a snapshot artifact by passing `from_snapshot=` as a peer ```python # Boot from a snapshot -sb = await Sandbox.create("worker", from_snapshot="after-pip-install") +sb = await Sandbox.create("worker", from_snapshot="baseline:after-pip-install") # Or from an image (existing flow, unchanged) sb = await Sandbox.create("worker", image="python:3.12") @@ -102,13 +132,15 @@ sb = await Sandbox.create("worker", image="python:3.12") ## Disk maintenance and incremental export +Exporting since a base omits its reusable disk layers and RAM objects. Full checkpoints keep the complete memory map and CPU/device state. The last-layers option only selects disk layers and leaves RAM payloads complete. Imports resolve dependencies from the batch or an explicitly named destination group. Use an external base for missing dependencies; direct restore still accepts an explicit base. + ```python worker = await Sandbox.get("worker") plan = await worker.compact(layers=3, dry_run=True) result = await worker.compact(layers=3) -await Snapshot.save("checkpoint-b", "changes.tar.zst", since="checkpoint-a") -await Snapshot.load("changes.tar.zst", base="checkpoint-a") -child = await Sandbox.create("child", from_snapshot="changes.tar.zst", snapshot_base="checkpoint-a") +await Snapshot.save("worker:checkpoint-b", "changes.msb", since="worker:checkpoint-a") +await Snapshot.load("changes.msb", base="worker:checkpoint-a") +child = await Sandbox.create("child", from_snapshot="changes.msb", snapshot_base="worker:checkpoint-a") ``` The count includes the oldest base but excludes the writable head. Omit `layers` to compact all sealed layers. Use `last_layers=n` instead of `since` to export the newest N sealed layers. Results are dictionaries with `input_layers`, `selected_layers`, `output_layers`, `materialized_bytes`, `total_us`, `pause_us`, and `dry_run`. See [disk-chain maintenance](/sandboxes/snapshots#export-changes-and-compact-a-disk-chain) for dependency and retention rules. @@ -222,9 +254,10 @@ Best-effort source-sandbox name ```python @staticmethod async def create( - name: str, + name: str = "", *, from_sandbox: str, + group: str | None = None, dest_dir: str | os.PathLike[str] | None = None, labels: dict[str, str] | None = None, force: bool = False, @@ -233,14 +266,14 @@ async def create( ) -> Snapshot ``` -Create a disk snapshot from a stopped or crashed sandbox, or set `full=True` to checkpoint a running sandbox. `name` is resolved under the default snapshots directory (`~/.microsandbox/snapshots//`), or under `dest_dir=` when given; `from_sandbox=` names the sandbox to capture and is required. +Create a disk snapshot, or set `full=True` to include memory and execution state. `name` identifies a member within `group`; an omitted name is generated and an omitted group uses the source sandbox's name. `dest_dir` selects the parent directory containing the group. `from_sandbox` names the sandbox to capture and is required.

Parameters

namestr
-
Bare snapshot name; resolved under the default snapshots directory.
+
Snapshot member name; generated when omitted.
from_sandboxstr
@@ -248,7 +281,7 @@ Create a disk snapshot from a stopped or crashed sandbox, or set `full=True` to
dest_dirstr | os.PathLike[str] | None
-
Parent directory to create the artifact in; the artifact lands at dest_dir/<name>. Defaults to the snapshots directory.
+
Parent directory containing snapshot groups. Defaults to the snapshots directory.
labelsdict[str, str] | None
@@ -256,7 +289,7 @@ Create a disk snapshot from a stopped or crashed sandbox, or set `full=True` to
forcebool
-
Overwrite an existing artifact with the same name. Default False.
+
Must remain False for installed group members, which are immutable. Direct archive capture supports overwriting its output file.
record_integritybool
@@ -301,6 +334,7 @@ async def create_archive( archive: str | os.PathLike[str], *, from_sandbox: str, + group: str | None = None, labels: dict[str, str] | None = None, force: bool = False, record_integrity: bool = False, @@ -314,7 +348,7 @@ Capture directly into an archive without installing a snapshot directory or inde ```python archive = await Snapshot.create_archive( "after-pip-install", - "/tmp/after-pip-install.tar.zst", + "/tmp/after-pip-install.msb", from_sandbox="baseline", ) ``` @@ -331,20 +365,20 @@ async def open(path_or_name: str) -> Snapshot ```python -snap = await Snapshot.open("after-pip-install") +snap = await Snapshot.open("baseline:after-pip-install") print(snap.image_ref) ``` -Open an existing artifact by bare name (resolved under the default snapshots directory) or path. Cheap metadata validation only; does **not** read the upper file. Use [`verify()`](#snap-verify) for content checks. +Open an existing artifact by group head, `group:member`, or path. This validates metadata without reading the full upper file. Use [`verify()`](#snap-verify) for content checks.

Parameters

path_or_namestr
-
Bare snapshot name or artifact directory path.
+
Group head, group:member, or artifact directory path.
@@ -367,20 +401,20 @@ async def get(name_or_digest: str) -> SnapshotHandle ```python -h = await Snapshot.get("after-pip-install") +h = await Snapshot.get("baseline:after-pip-install") print(h.digest) ``` -Look up a handle in the local index by name, digest, or path. +Look up a handle by group head, `group:member`, stable snapshot ID, descriptor digest, or artifact path. Global IDs and digests must resolve unambiguously.

Parameters

name_or_digeststr
-
Snapshot name, digest, or path.
+
Group head, group:member, stable snapshot ID, descriptor digest, or artifact path.
@@ -457,7 +491,7 @@ async def remove(path_or_name: str, *, force: bool = False) -> None ```python -await Snapshot.remove("after-pip-install", force=True) +await Snapshot.remove("baseline:after-pip-install", force=True) ``` @@ -469,7 +503,7 @@ Remove a snapshot artifact and its index row. Refuses if the snapshot has indexe
path_or_namestr
-
Bare snapshot name or artifact path.
+
Group head, group:member, or artifact path.
forcebool
@@ -534,22 +568,22 @@ async def save( ```python await Snapshot.save( - "after-pip-install", - "/tmp/after-pip-install.tar.zst", + "baseline:after-pip-install", + "/tmp/after-pip-install.msb", with_parents=True, ) ``` -Bundle a snapshot into a `.tar.zst` archive. The existing snapshot manifest is archived as-is; create the snapshot with recorded integrity when the archive will cross a trust boundary. +Bundle a snapshot into a `.msb` archive. The existing snapshot manifest is archived as-is; create the snapshot with recorded integrity when the archive will cross a trust boundary.

Parameters

name_or_pathstr
-
Snapshot bare name or artifact path to save.
+
Group head or group:member or artifact path to save.
outstr | os.PathLike
@@ -565,10 +599,21 @@ Bundle a snapshot into a `.tar.zst` archive. The existing snapshot manifest is a
plain_tarbool
-
Write an uncompressed .tar instead of .tar.zst. Default False.
+
Write an uncompressed .tar instead of .msb. Default False.
+ + +```python +await Snapshot.save( + "baseline:after-pip-install", + "/tmp/after-pip-install.msb", + with_parents=True, +) +``` + + ---

Move artifacts

@@ -584,17 +629,20 @@ async def load( archive: str | os.PathLike, *, dest: str | os.PathLike | None = None, + base: str | None = None, + group: str | None = None, + set_head: bool = False, ) -> SnapshotHandle ``` -Unpack a snapshot archive (`.tar.zst` or `.tar`) into the snapshots directory. Structural and archive-entry checks run during import; recorded payload integrity is preserved for explicit [`verify()`](#snap-verify). Compression is detected from magic bytes. +Unpack a snapshot archive (`.msb` or `.tar`) into the selected or generated group. The returned handle's `group` identifies the group and `head_update` reports the head selection outcome. Structural and archive-entry checks run during import; recorded payload integrity is preserved for explicit [`verify()`](#snap-verify). Compression is detected from magic bytes.

Parameters

archivestr | os.PathLike
-
Archive path (.tar.zst or .tar).
+
Archive path (.msb or .tar).
deststr | os.PathLike | None
@@ -614,7 +662,7 @@ Unpack a snapshot archive (`.tar.zst` or `.tar`) into the snapshots directory. S ```python -h = await Snapshot.load("/tmp/after-pip-install.tar.zst") +h = await Snapshot.load("/tmp/after-pip-install.msb") print(h.path) ``` @@ -777,7 +825,7 @@ async def open(self) -> Snapshot ```python -h = await Snapshot.get("after-pip-install") +h = await Snapshot.get("baseline:after-pip-install") snap = await h.open() print(snap.fstype) ``` @@ -804,13 +852,13 @@ async def remove(self, *, force: bool = False) -> None ```python -h = await Snapshot.get("after-pip-install") +h = await Snapshot.get("baseline:after-pip-install") await h.remove(force=False) ``` -Remove this snapshot artifact and its index row. Refuses if the snapshot has indexed children unless `force=True`. +Remove this installed snapshot copy and its index row using the handle's stored artifact path. Other groups containing the same snapshot ID or digest remain unchanged. Refuses if the snapshot has indexed children unless `force=True`.

Parameters

diff --git a/docs/sdk/rust/snapshots.mdx b/docs/sdk/rust/snapshots.mdx index 4ab10d26e..f9a3837a0 100644 --- a/docs/sdk/rust/snapshots.mdx +++ b/docs/sdk/rust/snapshots.mdx @@ -6,22 +6,58 @@ keywords: ["Rust SDK", "Rust snapshots", "microsandbox snapshots"] Local-only -Create disk snapshots of stopped sandboxes and full checkpoints of running sandboxes. See [Snapshots](/sandboxes/snapshots) for usage and lifecycle concepts. +Create disk snapshots of running, paused, stopped, or crashed sandboxes, or full checkpoints of running and paused sandboxes. See [Snapshots](/sandboxes/snapshots) for usage and lifecycle concepts. + +## Snapshot groups + +Installed snapshots belong to a group. A bare group selects its head; `group:member` selects a member by name or stable snapshot ID. Artifact paths remain valid selectors. Creation defaults to the source sandbox's group, and an empty builder name generates a member name. Imports without `LoadOpts.group` create a new generated group. `dest_dir` and `LoadOpts.dest` select the parent directory containing groups. + +```rust +use microsandbox::{Snapshot, snapshot::LoadOpts}; +use std::path::Path; + +let snap = Snapshot::builder("baseline").from_sandbox("box").group("work").create().await?; +let loaded = Snapshot::load_with_options(Path::new("changes.msb"), LoadOpts { + base: Some("work:baseline".into()), + group: Some("work".into()), + ..Default::default() +}).await?; +let head = Snapshot::group_head("work").await?; +let selected = Snapshot::group_head("work:baseline").await?; +``` + +`LoadOpts` contains `dest: Option`, `base: Option`, `group: Option`, and `set_head: bool`. `set_head: true` explicitly selects the imported member even when it is not a descendant. `load` and `load_with_base` use generated groups. `group_head` returns `HeadUpdate` with `group`, `previous`, `head`, `reason`, and `changed`; `previous` and `head` are stable snapshot IDs. Automatic head updates initialize an empty group or advance to a proven descendant; divergent imports and imports with unknown ancestry retain the current head. `Snapshot::head_update()` and `SnapshotHandle::head_update()` expose the create/import outcome. Direct archive capture creates no group and rejects `.group(...)`. + +## Load multiple archives + +```rust +let archives = vec!["changes.msb".into(), "base.msb".into()]; +let handles = Snapshot::load_many(&archives, LoadOpts { + group: Some("received".into()), + ..Default::default() +}).await?; +``` + +`Snapshot::load_many(archive_paths: &[PathBuf], opts: LoadOpts)` returns `MicrosandboxResult>`, with one handle per supplied archive in input order. Dependencies are resolved regardless of argument order from the batch and exact matching snapshots already installed in an explicitly named destination group. `opts.base` supplies an external snapshot or standalone archive when needed. Single-file `load_with_options` also reuses dependencies from its named destination group. Omitting `group` creates one generated group for the batch. + +All batch members are validated before publication. `set_head: true` requires a unique tip. With divergent tips and the default `false`, an existing head is retained (`HeadUpdateReason::AmbiguousCandidates`); a new group has no head and imported handles return `None` from `head_update()`. Select a member explicitly to give that group a head. The archive format is unchanged. The CLI equivalent is `msb snapshot load checkpoints/*.msb --group received`, with `--dest DIR` for another group-store root. ## Disk maintenance and incremental export +Exporting since a base omits its reusable disk layers and RAM objects. Full checkpoints keep the complete memory map and CPU/device state. The last-layers option only selects disk layers and leaves RAM payloads complete. Imports resolve dependencies from the batch or an explicitly named destination group. Use an external base for missing dependencies; direct restore still accepts an explicit base. + ```rust let worker = Sandbox::get("worker").await?; let plan = worker.compact().layers(3).dry_run().await?; let result = worker.compact().layers(3).apply().await?; -Snapshot::save("checkpoint-b", Path::new("changes.tar.zst"), SaveOpts { - since: Some("checkpoint-a".into()), +Snapshot::save("worker:checkpoint-b", Path::new("changes.msb"), SaveOpts { + since: Some("worker:checkpoint-a".into()), ..Default::default() }).await?; -Snapshot::load_with_base(Path::new("changes.tar.zst"), None, "checkpoint-a").await?; +Snapshot::load_with_base(Path::new("changes.msb"), None, "worker:checkpoint-a").await?; ``` -`layers` counts the oldest physical layers including the base, excluding the writable head. Omit it to compact all sealed layers. `last_layers: Some(n)` is an alternative to `since`; they cannot be combined with each other or `with_parents`. For direct restore, use `Sandbox::builder("child").from_snapshot("changes.tar.zst").snapshot_base("checkpoint-a")`. See [disk-chain maintenance](/sandboxes/snapshots#export-changes-and-compact-a-disk-chain) for dependency and retention rules. +`layers` counts the oldest physical layers including the base, excluding the writable head. Omit it to compact all sealed layers. `last_layers: Some(n)` is an alternative to `since`; they cannot be combined with each other or `with_parents`. For direct restore, use `Sandbox::builder("child").from_snapshot("changes.msb").snapshot_base("worker:checkpoint-a")`. See [disk-chain maintenance](/sandboxes/snapshots#export-changes-and-compact-a-disk-chain) for dependency and retention rules. ## Snapshot @@ -31,14 +67,14 @@ Snapshot::load_with_base(Path::new("changes.tar.zst"), None, "checkpoint-a").awa fn builder(name: impl Into) -> SnapshotBuilder ``` -Start configuring a new snapshot named `name`, resolved under the default snapshots directory (`~/.microsandbox/snapshots//`) or under [`dest_dir()`](#snapshot_builder-dest_dir) when set. The source sandbox is set with [`from_sandbox()`](#snapshot_builder-from_sandbox), which is required; the other setters cover labels and whether to record content integrity before capturing. See [`SnapshotBuilder`](#snapshotbuilder) for all options. +Start configuring a snapshot member named `name`, generated when empty. The default group is the source sandbox's name; `.group(name)` selects another group and [`dest_dir()`](#snapshot_builder-dest_dir) selects its parent directory. The source sandbox is set with [`from_sandbox()`](#snapshot_builder-from_sandbox), which is required; the other setters cover labels and whether to record content integrity before capturing. See [`SnapshotBuilder`](#snapshotbuilder) for all options.

Parameters

nameimpl Into<String>
-
Bare snapshot name. Must not be empty, contain /, or start with ..
+
Member name; generated when empty. Names must not contain path separators or start with ..
@@ -70,7 +106,7 @@ let snap = Snapshot::builder("baseline") async fn create(config: SnapshotConfig) -> MicrosandboxResult ``` -Create an installed snapshot artifact atomically, then best-effort update the rebuildable local index. Disk mode captures a stopped or crashed sandbox; full mode captures a running sandbox's checkpoint closure. Most callers use the [builder](#snapshotbuilder)'s [`create()`](#create) instead of constructing a [`SnapshotConfig`](#snapshotconfig) by hand. +Create an installed snapshot artifact atomically, then best-effort update the rebuildable local index. Disk mode supports running, paused, stopped, and crashed sources; full mode includes memory and execution state from a running or paused source. Most callers use the [builder](#snapshotbuilder)'s [`create()`](#create) instead of constructing a [`SnapshotConfig`](#snapshotconfig) by hand.

Parameters

@@ -117,7 +153,7 @@ Capture a disk or full snapshot directly into an archive. The operation creates ```rust let archive = Snapshot::create_archive( Snapshot::builder("baseline").from_sandbox("api").build()?, - "/tmp/baseline.tar.zst", + "/tmp/baseline.msb", false, ).await?; println!("{} {}", archive.id(), archive.path().display()); @@ -134,20 +170,20 @@ async fn open(path_or_name: impl AsRef) -> MicrosandboxResult ```rust -let snap = Snapshot::open("baseline").await?; +let snap = Snapshot::open("api:baseline").await?; println!("{}", snap.manifest().image.reference); ``` -Open an existing artifact by path or bare name. Bare names (no path separator, not starting with `.` or `~`) resolve under the default snapshots directory; anything else is treated as a path. This is a fast metadata operation: it verifies the manifest structure, recomputes the manifest digest, and checks that the upper file exists with the recorded size. It does **not** read the full upper contents; use [`verify()`](#snap-verify) for that. +Open an existing artifact by group head, `group:member`, or path. This is a fast metadata operation: it verifies the manifest structure, recomputes the manifest digest, and checks that the upper file exists with the recorded size. It does **not** read the full upper contents; use [`verify()`](#snap-verify) for that.

Parameters

path_or_nameimpl AsRef<str>
-
Bare snapshot name or filesystem path to an artifact directory.
+
Group head, group:member, or filesystem path to an artifact directory.
@@ -169,20 +205,20 @@ async fn get(name_or_digest: &str) -> MicrosandboxResult ```rust -let h = Snapshot::get("after-pip-install").await?; +let h = Snapshot::get("api:baseline").await?; println!("{} from {}", h.digest(), h.image_ref()); ``` -Look up a lightweight [`SnapshotHandle`](#snapshothandle) in the local index by name, digest (`sha256:`/`sha512:` prefix), or path. +Look up a lightweight [`SnapshotHandle`](#snapshothandle) by group head, `group:member`, stable snapshot ID, descriptor digest, or artifact path. Global IDs and digests must resolve unambiguously.

Parameters

name_or_digest&str
-
Snapshot name, manifest digest, or artifact path.
+
Group head, group:member, stable snapshot ID, descriptor digest, or artifact path.
@@ -257,19 +293,19 @@ async fn remove(path_or_name: &str, force: bool) -> MicrosandboxResult<()> ```rust -Snapshot::remove("after-pip-install", false).await?; +Snapshot::remove("api:baseline", false).await?; ``` -Remove a snapshot artifact (by digest, name, or path) and its index row. Refuses if the snapshot has indexed children unless `force` is set. The artifact directory is deleted on success and the parent's child count is decremented. +Remove a snapshot artifact by group selector, unambiguous ID or digest, or path, along with its index row. Refuses if the snapshot has indexed children unless `force` is set. A group's head cannot be removed while other members remain, even with `force`; select another head first.

Parameters

path_or_name&str
-
Snapshot digest, name, or artifact path.
+
Group head, group:member, unambiguous snapshot ID or digest, or artifact path.
forcebool
@@ -330,14 +366,14 @@ println!("indexed {n} snapshots"); async fn save(name_or_path: &str, out: &Path, opts: SaveOpts) -> MicrosandboxResult<()> ``` -Bundle a snapshot into a `.tar.zst` archive (or plain `.tar`) at `out`. Recorded payload integrity is preserved but not executed implicitly; call [`verify()`](#snap-verify) when an independent content scan is part of your workflow. See [`SaveOpts`](#saveopts) to also include ancestors and the OCI image cache. +Bundle a snapshot into a `.msb` archive (or plain `.tar`) at `out`. Recorded payload integrity is preserved but not executed implicitly; call [`verify()`](#snap-verify) when an independent content scan is part of your workflow. See [`SaveOpts`](#saveopts) to also include ancestors and the OCI image cache.

Parameters

name_or_path&str
-
Snapshot name or artifact path to save.
+
Group head, group:member, or artifact path to save.
out&Path
@@ -356,8 +392,8 @@ use microsandbox::snapshot::SaveOpts; use std::path::Path; Snapshot::save( - "baseline", - Path::new("/tmp/baseline.tar.zst"), + "api:baseline", + Path::new("/tmp/baseline.msb"), SaveOpts { with_parents: true, with_image: true, ..Default::default() }, ).await?; ``` @@ -378,13 +414,13 @@ async fn load(archive_path: &Path, dest: Option<&Path>) -> MicrosandboxResult -Unpack a snapshot archive (`.tar.zst` or `.tar`, detected from magic bytes) into the snapshots directory (or `dest`), routing any bundled image-cache entries into the global cache and registering everything found in the index. Structural and archive-entry checks remain mandatory, while recorded payload integrity is preserved for explicit [`verify()`](#snap-verify). Returns a handle for the head snapshot. +Unpack a snapshot archive (`.msb` or `.tar`, detected from magic bytes) into the snapshots directory (or `dest`), routing any bundled image-cache entries into the global cache and registering everything found in the index. Structural and archive-entry checks remain mandatory, while recorded payload integrity is preserved for explicit [`verify()`](#snap-verify). Returns a handle for the head snapshot.

Parameters

@@ -404,7 +440,7 @@ Unpack a snapshot archive (`.tar.zst` or `.tar`, detected from magic bytes) into
-
Handle for the head (last-listed) snapshot.
+
Handle for the archive-declared head snapshot, which may differ from the receiving group's selected head.
@@ -413,7 +449,7 @@ Unpack a snapshot archive (`.tar.zst` or `.tar`, detected from magic bytes) into ```rust use std::path::Path; -let h = Snapshot::load(Path::new("/tmp/baseline.tar.zst"), None).await?; +let h = Snapshot::load(Path::new("/tmp/baseline.msb"), None).await?; println!("loaded {}", h.digest()); ``` @@ -476,7 +512,7 @@ fn manifest(&self) -> &Manifest ```rust -let snap = Snapshot::open("baseline").await?; +let snap = Snapshot::open("api:baseline").await?; let m = snap.manifest(); println!("{} @ {}", m.image.reference, m.image.manifest_digest); ``` @@ -522,7 +558,7 @@ async fn verify(&self) -> MicrosandboxResult ```rust use microsandbox::snapshot::UpperVerifyStatus; -let snap = Snapshot::open("baseline").await?; +let snap = Snapshot::open("api:baseline").await?; match snap.verify().await?.upper { UpperVerifyStatus::Verified { algorithm, .. } => println!("ok via {algorithm}"), UpperVerifyStatus::NotRecorded => println!("no integrity hash recorded"), @@ -564,7 +600,7 @@ Manifest digest (`sha256:hex`), the canonical identity. fn name(&self) -> Option<&str> ``` -Name alias, or `None` for digest-only entries. +Member name within its group, or `None` when no alias is recorded. #### h.parent_digest() @@ -572,7 +608,7 @@ Name alias, or `None` for digest-only entries. fn parent_digest(&self) -> Option<&str> ``` -The parent snapshot's digest, or `None` for a root. Always `None` today; populated once chained snapshots land. +The captured parent snapshot's stable ID, or `None` when no parent is known. The accessor retains its existing `parent_digest` name. --- @@ -645,7 +681,7 @@ async fn open(&self) -> MicrosandboxResult ```rust -let h = Snapshot::get("baseline").await?; +let h = Snapshot::get("api:baseline").await?; let snap = h.open().await?; snap.verify().await?; ``` @@ -672,13 +708,13 @@ async fn remove(&self, force: bool) -> MicrosandboxResult<()> ```rust -let h = Snapshot::get("baseline").await?; +let h = Snapshot::get("api:baseline").await?; h.remove(false).await?; ``` -Remove this snapshot. Delegates to [`Snapshot::remove(self.digest(), force)`](#snapshotremove). +Remove this installed snapshot copy by its stored artifact path. Other groups containing the same snapshot ID or digest remain unchanged.

Parameters

@@ -703,7 +739,7 @@ fn from_snapshot(self, path_or_name: impl Into) -> Self ```rust let sb = Sandbox::builder("api-restored") - .from_snapshot("after-pip-install") + .from_snapshot("api:baseline") .create() .await?; ``` @@ -717,7 +753,7 @@ let sb = Sandbox::builder("api-restored")
path_or_nameimpl Into<String>
-
Bare name resolved under the default snapshots directory, or a path to an artifact directory or archive file.
+
Group head or group:member selector, or a path to an artifact directory or archive file.
@@ -737,14 +773,14 @@ Cold-boot only the disk state carried by a full snapshot. Chain after [`from_sna async fn snapshot(&self, name: &str) -> MicrosandboxResult ``` -`SandboxHandle` method. Snapshot this sandbox under a bare name in the default snapshots directory (`~/.microsandbox/snapshots//`). The sandbox must be stopped or crashed; running sandboxes are rejected with `SnapshotSandboxRunning`. Local handles only. To place the artifact elsewhere, use [`Snapshot::save()`](#snapshotsave) / [`Snapshot::load()`](#snapshotload) or move the self-contained artifact directory. +`SandboxHandle` method. Snapshot this sandbox's disk into its default group with the given member name. Use the returned artifact path or `sandbox:member` to open it later. Live captures are crash-consistent and preserve the source's running/paused state. Local handles only. To place the artifact elsewhere, use [`Snapshot::save()`](#snapshotsave) / [`Snapshot::load()`](#snapshotload) or move the self-contained artifact directory.

Parameters

name&str
-
Bare snapshot name.
+
Member name within the source sandbox's group.
@@ -757,18 +793,15 @@ async fn snapshot(&self, name: &str) -> MicrosandboxResult
-#### h.snapshot_to() - -```rust -async fn snapshot_to(&self, path: impl AsRef) -> MicrosandboxResult -``` - - + ```rust -let h = Sandbox::get("api").await?; -h.stop().await?; -let snap = h.snapshot_to("/data/snapshots/baseline").await?; +let snap = Snapshot::builder("baseline") + .from_sandbox("api") + .dest_dir("/data/snapshots") + .create() + .await?; +// snap.path() is /data/snapshots/api/. ``` @@ -795,7 +828,7 @@ Set the sandbox to capture. Required; [`build()`](#snapshot_builder-build) and [
source_sandboximpl Into<String>
-
Name of the source sandbox. Must be stopped or crashed, and rooted on an OCI image.
+
Name of the OCI-rooted source sandbox. Disk capture also supports running and paused sources.
@@ -808,7 +841,7 @@ Set the sandbox to capture. Required; [`build()`](#snapshot_builder-build) and [ fn dest_dir(self, dest_dir: impl Into) -> Self ``` -Create the artifact under this parent directory instead of the default snapshots store. The artifact directory is `dest_dir/`; the name stays the snapshot's identity either way. +Create the snapshot group under this parent directory instead of the default snapshots store. Member names are local aliases within the group; stable snapshot IDs identify immutable artifacts.

Parameters

@@ -846,7 +879,7 @@ Add a user label. Can be called multiple times. Labels are sorted by key in the fn force(self) -> Self ``` -Overwrite an existing artifact with the same name. Without this, creation fails with `SnapshotAlreadyExists` if the artifact directory exists. +Overwrite an existing direct archive output file. Installed group members are immutable, so installed creation rejects this option. #### snapshot_builder.record_integrity() @@ -911,11 +944,12 @@ Inputs to create a snapshot. A type alias for `SnapshotSpec`. Usually built via | Field | Type | Description | |-------|------|-------------| -| name | `String` | Bare snapshot name; always the artifact directory's basename | -| dest_dir | `Option` | Parent directory for the artifact; `None` = the default snapshots directory | -| source_sandbox | `String` | Name of the source sandbox; must be stopped | +| name | `String` | Member name within its group; generated when empty | +| group | `Option` | Destination group; defaults to the source sandbox's name | +| dest_dir | `Option` | Parent directory containing groups; `None` = the default snapshots directory | +| source_sandbox | `String` | Name of the source sandbox; disk capture preserves running/paused state | | labels | `Vec<(String, String)>` | User-supplied labels | -| force | `bool` | Overwrite an existing artifact with the same name | +| force | `bool` | Overwrite a direct archive output; rejected for installed group members | | record_integrity | `bool` | Compute and record upper-layer integrity at creation | | full | `bool` | Capture a running sandbox's full checkpoint instead of a stopped-sandbox disk snapshot | diff --git a/docs/sdk/typescript/sandbox.mdx b/docs/sdk/typescript/sandbox.mdx index 8cc23a776..3d3d20143 100644 --- a/docs/sdk/typescript/sandbox.mdx +++ b/docs/sdk/typescript/sandbox.mdx @@ -276,6 +276,8 @@ A running `Sandbox` exposes `name` (`string`), `id` (`string`), and `ownsLifecyc Command execution (`exec`, `execWith`, `execStream`, `execStreamWith`, `shell`, `shellStream`) and foreground attachment (`attach`, `attachWith`, `attachShell`) live on the [Execution](/sdk/typescript/execution) page. +Lifecycle calls do not wait for unrelated guest operations on the same handle to finish. For example, you can pause and resume while an `exec()` promise is pending. Await operations explicitly when their order matters; stopping a sandbox may interrupt pending commands or filesystem requests. + #### sandbox.config() ```typescript @@ -318,6 +320,8 @@ await sandbox.detach(); // keeps running in the background Release the handle without stopping the sandbox. The sandbox continues running as a background process. Reconnect later with [`Sandbox.get()`](#sandbox-get). +After detaching, this handle and its existing `fs()` objects reject new guest operations. Already-started operations retain their connection; await them before detaching if you need their results first. + #### sandbox.fs() ```typescript diff --git a/docs/sdk/typescript/snapshots.mdx b/docs/sdk/typescript/snapshots.mdx index ac54a1629..06de96c77 100644 --- a/docs/sdk/typescript/snapshots.mdx +++ b/docs/sdk/typescript/snapshots.mdx @@ -22,7 +22,7 @@ fromSnapshot(pathOrName: string): SandboxBuilder ```typescript const sb = await Sandbox.builder("worker") - .fromSnapshot("after-pip-install") + .fromSnapshot("baseline:after-pip-install") .create(); ``` @@ -37,7 +37,7 @@ Chain `.diskOnly()` after `.fromSnapshot()` to cold-boot only the disk state fro
pathOrNamestring
-
Bare name (resolved under the default snapshots directory) or filesystem path to an artifact directory or archive file.
+
Group head or group:member selector or filesystem path to an artifact directory or archive file.
@@ -58,14 +58,14 @@ Chain `.diskOnly()` after `.fromSnapshot()` to cold-boot only the disk state fro snapshot(name: string): Promise ``` -Snapshot this sandbox under a bare name in the default snapshots directory (`~/.microsandbox/snapshots//`). Called on a [`SandboxHandle`](/sdk/typescript/sandbox#sandboxhandle). The sandbox must be stopped or crashed; running sandboxes are rejected with a `SnapshotSandboxRunning` error. To place the artifact elsewhere, use [`Snapshot.save()`](#snapshot-save) / [`Snapshot.load()`](#snapshot-load) or move the self-contained artifact directory. +Snapshot this sandbox into its default group with the given member name. Called on a [`SandboxHandle`](/sdk/typescript/sandbox#sandboxhandle). Live disk captures preserve the source's running or paused state. Use the returned artifact path or `sandbox:member` to open it later.

Parameters

namestring
-
Bare name; becomes the artifact directory under the default snapshots dir.
+
Member name within the source sandbox's group.
@@ -90,16 +90,48 @@ const snap = await h.snapshot("after-pip-install"); --- +## Snapshot groups + +Installed snapshots belong to a group. A bare group selects its head; `group:member` selects a member by name or stable snapshot ID. Artifact paths remain valid selectors. Creation defaults to the source sandbox's group and generates a member name when omitted; imports without a group create a new generated group. `destDir` and `LoadOpts.dest` select the parent directory containing groups. + +```typescript +const snap = await Snapshot.builder("baseline").fromSandbox("box").group("work").create(); +const loaded = await Snapshot.loadWithOptions("changes.msb", { + base: "work:baseline", + group: "work", +}); +const head = await Snapshot.groupHead("work"); +const selected = await Snapshot.groupHead("work:baseline"); +await Snapshot.loadWithOptions("other.msb", { group: "work", setHead: true }); +``` + +`LoadOpts` contains optional `dest`, `base`, `group`, and `setHead` fields. `Snapshot.load(archive, dest?, base?)` uses a generated group. `Snapshot.groupHead(selector)` returns `HeadUpdate` with `group`, `previous`, `head`, `reason`, and `changed`; `previous` and `head` are stable snapshot IDs. Automatic head updates initialize an empty group or advance to a proven descendant; divergent imports and imports with unknown ancestry retain the current head. Reasons are `initialized`, `fast_forwarded`, `selected`, `unchanged`, `diverged`, `unknown_ancestry`, and `ambiguous_candidates`. Direct archive capture creates no group and rejects `.group(...)`. + +## Load multiple archives + +```typescript +const handles = await Snapshot.loadMany( + ["changes.msb", "base.msb"], + { group: "received" }, +); +``` + +`Snapshot.loadMany(archives: string[], opts?: LoadOpts)` returns `Promise`, with one handle per supplied archive in input order. Dependencies are resolved regardless of argument order from the batch and exact matching snapshots in an explicitly named destination group. `opts.base` supplies an external snapshot or standalone archive when needed. Single-file `loadWithOptions` also reuses dependencies from its named destination group. Omitting `group` creates one generated group for the batch. + +All batch members are validated before publication. `setHead: true` requires a unique tip. With divergent tips and the default `false`, an existing head is retained (`ambiguous_candidates`); a new group has no head and imported handles report `headUpdate: null`. Select a member explicitly to give that group a head. The archive format is unchanged. The CLI equivalent is `msb snapshot load checkpoints/*.msb --group received`, with `--dest DIR` for another group-store root. + ## Disk maintenance and incremental export +Exporting since a base omits its reusable disk layers and RAM objects. Full checkpoints keep the complete memory map and CPU/device state. The last-layers option only selects disk layers and leaves RAM payloads complete. Imports resolve dependencies from the batch or an explicitly named destination group. Use an external base for missing dependencies; direct restore still accepts an explicit base. + ```typescript const worker = await Sandbox.get("worker"); const plan = await worker.compact({ layers: 3, dryRun: true }); const result = await worker.compact({ layers: 3 }); -await Snapshot.save("checkpoint-b", "changes.tar.zst", { since: "checkpoint-a" }); -await Snapshot.load("changes.tar.zst", undefined, "checkpoint-a"); +await Snapshot.save("worker:checkpoint-b", "changes.msb", { since: "worker:checkpoint-a" }); +await Snapshot.load("changes.msb", undefined, "worker:checkpoint-a"); const child = await Sandbox.builder("child") - .fromSnapshot("changes.tar.zst").snapshotBase("checkpoint-a").create(); + .fromSnapshot("changes.msb").snapshotBase("worker:checkpoint-a").create(); ``` The count includes the oldest base but excludes the writable head. Omit `layers` to compact all sealed layers. `lastLayers` selects the newest N sealed export layers instead of `since`. Results expose physical counts, `materializedBytes`, `totalUs`, and `pauseUs`; materialized bytes are not reclaimed space. See [disk-chain maintenance](/sandboxes/snapshots#export-changes-and-compact-a-disk-chain) for dependency and retention rules. @@ -220,17 +252,17 @@ Best-effort source-sandbox name, if recorded. `null` when the manifest has no so #### Snapshot.builder() ```typescript -static builder(name: string): SnapshotBuilder +static builder(name?: string): SnapshotBuilder ``` -Begin building a new snapshot named `name`, resolved under the default snapshots directory (`~/.microsandbox/snapshots//`) or under [`.destDir()`](#snapshot-destdir) when set. The fluent builder is what powers the CLI internally. The source sandbox is set with [`.fromSandbox()`](#snapshot-fromsandbox), which is required. See [`SnapshotBuilder`](#snapshotbuilder) for all setters. +Begin building a snapshot member named `name`, generated when omitted. The default group is the source sandbox's name; `.group(name)` selects another group and [`.destDir()`](#snapshot-destdir) selects its parent directory. The fluent builder is what powers the CLI internally. The source sandbox is set with [`.fromSandbox()`](#snapshot-fromsandbox), which is required. See [`SnapshotBuilder`](#snapshotbuilder) for all setters.

Parameters

namestring
-
Bare snapshot name; becomes the artifact directory under the default snapshots dir.
+
Member name within its group; generated when omitted.
@@ -268,7 +300,7 @@ Capture directly into an archive without installing a snapshot directory or inde ```typescript const archive = await Snapshot.builder("after-pip-install") .fromSandbox("baseline") - .createArchive("/tmp/after-pip-install.tar.zst"); + .createArchive("/tmp/after-pip-install.msb"); ``` --- @@ -282,20 +314,20 @@ static open(pathOrName: string): Promise ```typescript -const snap = await Snapshot.open("after-pip-install"); +const snap = await Snapshot.open("baseline:after-pip-install"); console.log(snap.digest); ``` -Open an existing snapshot artifact. Bare names resolve under the default snapshots directory; anything else is treated as a path. Cheap metadata validation only; it does not read the upper file. Use [`verify()`](#snap-verify) for content checks. +Open an existing snapshot artifact by group head, `group:member`, or path. Cheap metadata validation only; it does not read the upper file. Use [`verify()`](#snap-verify) for content checks.

Parameters

pathOrNamestring
-
Bare name (resolved under the default snapshots dir) or filesystem path.
+
Group head or group:member selector or filesystem path.
@@ -317,7 +349,7 @@ static get(nameOrDigest: string): Promise ```typescript -const h = await Snapshot.get("after-pip-install"); +const h = await Snapshot.get("baseline:after-pip-install"); console.log(h.digest, h.createdAt); ``` @@ -405,19 +437,19 @@ static remove(pathOrName: string, opts?: { force?: boolean }): Promise ```typescript -await Snapshot.remove("after-pip-install", { force: true }); +await Snapshot.remove("baseline:after-pip-install", { force: true }); ``` -Remove a snapshot by path, name, or digest. Refuses if the snapshot has indexed children unless `force` is set. +Remove a snapshot by group selector, unambiguous ID or digest, or path. Refuses if the snapshot has indexed children unless `force` is set. A group's head cannot be removed while other members remain, even with `force`; select another head first.

Parameters

pathOrNamestring
-
Path, name, or digest of the snapshot to remove.
+
Group head, group:member, unambiguous snapshot ID or digest, or artifact path.
opts.forceboolean
@@ -469,14 +501,14 @@ Walk the snapshots directory (default: the configured snapshots dir) and rebuild static save(nameOrPath: string, out: string, opts?: SaveOpts): Promise ``` -Bundle a snapshot into a `.tar.zst` archive. The recorded manifest is archived as-is, so create the snapshot with [`recordIntegrity()`](#snapshot-recordintegrity) if receivers must verify content. See [`SaveOpts`](#saveopts-interface) for bundling options. +Bundle a snapshot into a `.msb` archive. The recorded manifest is archived as-is, so create the snapshot with [`recordIntegrity()`](#snapshot-recordintegrity) if receivers must verify content. See [`SaveOpts`](#saveopts-interface) for bundling options.

Parameters

nameOrPathstring
-
Name or path of the snapshot to bundle.
+
Group head, group:member, or artifact path to bundle.
outstring
@@ -491,7 +523,7 @@ Bundle a snapshot into a `.tar.zst` archive. The recorded manifest is archived a ```typescript -await Snapshot.save("after-pip-install", "./baseline.tar.zst", { +await Snapshot.save("baseline:after-pip-install", "./baseline.msb", { withImage: true, }); ``` @@ -504,10 +536,10 @@ await Snapshot.save("after-pip-install", "./baseline.tar.zst", {
staticasync
```typescript -static load(archive: string, dest?: string): Promise +static load(archive: string, dest?: string, base?: string): Promise ``` -Unpack a snapshot archive (`.tar.zst` or `.tar`) into the snapshots directory. Structural and archive-entry checks run during import; recorded payload integrity is preserved for explicit [`verify()`](#snap-verify). Compression is detected from magic bytes. +Unpack a snapshot archive (`.msb` or `.tar`) into the snapshots directory. Structural and archive-entry checks run during import; recorded payload integrity is preserved for explicit [`verify()`](#snap-verify). Compression is detected from magic bytes.

Parameters

@@ -534,7 +566,7 @@ Unpack a snapshot archive (`.tar.zst` or `.tar`) into the snapshots directory. S ```typescript -const h = await Snapshot.load("./baseline.tar.zst"); +const h = await Snapshot.load("./baseline.msb"); console.log("loaded", h.digest); ``` @@ -593,7 +625,7 @@ Manifest digest (`sha256:hex`), the canonical identity. `string \| null` -Convenience name; `null` for digest-only entries. +Member name within its group, or `null` when no alias is recorded. #### snapshotHandle.parentDigest @@ -651,10 +683,10 @@ Open and metadata-validate the underlying artifact. Throws if this handle is rea remove(opts?: { force?: boolean }): Promise ``` -Remove the artifact and its index row. Refuses if the snapshot has indexed children unless `force` is set. Throws if this handle is read-only. +Remove this installed snapshot copy and its index row using the handle's stored artifact path. Other groups containing the same snapshot ID or digest remain unchanged. Refuses if the snapshot has indexed children unless `force` is set. Throws if this handle is read-only. ```typescript -const h = await Snapshot.get("after-pip-install"); +const h = await Snapshot.get("baseline:after-pip-install"); const snap = await h.open(); // metadata-validated await h.remove({ force: false }); // refuse if it has children ``` @@ -692,7 +724,7 @@ Set the sandbox to capture. Required; [`.create()`](#snapshot-create) fails with destDir(destDir: string): this ``` -Create the artifact under this parent directory instead of the default snapshots store. The artifact directory is `destDir/`; the name stays the snapshot's identity either way. +Create the snapshot group under this parent directory instead of the default snapshots store. Member names are local aliases within the group; stable snapshot IDs identify immutable artifacts.

Parameters

@@ -730,7 +762,7 @@ Add a `key=value` label to the snapshot manifest. May be called repeatedly. force(): this ``` -Overwrite an existing artifact with the same name instead of failing on conflict. +Overwrite an existing direct archive output file. Installed group members are immutable, so installed creation rejects this option. #### snapshot.recordIntegrity() @@ -764,7 +796,6 @@ create(): Promise ```typescript const snap = await Snapshot.builder("baseline-v2") .fromSandbox("baseline") - .force() .recordIntegrity() .create(); ``` diff --git a/docs/snapshot-groups-explained.md b/docs/snapshot-groups-explained.md new file mode 100644 index 000000000..0f381d5b7 --- /dev/null +++ b/docs/snapshot-groups-explained.md @@ -0,0 +1,157 @@ +# Microsandbox snapshots: groups, checkpoints, and the head + +This describes the snapshot-group implementation on the development stack, not an already released CLI. + +## Start with the snapshot + +A snapshot is a saved point you can use to create another sandbox: + +```text +Running sandbox + | + +-- disk snapshot ----> new VM boots from the saved disk + | + +-- full snapshot ----> new VM resumes saved RAM, CPUs, devices, and disk +``` + +Disk snapshots also work when the source is paused or stopped. Full snapshots require resident execution state: a running or user-paused VM. Each capture produces a new immutable snapshot, even when unchanged disk layers or RAM objects are reused. Exporting a snapshot packages it as a `.msb` archive; loading an archive installs it without starting a VM. + +## A group gives those saved points a local home + +A **group** is a namespace containing snapshots and a selected **head**. Member names such as `cp01` are meaningful inside their group. Each snapshot also keeps its portable `snap_...` ID. A new group imported from competing branches can temporarily have no selected head; choose one explicitly before restoring by the bare group name. + +```text +~/.microsandbox/snapshots/ +| ++-- worker/ +| +-- group.json head = snap_B +| +-- snap_A/ +| | +-- snapshot.json ID, parent, disk/state references +| | +-- group-member.json name = cp01 +| | +-- layers/... disk-only payload +| +-- snap_B/ +| +-- snapshot.json parent = snap_A +| +-- group-member.json name = cp02 +| +-- checkpoint/... full checkpoint payload, when captured full +| ++-- imported/ + +-- group.json + +-- snap_A/... a separate local copy of the same snapshot +``` + +IDs above are shortened for readability. A disk-only member uses `layers/`; a full member uses `checkpoint/` with its disk layers, RAM objects, and execution/device state. Optional `metadata.json` stores labels. + +Groups do not magically make random IDs collision-proof. They keep local addresses separate. Within one group, the same ID with the same descriptor is reusable; the same ID with different descriptor bytes is rejected. A name already used by another member is also rejected. Nothing is silently overwritten. If a global ID resolves to multiple local copies, use the group-qualified address instead. + +## Create and restore + +```bash +msb create alpine --name worker --memory 512M + +# Group defaults to the source sandbox's name: worker. +msb snapshot create cp01 --from-sandbox worker --full +msb snapshot create cp02 --from-sandbox worker --full + +# A bare group selects its head, currently cp02. +msb create --name latest --from-snapshot worker --forked + +# A qualified name selects an exact checkpoint. +msb create --name earlier --from-snapshot worker:cp01 --forked + +# You can choose a different group, or let a member name be generated. +msb snapshot create --from-sandbox worker --group experiments --full +``` + +`--forked` shares clean restored RAM pages using copy-on-write; child writes remain private. It does not change which snapshot is selected. Omit `--full` at capture for a disk-only snapshot, and omit `--forked` when cold-booting disk state. + +## The head moves forward, not sideways by surprise + +Snapshots record their actual source ancestry. Neither timestamps, import order, nor an export's `--since` base defines that ancestry. + +```text +worker:cp01 ---- worker:cp02 ---- worker:cp03 <- head + \ + +-------- worker:experiment +``` + +The rules are small: + +- Empty group: a single capture or import initializes its head. A batch selects its one provably newest tip, if there is one. +- Known descendant of the current head: advance automatically. +- Same member, older member, sibling, unrelated history, or missing ancestry: keep the current head. The capture/import still succeeds. +- Explicit selection: choose any complete installed member, including an older one. + +```bash +msb snapshot head worker # Read the current head ID +msb snapshot head worker:experiment # Explicitly choose the other branch +msb snapshot head worker:cp01 # Explicitly rewind +``` + +There is no special `main` branch. The head is a selected snapshot, not a rule for guessing which future branch is preferred. + +### What if two separate operations publish siblings concurrently? + +```text + +---- snapshot A +head: cp02 ---------+ + +---- snapshot B + +A publishes first: head cp02 -> A +B publishes next: B is A's sibling, so head stays A + +Result: both snapshots exist. Only the first head update wins. +``` + +Publication checks and head replacement share a per-group lock. The losing sibling is not discarded or reported as a failed capture. If you want B, select it explicitly. Two captures of the *same* source are serialized and record a parent chain; they are not treated as sibling captures. + +A **single batch containing both siblings** is different: neither argument order nor which file finishes first chooses the head. An existing group retains its head; a new group imports both members with no selected head. Then use `msb snapshot head worker:` to choose. + +## Move a history to another machine + +```bash +# On the source machine: +mkdir -p checkpoints +msb snapshot save worker:cp01 checkpoints/cp01.msb +msb snapshot save worker:cp02 checkpoints/cp02.msb --since worker:cp01 + +# On the destination machine: +msb snapshot load checkpoints/*.msb --group received +msb create --name restored --from-snapshot received --forked +``` + +The shell expands `*.msb` into archive paths. Their order and filenames do not determine ancestry or load order. You can also list them explicitly, in any order: + +```bash +msb snapshot load checkpoints/cp02.msb checkpoints/cp01.msb --group received +``` + +Loading unpacks each supplied archive once, matches omitted disk layers and RAM objects to the available payloads, and validates the reconstructed snapshots before publishing members. It looks in the supplied batch first, then the explicitly selected destination group. This works for disk-only and full incremental archives. No intermediate VM runs. + +The same automatic lookup works when archives arrive separately: + +```bash +msb snapshot load checkpoints/cp01.msb --group received +msb snapshot load checkpoints/cp02.msb --group received +``` + +`--base` is only needed when the missing data is elsewhere, such as `--base another-group:cp01` or `--base /path/to/baseline.msb`. It supplies data; it does not define ancestry or select the group head. An external archive supplied as `--base` must be standalone; include dependent archives in the batch instead. Missing dependencies and conflicting IDs, names, or duplicate labels fail before publishing any incoming members. + +Current development limitation: disk-only captures reassign layer IDs, so `--since` between successive disk-only captures can reject the base. Use standalone disk-only exports for that workflow until capture identity preservation is fixed. Full-checkpoint incremental imports were live-tested successfully; the batch loader also handles dependency-correct disk-only archives. + +`--since` omits disk layers and reusable RAM objects supplied by the explicit base. Loading reconstructs a complete owned snapshot; the target does not depend on replaying earlier VMs. Each archive still includes the target's complete memory map and CPU/device state. The `.msb` archive does not contain a local group's mutable head file: its declared archive head is the import candidate, and the receiving group applies the rules above. + +Loading without `--group` creates one fresh generated group for the whole batch. The CLI prints a digest and installed artifact **path** for each input archive head, in input order. With one archive, the final line remains its installed path. With several archives, the final path is not necessarily the selected group head; use the group selector or `msb snapshot head received` instead. Repeating the same snapshot installs it only once. + +The destination directory is now an explicit `--dest DIR` option, leaving positional arguments for archive paths. Existing snapshot/archive formats are unchanged, including legacy readers. + +Importing an old checkpoint does not rewind an existing group. To deliberately select the imported archive's head: + +```bash +msb snapshot load checkpoints/cp01.msb --group received --set-head +``` + +For a batch, `--set-head` requires one unambiguous tip; it refuses competing tips rather than picking the last argument. Load those members without `--set-head`, then select the one you want. + +Missing historical checkpoints are okay when payload dependencies are complete. But a missing parent may prevent proving a fast-forward. Filling a history hole does not retrospectively select some other retained tip; select that tip explicitly or import it again once its ancestry is known. + +Direct archive capture (`snapshot create --archive`) and direct archive restore still skip installed snapshot directories. `msb branch` still creates a local child without publishing a durable snapshot. Neither operation implicitly moves a group's head; a later explicit capture can join a group using the child's recorded ancestry. diff --git a/justfile b/justfile index 8739d65d2..1ffc1fb09 100644 --- a/justfile +++ b/justfile @@ -218,6 +218,30 @@ build mode="debug": (build-msb mode) _ensure-libkrunfw [windows] build mode="debug": (build-msb mode) _ensure-libkrunfw +# Run snapshot/archive/group and checkpoint logic tests without starting VMs. +test-snapshot: + cargo test -p microsandbox --lib snapshot:: + cargo test -p microsandbox --test snapshot_artifact + cargo test -p microsandbox-runtime --lib checkpoint:: + cargo test -p microsandbox-cli --lib commands::snapshot::tests + {{ if os_family() == "windows" { "python" } else { "python3" } }} -m unittest discover -s scripts/smoke/cli -p test_snapshot_branch.py + +# Run the compact live snapshot/branch smoke. Forward arguments without shell re-parsing. +[unix] +[script("python3")] +[positional-arguments] +test-snapshot-live *args: + import runpy + runpy.run_path("scripts/smoke/cli/snapshot-branch.py", run_name="__main__") + +# Run the same smoke with the native Windows Python launcher. +[windows] +[script("python")] +[positional-arguments] +test-snapshot-live *args: + import runpy + runpy.run_path("scripts/smoke/cli/snapshot-branch.py", run_name="__main__") + # Install msb and libkrunfw to ~/.microsandbox/{bin,lib}/ and configure shell paths. Requires: just build. [linux] install: diff --git a/packages/microsandbox-types/rust/lib/domain.rs b/packages/microsandbox-types/rust/lib/domain.rs index b36c8354d..e5afdc942 100644 --- a/packages/microsandbox-types/rust/lib/domain.rs +++ b/packages/microsandbox-types/rust/lib/domain.rs @@ -782,29 +782,31 @@ pub struct SandboxPolicy { /// Inputs to create a snapshot. /// -/// The snapshot's name is its identity; the artifact directory is -/// `dest_dir.join(name)`, with `dest_dir` defaulting to the snapshots -/// store. Archive movement happens through save/load (the artifact -/// directory is also self-contained and safe to move directly). +/// Installed artifacts live at `dest_dir//`. A friendly name +/// is scoped to the group; it does not change the portable snapshot identity. +/// Save/load moves artifacts between stores without starting a VM. #[derive(Debug, Clone, Serialize, Deserialize)] #[cfg_attr(feature = "ts", derive(ts_rs::TS))] pub struct SnapshotSpec { - /// Snapshot name. Always the artifact directory's basename. + /// Friendly member name within a group; empty selects a generated name. pub name: String, - /// Parent directory to create the artifact in. `None` = the default - /// snapshots directory. + /// Local snapshot group; defaults to the source sandbox's name. + #[serde(default)] + pub group: Option, + + /// Group-store root. `None` selects the default snapshots directory. #[serde(default)] #[cfg_attr(feature = "ts", ts(type = "string | null"))] pub dest_dir: Option, - /// Name of the source sandbox. Must be stopped. + /// Source sandbox. Disk capture accepts running, paused, or stopped sources. pub source_sandbox: String, /// User-supplied labels. pub labels: Vec<(String, String)>, - /// Overwrite an existing artifact at the destination. + /// Overwrite a direct archive destination; installed members remain immutable. pub force: bool, /// Compute and record upper-layer content integrity at creation time. diff --git a/scripts/smoke/cli/branch-ownership.py b/scripts/smoke/cli/branch-ownership.py new file mode 100644 index 000000000..639757b46 --- /dev/null +++ b/scripts/smoke/cli/branch-ownership.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Live pin lifetime and same-name reservation checks for direct branching.""" + +import json +import os +from pathlib import Path +import subprocess + +binary = os.environ["MSB_PATH"] +home = Path(os.environ["MSB_HOME"]) +prefix = f"branch-own-{os.getpid()}" +names = [prefix, prefix + ".child", prefix + ".race"] + + +def call(*args, expected=0): + result = subprocess.run([binary, *args], capture_output=True, text=True, timeout=120) + if expected is not None: + assert result.returncode == expected, result.stderr + return result + + +def evictable(path): + # Same OS primitive used by production eviction. Never unlink or modify live backing. + with path.open("rb") as file: + if os.name == "nt": + import ctypes + from ctypes import wintypes + import msvcrt + + class Overlapped(ctypes.Structure): + _fields_ = [("internal", ctypes.c_size_t), ("internal_high", ctypes.c_size_t), + ("offset", wintypes.DWORD), ("offset_high", wintypes.DWORD), + ("event", wintypes.HANDLE)] + + kernel = ctypes.WinDLL("kernel32", use_last_error=True) + kernel.LockFileEx.argtypes = [wintypes.HANDLE, wintypes.DWORD, wintypes.DWORD, + wintypes.DWORD, wintypes.DWORD, ctypes.POINTER(Overlapped)] + kernel.UnlockFileEx.argtypes = [wintypes.HANDLE, wintypes.DWORD, wintypes.DWORD, + wintypes.DWORD, ctypes.POINTER(Overlapped)] + handle = msvcrt.get_osfhandle(file.fileno()) + overlap = Overlapped() + # Fail-immediately + exclusive, over the same whole-file range as production. + if not kernel.LockFileEx(handle, 3, 0, 0xffffffff, 0xffffffff, ctypes.byref(overlap)): + error = ctypes.get_last_error() + assert error == 33, ctypes.WinError(error) + return False + assert kernel.UnlockFileEx(handle, 0, 0xffffffff, 0xffffffff, ctypes.byref(overlap)) + return True + import fcntl + try: + fcntl.flock(file, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + return False + return True + + +try: + call("create", "alpine", "--name", prefix, "--root-disk", "tmpfs:128M", "--memory", "256M") + cache = home / "cache" / "memory" / "branches" + before = set(cache.glob("*.ram")) + call("branch", prefix, "--name", names[1]) + paths = set(cache.glob("*.ram")) - before + assert len(paths) == 1 + backing = paths.pop() + if os.name != "nt": + assert backing.stat().st_mode & 0o777 == 0o400 + assert not evictable(backing), "source/child pins disappeared" + assert not (home / "sandboxes" / names[1] / ".branch-restore").exists() + attempts = [subprocess.Popen([binary, "branch", prefix, "--name", names[2]], stdout=subprocess.PIPE, stderr=subprocess.PIPE) for _ in range(2)] + statuses = [] + for attempt in attempts: + attempt.communicate(timeout=120) + statuses.append(attempt.returncode) + assert sorted(statuses) == [0, 1], statuses + call("stop", prefix) + assert not evictable(backing), "child depended on the source's pin" + call("exec", names[1], "--", "true") + call("stop", names[1]) + assert evictable(backing), "pin leaked after final VM teardown" + print(json.dumps({"independent_child_pin": "pass", "release_after_teardown": "pass", "same_name_race": "pass", "dot_name": "pass"})) +finally: + for name in reversed(names): + call("stop", name, expected=None) diff --git a/scripts/smoke/cli/branch-timer-progress.py b/scripts/smoke/cli/branch-timer-progress.py new file mode 100644 index 000000000..8cc81cac9 --- /dev/null +++ b/scripts/smoke/cli/branch-timer-progress.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Repeated branching with retained siblings and a timer-driven guest workload. + +Use an isolated MSB_HOME, matching MSB_PATH/MSB_LIBKRUNFW_PATH, and unique +STACK8_PREFIX/STACK8_OUT. STACK8_REPEATS defaults to 100 and STACK8_RETAIN to 8. +Retained immutable RAM cache entries need disk space even after VMs stop. +""" +import json +import os +from pathlib import Path +import subprocess +import time + +binary = os.environ["MSB_PATH"] +out = Path(os.environ["STACK8_OUT"]) +out.mkdir(parents=True, exist_ok=True) +prefix = os.environ["STACK8_PREFIX"] +repeats = int(os.environ.get("STACK8_REPEATS", "100")) +retain = int(os.environ.get("STACK8_RETAIN", "8")) +assert repeats > 0 and retain > 0 +source = prefix + "-source" +names = [source] +live_children = [] +rows = [] + + +def run(label, *args, check=True): + start = time.monotonic() + try: + result = subprocess.run([binary, *args], capture_output=True, text=True, timeout=30) + except subprocess.TimeoutExpired: + rows.append({"case": label, "exit": "timeout"}) + raise + row = {"case": label, "exit": result.returncode, + "ms": round((time.monotonic() - start) * 1000, 2)} + rows.append(row) + (out / (label + ".stdout")).write_text(result.stdout) + (out / (label + ".stderr")).write_text(result.stderr) + if check: + assert result.returncode == 0, (row, result.stderr) + return result.stdout.strip() + + +try: + run("create", "create", "alpine", "--name", source, "--root-disk", "tmpfs:128M", + "--memory", "256M", "--cpus", "2") + # Atomic replacement prevents a concurrent reader mistaking a truncated + # counter file for a stalled timer. A background process survives exec exit. + run("prepare", "exec", source, "--", "sh", "-c", + "sh -c 'i=0; while :; do i=$((i+1)); echo $i > /dev/shm/count.next; " + "mv /dev/shm/count.next /dev/shm/count; sleep 0.02; done' " + ">/tmp/counter.log 2>&1 retain: + run("retire-" + str(index), "stop", live_children.pop(0)) + print(json.dumps({"branch": index, "timer_progress": "pass"}), flush=True) +finally: + # Failed creation must not be followed by exec/start: that would test an + # unintended cold boot instead of the failed restore. Stop is always safe. + for name in reversed(names): + try: + run("cleanup-" + name, "stop", name, check=False) + except Exception as error: + rows.append({"case": "cleanup-" + name, "error": str(error)}) + (out / "results.json").write_text(json.dumps(rows, indent=2)) diff --git a/scripts/smoke/cli/checkpoint-clock.py b/scripts/smoke/cli/checkpoint-clock.py new file mode 100644 index 000000000..ef058aa75 --- /dev/null +++ b/scripts/smoke/cli/checkpoint-clock.py @@ -0,0 +1,72 @@ +"""Cross-platform host runner for the static Linux guest clock fixture. + +Requires MSB_PATH, MSB_HOME, MSB_LIBKRUNFW_PATH, CLOCK_PROBE, CLOCK_OUT, +and a unique CLOCK_PREFIX. Never reuses an existing sandbox or snapshot. +""" +import json +import os +from pathlib import Path +import subprocess +import sys +import time + +binary = os.environ["MSB_PATH"] +out = Path(os.environ["CLOCK_OUT"]) +out.mkdir(parents=True, exist_ok=True) +prefix = os.environ["CLOCK_PREFIX"] +source, child, snapshot = prefix + "-source", prefix + "-child", prefix + "-full" +rows = [] + + +def run(label, *args, check=True, timeout=120): + started = time.perf_counter() + result = subprocess.run([binary, *args], capture_output=True, timeout=timeout) + (out / (label + ".stdout")).write_bytes(result.stdout) + (out / (label + ".stderr")).write_bytes(result.stderr) + row = {"case": label, "ms": round((time.perf_counter() - started) * 1000, 2), "exit": result.returncode} + rows.append(row) + print(json.dumps(row), flush=True) + if check and result.returncode: + raise RuntimeError(f"{label}: {result.stderr.decode(errors='replace')}") + return result + + +try: + run("create", "run", "-d", "-n", source, + "--root-disk", os.environ.get("CLOCK_LAYOUT", "flat:512M"), + "--cpus", os.environ.get("CLOCK_CPUS", "2"), "--memory", "256M", + "alpine", "--", "sh", "-c", + "while [ ! -x /clock-probe ]; do sleep 0.05; done; exec /clock-probe") + run("copy-probe", "copy", os.environ["CLOCK_PROBE"], source + ":/clock-probe") + run("chmod-probe", "exec", source, "--", "chmod", "+x", "/clock-probe") + for attempt in range(30): + if run("ready-" + str(attempt), "exec", source, "--", "test", "-s", "/tmp/clock-records.csv", check=False).returncode == 0: + break + time.sleep(0.05) + else: + raise RuntimeError("guest clock fixture did not start") + run("capture", "snapshot", "create", snapshot, "--from-sandbox", source, "--full", "--info") + if os.environ.get("CLOCK_INCREMENTAL") == "1": + snapshot = prefix + "-next" + run("capture-next", "snapshot", "create", snapshot, "--from-sandbox", source, "--full", "--info") + if os.environ.get("CLOCK_ARCHIVE") == "1": + archive = str(out / "clock.msb") + run("archive", "snapshot", "save", snapshot, archive) + snapshot = archive + run("stop-source", "stop", source) + time.sleep(float(os.environ.get("CLOCK_DELAY", "8"))) + (out / "restore-start.ns").write_text(str(time.time_ns())) + run("restore", "create", "-n", child, "--from-snapshot", snapshot, + *(["--forked"] if os.environ.get("CLOCK_FORKED") == "1" else []), "--info") + (out / "restore-end.ns").write_text(str(time.time_ns())) + time.sleep(6) + records = run("records", "exec", child, "--", "cat", "/tmp/clock-records.csv") + (out / "records.csv").write_bytes(records.stdout) + subprocess.run([sys.executable, str(Path(__file__).with_name("checkpoint-clock-analyze.py")), str(out)], check=True) +finally: + for name in (child, source): + try: + run("cleanup-" + name, "stop", name, check=False, timeout=20) + except Exception as error: + rows.append({"case": "cleanup-" + name, "error": str(error)}) + (out / "results.json").write_text(json.dumps(rows, indent=2)) diff --git a/scripts/smoke/cli/checkpoint-cpu-probe.rs b/scripts/smoke/cli/checkpoint-cpu-probe.rs new file mode 100644 index 000000000..b8aaf1f46 --- /dev/null +++ b/scripts/smoke/cli/checkpoint-cpu-probe.rs @@ -0,0 +1,34 @@ +//! Static Linux guest fixture: prove execution and timer wakeups on a chosen CPU. + +use std::fs; +use std::thread; +use std::time::{Duration, Instant}; + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +unsafe extern "C" { + fn sched_setaffinity(pid: i32, size: usize, mask: *const u64) -> i32; + fn sched_getcpu() -> i32; +} + +fn main() -> std::io::Result<()> { + let cpu: usize = std::env::args().nth(1).expect("CPU index").parse().unwrap(); + assert!(cpu < 64); + let mask = 1_u64 << cpu; + // A successful pin must be followed by observable execution on that CPU. + if unsafe { sched_setaffinity(0, 8, &mask) } != 0 { + return Err(std::io::Error::last_os_error()); + } + let start = Instant::now(); + for _ in 0..10 { + assert_eq!(unsafe { sched_getcpu() }, cpu as i32); + thread::sleep(Duration::from_millis(20)); + } + assert_eq!(unsafe { sched_getcpu() }, cpu as i32); + let marker = fs::read_to_string("/dev/shm/cow-marker")?; + assert_eq!(marker.trim(), "captured"); + println!("cpu={cpu} wakeups=10 elapsed_ms={}", start.elapsed().as_millis()); + Ok(()) +} diff --git a/scripts/smoke/cli/checkpoint-cpu-state.py b/scripts/smoke/cli/checkpoint-cpu-state.py new file mode 100644 index 000000000..993bcb3a0 --- /dev/null +++ b/scripts/smoke/cli/checkpoint-cpu-state.py @@ -0,0 +1,55 @@ +"""Live two-vCPU restore with CPU1 intentionally offline, then onlined again. + +Set MSB_PATH, MSB_HOME, MSB_LIBKRUNFW_PATH, CPU_PROBE (guest-architecture +checkpoint CPU fixture), CPU_PREFIX (unique), and CPU_OUT. This requires a +guest with CPU hotplug enabled; unsupported hotplug is a failure, not a pass. +""" +import json +import os +from pathlib import Path +import subprocess +import time + +binary = os.environ["MSB_PATH"] +out = Path(os.environ["CPU_OUT"]) +out.mkdir(parents=True, exist_ok=True) +prefix = os.environ["CPU_PREFIX"] +source, child = prefix + "-source", prefix + "-child" +rows = [] + + +def run(label, *args, check=True): + started = time.perf_counter() + result = subprocess.run([binary, *args], capture_output=True, timeout=90) + (out / (label + ".stdout")).write_bytes(result.stdout) + (out / (label + ".stderr")).write_bytes(result.stderr) + row = {"case": label, "ms": round((time.perf_counter() - started) * 1000, 2), "exit": result.returncode} + rows.append(row) + print(json.dumps(row), flush=True) + if check and result.returncode: + raise RuntimeError(f"{label}: {result.stderr.decode(errors='replace')}") + return result.stdout.strip() + + +try: + run("create", "create", "alpine", "-n", source, "--root-disk", "flat:512M", "--memory", "256M", "--cpus", "2") + run("marker", "exec", source, "--", "sh", "-c", "echo captured > /dev/shm/cow-marker") + run("copy-probe", "copy", os.environ["CPU_PROBE"], source + ":/cpu-probe") + run("chmod", "exec", source, "--", "chmod", "+x", "/cpu-probe") + run("cpu1-before", "exec", source, "--", "/cpu-probe", "1") + run("offline", "exec", source, "--", "sh", "-c", "echo 0 > /sys/devices/system/cpu/cpu1/online") + assert run("offline-before", "exec", source, "--", "cat", "/sys/devices/system/cpu/cpu1/online") == b"0" + run("capture", "snapshot", "create", prefix + "-full", "--from-sandbox", source, "--full", "--info") + run("restore", "create", "-n", child, "--from-snapshot", prefix + "-full", + *(["--forked"] if os.environ.get("CPU_FORKED") == "1" else []), "--info") + assert run("offline-after", "exec", child, "--", "cat", "/sys/devices/system/cpu/cpu1/online") == b"0" + run("cpu0-restored", "exec", child, "--", "/cpu-probe", "0") + run("online", "exec", child, "--", "sh", "-c", "echo 1 > /sys/devices/system/cpu/cpu1/online") + run("cpu1-restored", "exec", child, "--", "/cpu-probe", "1") +finally: + for name in (child, source): + try: + run("cleanup-" + name, "stop", name, check=False) + except Exception as error: + rows.append({"case": "cleanup-" + name, "error": str(error)}) + (out / "results.json").write_text(json.dumps(rows, indent=2)) diff --git a/scripts/smoke/cli/cow-memory-lifecycle.py b/scripts/smoke/cli/cow-memory-lifecycle.py new file mode 100644 index 000000000..a596b91f5 --- /dev/null +++ b/scripts/smoke/cli/cow-memory-lifecycle.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""Isolated #8 live smoke matrix; every started sandbox is stopped in finally.""" +import json +import os +from pathlib import Path +import subprocess +import time + +binary = os.environ["MSB_PATH"] +# Match CI's public mirror; callers may select an explicit local fixture instead. +image = os.environ.get("MSB_TEST_IMAGE", "mirror.gcr.io/library/alpine:latest") +root = Path(os.environ["STACK8_OUT"]) +root.mkdir(parents=True, exist_ok=True) +prefix = os.environ.get("STACK8_PREFIX", "cow8") +mode = os.environ.get("STACK8_MODE", "forked") +assert mode in ("forked", "eager") +restore_flags = ["--forked"] if mode == "forked" else [] +layout = os.environ.get("STACK8_LAYOUT", "flat:512M") +resize = os.environ.get("STACK8_LIVE_RESIZE") == "1" +rows = [] +names = [] + +def run(label, *args, expected=0, timeout=120): + started = time.perf_counter() + try: + result = subprocess.run([binary, *args], text=True, capture_output=True, timeout=timeout) + except subprocess.TimeoutExpired as error: + # TimeoutExpired can carry bytes even when text=True. Preserve the failed + # command in the evidence instead of recording only subsequent cleanup. + for stream in ("stdout", "stderr"): + captured = getattr(error, stream) or b"" + if isinstance(captured, bytes): + captured = captured.decode("utf-8", errors="replace") + (root / (label + "." + stream)).write_text(captured) + row = {"case": label, "ms": round((time.perf_counter() - started) * 1000, 2), + "exit": "timeout", "timeout_seconds": timeout} + rows.append(row) + print(json.dumps(row), flush=True) + raise + elapsed = (time.perf_counter() - started) * 1000 + (root / (label + ".stdout")).write_text(result.stdout) + (root / (label + ".stderr")).write_text(result.stderr) + row = {"case": label, "ms": round(elapsed, 2), "exit": result.returncode} + rows.append(row) + print(json.dumps(row), flush=True) + if expected is not None and result.returncode != expected: + raise RuntimeError(f"{label}: {result.stderr[-3000:]}") + return result + +try: + refused = prefix + "-forked-boot" + result = run("forked-boot-rejected", "create", image, "-n", refused, + "--forked", expected=None) + assert result.returncode != 0, "forked must require captured RAM" + source = prefix + "-source" + names.append(source) + run("fresh-" + mode, "run", "-d", "-n", source, + "--root-disk", layout, "--memory", "256M", "--cpus", "2", + *(["--max-memory", "512M"] if resize else []), image, + "--", "sh", "-c", "mkdir -p /dev/shm; echo captured > /dev/shm/cow-marker; i=0; while :; do echo $i > /tmp/cow-counter; i=$((i+1)); sleep 0.05; done") + # Detached launch acknowledges the runtime, not the application's first write. + for attempt in range(30): + ready = run("application-ready-" + str(attempt), "exec", source, "--", "test", "-s", "/dev/shm/cow-marker", expected=None) + if ready.returncode == 0: + break + time.sleep(0.1) + else: + raise RuntimeError("application did not initialize its marker") + run("marker-source", "exec", source, "--", "cat", "/dev/shm/cow-marker") + boot_id = run("boot-id-before", "exec", source, "--", "cat", "/proc/sys/kernel/random/boot_id").stdout.strip() + process = run("process-before", "exec", source, "--", "sh", "-c", "for p in /proc/[0-9]*/cmdline; do tr '\\0' ' ' < $p; echo; done").stdout + assert "cow-counter" in process + if resize: + baseline = int(run("memory-baseline", "exec", source, "--", "sh", "-c", + "awk '/MemTotal/ {print $2}' /proc/meminfo").stdout.strip()) + for step, target in enumerate((384, 256, 512, 256)): + run(f"memory-target-{step}", "modify", source, "--memory", f"{target}M", "--format", "json") + deadline = time.monotonic() + 30 + sample = 0 + while True: + observed = int(run(f"memory-convergence-{step}-{sample}", "exec", source, + "--", "sh", "-c", "awk '/MemTotal/ {print $2}' /proc/meminfo").stdout.strip()) + # Hotplug metadata consumes some newly onlined pages. Check actual guest + # capacity within 4 MiB, not just an accepted host target/configuration. + if abs(observed - (baseline + (target - 256) * 1024)) <= 4096: + break + assert time.monotonic() < deadline, f"memory target {target} did not converge: {observed} KiB" + sample += 1 + time.sleep(0.1) + assert run(f"memory-marker-{step}", "exec", source, "--", "cat", "/dev/shm/cow-marker").stdout.strip() == "captured" + snap = prefix + "-full" + run("first-full", "snapshot", "create", snap, "--from-sandbox", source, "--full", "--info") + run("pause", "pause", source) + run("pause-idempotent", "pause", source) + inspected = run("paused-inspect", "inspect", source, "--format", "json") + assert json.loads(inspected.stdout)["status"] == "Paused" + refusal = run("paused-exec", "exec", source, "--", "true", expected=None, timeout=10) + assert refusal.returncode != 0, "paused exec must fail promptly" + run("paused-full-1", "snapshot", "create", prefix + "-paused1", "--from-sandbox", source, "--full", "--info") + run("paused-full-2", "snapshot", "create", prefix + "-paused2", "--from-sandbox", source, "--full", "--info") + time.sleep(float(os.environ.get("STACK8_PAUSE_SECONDS", "5"))) + run("resume", "resume", source) + run("resume-idempotent", "resume", source) + assert run("boot-id-after", "exec", source, "--", "cat", "/proc/sys/kernel/random/boot_id").stdout.strip() == boot_id + guest_time = run("wall-clock-after", "exec", source, "--", "date", "+%s").stdout.strip() + assert abs(time.time() - int(guest_time)) < 3, f"guest wall clock stale: {guest_time}" + first_counter = run("counter-after", "exec", source, "--", "cat", "/tmp/cow-counter").stdout.strip() + time.sleep(0.2) + next_counter = run("counter-progress", "exec", source, "--", "cat", "/tmp/cow-counter").stdout.strip() + assert int(next_counter) > int(first_counter), "original workload must continue after resume" + run("marker-after-resume", "exec", source, "--", "cat", "/dev/shm/cow-marker") + for suffix in ("a", "b"): + child = prefix + "-" + suffix + names.append(child) + run("restore-" + suffix, "create", "-n", child, "--from-snapshot", snap, + *restore_flags, "--info") + result = run("marker-" + suffix, "exec", child, "--", "cat", "/dev/shm/cow-marker") + assert result.stdout.strip() == "captured" + run("mutate-a", "exec", prefix + "-a", "--", "sh", "-c", "echo private-a > /dev/shm/cow-marker") + assert run("isolation-b", "exec", prefix + "-b", "--", "cat", "/dev/shm/cow-marker").stdout.strip() == "captured" + assert run("isolation-source", "exec", source, "--", "cat", "/dev/shm/cow-marker").stdout.strip() == "captured" + # A restored child remains a normal capture source; no creation-time memory opt-in exists. + child_snapshot = prefix + "-child-full" + run("capture-restored-child", "snapshot", "create", child_snapshot, + "--from-sandbox", prefix + "-a", "--full", "--info") + grandchild = prefix + "-grandchild" + names.append(grandchild) + run("restore-grandchild", "create", "-n", grandchild, "--from-snapshot", child_snapshot, + *restore_flags, "--info") + assert run("grandchild-marker", "exec", grandchild, "--", "cat", "/dev/shm/cow-marker").stdout.strip() == "private-a" + archive = str(root / "direct.msb") + run("direct-full", "snapshot", "create", prefix + "-direct", "--from-sandbox", source, + "--full", "--archive", archive, "--info") + child = prefix + "-archive" + names.append(child) + run("direct-restore", "create", "-n", child, "--from-snapshot", archive, + *restore_flags, "--info") + if os.environ.get("STACK8_KEEP_ARCHIVE") != "1": + Path(archive).unlink() + assert run("archive-child-exec", "exec", child, "--", "cat", "/dev/shm/cow-marker").stdout.strip() == "captured" + run("pause-for-stop", "pause", source) + run("stop-paused", "stop", source, timeout=20) + disk_snapshot = prefix + "-disk" + run("stopped-disk-capture", "snapshot", "create", disk_snapshot, "--from-sandbox", source) + disk_archive = str(root / "disk.msb") + run("disk-archive", "snapshot", "save", disk_snapshot, disk_archive) + for label, snapshot in (("installed", disk_snapshot), ("archive", disk_archive)): + refused_name = prefix + "-refused-" + label + names.append(refused_name) + result = run("forked-disk-" + label, "create", "-n", refused_name, + "--from-snapshot", snapshot, "--forked", expected=None) + assert result.returncode != 0 and "forked requires a full snapshot" in result.stderr + inspected = run("refused-inspect-" + label, "inspect", refused_name, "--format", "json", expected=None) + assert inspected.returncode != 0, "invalid restore must not publish a sandbox row" + assert run("child-after-source-stop", "exec", prefix + "-a", "--", "cat", "/dev/shm/cow-marker").stdout.strip() == "private-a" +finally: + for name in reversed(names): + try: + run("cleanup-" + name, "stop", name, expected=None, timeout=20) + except Exception as error: + rows.append({"case": "cleanup-" + name, "error": str(error)}) + (root / "results.json").write_text(json.dumps(rows, indent=2)) diff --git a/scripts/smoke/cli/direct-branch.py b/scripts/smoke/cli/direct-branch.py new file mode 100644 index 000000000..7b7ed7204 --- /dev/null +++ b/scripts/smoke/cli/direct-branch.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Opt-in direct branch invariants and release timing; stops every test VM.""" + +import json +import os +from pathlib import Path +import subprocess +import time + +binary = os.environ["MSB_PATH"] +# Match CI's public mirror; callers may select an explicit local fixture instead. +image = os.environ.get("MSB_TEST_IMAGE", "mirror.gcr.io/library/alpine:latest") +root = Path(os.environ["STACK8_OUT"]) +root.mkdir(parents=True, exist_ok=True) +prefix = os.environ.get("STACK8_PREFIX", f"branch8-{os.getpid()}") +layout = os.environ.get("STACK8_LAYOUT", "flat:512M") +rows = [] +names = [] + + +def run(label, *args, expected=0): + started = time.perf_counter() + result = subprocess.run([binary, *args], text=True, capture_output=True, timeout=120) + elapsed = (time.perf_counter() - started) * 1000 + (root / f"{label}.stdout").write_text(result.stdout) + (root / f"{label}.stderr").write_text(result.stderr) + row = {"case": label, "ms": round(elapsed, 2), "exit": result.returncode} + rows.append(row) + print(json.dumps(row), flush=True) + if expected is not None and result.returncode != expected: + raise RuntimeError(f"{label}: {result.stderr[-4000:]}") + return result + + +def exec_guest(name, script, label): + return run(label, "exec", name, "--", "sh", "-c", script).stdout.strip() + + +def branch(source, child, label): + names.append(child) + run(label, "branch", source, "--name", child) + assert exec_guest(child, "cat /dev/shm/branch-marker", label + "-ready") == "source" + + +def capture(source, member, label): + result = run(label, "snapshot", "create", member, "--from-sandbox", source, "--full") + # A member's bare alias no longer identifies its installed group; use the returned path. + path = Path(result.stdout.strip().splitlines()[-1]) + assert (path / "snapshot.json").is_file(), path + return str(path) + + +def benchmark(source): + # One source and at most one measured child: do not let accumulating VMs distort later + # samples. Each CLI return includes activation; first guest command is recorded separately. + for i in range(10): + if os.environ.get("STACK8_BENCH_COMPACT") == "1" and i > 1: + run(f"setup-compact-branch-{i}", "modify", source, "--compact", "--format", "json") + child = prefix + f"-bench-{i}" + branch(source, child, f"branch-{i}") + run(f"stop-branch-{i}", "stop", child) + saved = prefix + "-warm" + if os.environ.get("STACK8_BENCH_COMPACT") == "1": + run("setup-compact-snapshot", "modify", source, "--compact", "--format", "json") + saved_path = capture(source, saved, "capture-warm-source") + for mode in ("forked", "eager"): + for i in range(8): + child = prefix + f"-{mode}-{i}" + names.append(child) + run(f"{mode}-restore-{i}", "create", "--name", child, "--from-snapshot", saved_path, + *(["--forked"] if mode == "forked" else [])) + assert exec_guest(child, "cat /dev/shm/branch-marker", f"{mode}-ready-{i}") == "source" + run(f"stop-{mode}-{i}", "stop", child) + for i in range(5): + if os.environ.get("STACK8_BENCH_COMPACT") == "1": + run(f"setup-compact-pipeline-{i}", "modify", source, "--compact", "--format", "json") + saved = prefix + f"-full-{i}" + child = prefix + f"-durable-{i}" + names.append(child) + saved_path = capture(source, saved, f"pipeline-capture-{i}") + run(f"pipeline-restore-{i}", "create", "--name", child, "--from-snapshot", saved_path, "--forked") + assert exec_guest(child, "cat /dev/shm/branch-marker", f"pipeline-ready-{i}") == "source" + run(f"stop-pipeline-{i}", "stop", child) + + +try: + source = prefix + "-source" + names.append(source) + run("boot", "create", image, "--name", source, "--root-disk", layout, + "--memory", "256M", "--cpus", "2") + exec_guest(source, "echo source > /dev/shm/branch-marker; echo source > /disk-marker; sh -c 'i=0; while :; do i=$((i+1)); echo $i > /dev/shm/branch-counter; sleep 0.02; done' >/tmp/branch-counter.log 2>&1 /dev/shm/branch-marker; echo child > /disk-marker", "mutate-child") + assert exec_guest(source, "cat /dev/shm/branch-marker; cat /disk-marker", "source-isolation") == "source\nsource" + assert exec_guest(prefix + "-repeat-0", "cat /dev/shm/branch-marker; cat /disk-marker", "sibling-isolation") == "source\nsource" + # Branch a branch after private writes; capturing its original file would lose these writes. + grandchild = prefix + "-grandchild" + names.append(grandchild) + run("branch-child", "branch", child, "--name", grandchild) + assert exec_guest(grandchild, "cat /dev/shm/branch-marker; cat /disk-marker", "grandchild-private-writes") == "child\nchild" + counter = int(exec_guest(grandchild, "cat /dev/shm/branch-counter", "counter-before")) + time.sleep(0.1) + assert int(exec_guest(grandchild, "cat /dev/shm/branch-counter", "counter-after")) > counter + run("pause-source", "pause", source) + branch(source, prefix + "-paused", "paused-branch") + rejected = run("source-still-paused", "exec", source, "--", "true", expected=None) + assert rejected.returncode != 0 + run("resume-source", "resume", source) + # Compare durable capture+forked-child against the same source and readiness endpoint. + for i in range(3): + snap = prefix + f"-saved-{i}" + snap_path = capture(source, snap, f"full-capture-{i}") + name = prefix + f"-restored-{i}" + names.append(name) + run(f"forked-restore-{i}", "create", "--name", name, "--from-snapshot", snap_path, "--forked") + assert exec_guest(name, "cat /dev/shm/branch-marker", f"restore-ready-{i}") == "source" + if os.environ.get("STACK8_MAINTENANCE") == "1" and not layout.startswith("tmpfs"): + run("grow-source", "modify", source, "--root-disk", "768M", "--format", "json") + run("compact-source", "modify", source, "--compact", "--format", "json") + branch(source, prefix + "-after-grow", "branch-after-grow") + run("stop-source", "stop", source) + run("stop-child", "stop", child) + assert exec_guest(grandchild, "cat /dev/shm/branch-marker; cat /disk-marker", "survives-source-stop") == "child\nchild" +finally: + for name in reversed(names): + run("cleanup-" + name, "stop", name, expected=None) + (root / "results.json").write_text(json.dumps(rows, indent=2)) diff --git a/scripts/smoke/cli/dirty-memory-checkpoint.py b/scripts/smoke/cli/dirty-memory-checkpoint.py new file mode 100644 index 000000000..de68a0ba9 --- /dev/null +++ b/scripts/smoke/cli/dirty-memory-checkpoint.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +"""Live no-sync checkpoint qualification in an isolated MSB_HOME. + +Requires a matching release runtime/guest agent. Checks dirty block-backed page +cache, shared/private mmap, heap, tmpfs, inherited incremental memory, and the +separate crash-consistent disk-only contract. It does not benchmark throughput. +""" +import argparse +import json +import os +from pathlib import Path +import subprocess +import tempfile +import time + + +WORKER = r''' +import json, mmap, os, time, uuid +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path + +# Make the dirty-memory case deliberate: normal background writeback must not +# turn this into a test that passes only because everything reached disk first. +Path('/proc/sys/vm/dirty_writeback_centisecs').write_text('0') +Path('/proc/sys/vm/dirty_expire_centisecs').write_text('600000') +Path('/proc/sys/vm/dirty_ratio').write_text('90') +Path('/proc/sys/vm/dirty_background_ratio').write_text('80') +Path('/persisted').write_bytes(b'persisted-before-checkpoint') +fd = os.open('/persisted', os.O_RDONLY); os.fsync(fd); os.close(fd) +fd = os.open('/', os.O_RDONLY); os.fsync(fd); os.close(fd) +size = 64 * 1024 * 1024 +fd = os.open('/dirty-cache', os.O_RDWR | os.O_CREAT | os.O_TRUNC, 0o600) +os.ftruncate(fd, size) +shared = mmap.mmap(fd, size, access=mmap.ACCESS_WRITE) +shared[:] = b'A' * size +private = mmap.mmap(fd, size, access=mmap.ACCESS_COPY) +private[:8] = b'private0' +heap = bytearray(b'heap0000') +Path('/dev/shm/latch-marker').write_bytes(b'tmpfs000') +nonce = str(uuid.uuid4()) +class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): pass + def do_GET(self): + if self.path == '/mutate': + shared[:8] = b'shared01'; private[:8] = b'private1'; heap[:] = b'heap0001' + Path('/dev/shm/latch-marker').write_bytes(b'tmpfs001') + dirty = next(int(x.split()[1]) for x in Path('/proc/meminfo').read_text().splitlines() if x.startswith('Dirty:')) + # Poll only the marker: copying 64 MiB per request dirties unrelated heap pages + # and can turn this sparse mutation check into a legitimate dense full capture. + with open('/dirty-cache', 'rb') as disk_file: + disk_cache = disk_file.read(8).decode() + body = json.dumps(dict(nonce=nonce, heap=heap.decode(), shared=shared[:8].decode(), + private=private[:8].decode(), tmpfs=Path('/dev/shm/latch-marker').read_text(), + disk_cache=disk_cache, dirty_kib=dirty, + clock=time.time())).encode() + self.send_response(200); self.send_header('Content-Length', str(len(body))); self.end_headers(); self.wfile.write(body) +HTTPServer(('0.0.0.0', 8080), Handler).serve_forever() +''' + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--binary', required=True) + parser.add_argument('--output', required=True, type=Path) + parser.add_argument('--layout', choices=['flat', 'layered'], required=True) + parser.add_argument('--home-parent', type=Path, help='Filesystem for the isolated sandbox home') + args = parser.parse_args() + args.output.mkdir(parents=True, exist_ok=False) + parent = args.home_parent or Path('/private/tmp' if os.uname().sysname == 'Darwin' else '/tmp') + home = Path(tempfile.mkdtemp(prefix='dirty-', dir=parent)) + env = dict(os.environ, MSB_HOME=str(home), MSB_BACKEND='local') + names, rows = [], [] + active = 'source' + report = dict(home=str(home), binary=args.binary, layout=args.layout, rows=rows, status='running') + + def run(label, *command): + start = time.monotonic() + result = subprocess.run([args.binary, *command], env=env, capture_output=True, text=True, timeout=180) + row = dict(case=label, seconds=time.monotonic()-start, returncode=result.returncode, + stdout=result.stdout, stderr=result.stderr) + rows.append(row) + (args.output / 'report.json').write_text(json.dumps(report, indent=2)) + print(json.dumps({k: row[k] for k in ('case', 'seconds', 'returncode')}), flush=True) + assert result.returncode == 0, row + return result.stdout + + def state(path='/'): + # Use the guest loopback endpoint. A branch must not inherit host port + # publications, and this test is about memory, not ingress reconfiguration. + return json.loads(run('state-' + active, 'exec', active, '--', 'python3', '-c', + f"import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080{path}').read().decode())")) + + def matches(expected, actual=None): + actual = state() if actual is None else actual + for key in ('nonce', 'heap', 'shared', 'private', 'tmpfs', 'disk_cache'): + assert actual[key] == expected[key], (key, actual, expected) + assert abs(actual['clock'] - time.time()) < 3, actual + return actual + + def stop(name): + run('stop-' + name, 'stop', name) + + try: + source = 'source'; names.append(source) + run('create', 'run', '-d', '--name', source, '--memory', '512M', '--cpus', '2', + '--root-disk', 'flat:1G' if args.layout == 'flat' else '1G', + 'python:3.13-alpine3.22', '--', 'python3', '-u', '-c', WORKER) + deadline = time.monotonic() + 30 + while True: + try: initial = state(); break + except Exception: + if time.monotonic() >= deadline: raise + time.sleep(.1) + assert initial['dirty_kib'] >= 32 * 1024, initial + report['initial'] = initial + run('full-dirty', 'snapshot', 'create', 'dirty-full', '--from-sandbox', source, '--full') + matches(initial) + names.append('running-branch') + run('branch-dirty', 'branch', source, '--name', 'running-branch') + matches(initial, json.loads(run('branch-state', 'exec', 'running-branch', '--', 'python3', '-c', + "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080').read().decode())"))) + branch_changed = json.loads(run('mutate-branch', 'exec', 'running-branch', '--', 'python3', '-c', + "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/mutate').read().decode())")) + matches(initial) + names.append('branch-grandchild') + run('branch-of-branch', 'branch', 'running-branch', '--name', 'branch-grandchild') + matches(branch_changed, json.loads(run('branch-grandchild-state', 'exec', 'branch-grandchild', '--', 'python3', '-c', + "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080').read().decode())"))) + stop('branch-grandchild') + stop('running-branch'); matches(initial) + run('pause', 'pause', source) + for suffix in ('one', 'two'): + run('capture-paused-' + suffix, 'snapshot', 'create', 'paused-' + suffix, '--from-sandbox', source, '--full') + assert json.loads(run('inspect-paused-' + suffix, 'inspect', source, '--format', 'json'))['status'] == 'Paused' + names.append('paused-branch') + run('branch-paused-dirty', 'branch', source, '--name', 'paused-branch') + matches(initial, json.loads(run('paused-branch-state', 'exec', 'paused-branch', '--', 'python3', '-c', + "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080').read().decode())"))) + assert json.loads(run('source-still-paused', 'inspect', source, '--format', 'json'))['status'] == 'Paused' + stop('paused-branch') + run('resume', 'resume', source); matches(initial) + # Source shutdown also proves restored state does not rely on live source RAM. + stop(source) + for mode in ('eager', 'forked'): + child = mode; names.append(child) + run('restore-' + mode, 'create', '--name', child, '--from-snapshot', 'source:dirty-full', + *(['--forked'] if mode == 'forked' else [])) + active = child + matches(initial) + if mode == 'forked': + # A restored child starts a fresh dirty-tracking baseline while retaining + # snapshot ancestry. Capture before mutating, then verify an incremental cut. + run('child-baseline', 'snapshot', 'create', 'child-baseline', '--from-sandbox', child, '--full') + changed = state('/mutate') + assert changed['private'] == 'private1' + captured = run('incremental-dirty', 'snapshot', 'create', 'dirty-incremental', '--from-sandbox', child, '--full') + # Capture returns the exact installed member path, independent of its alias. + checkpoint = Path(captured.strip().splitlines()[-1]) / 'checkpoint' + descriptor = json.loads((checkpoint / 'checkpoint.json').read_text()) + algorithm, digest = descriptor['memory'].split(':', 1) + memory = json.loads((checkpoint / 'objects' / algorithm / digest[:2] / digest).read_text()) + report['incremental_capture_mode'] = memory['capture_mode'] + assert memory['capture_mode'] == 'incremental', memory['capture_mode'] + matches(changed) + stop(child) + names.append('grandchild') + run('restore-incremental', 'create', '--name', 'grandchild', '--from-snapshot', 'forked:dirty-incremental', '--forked') + active = 'grandchild' + matches(changed); stop('grandchild') + names.append('disk-only') + run('disk-only', 'create', '--name', 'disk-only', '--from-snapshot', 'source:dirty-full', '--disk-only') + assert run('persisted-disk-data', 'exec', 'disk-only', '--', 'cat', '/persisted').strip() == 'persisted-before-checkpoint' + run('no-tmpfs-in-disk-view', 'exec', 'disk-only', '--', 'test', '!', '-e', '/dev/shm/latch-marker') + # Unsynced disk bytes are deliberately not asserted either present or absent. + report['status'] = 'passed' + except Exception as error: + report.update(status='failed', error=repr(error)); raise + finally: + errors = [] + for name in reversed(names): + result = subprocess.run([args.binary, 'stop', name], env=env, capture_output=True, text=True, timeout=30) + if result.returncode and 'already stopped' not in result.stderr.lower(): + errors.append(dict(name=name, error=result.stderr)) + report['cleanup_errors'] = errors + (args.output / 'report.json').write_text(json.dumps(report, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/scripts/smoke/cli/disk-compaction-export.sh b/scripts/smoke/cli/disk-compaction-export.sh index 0381f11d4..a03b29908 100644 --- a/scripts/smoke/cli/disk-compaction-export.sh +++ b/scripts/smoke/cli/disk-compaction-export.sh @@ -46,10 +46,10 @@ for layout in managed flat; do measure "$layout-save-last" msb snapshot save "$name-4" "$QUAL_ROOT/$layout-last.tar" --last-layers 2 --plain-tar measure "$layout-save-base" msb snapshot save "$name-2" "$QUAL_ROOT/$layout-base.tar.zst" measure "$layout-invalid-last-zero" refuse msb snapshot save "$name-4" "$QUAL_ROOT/invalid.tar" --last-layers 0 - measure "$layout-missing-base" refuse msb snapshot load "$QUAL_ROOT/$layout-delta.tar.zst" "$QUAL_ROOT/$layout-missing" - measure "$layout-wrong-base" refuse msb snapshot load "$QUAL_ROOT/$layout-delta.tar.zst" "$QUAL_ROOT/$layout-wrong" --base "$name-1" - measure "$layout-load-delta" msb snapshot load "$QUAL_ROOT/$layout-delta.tar.zst" "$QUAL_ROOT/$layout-import" --base "$name-2" - measure "$layout-load-base-archive" msb snapshot load "$QUAL_ROOT/$layout-last.tar" "$QUAL_ROOT/$layout-base-import" --base "$QUAL_ROOT/$layout-base.tar.zst" + measure "$layout-missing-base" refuse msb snapshot load "$QUAL_ROOT/$layout-delta.tar.zst" --dest "$QUAL_ROOT/$layout-missing" + measure "$layout-wrong-base" refuse msb snapshot load "$QUAL_ROOT/$layout-delta.tar.zst" --dest "$QUAL_ROOT/$layout-wrong" --base "$name-1" + measure "$layout-load-delta" msb snapshot load "$QUAL_ROOT/$layout-delta.tar.zst" --dest "$QUAL_ROOT/$layout-import" --base "$name-2" + measure "$layout-load-base-archive" msb snapshot load "$QUAL_ROOT/$layout-last.tar" --dest "$QUAL_ROOT/$layout-base-import" --base "$QUAL_ROOT/$layout-base.tar.zst" # A long-running agentd-managed writer remains active across preparation and the switch. msb exec "$name" -- sh -c 'i=0; while [ ! -e /writer-stop ]; do i=$((i+1)); echo "$i" >>/writes; sync; done' >"$QUAL_ROOT/logs/$layout-writer.out" 2>"$QUAL_ROOT/logs/$layout-writer.err" & diff --git a/scripts/smoke/cli/disk-compaction-negative.sh b/scripts/smoke/cli/disk-compaction-negative.sh index 2619e0e80..21ffbbbf6 100644 --- a/scripts/smoke/cli/disk-compaction-negative.sh +++ b/scripts/smoke/cli/disk-compaction-negative.sh @@ -18,9 +18,9 @@ msb stop compact-neg-owned refuse msb modify compact-neg-owned --compact msb snapshot save "$QUAL_SOURCE-4" "$QUAL_ROOT/truncated.tar" --last-layers 2 --plain-tar truncate -s 2048 "$QUAL_ROOT/truncated.tar" -refuse msb snapshot load "$QUAL_ROOT/truncated.tar" "$QUAL_ROOT/truncated-import" --base "$QUAL_SOURCE-2" +refuse msb snapshot load "$QUAL_ROOT/truncated.tar" --dest "$QUAL_ROOT/truncated-import" --base "$QUAL_SOURCE-2" msb snapshot save "$QUAL_SOURCE-4" "$QUAL_ROOT/complete-last.tar" --last-layers 4 --plain-tar -msb snapshot load "$QUAL_ROOT/complete-last.tar" "$QUAL_ROOT/complete-last-import" +msb snapshot load "$QUAL_ROOT/complete-last.tar" --dest "$QUAL_ROOT/complete-last-import" msb snapshot save "$QUAL_SOURCE-4" "$QUAL_ROOT/same.tar" --since "$QUAL_SOURCE-4" --plain-tar -msb snapshot load "$QUAL_ROOT/same.tar" "$QUAL_ROOT/same-import" --base "$QUAL_SOURCE-4" +msb snapshot load "$QUAL_ROOT/same.tar" --dest "$QUAL_ROOT/same-import" --base "$QUAL_SOURCE-4" echo 'tmpfs/user-owned rejection, truncated refusal, all-layer standalone and equal-base export PASS' diff --git a/scripts/smoke/cli/failed-restore.py b/scripts/smoke/cli/failed-restore.py new file mode 100644 index 000000000..5ad94035a --- /dev/null +++ b/scripts/smoke/cli/failed-restore.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Late restore failure must preserve sealed bytes and refuse cold-boot paths. + +Run only with an isolated MSB_HOME: this temporarily blocks its memory cache. +""" +import hashlib +import json +import os +from pathlib import Path +import subprocess +import time + +assert os.environ.get("MSB_TEST_DISPOSABLE_HOME") == "1", "requires an isolated test home" +binary = os.environ["MSB_PATH"] +home = Path(os.environ["MSB_HOME"]) +out = Path(os.environ["STACK8_OUT"]) +out.mkdir(parents=True, exist_ok=True) +prefix = os.environ.get("STACK8_PREFIX", "failed-restore-" + str(os.getpid())) +source, child, healthy = (prefix + suffix for suffix in ("-source", "-failed", "-healthy")) +snapshot = prefix + "-saved" +cache = home / "cache" / "memory" +held = cache.with_name("memory-held-" + prefix) +rows = [] + +def call(label, *args, expected=0): + start = time.monotonic() + result = subprocess.run([binary, *args], capture_output=True, text=True, timeout=120) + rows.append(dict(case=label, exit=result.returncode, ms=(time.monotonic()-start)*1000, + stdout=result.stdout, stderr=result.stderr)) + if expected is not None: + assert result.returncode == expected, rows[-1] + return result + +def layers(): + root = home / "snapshots" / snapshot + paths = sorted(p for p in root.rglob('*') if p.is_file() and p.suffix in ('.raw', '.qcow2', '.ext4')) + assert paths, "fixture must include sealed disk bytes" + result = {} + for path in paths: + with path.open('rb') as file: + digest = hashlib.sha256() + for chunk in iter(lambda: file.read(1024 * 1024), b''): + digest.update(chunk) + result[str(path.relative_to(root))] = digest.hexdigest() + return result + +blocked = False +try: + call("source", "create", "alpine", "--name", source, "--root-disk", os.environ.get("STACK8_LAYOUT", "flat:512M"), "--memory", "256M") + call("marker", "exec", source, "--", "sh", "-c", "echo preserved > /dev/shm/restore-marker; echo disk-preserved > /restore-disk-marker") + call("capture", "snapshot", "create", snapshot, "--from-sandbox", source, "--full") + before = layers() + # Trigger a real host I/O error during memory installation, after child staging + # and DB insertion. No production test-only failure hook is necessary. + assert not held.exists() + if cache.exists(): + cache.rename(held) + cache.write_bytes(b"intentional isolated test obstruction") + blocked = True + failure = call("restore-fails", "create", "--name", child, "--from-snapshot", snapshot, "--forked", expected=None) + assert failure.returncode != 0 + call("failed-row-exists", "inspect", child) + cache.unlink() + blocked = False + if held.exists(): + held.rename(cache) + for label, args in [("start", ["start", child]), ("exec", ["exec", child, "--", "true"]), + ("modify", ["modify", child, "--root-disk", "8G"]), + ("compact", ["modify", child, "--compact"]), + ("snapshot", ["snapshot", "create", prefix+'-invalid', "--from-sandbox", child])]: + refused = call(label + "-refused", *args, expected=None) + assert refused.returncode != 0 and "incomplete restore" in refused.stderr, rows[-1] + assert layers() == before, "failed restore or later lifecycle mutated sealed disk bytes" + call("healthy-restore", "create", "--name", healthy, "--from-snapshot", snapshot, "--forked") + assert call("healthy-marker", "exec", healthy, "--", "cat", "/dev/shm/restore-marker").stdout.strip() == "preserved" + call("healthy-stop", "stop", healthy) + call("healthy-later-start", "start", healthy) + assert call("healthy-disk-marker", "exec", healthy, "--", "cat", "/restore-disk-marker").stdout.strip() == "disk-preserved" + assert layers() == before, "ordinary later startup mutated the original sealed snapshot" + print(json.dumps({"late_failure": "pass", "start_exec_modify_compact_snapshot_refused": "pass", "sealed_bytes": "unchanged", "fresh_restore_and_later_start": "pass"})) +finally: + if blocked: + cache.unlink() + if held.exists(): + held.rename(cache) + for name in (healthy, child, source): + call("cleanup-" + name, "stop", name, expected=None) + (out / "results.json").write_text(json.dumps(rows, indent=2)) diff --git a/scripts/smoke/cli/incremental-full-archive.py b/scripts/smoke/cli/incremental-full-archive.py new file mode 100644 index 000000000..fa7a9d8bb --- /dev/null +++ b/scripts/smoke/cli/incremental-full-archive.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +"""Opt-in full archive dependency chain: real guest RAM/disk, eager/forked restore, cleanup. + +Set MSB_PATH, STACK8_OUT, and optionally STACK8_LAYOUT (512M, flat:512M, tmpfs:128M). +Each run creates isolated source/destination MSB_HOME directories below STACK8_OUT. +""" + +import json +import os +from pathlib import Path +import shutil +import subprocess +import time + +binary = os.environ["MSB_PATH"] +root = Path(os.environ["STACK8_OUT"]) +root.mkdir(parents=True, exist_ok=False) +layout = os.environ.get("STACK8_LAYOUT", "flat:512M") +source_home = root / "source-home" +dest_home = root / "destination-home" +prefix = f"ramdelta-{os.getpid()}" +rows = [] +names = [] + + +def run(label, home, *args, fail=False): + env = dict(os.environ, MSB_HOME=str(home)) + started = time.perf_counter() + result = subprocess.run([binary, *map(str, args)], env=env, capture_output=True, text=True, timeout=180) + elapsed = round((time.perf_counter() - started) * 1000, 2) + (root / f"{label}.stdout").write_text(result.stdout) + (root / f"{label}.stderr").write_text(result.stderr) + rows.append({"case": label, "ms": elapsed, "exit": result.returncode}) + print(json.dumps(rows[-1]), flush=True) + if (result.returncode != 0) != fail: + raise RuntimeError(f"{label}: exit={result.returncode}: {result.stderr[-3000:]}") + return result.stdout.strip() + + +def guest(label, home, name, script): + return run(label, home, "exec", name, "--", "sh", "-ec", script) + + +def inventory(archive): + # System tar decodes plain and zstd input by magic, independently of the SDK loader. + return json.loads(subprocess.check_output(["tar", "-xOf", str(archive), "archive.json"])) + + +def restore(label, snapshot, expected, base=None, forked=False): + name = prefix + "-" + label + names.append((dest_home, name)) + args = ["create", "--name", name, "--from-snapshot", str(snapshot)] + if base: + args += ["--snapshot-base", str(base)] + if forked: + args += ["--forked"] + run(label, dest_home, *args) + actual = guest(label + "-read", dest_home, name, "cat /dev/shm/marker; cat /disk-marker; sha256sum /dev/shm/blob | cut -d' ' -f1") + assert actual == f"{expected}\n{expected}\n{blob_hash}", actual + guest(label + "-write", dest_home, name, "echo child > /dev/shm/marker; echo child > /disk-marker") + return name + + +try: + source = prefix + "-source" + names.append((source_home, source)) + # Reuse only immutable OCI artifacts, never a prior VM or memory backing cache. + seed = os.environ.get("STACK8_SEED_CACHE") + if seed: + for kind in ("layers", "manifests", "fsmeta", "vmdk"): + origin = Path(seed) / kind + if origin.exists(): + shutil.copytree(origin, source_home / "cache" / kind) + run("boot", source_home, "create", "alpine", "--name", source, "--root-disk", layout, "--memory", "256M", "--cpus", "2") + # Image transport is separate from checkpoint dependencies. Independently populate the + # destination's OCI cache so this matrix does not depend on --with-image or source paths. + seed_vm = prefix + "-image-seed" + names.append((dest_home, seed_vm)) + run("prepare-destination-image", dest_home, "create", "alpine", "--name", seed_vm, "--root-disk", layout, "--memory", "256M", "--cpus", "2") + run("stop-image-seed", dest_home, "stop", seed_vm) + blob_hash = guest("prepare", source_home, source, "dd if=/dev/urandom of=/dev/shm/blob bs=1M count=24 2>/dev/null; sha256sum /dev/shm/blob | cut -d' ' -f1") + previous_source = None + installed_base = None + archive_rows = [] + for n in range(1, 13): + guest(f"mutate-{n}", source_home, source, f"echo {n} > /dev/shm/marker; echo {n} > /disk-marker; dd if=/dev/zero of=/dev/shm/zero bs=4096 count=1 2>/dev/null") + if n == 6: + run("pause-source", source_home, "pause", source) + name = f"cp{n:02}" + run(f"capture-{n}", source_home, "snapshot", "create", name, "--from-sandbox", source, "--full") + if n == 6: + run("resume-source", source_home, "resume", source) + artifact = source_home / "snapshots" / name + archive = root / f"cp{n:02}.msb" + args = ["snapshot", "save", artifact, archive] + if previous_source: + args += ["--since", previous_source] + run(f"export-{n}", source_home, *args) + inv = inventory(archive) + omitted_ram = [e for e in inv["entries"] if not e["included"] and e["kind"] == "checkpoint-object"] + if n > 1: + assert omitted_ram, "incremental export did not omit any reusable RAM objects" + assert all(e["kind"] in ("checkpoint-object", "checkpoint-disk-layer") for e in inv["entries"] if not e["included"]) + archive_rows.append({"checkpoint": n, "archive_bytes": archive.stat().st_size, "omitted_ram_objects": len(omitted_ram), "omitted_ram_bytes": sum(e["apparent_size"] for e in omitted_ram)}) + if n == 2: + run("missing-base", dest_home, "snapshot", "load", archive, fail=True) + run("wrong-base", dest_home, "snapshot", "load", archive, "--base", root / "absent", fail=True) + if n == 12: + final_archive, final_artifact = archive, artifact + break + load_args = ["snapshot", "load", archive] + if installed_base: + load_args += ["--base", installed_base] + old_base = installed_base + installed_base = Path(run(f"load-{n}", dest_home, *load_args).splitlines()[-1]) + assert installed_base.parent == dest_home / "snapshots" + if old_base: + run(f"remove-base-{n-1}", dest_home, "snapshot", "remove", old_base) + previous_source = artifact + run("stop-source", source_home, "stop", source) + before = set((dest_home / "snapshots").iterdir()) + eager = restore("direct-eager", final_archive, 12, installed_base) + forked = restore("direct-forked", final_archive, 12, installed_base, True) + assert set((dest_home / "snapshots").iterdir()) == before, "direct restore installed the target snapshot" + final_loaded = Path(run("load-final", dest_home, "snapshot", "load", final_archive, "--base", installed_base).splitlines()[-1]) + run("remove-final-base", dest_home, "snapshot", "remove", installed_base) + # Live children must retain both their captured RAM backing and private disk writes + # after the explicit base disappears. The installed target must remain independent too. + for mode, child in (("eager", eager), ("forked", forked)): + actual = guest("deleted-base-" + mode, dest_home, child, "cat /dev/shm/marker; cat /disk-marker; sha256sum /dev/shm/blob | cut -d' ' -f1") + assert actual == f"child\nchild\n{blob_hash}", actual + run("stop-" + mode, dest_home, "stop", child) + run("verify-final", dest_home, "snapshot", "verify", final_loaded) + for mode in ("eager", "forked"): + child = restore("installed-" + mode, final_loaded, 12, forked=mode == "forked") + run("stop-installed-" + mode, dest_home, "stop", child) + complete = root / "standalone.msb" + run("export-standalone", source_home, "snapshot", "save", final_artifact, complete) + assert all(e["included"] for e in inventory(complete)["entries"]) + archive_rows[-1]["standalone_bytes"] = complete.stat().st_size + (root / "archive-sizes.json").write_text(json.dumps(archive_rows, indent=2)) + print(json.dumps({"pass": True, "layout": layout, "archives": archive_rows}), flush=True) +finally: + # Stop only test-owned names, including children whose creation failed part-way through. + for home, name in reversed(names): + result = subprocess.run([binary, "stop", name], env=dict(os.environ, MSB_HOME=str(home)), capture_output=True, text=True, timeout=30) + (root / ("cleanup-" + name + ".log")).write_text(result.stdout + result.stderr) + (root / "results.json").write_text(json.dumps(rows, indent=2)) diff --git a/scripts/smoke/cli/live-disk-snapshot.py b/scripts/smoke/cli/live-disk-snapshot.py new file mode 100644 index 000000000..e55d1b0b0 --- /dev/null +++ b/scripts/smoke/cli/live-disk-snapshot.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Live disk-only capture on an isolated MSB_HOME with a matching runtime/firmware. + +MSB_PATH, MSB_HOME, STACK8_PREFIX and STACK8_OUT are required. The fixture stops +its own VMs and records whole CLI times, not just the VM pause interval. +""" +import hashlib +import json +import os +from pathlib import Path +import subprocess +import time + +binary = os.environ["MSB_PATH"] +home = Path(os.environ["MSB_HOME"]) +out = Path(os.environ["STACK8_OUT"]) +out.mkdir(parents=True, exist_ok=True) +prefix = os.environ["STACK8_PREFIX"] +rows = [] +names = [] + + +def run(label, *args, ok=True): + start = time.monotonic() + result = subprocess.run([binary, *args], capture_output=True, text=True, timeout=180) + row = dict(case=label, ms=round((time.monotonic() - start) * 1000, 2), + exit=result.returncode, stdout=result.stdout, stderr=result.stderr) + rows.append(row) + (out / "results.json").write_text(json.dumps(rows, indent=2)) + if ok: + assert result.returncode == 0, row + return result + + +def files(root): + return {str(p.relative_to(root)): (p.stat().st_size, p.stat().st_mtime_ns) + for p in root.rglob("*") if p.is_file()} if root.exists() else {} + + +def sealed_hashes(root): + result = {} + for path in root.rglob("*"): + if path.is_file() and path.suffix in (".raw", ".qcow2", ".ext4"): + digest = hashlib.sha256() + with path.open("rb") as file: + for chunk in iter(lambda: file.read(1024 * 1024), b""): + digest.update(chunk) + result[str(path)] = digest.hexdigest() + assert result + return result + + +try: + for layout in ("flat", "managed"): + source = prefix + "-" + layout + names.append(source) + run("create-" + layout, "create", "alpine", "--name", source, + "--root-disk", "flat:512M" if layout == "flat" else "512M", "--memory", "256M") + run("prepare-" + layout, "exec", source, "--", "sh", "-c", + "echo before > /disk-marker; echo ram-only > /dev/shm/ram-marker; sync") + boot = run("boot-" + layout, "exec", source, "--", "cat", "/proc/sys/kernel/random/boot_id").stdout + runtime = home / "sandboxes" / source / "runtime" + memory_before = files(runtime / "checkpoint-store") + cache_before = files(home / "cache" / "memory") + for mode in ("installed", "integrity", "archive", "plain", "paused"): + snap = source + "-" + mode + child = snap + "-child" + names.append(child) + run("reset-" + snap, "exec", source, "--", "sh", "-c", "echo before > /disk-marker; sync") + if mode == "paused": + run("pause-" + layout, "pause", source) + args = ["snapshot", "create", snap, "--from-sandbox", source] + archive = out / (snap + (".tar" if mode == "plain" else ".msb")) + installed_before = set((home / "snapshots").glob("*")) + if mode in ("archive", "plain"): + args += ["--archive", str(archive)] + if mode == "plain": + args += ["--plain-tar"] + if mode == "integrity": + args += ["--integrity"] + run("capture-" + snap, *args) + assert files(runtime / "checkpoint-store") == memory_before, "disk capture touched the RAM/object store" + assert files(home / "cache" / "memory") == cache_before, "disk capture touched the RAM cache" + assert not list((runtime / "checkpoints").iterdir()), "consumed disk staging leaked" + if mode == "paused": + # Public repeated pause must be idempotent, and an exec must not release it. + paused_exec = run("paused-exec-" + layout, "exec", source, "--", "true", ok=False) + assert paused_exec.returncode != 0 + run("resume-" + layout, "resume", source) + assert run("source-boot-" + snap, "exec", source, "--", "cat", "/proc/sys/kernel/random/boot_id").stdout == boot + assert run("source-ram-" + snap, "exec", source, "--", "cat", "/dev/shm/ram-marker").stdout.strip() == "ram-only" + run("diverge-" + snap, "exec", source, "--", "sh", "-c", "echo after > /disk-marker; sync") + if mode in ("archive", "plain"): + assert set((home / "snapshots").glob("*")) == installed_before, "direct archive installed a snapshot" + from_snapshot = str(archive) + else: + artifact = home / "snapshots" / snap + manifest = json.loads((artifact / "snapshot.json").read_text()) + assert manifest["scope"] == "file" and manifest["state"]["kind"] == "file", manifest + assert not (artifact / "checkpoint").exists() + before = sealed_hashes(artifact) + from_snapshot = snap + run("restore-" + snap, "create", "--name", child, "--from-snapshot", from_snapshot) + assert run("child-disk-" + snap, "exec", child, "--", "cat", "/disk-marker").stdout.strip() == "before" + run("child-no-ram-" + snap, "exec", child, "--", "sh", "-c", "test ! -e /dev/shm/ram-marker") + assert run("child-boot-" + snap, "exec", child, "--", "cat", "/proc/sys/kernel/random/boot_id").stdout != boot + run("child-write-" + snap, "exec", child, "--", "sh", "-c", "echo child > /disk-marker; sync") + assert run("source-disk-" + snap, "exec", source, "--", "cat", "/disk-marker").stdout.strip() == "after" + if mode in ("archive", "plain"): + archive.unlink() + else: + assert sealed_hashes(artifact) == before, "source or child mutated sealed layers" + run("remove-" + snap, "snapshot", "remove", snap) + assert run("after-delete-" + snap, "exec", child, "--", "cat", "/disk-marker").stdout.strip() == "child" + run("stop-child-" + snap, "stop", child) + # Exercise the cut while the guest actually submits writes, not only after sync. + # Keep ownership in the guest, as in the branch timer fixture. This does not depend + # on an additional host client's stdin/console lifetime while capture runs. + run("start-writer-" + layout, "exec", source, "--", "sh", "-c", + "sh -c 'i=1; while [ ! -e /stop-counter ]; do echo $i > /counter.tmp; " + "mv /counter.tmp /counter; sync; i=$((i+1)); sleep 0.02; done; " + "touch /counter-done' >/tmp/counter.log 2>&1 = start_counter + run("stop-busy-child-" + layout, "stop", busy_child) + # A later full checkpoint must still work after disk-only generations. + full = source + "-full" + run("full-after-disk-" + layout, "snapshot", "create", full, "--from-sandbox", source, "--full") + full_child = source + "-full-child" + names.append(full_child) + run("full-restore-" + layout, "create", "--name", full_child, "--from-snapshot", full) + assert run("full-ram-" + layout, "exec", full_child, "--", "cat", "/dev/shm/ram-marker").stdout.strip() == "ram-only" + # A disk-only cut between full captures must not consume/advance the RAM baseline. + memory_before = files(runtime / "checkpoint-store") + run("disk-between-full-" + layout, "snapshot", "create", source + "-between", "--from-sandbox", source) + assert files(runtime / "checkpoint-store") == memory_before + run("change-ram-" + layout, "exec", source, "--", "sh", "-c", "echo updated > /dev/shm/ram-marker") + run("second-full-" + layout, "snapshot", "create", full + "-next", "--from-sandbox", source, "--full") + next_child = source + "-next-child" + names.append(next_child) + run("second-full-restore-" + layout, "create", "--name", next_child, "--from-snapshot", full + "-next") + assert run("second-full-ram-" + layout, "exec", next_child, "--", "cat", "/dev/shm/ram-marker").stdout.strip() == "updated" + # A name collision must fail before publication and leave both the source and snapshot usable. + refused = run("duplicate-refused-" + layout, "snapshot", "create", source + "-between", "--from-sandbox", source, ok=False) + assert refused.returncode != 0 + run("source-after-refusal-" + layout, "exec", source, "--", "true") + run("stop-source-" + layout, "stop", source) + run("stopped-after-live-" + layout, "snapshot", "create", source + "-stopped", "--from-sandbox", source) + tmpfs = prefix + "-tmpfs" + names.append(tmpfs) + run("tmpfs-create", "create", "alpine", "--name", tmpfs, "--root-disk", "tmpfs:128M", "--memory", "256M") + refused = run("tmpfs-refused", "snapshot", "create", tmpfs + "-bad", "--from-sandbox", tmpfs, ok=False) + assert refused.returncode != 0 and "tmpfs" in refused.stderr + run("tmpfs-still-running", "exec", tmpfs, "--", "true") + print(json.dumps({"result": "pass", "layouts": ["flat", "managed"], "modes": ["installed", "integrity", "archive", "plain", "paused"]})) +finally: + for name in reversed(names): + try: + run("cleanup-" + name, "stop", name, ok=False) + except Exception as error: + rows.append({"case": "cleanup-" + name, "error": str(error)}) + (out / "results.json").write_text(json.dumps(rows, indent=2)) diff --git a/scripts/smoke/cli/snapshot-branch.py b/scripts/smoke/cli/snapshot-branch.py new file mode 100644 index 000000000..4e8523deb --- /dev/null +++ b/scripts/smoke/cli/snapshot-branch.py @@ -0,0 +1,393 @@ +#!/usr/bin/env python3 +"""Small live snapshot/branch regression suite; no benchmark repetitions or shared VM state.""" + +import argparse +from contextlib import contextmanager +import csv +import io +import json +import os +from pathlib import Path +import signal +import shutil +import sqlite3 +import subprocess +import sys +import tempfile +import time + + +REPOSITORY = Path(__file__).resolve().parents[3] + + +def positive_seconds(value): + seconds = float(value) + if not 0 < seconds < float("inf"): + raise argparse.ArgumentTypeError("timeout must be finite and positive") + return seconds + + +def output_text(value): + # TimeoutExpired carries bytes even when subprocess.run requested text output. + return value.decode(errors="replace") if isinstance(value, bytes) else value or "" + + +@contextmanager +def defer_interrupts(): + # A second Ctrl-C/SIGTERM must not abandon the remaining bounded stop attempts. + received = [] + previous = {sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM)} + try: + for sig in previous: + signal.signal(sig, lambda signum, _frame: received.append(signum)) + yield received + finally: + for sig, handler in previous.items(): + signal.signal(sig, handler) + + +class Smoke: + def __init__(self, args): + self.args = args + self.binary = args.binary.expanduser().resolve(strict=True) + if not self.binary.is_file() or not os.access(self.binary, os.X_OK): + raise ValueError(f"msb binary is not executable: {self.binary}") + if args.output: + self.root = args.output.expanduser().resolve() + self.root.mkdir(mode=0o700, parents=True, exist_ok=False) + else: + # macOS's default per-user temp path is too long for the runtime's Unix sockets. + self.root = Path(tempfile.mkdtemp(prefix="msb-smoke-", dir="/tmp" if os.name == "posix" else None)) + self.logs = self.root / "logs" + self.logs.mkdir() + self.home = self.root / "home" + # Never borrow the caller's catalog, cloud backend, project config, or VM names. + # Explicit matching runtime/firmware overrides remain available to development builds. + self.env = dict(os.environ) + for key in ("MSB_HOME", "MSB_CONFIG_PATH", "MSB_BACKEND", "MSB_PROFILE"): + self.env.pop(key, None) + self.env.update(MSB_HOME=str(self.home), MSB_BACKEND="local", NO_COLOR="1") + self.names = [] + self.active = [] + self.deadline = None + self.report = { + "status": "running", "binary": str(self.binary), "home": str(self.home), + "layout": args.layout, "image": args.image, "commands": [], "cleanup": [], + "setup_ms": 0, "operations_ms": 0, "cleanup_ms": 0, + } + self.persist() + + def persist(self): + (self.root / "report.json").write_text(json.dumps(self.report, indent=2) + "\n") + + def run(self, case, *arguments, expected_failure=False, phase="operations", timeout=None): + limit = self.args.timeout if timeout is None else timeout + if phase == "operations" and self.deadline is not None: + remaining = self.deadline - time.monotonic() + if remaining <= 0: + raise RuntimeError(f"suite deadline exceeded before {case}") + limit = min(limit, remaining) + command = [str(self.binary), *map(str, arguments)] + started = time.monotonic() + stdout, stderr, code, timed_out = "", "", None, False + try: + result = subprocess.run(command, env=self.env, cwd=self.root, capture_output=True, + text=True, timeout=limit) + stdout, stderr, code = result.stdout, result.stderr, result.returncode + except subprocess.TimeoutExpired as error: + stdout, stderr = output_text(error.stdout), output_text(error.stderr) + timed_out = True + finally: + elapsed = round((time.monotonic() - started) * 1000, 2) + prefix = f"{len(self.report['commands']):03d}-{case}" + (self.logs / f"{prefix}.stdout.log").write_text(stdout) + (self.logs / f"{prefix}.stderr.log").write_text(stderr) + row = dict(case=case, phase=phase, argv=command[1:], ms=elapsed, exit=code, + expected_failure=expected_failure, timed_out=timed_out) + self.report["commands"].append(row) + self.persist() + print(f"{case}: {elapsed:.2f} ms (exit={code})", flush=True) + if timed_out: + raise RuntimeError(f"{case} timed out after {limit:.2f}s") + # A crash is not proof that a deliberately unsupported operation was refused cleanly. + if code is None or code < 0 or code > 255 or (code != 0) != expected_failure: + raise RuntimeError(f"{case} failed (exit={code}): {stderr[-2000:]}") + return stdout.strip(), stderr.strip() + + def guest(self, case, name, script): + return self.run(case, "exec", name, "--", "sh", "-ec", script)[0] + + def remember(self, name): + # Register before create/branch: a timed-out client may have started a detached VM. + self.names.append(name) + self.active.append(name) + + def create(self, name, *options): + self.remember(name) + self.run("create-" + name, "create", "--name", name, *options) + + def branch(self, source, child): + self.remember(child) + self.run("branch-" + child, "branch", source, "--name", child) + + def stop(self, name): + self.run("stop-" + name, "stop", name, "--timeout", "5") + self.active.remove(name) + + def check_markers(self, name, value, ram=True): + script = "cat /smoke-marker; " + ( + "cat /dev/shm/smoke-marker" if ram else "test ! -e /dev/shm/smoke-marker") + actual = self.guest("markers-" + name, name, script) + expected = value + "\n" + value if ram else value + if actual != expected: + raise RuntimeError(f"{name}: expected {expected!r}, got {actual!r}") + + def check_status(self, name, expected): + entries = json.loads(self.run("status-" + name, "list", "--format", "json")[0]) + actual = {entry["name"]: entry["status"] for entry in entries}.get(name) + if actual != expected: + raise RuntimeError(f"{name}: expected status {expected}, got {actual}") + + def capture(self, member, full=False): + options = ["--full"] if full else [] + output = self.run("capture-" + member, "snapshot", "create", member, + "--from-sandbox", "source", "--group", "work", *options)[0] + path = Path(output.splitlines()[-1]) + descriptor = json.loads((path / "snapshot.json").read_text()) + if (path.parent != self.home / "snapshots/work" + or path.name != descriptor["snapshot_id"] + or descriptor["state"]["kind"] != ("checkpoint" if full else "file")): + raise RuntimeError(f"unexpected captured artifact: {path}") + return descriptor + + def exercise(self): + layout = "flat:512M" if self.args.layout == "flat" else "512M" + self.create("source", self.args.image, "--root-disk", layout, + "--memory", "256M", "--cpus", "2") + self.guest("seed-source", "source", + "echo source > /smoke-marker; echo source > /dev/shm/smoke-marker; sync") + before = set((self.home / "snapshots").rglob("snapshot.json")) + self.branch("source", "child") + self.check_markers("child", "source") + self.guest("write-private-child", "child", + "echo child > /smoke-marker; echo child > /dev/shm/smoke-marker") + self.check_markers("source", "source") + self.branch("child", "grandchild") + self.check_markers("grandchild", "child") + if before != set((self.home / "snapshots").rglob("snapshot.json")): + raise RuntimeError("direct branching installed a durable snapshot") + _, error = self.run("duplicate-child-refused", "branch", "source", "--name", "child", + expected_failure=True) + if "already exists" not in error.lower(): + raise RuntimeError(f"duplicate branch failed for an unrelated reason: {error}") + self.check_markers("child", "child") + self.stop("child") + self.check_markers("grandchild", "child") + self.stop("grandchild") + + self.run("pause-source", "pause", "source") + self.check_status("source", "Paused") + self.branch("source", "paused-child") + self.check_markers("paused-child", "source") + self.stop("paused-child") + full = self.capture("full", full=True) + self.check_status("source", "Paused") + _, error = self.run("paused-exec-refused", "exec", "source", "--", "true", + expected_failure=True) + if "paused" not in error.lower(): + raise RuntimeError(f"paused exec failed for an unrelated reason: {error}") + self.run("resume-source", "resume", "source") + self.check_markers("source", "source") + + # A full snapshot preserves tmpfs; disk-only cold boot must not restore it. + disk = self.capture("disk") + self.check_status("source", "Running") + if disk["parent"] != full["snapshot_id"]: + raise RuntimeError("capture lineage was not retained") + self.create("disk-child", "--from-snapshot", "work:disk") + self.check_markers("disk-child", "source", ram=False) + self.stop("disk-child") + + full_archive, disk_archive = self.root / "full.msb", self.root / "disk.msb" + self.run("save-full", "snapshot", "save", "work:full", full_archive) + self.run("save-disk", "snapshot", "save", "work:disk", disk_archive) + self.run("load-batch-reversed", "snapshot", "load", disk_archive, full_archive, + "--group", "received") + head = json.loads(self.run("batch-head", "snapshot", "head", "received", + "--format", "json")[0]) + if head["head"] != disk["snapshot_id"]: + raise RuntimeError("batch head followed argument order instead of ancestry") + self.run("verify-imported-full", "snapshot", "verify", "received:full") + self.create("eager", "--from-snapshot", "received:full") + self.check_markers("eager", "source") + self.run("pause-eager", "pause", "eager") + self.stop("eager") + + before = set((self.home / "snapshots").rglob("snapshot.json")) + self.create("forked", "--from-snapshot", full_archive, "--forked") + if before != set((self.home / "snapshots").rglob("snapshot.json")): + raise RuntimeError("direct archive restore installed an intermediate snapshot") + # Unlink only archives created by this test, after the child is ready. + full_archive.unlink() + disk_archive.unlink() + self.check_markers("forked", "source") + self.guest("write-private-forked", "forked", + "echo forked > /smoke-marker; echo forked > /dev/shm/smoke-marker") + self.check_markers("source", "source") + self.stop("source") + self.check_markers("forked", "forked") + self.stop("forked") + + def runtime_pids(self): + # The CLI catalog can become terminal before the host process has exited. Read only + # this test's private run history, including VMs stopped earlier in the suite. + database = self.home / "db/msb.db" + if not database.exists(): + return [] + with sqlite3.connect(database.as_uri() + "?mode=ro", uri=True, timeout=2) as db: + pids = sorted({row[0] for row in db.execute('SELECT pid FROM "run" WHERE pid > 0')}) + remaining = [] + for pid in pids: + if os.name == "nt": + result = subprocess.run(["tasklist", "/FI", f"PID eq {pid}", "/FO", "CSV", "/NH"], + capture_output=True, text=True, timeout=2, check=True) + alive = any(len(row) > 1 and row[1] == str(pid) + for row in csv.reader(io.StringIO(result.stdout))) + else: + result = subprocess.run(["ps", "-p", str(pid), "-o", "stat="], + capture_output=True, text=True, timeout=2) + if result.returncode not in (0, 1): + raise RuntimeError(f"could not inspect runtime PID {pid}: {result.stderr}") + alive = bool(result.stdout.strip()) and not result.stdout.strip().startswith("Z") + if alive: + remaining.append(pid) + # Never signal bare recorded PIDs: PID reuse must not endanger another process. + return remaining + + def cleanup(self): + with defer_interrupts() as interrupted: + errors = self.cleanup_owned() + if interrupted: + errors.append("interrupted during cleanup; completed bounded stop attempts") + return errors + + def cleanup_owned(self): + errors = [] + for name in reversed(self.active): + try: + self.run("cleanup-" + name, "stop", name, "--timeout", "5", + phase="cleanup", timeout=10) + self.report["cleanup"].append(dict(name=name, stopped=True)) + except Exception as error: + # A failed create may not have a catalog row. Preserve those diagnostics, then + # check both the catalog and process history after a bounded force-stop attempt. + self.report["cleanup"].append(dict(name=name, error=str(error))) + try: + self.run("force-stop-" + name, "stop", name, "--force", + phase="cleanup", timeout=10) + except Exception as forced: + errors.append(str(forced)) + try: + entries = json.loads(self.run("cleanup-inventory", "list", "--format", "json", + phase="cleanup", timeout=10)[0]) + resident = [entry for entry in entries if entry["status"] not in ("Stopped", "Crashed")] + self.report["remaining_sandboxes"] = resident + if resident: + errors.append(f"test sandboxes remain resident: {resident}") + except Exception as error: + errors.append(f"could not verify VM cleanup: {error}") + try: + deadline = time.monotonic() + 5 + while True: + remaining = self.runtime_pids() + if not remaining or time.monotonic() >= deadline: + break + time.sleep(0.1) + self.report["remaining_runtime_pids"] = remaining + if remaining: + errors.append(f"recorded runtime PIDs are still alive: {remaining}") + except Exception as error: + errors.append(f"could not verify runtime process exit: {error}") + return errors + + def execute(self): + started = time.monotonic() + failure = None + operations_started = None + try: + # Build/download/materialization time is not snapshot latency. Preparing both layouts + # also avoids deferred image work becoming part of the first measured create. + self.run("prepare-image", "pull", self.args.image, "--materialize", "all", + phase="setup", timeout=180) + self.report["setup_ms"] = round((time.monotonic() - started) * 1000, 2) + operations_started = time.monotonic() + self.deadline = operations_started + self.args.suite_timeout + self.exercise() + except (Exception, KeyboardInterrupt) as error: + failure = str(error) or "interrupted" + finally: + if operations_started is not None: + self.report["operations_ms"] = round( + (time.monotonic() - operations_started) * 1000, 2) + else: + self.report["setup_ms"] = round((time.monotonic() - started) * 1000, 2) + cleanup_started = time.monotonic() + # Cleanup is independent of the expired operation deadline and tries every owned VM. + cleanup_errors = self.cleanup() + if failure or cleanup_errors: + for name in self.names: + try: + self.run("diagnostics-" + name, "logs", "--source", "system", name, + phase="diagnostics", timeout=5) + except Exception: + pass + self.report["home_removed"] = False + if not failure and not cleanup_errors and self.home.exists(): + # This home was created beneath our exclusive output directory. Once both + # catalog and process checks pass, retain text evidence, not large RAM/disks. + try: + shutil.rmtree(self.home) + self.report["home_removed"] = True + except OSError as error: + cleanup_errors.append(f"could not remove owned test home: {error}") + self.report.update( + status="failed" if failure or cleanup_errors else "passed", error=failure, + cleanup_errors=cleanup_errors, + cleanup_ms=round((time.monotonic() - cleanup_started) * 1000, 2), + total_ms=round((time.monotonic() - started) * 1000, 2), + ) + self.persist() + print(json.dumps({key: self.report[key] for key in + ("status", "setup_ms", "operations_ms", "cleanup_ms", "total_ms")}), + flush=True) + print(f"Report: {self.root / 'report.json'}", flush=True) + if self.report["error"] or self.report["cleanup_errors"]: + print(self.report["error"] or "; ".join(self.report["cleanup_errors"]), file=sys.stderr) + return 0 if self.report["status"] == "passed" else 1 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--binary", type=Path, + default=REPOSITORY / "build" / ("msb.exe" if os.name == "nt" else "msb")) + parser.add_argument("--output", type=Path, help="New directory for an isolated home and logs") + parser.add_argument("--layout", choices=("managed", "flat"), default="managed") + parser.add_argument("--image", default="mirror.gcr.io/library/alpine:3.20") + parser.add_argument("--timeout", type=positive_seconds, default=30, + help="Per-operation timeout in seconds (default: 30)") + parser.add_argument("--suite-timeout", type=positive_seconds, default=120, + help="Operation-suite deadline, excluding image setup/cleanup (default: 120)") + args = parser.parse_args() + + def interrupted(_signum, _frame): + raise KeyboardInterrupt("interrupted; cleaning up owned test VMs") + + signal.signal(signal.SIGTERM, interrupted) + try: + return Smoke(args).execute() + except (OSError, ValueError) as error: + parser.exit(1, f"snapshot smoke: {error}\n") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/smoke/cli/snapshot-groups.py b/scripts/smoke/cli/snapshot-groups.py new file mode 100644 index 000000000..dece084a7 --- /dev/null +++ b/scripts/smoke/cli/snapshot-groups.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""Opt-in live snapshot-group qualification in an isolated MSB_HOME. + +Set MSB_PATH and GROUP_TEST_OUT; optionally GROUP_TEST_LAYOUT=512M, flat:512M, or tmpfs:128M. +All created VMs are stopped in finally, and every command/timing is retained. +""" + +from concurrent.futures import ThreadPoolExecutor +import json +import os +from pathlib import Path +import subprocess +import time + +binary = os.environ["MSB_PATH"] +# Match CI's public mirror; callers may select an explicit local fixture instead. +image = os.environ.get("MSB_TEST_IMAGE", "mirror.gcr.io/library/alpine:latest") +root = Path(os.environ["GROUP_TEST_OUT"]) +root.mkdir(parents=True, exist_ok=False) +home = root / "home" +env = dict(os.environ, MSB_HOME=str(home)) +layout = os.environ.get("GROUP_TEST_LAYOUT", "flat:512M") +full_only = layout.startswith("tmpfs:") +rows = [] +names = [] + + +def run(label, *args, fail=False): + started = time.perf_counter() + result = subprocess.run([binary, *map(str, args)], env=env, capture_output=True, + text=True, timeout=180) + row = {"case": label, "ms": round((time.perf_counter() - started) * 1000, 2), + "exit": result.returncode} + rows.append(row) + print(json.dumps(row), flush=True) + (root / f"{label}.stdout").write_text(result.stdout) + (root / f"{label}.stderr").write_text(result.stderr) + if (result.returncode != 0) != fail: + raise RuntimeError(f"{label}: {result.stderr[-3000:]}") + return result.stdout.strip() + + +def guest(label, name, script): + return run(label, "exec", name, "--", "sh", "-ec", script) + + +def create(name, snapshot=None, forked=False): + names.append(name) + args = ["create", "--name", name] + if snapshot: + args += ["--from-snapshot", snapshot] + else: + args += [image, "--root-disk", layout, "--memory", "256M", "--cpus", "2"] + if forked: + args.append("--forked") + run("create-" + name, *args) + + +def capture(label, source, member, group="work", full=False, fail=False): + args = ["snapshot", "create", member, "--from-sandbox", source, "--group", group] + if full or full_only: + args.append("--full") + output = run(label, *args, fail=fail) + if fail: + return None + path = Path(output.splitlines()[-1]) + descriptor = json.loads((path / "snapshot.json").read_text()) + assert path.parent == home / "snapshots" / group, path + assert path.name == descriptor["snapshot_id"], path + return path, descriptor + + +def head(label, selector): + return json.loads(run(label, "snapshot", "head", selector, "--format", "json")) + + +try: + create("source") + guest("source-one", "source", "echo one > /disk-marker; echo ram-one > /dev/shm/marker; sync") + cp1, d1 = capture("capture-cp1", "source", "cp1") + assert d1["parent"] is None + assert head("initial-head", "work")["head"] == d1["snapshot_id"] + guest("source-two", "source", "echo two > /disk-marker; sync") + cp2, d2 = capture("capture-cp2", "source", "cp2") + assert d2["parent"] == d1["snapshot_id"] + assert head("advanced-head", "work")["head"] == d2["snapshot_id"] + + create("old-child", "work:cp1") + assert guest("read-old-child", "old-child", "cat /disk-marker") == "one" + branch, db = capture("capture-old-child", "old-child", "experiment") + assert db["parent"] == d1["snapshot_id"] + assert head("sibling-keeps-head", "work")["head"] == d2["snapshot_id"] + assert head("select-sibling", "work:experiment")["head"] == db["snapshot_id"] + create("selected-child", "work") + assert guest("read-selected-head", "selected-child", "cat /disk-marker") == "one" + run("stop-selected-child", "stop", "selected-child") + head("select-cp2", "work:cp2") + + # Same source serialization creates ancestry; different children of one head are siblings. + create("race-a", "work:cp2") + create("race-b", "work:cp2") + with ThreadPoolExecutor(max_workers=2) as pool: + pending = [pool.submit(capture, "capture-" + child, child, child) + for child in ("race-a", "race-b")] + siblings = [future.result() for future in pending] + ids = {desc["snapshot_id"] for _, desc in siblings} + assert all(desc["parent"] == d2["snapshot_id"] for _, desc in siblings) + assert head("race-head", "work")["head"] in ids + assert all(path.exists() for path, _ in siblings) + for name in ("old-child", "race-a", "race-b"): + run("stop-" + name, "stop", name) + head("select-cp2-again", "work:cp2") + + # Full capture, direct local branch ancestry, and paused-source preservation. + full1, f1 = capture("capture-full1", "source", "full1", full=True) + assert f1["parent"] == d2["snapshot_id"] + names.append("local-child") + run("local-branch", "branch", "source", "--name", "local-child") + local_snap, dl = capture("capture-local-child", "local-child", "local-child", full=True) + assert dl["parent"] == f1["snapshot_id"] + guest("source-three", "source", "echo three > /disk-marker; echo ram-three > /dev/shm/marker; sync") + run("pause-source", "pause", "source") + full2, f2 = capture("capture-paused-full2", "source", "full2", full=True) + assert f2["parent"] == f1["snapshot_id"] + run("resume-source", "resume", "source") + assert guest("source-still-live", "source", "cat /disk-marker") == "three" + run("stop-local-child", "stop", "local-child") + + # A rejected alias collision cannot replace the artifact or advance the cursor/head. + before = (full2 / "snapshot.json").read_bytes() + current = head("head-before-conflict", "work")["head"] + capture("duplicate-name-refused", "source", "full2", fail=True) + assert (full2 / "snapshot.json").read_bytes() == before + assert head("head-after-conflict", "work")["head"] == current + full3, f3 = capture("capture-after-conflict", "source", "full3", full=True) + assert f3["parent"] == f2["snapshot_id"] + + base_archive = root / "base.msb" + delta_archive = root / "delta.msb" + run("export-base", "snapshot", "save", full1, base_archive) + run("export-delta", "snapshot", "save", full2, delta_archive, "--since", full1) + inventory = json.loads(subprocess.check_output(["tar", "-xOf", str(delta_archive), "archive.json"])) + dependent = inventory["completeness"] == "dependent" + # A RAM-only cut can have no reusable objects: --since then emits a standalone archive. + # Require a base exactly when the archive actually omitted required payloads. + run("missing-base" + ("-refused" if dependent else "-not-needed"), "snapshot", "load", + delta_archive, "--group", "missing", fail=dependent) + if dependent: + assert not (home / "snapshots" / "missing").exists() + imported_base = Path(run("import-base", "snapshot", "load", base_archive, + "--group", "received").splitlines()[-1]) + imported_delta = Path(run("import-delta", "snapshot", "load", delta_archive, + "--group", "received", "--base", "received:full1").splitlines()[-1]) + assert imported_base.name == f1["snapshot_id"] and imported_delta.name == f2["snapshot_id"] + assert head("import-advanced-head", "received")["head"] == f2["snapshot_id"] + run("reimport-old", "snapshot", "load", base_archive, "--group", "received") + assert head("old-import-kept-head", "received")["head"] == f2["snapshot_id"] + run("reimport-set-head", "snapshot", "load", base_archive, "--group", "received", "--set-head") + assert head("old-import-explicit-head", "received")["head"] == f1["snapshot_id"] + run("head-removal-refused", "snapshot", "remove", "received:full1", "--force", fail=True) + + duplicate = Path(run("import-second-group", "snapshot", "load", base_archive).splitlines()[-1]) + assert duplicate.name == imported_base.name and duplicate.parent != imported_base.parent + run("ambiguous-id-refused", "snapshot", "inspect", f1["snapshot_id"], fail=True) + run("reindex", "snapshot", "reindex") + for mode in ("eager", "forked"): + child = "restored-" + mode + create(child, "received:full2", forked=mode == "forked") + assert guest("restored-state-" + mode, child, + "cat /disk-marker; cat /dev/shm/marker") == "three\nram-three" + run("stop-" + child, "stop", child) + + # Direct archive capture records ancestry but never creates an installed member. + before_members = sorted(str(p) for p in (home / "snapshots").rglob("snapshot.json")) + direct = root / "direct.msb" + run("direct-capture", "snapshot", "create", "direct", "--from-sandbox", "source", "--full", "--archive", direct) + assert sorted(str(p) for p in (home / "snapshots").rglob("snapshot.json")) == before_members + create("direct-restored", str(direct), forked=True) + assert guest("direct-restored-state", "direct-restored", "cat /dev/shm/marker") == "ram-three" + assert sorted(str(p) for p in (home / "snapshots").rglob("snapshot.json")) == before_members + run("stop-direct-restored", "stop", "direct-restored") + run("stop-source", "stop", "source") + if full_only: + capture("stopped-tmpfs-refused", "source", "stopped", fail=True) + else: + stopped, stopped_desc = capture("stopped-capture", "source", "stopped") + assert stopped_desc["parent"] != f3["snapshot_id"], "direct archive did not advance source ancestry" + create("stopped-restored", "work:stopped") + assert guest("stopped-restored-state", "stopped-restored", "cat /disk-marker") == "three" + print(json.dumps({"pass": True, "layout": layout, "commands": len(rows)}), flush=True) +finally: + for name in reversed(names): + result = subprocess.run([binary, "stop", name], env=env, capture_output=True, + text=True, timeout=30) + (root / ("cleanup-" + name + ".log")).write_text(result.stdout + result.stderr) + (root / "results.json").write_text(json.dumps(rows, indent=2)) diff --git a/scripts/smoke/cli/snapshot-load-batch.py b/scripts/smoke/cli/snapshot-load-batch.py new file mode 100644 index 000000000..44af10d45 --- /dev/null +++ b/scripts/smoke/cli/snapshot-load-batch.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 +"""Qualify unordered batch imports using already captured real VM checkpoints. + +The fixture root comes from snapshot-groups.py and must contain home/snapshots/work +members full1, full2, full3, local-child, and experiment. No fixture is modified. +Pass --live to restore eager/forked children after imports; all owned VMs are stopped. +Pass --file-only to qualify disk-only cp1/cp2 archives and their cold-boot restore. +""" + +import argparse +import json +import os +from pathlib import Path +import shlex +import subprocess +import time + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--binary", required=True) + parser.add_argument("--fixtures", required=True, type=Path) + parser.add_argument("--image-home", type=Path, + help="Optional home containing the fixture's fully materialized image cache") + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--live", action="store_true") + parser.add_argument("--file-only", action="store_true") + parser.add_argument("--fresh-file", action="store_true", + help="Capture two new disk-only checkpoints before the file-only matrix") + parser.add_argument("--file-standalone", action="store_true", + help="Qualify complete disk archives instead of dependent --since exports") + args = parser.parse_args() + if args.fresh_file and not (args.file_only and args.live): + parser.error("--fresh-file requires --file-only --live") + if args.file_standalone and not args.file_only: + parser.error("--file-standalone requires --file-only") + args.output.mkdir(parents=True, exist_ok=False) + home = args.output / "home" + inputs = args.output / "inputs" + inputs.mkdir() + env = dict(os.environ, MSB_HOME=str(home), MSB_BACKEND="local") + # Flat-only fixtures may not have the layered image cache required by --with-image. + # An explicitly supplied cache home must hold the same pinned image digest; export + # validates that match while reading snapshot artifacts from their original paths. + source_env = dict(env, MSB_HOME=str(args.image_home or args.fixtures / "home")) + rows, names = [], [] + report = dict(status="running", binary=args.binary, fixture=str(args.fixtures), + image_home=source_env["MSB_HOME"], home=str(home), live=args.live, + file_only=args.file_only, fresh_file=args.fresh_file, + file_standalone=args.file_standalone, rows=rows) + + def persist(): + (args.output / "report.json").write_text(json.dumps(report, indent=2)) + + def run(label, *command, fail=False, source=False, shell=False): + started = time.perf_counter() + argv = ["/bin/sh", "-c", command[0]] if shell else [args.binary, *map(str, command)] + result = subprocess.run(argv, + env=source_env if source else env, + capture_output=True, text=True, timeout=180) + row = dict(case=label, ms=round((time.perf_counter() - started) * 1000, 2), + exit=result.returncode, expected_failure=fail, + stdout=result.stdout, stderr=result.stderr) + rows.append(row) + persist() + print(json.dumps({key: row[key] for key in ("case", "ms", "exit")}), flush=True) + assert (result.returncode != 0) == fail, row + return result.stdout.strip() + + def group_head(group): + return json.loads((home / "snapshots" / group / "group.json").read_text())["head"] + + def members(group): + return sorted(path.parent.name for path in (home / "snapshots" / group).glob("*/snapshot.json")) + + def restored(mode, group, marker): + name = "batch-" + mode + names.append(name) + options = ["--forked"] if mode == "forked" else [] + run("restore-" + mode, "create", "--name", name, "--from-snapshot", group, *options) + actual = run("state-" + mode, "exec", name, "--", "sh", "-ec", + "cat /disk-marker; cat /dev/shm/marker") + assert actual == marker, actual + run("stop-" + mode, "stop", name) + + fixtures = {} + for metadata in (args.fixtures / "home/snapshots/work").glob("*/group-member.json"): + name = json.loads(metadata.read_text())["name"] + descriptor = json.loads((metadata.parent / "snapshot.json").read_text()) + fixtures[name] = (metadata.parent, descriptor) + required = (("cp1", "cp2") if args.file_only + else ("full1", "full2", "full3", "local-child", "experiment")) + assert all(name in fixtures for name in required), sorted(fixtures) + ids = [fixtures[name][1]["snapshot_id"] for name in required[:3]] + archives = [inputs / (name + ".msb") for name in ("full1", "full2", "full3")] + branch = inputs / "branch.msb" + unrelated = inputs / "unrelated.msb" + try: + if args.file_only: + if args.fresh_file: + # Start only a disposable child, never the retained fixture sandbox. + seed = inputs / "seed.msb" + run("export-file-seed", "snapshot", "save", fixtures["cp2"][0], seed, + "--with-image", source=True) + run("load-file-seed", "snapshot", "load", seed, "--group", "seed") + names.append("batch-capture") + run("create-file-source", "create", "--name", "batch-capture", "--from-snapshot", "seed") + for member, marker in (("cp1", "one"), ("cp2", "two")): + run("write-file-" + member, "exec", "batch-capture", "--", "sh", "-ec", + "echo " + marker + " > /disk-marker; sync") + output = run("capture-file-" + member, "snapshot", "create", member, + "--from-sandbox", "batch-capture", "--group", "fresh") + artifact = Path(output.splitlines()[-1]) + fixtures[member] = (artifact, json.loads((artifact / "snapshot.json").read_text())) + run("stop-file-source", "stop", "batch-capture") + ids = [fixtures[name][1]["snapshot_id"] for name in required] + # These are actual disk-only artifacts, not full checkpoints restored with + # --disk-only: their inherited payloads use the file-archive layer pool. + assert all(fixtures[name][1]["state"]["kind"] == "file" for name in required) + first, second = inputs / "cp1.msb", inputs / "cp2.msb" + run("export-file-base", "snapshot", "save", fixtures["cp1"][0], first, + "--with-image", source=True) + delta_options = [] if args.file_standalone else ["--since", fixtures["cp1"][0]] + run("export-file-complete" if args.file_standalone else "export-file-delta", + "snapshot", "save", fixtures["cp2"][0], second, *delta_options, source=True) + inventory = json.loads(subprocess.check_output(["tar", "-xOf", str(second), "archive.json"])) + assert inventory["completeness"] == ("boot-complete" if args.file_standalone else "dependent") + report["file_inventory"] = inventory + if not args.file_standalone: + run("missing-file-base-refused", "snapshot", "load", second, "--group", "missing", fail=True) + assert members("missing") == [] + run("load-file-reverse", "snapshot", "load", second, first, "--group", "file") + assert members("file") == sorted(ids) + assert group_head("file") == ids[1] + for name in required: + run("verify-file-" + name, "snapshot", "verify", "file:" + name) + run("install-file-base", "snapshot", "load", first, "--group", "automatic") + run("file-auto-base", "snapshot", "load", second, "--group", "automatic") + assert group_head("automatic") == ids[1] + # The surviving member must own the full disk closure after inputs and its + # installed historical parent disappear, including borrowed archive layers. + first.unlink() + second.unlink() + run("remove-file-ancestor", "snapshot", "remove", "file:" + ids[0], "--force") + run("verify-owned-file", "snapshot", "verify", "file") + if args.live: + name = "batch-file" + names.append(name) + run("restore-file", "create", "--name", name, "--from-snapshot", "file") + actual = run("state-file", "exec", name, "--", "sh", "-ec", + "cat /disk-marker; test ! -e /dev/shm/marker") + assert actual == "two", actual + run("stop-file", "stop", name) + report["status"] = "passed" + return + # Include the pinned image once; importing the batch remains offline-capable. + run("export-baseline", "snapshot", "save", fixtures["full1"][0], archives[0], + "--with-image", source=True) + for index in (1, 2): + run("export-delta-" + str(index), "snapshot", "save", + fixtures["full" + str(index + 1)][0], archives[index], + "--since", fixtures["full" + str(index)][0], source=True) + run("export-branch", "snapshot", "save", fixtures["local-child"][0], branch, source=True) + run("export-unrelated", "snapshot", "save", fixtures["experiment"][0], unrelated, source=True) + for index in (1, 2): + inventory = json.loads(subprocess.check_output(["tar", "-xOf", str(archives[index]), "archive.json"])) + assert inventory["completeness"] == "dependent", "fixture must omit real dependencies" + report["delta_" + str(index) + "_inventory"] = inventory + + for label, order in (("reverse", (2, 1, 0)), ("shuffled", (1, 0, 2))): + output = run("load-" + label, "snapshot", "load", *(archives[index] for index in order), "--group", label) + # Results correspond to input archive heads; input order does not select the + # group's head. In reverse order, the final printed path is the oldest member. + printed_paths = [Path(line) for line in output.splitlines() if line.startswith(str(home))] + assert [path.name for path in printed_paths] == [ids[index] for index in order] + assert members(label) == sorted(ids) + assert group_head(label) == ids[2] + for name in ("full1", "full2", "full3"): + run("verify-" + label + "-" + name, "snapshot", "verify", label + ":" + name) + + wildcard_inputs = inputs / "wildcard" + wildcard_inputs.mkdir() + for archive in archives: + os.link(archive, wildcard_inputs / archive.name) + # This is a real shell expansion, unlike the explicit argument arrays above. + wildcard_command = (shlex.join([args.binary, "snapshot", "load"]) + " " + + shlex.quote(str(wildcard_inputs)) + "/*.msb --group wildcard") + run("shell-wildcard", wildcard_command, shell=True) + assert members("wildcard") == sorted(ids) + assert group_head("wildcard") == ids[2] + + run("group-base-install", "snapshot", "load", archives[0], "--group", "automatic") + run("group-base-auto", "snapshot", "load", archives[2], archives[1], "--group", "automatic") + assert group_head("automatic") == ids[2] + assert members("automatic") == sorted(ids) + + # A destination is a group-store root, not another archive argument. Automatic + # dependency lookup must also respect an explicitly selected nondefault root. + alternate = args.output / "alternate-store" + run("alternate-base", "snapshot", "load", archives[0], "--dest", alternate, "--group", "work") + run("alternate-auto-base", "snapshot", "load", archives[2], archives[1], + "--dest", alternate, "--group", "work") + assert json.loads((alternate / "work/group.json").read_text())["head"] == ids[2] + run("verify-alternate", "snapshot", "verify", alternate / "work" / ids[2]) + + run("missing-base-refused", "snapshot", "load", archives[2], "--group", "missing", fail=True) + assert members("missing") == [] + run("unrelated-base-refused", "snapshot", "load", archives[2], "--base", unrelated, + "--group", "unrelated", fail=True) + assert members("unrelated") == [] + # A complete external base supplies payloads, not mandatory imported history. + run("external-base", "snapshot", "load", archives[2], "--base", fixtures["full2"][0], + "--group", "hole") + assert members("hole") == [ids[2]] + assert group_head("hole") == ids[2] + run("verify-history-hole", "snapshot", "verify", "hole") + + run("duplicates", "snapshot", "load", archives[0], archives[0], "--group", "duplicates") + assert members("duplicates") == [ids[0]] + run("ambiguous-new", "snapshot", "load", archives[1], branch, archives[0], "--group", "branches") + assert group_head("branches") is None + assert len(members("branches")) == 3 + run("headless-restore-refused", "snapshot", "inspect", "branches", fail=True) + run("explicit-branch-selection", "snapshot", "head", "branches:local-child") + assert group_head("branches") == fixtures["local-child"][1]["snapshot_id"] + + run("ambiguity-base", "snapshot", "load", archives[0], "--group", "retained") + run("ambiguity-retains", "snapshot", "load", archives[1], branch, "--group", "retained") + assert group_head("retained") == ids[0] + run("ambiguous-forced-head-refused", "snapshot", "load", archives[1], branch, archives[0], + "--group", "rejected", "--set-head", fail=True) + assert members("rejected") == [] + + # Delete only this harness's exports, never the retained source fixtures. Loaded + # snapshots must keep working with their own dependency-complete disk/RAM files. + for archive in inputs.rglob("*.msb"): + assert archive.is_file() + archive.unlink() + for index in (0, 1): + run("remove-installed-ancestor-" + str(index), "snapshot", "remove", + "reverse:" + ids[index], "--force") + run("verify-owned-final", "snapshot", "verify", "reverse") + assert members("reverse") == [ids[2]] + if args.live: + restored("eager", "reverse", "three\nram-three") + restored("forked", "reverse", "three\nram-three") + report["status"] = "passed" + except Exception as error: + report.update(status="failed", error=repr(error)) + raise + finally: + cleanup = [] + for name in names: + result = subprocess.run([args.binary, "stop", name], env=env, + capture_output=True, text=True, timeout=30) + cleanup.append(dict(name=name, exit=result.returncode, stderr=result.stderr)) + report["cleanup"] = cleanup + persist() + + +if __name__ == "__main__": + main() diff --git a/scripts/smoke/cli/test_snapshot_branch.py b/scripts/smoke/cli/test_snapshot_branch.py new file mode 100644 index 000000000..fde5afa6d --- /dev/null +++ b/scripts/smoke/cli/test_snapshot_branch.py @@ -0,0 +1,446 @@ +#!/usr/bin/env python3 +"""VM-free tests for snapshot-branch smoke isolation, reporting, and cleanup.""" + +from __future__ import annotations + +import argparse +import contextlib +import importlib.util +import io +import json +import os +from pathlib import Path +import signal +import sqlite3 +import subprocess +import sys +import tempfile +import time +from types import SimpleNamespace +import unittest +from unittest import mock + + +SPEC = importlib.util.spec_from_file_location( + "snapshot_branch_smoke", Path(__file__).with_name("snapshot-branch.py") +) +assert SPEC is not None and SPEC.loader is not None +HARNESS = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(HARNESS) + + +class SnapshotBranchSmokeTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix="snapshot-branch-unit-") + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + contexts = contextlib.ExitStack() + self.addCleanup(contexts.close) + contexts.enter_context(contextlib.redirect_stdout(io.StringIO())) + contexts.enter_context(contextlib.redirect_stderr(io.StringIO())) + # Patch the process boundary globally for each test: no accidental command can boot a VM. + self.process = contexts.enter_context(mock.patch.object(HARNESS.subprocess, "run")) + self.process.side_effect = self.successful_process + + @staticmethod + def successful_process(command, **_kwargs): + stdout = "[]" if command[1] == "list" else "" + return subprocess.CompletedProcess(command, 0, stdout, "") + + def smoke(self, output="run"): + return HARNESS.Smoke(argparse.Namespace( + binary=Path(sys.executable), output=self.root / output, + layout="managed", image="test-image", timeout=30, suite_timeout=120, + )) + + @staticmethod + def report(smoke): + return json.loads((smoke.root / "report.json").read_text()) + + def commands(self): + return [call.args[0][1:] for call in self.process.call_args_list] + + @staticmethod + def run_history(smoke, rows): + database = smoke.home / "db/msb.db" + database.parent.mkdir(parents=True) + with sqlite3.connect(database) as db: + db.execute('CREATE TABLE "run" (pid INTEGER, status TEXT)') + db.executemany('INSERT INTO "run" (pid, status) VALUES (?, ?)', rows) + return database + + def test_defer_interrupts_records_signals_and_restores_handlers_on_exception(self): + original = {signal.SIGINT: mock.Mock(), signal.SIGTERM: mock.Mock()} + handlers = dict(original) + with mock.patch.object(HARNESS.signal, "getsignal", side_effect=handlers.__getitem__), \ + mock.patch.object(HARNESS.signal, "signal", side_effect=handlers.__setitem__): + with self.assertRaisesRegex(RuntimeError, "cleanup failed"): + with HARNESS.defer_interrupts() as received: + handlers[signal.SIGINT](signal.SIGINT, None) + handlers[signal.SIGTERM](signal.SIGTERM, None) + self.assertEqual(received, [signal.SIGINT, signal.SIGTERM]) + raise RuntimeError("cleanup failed") + self.assertEqual(handlers, original) + for handler in original.values(): + handler.assert_not_called() + + def test_isolates_home_backend_config_and_command_working_directory(self): + ambient = { + "MSB_HOME": str(self.root / "caller-home"), + "MSB_CONFIG_PATH": str(self.root / "caller-config.json"), + "MSB_BACKEND": "cloud", "MSB_PROFILE": "production", + "MSB_AGENTD_PATH": "/matching/agentd", "NO_COLOR": "0", + } + with mock.patch.dict(os.environ, ambient): + smoke = self.smoke() + smoke.run("probe", "list", "--format", "json") + self.assertEqual(os.environ["MSB_HOME"], ambient["MSB_HOME"]) + passed = self.process.call_args.kwargs + self.assertEqual(passed["cwd"], smoke.root) + self.assertEqual(passed["env"]["MSB_HOME"], str(smoke.root / "home")) + self.assertEqual(passed["env"]["MSB_BACKEND"], "local") + self.assertEqual(passed["env"]["NO_COLOR"], "1") + self.assertEqual(passed["env"]["MSB_AGENTD_PATH"], "/matching/agentd") + self.assertNotIn("MSB_CONFIG_PATH", passed["env"]) + self.assertNotIn("MSB_PROFILE", passed["env"]) + self.assertEqual(self.report(smoke)["home"], str(smoke.home)) + self.assertFalse((self.root / "caller-home").exists()) + + def test_refuses_existing_output_without_overwriting_it(self): + output = self.root / "existing" + output.mkdir() + sentinel = output / "report.json" + sentinel.write_text("keep existing report") + with self.assertRaises(FileExistsError): + self.smoke("existing") + self.assertEqual(sentinel.read_text(), "keep existing report") + self.process.assert_not_called() + + def test_records_success_and_clean_expected_failures(self): + smoke = self.smoke() + for code, expected_failure in [(0, False), (1, True), (2, True), (255, True)]: + with self.subTest(code=code, expected_failure=expected_failure): + self.process.side_effect = None + self.process.return_value = subprocess.CompletedProcess( + [], code, " output\n", " diagnostic\n" + ) + self.assertEqual( + smoke.run(f"exit-{code}", "probe", expected_failure=expected_failure), + ("output", "diagnostic"), + ) + row = self.report(smoke)["commands"][-1] + self.assertEqual(row["exit"], code) + self.assertEqual(row["expected_failure"], expected_failure) + self.assertFalse(row["timed_out"]) + + def test_unexpected_success_failure_and_crashes_are_not_accepted(self): + smoke = self.smoke() + # Windows exception statuses are positive; they must not pass as ordinary refusals. + for code, expected_failure in [ + (0, True), (1, False), (-9, False), (-9, True), + (256, True), (0xC0000005, False), (0xC0000005, True), + ]: + with self.subTest(code=code, expected_failure=expected_failure): + self.process.side_effect = None + self.process.return_value = subprocess.CompletedProcess([], code, "", "failed") + with self.assertRaisesRegex(RuntimeError, "failed"): + smoke.run("rejected", "probe", expected_failure=expected_failure) + self.assertEqual(self.report(smoke)["commands"][-1]["exit"], code) + + def test_timeout_preserves_partial_bytes_in_logs_and_report(self): + smoke = self.smoke() + self.process.side_effect = subprocess.TimeoutExpired( + [sys.executable, "probe"], 30, output=b"partial\xff\n", stderr=b"waiting\xfe" + ) + with self.assertRaisesRegex(RuntimeError, "timed out"): + smoke.run("slow", "probe", expected_failure=True) + row = self.report(smoke)["commands"][-1] + self.assertTrue(row["timed_out"]) + self.assertIsNone(row["exit"]) + self.assertEqual(next(smoke.logs.glob("*slow.stdout.log")).read_text(), "partial\ufffd\n") + self.assertEqual(next(smoke.logs.glob("*slow.stderr.log")).read_text(), "waiting\ufffd") + + def test_expired_suite_deadline_still_allows_bounded_cleanup(self): + smoke = self.smoke() + smoke.remember("owned") + smoke.deadline = time.monotonic() - 1 + with self.assertRaisesRegex(RuntimeError, "suite deadline exceeded"): + smoke.run("too-late", "probe") + self.process.assert_not_called() + self.assertEqual(smoke.cleanup(), []) + self.assertIn(["stop", "owned", "--timeout", "5"], self.commands()) + self.assertEqual(self.commands()[-1], ["list", "--format", "json"]) + self.assertTrue(all(call.kwargs["timeout"] > 0 for call in self.process.call_args_list)) + self.assertTrue(all(row["phase"] == "cleanup" for row in smoke.report["commands"])) + + def test_failed_create_is_still_registered_for_cleanup(self): + smoke = self.smoke() + self.process.side_effect = subprocess.TimeoutExpired([sys.executable, "create"], 30) + with self.assertRaisesRegex(RuntimeError, "timed out"): + smoke.create("possibly-started", "test-image") + self.process.side_effect = self.successful_process + self.assertEqual(smoke.cleanup(), []) + self.assertIn(["stop", "possibly-started", "--timeout", "5"], self.commands()) + + def test_cleanup_attempts_every_vm_and_force_fallback_despite_stop_failures(self): + smoke = self.smoke() + for name in ("first", "second", "third"): + smoke.remember(name) + + def process(command, **kwargs): + if command[1] == "stop": + name, forced = command[2], "--force" in command + # One fallback succeeds; another fails. Neither may prevent the next VM's stop. + code = int(name == "third" or (name == "second" and not forced)) + return subprocess.CompletedProcess(command, code, "", "stop refused") + return self.successful_process(command, **kwargs) + + self.process.side_effect = process + errors = smoke.cleanup() + for name in ("first", "second", "third"): + self.assertIn(["stop", name, "--timeout", "5"], self.commands()) + for name in ("second", "third"): + self.assertIn(["stop", name, "--force"], self.commands()) + self.assertEqual(self.commands()[-1], ["list", "--format", "json"]) + self.assertEqual(len(smoke.report["cleanup"]), 3) + self.assertTrue(any("third" in error for error in errors)) + + def test_cleanup_rejects_resident_inventory_but_accepts_terminal_entries(self): + smoke = self.smoke() + resident = [{"name": "running", "status": "Running"}, + {"name": "paused", "status": "Paused"}] + terminal = [{"name": "stopped", "status": "Stopped"}, + {"name": "crashed", "status": "Crashed"}] + self.process.side_effect = None + self.process.return_value = subprocess.CompletedProcess( + [], 0, json.dumps(resident + terminal), "" + ) + self.assertTrue(any("remain resident" in error for error in smoke.cleanup())) + self.assertEqual(smoke.report["remaining_sandboxes"], resident) + self.process.return_value = subprocess.CompletedProcess([], 0, json.dumps(terminal), "") + self.assertEqual(smoke.cleanup(), []) + self.assertEqual(smoke.report["remaining_sandboxes"], []) + + def test_cleanup_inventory_failure_is_reported(self): + smoke = self.smoke() + self.process.side_effect = None + self.process.return_value = subprocess.CompletedProcess([], 0, "not JSON", "") + self.assertTrue(any("could not verify VM cleanup" in error for error in smoke.cleanup())) + + def test_runtime_pids_checks_stopped_history_and_ignores_zombies_and_absent_processes(self): + smoke = self.smoke() + database = self.run_history(smoke, [ + (41001, "Stopped"), (41002, "Stopped"), (41003, "Stopped"), + (41003, "Crashed"), (0, "Running"), (-1, "Running"), (None, "Stopped"), + ]) + before = database.read_bytes() + states = {41001: (0, "Z+\n"), 41002: (1, ""), 41003: (0, "S+\n")} + + def process(command, **_kwargs): + self.assertEqual(command[0], "ps") + code, state = states[int(command[2])] + return subprocess.CompletedProcess(command, code, state, "") + + self.process.side_effect = process + platform = SimpleNamespace(name="posix", kill=mock.Mock()) + with mock.patch.object(HARNESS, "os", platform), \ + mock.patch.object(HARNESS.sqlite3, "connect", wraps=sqlite3.connect) as connect: + self.assertEqual(smoke.runtime_pids(), [41003]) + self.assertEqual(connect.call_args.args[0], database.as_uri() + "?mode=ro") + self.assertTrue(connect.call_args.kwargs["uri"]) + self.assertEqual([int(call.args[0][2]) for call in self.process.call_args_list], + [41001, 41002, 41003]) + self.assertEqual(database.read_bytes(), before) + platform.kill.assert_not_called() + + def test_runtime_pids_reads_windows_tasklist_csv_and_requires_exact_pid(self): + smoke = self.smoke() + self.run_history(smoke, [(42001, "Stopped"), (42002, "Stopped"), (42003, "Stopped")]) + outputs = { + 42001: '"msb.exe","42001","Console","1","8,192 K"\r\n', + 42002: "INFO: No tasks are running which match the specified criteria.\r\n", + 42003: '"msb.exe","420030","Console","1","4,096 K"\r\n', + } + + def process(command, **kwargs): + self.assertEqual(command[0], "tasklist") + self.assertTrue(kwargs["check"]) + pid = int(command[2].split()[-1]) + return subprocess.CompletedProcess(command, 0, outputs[pid], "") + + self.process.side_effect = process + platform = SimpleNamespace(name="nt", kill=mock.Mock()) + with mock.patch.object(HARNESS, "os", platform): + self.assertEqual(smoke.runtime_pids(), [42001]) + self.assertEqual(self.process.call_count, 3) + platform.kill.assert_not_called() + + def test_runtime_pids_does_not_create_a_missing_database(self): + smoke = self.smoke() + self.assertEqual(smoke.runtime_pids(), []) + self.assertFalse((smoke.home / "db/msb.db").exists()) + self.process.assert_not_called() + + def test_runtime_pids_reports_process_inspection_failure(self): + smoke = self.smoke() + self.run_history(smoke, [(43001, "Stopped")]) + self.process.side_effect = None + self.process.return_value = subprocess.CompletedProcess([], 2, "", "ps unavailable") + with mock.patch.object(HARNESS, "os", SimpleNamespace(name="posix")): + with self.assertRaisesRegex(RuntimeError, "could not inspect runtime PID 43001"): + smoke.runtime_pids() + + def test_cleanup_detects_live_runtime_pid_after_catalog_is_stopped_without_real_wait(self): + smoke = self.smoke() + self.run_history(smoke, [(44001, "Stopped")]) + + def process(command, **_kwargs): + if command[0] == "ps": + return subprocess.CompletedProcess(command, 0, "S\n", "") + return subprocess.CompletedProcess( + command, 0, json.dumps([{"name": "owned", "status": "Stopped"}]), "" + ) + + self.process.side_effect = process + # Advance a synthetic clock on each read, so the real grace period costs no wall time. + with mock.patch.object(HARNESS, "os", SimpleNamespace(name="posix")), \ + mock.patch.object(HARNESS.time, "monotonic", side_effect=iter(range(0, 100, 2))), \ + mock.patch.object(HARNESS.time, "sleep") as sleep: + errors = smoke.cleanup() + sleep.assert_called() + self.assertEqual(smoke.report["remaining_sandboxes"], []) + self.assertEqual(smoke.report["remaining_runtime_pids"], [44001]) + self.assertTrue(any("runtime PIDs are still alive" in error for error in errors)) + + def test_cleanup_succeeds_when_runtime_exits_during_grace_period(self): + smoke = self.smoke() + self.run_history(smoke, [(45001, "Stopped")]) + states = iter([(0, "S\n"), (1, "")]) + + def process(command, **kwargs): + if command[0] == "ps": + code, state = next(states) + return subprocess.CompletedProcess(command, code, state, "") + return self.successful_process(command, **kwargs) + + self.process.side_effect = process + with mock.patch.object(HARNESS, "os", SimpleNamespace(name="posix")), \ + mock.patch.object(HARNESS.time, "sleep") as sleep: + self.assertEqual(smoke.cleanup(), []) + sleep.assert_called_once() + self.assertEqual(smoke.report["remaining_runtime_pids"], []) + + def test_cleanup_defers_interrupt_and_still_attempts_all_owned_vms(self): + smoke = self.smoke() + smoke.remember("first") + smoke.remember("second") + original = {signal.SIGINT: mock.Mock(), signal.SIGTERM: mock.Mock()} + handlers = dict(original) + + def process(command, **kwargs): + if command[1] == "stop" and command[2] == "second": + handlers[signal.SIGTERM](signal.SIGTERM, None) + return self.successful_process(command, **kwargs) + + self.process.side_effect = process + with mock.patch.object(HARNESS.signal, "getsignal", side_effect=handlers.__getitem__), \ + mock.patch.object(HARNESS.signal, "signal", side_effect=handlers.__setitem__): + errors = smoke.cleanup() + self.assertEqual(handlers, original) + self.assertTrue(any("interrupted during cleanup" in error for error in errors)) + for name in ("first", "second"): + self.assertIn(["stop", name, "--timeout", "5"], self.commands()) + self.assertIn(["list", "--format", "json"], self.commands()) + + def test_execute_cleans_and_persists_failure_even_when_interrupted(self): + for index, error in enumerate((RuntimeError("operation failed"), KeyboardInterrupt())): + with self.subTest(error=type(error).__name__): + smoke = self.smoke(f"failure-{index}") + self.process.reset_mock() + + def exercise(): + smoke.remember("possibly-started") + raise error + + with mock.patch.object(smoke, "exercise", side_effect=exercise): + self.assertEqual(smoke.execute(), 1) + report = self.report(smoke) + self.assertEqual(report["status"], "failed") + self.assertEqual(report["error"], str(error) or "interrupted") + self.assertIn(["stop", "possibly-started", "--timeout", "5"], self.commands()) + self.assertIn(["list", "--format", "json"], self.commands()) + self.assertTrue(any(command[0] == "logs" for command in self.commands())) + self.assertEqual(report["cleanup_errors"], []) + + def test_execute_setup_failure_still_verifies_cleanup(self): + smoke = self.smoke() + + def process(command, **kwargs): + if command[1] == "pull": + return subprocess.CompletedProcess(command, 1, "", "image preparation failed") + return self.successful_process(command, **kwargs) + + self.process.side_effect = process + with mock.patch.object(smoke, "exercise") as exercise: + self.assertEqual(smoke.execute(), 1) + exercise.assert_not_called() + report = self.report(smoke) + self.assertEqual(report["status"], "failed") + self.assertIn("prepare-image", report["error"]) + self.assertEqual(report["operations_ms"], 0) + self.assertIn(["list", "--format", "json"], self.commands()) + + def test_execute_fails_if_cleanup_leaves_a_resident_vm(self): + smoke = self.smoke() + + def process(command, **kwargs): + if command[1] == "list": + return subprocess.CompletedProcess( + command, 0, json.dumps([{"name": "owned", "status": "Running"}]), "" + ) + return self.successful_process(command, **kwargs) + + self.process.side_effect = process + with mock.patch.object(smoke, "exercise", side_effect=lambda: smoke.remember("owned")): + self.assertEqual(smoke.execute(), 1) + report = self.report(smoke) + self.assertEqual(report["status"], "failed") + self.assertIsNone(report["error"]) + self.assertTrue(report["cleanup_errors"]) + + def test_execute_reports_success_only_after_clean_inventory(self): + smoke = self.smoke() + with mock.patch.object(smoke, "exercise"): + self.assertEqual(smoke.execute(), 0) + report = self.report(smoke) + self.assertEqual(report["status"], "passed") + self.assertIsNone(report["error"]) + self.assertEqual(report["cleanup_errors"], []) + self.assertEqual(self.commands()[-1], ["list", "--format", "json"]) + + def test_success_removes_only_owned_home_and_keeps_text_evidence(self): + smoke = self.smoke() + smoke.home.mkdir() + (smoke.home / "test-ram").write_bytes(b"fixture") + sibling = self.root / "unrelated" + sibling.write_text("keep") + with mock.patch.object(smoke, "exercise"): + self.assertEqual(smoke.execute(), 0) + self.assertFalse(smoke.home.exists()) + self.assertTrue(self.report(smoke)["home_removed"]) + self.assertTrue(any(smoke.logs.iterdir())) + self.assertEqual(sibling.read_text(), "keep") + + def test_failure_retains_owned_home_for_investigation(self): + smoke = self.smoke() + smoke.home.mkdir() + fixture = smoke.home / "test-ram" + fixture.write_bytes(b"fixture") + with mock.patch.object(smoke, "exercise", side_effect=RuntimeError("injected failure")): + self.assertEqual(smoke.execute(), 1) + self.assertEqual(fixture.read_bytes(), b"fixture") + self.assertFalse(self.report(smoke)["home_removed"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/smoke/cli/test_transport_checkpoint.py b/scripts/smoke/cli/test_transport_checkpoint.py new file mode 100644 index 000000000..27abc61da --- /dev/null +++ b/scripts/smoke/cli/test_transport_checkpoint.py @@ -0,0 +1,551 @@ +#!/usr/bin/env python3 +"""VM-free checks for transport workload integrity, measurements, and process cleanup.""" + +import argparse +import contextlib +import hashlib +import importlib.util +import io +import json +import os +from pathlib import Path +import select +import shutil +import subprocess +import sys +import tempfile +import threading +import time +from types import SimpleNamespace +import unittest +from unittest import mock + + +SPEC = importlib.util.spec_from_file_location( + "transport_checkpoint_smoke", Path(__file__).with_name("transport-checkpoint.py")) +HARNESS = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(HARNESS) + + +class TransportCheckpointTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix="transport-unit-") + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.firmware = self.root / "fixture-firmware" + self.firmware.write_bytes(b"not a live firmware") + self.output = self.root / "logs" + self.output.mkdir() + + def smoke(self, home_parent=Path("/tmp")): + smoke = HARNESS.TransportSmoke(argparse.Namespace( + binary=Path(sys.executable), output=self.root / "run", label="candidate", + firmware=self.firmware, agentd=None, image="never-pull-this-unit-fixture", + layout="managed", timeout=3, suite_timeout=20, samples=1, stdin_mib=2, bulk_mib=1, + gate_delay=20, tcp_mib=64, home_parent=home_parent, + cases=["idle", "throughput", "paused", "full", "branch"], + input_modes=["pipe", "pty"], + )) + self.addCleanup(shutil.rmtree, smoke.home, True) + return smoke + + def test_custom_home_parent_allocates_fresh_owned_directory(self): + parent = self.root / "homes" + parent.mkdir() + existing = parent / "keep" + existing.write_text("not owned by the harness") + smoke = self.smoke(home_parent=parent) + self.assertEqual(smoke.home.parent, parent) + self.assertTrue(smoke.home.name.startswith("msb-t-")) + self.assertTrue(smoke.home.is_dir()) + self.assertEqual(smoke.env["MSB_HOME"], str(smoke.home)) + self.assertEqual(existing.read_text(), "not owned by the harness") + + def stream(self, program=None, arguments=None, is_pty=False, control=False): + # Only the small guest Python fixture runs, as a local child. The real CLI invocation + # is translated here before Popen: these tests cannot accidentally launch an msb VM. + smoke = SimpleNamespace(binary=Path(sys.executable), logs=self.output, + env=dict(os.environ), root=self.root, + track=lambda job: job, limit=lambda: 5) + actual_job = HARNESS.Job + + def local_python_job(command, env, cwd, **stdio): + position = command.index("--") + self.assertEqual(command[position + 1], "python3") + return actual_job([sys.executable, *command[position + 2:]], env, cwd, **stdio) + + if arguments is None: + arguments = [str(self.root / "gate"), str(self.root / "receipt"), "1" if is_pty else "0"] + with mock.patch.object(HARNESS, "Job", side_effect=local_python_job): + stream = HARNESS.Stream(smoke, "fixture", program or HARNESS.INPUT_PROGRAM, + arguments, is_pty, control) + self.addCleanup(stream.close) + return stream + + def test_nearest_rank_percentiles_include_real_sample_counts(self): + self.assertEqual(HARNESS.distribution([]), {"n": 0, "p50": None, "p95": None}) + self.assertEqual(HARNESS.distribution([3, 1, 2]), {"n": 3, "p50": 2, "p95": 3}) + for invalid in (-1, float("inf"), float("nan")): + with self.assertRaises(ValueError): + HARNESS.distribution([invalid]) + + def test_cpu_time_parser_keeps_fractional_and_day_values(self): + self.assertAlmostEqual(HARNESS.cpu_seconds("02:03.45"), 123.45) + self.assertEqual(HARNESS.cpu_seconds("1-02:03:04"), 93784) + with self.assertRaises(ValueError): + HARNESS.cpu_seconds("not available") + + def test_receipts_reject_corruption_truncation_and_wrong_eof(self): + receipt = dict(bytes=123, sha256="expected", eof=True, eof_kind="pipe-close") + HARNESS.verify_receipt(receipt, 123, "expected", False) + for key, value in [("bytes", 122), ("sha256", "corrupt"), ("eof", False), + ("eof_kind", "pty-veof")]: + with self.subTest(key=key), self.assertRaisesRegex(RuntimeError, "mismatch"): + HARNESS.verify_receipt({**receipt, key: value}, 123, "expected", False) + + def test_line_fixtures_are_canonical_bounded_and_sequence_sensitive(self): + self.assertEqual(len(HARNESS.INPUT_LINE), 512) + self.assertTrue(HARNESS.INPUT_LINE.endswith(b"\n")) + self.assertEqual(len(HARNESS.control_line(0)), 256) + self.assertNotEqual(HARNESS.control_line(0), HARNESS.control_line(1)) + cutoff = HARNESS.MIB + payload = HARNESS.input_bytes(0, cutoff + 512, cutoff) + self.assertEqual(len(payload), cutoff + 512) + self.assertNotEqual(payload[:512], payload[512:1024]) + self.assertTrue(payload[:512].startswith(b"BEFORE:")) + self.assertTrue(payload[cutoff:].startswith(b"AFTER!:")) + self.assertEqual(HARNESS.input_bytes(997, 70001, cutoff), payload[997:70998]) + + def test_child_receipt_requires_valid_bounded_prefix_and_correct_eof(self): + cutoff = HARNESS.MIB + value = dict(bytes=70001, sha256=HARNESS.input_digest(70001, cutoff), + eof=True, eof_kind="pipe-close") + HARNESS.verify_child_receipt(value, HARNESS.PREFIX_BYTES, cutoff, False) + for key, wrong in (("bytes", HARNESS.PREFIX_BYTES - 1), ("bytes", cutoff + 1), + ("sha256", "corrupt"), ("eof", False), ("eof_kind", "pty-quiescent")): + with self.subTest(key=key, wrong=wrong), self.assertRaises(RuntimeError): + HARNESS.verify_child_receipt({**value, key: wrong}, HARNESS.PREFIX_BYTES, cutoff, False) + pty_value = {**value, "eof": False, "eof_kind": "pty-quiescent"} + HARNESS.verify_child_receipt(pty_value, HARNESS.PREFIX_BYTES, cutoff, True) + with self.assertRaises(RuntimeError): + HARNESS.verify_child_receipt(value, HARNESS.PREFIX_BYTES, cutoff, True) + + def test_gate_timing_uses_conservative_clock_bounds(self): + before = dict(host_before_ns=1000, guest_ns=1500, host_after_ns=1200) + after = dict(host_before_ns=9000, guest_ns=9600, host_after_ns=9300) + operations = [dict(wall_end_ns=4000)] + proof = HARNESS.verify_gate_timing(dict(unix_ns=5000), before, after, operations) + self.assertEqual(proof["earliest_host_open_ns"], 4400) + with self.assertRaisesRegex(RuntimeError, "overlap unproven"): + HARNESS.verify_gate_timing(dict(unix_ns=4600), before, after, operations) + + def test_fresh_source_and_child_exec_must_finish_before_autonomous_gate(self): + before = dict(host_before_ns=1000, guest_ns=1500, host_after_ns=1200) + after = dict(host_before_ns=9000, guest_ns=9600, host_after_ns=9300) + operations = [dict(kind="restore_ready", wall_end_ns=4000)] + opened = dict(unix_ns=5000) + HARNESS.verify_gate_timing(opened, before, after, operations) + # A fast restore followed by control credit starvation is not a passing restore: + # the first independent exec must complete while inherited input is still blocked. + for kind in ("child-ready-exec", "source-ready-exec"): + with self.subTest(kind=kind), self.assertRaisesRegex(RuntimeError, "overlap unproven"): + HARNESS.verify_gate_timing(opened, before, after, + [*operations, dict(kind=kind, wall_end_ns=4500)]) + + def test_child_ready_records_first_exec_in_gate_deadline_operations(self): + smoke = self.smoke() + row = dict(case="full-pipe-0", operations=[dict(kind="restore_ready")]) + with mock.patch.object(smoke, "guest", return_value="CHILD_FRAME_OK") as guest, \ + mock.patch.object(HARNESS.time, "monotonic", side_effect=[1.25, 1.5]), \ + mock.patch.object(HARNESS.time, "time_ns", side_effect=[1000, 1250]): + smoke.child_ready(row, "child") + self.assertEqual(guest.call_args.args[:2], ("full-pipe-0-child-ready-exec", "child")) + self.assertNotIn("touch ", guest.call_args.args[2]) + self.assertTrue(row["child_exec_independent"]) + self.assertEqual(row["child_ready_exec_ms"], 250) + self.assertEqual(row["operations"], [dict(kind="restore_ready"), + dict(kind="child-ready-exec", start=1.25, end=1.5, + wall_start_ns=1000, wall_end_ns=1250)]) + + def test_source_ready_records_fresh_exec_without_opening_consumer_gate(self): + smoke = self.smoke() + row = dict(case="full-pipe-0", operations=[dict(kind="capture")]) + with mock.patch.object(smoke, "guest", return_value="SOURCE_FRAME_OK") as guest, \ + mock.patch.object(HARNESS.time, "monotonic", side_effect=[2.0, 2.125]), \ + mock.patch.object(HARNESS.time, "time_ns", side_effect=[2000, 2125]): + smoke.source_ready(row) + guest.assert_called_once_with("full-pipe-0-source-ready-exec", "source", + "printf 'SOURCE_FRAME_OK\\n'") + self.assertTrue(row["source_exec_independent"]) + self.assertEqual(row["source_ready_exec_ms"], 125) + self.assertEqual(row["operations"], [dict(kind="capture"), + dict(kind="source-ready-exec", start=2.0, end=2.125, + wall_start_ns=2000, wall_end_ns=2125)]) + + def test_source_ready_rejects_corrupt_marker_and_keeps_timing_evidence(self): + smoke = self.smoke() + row = dict(case="paused-pipe-0") + with mock.patch.object(smoke, "guest", return_value="SOURCE_FRAME_OK extra"): + with self.assertRaisesRegex(RuntimeError, "source framing mismatch"): + smoke.source_ready(row) + self.assertNotIn("source_exec_independent", row) + self.assertEqual(row["operations"][0]["kind"], "source-ready-exec") + self.assertGreaterEqual(row["source_ready_exec_ms"], 0) + + def test_source_ready_latency_is_summarized_only_for_passing_samples(self): + samples = [dict(kind="full", tty=False, status="passed", source_ready_exec_ms=12), + dict(kind="full", tty=False, status="passed", source_ready_exec_ms=18), + dict(kind="full", tty=False, status="failed", source_ready_exec_ms=900)] + self.assertEqual(HARNESS.summarize(samples)["full/pipe/source_ready_exec_ms"], + dict(n=2, p50=12, p95=18)) + + def test_regression_timer_opens_without_an_ordinary_control_exec(self): + stream = self.stream(arguments=[str(self.root / "absent-gate"), str(self.root / "receipt"), + "0", str(HARNESS.PREFIX_BYTES), ".35"]) + stream.await_ready() + stream.feed(2 * HARNESS.MIB, regression=True) + stream.await_prefix() + stream.offer.set() + stream.await_pressure() + stream.after_cut.set() + self.assertEqual(stream.finish()["receipt"]["bytes"], 2 * HARNESS.MIB) + self.assertFalse((self.root / "absent-gate").exists()) + self.assertTrue((self.root / "receipt.gate").exists()) + + def test_sequence_sensitive_pty_regression_keeps_suffix_and_canonical_eof(self): + stream = self.stream(arguments=[str(self.root / "gate"), str(self.root / "receipt"), + "1", str(HARNESS.PREFIX_BYTES)], is_pty=True) + stream.await_ready() + stream.feed(2 * HARNESS.MIB, regression=True) + stream.await_prefix() + stream.offer.set() + self.assertGreater(stream.await_pressure()["eagain_count"], 0) + stream.after_cut.set() + (self.root / "gate").touch() + value = stream.finish()["receipt"] + self.assertEqual(value["bytes"], 2 * HARNESS.MIB) + self.assertEqual(value["eof_kind"], "pty-veof") + + def test_runtime_and_home_isolation_discards_ambient_overrides(self): + with mock.patch.dict(os.environ, {"MSB_HOME": "/caller", "MSB_BACKEND": "cloud", + "MSB_PROFILE": "production", "MSB_PATH": "/wrong/msb", + "MSB_AGENTD_PATH": "/wrong/agentd", + "MSB_CONFIG_PATH": "/caller/config"}): + smoke = self.smoke() + self.assertEqual(smoke.env["MSB_PATH"], str(Path(sys.executable).resolve())) + self.assertEqual(smoke.env["MSB_LIBKRUNFW_PATH"], str(self.firmware.resolve())) + self.assertEqual(smoke.env["MSB_HOME"], str(smoke.home)) + self.assertTrue(str(smoke.home).startswith("/tmp/msb-t-")) + self.assertEqual(smoke.env["MSB_BACKEND"], "local") + for key in ("MSB_CONFIG_PATH", "MSB_PROFILE", "MSB_AGENTD_PATH"): + self.assertNotIn(key, smoke.env) + + def test_refuses_existing_output_directory(self): + smoke = self.smoke() + with self.assertRaises(FileExistsError): + self.smoke() + self.assertTrue((smoke.root / "report.json").exists()) + + def test_pipe_backpressure_preserves_partial_write_bytes_and_eof(self): + stream = self.stream() + stream.await_ready() + actual_write = os.write + + def short_write(fd, data): + return actual_write(fd, data[:997]) # Force non-line-aligned short writes. + + with mock.patch.object(HARNESS.os, "write", side_effect=short_write): + stream.feed(2 * HARNESS.MIB) + proof = stream.await_pressure() + self.assertGreater(proof["eagain_count"], 0) + self.assertLess(proof["forwarded_bytes"], 2 * HARNESS.MIB) + (self.root / "gate").touch() + result = stream.finish() + self.assertEqual(result["receipt"]["bytes"], 2 * HARNESS.MIB) + self.assertEqual(result["receipt"]["eof_kind"], "pipe-close") + stream.close() + stream.close() # Global cleanup may revisit an already-finished stream. + + @unittest.skipUnless(os.name == "posix", "PTY workload requires POSIX") + def test_pty_backpressure_preserves_bytes_and_canonical_eof(self): + stream = self.stream(is_pty=True) + stream.await_ready() + stream.feed(2 * HARNESS.MIB) + self.assertGreater(stream.await_pressure()["eagain_count"], 0) + (self.root / "gate").touch() + result = stream.finish() + self.assertEqual(result["receipt"]["eof_kind"], "pty-veof") + self.assertEqual(result["receipt"]["bytes"], 2 * HARNESS.MIB) + stream.close() + stream.close() + + def test_regression_producer_acknowledges_prefix_and_withholds_post_cut_bytes(self): + stream = self.stream(arguments=[str(self.root / "gate"), str(self.root / "receipt"), + "0", str(HARNESS.PREFIX_BYTES)]) + stream.await_ready() + stream.feed(2 * HARNESS.MIB, regression=True) + prefix = stream.await_prefix() + self.assertEqual(prefix["bytes"], HARNESS.PREFIX_BYTES) + self.assertEqual(stream.sent, HARNESS.PREFIX_BYTES) + stream.offer.set() + self.assertGreater(stream.await_pressure()["eagain_count"], 0) + (self.root / "gate").touch() + deadline = time.monotonic() + 3 + while stream.sent < stream.cutoff and time.monotonic() < deadline: + time.sleep(.01) + self.assertEqual(stream.sent, stream.cutoff) + self.assertIsNone(stream.receipt) # No host EOF or AFTER! suffix before explicit release. + stream.after_cut.set() + self.assertEqual(stream.finish()["receipt"]["bytes"], 2 * HARNESS.MIB) + + def test_detached_child_fixtures_pipe_eof_and_pty_no_half_close(self): + # Simulate only the fixture's inherited input endpoint, not VM checkpoint machinery: + # pipe closes after its prefix; PTY stays open and reports bounded quiescence. + for is_pty in (False, True): + with self.subTest(is_pty=is_pty): + gate = self.root / ("child-gate-" + str(is_pty)) + receipt = self.root / ("child-receipt-" + str(is_pty)) + stream = self.stream(arguments=[str(gate), str(receipt), "1" if is_pty else "0", + str(HARNESS.PREFIX_BYTES)], is_pty=is_pty) + stream.await_ready() + stream.cutoff = HARNESS.MIB + data = HARNESS.input_bytes(0, HARNESS.PREFIX_BYTES + 4096, stream.cutoff) + + def send_prefix(): + offset = 0 + while offset < len(data): + try: + offset += os.write(stream.input_fd, data[offset:offset + 997]) + except BlockingIOError: + select.select([], [stream.input_fd], [], .02) + if not is_pty: + stream.job.process.stdin.close() + + writer = threading.Thread(target=send_prefix, daemon=True) + writer.start() + stream.await_prefix() + Path(str(receipt) + ".probe").touch() + gate.touch() + writer.join(3) + self.assertFalse(writer.is_alive()) + self.assertEqual(stream.job.wait(3), 0) + stream.reader.join(3) + self.assertIsNone(stream.error) + HARNESS.verify_child_receipt(stream.receipt, HARNESS.PREFIX_BYTES, + stream.cutoff, is_pty) + self.assertEqual(stream.receipt["bytes"], len(data)) + stream.close() + + def test_stream_missing_receipt_is_not_successful_exit(self): + stream = self.stream("print('INPUT_READY', flush=True)", []) + stream.await_ready() + stream.size, stream.expected_digest = 0, hashlib.sha256(b"").hexdigest() + with self.assertRaisesRegex(RuntimeError, "without final receipt"): + stream.finish() + + def test_control_output_has_exact_sequenced_framing(self): + stop = self.root / "control-stop" + stream = self.stream(HARNESS.CONTROL_PROGRAM, [str(stop)], control=True) + self.assertTrue(stream.job.process.stdin.closed) + stream.await_ready() + deadline = time.monotonic() + 2 + while stream.control_bytes < 8192 and time.monotonic() < deadline: + time.sleep(.01) + stop.touch() + result = stream.finish() + self.assertGreaterEqual(result["bytes"], 8192) + + def test_control_reordered_or_duplicate_frame_fails(self): + program = f"import sys; sys.stdout.buffer.write(b'CONTROL_READY\\n' + {HARNESS.control_line(1)!r}); sys.stdout.flush()" + stream = self.stream(program, [], control=True) + stream.await_ready() + with self.assertRaisesRegex(RuntimeError, "unexpected stream frame"): + stream.finish() + + def test_owned_process_group_termination_is_bounded(self): + job = HARNESS.Job([sys.executable, "-c", "import time; time.sleep(60)"], + dict(os.environ), self.root, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, stdin=subprocess.DEVNULL) + self.addCleanup(job.terminate) + started = time.monotonic() + job.terminate() + self.assertIsNotNone(job.process.poll()) + self.assertLess(time.monotonic() - started, 4) + + def test_cleanup_closes_new_operations_but_permits_cleanup_commands(self): + smoke = self.smoke() + smoke.cleaning = True + with mock.patch.object(HARNESS, "Job") as job: + with self.assertRaisesRegex(RuntimeError, "closed admission"): + smoke.invoke("late-copy", "copy", "fixture", "source:/fixture") + job.assert_not_called() + # A real local Python process proves cleanup is independent of an expired deadline; + # no msb process is involved because binary=sys.executable in smoke(). + smoke.deadline = time.monotonic() - 1 + result = smoke.run("cleanup-probe", "-c", "print('cleanup')", phase="cleanup", timeout=2) + self.assertEqual(result[0], "cleanup") + + def test_command_timeout_records_failure_and_reaps_owned_process(self): + smoke = self.smoke() + with self.assertRaisesRegex(RuntimeError, "timeout=True"): + smoke.invoke("timeout", "-c", "import time; time.sleep(60)", timeout=.05) + self.assertTrue(smoke.report["commands"][-1]["timed_out"]) + self.assertIsNotNone(smoke.jobs[-1].process.poll()) + + def test_cleanup_attempts_every_owned_client_before_vm_cleanup(self): + smoke = self.smoke() + smoke.streams = [mock.Mock(), mock.Mock()] + smoke.streams[0].close.side_effect = RuntimeError("injected stream error") + smoke.jobs = [mock.Mock(), mock.Mock()] + worker = mock.Mock() + worker.thread.is_alive.return_value = False + smoke.bulk_workers = [worker] + with mock.patch.object(HARNESS.BASE.Smoke, "cleanup_owned", return_value=[]) as base: + errors = smoke.cleanup_owned() + self.assertTrue(smoke.cleaning) + self.assertEqual(len(errors), 1) + for stream in smoke.streams: + stream.close.assert_called_once() + for job in smoke.jobs: + job.terminate.assert_called_once() + worker.stop.set.assert_called_once() + worker.thread.join.assert_called_once_with(3) + base.assert_called_once() + + def test_stdin_only_throughput_does_not_create_control_or_bulk_workers(self): + smoke = self.smoke() + size = smoke.args.stdin_mib * HARNESS.MIB + receipt = dict(bytes=size, sha256=HARNESS.input_digest(size), eof=True, eof_kind="pipe-close") + stream = mock.Mock(size=size, expected_digest=receipt["sha256"]) + stream.finish.return_value = dict(bytes=size, seconds=1, receipt=receipt) + + def guest(_label, _name, command): + if command.startswith("touch "): + return "" + if command.startswith("cat "): + return json.dumps(receipt) + return "POST_FRAME_OK" + + with mock.patch.object(HARNESS, "Stream", return_value=stream) as constructor, \ + mock.patch.object(HARNESS, "Bulk") as bulk, \ + mock.patch.object(smoke, "guest", side_effect=guest), \ + contextlib.redirect_stdout(io.StringIO()): + smoke.scenario("stdin-throughput", False, 0) + constructor.assert_called_once() + bulk.assert_not_called() + stream.feed.assert_called_once_with(size) + stream.await_pressure.assert_not_called() + sample = smoke.report["samples"][0] + self.assertEqual(sample["status"], "passed") + self.assertEqual(sample["bulk"], {}) + self.assertNotIn("control_stdout", sample) + + def test_source_exec_precedes_post_cut_release_and_source_stdin_finish(self): + smoke = self.smoke() + size = smoke.args.stdin_mib * HARNESS.MIB + receipt = dict(bytes=size, sha256=HARNESS.input_digest(size, size - HARNESS.LATE_BYTES), + eof=True, eof_kind="pipe-close") + stream = mock.Mock(size=size, expected_digest=receipt["sha256"]) + stream.after_cut = threading.Event() + stream.await_prefix.return_value = dict(bytes=HARNESS.PREFIX_BYTES) + stream.await_pressure.return_value = dict(eagain_count=1) + events = [] + + def guest(_label, _name, command): + if command == "printf 'SOURCE_FRAME_OK\\n'": + self.assertFalse(stream.after_cut.is_set()) + self.assertEqual(events, ["pause", "resume"]) + events.append("source-ready-exec") + return "SOURCE_FRAME_OK" + return json.dumps(receipt) if command.startswith("cat ") else "POST_FRAME_OK" + + def finish(): + self.assertTrue(stream.after_cut.is_set()) + self.assertEqual(events[-1], "source-ready-exec") + events.append("stdin-finish") + return dict(bytes=size, seconds=1, receipt=receipt) + + def gate_proof(row, _receipt): + self.assertEqual(row["operations"][-1]["kind"], "source-ready-exec") + self.assertTrue(row["source_exec_independent"]) + + stream.finish.side_effect = finish + with mock.patch.object(HARNESS, "Stream", return_value=stream), \ + mock.patch.object(smoke, "guest", side_effect=guest), \ + mock.patch.object(smoke, "run", side_effect=lambda _label, kind, *args: events.append(kind)), \ + mock.patch.object(smoke, "clock_probe", return_value={}), \ + mock.patch.object(smoke, "check_status"), \ + mock.patch.object(smoke, "gate_proof", side_effect=gate_proof), \ + contextlib.redirect_stdout(io.StringIO()): + smoke.scenario("paused", False, 0) + self.assertEqual(events, ["pause", "resume", "source-ready-exec", "stdin-finish"]) + self.assertEqual(smoke.report["samples"][0]["status"], "passed") + + def test_tcp_source_exec_follows_child_probe_and_precedes_input_finish(self): + smoke = self.smoke() + smoke.args.tcp_mib = 1 + receipt = dict(bytes=HARNESS.MIB, sha256=HARNESS.input_digest(HARNESS.MIB, HARNESS.MIB), + eof=True, eof_kind="pipe-close") + server, client = mock.Mock(), mock.Mock() + server.finish.return_value = dict(receipt=receipt) + client.await_pressure.return_value = dict(eagain_count=1) + events = [] + + def guest(_label, _name, command): + if command == "printf 'SOURCE_FRAME_OK\\n'": + self.assertEqual(events, ["child-ready-exec"]) + events.append("source-ready-exec") + return "SOURCE_FRAME_OK" + return "TCP_POST_FRAME_OK" + + def finish(): + self.assertEqual(events, ["child-ready-exec", "source-ready-exec"]) + events.append("tcp-finish") + return receipt + + def gate_proof(row, _receipt): + self.assertEqual(row["operations"][-1]["kind"], "source-ready-exec") + self.assertTrue(row["source_exec_independent"]) + + client.finish.side_effect = finish + with mock.patch.object(HARNESS, "Stream", return_value=server), \ + mock.patch.object(HARNESS.TCP, "InlineTcp", return_value=client), \ + mock.patch.object(smoke, "guest", side_effect=guest), \ + mock.patch.object(smoke, "run"), \ + mock.patch.object(smoke, "clock_probe", return_value={}), \ + mock.patch.object(smoke, "child_ready", side_effect=lambda *args: events.append("child-ready-exec")), \ + mock.patch.object(smoke, "gate_proof", side_effect=gate_proof), \ + mock.patch.object(smoke, "stop"), \ + contextlib.redirect_stdout(io.StringIO()): + smoke.tcp_scenario("tcp-branch", 0) + self.assertEqual(events, ["child-ready-exec", "source-ready-exec", "tcp-finish"]) + self.assertEqual(smoke.report["samples"][0]["status"], "passed") + + def test_failed_samples_and_failed_baselines_do_not_generate_performance_claims(self): + sample = dict(kind="full", tty=False, status="failed", capture_ms=12) + self.assertEqual(HARNESS.summarize([sample]), {}) + sample["status"] = "passed" + stats = HARNESS.summarize([sample]) + self.assertEqual(stats["full/pipe/capture_ms"]["n"], 1) + reports = {"baseline": dict(status="failed", statistics=stats), + "candidate": dict(status="passed", statistics=stats)} + self.assertEqual(HARNESS.compare(reports), {}) + reports["baseline"]["status"] = "passed" + for report in reports.values(): + report["image_manifest_digest"] = "sha256:fixture" + report["parameters"] = {"samples": 1} + report["firmware_sha256"] = "same-firmware" + report["host"] = "same-host" + report["host_node"] = "same-machine" + self.assertEqual(HARNESS.compare(reports)["full/pipe/capture_ms"]["candidate_over_baseline_p50"], 1) + for field in ("firmware_sha256", "host", "host_node"): + previous = reports["candidate"][field] + reports["candidate"][field] = "different" + self.assertEqual(HARNESS.compare(reports), {}) + reports["candidate"][field] = previous + reports["candidate"]["image_manifest_digest"] = "sha256:different-image" + self.assertEqual(HARNESS.compare(reports), {}) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/smoke/cli/test_transport_tcp.py b/scripts/smoke/cli/test_transport_tcp.py new file mode 100644 index 000000000..59ec0c878 --- /dev/null +++ b/scripts/smoke/cli/test_transport_tcp.py @@ -0,0 +1,116 @@ +"""VM-free checks for the opt-in current-wire inline TCP checkpoint probe.""" + +import hashlib +import importlib.util +import json +from pathlib import Path +import socket +import struct +import tempfile +import threading +import unittest + + +SPEC = importlib.util.spec_from_file_location("transport_tcp", Path(__file__).with_name("transport_tcp.py")) +TCP = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(TCP) + + +class TcpProbeTests(unittest.TestCase): + def test_cbor_matches_fixed_wire_fixture_and_rejects_ambiguous_values(self): + self.assertEqual(TCP.cbor({"data": b"abc"}), bytes.fromhex("a1646461746143616263")) + self.assertEqual(TCP.uncbor(bytes.fromhex("a1646461746143616263")), {"data": b"abc"}) + for value in (0, 23, 24, 255, 256, 65535, 65536, 2 ** 32, "é", b"z" * 65536): + self.assertEqual(TCP.uncbor(TCP.cbor(value)), value) + for raw in (b"", b"\xbf\xff", b"\x00\x00", b"\x63x", bytes.fromhex("a2617800617801")): + with self.subTest(raw=raw), self.assertRaises(ValueError): + TCP.uncbor(raw) + with self.assertRaises(ValueError): + TCP.cbor(-1) + + def test_connect_frame_omits_bulk_offer_and_uses_session_start_flag(self): + encoded = TCP.frame(123, 9, "core.tcp.connect", {"host": "127.0.0.1", "port": 32017}) + self.assertEqual(struct.unpack(">I", encoded[:4])[0], len(encoded) - 4) + self.assertEqual(struct.unpack(">IB", encoded[4:9]), (123, 2)) + envelope = TCP.uncbor(encoded[9:]) + self.assertEqual(envelope["v"], 9) + self.assertEqual(TCP.uncbor(envelope["p"]), {"host": "127.0.0.1", "port": 32017}) + with self.assertRaises(ValueError): + TCP.frame(1, 9, "core.tcp.data", {"data": b"x" * TCP.MAX_FRAME}) + + def test_bounded_local_relay_saturation_preserves_data_and_tcp_eof(self): + with tempfile.TemporaryDirectory(prefix="tcp-unit-", dir="/tmp") as directory: + path = Path(directory) / "agent.sock" + gate, errors = threading.Event(), [] + listener = socket.socket(socket.AF_UNIX) + self.addCleanup(listener.close) + listener.bind(str(path)) + listener.listen(1) + listener.settimeout(5) + payload = bytes(range(256)) * 8192 + expected = dict(bytes=len(payload), sha256=hashlib.sha256(payload).hexdigest(), + eof=True, eof_kind="pipe-close") + + def relay(): + try: + peer, _ = listener.accept() + with peer: + peer.settimeout(5) + + def exact(size): + data = bytearray() + while len(data) < size: + block = peer.recv(size - len(data)) + if not block: + raise RuntimeError("unexpected test client EOF") + data.extend(block) + return bytes(data) + + def message(): + length = struct.unpack(">I", exact(4))[0] + raw = exact(length) + self.assertEqual(struct.unpack(">I", raw[:4])[0], 11) + return TCP.uncbor(raw[5:]) + + peer.sendall(struct.pack(">II", 11, 111) + TCP.frame(0, 9, "core.ready", {})) + self.assertEqual(message()["t"], "core.tcp.connect") + peer.sendall(TCP.frame(11, 9, "core.tcp.connected", {})) + if not gate.wait(5): + raise RuntimeError("test gate did not release") + received = bytearray() + while True: + current = message() + if current["t"] == "core.tcp.eof": + break + self.assertEqual(current["t"], "core.tcp.data") + received.extend(TCP.uncbor(current["p"])["data"]) + self.assertEqual(received, payload) + terminal = bytearray(TCP.frame(11, 9, "core.tcp.closed", {})) + terminal[8] = 1 + output = (TCP.frame(11, 9, "core.tcp.data", {"data": json.dumps(expected).encode()}) + + TCP.frame(11, 9, "core.tcp.eof", {}) + terminal) + for offset in range(0, len(output), 7): + peer.sendall(output[offset:offset + 7]) + except BaseException as error: + errors.append(error) + + thread = threading.Thread(target=relay, daemon=True) + thread.start() + client = TCP.InlineTcp(path, 32017, lambda: 5) + try: + client.feed(len(payload), lambda offset, count: payload[offset:offset + count]) + proof = client.await_pressure() + self.assertGreater(proof["eagain_count"], 0) + self.assertLess(proof["payload_bytes_framed"], len(payload)) + gate.set() + self.assertEqual(client.finish(), expected) + finally: + gate.set() + client.close() + thread.join(5) + self.assertFalse(thread.is_alive()) + self.assertEqual(errors, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/smoke/cli/transport-checkpoint.py b/scripts/smoke/cli/transport-checkpoint.py new file mode 100644 index 000000000..bda5f98a2 --- /dev/null +++ b/scripts/smoke/cli/transport-checkpoint.py @@ -0,0 +1,1092 @@ +#!/usr/bin/env python3 +"""Isolated live transport/checkpoint regression and baseline/candidate comparison. + +Example (use matching, codesigned runtime/agentd/firmware builds): + python3 scripts/smoke/cli/transport-checkpoint.py \ + --baseline /build/before/msb --baseline-firmware /build/before/libkrunfw.dylib \ + --candidate /build/after/msb --candidate-firmware /build/after/libkrunfw.dylib + +Images are pulled/materialized before measurements; timed programs use no external network. +Pipe cases restore installed snapshots eagerly; PTY cases restore .msb archives with --forked. +Use --cases idle throughput for a passing pre/post performance baseline independently of the +freeze/thaw regression cases. Opt-in tcp-paused/tcp-full/tcp-branch cases exercise inline +TCP backpressure without BulkOffer; restored TCP connections are intentionally not inherited. +Use --cases stdin-throughput to isolate stdin from concurrent control output and bulk copies. +Results retain individual samples, correctness checks, and nearest-rank p50/p95. PTY EOF +means canonical VEOF, not a nonexistent PTY half-close. Bulk overlap is measured at the CLI +operation boundary, not claimed as an instrumented guest-frame boundary. This is a POSIX +harness. It never reads or stops sandboxes from the caller's MSB_HOME. +Use --home-parent /short/disk/path when /tmp has a RAM or per-user quota; every run still +creates and owns a fresh home underneath that directory. +""" + +import argparse +import errno +import hashlib +import importlib.util +import json +import math +import os +from pathlib import Path +import platform +import pty +import resource +import select +import shlex +import signal +import sqlite3 +import subprocess +import sys +import tempfile +import threading +import time +import tty + + +SPEC = importlib.util.spec_from_file_location( + "snapshot_branch_smoke", Path(__file__).with_name("snapshot-branch.py")) +BASE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(BASE) +TCP_SPEC = importlib.util.spec_from_file_location( + "transport_tcp_probe", Path(__file__).with_name("transport_tcp.py")) +TCP = importlib.util.module_from_spec(TCP_SPEC) +TCP_SPEC.loader.exec_module(TCP) +MIB = 1024 * 1024 +INPUT_LINE = b"0123456789abcdef" * 31 + b"0123456789abcde\n" # 512-byte canonical lines. +PREFIX_BYTES = 64 * 1024 +LATE_BYTES = 64 * 1024 +CONTROL_SUFFIX = b"0123456789abcdef" * 14 + b"abcdef\n" + +# Regression consumers acknowledge a prefix, then wait behind an autonomous timer gate. +# The independent gate timestamp proves fresh metadata exec completed without draining input. +# Throughput and VM-free fixture modes can still open a file gate explicitly. +INPUT_PROGRAM = r''' +import hashlib, json, os, select, sys, termios, time +gate, receipt, is_pty = sys.argv[1], sys.argv[2], sys.argv[3] == "1" +prefix = int(sys.argv[4]) if len(sys.argv) > 4 else 0 +delay = float(sys.argv[5]) if len(sys.argv) > 5 else 0 +if is_pty: + attrs = termios.tcgetattr(0) + attrs[0] &= ~(termios.ICRNL | termios.INLCR | termios.IGNCR) + attrs[1] &= ~termios.OPOST + attrs[3] = (attrs[3] | termios.ICANON) & ~(termios.ECHO | termios.ECHONL) + attrs[6][termios.VEOF] = b"\x04" + termios.tcsetattr(0, termios.TCSANOW, attrs) +print("INPUT_READY", flush=True) +h, size = hashlib.sha256(), 0 +while size < prefix: + block = os.read(0, min(65536, prefix - size)) + if not block: raise RuntimeError("EOF before acknowledged prefix") + h.update(block); size += len(block) +if prefix: + value = dict(bytes=size, sha256=h.hexdigest()) + with open(receipt + ".prefix", "w") as f: json.dump(value, f) + print("INPUT_PREFIX " + json.dumps(value), flush=True) +deadline = time.monotonic() + delay if delay else float("inf") +while not os.path.exists(gate) and time.monotonic() < deadline: time.sleep(.005) +if prefix: + with open(receipt + ".gate", "w") as f: + json.dump(dict(unix_ns=time.time_ns(), monotonic_ns=time.monotonic_ns()), f) +# A restored PTY deliberately has no inherited host input/half-close. Its bounded probe +# drains available canonical lines, then records quiescence, not an invented EOF. +probe = False +eof = False +while True: + if is_pty and prefix: + # Detect the child marker even if a slow restore outlived the autonomous timer. + # Otherwise that child could enter a blocking read before fresh exec creates it. + probe = os.path.exists(receipt + ".probe") + if not select.select([0], [], [], 1 if probe else .05)[0]: + if probe: break + continue + block = os.read(0, 65536) + if not block: + eof = True + break + h.update(block); size += len(block) +value = dict(bytes=size, sha256=h.hexdigest(), eof=eof, + eof_kind="pty-quiescent" if probe else "pty-veof" if is_pty else "pipe-close") +with open(receipt + ".tmp" if prefix else receipt, "w") as f: json.dump(value, f) +if prefix: os.replace(receipt + ".tmp", receipt) # No partial JSON for child observers. +print("INPUT_RESULT " + json.dumps(value), flush=True) +''' + +CONTROL_PROGRAM = r''' +import os, sys, time +stop = sys.argv[1] +suffix = b"0123456789abcdef" * 14 + b"abcdef\n" +sys.stdout.buffer.write(b"CONTROL_READY\n"); sys.stdout.buffer.flush() +n = 0 +while not os.path.exists(stop): + block = b"".join(b"CONTROL:" + f"{i:016x}".encode() + b":" + suffix for i in range(n, n + 32)) + sys.stdout.buffer.write(block); sys.stdout.buffer.flush(); n += 32 + time.sleep(.001) +print("CONTROL_DONE " + str(n), flush=True) +''' + +TCP_PROGRAM = r''' +import hashlib, json, os, socket, sys, time +gate, receipt, port, delay = sys.argv[1], sys.argv[2], int(sys.argv[3]), float(sys.argv[4]) +listener = socket.socket() +listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +listener.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 16384) +listener.bind(("127.0.0.1", port)); listener.listen(1) +print("INPUT_READY", flush=True) +peer, _ = listener.accept(); listener.close() +deadline = time.monotonic() + delay +while time.monotonic() < deadline: time.sleep(.005) +with open(receipt + ".gate", "w") as f: + json.dump(dict(unix_ns=time.time_ns(), monotonic_ns=time.monotonic_ns()), f) +h, size = hashlib.sha256(), 0 +while True: + block = peer.recv(65536) + if not block: break + h.update(block); size += len(block) +value = dict(bytes=size, sha256=h.hexdigest(), eof=True, eof_kind="pipe-close") +with open(receipt + ".tmp", "w") as f: json.dump(value, f) +os.replace(receipt + ".tmp", receipt) +peer.sendall(json.dumps(value).encode() + b"\n") +peer.shutdown(socket.SHUT_WR); peer.close() +print("INPUT_RESULT " + json.dumps(value), flush=True) +''' + + +def bounded_int(value): + number = int(value) + if not 1 <= number <= 1024: + raise argparse.ArgumentTypeError("value must be between 1 and 1024") + return number + + +def distribution(values): + """Do not invent percentiles for failed/missing measurements or interpolate tiny samples.""" + values = sorted(values) + if not values: + return {"n": 0, "p50": None, "p95": None} + if any(not math.isfinite(v) or v < 0 for v in values): + raise ValueError("samples must be finite and nonnegative") + return {"n": len(values), "p50": values[math.ceil(len(values) * .50) - 1], + "p95": values[math.ceil(len(values) * .95) - 1]} + + +def file_digest(path): + digest = hashlib.sha256() + with path.open("rb") as source: + for block in iter(lambda: source.read(MIB), b""): + digest.update(block) + return digest.hexdigest() + + +def control_line(index): + return b"CONTROL:" + f"{index:016x}".encode() + b":" + CONTROL_SUFFIX + + +def cpu_seconds(text): + """Parse ps TIME ([[days-]hours:]minutes:seconds), retaining its coarse precision.""" + text = text.strip() + days = 0 + if "-" in text: + day, text = text.split("-", 1) + days = int(day) + fields = [float(part) for part in text.split(":")] + if len(fields) not in (2, 3): + raise ValueError(f"unrecognized ps CPU time: {text!r}") + value = days * 86400 + for field, multiplier in zip(reversed(fields), (1, 60, 3600)): + value += field * multiplier + return value + + +def verify_receipt(value, size, digest, is_pty): + expected = dict(bytes=size, sha256=digest, eof=True, + eof_kind="pty-veof" if is_pty else "pipe-close") + if value != expected: + raise RuntimeError(f"stdin bytes/EOF mismatch: expected {expected}, got {value}") + + +def input_bytes(offset, size, cutoff=None): + """Regression lines carry positions; throughput keeps its original measured workload.""" + if cutoff is None: + block = INPUT_LINE * ((offset % 512 + size + 511) // 512) + else: + block = b"".join( + ((b"BEFORE" if index * 512 < cutoff else b"AFTER!") + + f":{index:016x}:".encode() + b"x" * 487 + b"\n") + for index in range(offset // 512, (offset + size + 511) // 512)) + return block[offset % 512:offset % 512 + size] + + +def input_digest(size, cutoff=None): + digest = hashlib.sha256() + if cutoff is None: + for _ in range(size // len(INPUT_LINE)): + digest.update(INPUT_LINE) + digest.update(INPUT_LINE[:size % len(INPUT_LINE)]) + return digest.hexdigest() + for offset in range(0, size, 65536): + digest.update(input_bytes(offset, min(65536, size - offset), cutoff)) + return digest.hexdigest() + + +def verify_child_receipt(value, acknowledged, cutoff, is_pty): + """Host enqueue is not guest admission: validate a prefix range, never the whole stream.""" + size = value.get("bytes") + if type(size) is not int or not acknowledged <= size <= cutoff: + raise RuntimeError(f"child input is outside the acknowledged/pre-cut range: {value}") + expected = dict(bytes=size, sha256=input_digest(size, cutoff), eof=not is_pty, + eof_kind="pty-quiescent" if is_pty else "pipe-close") + if value != expected: + raise RuntimeError(f"child input prefix/EOF mismatch: expected {expected}, got {value}") + + +def verify_gate_timing(opened, before, after, operations): + # Clock probes bracket the guest read with host reads. Use the union of both offset + # intervals, not optimistic midpoint alignment. A changing clock is evidence we cannot + # qualify, never permission to count a timer that opened before checkpoint completion. + earliest = opened["unix_ns"] + min(probe["host_before_ns"] - probe["guest_ns"] + for probe in (before, after)) + completed = max(operation["wall_end_ns"] for operation in operations) + if earliest <= completed: + raise RuntimeError("autonomous input gate may have opened before operation completion; overlap unproven") + return dict(earliest_host_open_ns=earliest, latest_operation_end_ns=completed, + minimum_margin_ms=(earliest - completed) / 1e6, + clock_assumption="guest-host wall offset remained within pre/post probe bounds") + + +class Job: + """A direct child/process group owned by this test, never a recycled catalog PID.""" + + def __init__(self, command, env, cwd, **stdio): + self.started = time.monotonic() + self.ended = None + self.process = subprocess.Popen(command, env=env, cwd=cwd, + start_new_session=True, **stdio) + + def wait(self, timeout): + code = self.process.wait(timeout=timeout) + self.ended = time.monotonic() + return code + + def terminate(self): + if self.process.poll() is not None: + return + # This child has not been reaped, so its PID cannot have been reused. Never use this + # operation for historical PIDs read from the runtime database. + try: + os.killpg(self.process.pid, signal.SIGTERM) + self.wait(1) + except subprocess.TimeoutExpired: + os.killpg(self.process.pid, signal.SIGKILL) + self.wait(2) + except ProcessLookupError: + self.wait(2) + + +class Stream: + """Bounded streaming reader plus a nonblocking stdin producer, with explicit evidence.""" + + def __init__(self, smoke, label, program, arguments, is_pty=False, control=False, no_input=False): + self.smoke, self.label = smoke, label + self.is_pty, self.control = is_pty, control + self.ready, self.done, self.cancel = threading.Event(), threading.Event(), threading.Event() + self.prefix_ready, self.offer, self.after_cut = (threading.Event() for _ in range(3)) + self.prefix_receipt = None + self.error, self.receipt = None, None + self.sent, self.blocked, self.control_bytes = 0, 0, 0 + self.producer = None + self.master = None + self.closed = False + self.started = time.monotonic() + command = [str(smoke.binary), "exec", "source", "--tty" if is_pty else "--stream", + "--", "python3", "-u", "-c", program, *arguments] + self.stderr = (smoke.logs / (label + ".stderr.log")).open("wb") + if is_pty: + self.master, slave = pty.openpty() + tty.setraw(slave) + try: + self.job = smoke.track(Job(command, smoke.env, smoke.root, + stdin=slave, stdout=slave, stderr=self.stderr)) + finally: + os.close(slave) + self.input_fd, self.output_fd = self.master, self.master + else: + self.job = smoke.track(Job(command, smoke.env, smoke.root, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=self.stderr)) + self.input_fd = self.job.process.stdin.fileno() + self.output_fd = self.job.process.stdout.fileno() + os.set_blocking(self.input_fd, False) + if control or no_input: + # This workload has no input. Leaving a pipe open strands the CLI's blocking + # Tokio stdin reader during runtime shutdown even after the guest has exited. + self.job.process.stdin.close() + self.reader = threading.Thread(target=self.read, daemon=True) + self.reader.start() + + def read(self): + buffer, lines, ended = b"", 0, False + try: + while not self.cancel.is_set(): + if not select.select([self.output_fd], [], [], .1)[0]: + continue + try: + block = os.read(self.output_fd, 65536) + except OSError as error: + if self.is_pty and error.errno == errno.EIO: + block = b"" # A PTY master reports slave closure as EIO on Linux. + else: + raise + if not block: + if buffer: + raise RuntimeError("unterminated stream framing") + break + buffer += block + if len(buffer) > MIB: + raise RuntimeError("stream exceeded bounded line buffer") + while b"\n" in buffer: + line, buffer = buffer.split(b"\n", 1) + line = line.removesuffix(b"\r") + if ended: + raise RuntimeError("bytes followed final stream receipt") + if line == (b"CONTROL_READY" if self.control else b"INPUT_READY"): + if self.ready.is_set(): + raise RuntimeError("duplicate stream readiness marker") + self.ready.set() + elif self.control and line + b"\n" == control_line(lines): + if not self.ready.is_set(): + raise RuntimeError("control bytes arrived before readiness") + lines += 1 + self.control_bytes += len(control_line(lines - 1)) + elif self.control and line.startswith(b"CONTROL_DONE "): + if int(line.split()[1]) != lines: + raise RuntimeError("control stdout sequence lost or duplicated bytes") + ended = True + elif not self.control and line.startswith(b"INPUT_RESULT "): + self.receipt = json.loads(line[len(b"INPUT_RESULT "):]) + ended = True + elif not self.control and line.startswith(b"INPUT_PREFIX "): + if not self.ready.is_set() or self.prefix_ready.is_set(): + raise RuntimeError("invalid or duplicate consumed-prefix acknowledgment") + self.prefix_receipt = json.loads(line[len(b"INPUT_PREFIX "):]) + self.prefix_ready.set() + else: + raise RuntimeError(f"unexpected stream frame: {line[:160]!r}") + if not ended and not self.cancel.is_set(): + raise RuntimeError("stream ended without final receipt") + except Exception as error: + self.error = error + finally: + self.done.set() + + def await_ready(self): + until = time.monotonic() + self.smoke.limit() + while not self.ready.wait(.02): + if self.done.is_set() or time.monotonic() >= until: + raise RuntimeError(f"{self.label}: readiness failed: {self.error}") + + def feed(self, size, regression=False): + self.size = size + self.cutoff = size - LATE_BYTES if regression else None + if regression and self.cutoff <= PREFIX_BYTES: + raise ValueError("regression input must exceed prefix plus withheld suffix") + self.expected_digest = input_digest(size, self.cutoff) + + def write(): + try: + throughput_block = INPUT_LINE * 128 + while self.sent < size and not self.cancel.is_set(): + boundary = size + if regression: + boundary = PREFIX_BYTES if self.sent < PREFIX_BYTES else self.cutoff + event = self.offer if self.sent == PREFIX_BYTES else self.after_cut + if self.sent in (PREFIX_BYTES, self.cutoff): + while not event.wait(.02): + if self.cancel.is_set(): + return + if self.sent >= self.cutoff: + boundary = size + try: + if regression: + block = input_bytes(self.sent, min(65536, boundary - self.sent), self.cutoff) + else: + offset = self.sent % len(throughput_block) + block = throughput_block[offset:offset + min( + len(throughput_block) - offset, size - self.sent)] + count = os.write(self.input_fd, block) + self.sent += count + except BlockingIOError: + self.blocked += 1 + select.select([], [self.input_fd], [], .02) + if self.cancel.is_set(): + return + if self.is_pty: + # Every data chunk ends at a newline. VEOF now yields a genuine zero-length + # guest read without closing the bidirectional host PTY or losing its output. + while not self.cancel.is_set(): + try: + os.write(self.input_fd, b"\x04") + break + except BlockingIOError: + select.select([], [self.input_fd], [], .02) + else: + self.job.process.stdin.close() + except Exception as error: + self.error = error + + self.producer = threading.Thread(target=write, daemon=True) + self.producer.start() + + def await_prefix(self): + until = time.monotonic() + self.smoke.limit() + while not self.prefix_ready.wait(.02): + if self.error or self.done.is_set() or time.monotonic() >= until: + raise RuntimeError(f"{self.label}: consumed-prefix acknowledgment failed: {self.error}") + expected = dict(bytes=PREFIX_BYTES, sha256=input_digest(PREFIX_BYTES, self.cutoff)) + if self.prefix_receipt != expected: + raise RuntimeError(f"invalid consumed-prefix acknowledgment: {self.prefix_receipt}") + return self.prefix_receipt + + def await_pressure(self): + until = time.monotonic() + self.smoke.limit() + previous, stable_since = -1, time.monotonic() + while time.monotonic() < until: + if self.error: + raise self.error + if self.sent >= (self.cutoff if self.cutoff is not None else self.size): + raise RuntimeError("input fit in forwarding queues; increase --stdin-mib; saturation unproven") + if self.sent != previous: + previous, stable_since = self.sent, time.monotonic() + elif self.blocked and time.monotonic() - stable_since >= .2: + return {"forwarded_bytes": self.sent, "eagain_count": self.blocked, + "stable_blocked_seconds": time.monotonic() - stable_since} + time.sleep(.01) + raise RuntimeError("could not demonstrate sustained stdin backpressure") + + def finish(self): + if self.producer: + self.producer.join(self.smoke.limit()) + if self.producer.is_alive(): + raise RuntimeError(f"{self.label}: stdin did not drain") + code = self.job.wait(self.smoke.limit()) + self.reader.join(self.smoke.limit()) + if self.reader.is_alive() or self.error or code: + raise RuntimeError(f"{self.label}: stream failed (exit={code}): {self.error}") + if not self.control: + verify_receipt(self.receipt, self.size, self.expected_digest, self.is_pty) + return {"bytes": self.control_bytes if self.control else self.size, + "seconds": time.monotonic() - self.started, "receipt": self.receipt} + + def close(self): + if self.closed: + return + self.closed = True + self.cancel.set() + self.job.terminate() + for thread in (self.producer, self.reader): + if thread: + thread.join(3) + if self.master is not None: + os.close(self.master) + self.master = None + else: + for file in (self.job.process.stdin, self.job.process.stdout): + file.close() + self.stderr.close() + + +class Bulk: + """Continuous finite, independently verified uploads/downloads around a checkpoint.""" + + def __init__(self, smoke, label, direction): + self.smoke, self.label, self.direction = smoke, label, direction + self.stop = threading.Event() + self.error, self.current = None, None + self.samples = [] + self.thread = threading.Thread(target=self.work, daemon=True) + self.thread.start() + + def work(self): + try: + index = 0 + while not self.stop.is_set(): + label = f"{self.label}-{self.direction}-{index}" + destination = self.smoke.root / "artifacts" / (label + ".bin") + args = ([str(self.smoke.blob), "source:/transport-upload.bin"] + if self.direction == "upload" + else ["source:/transport-download.bin", str(destination)]) + _, _, row = self.smoke.invoke(label, "copy", *args, + started=lambda job: setattr(self, "current", job)) + self.current = None + if self.direction == "upload": + digest = self.smoke.guest(label + "-hash", "source", + "sha256sum /transport-upload.bin").split()[0] + else: + digest = file_digest(destination) + destination.unlink() + if digest != self.smoke.blob_digest: + raise RuntimeError(f"{label}: bulk checksum mismatch") + self.samples.append({"start": row["start"], "end": row["end"], + "bytes": self.smoke.args.bulk_mib * MIB, + "seconds": row["ms"] / 1000, "sha256": digest}) + index += 1 + except Exception as error: + self.error = error + + def finish(self): + self.stop.set() + self.thread.join(self.smoke.limit()) + if self.thread.is_alive() or self.error: + raise RuntimeError(f"{self.label}/{self.direction}: {self.error or 'copy did not finish'}") + return self.samples + + +class TransportSmoke(BASE.Smoke): + def __init__(self, args): + self.lock = threading.RLock() + self.jobs, self.streams, self.bulk_workers, self.tcp_clients = [], [], [], [] + self.cleaning = False + super().__init__(args) + # Reports may live under a long build directory; Unix sockets cannot. Allocate the + # home independently, never borrow an existing path. Base cleanup still owns exactly + # this freshly allocated directory and only removes it after catalog/PID verification. + # Some Linux hosts mount /tmp as quota-limited tmpfs. Keep the short-path default, + # but permit an explicit disk-backed parent for archive and CoW memory fixtures. + self.home = Path(tempfile.mkdtemp(prefix="msb-t-", dir=args.home_parent)) + self.env["MSB_HOME"] = str(self.home) + self.report["home"] = str(self.home) + # Make the CLI and its launched runtime an explicit matching pair. Do not inherit a + # caller's agentd/runtime override, profile, or alternate config file accidentally. + for key in ("MSB_PATH", "MSB_LIBKRUNFW_PATH", "MSB_AGENTD_PATH"): + self.env.pop(key, None) + self.env["MSB_PATH"] = str(self.binary) + self.env["MSB_LIBKRUNFW_PATH"] = str(args.firmware.resolve(strict=True)) + if args.agentd: + self.env["MSB_AGENTD_PATH"] = str(args.agentd.resolve(strict=True)) + self.report.update(label=args.label, firmware=self.env["MSB_LIBKRUNFW_PATH"], + agentd=self.env.get("MSB_AGENTD_PATH", "embedded"), samples=[], + binary_sha256=file_digest(self.binary), + firmware_sha256=file_digest(Path(self.env["MSB_LIBKRUNFW_PATH"])), + agentd_sha256=(file_digest(Path(self.env["MSB_AGENTD_PATH"])) + if "MSB_AGENTD_PATH" in self.env else None), + host=platform.platform(), host_node=platform.node(), harness_python=sys.version, + parameters={key: getattr(args, key) for key in + ("samples", "stdin_mib", "bulk_mib", "image", "cases", "input_modes")}, + latency_scope="CLI completion; independent source/child exec verifies readiness before input drains", + throughput_scope="end-to-end CLI streams including startup/gating, not raw device throughput", + cpu_scope="source runtime ps TIME; CLI children rusage; harness process_time; not total guest CPU", + bulk_overlap_scope="checksum-verified CLI operation lifetimes, not guest-frame instrumentation") + self.report["harness_sha256"] = { + name: file_digest(Path(__file__).with_name(name)) + for name in ("transport-checkpoint.py", "transport_tcp.py", "snapshot-branch.py")} + (self.root / "artifacts").mkdir() + if any(case.startswith("tcp-") for case in args.cases): + self.report["parameters"]["tcp_mib"] = args.tcp_mib + if any(case not in ("idle", "throughput", "stdin-throughput") for case in args.cases): + self.report["parameters"]["gate_delay"] = args.gate_delay + self.persist() + + def persist(self): + with self.lock: + super().persist() + + def track(self, job): + with self.lock: + self.jobs.append(job) + return job + + def limit(self): + remaining = self.args.timeout if self.deadline is None else self.deadline - time.monotonic() + if remaining <= 0: + raise RuntimeError("transport suite deadline exceeded") + return min(self.args.timeout, remaining) + + def invoke(self, case, *arguments, expected_failure=False, phase="operations", timeout=None, + started=None): + if self.cleaning and phase == "operations": + raise RuntimeError("cleanup has closed admission of new operation clients") + limit = timeout or (self.limit() if phase == "operations" else self.args.timeout) + with self.lock: + sequence = len(self.report["commands"]) + row = dict(case=case, argv=list(map(str, arguments)), phase=phase, start=time.monotonic()) + self.report["commands"].append(row) + prefix = self.logs / f"{sequence:04d}-{case}" + outpath, errpath = prefix.with_suffix(".stdout.log"), prefix.with_suffix(".stderr.log") + job, code, timed_out = None, None, False + try: + with outpath.open("wb") as out, errpath.open("wb") as err: + job = self.track(Job([str(self.binary), *map(str, arguments)], self.env, self.root, + stdin=subprocess.DEVNULL, stdout=out, stderr=err)) + if started: + started(job) + code = job.wait(limit) + except subprocess.TimeoutExpired: + timed_out = True + job.terminate() + finally: + row.update(end=time.monotonic(), exit=code, timed_out=timed_out) + row["ms"] = (row["end"] - row["start"]) * 1000 + self.persist() + stdout, stderr = outpath.read_text(errors="replace"), errpath.read_text(errors="replace") + if timed_out or code is None or code < 0 or code > 255 or (code != 0) != expected_failure: + raise RuntimeError(f"{case}: exit={code}, timeout={timed_out}: {stderr[-2000:]}") + return stdout.strip(), stderr.strip(), row + + def run(self, case, *arguments, **kwargs): + out, err, _ = self.invoke(case, *arguments, **kwargs) + return out, err + + def source_cpu(self): + # Read only this harness's catalog. Missing/unsupported CPU observations are explicitly + # null, never zero. No catalog PID is signalled by the harness. + try: + database = self.home / "db/msb.db" + with sqlite3.connect(database.as_uri() + "?mode=ro", uri=True, timeout=2) as db: + pid = db.execute('SELECT pid FROM "run" ORDER BY id ASC LIMIT 1').fetchone()[0] + result = subprocess.run(["ps", "-p", str(pid), "-o", "time="], + capture_output=True, text=True, timeout=2, check=True) + return cpu_seconds(result.stdout) + except (OSError, ValueError, TypeError, IndexError, sqlite3.Error, subprocess.SubprocessError): + return None + + def operation(self, row, metric, *arguments): + started = time.monotonic() + wall_start = time.time_ns() + self.run(row["case"] + "-" + metric, *arguments) + ended = time.monotonic() + row[metric + "_ms"] = (ended - started) * 1000 + row.setdefault("operations", []).append(dict(kind=metric, start=started, end=ended, + wall_start_ns=wall_start, wall_end_ns=time.time_ns())) + + def clock_probe(self, label): + before = time.time_ns() + guest = int(self.guest(label, "source", "python3 -c 'import time; print(time.time_ns())'")) + return dict(host_before_ns=before, guest_ns=guest, host_after_ns=time.time_ns()) + + def gate_proof(self, row, receipt): + after = self.clock_probe(row["case"] + "-clock-after") + opened = json.loads(self.guest(row["case"] + "-gate-proof", "source", f"cat {receipt}.gate")) + evidence = dict(opened=opened, clock_after=after, + waiting_excluded_from_checkpoint_latency=True) + row["autonomous_gate"] = evidence + try: + evidence.update(verify_gate_timing(opened, row["gate_clock_before"], after, row["operations"])) + except RuntimeError as error: + evidence["qualification_error"] = str(error) + self.persist() # Retain the raw opening/clock observations even when overlap is unproven. + raise + + def child_ready(self, row, name): + started, wall_start = time.monotonic(), time.time_ns() + actual = self.guest(row["case"] + "-child-ready-exec", name, + "test \"$(cat /transport-marker)\" = source; " + "test \"$(cat /dev/shm/transport-marker)\" = source; " + "echo child > /transport-marker; echo child > /dev/shm/transport-marker; " + "printf 'CHILD_FRAME_OK\\n'") + ended = time.monotonic() + row["child_ready_exec_ms"] = (ended - started) * 1000 + # Restore completion alone cannot prove fresh control traffic escapes inherited + # input debt. This command must also finish before the autonomous read gate opens; + # child_input deliberately opens the child's gate only after this check returns. + row.setdefault("operations", []).append(dict(kind="child-ready-exec", start=started, + end=ended, wall_start_ns=wall_start, wall_end_ns=time.time_ns())) + if actual != "CHILD_FRAME_OK": + raise RuntimeError(f"child framing mismatch: {actual!r}") + row["child_exec_independent"] = True + + def source_ready(self, row): + started, wall_start = time.monotonic(), time.time_ns() + actual = self.guest(row["case"] + "-source-ready-exec", "source", + "printf 'SOURCE_FRAME_OK\\n'") + ended = time.monotonic() + row["source_ready_exec_ms"] = (ended - started) * 1000 + # Unlike the restored child's empty host queue, this source still owns queued input. + # Do not open its gate: completion before the autonomous timestamp must prove that + # fresh metadata can pass credit-blocked data while the input consumer stays blocked. + row.setdefault("operations", []).append(dict(kind="source-ready-exec", start=started, + end=ended, wall_start_ns=wall_start, wall_end_ns=time.time_ns())) + if actual != "SOURCE_FRAME_OK": + raise RuntimeError(f"source framing mismatch: {actual!r}") + row["source_exec_independent"] = True + + def child_input(self, row, name, gate, receipt, stream): + # This read happens before the host releases AFTER! bytes. The source's closed gate + # is an independent copy, so inspecting the child cannot drain the source stream. + prefix = json.loads(self.guest(row["case"] + "-child-prefix", name, + f"cat {receipt}.prefix")) + if prefix != stream.prefix_receipt: + raise RuntimeError(f"child lost the consumed-prefix marker: {prefix}") + wait = ("import json, os, sys, time; path=sys.argv[1]; " + "deadline=time.monotonic()+float(sys.argv[2]);\n" + "while not os.path.exists(path):\n" + " if time.monotonic() >= deadline: raise RuntimeError('child input did not settle')\n" + " time.sleep(.01)\n" + "print(open(path).read())") + command = (f"touch {receipt}.probe; touch {gate}; python3 -c {shlex.quote(wait)} " + f"{receipt} {max(.1, self.limit() - 2)}") + value = json.loads(self.guest(row["case"] + "-child-input", name, command)) + verify_child_receipt(value, PREFIX_BYTES, stream.cutoff, stream.is_pty) + row["child_stdin"] = dict(receipt=value, acknowledged_prefix=prefix, + offered_pre_cut_upper_bound=stream.cutoff, + withheld_post_cut_bytes=LATE_BYTES, + exact_guest_admission_frontier_observed=False) + + def wait_bulk_active(self, workers): + until = time.monotonic() + self.limit() + while time.monotonic() < until: + if any(worker.error for worker in workers): + raise RuntimeError(f"bulk start failed: {[str(w.error) for w in workers]}") + if all(w.current and w.current.process.poll() is None for w in workers): + return + time.sleep(.002) + raise RuntimeError("bidirectional bulk operations never overlapped") + + def scenario(self, kind, is_pty, index): + label = f"{kind}-{'pty' if is_pty else 'pipe'}-{index}" + row = dict(case=label, kind=kind, tty=is_pty, index=index, status="running") + self.report["samples"].append(row) + started, host_cpu = time.monotonic(), time.process_time() + source_before = self.source_cpu() + cli_before = resource.getrusage(resource.RUSAGE_CHILDREN) + gate, receipt, stop = (f"/transport-{label}-{suffix}" for suffix in ("go", "receipt", "stop")) + regression = kind not in ("throughput", "stdin-throughput") + if regression: + row["gate_clock_before"] = self.clock_probe(label + "-clock-before") + stream = Stream(self, label, INPUT_PROGRAM, + [gate, receipt, "1" if is_pty else "0", str(PREFIX_BYTES if regression else 0), + str(self.args.gate_delay if regression else 0)], + is_pty) + self.streams.append(stream) + stream.await_ready() + control, workers, child, member = None, [], None, None + if kind in ("full", "branch", "throughput"): + control = Stream(self, label + "-control", CONTROL_PROGRAM, [stop], control=True) + self.streams.append(control) + control.await_ready() + workers = [Bulk(self, label, direction) for direction in ("upload", "download")] + self.bulk_workers.extend(workers) + self.wait_bulk_active(workers) + if regression: + stream.feed(self.args.stdin_mib * MIB, regression=True) + row["acknowledged_prefix"] = stream.await_prefix() + if kind == "paused": + self.operation(row, "pause", "pause", "source") + self.check_status("source", "Paused") + if not regression: + self.guest(label + "-go", "source", f"touch {gate}") + stream.feed(self.args.stdin_mib * MIB) + if regression: + stream.offer.set() + row["backpressure"] = stream.await_pressure() + if kind == "paused": + self.operation(row, "resume", "resume", "source") + elif kind == "branch": + child = label + "-child" + self.remember(child) + self.wait_bulk_active(workers) + self.operation(row, "branch_ready", "branch", "source", "--name", child) + self.child_ready(row, child) + self.child_input(row, child, gate, receipt, stream) + elif kind == "full": + member = label + self.wait_bulk_active(workers) + self.operation(row, "capture", "snapshot", "create", member, + "--from-sandbox", "source", "--group", "transport", "--full") + child = label + "-child" + self.remember(child) + # Exercise both durable restore routes on every repetition without silently pooling + # their latency distributions: pipe=eager/installed; PTY=forked/archive. + source = "transport:" + member + row["restore_source"] = "archive" if is_pty else "installed" + row["restore_memory"] = "forked" if is_pty else "eager" + if is_pty: + archive = self.root / "artifacts" / (label + ".msb") + self.run(label + "-save", "snapshot", "save", source, archive) + source = str(archive) + self.wait_bulk_active(workers) + self.operation(row, "restore_ready", "create", "--name", child, + "--from-snapshot", source, "--pull", "never", + *(["--forked"] if is_pty else [])) + self.child_ready(row, child) + self.child_input(row, child, gate, receipt, stream) + if regression: + self.source_ready(row) + stream.after_cut.set() + # The timer remains independent evidence, not a metadata-progress workaround. + # Neither readiness exec opens the source gate before its queued input drains. + row["stdin"] = stream.finish() + stream.close() + verify_receipt(json.loads(self.guest(label + "-source-receipt", "source", f"cat {receipt}")), + stream.size, stream.expected_digest, is_pty) + if regression: + self.gate_proof(row, receipt) + if control: + self.guest(label + "-control-stop", "source", f"touch {stop}") + row["control_stdout"] = control.finish() + control.close() + for worker in workers: + worker.stop.set() + row["bulk"] = {worker.direction: worker.finish() for worker in workers} + if kind in ("full", "branch"): + # Require a verified operation interval covering the *start* of every checkpoint + # action. A transfer that starts only after thaw is not counted as overlap. + for operation in row["operations"]: + if operation["kind"] in ("child-ready-exec", "source-ready-exec"): + continue # This probes control readiness, not checkpoint/bulk overlap. + for direction, samples in row["bulk"].items(): + overlaps = [s for s in samples if s["start"] <= operation["start"] < s["end"]] + if not overlaps: + raise RuntimeError(f"{label}/{operation['kind']}: no verified {direction} overlap; increase --bulk-mib") + # Exact fresh framing and disk/RAM identity checks after the busy session's EOF. + actual = self.guest(label + "-framing", "source", + "test \"$(cat /transport-marker)\" = source; " + "test \"$(cat /dev/shm/transport-marker)\" = source; printf 'POST_FRAME_OK\\n'") + if actual != "POST_FRAME_OK": + raise RuntimeError(f"post-checkpoint framing mismatch: {actual!r}") + cli_after = resource.getrusage(resource.RUSAGE_CHILDREN) + source_after = self.source_cpu() + row.update(status="passed", elapsed_ms=(time.monotonic() - started) * 1000, + harness_cpu_seconds=time.process_time() - host_cpu, + cli_cpu_seconds=(cli_after.ru_utime + cli_after.ru_stime + - cli_before.ru_utime - cli_before.ru_stime), + source_runtime_cpu_seconds=(source_after - source_before + if source_before is not None and source_after is not None else None)) + self.persist() + if child: + self.stop(child) + if member: + self.run(label + "-remove", "snapshot", "remove", "transport:" + member) + archive = self.root / "artifacts" / (label + ".msb") + archive.unlink(missing_ok=True) + print(f"{self.args.label}/{label}: correctness passed", flush=True) + + def tcp_scenario(self, kind, index): + label = f"{kind}-{index}" + row = dict(case=label, kind=kind, index=index, status="running", tcp_mode="inline-no-bulk-offer") + self.report["samples"].append(row) + gate, receipt = f"/transport-{label}-go", f"/transport-{label}-receipt" + row["gate_clock_before"] = self.clock_probe(label + "-clock-before") + server = Stream(self, label, TCP_PROGRAM, + [gate, receipt, "32017", str(self.args.gate_delay)], no_input=True) + self.streams.append(server) + server.await_ready() + # This canonical endpoint is scoped to our fresh home and exact source name. Never + # discover a global socket or inherit a caller's agent-client connection. + socket_path = self.home / "run/sandboxes" / hashlib.sha256(b"source").hexdigest()[:24] / "agent.sock" + client = TCP.InlineTcp(socket_path, 32017, self.limit) + self.tcp_clients.append(client) + size = self.args.tcp_mib * MIB + server.size, server.expected_digest = size, input_digest(size, size) + client.feed(size, lambda offset, count: input_bytes(offset, count, size)) + row["backpressure"] = client.await_pressure() + child, member = None, None + if kind == "tcp-paused": + self.operation(row, "pause", "pause", "source") + self.check_status("source", "Paused") + self.operation(row, "resume", "resume", "source") + elif kind == "tcp-branch": + child = label + "-child" + self.remember(child) + self.operation(row, "branch_ready", "branch", "source", "--name", child) + else: + member, child = label, label + "-child" + self.operation(row, "capture", "snapshot", "create", member, + "--from-sandbox", "source", "--group", "transport", "--full") + self.remember(child) + self.operation(row, "restore_ready", "create", "--name", child, + "--from-snapshot", "transport:" + member, "--pull", "never") + if child: + self.child_ready(row, child) + row["child_tcp_expectation"] = "old connection detached; only fresh child exec asserted" + self.source_ready(row) + row["tcp_receipt"] = client.finish() + verify_receipt(row["tcp_receipt"], size, server.expected_digest, False) + row["server"] = server.finish() + client.close() + server.close() + self.gate_proof(row, receipt) + if self.guest(label + "-framing", "source", "printf 'TCP_POST_FRAME_OK\\n'") != "TCP_POST_FRAME_OK": + raise RuntimeError("post-TCP fresh exec framing failed") + row["status"] = "passed" + self.persist() + if child: + self.stop(child) + if member: + self.run(label + "-remove", "snapshot", "remove", "transport:" + member) + print(f"{self.args.label}/{label}: correctness passed", flush=True) + + def exercise(self): + self.blob = self.root / "artifacts" / "payload.bin" + block = bytes(range(256)) * 4096 + with self.blob.open("wb") as destination: + for _ in range(self.args.bulk_mib): + destination.write(block) + self.blob_digest = file_digest(self.blob) + disk_mib = max(2048, self.args.bulk_mib * 2 + 512) + self.create("source", self.args.image, "--root-disk", f"{disk_mib}M", + "--memory", "512M", "--cpus", "2", "--pull", "never") + inspected = json.loads(self.run("source-provenance", "inspect", "source", "--format", "json")[0]) + self.report["image_manifest_digest"] = inspected["config"].get("manifest_digest") + if not self.report["image_manifest_digest"]: + raise RuntimeError("source did not expose a pinned image manifest for benchmark provenance") + self.guest("prepare-source", "source", + "python3 --version; echo source > /transport-marker; " + "echo source > /dev/shm/transport-marker; sync") + self.run("seed-download", "copy", self.blob, "source:/transport-download.bin") + if self.guest("seed-download-hash", "source", "sha256sum /transport-download.bin").split()[0] != self.blob_digest: + raise RuntimeError("initial bulk payload checksum mismatch") + for index in range(self.args.samples): + if "idle" in self.args.cases: + idle = dict(case=f"idle-{index}", kind="idle", status="running") + self.report["samples"].append(idle) + self.operation(idle, "pause", "pause", "source") + self.operation(idle, "resume", "resume", "source") + self.guest(f"idle-probe-{index}", "source", "true") + idle["status"] = "passed" + for mode in self.args.input_modes: + is_pty = mode == "pty" + for kind in ("throughput", "stdin-throughput", "paused", "full", "branch"): + if kind in self.args.cases: + self.scenario(kind, is_pty, index) + for kind in ("tcp-paused", "tcp-full", "tcp-branch"): + if kind in self.args.cases: + self.tcp_scenario(kind, index) + + def cleanup_owned(self): + errors = [] + self.cleaning = True + # First stop admission of new clients, then kill owned client process groups, then stop + # every registered VM. Cleanup is independent of the expired operation deadline. + for worker in self.bulk_workers: + worker.stop.set() + for client in self.tcp_clients: + try: + client.close() + except Exception as error: + errors.append(f"TCP client cleanup: {error}") + for stream in self.streams: + try: + stream.close() + except Exception as error: + errors.append(f"stream cleanup: {error}") + for job in list(self.jobs): + try: + job.terminate() + except Exception as error: + errors.append(f"client cleanup: {error}") + for worker in self.bulk_workers: + worker.thread.join(3) + if worker.thread.is_alive(): + errors.append("bulk worker remained alive after owned clients were terminated") + errors.extend(super().cleanup_owned()) + for row in self.report["samples"]: + if row["status"] == "running": + row["status"] = "incomplete" + self.report["statistics"] = summarize(self.report["samples"]) + return errors + + +def summarize(samples): + groups = {} + for row in samples: + if row["status"] != "passed": + continue + prefix = row["kind"] + ("/pty" if row.get("tty") else "/pipe") + for key in ("pause_ms", "resume_ms", "capture_ms", "branch_ready_ms", "restore_ready_ms", + "child_ready_exec_ms", "source_ready_exec_ms", + "elapsed_ms", "harness_cpu_seconds", "cli_cpu_seconds", "source_runtime_cpu_seconds"): + if row.get(key) is not None: + groups.setdefault(prefix + "/" + key, []).append(row[key]) + for channel in ("stdin", "control_stdout"): + if channel in row: + value = row[channel] + groups.setdefault(prefix + "/" + channel + "_mib_s", []).append( + value["bytes"] / MIB / value["seconds"]) + for direction, transfers in row.get("bulk", {}).items(): + for value in transfers: + groups.setdefault(prefix + "/" + direction + "_mib_s", []).append( + value["bytes"] / MIB / value["seconds"]) + return {key: distribution(values) for key, values in sorted(groups.items())} + + +def compare(reports): + if any(reports.get(label, {}).get("status") != "passed" for label in ("baseline", "candidate")): + return {} # Failed/partial runs are correctness evidence, not a fair performance baseline. + baseline, candidate = reports["baseline"], reports["candidate"] + if (not baseline.get("image_manifest_digest") + or baseline["image_manifest_digest"] != candidate.get("image_manifest_digest") + or baseline.get("parameters") != candidate.get("parameters") + or not baseline.get("firmware_sha256") + or baseline["firmware_sha256"] != candidate.get("firmware_sha256") + or not baseline.get("host") or baseline["host"] != candidate.get("host") + or not baseline.get("host_node") or baseline["host_node"] != candidate.get("host_node")): + return {} # Changed image, workload, firmware, or host is not an isolated binary delta. + before = reports.get("baseline", {}).get("statistics", {}) + after = reports.get("candidate", {}).get("statistics", {}) + return {key: {"baseline": before[key], "candidate": after[key], + "candidate_over_baseline_p50": (after[key]["p50"] / before[key]["p50"] + if before[key]["p50"] else None)} for key in before.keys() & after.keys()} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + for label in ("baseline", "candidate"): + parser.add_argument("--" + label, type=Path, help="Matching msb CLI/runtime binary") + parser.add_argument("--" + label + "-firmware", type=Path) + parser.add_argument("--" + label + "-agentd", type=Path, help="Otherwise use embedded agentd") + parser.add_argument("--output", type=Path, help="Exclusive new report directory") + parser.add_argument("--home-parent", type=Path, default=Path("/tmp"), + help="Existing short directory for a fresh isolated home (default: /tmp)") + parser.add_argument("--samples", type=bounded_int, default=3) + parser.add_argument("--input-modes", nargs="+", choices=("pipe", "pty"), default=["pipe", "pty"]) + parser.add_argument("--cases", nargs="+", choices=("idle", "throughput", "stdin-throughput", "paused", "full", "branch", + "tcp-paused", "tcp-full", "tcp-branch"), + default=["idle", "throughput", "paused", "full", "branch"], + help="Select regression cases, or idle throughput for a standalone performance run") + parser.add_argument("--stdin-mib", type=bounded_int, default=16) + parser.add_argument("--bulk-mib", type=bounded_int, default=64) + parser.add_argument("--tcp-mib", type=bounded_int, default=64, + help="Finite input for opt-in inline TCP backpressure cases") + parser.add_argument("--gate-delay", type=BASE.positive_seconds, default=20, + help="Guest-autonomous regression gate delay; actual opening must follow checkpoint completion") + parser.add_argument("--image", default="mirror.gcr.io/library/python:3.13-alpine") + parser.add_argument("--timeout", type=BASE.positive_seconds, default=45) + parser.add_argument("--suite-timeout", type=BASE.positive_seconds, default=900) + args = parser.parse_args() + labels = [label for label in ("baseline", "candidate") if getattr(args, label)] + if not labels or os.name != "posix": + parser.error("provide --baseline and/or --candidate; this harness requires POSIX PTYs") + for label in labels: + if not getattr(args, label + "_firmware"): + parser.error(f"--{label}-firmware is required for an explicit matching runtime pair") + root = args.output.expanduser().resolve() if args.output else Path( + tempfile.mkdtemp(prefix="msb-transport-", dir="/tmp")) + if args.output: + root.mkdir(parents=True, mode=0o700, exist_ok=False) + reports, failed = {}, False + + def interrupt(_signum, _frame): + raise KeyboardInterrupt("interrupted; cleaning up owned transport-test VMs") + + signal.signal(signal.SIGTERM, interrupt) + for label in labels: + selected = argparse.Namespace(**vars(args)) + selected.binary, selected.output, selected.label = getattr(args, label), root / label, label + selected.firmware, selected.agentd = getattr(args, label + "_firmware"), getattr(args, label + "_agentd") + selected.layout = "managed" + smoke = TransportSmoke(selected) + failed |= bool(smoke.execute()) + reports[label] = smoke.report + if smoke.report["status"] == "passed": + smoke.blob.unlink(missing_ok=True) + if smoke.report["cleanup_errors"]: + break # Do not benchmark the next binary alongside an unverified surviving runtime. + result = dict(reports={key: str(root / key / "report.json") for key in reports}, + comparison=compare(reports), status="failed" if failed else "passed", + comparison_requirement="two completely passing runs with identical host, firmware, pinned image, and workload parameters") + (root / "comparison.json").write_text(json.dumps(result, indent=2) + "\n") + print(f"Comparison: {root / 'comparison.json'}", flush=True) + return int(failed) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/smoke/cli/transport_tcp.py b/scripts/smoke/cli/transport_tcp.py new file mode 100644 index 000000000..8b02d118d --- /dev/null +++ b/scripts/smoke/cli/transport_tcp.py @@ -0,0 +1,226 @@ +"""Small current-wire-only TCP probe; not an SDK, legacy codec, or transport upgrade client. + +Wire sources: protocol/lib/{codec,message,tcp}.rs and agent-client/rust/lib/client.rs. +Omitting BulkOffer deliberately exercises inline TcpData, unlike current SSH forwarding. +""" + +import json +import select +import socket +import struct +import threading +import time + + +MAX_FRAME = 4 * 1024 * 1024 + + +def cbor(value): + def head(major, number): + if number < 24: + return bytes([major * 32 + number]) + for width, marker in ((1, 24), (2, 25), (4, 26), (8, 27)): + if number < 1 << (8 * width): + return bytes([major * 32 + marker]) + number.to_bytes(width, "big") + raise ValueError("CBOR integer overflow") + + if type(value) is int and value >= 0: + return head(0, value) + if isinstance(value, bytes): + return head(2, len(value)) + value + if isinstance(value, str): + raw = value.encode() + return head(3, len(raw)) + raw + if isinstance(value, dict): + return head(5, len(value)) + b"".join(cbor(k) + cbor(v) for k, v in value.items()) + raise ValueError(f"unsupported probe CBOR value: {type(value)}") + + +def uncbor(raw): + position = 0 + + def take(size): + nonlocal position + if position + size > len(raw): + raise ValueError("truncated CBOR") + value = raw[position:position + size] + position += size + return value + + def item(depth=0): + if depth > 8: + raise ValueError("CBOR nesting exceeded") + initial = take(1)[0] + major, number = initial >> 5, initial & 31 + if number >= 24: + if number not in (24, 25, 26, 27): + raise ValueError("indefinite/reserved CBOR is outside the probe contract") + number = int.from_bytes(take(1 << (number - 24)), "big") + if major == 0: + return number + if major in (2, 3): + value = take(number) + return value if major == 2 else value.decode() + if major == 5 and number <= 32: + value = {} + for _ in range(number): + key, member = item(depth + 1), item(depth + 1) + if key in value: + raise ValueError("duplicate CBOR key") + value[key] = member + return value + raise ValueError("unsupported probe CBOR type") + + value = item() + if position != len(raw): + raise ValueError("trailing CBOR bytes") + return value + + +def frame(identifier, version, kind, payload): + flags = 2 if kind == "core.tcp.connect" else 0 + body = struct.pack(">IB", identifier, flags) + cbor(dict(v=version, t=kind, p=cbor(payload))) + if len(body) > MAX_FRAME: + raise ValueError("probe frame exceeds protocol limit") + return struct.pack(">I", len(body)) + body + + +class InlineTcp: + def __init__(self, path, port, limit): + self.limit = limit + self.cancel = threading.Event() + self.connected = threading.Event() + self.error, self.receipt = None, None + self.wire_sent, self.sent, self.blocked = 0, 0, 0 + self.writer, self.reader = None, None + self.socket = socket.socket(socket.AF_UNIX) + try: + self.socket.settimeout(limit()) + self.socket.connect(str(path)) + self.socket.setblocking(False) + minimum, maximum = struct.unpack(">II", self.receive(8)) + if not 0 < minimum < maximum: + raise RuntimeError("probe requires a current relay ID-range handshake") + _, _, ready = self.message() + if ready["t"] != "core.ready" or not 4 <= ready["v"] <= 9: + raise RuntimeError("probe requires known current-wire TCP protocol generation") + self.identifier, self.version = minimum, ready["v"] + self.send(frame(minimum, self.version, "core.tcp.connect", {"host": "127.0.0.1", "port": port})) + self.reader = threading.Thread(target=self.read, daemon=True) + self.reader.start() + if not self.connected.wait(limit()) or self.error: + raise RuntimeError(f"TCP connect failed: {self.error}") + except BaseException: + self.close() + raise + + def receive(self, size): + data = bytearray() + deadline = time.monotonic() + self.limit() + while len(data) < size: + if self.cancel.is_set() or time.monotonic() >= deadline: + raise RuntimeError("TCP probe read cancelled or timed out") + if not select.select([self.socket], [], [], .02)[0]: + continue + block = self.socket.recv(size - len(data)) + if not block: + raise RuntimeError("relay closed before TCP terminal frame") + data.extend(block) + return bytes(data) + + def message(self): + length = struct.unpack(">I", self.receive(4))[0] + if not 5 <= length <= MAX_FRAME: + raise RuntimeError(f"invalid probe frame length: {length}") + raw = self.receive(length) + identifier, flags = struct.unpack(">IB", raw[:5]) + return identifier, flags, uncbor(raw[5:]) + + def send(self, data): + offset = 0 + deadline = time.monotonic() + self.limit() + while offset < len(data): + if self.cancel.is_set() or time.monotonic() >= deadline: + raise RuntimeError("TCP probe write cancelled or timed out") + try: + count = self.socket.send(data[offset:]) + if not count: + raise RuntimeError("zero-length TCP probe socket write") + offset += count + self.wire_sent += count + except BlockingIOError: + self.blocked += 1 + select.select([], [self.socket], [], .02) + + def read(self): + data, eof = bytearray(), False + try: + while True: + identifier, flags, message = self.message() + if identifier != self.identifier or flags & 8: + raise RuntimeError("unexpected TCP correlation ID or raw-bulk frame") + kind, payload = message["t"], uncbor(message["p"]) + if kind == "core.tcp.connected" and not self.connected.is_set(): + self.connected.set() + elif kind == "core.tcp.data" and self.connected.is_set() and not eof: + data.extend(payload["data"]) + if len(data) > 4096: + raise RuntimeError("TCP receipt exceeded bound") + elif kind == "core.tcp.eof" and not eof: + eof = True + elif kind == "core.tcp.closed" and eof and flags & 1: + self.receipt = json.loads(data) + return + else: + raise RuntimeError(f"unexpected TCP reply: {kind}: {payload}") + except Exception as error: + if not self.cancel.is_set(): + self.error = error + self.connected.set() + + def feed(self, size, payload): + self.size = size + + def write(): + try: + for offset in range(0, size, 65536): + block = payload(offset, min(65536, size - offset)) + self.send(frame(self.identifier, self.version, "core.tcp.data", {"data": block})) + self.sent += len(block) + self.send(frame(self.identifier, self.version, "core.tcp.eof", {})) + except Exception as error: + if not self.cancel.is_set(): + self.error = error + + self.writer = threading.Thread(target=write, daemon=True) + self.writer.start() + + def await_pressure(self): + previous, stable = -1, time.monotonic() + deadline = time.monotonic() + self.limit() + while time.monotonic() < deadline: + if self.error: + raise self.error + if self.sent == self.size: + raise RuntimeError("TCP input fit in forwarding queues; increase --tcp-mib") + if self.wire_sent != previous: + previous, stable = self.wire_sent, time.monotonic() + elif self.blocked and time.monotonic() - stable >= .2: + return dict(payload_bytes_framed=self.sent, wire_bytes_written=self.wire_sent, + eagain_count=self.blocked, stable_blocked_seconds=time.monotonic() - stable) + time.sleep(.01) + raise RuntimeError("TCP sustained backpressure unproven") + + def finish(self): + for thread in (self.writer, self.reader): + thread.join(self.limit()) + if thread.is_alive() or self.error: + raise RuntimeError(f"TCP stream failed: {self.error or 'deadline exceeded'}") + return self.receipt + + def close(self): + self.cancel.set() + for thread in (self.writer, self.reader): + if thread: + thread.join(1) + self.socket.close() diff --git a/scripts/smoke/reports/cow-memory-lifecycle-2026-09-07.md b/scripts/smoke/reports/cow-memory-lifecycle-2026-09-07.md new file mode 100644 index 000000000..d75301788 --- /dev/null +++ b/scripts/smoke/reports/cow-memory-lifecycle-2026-09-07.md @@ -0,0 +1,60 @@ +# CoW memory and resident lifecycle — 2026-09-07 + +Archive names in this report use the current `.msb` convention. Retained raw logs preserve the filenames used in the original runs. + +Follow-up: [execution-state fixes and qualification](execution-state-2026-09-07.md) supersedes the Windows post-restore failure status below and adds Linux ARM64 coverage. The observations below describe the earlier backend revision. + +Status: development integration and live smoke coverage, not full platform or performance qualification. Microsandbox #8 remains stacked directly on #7 `ce04099b`, with libkrun `94d680b21bf7ea7c2bed5262ed211833bd4379bd` and firmware `6cca413ac248f63e65d4ea4748b3bc36cd1b22f3`. The kernel and agentd used below were built from matching development sources on the authorized OVH host, including the ARM64 guest artifacts used on macOS and Windows. + +## Reproduce + +Use `scripts/smoke/cli/cow-memory-lifecycle.py` with `MSB_PATH`, an isolated `MSB_HOME`, matching `MSB_LIBKRUNFW_PATH`, an output directory in `STACK8_OUT`, and a fresh `STACK8_PREFIX`. Select `STACK8_LAYOUT=flat:512M`, `512M`, or `tmpfs`, `STACK8_MODE=cow` or `standard`, and optionally `STACK8_PAUSE_SECONDS=10`. The workload uses Alpine, 256 MiB RAM, and two vCPUs. The runner records each command's elapsed wall time, stdout, stderr, and exit code, and attempts to stop every sandbox it starts in `finally`. + +Optional `STACK8_LIVE_RESIZE=1` adds a 512 MiB ceiling and exercises targets 384, 256, 512, and 256 MiB before capture. It checks actual guest MemTotal convergence relative to the baseline within 4 MiB and preserves a tmpfs marker. `STACK8_CHECK_COW_REJECTION=1` checks unsupported-platform rejection without a running/paused fallback. `STACK8_KEEP_ARCHIVE=1` retains the direct archive for diagnosis instead of testing unlink survival. Timed-out commands are recorded explicitly, with partial output, before cleanup runs. + +The opt-in language SDK tests are `sdk/python/tests/test_cow_lifecycle.py`, `sdk/node-ts/tests/cow-lifecycle.test.ts`, and `sdk/go/cow_lifecycle_test.go`. Set `MSB_COW_LIVE=1`; Go also requires tags `cow_live microsandbox_ffi_path` and `MICROSANDBOX_FFI_PATH` pointing to its matching native library. + +## Observed coverage + +Linux/KVM x86-64 passed CoW flat, managed, and tmpfs roots, plus a standard-memory flat-root baseline. macOS/HVF ARM64 passed the same root/memory variants. The checks exercise fresh construction, running full capture, idempotent pause/resume, host-observed Paused status, prompt rejection of new guest exec while paused, two successive full captures while retaining pause, installed-snapshot restore into two children, private child writes, direct full `.msb` capture/restore, survival after input-archive unlink, and stop from paused. Completed runs stopped their test VMs; retained snapshot/cache artifacts remain in the isolated test homes for inspection. + +The later Linux flat/managed/tmpfs/standard runs and macOS tmpfs run additionally assert unchanged Linux boot ID across ordinary pause/resume, resumed progress of the original counter workload, and guest wall clock within three seconds of the host after a ten-second pause. These checks do not constitute host-suspend, every clock-failure, or VM Generation ID notification testing. + +The Python, Node, and Go live SDK checks passed on macOS: create with explicit CoW, pause, capture while paused, resume through a handle, restore a child, verify tmpfs contents and source/child write isolation, and stop a paused child. A further Linux flat-root CoW run passed the live-resize sequence described above and the subsequent full lifecycle matrix. This verifies basic capacity convergence and retained marker contents, not physical host-memory reclamation or every unplug/replug invariant. + +Windows ARM64 firmware build/export/load checks and native `aarch64-pc-windows-msvc` runtime compilation passed. Explicit CoW rejection passed without starting a VM. Standard-memory lifecycle qualification failed as detailed below; Windows CoW remains unsupported. + +### Windows post-restore failure + +The first standard-memory flat-root run passed fresh boot, running full capture, pause/idempotence/status/admission checks, two paused captures, resume/idempotence, boot identity, ten-second pause clock correction, workload progress, and two installed-snapshot child restores with write isolation. Direct full archive creation took 10,247.52 ms and produced 20,630,604 bytes. Direct restore returned success after 4,912.00 ms, but the first guest command timed out after 120 seconds. That is a failed usable-restore result, not a 4.9-second successful restore benchmark. + +A repeat intended to retain the archive failed earlier: its first installed-snapshot child reported restored in 2,934.15 ms but its first guest command timed out after 120 seconds. No direct archive or archive unlink had been reached, ruling out archive deletion as the sole explanation. Three bounded restores of that retained installed snapshot subsequently all reported activation in 3,066.11–3,182.01 ms and all timed out at the 15-second guest-command bound. Do not infer successful activation implies a usable guest or use these values as successful restore latency. + +Trace evidence shows successful generation/clock acknowledgement and workload-thaw exchange, subsequent command bytes delivered to the virtual console, and continuing filesystem device activity during the command stall. The exact cause remains unresolved; the evidence does not establish a whole-VM freeze, archive corruption, or a specific interrupt/agent defect. All four VMs started by the first matrix and both started by the repeat received successful stop responses; cleanup also covered the rejected CoW sandbox record. A process inventory confirmed no remaining `cow8-windows-standard` or `cow8-windows-keep` runtime. Each bounded diagnostic also runs stop in `finally`. A separate fresh diagnostic snapshot reproduced the command timeout on all three children, but its attempted guest task-stack output did not reach `kernel.log` and supplies no task-stack diagnosis. Earlier unrelated development VMs were not terminated. + +## Individual debug-build timings + +These are single observations, not p50/p95, release benchmarks, or claims of speedup. Build activity and cache warmth varied. CLI wall time includes client/process/setup work and must not be quoted as stop-the-world duration. + +| Operation | Linux managed CoW (ms) | macOS managed CoW (ms) | macOS tmpfs CoW (ms) | +| --- | ---: | ---: | ---: | +| First running full capture | 2389.55 | 2000.17 | 1741.27 | +| Resident pause | 9.26 | 19.24 | 14.58 | +| First capture while user-paused | 2679.02 | 2132.41 | 1039.84 | +| Second capture while user-paused | 471.60 | 743.66 | 452.17 | +| Resident resume | 10.46 | 16.22 | 35.71 | +| Restore child A | 158.56 | 572.77 | 259.46 | +| Restore child B | 155.39 | 590.81 | 241.84 | +| Direct full archive capture | 1968.17 | 3050.15 | 2925.98 | +| Direct full archive restore | 396.31 | 1502.71 | 990.42 | +| Stop user-paused source | 111.19 | 119.05 | 116.72 | + +The macOS flat run logged APFS reflink reuse for repeated cache construction, including approximately 8 ms for one unchanged paused generation. This is cache preparation only, not total snapshot latency. A warm cache lookup logged 124 microseconds in one restore; that is not end-to-end restore latency or a physical-sharing measurement. + +## Other validation and remaining work + +Rust SDK library tests: 668 passed, three ignored. Runtime tests: 179 passed. CLI library tests: 315 passed, plus three enabled CLI integration checks; platform-dependent ignored tests remain ignored. Node: 137 unit tests and typecheck passed. Go unit and native-FFI smoke tests passed. Python's focused API/stub tests passed (33 tests). The new backend-binding regression test verifies that pause observation and lifecycle requests use the handle's local backend rather than an ambient backend with the same sandbox name. CoW cache location is also passed from the owning backend at launch. + +CoW virtio-mem unplug currently writes private zeros to prevent old backing bytes from reappearing, but does not reclaim those pages' host RAM. NUMA plus CoW is explicitly rejected. These are outstanding integration/performance limitations, not completed acceptance items. Further work includes backing-aware physical reclamation, deeper resize/balloon live invariants beyond the basic Linux sequence, real shared/private resident-memory measurements, cache eviction and publication failure races, cancellation and lifecycle/maintenance concurrency, unsupported guest preparation, recovery/resume failures, fixing and qualifying the Windows post-restore failure, and repeated release-build performance distributions. Public cache inspection/eviction workflow and comprehensive archive compatibility variants also remain to be completed. Do not mark #8 complete from these smoke results. + +Evidence locations: OVH `/home/ubuntu/msb-stack8.ElfKzf/`; macOS `/private/tmp/msb-stack8-mac-{results,managed-results,tmpfs-results,standard-results}/`; Windows isolated worktree `C:\Users\Stephen\AppData\Local\Temp\msb-stack8-20260907`. These are development outputs, not shipped artifacts. diff --git a/scripts/smoke/reports/cow-platform-fixes-2026-09-09.md b/scripts/smoke/reports/cow-platform-fixes-2026-09-09.md new file mode 100644 index 000000000..cb6c59105 --- /dev/null +++ b/scripts/smoke/reports/cow-platform-fixes-2026-09-09.md @@ -0,0 +1,56 @@ +# CoW platform fixes — 2026-09-09 + +Development qualification of the #8 changes on top of Microsandbox `49b6670d`. The tested companion sources are published and pinned to libkrun `862d68422b40ac7c26c731b3577025a5b1f64b89` and rust-vmm `f798d4f274db22a3c458ba756900db2cd03e6fe9`. Firmware is unchanged from the matching execution-state test build. The live test builds used explicit local source overrides for those sources before publication. These are debug-build correctness runs, not release performance benchmarks. + +## Changes + +- Windows uses native private file views for restored RAM. All GPA slots are slices of one owned view, so 4 KiB slot offsets do not require separate 64 KiB-aligned mappings. Siblings cannot change each other's RAM or the backing file. The final view owner unmaps it exactly once. +- The Windows RAM cache has owner/SYSTEM-only directory ACLs, immutable reader sharing, shared lifetime pins, and identity-checked exclusive eviction. The completed writer closes before readers open the published file. +- Pending restore intent survives database insertion and process failure. It is cleared only after restore activation and create finalization. A failed child cannot later start as an ordinary VM, auto-start through exec, grow or compact its staged disk, or create a snapshot. Remove/recreate it from the original input. Successful children retain normal later stop/start behavior. +- ARM64 KVM restores distributor control before pending levels and interrupt enables, with every vCPU still paused. KVM's userspace `GICD_CTLR` setter changes the enable flag without requeueing pending interrupts; enabling it last can strand an already-high timer line. See the [Linux VGIC implementation](https://github.com/torvalds/linux/blob/v6.12/arch/arm64/kvm/vgic/vgic-mmio-v3.c). +- Resume acknowledgements now get the configured one-second barrier deadline. Only pause uses ten-millisecond sub-waits for periodic kicks. This corrects an accidental ten-millisecond resume deadline, without adding delay to successful acknowledgements. + +## Live coverage + +| Check | macOS ARM64/HVF | Linux x86-64/KVM | Windows ARM64/WHP | +| --- | --- | --- | --- | +| Full eager/forked restore, flat and layered roots | Pass | Pass | Pass | +| Direct full archive capture/restore and input-archive unlink | Pass | Pass | Pass | +| Pause/resume, idempotence, capture while paused | Pass | Pass | Pass | +| Source/sibling RAM isolation and modified-child capture/restore | Pass | Pass | Pass | +| Direct branch, paused-source branch, branch of branch; flat/layered/tmpfs | Pass | Pass | Pass | +| Independent backing pins, final release, same-name race | Pass | Pass | Pass after porting the test's Unix-only lock probe | +| Failed restore refuses start/exec/modify/compact/snapshot; sealed bytes unchanged | Pass | Pass | Pass | +| Fresh restore after failure, then ordinary stop/start | Pass | Pass | Pass | +| Retained-sibling timer-progress regression | 20/20 | 20/20 | 20/20 | +| Branch after live root growth and compaction | Previously passed; not added to final Mac rerun | Previously passed; not added to final x86 rerun | Pass, flat and layered | +| Delayed incremental archive forked restore: wall clock, monotonic/boottime, timers | Prior coverage retained | Prior coverage retained | Pass | +| Offline CPU retained and subsequently onlined after forked restore | Prior eager coverage retained | Prior eager coverage retained | Pass | + +Linux ARM64 nested-KVM also passed all eight final matrix suites, including branch-after-growth/compaction for flat and layered disks. Its failed-restore test passed every refusal, snapshot byte preservation, fresh restore, and later stop/start. Delayed incremental archive forked restore passed wall-clock, monotonic/boottime, and relative-timer checks. The offline-CPU forked restore and later CPU online check passed. The committed timer-progress fixture passed another 8/8 iterations after the 100-iteration diagnostic run. Windows x86-64 was deliberately not tested. This report does not claim every possible failure injection, memory-pressure scenario, NUMA configuration, or SDK-language/platform combination has been qualified. + +## ARM64 diagnosis and regression + +The original retained-sibling loop repeatedly hung after successful restore activation. The same failure occurred with forced full RAM capture, eager anonymous memory, and a single vCPU; those experiments did not fix it. A cold-start control and an ordinary pause/resume control each passed 100 iterations. + +A stuck guest had both CPUs idle, virtual timers enabled/unmasked with expired deadlines, and timer interrupt line levels high, but no active interrupt. Reasserting the pending interrupt recovered workload progress in a disposable diagnostic guest. That experiment was not shipped as a workaround. Inspection of KVM's userspace distributor-enable semantics identified the ordering bug above. No forced eager fallback, synthetic timer injection, periodic wakeup, or disabled incremental capture remains in the production changes. + +The corrected-order run first exposed the separate resume-deadline bug. With both fixes, a subsequent run passed 20 retained-child iterations before the disposable 32 GiB VM exhausted disk space. Unpinned local RAM cache entries were evicted under exclusive file locks; snapshots and pinned backings were not removed. A fresh run then passed all 100 retained-child iterations, including first exec and continuing timer-driven workload progress, with no failed cleanup. The final broader ARM64 matrix and focused regressions also passed. + +## Reproduce and evidence + +- `scripts/smoke/cli/failed-restore.py`: requires `MSB_TEST_DISPOSABLE_HOME=1`, deliberately obstructs only the disposable memory cache, retains the failed database row, verifies every refusal and sealed-file digest, and tests a fresh child's later stop/start. +- `scripts/smoke/cli/branch-timer-progress.py`: repeated direct branches, retained siblings, atomic counter publication, first-command success and continued guest timer progress. Set unique `STACK8_PREFIX`, `STACK8_OUT`, optional `STACK8_REPEATS` and `STACK8_RETAIN`, plus the usual runtime/home/firmware environment. +- Existing `cow-memory-lifecycle.py`, `direct-branch.py`, `branch-ownership.py`, `checkpoint-clock.py`, and `checkpoint-cpu-state.py` supply the broader matrix. `CPU_FORKED=1` selects forked restore for the offline-CPU fixture. +- Mac final matrix: `/private/tmp/msb-cow-fix-final-mac-results`; failure/restart: `/private/tmp/msb-cow-fix-restart-mac`; timer stress: `/private/tmp/msb-cow-fix-mac-stress`. +- OVH final matrix: `/home/ubuntu/msb-cow-fix-final-results`; failure/restart: `/home/ubuntu/msb-cow-fix-restart-results`. A separate benchmark task overlapped some qualification work; do not use these timings as an uncontended performance comparison. +- Surface evidence under `C:\Users\Stephen\AppData\Local\Temp`: `msb-cow-fix-49b6670d\results`, `msb-cow-fix-restart-results`, `msb-cow-fix-maint-flat`, `msb-cow-fix-maint-layered`, `msb-cow-fix-clock-results`, `msb-cow-fix-cpu-results`, and `msb-cow-fix-win-stress`. +- Nested ARM64: `/root/msb-cow-fix-qualified-results` and `/root/msb-cow-fix-arm-final-results` in the disposable QEMU VM; text-only evidence copied to `/private/tmp/msb-cow-fix-evidence/arm64` on the Mac. + +All runners stop their own test VMs in `finally`; final process checks found no remaining runtimes from this pass on the Mac, nested ARM64 VM, or Surface. Three pre-existing Surface branch-5 runtimes were left untouched. The disposable nested ARM64 VM was shut down after evidence collection. Unrelated development VMs are not targets for cleanup. Raw artifacts can contain guest state and are not committed. + +## Other validation + +After publication, the Mac CLI build passed with the pinned Git dependencies and no local source overrides: `cargo build -p microsandbox-cli --no-default-features --features net,ssh`. The generated lockfile changes only the twelve intended dependency sources. Formatting, diff whitespace checks, and Python smoke-script compilation passed. + +The Mac libkrun VMM library tests passed 52/52, including the resume-deadline regression. ARM64 VGIC tests passed 3/3, including the distributor/pending-state ordering regression. The failed-restore database regression passed. Runtime checkpoint tests passed 31/31 on Mac and Windows. The Windows public Rust mapping harness passed alias/sibling/backing isolation, bounds, parent-drop, and mapped-file-unlink checks. The standalone vm-memory unit harness cannot currently compile on Windows because its existing vmm-sys-util development dependency references Unix clock APIs; this is not reported as a passing unit run. The production mapping code was exercised by both the standalone Rust binary and the live WHP matrix. diff --git a/scripts/smoke/reports/execution-state-2026-09-07.md b/scripts/smoke/reports/execution-state-2026-09-07.md new file mode 100644 index 000000000..7f780b9a1 --- /dev/null +++ b/scripts/smoke/reports/execution-state-2026-09-07.md @@ -0,0 +1,70 @@ +# Execution-state fixes — 2026-09-07 + +Follow-up: [CoW platform fixes and qualification](cow-platform-fixes-2026-09-09.md) covers Windows private memory, failed-restore lifecycle protection, and the later ARM64 pending-interrupt regression. The backend revision and observations below remain historical. + +The Windows ARM64 post-restore command hang is fixed in the tested cases. Linux ARM64 now has working full execution-state capture/restore with VGICv3. Windows x86-64 has a new implementation with successful cross-compilation and executable userspace tests, but no native x86 WHP live qualification. This report does not mark all of #8 complete. + +Backend revision: libkrun `51c1ed3b83dc826c02800fb297995538e4eac55d`. Firmware remains `6cca413ac248f63e65d4ea4748b3bc36cd1b22f3`, using matching ARM64 kernel and agentd builds. Microsandbox additionally captures device state before interrupt-controller state, and captures RAM afterward. Public CLI/SDK signatures and disk-only snapshot formats are unchanged. Old Windows ARM64 development full snapshots require recapture because the internal execution-state ABI now includes CPU activity and clock-frequency state. + +## What changed + +- Windows ARM64: capture and restore the architecture-specific internal activity register, including StartupSuspend. Previously, an online secondary CPU could remain startup-suspended after restore, leaving guest work waiting indefinitely. Preserve genuinely offline CPUs too; do not force all CPUs online. +- Windows ARM64 timers: freeze partition time at the all-vCPU pause barrier. After WHP activates partition time, reinstall saved timer compare/control values before releasing any vCPU. Installing them earlier allows WHP's time activation to rebase the deadline incorrectly. Preserve enable/mask state and validate counter frequency. +- Linux ARM64: enumerate complete variable-width KVM registers, retain MP and exception state, save VGICv3 private/global interrupt state, and retain virtual/physical counter origins and timer controls across pause. Restore VM-wide counter offsets once, not separately for each vCPU. Use the architectural host frequency rather than a nonexistent CNTFRQ GET_ONE_REG interface. Accommodate VGIC initialization ordering on Linux 6.12 without dropping the IIDR handshake. +- Windows x86-64: add register/MSR, XSAVE, local APIC, SynIC, userspace IOAPIC, partition capability/frequency and AP startup state. An already-running restored AP skips SIPI initialization so its saved instruction pointer is not overwritten. +- Shared controller: discard stale capture/restore replies when waiting for a later pause/resume acknowledgement. A failed capture on one CPU must not poison another CPU's source-resume response queue. +- Microsandbox: quiesce/capture device workers before capturing interrupt-controller state. Otherwise the saved queues and saved interrupt state can disagree about a late device completion. KVM's LPI pending-table flush must still precede RAM capture. + +## Live coverage + +All live VMs below used Alpine, two vCPUs, 256 MiB memory and a 512 MiB root. These are debug builds. Linux ARM64 ran inside Debian 13/Linux 6.12.107 on QEMU/HVF with EL2 exposed on the M5 Max Mac: `/dev/kvm` and nVHE initialization were verified, and Microsandbox used KVM, not software CPU emulation. This qualifies the exercised nested-KVM configuration, not every ARM host or kernel. + +| Check | Windows ARM64/WHP | Linux ARM64/KVM | +| --- | --- | --- | +| Running full capture and usable restore | Pass, standard memory | Pass, standard and CoW memory | +| Flat / layered root coverage | Flat clock/offline-CPU tests; layered full lifecycle; earlier flat full lifecycle | Flat standard full lifecycle; layered CoW full lifecycle | +| Idempotent pause/resume, status and paused exec rejection | Pass | Pass | +| Two captures while remaining user-paused | Pass | Pass | +| 15-second pause, boot identity and original workload progress | Pass | Pass | +| Two children, private RAM writes, source/child isolation | Pass | Pass | +| Direct full archive capture/restore and archive unlink survival | Pass | Pass | +| Stop paused source, child still usable | Pass | Pass | +| Six restores from the later paused snapshot; CPU0 and CPU1 pinned timer work on every child | Pass, 6/6 | Pass, 6/6 | +| CPU1 offline at capture, still offline after restore, then successfully onlined | Pass | Pass | +| Delayed restore: first-thaw wall time, elapsed clocks, timer expiration/cancellation | Pass | Pass, CoW | + +The clock fixture has four concurrently scheduled readers. Windows observed a 20,638.76 ms wall-time gap with only 46.29 ms monotonic/boottime advancement across the checkpoint boundary; its five-second elapsed-time timer fired once at 5,004.89 ms. Linux observed a 10,519.05 ms wall gap, 96.74 ms elapsed-clock advancement, and one timer expiration at 5,012.80 ms. Both reported no backward readings, expired the absolute wall timer once, canceled the cancel-on-clock-set timer, and passed the first-thaw wall-time bound. These are guest application observations, not just successful host API calls. + +macOS/HVF ARM64 and OVH Linux/KVM x86-64 also passed a fresh flat-root CoW lifecycle regression, including 15-second pause, full and repeated paused captures, independent children, direct archive restore and cleanup. This is a regression check of the shared ordering change, not a repeat of every earlier SDK/platform test. + +## Observed elapsed times + +Milliseconds, individual CLI wall-time observations. The hosts and memory modes differ; these are not controlled speedup comparisons, percentiles, or stop-the-world durations. The first diagnostic nested-KVM restore took 13,947 ms; subsequent lifecycle restores below were much shorter. Retain that cold/diagnostic outlier rather than claiming a stable distribution. + +| Operation | Windows ARM64 standard, layered | Linux ARM64 standard, flat | Linux ARM64 CoW, layered | +| --- | ---: | ---: | ---: | +| Running full capture | 7,542.96 | 1,084.90 | 1,238.35 | +| Resident pause | 73.82 | 26.57 | 34.11 | +| First / second paused capture | 2,988.71 / 2,754.49 | 360.17 / 340.20 | 506.82 / 315.73 | +| Resident resume | 81.48 | 71.15 | 87.20 | +| Restore child A / B | 2,647.74 / 3,083.40 | 781.67 / 798.37 | 337.50 / 359.42 | +| Direct full archive capture | 13,993.57 | 2,484.29 | 2,351.41 | +| Direct full archive restore | 4,019.68 | 1,727.62 | 1,090.74 | + +Six later-paused-snapshot restores took 2,906–3,296 ms on Windows ARM64 and 786.70–972.59 ms on Linux ARM64. Every child executed the marker read and both CPU-pinned probes successfully. The macOS regression's two CoW children restored in 975.91–1,007.92 ms; the OVH x86 Linux regression's children restored in 165.04–166.40 ms. + +## Reproduction and evidence + +Use `scripts/smoke/cli/cow-memory-lifecycle.py` as described in the preceding lifecycle report. The runner now waits for the detached application's marker before checking it, rather than assuming runtime readiness means the application's first write has happened. + +Build `checkpoint-clock-probe.rs` and `checkpoint-cpu-probe.rs` as static Linux musl binaries for the guest architecture. The portable host runners are `checkpoint-clock.py` and `checkpoint-cpu-state.py`. Their module docstrings specify the required environment variables. The CPU test deliberately offlines CPU1 before capture, checks the offline value after restore, and brings CPU1 online again before running the affinity/timer probe. The clock test runs `checkpoint-clock-analyze.py` against the captured application's observations. Every runner attempts source/child cleanup in `finally`. + +Local raw evidence: `/private/tmp/msb-execution-state-20260907/` (ARM64 Linux and Windows), `/private/tmp/mac-arch-fix-cow/` (macOS). OVH evidence: `/home/ubuntu/msb-stack8.ElfKzf/arch-fix-results/`. The nested Linux host retained `/root/stack8/` until shutdown; its test disk remains in `/private/tmp/msb-linux-arm64.lXCCOl/`. Snapshots/cache files remain isolated development artifacts; no test workload is intentionally left running. + +## Validation limits + +The native ARM64 VMM suite reported 67 passing tests, two ignored tests and two failures in existing MMIO test setup: `test_register_virtio_device` and `test_register_too_many_devices` call `create_irq_chip()` on ARM and fail with EINVAL. The new ARM execution-state tests passed (3), and focused VGIC tests passed (2). The ARM-only vCPU creation test's declaration-order error was fixed. Microsandbox's focused checkpoint suite passed 27 tests. macOS controller barrier tests passed, including the new stale-capture recovery case. + +Windows x86 compiled for `x86_64-pc-windows-msvc`. Executable tests on the Surface's x86 emulation passed for restored-running AP startup, pending SIPI, IOAPIC programming/serialization, and stale-capture response recovery. That does not exercise an x86 virtual processor, XSAVE/APIC restoration, or x86 timer delivery under WHP; a native x86 Windows machine is still required. + +Windows CoW memory remains explicitly unavailable at the protected shared-cache integration boundary; these fixes qualify standard-memory execution restore, not a Windows CoW cache implementation. Linux ARM64 execution snapshots require VGICv3: VGICv2 does not expose the same complete input-line/latch state interface, and its ordinary boot path remains unchanged. Host suspend, cross-host CPU compatibility, every interrupt injection race, all failure-injection cases and repeated release-build performance distributions remain outside this report. diff --git a/scripts/smoke/reports/incremental-full-archives-2026-09-10.md b/scripts/smoke/reports/incremental-full-archives-2026-09-10.md new file mode 100644 index 000000000..29e0dcb2a --- /dev/null +++ b/scripts/smoke/reports/incremental-full-archives-2026-09-10.md @@ -0,0 +1,130 @@ +# Incremental full-snapshot archive qualification — 2026-09-10 + +## Result and scope + +RAM-aware `snapshot save --since` passed 12-checkpoint live chains on macOS ARM64/HVF and OVH Linux x86-64/KVM with flat, managed/layered, and tmpfs roots. The final checkpoint resumed with its expected RAM and filesystem contents through direct-archive and installed-snapshot restore, both eager and forked. This qualifies the archive dependency change, not unrelated concurrent lifecycle work. Linux ARM64 and Windows were not rerun for this change. + +The implementation replaces the unreleased disk-only dependency encoding with `msb-snapshot-dependencies-v1`. It omits reusable RAM objects as well as the exact physical disk prefix, keeps the target memory manifest and CPU/device state complete, and resolves the explicit base into destination-owned staging. `--last-layers` remains disk-only selection. Standalone archives and snapshot descriptors are unchanged; there is no compatibility shim for earlier unreleased #8 dependency archives. + +## macOS build and fixture + +- Base commit: `18c8fb8693957ac6e8ded964e42cfbe63f8506ad`, plus this change's archive implementation, tests, and CLI help. An isolated worktree excluded concurrent agentd, database, and control-path edits in the shared #8 checkout. +- Host: macOS 26.3, build `25D2125`, ARM64/HVF. +- Binary: debug build, `net,ssh` features, codesigned with `msb-entitlements.plist`. SHA-256: `dc21397227f028bb32ffeed7f8322f33caf2767f16bac264fae2ede80a8f6cdb`. +- Guest agent SHA-256: `4c467d1e93d9ba78168d0eb3ec6c0a5d03793b972c496e250fe0288a42e51e43`. +- Firmware SHA-256: `ea0d458cdc12a0fa6dac8d192542ddc39717f816da41176582905e31a8bf868c`. +- Each source: Alpine, 256 MiB RAM, two vCPUs. Flat/managed disks: 512 MiB; tmpfs root: 128 MiB. +- Workload: a retained 24 MiB random RAM file plus a RAM marker and root-filesystem marker changed before each capture. Checkpoint 6 was captured while explicitly paused; other captures were from a running source. +- Source and destination used separate `MSB_HOME` directories. Each destination independently populated its OCI image cache. Image bundling was not part of this test. + +## macOS size and timing + +These are individual end-to-end CLI wall times from sequential debug-build qualification runs, not release-build latency claims. Filesystem caches were warm, and forked restores ran after eager restores. MB means decimal megabytes. Capture times include publication, not just the pause interval. Export comparisons below use the same checkpoint 12 with and without `--since`. + +| Checkpoint 12 result | Flat | Managed/layered | Tmpfs | +| --- | ---: | ---: | ---: | +| Standalone archive bytes | 51,778,517 | 48,269,890 | 47,274,984 | +| Incremental archive bytes | 535,798 | 598,498 | 501,731 | +| Archive reduction | 98.97% | 98.76% | 98.94% | +| Omitted RAM objects | 13 | 13 | 13 | +| Omitted RAM object bytes | 131,334,144 | 135,737,344 | 126,586,880 | +| Full capture | 927.64 ms | 733.54 ms | 312.09 ms | +| Standalone export | 3,228.32 ms | 3,020.35 ms | 3,233.28 ms | +| Incremental export | 1,272.95 ms | 771.61 ms | 189.83 ms | +| Load final dependent archive | 2,222.48 ms | 1,717.76 ms | 1,042.21 ms | +| Direct archive → eager child | 3,361.47 ms | 2,687.48 ms | 1,836.25 ms | +| Direct archive → forked child | 3,367.98 ms | 2,936.96 ms | 1,680.98 ms | +| Installed snapshot → eager child | 1,256.04 ms | 1,076.20 ms | 752.70 ms | +| Installed snapshot → forked child | 792.42 ms | 602.98 ms | 237.71 ms | +| Sum of timed test commands | 75.84 s | 52.63 s | 38.96 s | + +The size reduction is workload-dependent. Reuse is at the existing RAM-object granularity, not a new byte-diff encoding. An object can contain ranges not selected by the current memory map, so omitted object bytes are not the number of live unchanged guest bytes. Loading still reconstructs a complete local snapshot and verifies the borrowed RAM objects; a small archive does not imply equally small local storage or load work. Capture, live RAM allocation, and CoW materialization were not redesigned here. + +## macOS live invariants + +| Check | Flat | Managed/layered | Tmpfs | +| --- | --- | --- | --- | +| Standalone baseline plus 11 dependent exports | Pass | Pass | Pass | +| Every dependent export actually omits RAM payloads | Pass | Pass | Pass | +| Load checkpoints 1–11 without starting checkpoint VMs | Pass | Pass | Pass | +| Delete each prior installed base after resolving the next | Pass | Pass | Pass | +| Missing/nonexistent explicit base fails | Pass | Pass | Pass | +| Capture while paused, then resume source | Pass | Pass | Pass | +| Direct archive eager/forked restore of checkpoint 12 | Pass | Pass | Pass | +| Direct restore does not install the target snapshot | Pass | Pass | Pass | +| Restored RAM blob hash and both markers match checkpoint 12 | Pass | Pass | Pass | +| Child changes do not alter other restores or installed snapshot | Pass | Pass | Pass | +| Installed snapshot eager/forked restore after deleting its base | Pass | Pass | Pass | +| Live children retain RAM and disk writes after base deletion | Not separately injected | Pass | Pass | +| Final snapshot verification and standalone re-export | Pass | Pass | Pass | +| All test-owned VM processes stopped | Pass | Pass | Pass | + +The tmpfs root marker is RAM-backed: this exercises a dependent archive with no disk-layer omissions. The flat run stopped its direct children before deleting the base; the later managed/tmpfs runs additionally deleted the base while those children remained running. + +## Automated checks + +- `cargo test --locked -p microsandbox --no-default-features --features net,ssh --lib snapshot --offline --target-dir /private/tmp/rd-target-10`: **44 passed** in 7.43 s. +- New archive fixtures cover disk+RAM and RAM-only 12-generation chains, packed-object offsets, zero extents, changed object identities, complete CPU/device metadata, installed and standalone-archive bases, direct restore staging, base deletion, corrupted/missing RAM, malformed omissions, undeclared or unreferenced dependencies, truncated input, and standalone re-export. Existing disk-only archive and released-format regression tests remain passing. +- `cargo build --locked -p microsandbox-cli --no-default-features --features net,ssh --offline --target-dir /private/tmp/rd-target-10`: passed. +- `cargo fmt --all -- --check`, `git diff --check`, and Python smoke-script syntax compilation: passed in the isolated worktree. +- Strict Clippy is blocked by pre-existing warnings: `derivable_impls` in `crates/image/lib/snapshot/manifest.rs` and `too_many_arguments` in `sdk/rust/lib/snapshot/create.rs`. The scoped SDK check passed with `--no-deps -- -D warnings -A clippy::too_many_arguments`; no source lint allowances were added. +- A default-stack regression test exposed a large nested async future introduced by RAM verification. Boxing the buffered reader fixed it; the passing suite uses the default test stack, not an increased stack limit. + +## Reproduction and local evidence + +The repository smoke harness is `scripts/smoke/cli/incremental-full-archive.py`. Use a short, fresh output directory on macOS because runtime socket paths have a length limit: + +```bash +MSB_PATH=/path/to/codesigned/msb \ +MSB_LIBKRUNFW_PATH=/path/to/matching/libkrunfw.5.dylib \ +STACK8_OUT=/private/tmp/ram-delta-flat \ +STACK8_LAYOUT=flat:512M \ +python3 scripts/smoke/cli/incremental-full-archive.py +``` + +For managed roots use `STACK8_LAYOUT=512M`; for tmpfs use `STACK8_LAYOUT=tmpfs:128M`. An optional `STACK8_SEED_CACHE` reuses immutable OCI cache artifacts, never VM RAM caches. The harness records command logs, `results.json`, and `archive-sizes.json`, and stops only its own named VMs in cleanup. + +Successful run directories: `/private/tmp/rd-f11`, `/private/tmp/rd-m12`, and `/private/tmp/rd-t11`. These include test RAM and disk artifacts and are local evidence, not files to publish. An explicit process check after all runs found no remaining test VMs. + +Discarded fixture attempts are not counted as passes: an older installed firmware lacked the matching VMGenID readiness support; an overly long temporary path exceeded the socket-path limit; optional `--with-image` export from a flat-only cache lacked fsmeta; and `managed:512M` was not valid CLI syntax. Final runs used matching firmware, short directories, independent OCI caches, and the verified root-disk syntax. The `--with-image` cache limitation was not fixed by this archive dependency change. + +## Linux x86-64/KVM follow-up + +All three root layouts passed the same 12-checkpoint live harness on OVH, including deleting the supplied base while direct eager and forked children remained running. Each run exported the baseline standalone, exported checkpoints 2–12 with `--since`, loaded 1–11 sequentially with `--base`, and restored checkpoint 12 both directly from its archive and from its installed snapshot. Guest reads verified the checkpoint-12 RAM/root markers and the original 24 MiB RAM-file hash. Child writes remained private. Missing bases failed, capture while explicitly paused passed, and final verification and standalone re-export passed. No additional implementation fix was needed for Linux. + +### Linux build and storage + +- Same isolated `18c8fb8693957ac6e8ded964e42cfbe63f8506ad` baseline plus archive edits; no concurrent #8 lifecycle changes. The transferred `delta.rs` SHA-256 matched the Mac source: `25894fc39499d9d664c02d6bd126dfd73a3c6c325c55d4f39fd879c23af6460e`. +- Host kernel: `7.0.0-28-generic`, x86-64/KVM. Rust: `1.97.1`. Guest fixture sizes and workload match the Mac tests. +- Rebuilt the matching static x86-64 guest agent from source; its release build took 20.83 s. SHA-256: `31c648803053be07bc4dc8491b2e16035b44dbf79d21da097a69e609c2658814`. +- Debug CLI build with `--locked --no-default-features --features net,ssh --offline` took 38.62 s. SHA-256: `9bdd6c7ed090520be31e95da7776655339fd62d819640f4a8c87af5553acd10b`. +- Matching existing firmware SHA-256: `6acfb3c81238e64f60ab1dcac95e7b2c2c57161a5e6ab10fbab18e457a205d80`. +- Final archive, cache, sandbox, and temporary staging directories were on the host's ext4 `/dev/md3` filesystem. `TMPDIR=/home/ubuntu/rdl.pa6szm` kept SDK temporary staging and launch-configuration files there too. These results are not from host tmpfs storage; the guest tmpfs-root case still correctly stores its filesystem contents in guest RAM. +- The Linux snapshot regression suite passed **44/44 tests** in 0.45 s with the same test command and features as the Mac. The new host build and all live tests used the rebuilt matching agent. + +### Linux measurements + +One sequential debug-build run per layout, warm filesystem caches; these are end-to-end CLI times, not pause durations or release latency claims. Forked restores ran after eager restores. The final standalone and incremental exports use the same checkpoint 12. + +| Checkpoint 12 result | Flat | Managed/layered | Tmpfs | +| --- | ---: | ---: | ---: | +| Standalone archive bytes | 43,430,398 | 39,755,415 | 39,378,027 | +| Incremental archive bytes | 258,923 | 257,546 | 232,087 | +| Archive reduction | 99.40% | 99.35% | 99.41% | +| Omitted RAM objects | 13 | 13 | 13 | +| Omitted RAM object bytes | 112,947,200 | 121,892,864 | 106,696,704 | +| Full capture | 128.87 ms | 125.26 ms | 84.44 ms | +| Standalone export | 992.76 ms | 900.48 ms | 838.71 ms | +| Incremental export | 139.09 ms | 128.20 ms | 104.27 ms | +| Load final dependent archive | 1,592.66 ms | 1,690.74 ms | 1,459.00 ms | +| Direct archive → eager child | 2,382.62 ms | 2,501.35 ms | 2,179.46 ms | +| Direct archive → forked child | 2,451.04 ms | 2,565.34 ms | 2,244.45 ms | +| Installed snapshot → eager child | 806.07 ms | 809.40 ms | 732.56 ms | +| Installed snapshot → forked child | 235.81 ms | 239.48 ms | 198.96 ms | +| Sum of timed test commands | 50.36 s | 44.48 s | 40.72 s | + +### Linux evidence and fixture corrections + +The isolated source/build directory is `/home/ubuntu/ram-delta-linux.mdyJh7`. Successful disk-backed run directories are `/home/ubuntu/rdl.pa6szm/{flat-2,layered-2,tmpfs-2}`. Timing and size JSON files were copied to `/private/tmp/rd-linux-evidence-10/{flat,layered,tmpfs}` on the Mac; raw RAM and disk artifacts were not copied or published. The final process check found no remaining runtime using the isolated test binary, including the diagnostic probe. + +An initial long runtime path exceeded the Unix socket-path limit. A subsequent flat-root run under `/tmp/rdl.W5axYo/flat` passed, but `/tmp` on this host is RAM-backed, so its timings are excluded from the table. The first disk-backed rerun then encountered `EDQUOT` while writing an anonymous launch-configuration file in `/tmp`, not while writing a snapshot. A scoped syscall trace identified that location. Setting only the test process's `TMPDIR` to the private disk-backed directory resolved it; no host quota, system configuration, or unrelated files were changed. The three final disk-backed runs above completed successfully. diff --git a/scripts/smoke/reports/live-disk-snapshot-2026-09-09.md b/scripts/smoke/reports/live-disk-snapshot-2026-09-09.md new file mode 100644 index 000000000..8ed0b7588 --- /dev/null +++ b/scripts/smoke/reports/live-disk-snapshot-2026-09-09.md @@ -0,0 +1,69 @@ +# Live disk-only snapshots — 2026-09-09 + +Archive names in this report use the current `.msb` convention. Retained raw logs preserve the filenames used in the original runs. + +Implemented on the #8 branch above Microsandbox `f68c1329`, using existing pinned libkrun `862d6842`, rust-vmm `f798d4f2`, and matching firmware. No companion changes, dependency overrides, schema change, or new public flags were required. + +## Behavior + +```sh +# Source stays running. No RAM checkpoint or --full workaround. +msb snapshot create saved --from-sandbox source +msb create --name child --from-snapshot saved + +# Direct archive: no installed snapshot directory or index row. +msb snapshot create exported --from-sandbox source --archive ./exported.msb +msb create --name archive-child --from-snapshot ./exported.msb +``` + +Running and user-paused managed/flat OCI roots use the serialized runtime control executor and a distinct capability-gated `disk_checkpoint_create` operation. Rollover seals the disk and selects a private successor. Running sources resume before SDK packaging; user-paused sources stay paused. Stopped/crashed copies retain their lifecycle lock. The SDK packages only the immutable disk closure, rechecks source identity, and removes consumed staging. + +Live disk cuts are **crash-consistent**, not application/filesystem-quiesced. Unsaved application buffers, guest page-cache writes not submitted to disk, and tmpfs are not promised. Children cold-boot with private writable heads. The block worker is inspected by existing rollover machinery, but no RAM, full CPU/device checkpoint, freezer handshake, or RAM-baseline update is required. Full capture retains its existing behavior. Cloud, tmpfs disk-only, and user-owned disk-image restrictions remain unchanged. Unsupported old runtimes are refused without a full-capture or writable-file-copy fallback. + +Existing Rust, Python, TypeScript and Go snapshot APIs already route through the shared Rust implementation. Documentation was corrected; no duplicate API was added. Native language packages were not rebuilt/live-tested in this pass. + +## Live coverage + +The committed `scripts/smoke/cli/live-disk-snapshot.py` passed on macOS ARM64/HVF, OVH Linux x86-64/KVM, and Surface Windows ARM64/WHP. Each tested flat and layered roots with 512 MiB disk capacity, 256 MiB RAM, and a cached Alpine image. + +| Checks, on both root layouts | Mac | Linux | Windows | +| --- | --- | --- | --- | +| Running installed capture and cold restore; optional integrity | Pass | Pass | Pass | +| Direct compressed `.msb` and plain `.tar`; no installed intermediate | Pass | Pass | Pass | +| User-paused capture remains paused; exec refuses until explicit resume | Pass | Pass | Pass | +| Source RAM/boot ID retained; disk child has a new boot ID and no tmpfs marker | Pass | Pass | Pass | +| Source/child writes isolated; sealed payload hashes unchanged | Pass | Pass | Pass | +| Child usable after snapshot/archive deletion | Pass | Pass | Pass | +| Disk capture leaves RAM store/cache untouched; transient staging released | Pass | Pass | Pass | +| Full after disk; disk between full captures; updated RAM restores | Pass | Pass | Pass | +| Stopped capture after live rollover; duplicate-name and tmpfs refusal | Pass | Pass | Pass | +| Capture during active writes; writer progresses; captured counter cold-boots | Pass | Pass | Pass | + +An initial live test caught incorrect acquisition of the runtime-owned lifecycle lock; live SDK capture now uses runtime serialization, while stopped capture retains the lock. The first Windows active-write fixture failed to start its separate long-running host client before capture. The final portable fixture uses a guest-owned background worker, as existing branch tests do, and verifies startup, progress, and termination. + +## Timings + +Debug-build whole CLI wall times in milliseconds, one observation per case. These are correctness-run observations, not release benchmarks, percentiles, or controlled host-to-host comparisons. Cache state, hardware, sequential chain depth, and workload matter. Guest contents and command success were checked separately after create returned. + +| Host / root | Installed capture | Installed cold restore | Compressed archive capture | Archive cold restore | +| --- | ---: | ---: | ---: | ---: | +| Mac / flat | 203.20 | 415.35 | 366.84 | 629.40 | +| Mac / layered | 94.74 | 311.51 | 139.37 | 402.38 | +| Linux / flat | 24.57 | 319.45 | 144.48 | 346.42 | +| Linux / layered | 14.18 | 320.52 | 22.19 | 335.16 | +| Windows / flat | 234.00 | 2359.00 | 906.00 | 1328.00 | +| Windows / layered | 157.00 | 2016.00 | 250.00 | 1141.00 | + +These samples come from Mac `live-disk-mac-h`, Linux `live-disk-linux-d`, and Windows `live-disk-win-c`. Runtime `capture_disk.pause_us` separately measures pause request through resume acknowledgement for running sources. First installed captures measured 174.32/70.56 ms for flat/layered on Mac and 145.21/49.04 ms on Windows. Linux's preceding `live-disk-linux-c` run measured 5.91/3.32 ms. User-paused measurements describe rollover work, not the user's entire suspension interval. + +Disk-only skips RAM capture but is not constant-time or zero-I/O. Existing rollover still performs layer integrity work and prepares its immutable closure while paused. Allocated bytes, chain depth, copying fallback, outstanding I/O and durability latency can increase pause time; this change does not optimize that existing hot path away. SDK publication and archive compression follow source resume. + +## Reproduction and limits + +Set `MSB_PATH`, isolated `MSB_HOME`, matching `MSB_LIBKRUNFW_PATH`, unique `STACK8_PREFIX`, and `STACK8_OUT`, then run `python3 scripts/smoke/cli/live-disk-snapshot.py`. The fixture stops only its own VMs in `finally`. + +Final portable-fixture evidence: `/private/tmp/msb-live-disk-mac-i/results.json`; `/home/ubuntu/msb-live-disk-linux-e/results.json` on OVH; `C:\Users\Stephen\AppData\Local\Temp\msb-live-disk-win-e\results.json` on Surface. Earlier timing runs retain the corresponding result directories. Guest disks/RAM are not committed. + +Runtime control tests passed 6/6; checkpoint-filtered runtime tests 32/32; SDK snapshot-filtered tests 31/31. Formatting and diff whitespace checks passed. CLI builds passed on all three hosts with pinned Git sources; the Mac runnable binary was codesigned. + +Linux ARM64/nested KVM and Windows x86-64 were not rerun for this addition. Native SDK packages, exhaustive crash/publication fault injection, disk-full, near-limit chains, concurrent maintenance, large cold-cache workloads and release p50/p95 performance are not qualified by this pass. diff --git a/scripts/smoke/reports/merge-reconciliation-2026-09-10.md b/scripts/smoke/reports/merge-reconciliation-2026-09-10.md new file mode 100644 index 000000000..a8f6c6f5c --- /dev/null +++ b/scripts/smoke/reports/merge-reconciliation-2026-09-10.md @@ -0,0 +1,115 @@ +# #8 merge reconciliation — 2026-09-10 + +This records reconciliation and focused regression checks, not merge approval or a complete live-platform qualification. The initial repair pass removed incomplete private workload-control plumbing; the subsequent coordinated implementation and its remaining failures are recorded below. The maintainer requested committing and pushing this progress to #8 with those failures still open. That publication does not authorize merging the PR or claim the stack is ready. + +## Transport follow-up — live qualification in progress + +The maintainer subsequently requested and approved the coordinated transport fix, including both host writers, readers, guest dispatch, the coordinator, and saved transport counters. The integrated implementation compiles. Freeze/thaw uses a bounded in-process route to the existing primary writer; ordinary input remains ordered and gated at complete-frame boundaries. There is no additional socket or public API. Native guest execution and fresh Mac/Linux live qualification are being completed; compilation alone is not a qualification pass. + +The initial safety-review hold was resolved by the maintainer's subsequent approval to complete the coordinated repair. Full snapshots produced by earlier unreleased development builds without the required transport counters are rejected. Released disk snapshots and the frozen generation-8 protocol schema are unchanged. + +The agreed input accounting uses cumulative control/bulk wire-byte and frame counters, including headers, with 8 MiB/256 ordinary control frames and 32 MiB/256 bulk records. Combined-port raw bulk records use bulk accounting too. These limits fit the existing 512-entry guest consumer queues when charges remain owned through consumption. Accepted guest input and its remaining credit debt survive restore; unadmitted host input remains source-owned. No counter reset or new public socket is intended. Private replies must match operation and attempt, and gate release requires confirmed thaw. Reverse output backpressure still needs a bounded-prefix review; an ordinary output-budget wait cannot be described as consumer-independent lifecycle progress. + +The integrated follow-up passes 271 host runtime tests, 196 native Linux guest tests, 61 protocol tests, four schema checks, and 21 Unix agent-client tests. Runner/client-only runtime and local SDK library Clippy checks pass; native Linux guest Clippy also passes with `-D warnings`. The ARM64 musl guest cross-check passes (existing musl `time_t` deprecation is allowed only in that cross-Clippy invocation). Released `schema/gen-8.json` remains byte-identical with SHA-256 `0b9ec1348f019430fdcb2d4acf3cac48f2267b1091523ed523be1daf5831d911`. The live harness has 26 passing transport helper tests and reuses 24 passing cleanup tests. Local socket fixtures require execution permission to bind their temporary Unix socket; the permitted rerun passes. These are not candidate VM qualification. + +The tested source bundle is `/private/tmp/msb-transport-repair.PF6XsF/candidate-source-2.tar.gz`, SHA-256 `5ef502deb821420a3bc671622e450999dd8e6a22e5527608108189d593b7b60e`; OVH extracted it into `/home/ubuntu/msb-transport-repair.AKNpfY/candidate-2`. Guest tests and release guest builds use that identical source. The Mac release binary is codesigned with this checkout's hypervisor entitlements. Baseline binaries remain preserved separately. + +### Candidate-2 live finding: inherited stdin exhausts control frames + +The Mac pilot passed idle pause/resume, blocked pipe resume, exact source 16 MiB input plus EOF, and the restored child's retained prefix plus pipe EOF. However, full capture took 667.57 ms and the user-visible child restore command took 19,888.95 ms, coinciding with the autonomous consumer gate. This is a qualification failure, not an acceptable restore result. The runtime itself activated in 61.042 ms (626 µs private thaw), then fresh child clients waited roughly 19.6 seconds. The checkpoint has both sent and granted `control_frames=2433`: all 256 outstanding frame credits belong to 8 KiB stdin messages, leaving no frame credit for new leases or exec requests despite 6,279,936 bytes of headroom. A longer fixture timer would hide nothing; it would merely extend the wait. + +The proposed correction is to charge inline stdin, TCP input/EOF, filesystem input/EOF and raw records to the existing data window, leaving control credit for leases and ordinary requests. It reuses already-decoded message types and existing writers/FIFOs; it does not reset debt or add a queue/socket. New `data_*` counter names would reject superseded unreleased captures. This material accounting change is awaiting maintainer approval. The pilot's raw report is `/private/tmp/msb-transport-repair.PF6XsF/mac-candidate-pilot/candidate/report.json`; its isolated home `/tmp/msb-t-b556ro7o` is retained for logs/artifacts, with all owned VMs stopped and no cleanup errors. + +The matching Linux pilot subsequently reproduced the same gate-overlap failure after blocked pipe pause/resume passed; it did not proceed to branch. Raw report: `/home/ubuntu/msb-transport-repair.AKNpfY/transport-candidate-harness.3IOpBC/linux-candidate2-pilot/candidate/report.json`. Its failed evidence home `/tmp/msb-t-yg7zr1vs` is retained but stopped. Both hosts have no remaining owned test VMs. + +The exact pre-follow-up source baseline is preserved at `/private/tmp/msb-transport-repair.PF6XsF/baseline-source.tar.gz`, SHA-256 `9f624caa07a16f496698ca64dd591dba49f6329d6c4dd1e3c3b258409d586dc8`. Fresh Mac and Linux release baseline binaries and matching ARM64/x86-64 guest agents were built. Initial live harness attempts exposed harness socket-path and output-only stdin-lifetime problems; their partial timings are not a valid performance comparison. Candidate live testing and before/after performance qualification remain outstanding. + +Corrected baseline runs subsequently passed on macOS ARM64 and OVH Linux x86-64: three idle pause/resume cycles, three pipe and three PTY workloads, exact 16 MiB stdin hash/EOF checks, sequenced control output, checksummed 64 MiB uploads/downloads, and subsequent framing checks. Both reports confirm no cleanup errors, no remaining owned sandboxes/runtime PIDs, and removal of the isolated homes. No blocked-input capture, branch, or restore qualification was performed in these baseline runs. + +| Baseline, three samples per measurement | macOS ARM64 p50 / p95 | Linux x86-64 p50 / p95 | +| --- | --- | --- | +| Idle pause | 12.46 / 13.43 ms | 7.56 / 7.65 ms | +| Idle resume | 11.88 / 12.83 ms | 3.53 / 7.48 ms | +| Pipe stdin throughput | 48.78 / 51.86 MiB/s | 40.22 / 47.65 MiB/s | +| PTY stdin throughput | 11.49 / 11.50 MiB/s | 20.05 / 20.41 MiB/s | + +These are pre-fix baseline observations, not improvements or candidate passes. Raw reports: `/private/tmp/msb-transport-repair.PF6XsF/mac-baseline-perf-3/baseline/report.json` and `/private/tmp/msb-transport-repair.PF6XsF/linux-baseline-evidence/linux-baseline-perf-3/baseline/report.json`. The reports retain workload parameters, binary/agent/firmware hashes, architecture-specific image digests, CPU/throughput samples, and cleanup evidence. Percentiles use nearest rank; with only three samples, p95 is the maximum observed sample. + +Candidate-2's separate idle/throughput runs pass byte, EOF, framing, and cleanup checks on both hosts, but do not establish capture/restore correctness or acceptable final performance. Each uses three samples, the same image digest per host, 16 MiB stdin, and concurrent 64 MiB bulk transfers. The following p50 observations are provisional; a same-session Linux baseline repeat is being used to check host/time variability. + +| Metric | Mac baseline → candidate-2 | Linux baseline → candidate-2 | +| --- | --- | --- | +| Idle pause | 12.46 → 13.18 ms | 7.56 → 3.57 ms | +| Idle resume | 11.88 → 11.57 ms | 3.53 → 3.50 ms | +| Pipe stdin | 48.78 → 67.15 MiB/s | 40.22 → 24.24 MiB/s | +| PTY stdin | 11.49 → 9.64 MiB/s | 20.05 → 5.73 MiB/s | + +The Linux PTY slowdown is significant and unresolved, not an accepted cost of correctness. Source runtime CPU measurements include concurrent filesystem transfers, which run longer and transfer more total data in a slower stdin case; they do not isolate stdin overhead. The host writer comparison found extra admission locking/notification work, but no new payload copy, changed console batching, or verified busy loop. These are optimization candidates, not a proven causal explanation. Reports: `/private/tmp/msb-transport-repair.PF6XsF/mac-candidate2-perf/candidate/report.json` and `/private/tmp/msb-transport-repair.PF6XsF/linux-candidate2-evidence/linux-candidate-perf/candidate/report.json`. Both runs removed their isolated homes and report no remaining owned VM processes. + +Read-only follow-up identified a guest scheduling candidate: after the first partial stdin write, subsequent input queues without probing newly available pipe/PTY capacity, and queued input drains only in bounded turns of the outer reactor loop. A bounded, non-awaiting FIFO drain per decoded input batch may reduce that backlog without reinstating the original blocking write. Separately, host admission currently registers notifications and locks state on two loop turns per frame; arming a waiter only when blocked could reduce overhead while preserving atomic gate checks. Neither optimization has been implemented or proven causal. The Linux mixed-load PTY test completed 30 downloads and 11 uploads in candidate-2 versus five and three in the original baseline; fixed-volume/stdin-only checks are needed before interpreting this as an intrinsic per-byte slowdown. + +A subsequent Linux baseline repeat with the same current mixed-load harness passed all nine samples and cleanup. Pipe/PTY stdin p50 was 38.93/20.16 MiB/s, close to the original 40.22/20.05 MiB/s. The candidate-2 mixed-load slowdown therefore persists against a nearby baseline, while the different amount of competing bulk work still prevents causal attribution. The new stdin-only harness case has helper coverage but has not been live-run; further runs are held pending the accounting decision. Baseline-repeat report: `/home/ubuntu/msb-transport-repair.AKNpfY/transport-candidate-harness.3IOpBC/linux-baseline-interleaved/baseline/report.json`. + +## Pinned inputs + +- Original #8: `24152183b0eace990798e31f1a53f024980f938f`. +- Updated #7 merge input: `2830a4e2465fb1572a2b007e4cb5f3d790bca74c`. +- Common ancestor: `ce04099b660adcd75e4b63a985bc51fb30d30504`. +- Isolated candidate: `/private/tmp/msb-source-flag.KUTTO5/pr8`. During qualification, reviewed working-tree resolutions were deliberately left unstaged in the merge index. The maintainer's subsequent commit/push request includes recording those resolutions; it does not resolve the behavioral gaps below. + +The comparison uses both pinned parents, not newer unrelated `main`. The candidate's root `Cargo.toml` and `Cargo.lock` remain identical to the updated #7 input. Published `msb_krun` 0.1.34 and `msb-imago` 0.1.7 remain selected; no new Git dependency override was introduced. + +## Intersection map and repairs + +| Intersection | Reconciled behavior | Evidence / limitation | +| --- | --- | --- | +| Release lifecycle identity × #8 pause/branch | Retained Sandbox and SandboxHandle receivers reject same-name replacements. Control selection binds the sandbox row, run row, and PID, then checks the connected endpoint's server PID before writing. Branch revalidates the selected run before capture. | Stale receiver, endpoint mismatch, and source-restart regression tests pass. This does not serialize every live configuration mutation; see remaining work. | +| Release startup ownership × #8 restore finalization | The actual child process stays owned through readiness, catalog publication, and restore finalization. Detached creation disarms ownership only after success; failure cleanup avoids reacquiring the transition lock already held by the creator. | Startup/kill and abandoned-Starting tests pass. Pending restore intent remains fail-closed. | +| Windows lock handoff × startup acknowledgement | The parent explicitly releases the non-inheritable Windows lifecycle guard immediately before spawn while retaining the name-transition guard. | A Windows helper-process regression test was added, but native execution and Windows typechecking remain unqualified. | +| Release removal/maintenance × #8 lineage publication | Snapshot lineage uses one stable lock outside the removable sandbox directory. Removal/replacement coordinates with it. Ephemeral maintenance uses a nonblocking claim, including the exit observer that already owns lifecycle, and retries without deleting artifacts when capture owns lineage. | SDK removal/lineage tests and runtime cleanup tests pass. Never wait for lineage while holding lifecycle. | +| Live disk rollover × failure recovery | Tentative sealed-layer hashes no longer mutate the live head before successful rollover. A journal replacement/directory-sync failure is treated as possibly published and leaves the source paused for recovery. | Raw/qcow retry and injected post-rename failure tests pass; these are not live power-loss tests. | +| Portable memory cache × eviction | Publication holds a backing pin before exposing a new cache entry; an existing winning entry is also pinned before use. | Publication/eviction race tests pass. | +| Paused guest refusal × release filesystem bulk protocol | Filesystem negotiation and credit waits preserve terminal `core.error` instead of losing its useful rejection reason. | Both added tests failed before the change and pass afterward; all seven focused filesystem pause tests pass. | +| Released database history × snapshot identity/groups | The complete 25-migration release prefix remains unchanged, followed by snapshot identity and then snapshot groups. All 26 executable migration entries and payloads from updated #7 remain intact. | 31 migration tests pass, including canonical ordering and group migration/refusal checks on synthetic catalogs. | +| Release protocol/feature split × #8 restore | Generation-8 schema bytes are unchanged. Strict restore intent and the internal `--restore` argument remain alongside release file mounts, resolved network configuration, and client/runner feature separation. | Bounded two-parent source comparison and native focused builds/tests. No old-binary live compatibility run in this pass. | +| CLI/SDK/docs × combined surface | Snapshot groups, batch loading, `--from-sandbox`, cloud rejection, and forked restore constraints remain. Python `connect_or_create` now declares the `forked` option already accepted by its native builder. Lifecycle docs describe Paused and SDK resume correctly. | Four Python AST contract checks and documentation language-order checks pass. | +| Release kernel cache action × #8 firmware pin | Retain firmware `6cca413ac248f63e65d4ea4748b3bc36cd1b22f3` with the clock-only restore path. Teach the cache action the exact kernel 6.12.108 tarball checksum. | Resolver tests accept the exact 6.12.99/6.12.108 checksums and URLs and reject an unknown version. No kernel rebuild in this pass. | + +The kernel checksum was compared against kernel.org's [signed checksum-index text](https://cdn.kernel.org/pub/linux/kernel/v6.x/sha256sums.asc) retrieved over HTTPS; this is not a claim that its OpenPGP signature was independently verified. + +## Focused checks + +Native Rust checks used macOS ARM64, principally `CARGO_TARGET_DIR=/private/tmp/rd-target-10`. `MSB_AGENTD_PATH=/private/tmp/msb-cow-8.PIhgYp/build/agentd` was only a compile fixture; it is not evidence of a fresh matching live guest build. Migration/client-only checks used `/private/tmp/msb-merge-client-target`. + +| Check | Result | +| --- | --- | +| SDK `backend::local` tests, `local,net` | 60 passed | +| SDK `sandbox::pause` tests | 4 passed | +| SDK persisted-removal tests | 3 passed | +| SDK `snapshot::lineage` tests | 6 passed | +| SDK `sandbox::fs::pause_tests` | 7 passed | +| Runtime maintenance tests after ephemeral-cleanup repair | 15 passed | +| Runtime checkpoint tests, client-only / runner | 18 / 48 passed | +| Runtime `restored_` tests | 7 passed; includes the two characterization tests described below | +| `cargo test -p microsandbox-migration --lib --offline` | 31 passed | +| `cargo clippy -p microsandbox --lib --no-default-features --features local,net -- -D warnings` | Passed | +| `cargo clippy -p microsandbox-runtime --lib --no-default-features --features runner,net -- -D warnings` | Passed | +| Runtime Clippy with `--all-targets` | Fails `items_after_test_module` in client/control.rs, client/logging.rs, and runner/exec_log.rs; not silently suppressed | +| Python creation-stub contract functions | 4 passed using `runpy`; system Python lacks pytest, so this is not a pytest-suite pass | +| Node SDK native build, TypeScript build/typecheck, and unit tests | Passed; 153 tests across 12 files, using the newly built native binding | +| `python3 -m unittest discover -s scripts/smoke/cli -p test_snapshot_branch.py -v` | 24 passed; harness tests, not live sandbox tests | +| `python3 scripts/check-docs-language-order.py` | 117 multilingual groups and SDK navigation passed | +| `cargo fmt --all -- --check`; `git diff --check` | Passed before the final report update | + +Local Unix-socket fixtures initially failed to bind under tool sandbox restrictions and passed with the required permissions. Windows ARM64 cross-check stopped in dependency C compilation because the host lacks Windows CRT headers (`stdlib.h`, `assert.h`, and `setjmp.h`); it did not establish Windows type correctness. The Surface SSH connection timed out. + +Node checks ran in `/private/tmp/msb-merge-node.K1eep5`, a copy of this candidate's SDK. The repository's CI pruning script removed unpublished platform optional dependencies in that disposable copy before offline `npm ci`; tracked package manifests and lockfiles were not changed. Tests initially lacked built JavaScript/native artifacts, then passed after `npm run build:ts` and a fresh `cargo build --offline -p microsandbox-node`. That build used a disposable `MSB_HOME` and `/private/tmp/msb-v070-integration.BKd4tB/artifacts` solely to satisfy the build-time runtime-pair requirement; no VM was started from those artifacts. The debug native link reported a large `__eh_frame` compact-unwind warning. No release-build performance claim is made. + +## Remaining work — not repaired or qualified + +1. **Transport qualification and retained-input control starvation.** The original loopback/FIFO deadlock and missing frame-boundary plumbing have been replaced by the coordinated implementation described above. Native tests pass, but the fresh Mac pilot exposed inherited stdin exhausting ordinary control-frame credits. The data/control accounting correction awaits approval. Complete blocked-input capture/branch/restore and performance qualification remain required after that correction; the candidate is not merge-ready. +2. **Concurrent host-policy changes during branch.** Exact run validation prevents selecting one runtime and capturing another, but a same-run live secret update can still race with cloning the source's host-side configuration. The capture refreshes CPU/memory/interface values, not a transactionally consistent host secret policy. This inherited branch/modify interaction remains a separate decision and implementation item; no broad modification-lock refactor was made. +3. **Older-macOS dependency delivery.** The separately prepared libkrun optional-HVF-symbol fix at `31f3ce3` is not part of the published `msb_krun` 0.1.34 dependency used by this candidate. This pass did not publish a replacement crate or change that pin. +4. **Fresh platform qualification.** The initial repair pass used no live VMs. The transport follow-up now has matching release binaries, native guest tests, and fresh Mac/Linux live evidence as detailed above; it is not a complete platform pass. Windows ARM64 is not tested in this follow-up. Historical live reports must not be presented as tests of this candidate. + +The isolated merge remains available for inspection and continuation. None of these checks establishes that #8 or the full release stack is merge-ready. diff --git a/scripts/smoke/reports/node-lifecycle-concurrency-2026-09-10.md b/scripts/smoke/reports/node-lifecycle-concurrency-2026-09-10.md new file mode 100644 index 000000000..c5b1408ed --- /dev/null +++ b/scripts/smoke/reports/node-lifecycle-concurrency-2026-09-10.md @@ -0,0 +1,50 @@ +# Node lifecycle concurrency qualification — 2026-09-10 + +The approved Node ownership change is implemented. The wrapper now acquires an `Arc` under a short admission lock instead of holding the lock across guest execution, filesystem I/O, or lifecycle waits. Normal operations do not clone the sandbox configuration. Detach/removal consume the shared slot, so subsequent calls through the wrapper or an existing filesystem facade fail; admitted operations retain their references. The runtime still decides whether concurrent operations are valid. This is not a guarantee that work completes successfully after stopping or removing its sandbox. + +Live tests also found an existing filesystem response-decoding bug: the host relay rejects new guest work while paused using `core.error`, but filesystem helpers attempted to deserialize it as `FsResponse` or ignored it on streams. They now preserve the existing diagnostic through the SDK's existing unexpected-response helper. The wire protocol, snapshot format, and public method signatures are unchanged. No additional state lookup, guest round trip, or retry was introduced. + +## Release timing + +Each sample measures the Node promise from invocation to completion using `performance.now()`, while a command on the same object is already executing and waiting for an explicit release marker. There are 20 pause/resume pairs per host in the final sample. Sandbox configuration is Alpine, 256 MiB RAM, and a 512 MiB managed root disk. These are SDK call timings, not CLI startup, snapshot restore, or application-response timings. + +| Host | Pause median | Resume median | Pause min–max | Resume min–max | +| --- | --- | --- | --- | --- | +| macOS ARM64/HVF | 0.216 ms | 0.182 ms | 0.187–0.407 ms | 0.170–0.367 ms | +| Linux x86-64/KVM | 0.195 ms | 0.313 ms | 0.178–0.457 ms | 0.299–0.549 ms | + +A freshly built old macOS Node binding, using the same runtime and firmware, fails the pending-exec regression at the 2,000 ms pause deadline. The command intentionally waits indefinitely for a marker, so this is a demonstrated lock blockage, not a baseline latency distribution or a meaningful speedup ratio. The new binding completes pause/resume without releasing that command, then verifies the original command completes correctly. The old binding's failure and final candidate reports are retained separately. CH/FC and the snapshot/restore benchmark matrix were not rerun in this follow-up. + +## Live coverage + +Fixture: `sdk/node-ts/tests/lifecycle-concurrency.test.ts`, enabled with `MSB_NODE_LIFECYCLE_LIVE=1`. Set `MSB_NODE_TIMINGS` to save raw call timings. Use a short isolated `MSB_HOME` and explicitly select the matching development `MSB_PATH` and `MSB_LIBKRUNFW_PATH`. + +| Case | macOS ARM64 | Linux x86-64 | +| --- | --- | --- | +| Pending exec: 20 pause/resume cycles, public paused state, original command completion | Pass | Pass | +| Filesystem request while paused: useful rejection and successful read after resume | Pass | Pass | +| Detach during exec: consumed-handle errors, idempotent detach, original command and VM survive | Pass | Pass | +| Blocked filesystem upload: pause/resume/detach progress, existing facade rejects new work, admitted upload finishes with correct bytes | Pass | Pass | +| Removal during exec: wrapper consumption is independent of runtime lifecycle locking; admitted command completes | Pass | Pass | +| Terminal-state wait concurrent with stop, followed by successful removal and consumed-handle errors | Pass | Pass | + +The upload fixture uses a host FIFO and observes the first bytes in the guest before pausing, proving the guest stream has started rather than racing its initial admission. The removal fixture permits the runtime's existing lifecycle lock to defer removal until shutdown or reject it. If removal wins before stop's final database observation, a missing-row stop error is accepted only after verifying that the concurrent removal succeeded. The tests do not weaken runtime lifecycle locking or silently ignore arbitrary cleanup errors. + +Final live test-body totals, including VM creation and cleanup: macOS 1.72 seconds; Linux approximately 13.53 seconds. The longer Linux fixture total includes shutdown and is not the pause/resume latency shown above. Both final runs passed all six cases. No sandbox records or matching runtime processes remained in either isolated test home after cleanup. + +Other validation: 137 Node unit tests passed on each host; three Rust shared-ownership tests passed (pending operation, competing consumers, cancellation/lifetime); five Rust filesystem response tests passed (normal success/failure, paused diagnostic, unexpected envelope, read-stream rejection, terminal response without waiting for channel close); TypeScript build/typecheck and separate live-fixture typecheck passed; Node native Clippy with `-D warnings`, focused Rust formatting, and `git diff --check` passed. + +Initial attempts are retained rather than counted as passes: the first Mac home exceeded Unix socket path limits; Linux's first runner installation omitted its optional native bundler dependency; macOS archive metadata sidecars were initially collected as tests on Linux and were excluded with `--exclude '**/._*'` while all 137 actual tests ran. Early upload/removal assertions raced guest admission or incorrectly imposed a 2-second shutdown deadline; the final fixture checks the actual contracts above. The paused-filesystem decoding failure was a product bug and was fixed, not suppressed in the test. + +## Artifacts and scope + +- Local stage: `/private/tmp/msb-resident-perf.qHpm0p`; candidate package `node-candidate`, old binding `node-baseline`, final reports `node-live-results-final.json`, `node-timings-final.json`, and `node-unit-results-final.json`. Linux live/timing reports are also copied here with a `linux-` prefix. +- Linux stage: `/home/ubuntu/msb-resident-perf.zqr4vR`; final reports use the same names. Runtime `/bin/msb` under that stage is the previously qualified lifecycle candidate; live tests explicitly override the SDK's prebuilt discovery paths. +- Test homes: `/private/tmp/nl.u0biju` and `/home/ubuntu/nl.eRfSLi`. Test sandboxes were removed; image caches and reports are retained. +- macOS candidate Node binding SHA-256: `3ee1a932985d102e6977cf23da37a63eac374429906dbf02a8e0df2f5814147f`. +- macOS old Node binding SHA-256: `dae1ef24eb8d9716521f2c803d3c258d690e8902d7af8a2e4e53c12f9fab4bca`. +- Linux candidate Node binding SHA-256: `5467985868389923447bf0495b0cf7b42d8f49619b9b9ae20b4d7182170fda33`. + +Runtime and guest-firmware provenance is recorded in [the resident lifecycle report](resident-lifecycle-2026-09-10.md). The local worktree advanced from `18c8fb86` to `e20269e1` through another contributor's archive commit during this work; those unrelated changes were preserved. Linux uses the existing isolated lifecycle source stage plus this Node/FS patch. Qualification preceded the separately authorized commit and push. + +This qualifies the Node ownership follow-up on macOS ARM64 and Linux x86-64, not every mobility invariant. Windows, Linux ARM64, cloud execution, every SSH/streaming variant, retained-handle boot fencing, Python ownership optimization, and companion clock-patch integration are outside this run. No generated binding declarations, dependency versions, lockfiles, or submodule pointers were changed by this work. diff --git a/scripts/smoke/reports/registry-dependencies-2026-09-09.md b/scripts/smoke/reports/registry-dependencies-2026-09-09.md new file mode 100644 index 000000000..3a2bb5dac --- /dev/null +++ b/scripts/smoke/reports/registry-dependencies-2026-09-09.md @@ -0,0 +1,49 @@ +# Registry dependency qualification — 2026-09-09 + +This follow-up to #8 replaces development Cargo Git patches with published crates. It starts from Microsandbox `8e68722c47d7f06f76e51b78f44563939d47b4da`, retains the #7 stack base, and does not change the firmware pin, agent protocol, snapshot format, or public API. These are debug-build correctness checks, not release performance measurements. + +## Published dependencies + +| Package | Registry version | Release-source commit | +| --- | --- | --- | +| `msb-vm-memory` | `0.18.0-msb.2` | rust-vmm `c8aad4c`, on `appcypher/windows-private-memory` | +| `msb-imago` | `0.1.7` | imago `cba9c0c`, on `appcypher/release-memory-dependency` | +| All 15 `msb_krun` family crates | `0.1.34` | libkrun `b20d31a`, on `appcypher/registry-memory-dependencies` | + +All uploads completed and Cargo confirmed registry availability. Release-source branches were pushed; this does not claim they have merged into their default branches. No Microsandbox package was published. + +The memory release contains the Windows private-view implementation already used by #8's pinned `f798d4f` source. Imago's exact memory dependency had to move with it to avoid incompatible copies of the shared memory types; `0.1.7` also retains the previously released `0.1.6` tail-discard fix. Libkrun now uses those registry dependencies without a Git override. Microsandbox's lockfile changes only the intended 13 packages and resolves one `msb-vm-memory` version across imago and libkrun. + +## Release checks + +- Memory: Linux x86-64 passed 127 unit tests and 34 doctests, plus Clippy. macOS Clippy passed; 121 unit tests passed and five existing 4 KiB page-assumption tests failed on the 16 KiB-page host. Comparing against `0.18.0-msb.1` confirmed no Unix memory source changes. Windows ARM64 and x86-64 production-feature compilation passed; this is not a native Windows test run. Packaged dry run passed. +- Imago: all-feature build, Clippy, formatting, packaged dry run, 37 unit tests and four doctests passed; two tests were ignored. Windows ARM64 and x86-64 compilation passed. +- Libkrun: `cargo build --all --locked`, `cargo test --all --locked` (281 passed, six ignored), `cargo clippy --all --locked -- -D warnings`, formatting, and the coordinated 15-crate packaged dry run passed. `cargo check --locked -p msb_krun --features blk --target aarch64-pc-windows-msvc` passed with registry dependencies. + +## Microsandbox checks + +```bash +cargo build --locked -p microsandbox-cli --no-default-features --features net,ssh +cargo test --locked -p microsandbox-runtime --no-default-features --features net --lib checkpoint +cargo test --locked -p microsandbox --no-default-features --features net,ssh --lib snapshot +cargo fmt --all -- --check +git diff --check +``` + +Build and formatting passed; checkpoint tests passed 32/32 and snapshot tests passed 39/39. Cargo reported the existing future-incompatibility warning for `proc-macro-error2 2.0.1`. The CLI was codesigned with `msb-entitlements.plist` before live testing. + +## Mac live checks + +The existing `scripts/smoke/cli/direct-branch.py` harness ran with `STACK8_MAINTENANCE=1`, a new disposable `MSB_HOME`, flat 512 MiB disk, 256 MiB RAM, and two vCPUs. All 56 recorded steps had their expected result, including the two deliberate refusals: + +- First branch plus five repeated branches with retained siblings, without installed snapshot publication. +- Same-name refusal, private RAM/disk writes, source/sibling isolation, and a grandchild retaining its parent's private writes. +- Continuing guest timer-driven counter progress. +- Branch from a paused source, refusal to execute on the still-paused source, and ordinary resume. +- Three durable full captures and forked restores with subsequent guest reads. +- Live root growth to 768 MiB, explicit compaction, and another usable branch. +- Grandchild survival after source and parent stop; successful cleanup of all 13 test VMs. A final process check found no remaining runtimes with the test prefix. + +Evidence: `/private/tmp/msb8-registry-live.b1LAEv/results/results.json` and adjacent per-command logs. The signed debug CLI SHA-256 was `cdbfc66c9fae75d1a8d3e4015020889fa0f48b5ee01932c08de87a01282c8ab7`. Firmware SHA-256 was `ea0d458cdc12a0fa6dac8d192542ddc39717f816da41176582905e31a8bf868c`; the embedded agent SHA-256 was `4c467d1e93d9ba78168d0eb3ec6c0a5d03793b972c496e250fe0288a42e51e43`, matching the existing stack test artifacts. Raw guest state is not committed. + +This registry-only update was not live-rerun on Linux or Windows, and does not claim a new full platform matrix or language-SDK qualification. The Surface SSH connection timed out during this pass. Earlier successful platform coverage remains recorded separately in [CoW platform fixes](cow-platform-fixes-2026-09-09.md) and [live disk snapshot](live-disk-snapshot-2026-09-09.md); their historical source versions and measurements are unchanged. diff --git a/scripts/smoke/reports/resident-lifecycle-2026-09-10.md b/scripts/smoke/reports/resident-lifecycle-2026-09-10.md new file mode 100644 index 000000000..9cacd7e08 --- /dev/null +++ b/scripts/smoke/reports/resident-lifecycle-2026-09-10.md @@ -0,0 +1,69 @@ +# Resident pause/resume optimization qualification + +## Scope and implementation + +This pass implements the approved change from automatic guest-wide filesystem syncing to execution-state preservation: resident pause, full capture, and direct branch no longer call guest `sync()`. Full restore retains dirty guest memory; deliberately extracting only the disk is crash-consistent. Host block draining, sealed-layer durability, workload freezing, heartbeat gating, and clock acknowledgement before workload thaw remain in place. + +The CLI avoids a second command-tree construction unless `--tree` is present and uses a current-thread Tokio runtime for pause/resume. Ambient local backend resolution reuses its loaded configuration document. A healthy current-catalog control lookup uses a WAL-aware read-only pool without snapshot reconciliation or writer initialization. Schema/install/downgrade checks remain; older catalogs and stale targets use the existing slow path. Go's name-based pause/resume calls use that observation-free lookup. + +The independent libkrun clock-observation change suppresses notifications when the entire observed predicate is unchanged, under the same mutex used by waiters. It passes device unit tests but is **not included in the release binaries benchmarked below**; these use published `msb_krun` 0.1.34. No firmware or protocol version was bumped. + +The Node shared-ownership rewrite was approved and implemented in the subsequent [Node ownership follow-up](node-lifecycle-concurrency-2026-09-10.md); it is not part of the CLI/direct measurements below. Retained-handle boot fencing and the Python ownership micro-optimization are not implemented in this pass. Existing runtime-authoritative control remains; this report does not claim to have eliminated the existing name-reuse race for stale retained handles. + +## Release benchmark results + +Measurements are complete request-to-acknowledgement durations, not isolated hypervisor pause instructions. The direct route opens the same existing control socket and invokes the same runtime operation as the CLI. No snapshot occurs between resident pause and resume. Each entry is the median of 30 samples; CLI/direct order alternates within each fixture. Baseline and candidate each use fresh Small/Large fixtures, run sequentially per host, not randomized across builds. Treat small differences and tail estimates accordingly. + +| Host | Profile | Interface | Pause before → after, ms | Resume ACK before → after, ms | Resume through app response before → after, ms | +| --- | --- | --- | --- | --- | --- | +| macOS ARM64/HVF | Small | CLI | 11.980 → 5.305 | 6.582 → 4.836 | 7.236 → 5.417 | +| macOS ARM64/HVF | Small | Direct | 5.531 → 0.410 | 0.166 → 0.160 | 0.615 → 0.518 | +| macOS ARM64/HVF | Large | CLI | 11.957 → 5.348 | 6.449 → 4.970 | 7.081 → 5.585 | +| macOS ARM64/HVF | Large | Direct | 5.561 → 0.428 | 0.183 → 0.175 | 0.652 → 0.547 | +| Linux x86-64/KVM | Small | CLI | 2.717 → 1.916 | 2.795 → 2.078 | 3.223 → 2.532 | +| Linux x86-64/KVM | Small | Direct | 0.275 → 0.228 | 0.347 → 0.343 | 0.800 → 0.804 | +| Linux x86-64/KVM | Large | CLI | 2.773 → 1.944 | 2.748 → 2.018 | 3.182 → 2.486 | +| Linux x86-64/KVM | Large | Direct | 0.307 → 0.244 | 0.307 → 0.307 | 0.763 → 0.767 | + +All 480 measured baseline/candidate cycles completed successfully, with application identity, memory state, progress, and SQLite integrity checks. This is 240 candidate cycles and 240 baseline cycles across both hosts, including both interfaces. Small uses one vCPU/256 MiB RAM; Large uses two vCPUs/4096 MiB RAM. These are release builds with matching guest agents; macOS binaries were codesigned with the repository's VM entitlements. The existing bounded stage tracing is unchanged between builds; no new per-pause diagnostic logging was added. + +Mac Small direct pause improves about 13.5×; its CLI pause improves about 2.26×. Linux Small CLI pause improves about 1.42×. Direct resume is essentially unchanged, particularly on Linux. The remaining CLI cost must not be described as a slow VM resume primitive. This pass does not rerun CH/FC, isolate each optimization's individual contribution, or qualify p99 latency. Mac Large has one candidate CLI pause outlier of 64.8 ms; median improvements do not eliminate scheduling tails. + +## Correctness coverage + +The live fixture is `scripts/smoke/cli/dirty-memory-checkpoint.py`. It disables ordinary periodic guest dirty-page writeback for the fixture, sets generous dirty thresholds, dirties a 64 MiB shared mapping, and requires at least 32 MiB reported dirty before capture. It separately retains a private mapping, heap bytes, tmpfs data, and a file explicitly persisted with file/directory `fsync`. Guest settings affect only the disposable test VM. + +| Invariant | macOS flat | macOS layered | Linux flat | Linux layered | +| --- | --- | --- | --- | --- | +| Running full capture retains dirty page-cache/shared mmap, private mmap, heap and tmpfs | Pass | Pass | Pass | Pass | +| Eager and forked restore after source shutdown | Pass | Pass | Pass | Pass | +| Running/paused direct branch; source remains paused after paused capture/branch | Pass | Pass | Pass | Pass | +| Mutated direct child stays private; branch-of-branch retains mutations | Pass | Pass | Pass | Pass | +| Dirty incremental capture verified by `capture_mode = incremental`, then restored | Pass | Pass | Pass | Pass | +| Disk-only extraction retains explicitly persisted file and excludes tmpfs | Pass | Pass | Pass | Pass | + +The first fixture iteration captured a restored child's initial full baseline and therefore did not prove incremental dirty capture. The tightened fixture creates the child's baseline first, mutates it, checks the next memory manifest explicitly says `incremental`, and restores that generation. Only the tightened run is credited for the incremental invariant. The paused checks validate public paused state and resulting memory; they are not an instruction-entry-counter proof that no guest instruction executed during capture. + +The tightened Linux tmpfs run returned an empty control response during baseline capture from a forked child, followed by SQLite disk-I/O errors during cleanup. Its runtime log ends during capture preparation without an explicit panic; no matching kernel crash/OOM report was found. The shared `/tmp` is a 32 GiB tmpfs and was heavily occupied. This is a suspected host-storage/resource interaction, **not an established root cause** and not a fixed product defect. The failed fixture is retained at `/tmp/dirty-42ts79bx` on OVH; none of its runtime processes remained alive when checked. Disk-backed reruns are tracked separately below. + +Disk-backed tightened Linux reruns passed both layouts after freeing tmpfs quota and directing temporary files to disk-backed storage. The intermediate disk-backed attempt had also failed at creation with explicit `Disk quota exceeded (os error 122)` because helper temporary files still used `/tmp`; mount inspection confirmed `/tmp` has `usrquota`. Two completed, stopped earlier fixture homes were moved intact from `/tmp/dirty-aic065db` and `/tmp/dirty-1ob_e60_` to `retained-dirty-flat` and `retained-dirty-layered` under the remote stage, freeing about 5 GiB without deleting their artifacts. Final reports are `dirty-flat-disk-r2/report.json` and `dirty-layered-disk-r2/report.json` under that stage. The quota failure is established; attribution of the earlier empty-response incident specifically to it remains an inference, not a proved runtime fix. + +Other checks passed: 17 Linux guest-freezer tests (predicate/event races, EINTR, error paths, bounded fast polling/backoff, timeout and latch ownership); 4 read-pool tests (including WAL visibility and write refusal); 4 control-lookup tests; 25 backend/profile tests; 2 control-socket reply tests; 15 libkrun clock-device tests; Go native binding `cargo check`; CLI `--tree`, `pause --tree`, and `pause --help`; focused Rust formatting and `git diff --check`. The restricted socket test initially failed to bind sockets and passed when rerun with appropriate permissions. The Go check initially lacked network/prebuilt artifacts and passed with an isolated development bundle and dependency access. + +Windows ARM64 and Linux ARM64/KVM were not live-tested in this pass. The subsequent [Node ownership follow-up](node-lifecycle-concurrency-2026-09-10.md) qualifies same-object exec/filesystem/lifecycle concurrency on Mac and Linux x86-64. Other unqualified cases include runtime-share custom files/mappings, detailed injected block-queue/ENOSPC failures, and a full per-language SDK timing matrix. Do not present these reports as exhaustive cross-platform correctness qualification or completion of every research recommendation. + +## Artifacts and provenance + +Base Microsandbox commit: `18c8fb8693957ac6e8ded964e42cfbe63f8506ad`. Worktree: `/private/tmp/msb-cow-8.PIhgYp`, branch `appcypher/cow-memory-lifecycle`. Release source staging copied that commit plus the lifecycle implementation files only, excluding concurrent incremental-archive changes in the shared checkout. Test-only/formatting additions made after staging do not change the measured runtime behavior. Qualification preceded the separately authorized commit and push. + +- Local build, harnesses, raw benchmark JSON, and live reports: `/private/tmp/msb-resident-perf.qHpm0p`. +- Linux build, raw results, and logs: `/home/ubuntu/msb-resident-perf.zqr4vR` on OVH. +- macOS candidate binary SHA-256: `23d7d9db282f63cc9d1ee1b1b2e49bed0661814f3cce3e98cbde2f5e4bc2dd101`. +- macOS baseline binary SHA-256: `82eb66d6cfc6370f172c3af597913069d03cb197688ee129bdde125308a01126`. +- Linux candidate binary SHA-256: `f87b4063dd1adc64ce996d306e0d334c3816f7ff9ca3b7a9474dad4acce25372`. +- ARM64 guest agent SHA-256: `1425dc4b6974c10983db03e023c2b868c890fc197857ea45d6289a83df593aa8`. +- x86-64 guest agent SHA-256: `ae70a9d63c7df953340c8d6dc2c674ccad18e9f490c114977ba7972fe63bb6d0`. +- macOS firmware SHA-256: `ea0d458cdc12a0fa6dac8d192542ddc39717f816da41176582905e31a8bf868c`. +- Companion libkrun worktree: `/private/tmp/krun-registry-release.Ordgfp`, baseline `b20d31aba2fcf996512b0540bde0176dd7db7ad8`; only the clock-observation file is modified. + +Passed live fixtures stop their own VMs in `finally`; artifacts are retained for inspection rather than deleting unrelated test data. Initial harness failures (old system Python, prohibited inherited host ports, and incorrect layered-root CLI spelling) are retained as failed attempts and are not counted as product passes. diff --git a/scripts/smoke/reports/restore-performance-2026-09-08.md b/scripts/smoke/reports/restore-performance-2026-09-08.md new file mode 100644 index 000000000..882069279 --- /dev/null +++ b/scripts/smoke/reports/restore-performance-2026-09-08.md @@ -0,0 +1,82 @@ +# Restore optimization qualification — 2026-09-08 + +The optimization batch is implemented and live-tested on macOS/HVF ARM64. This report does not declare the entire #8 branch or every proposed performance optimization complete. Linux and Windows were not rerun in this pass. + +## Changes qualified + +- Preserve GNU sparse encoding for long checkpoint layer names, with the exact GNU long-name marker already accepted by the previous reader. Increase archive input and extraction buffers to 1 MiB while retaining bounded reads, transport hashing, sparse-map validation, and truncation rejection. +- Merge ordered incremental memory ranges with a consuming sweep rather than rescanning and sorting the entire evolving map for each update. Preserve zero ranges, object identity, slice offsets, and overlap/overflow rejection. +- Resolve flat snapshot image configuration without downloading or materializing unused OCI disks. Probe pinned/original cache keys before the existing digest scan. Keep digest pinning, registry authentication/TLS settings, `--pull never` refusal when metadata is absent, and full artifact requirements for layered roots. +- Make the stopped-sandbox startup check respect flat root disks. Live testing caught its unconditional VMDK requirement when the new metadata-only cache contained no VMDK; the fix was rebuilt and requalified. +- Avoid directory syncs for disposable child checkpoint staging, but keep durable installed-snapshot publication, persistent disk publication, and immutable RAM-cache synchronization. Avoid syncing diagnostic boot/activation records; guest activation/clock acknowledgement/thaw ordering is unchanged. +- Inspect only the bounded, identity-verified checkpoint root during builder planning, and remove the redundant pre-copy source validation. Child/runtime payload admission still runs before consumption. +- Serialize same-identity CoW cache misses with a process lock and recheck after acquiring it. Warm hits remain outside the build lock. Keep no-replacement inode publication for interoperability with earlier builders and pinned VMs. + +No snapshot schema, agent protocol, dependency pin, CLI flag, or public language-SDK option changed. + +## Environment and reproducibility + +Apple M5 Max, 36 GiB RAM, macOS 26.3, APFS, native ARM64/HVF. Guest: Alpine pinned at `sha256:e7a1a92a5bfeee40966aea60f0796b0e7917cc35591542701834f03a68fa3d18`, 256 MiB RAM, two vCPUs, flat 512 MiB root disk, 32 MiB random tmpfs payload, disk/RAM markers, and a shell workload. This is a development workstation, not an otherwise idle dedicated performance host. + +Source base: Microsandbox `15ab8838da58061df5dd9560cbd85d648bb72351` plus this optimization commit. Libkrun remains pinned to `51c1ed3b83dc826c02800fb297995538e4eac55d`. Matching guest agent: `/private/tmp/msb-stack8-agentd-arm`; firmware: `/private/tmp/msb-stack8-libkrunfw.5.dylib`. The final signed executable SHA-256 is `32530a3f4385a090fb385d8a5015429638a43b16d990353818c0ecba219c3d9f`. + +Build: + +```sh +MSB_AGENTD_PATH=/private/tmp/msb-stack8-agentd-arm cargo build --release --locked --no-default-features --features net,ssh,prebuilt -p microsandbox-cli +codesign --entitlements msb-entitlements.plist --force -s - /private/tmp/msb-resume-final.P83RaS/msb +``` + +The build executable was copied to the final test directory before signing. Final evidence and executable: `/private/tmp/msb-resume-final.P83RaS/`. `bench.py`, `qualify.py`, and `fanout.py` retain the exact test procedures; `results.json`, `summary.json`, `qualification.json`, `qualification-summary.json`, and `concurrent.json` contain individual times and exit codes. Tests use isolated homes, bounded subprocess timeouts, and cleanup in `finally`. Use fresh directories/names when repeating them. The earlier optimization run and corruption test are retained in `/private/tmp/msb-resume-opt.locaTL/`; the pre-optimization benchmark and actual older executable are in `/private/tmp/msb-resume-profile.osiuRj/`. + +## Performance + +These times measure process launch through successful CLI completion, not just RAM mapping, stop-the-world time, or application readiness. “First command” measures from that same initial launch until a subsequent guest command completes, including a second CLI process. RAM hash checks occur afterward. Warm CoW means the prepared backing exists; cache miss means that file was absent, not that the OS page cache was purged. No p95 claim is made from these sample counts. + +The strongest A/B comparison alternates the actual pre-optimization and final release binaries against the same installed snapshot and home, six samples per binary/mode: + +| Installed full restore | Before median | After median | Reduction | After range | +| --- | ---: | ---: | ---: | ---: | +| Standard RAM | 278.15 ms | 153.56 ms | 44.8% | 145.83–159.68 ms | +| Warm CoW | 229.64 ms | 115.18 ms | 49.8% | 112.22–116.87 ms | + +The complete final-binary matrix below compares with the previous measurement pass. Unlike the alternating A/B above, those passes used separate captures and ran at different times. Improvements describe the combined batch, not an isolated contribution from each optimization. + +| Path | n | Previous CLI median | Final CLI median | Final first-command median | +| --- | ---: | ---: | ---: | ---: | +| Resident resume, standard | 8 | 12.97 ms | 11.40 ms | 25.81 ms | +| Resident resume, CoW | 8 | 12.91 ms | 10.57 ms | 24.04 ms | +| Fresh boot, standard | 5 | 180.48 ms | 158.89 ms | 194.53 ms | +| Fresh boot, CoW configured | 5 | 179.43 ms | 166.89 ms | 219.02 ms | +| Stopped disk snapshot | 5 | 206.80 ms | 190.45 ms | 226.67 ms | +| Full snapshot, disk-only restore | 5 | 209.38 ms | 181.63 ms | 218.00 ms | +| Installed full, standard | 8 | 278.76 ms | 152.31 ms | 219.32 ms | +| Installed full, warm CoW | 8 | 228.47 ms | 111.66 ms | 176.29 ms | +| Installed full, CoW cache miss | 5 | 301.18 ms | 162.85 ms | 232.38 ms | +| Full archive, standard | 5 | 405.12 ms | 236.38 ms | 287.45 ms | +| Full archive, warm CoW | 5 | 347.50 ms | 200.94 ms | 256.26 ms | + +The final full archive was 54,767,005 bytes. A genuine incremental checkpoint produced by repeated capture while paused restored in median 165.20 ms with standard RAM and 120.20 ms with CoW (five samples each). That paused incremental case has no intentionally dirtied RAM between captures; it verifies the incremental path but does not measure heavily fragmented dirty-memory merge throughput. A separate post-workload capture fell back to full and is correctly recorded as `delta-*` with `capture_mode=full`, not presented as incremental performance. + +Single observations, not distributions: full capture 493.35 ms; subsequent full-fallback capture 466.94 ms; direct full archive capture 442.86 ms; stopped disk capture 37.38 ms. In the qualification run, a full baseline capture took 458.52 ms and the following genuinely incremental paused capture took 247.57 ms. These are caller-observed operation durations, not vCPU pause durations. + +Three simultaneous cold-CoW archive restores completed in 278.70–286.37 ms each and published one prepared RAM file. This is one fanout experiment, not a concurrency percentile. An empty image-cache archive restore took 3,138.93 ms including registry network access; only one metadata JSON file was cached, with no OCI disk artifacts. A subsequent offline `--pull never` CoW restore succeeded. Avoid interpreting network-bound cold metadata fetch as warm restore latency. + +## Correctness and compatibility evidence + +- 92 focused tests passed: 27 runtime checkpoint tests, 28 registry tests, 31 SDK snapshot tests, five checkpoint resolver tests, and the new flat-restart regression test. +- The memory merge was compared against an independent per-byte oracle over 1,000 generated fragmented cases, including holes, zero updates, nonzero object offsets, unsorted input, and updates spanning multiple old extents. Malformed overlap, empty, and overflowing ranges are rejected. +- Sparse long-name output is checked with a separate tar parser and encoded-size assertion, then loaded and restored. The actual pre-optimization #8 release binary successfully restored the new plain-tar and zstd archives without reader changes. The final binary restored a real pre-optimization archive and verified its tmpfs payload hash. macOS system tar listed both new archive encodings successfully. This is not qualification against every historical release. +- The live matrix checked full-state RAM SHA-256, retained workload markers, cold disk-only semantics, standard/CoW modes, installed/direct-archive inputs, prepared-cache hits/misses, resident resume, unchanged boot ID across resident resume, and cleanup. +- Missing metadata with `--pull never` refused cleanly; metadata-only network fetch succeeded; reuse with `--pull never` succeeded. Unit tests retain layered artifact requirements and reject moved-tag digest mismatches. +- Two children retained independent RAM/disk contents after deleting their input archive. Stopping/restarting the modified child preserved its private disk marker without OCI layers or VMDK. An isolated snapshot copy with corrupted execution-state object bytes was refused before successful VM creation. +- Three concurrent restores shared a cold cache entry, retained their markers, and isolated private RAM writes. A truncated zstd archive was refused with `zstd stream did not finish`. +- A process inventory after the runs found no runtimes belonging to either optimization test directory. Two unrelated pre-existing development VMs were left untouched. Test artifacts were retained for inspection. + +The first live cold-cache attempt used an overlong Unix socket path and was rerun with a shorter home. The later restart check exposed and fixed the real unconditional-VMDK bug described above. Both are retained in the earlier run's evidence; neither is counted as a passing initial attempt. + +`cargo fmt --all -- --check` and `git diff --check` passed. Strict Clippy was blocked by the existing `derivable_impls` warning in `crates/image/lib/snapshot/manifest.rs`; allowing that lint exposed the existing `too_many_arguments` warning on `RuntimeControlExecutor::new`. No unrelated lint cleanup was included. This pass did not rerun every workspace or language-SDK test. + +## Remaining performance work + +Warm full CoW branching is roughly twice as fast, but it is not consistently below 90 ms. The broader proposals for startup readiness notification, pipelined eager object reads, restore-only zero-preserving RAM construction, and avoiding immutable qcow2 ancestor relocation copies are not implemented by this batch. The remaining first-command latency also deserves separate measurement; CLI completion is not a substitute for measuring a ready application or a persistent SDK connection. Large-memory, deep-chain, heavily fragmented dirty-memory throughput, cache-builder process-death, and cross-platform performance qualification remain outside this pass. diff --git a/scripts/smoke/reports/snapshot-groups-2026-09-10.md b/scripts/smoke/reports/snapshot-groups-2026-09-10.md new file mode 100644 index 000000000..7b3d2c062 --- /dev/null +++ b/scripts/smoke/reports/snapshot-groups-2026-09-10.md @@ -0,0 +1,110 @@ +# Snapshot groups qualification — 2026-09-10 + +## Result + +Snapshot groups, scoped selectors, ancestry-aware head advancement, explicit selection, archive import, and CLI/Rust/Python/TypeScript/Go surfaces were initially implemented and qualified in an isolated detached worktree based on `e20269e1b260149e4cf629f43fac142a35e1458b`. The initial qualification below did not edit the concurrent #8 checkout. Subsequent integration checks are recorded separately so these original measurements retain their context. + +The final macOS ARM64/HVF live matrix passed with flat, managed/layered, and tmpfs roots. These runs verify group integration with existing execution state and archive workflows; they are not a fresh Linux or Windows qualification of the stack. + +## Live coverage + +| Check | Flat 512 MiB | Managed 512 MiB | Tmpfs 128 MiB | +| --- | --- | --- | --- | +| Capture, stable ID directory, alias, recorded parent | Pass | Pass | Pass, full only | +| Automatic fast-forward and explicit head selection | Pass | Pass | Pass | +| Restore exact old member versus selected group head | Pass | Pass | Pass | +| Concurrent sibling captures retain both members; one wins head | Pass | Pass | Pass | +| Full capture and capture from user-paused source | Pass | Pass | Pass | +| Direct branch child records source's last captured ancestor | Pass | Pass | Pass | +| Duplicate name rejects without changing existing member/head/cursor | Pass | Pass | Pass | +| Base + `--since` export/load, preserved aliases and IDs | Pass | Pass | Pass, self-contained export | +| Missing required base rejected before group creation | Pass | Pass | No dependency omitted in this workload | +| Old/idempotent import retains head; `--set-head` explicitly rewinds | Pass | Pass | Pass | +| Same archive in another group; ambiguous global ID refused | Pass | Pass | Pass | +| Head deletion protection, even with `--force` | Pass | Pass | Pass | +| Eager and forked full restore reproduce disk/RAM markers | Pass | Pass | Pass | +| Direct full archive capture and restore install no snapshot member | Pass | Pass | Pass | +| Stopped-source disk capture and restore | Pass | Pass | Correctly refused | + +The final matrices ran 66, 66, and 64 CLI commands respectively, including expected negative cases and explicit stops. Test-owned VMs are stopped in `finally`, including failed creates. The tmpfs source's `--since` archive had no reusable object omissions, so its inventory correctly declared `boot-complete`; the fixture checks whether a base is required from the actual inventory rather than assuming every `--since` request produces a dependent archive. RAM dependency reconstruction is independently covered by the 12-checkpoint RAM-only and disk-plus-RAM artifact tests. + +## Timing context + +These are individual end-to-end **debug-build** CLI wall times, not pause duration, release performance, or a regression comparison. The flat/managed matrices overlapped, while the final tmpfs retry ran separately. Sources had two vCPUs and 256 MiB of RAM with small disk/RAM marker files. They are intentionally small correctness fixtures, not representative large working sets. + +| Operation | Flat | Managed | Tmpfs | +| --- | ---: | ---: | ---: | +| Initial group head read | 7.98 ms | 7.73 ms | 10.15 ms | +| Second disk capture | 123.37 ms | 128.96 ms | Not applicable | +| Full capture (`full1`) | 1356.53 ms | 1099.99 ms | 791.07 ms | +| Capture while paused | 1260.54 ms | 1162.57 ms | 770.73 ms | +| Installed eager full restore | 936.21 ms | 725.02 ms | 522.50 ms | +| Installed forked full restore | 920.67 ms | 733.85 ms | 546.03 ms | +| Direct local branch | 922.42 ms | 865.51 ms | 607.58 ms | +| Direct full archive capture | 2279.01 ms | 1863.23 ms | 1493.84 ms | +| Direct forked archive restore | 2017.03 ms | 1650.82 ms | 1257.52 ms | + +Raw results are `/private/tmp/sgf2/results.json`, `/private/tmp/sgm2/results.json`, and `/private/tmp/sgt2/results.json`, with per-command stdout/stderr alongside them. Initial retries documented fixture issues: an omitted firmware override, selecting an older installed firmware without VM Generation ID readiness, a too-long macOS socket path, and the tmpfs dependency assumption above. No failed run is counted as a passing matrix. + +## Automated checks + +- Snapshot library tests: 61 passed, including 15 group tests, five ancestry tests, duplicate IDs, generated-name retry without recapture, cancellation-safe cursor locking, missing history, source replacement, scoped deletion, and RAM-aware archive reconstruction. +- Artifact integration tests: 45 passed, including released-flat import normalization, repeated imports, alias-preserving re-export, and explicit legacy paths. +- Migration tests: 27 passed; downgrade preflight and rollback tests passed separately. +- Same-name/different-backend full-capture routing and post-publication resume-failure handling: passed with local socket permissions. +- CLI snapshot tests: 10 passed. TypeScript snapshot/native contract tests: 11 passed. Go unit tests passed; Go integration tests compiled. Native Rust bindings checked for Python, TypeScript, and Go; TypeScript declarations regenerated and typecheck passed. +- `cargo fmt --all -- --check` and `git diff --check`: passed. +- Strict broad Clippy encountered pre-existing `derivable_impls` in `crates/image/lib/snapshot/manifest.rs`; targeted `--no-deps -D warnings` encountered pre-existing `too_many_arguments` in `build_artifact`. Targeted Clippy passed with only that existing lint allowed via command-line `-A clippy::too_many_arguments`. No unrelated source was changed to silence lints. +- Full Python/Go VM suites, Linux, Windows, workspace-wide tests, and a release-build performance comparison were not rerun for this change. Python stub Ruff retains two unrelated pre-existing line-length findings. + +## Reproduce + +Use a short, fresh output path on macOS: + +```bash +MSB_PATH=/path/to/codesigned/msb \ +MSB_LIBKRUNFW_PATH=/path/to/matching/libkrunfw.5.dylib \ +GROUP_TEST_OUT=/tmp/sg-test \ +GROUP_TEST_LAYOUT=flat:512M \ +python3 scripts/smoke/cli/snapshot-groups.py +``` + +Repeat with `512M` and `tmpfs:128M`. The script owns an isolated `MSB_HOME` under its output directory and never stops unrelated sandboxes. + +The qualified binary SHA-256 was `6b71b08c9d89f87c59a60b5a218903ec048d6ab47066846d9d83c7d674917fa7`; firmware SHA-256 was `ea0d458cdc12a0fa6dac8d192542ddc39717f816da41176582905e31a8bf868c`; embedded agentd SHA-256 was `1425dc4b6974c10983db03e023c2b868c890fc197857ea45d6289a83df593aa8`. This uses the existing #8 guest artifacts; no kernel or agent implementation changes are part of the group feature. + +## Publication and recovery boundaries + +Group/head and per-source ancestry locks are process-held filesystem locks, not a daemon or a distributed authority. Head/ID lookup opens only the selected member; aliases scan local member-name metadata. An interrupted operation may have published a complete member, so callers should inspect before retrying. A process crash between member/head publication and cursor persistence may leave conservative ancestry that needs explicit head selection; it cannot silently overwrite a sibling. Group identity/name conflicts fail before moving staged members. Missing ancestry is distinct from missing payload data. Explicit removal racing an already resolved open may fail that operation; it never silently changes the chosen snapshot or cold-boots instead. + +Friendly names and group heads are local metadata, not new cryptographic identities. Descriptors retain their existing schema and portable IDs. Released flat artifacts remain readable by explicit path; there is no automatic relocation of old directories. + +## Integration onto updated #8 + +The group commit was replayed onto `86873fa68806914a8417cd0fa4e5f6eaa068105b`, retaining both newer commits: `8713ded2` (resident lifecycle performance) and `86873fa6` (Node ownership locking). No newer-main changes were imported. The newer dirty-memory smoke script was updated to use qualified member selectors and the installed path returned by capture. SDK reference examples were checked against the grouped APIs. + +Post-integration checks passed: 61 snapshot tests, 45 artifact tests, 10 CLI snapshot tests, four read-only control-lookup tests, the unindexed-group downgrade refusal test, and the same-name backend capture-routing test. All 27 migration tests also passed during integration preparation. Native bindings checked for Python, Node and Go; TypeScript typecheck and 16 focused unit tests passed; Go unit tests passed. Formatting and targeted Clippy passed with the previously documented `too_many_arguments` allowance. Native checks used the existing isolated SDK bootstrap home after the default prebuilt installer attempted an unavailable network download; no installed runtime was replaced. + +The complete macOS group matrices passed again: 66 commands each for flat and managed roots and 64 for tmpfs. Raw results are `/private/tmp/sgpf/results.json`, `/private/tmp/sgpm/results.json`, and `/private/tmp/sgpt/results.json`. The integrated, codesigned debug binary SHA-256 is `56f901eba49a3e748608cd2c987f9c7495959c4d6e43981bc0e5aa3c47efa067`, retained at `/private/tmp/sg-push-integrated-msb`; firmware and embedded agentd are unchanged from the initial qualification. + +| Integrated debug CLI operation | Flat | Managed | Tmpfs | +| --- | ---: | ---: | ---: | +| Initial group head read | 6.80 ms | 9.00 ms | 9.58 ms | +| Second disk capture | 114.73 ms | 137.68 ms | Not applicable | +| Full capture | 1275.77 ms | 966.86 ms | 814.14 ms | +| Capture while paused | 1260.71 ms | 993.24 ms | 777.49 ms | +| Installed eager restore | 949.10 ms | 697.05 ms | 504.40 ms | +| Installed forked restore | 947.55 ms | 718.54 ms | 534.55 ms | +| Direct branch | 843.45 ms | 777.08 ms | 626.83 ms | +| Direct full archive capture | 2527.27 ms | 1788.07 ms | 1502.21 ms | +| Direct forked archive restore | 2510.29 ms | 1571.97 ms | 1254.42 ms | + +These remain small debug-build correctness fixtures; some runs overlapped other builds or tests. They are not isolated performance comparisons or VM pause measurements. Linux, Windows, and release-performance qualification were not rerun during this push. + +### Existing dirty-memory qualification gap + +The additional `dirty-memory-checkpoint.py` smoke test did **not** pass. Its second full capture from a forked child reported `capture_mode: full` rather than the asserted `incremental`. Source/child RAM checks, direct branching, branch-of-branch, paused captures, and eager/forked restoration passed up to that assertion; the later grandchild and disk-only checks in that script were not reached. These failures do not invalidate the separate complete group matrices above, but they must not be described as a complete dirty-memory qualification. + +The original poll allocated and copied the entire 64 MiB disk-cache file merely to inspect eight bytes. The integration changes limit that read to eight bytes while retaining the dirty 64 MiB shared mapping and every assertion. This reduced observer overhead but did not resolve the full-capture result: both flat and managed reruns still failed. The runtime has an existing density fallback that chooses a full capture at 60% dirty RAM, but the failed runs do not log the decision reason, so that explanation remains unproven. + +Crucially, the exact pre-group #8 code at `86873fa6`, rebuilt separately with the same guest artifacts, reproduces the same assertion using its original script on a managed root. This establishes that the failure exists before the group change; no runtime workaround or relaxed assertion was added. Baseline binary SHA-256: `6d1eb6edaba2794d564c2b85f4c48598bb6a3d95b8726d861fccac97ac8a9c57` (`/private/tmp/sg-push-baseline-msb`). Reports: `/private/tmp/sg-dirty-push-flat/report.json`, `/private/tmp/sg-dirty-push-flat2/report.json`, `/private/tmp/sg-dirty-push-managed2/report.json`, and `/private/tmp/sg-dirty-baseline/report.json`. Every dirty-memory run reported an empty cleanup-error list. diff --git a/scripts/smoke/reports/snapshot-load-batch-2026-09-10.md b/scripts/smoke/reports/snapshot-load-batch-2026-09-10.md new file mode 100644 index 000000000..f7562e239 --- /dev/null +++ b/scripts/smoke/reports/snapshot-load-batch-2026-09-10.md @@ -0,0 +1,116 @@ +# Snapshot batch-load qualification — 2026-09-10 + +Archive names in the original qualification below use the current `.msb` convention. Retained raw logs preserve the filenames used at the time; post-rename qualification is recorded separately. + +## Result + +Batch loading passed on macOS ARM64/HVF with real full-checkpoint archives for both flat and managed root disks, including eager and forked restoration. Real complete disk-only archive batches also passed for both layouts. Fresh disk-only `--since` export is blocked by an existing producer limitation, described below; this is not counted as passing dependent File-archive live coverage. + +The final signed development binary was `/private/tmp/msb-batch-load-final`, SHA-256 `269bcba93e00b1e04b98a92386a301571dc0b59f2f2b0d90e52a2aa46e28541a`. Runtime tests used `/private/tmp/msb-forked-build.8HN0Ie/lib/libkrunfw.5.dylib` and `/private/tmp/msb-cow-8.PIhgYp/build/agentd`. Existing real fixtures were 256 MiB RAM, two vCPUs, and 512 MiB root disks. The full matrices ran in parallel in isolated homes. These are CLI wall times from qualification runs, not release-build performance benchmarks or stop-the-world measurements. + +## Live coverage + +| Check | Flat root | Managed root | +| --- | --- | --- | +| Three full archives, supplied in reverse and shuffled order | Pass | Pass | +| Actual shell-expanded `*.msb` batch | Pass | Pass | +| Returned paths retain input order while group head selects the unique tip | Pass | Pass | +| Automatic dependency resolution from the named destination group | Pass | Pass | +| `--dest` store root and automatic base lookup there | Pass | Pass | +| Missing disk/RAM closure and unrelated external base rejected before member publication | Pass | Pass | +| Complete external base supplies payloads without importing historical ancestors | Pass | Pass | +| Duplicate inputs install once | Pass | Pass | +| Ambiguous new group remains headless; explicit head selection works | Pass | Pass | +| Ambiguous existing group retains its head | Pass | Pass | +| Ambiguous `--set-head` rejected without published members | Pass | Pass | +| Full imported final member survives removal of inputs and its installed ancestors | Pass | Pass | +| Eager/forked restore retains `/disk-marker` and `/dev/shm/marker` | Pass | Pass | +| Complete disk-only archives load in reverse order and select the child head | Pass | Pass | +| Disk-only final member survives removal of inputs and its installed ancestor | Pass | Pass | +| Disk-only cold boot retains disk marker and does not restore tmpfs marker | Pass | Pass | +| Fresh disk-only dependent `--since` archive creation | Blocked: existing layer-ID issue | Blocked: existing layer-ID issue | + +Each final full matrix checked 39 command outcomes, including expected errors. Each complete disk-only matrix checked 12: 102 checked command outcomes in the four successful runs. All six VMs started by those final matrices were stopped, as were the two additional fresh-capture probe VMs. A final process inspection found no matching test harness or VM process remaining. + +## Observed timings + +| CLI operation | Flat root | Managed root | +| --- | ---: | ---: | +| Load three full archives, reverse order | 3,853.53 ms | 2,511.74 ms | +| Load three full archives, shuffled order | 3,604.32 ms | 3,234.55 ms | +| Load full archive shell wildcard | 3,621.72 ms | 2,752.73 ms | +| Load two full deltas using installed group base | 2,070.18 ms | 1,467.89 ms | +| Same automatic dependency lookup under `--dest` | 2,058.32 ms | 1,467.14 ms | +| Load final full delta with explicit complete external base | 1,074.19 ms | 751.28 ms | +| Restore imported final checkpoint, eager | 958.19 ms | 763.53 ms | +| Restore imported final checkpoint, forked | 970.45 ms | 774.82 ms | +| Load two complete disk-only archives, reverse order | 707.11 ms | 474.26 ms | +| Cold boot imported final disk-only snapshot | 422.90 ms | 329.61 ms | + +Restore times cover the `msb create` invocation. Guest marker checks were separate successful `msb exec` commands. No claim is made that these times isolate memory mapping, disk preparation, or guest readiness from the rest of the CLI pipeline. + +## Producer limitations found during qualification + +1. `--with-image` currently requires materialized layered image-cache artifacts, including fsmeta and VMDK, even for a flat snapshot. The old flat-only fixture lacked those files. The successful flat runs exported the original flat snapshot paths using the managed fixture's already populated image cache with the identical pinned image digest. The harness exposes this as `--image-home`; it does not modify either fixture. The first failed attempt is retained in `/private/tmp/sblf/report.json`. +2. Consecutive disk-only snapshots currently receive fresh IDs for every copied disk layer. `build_artifact` and `new_file_manifest` in `sdk/rust/lib/snapshot/create.rs` allocate random layer IDs for the whole source closure. The physical-prefix requirement therefore rejects `msb snapshot save work:child child.msb --since work:parent` even when parent and child were captured consecutively from the same running sandbox. This was reproduced on the final binary with new captures on both layouts, not only old fixtures. No producer implementation was changed in this work, and no archive metadata was fabricated to make the live test pass. The strict dependent-File test remains available and fails at export. Complete File batch import was qualified separately. Synthetic dependent-File import tests cover the loader independently of this existing exporter limitation. + +## Automated checks + +- Snapshot library: 75 tests passed, including a six-archive mixed-codec reverse-order chain, RAM-only and disk-plus-RAM dependency resolution, missing/corrupt RAM, borrowed File payload checks, and dependent File archives in both input orders. +- Snapshot artifact integration: 55 tests passed, covering legacy single load, ordering, duplicates, conflicting aliases/IDs/labels, branch/head behavior, corruption before publication, missing historical ancestors with complete payloads, and independent installed payload ownership. +- CLI snapshot tests: 12 passed. +- Rust native checks: Python, Node, and Go bindings passed. +- Node: native build, 14 focused tests, and type checking passed. +- Python: one stub-surface test and Ruff passed. +- Go: unit/native checks and integration test compilation passed; VM-backed Go integration execution was not run. +- Targeted Microsandbox/CLI Clippy passed with `--no-deps -D warnings -A clippy::too_many_arguments`; formatting and diff checks passed. + +The automated counts above include checks run by the coordinating agent and the SDK agent. Live qualification in this report was macOS-only; Linux and Windows were not rerun for this batch-load change. + +## Reproduction and retained evidence + +Use a fresh output directory for each invocation. The harness never restarts or deletes the retained source fixtures. It removes only its own exported input archives and selected imported ancestor snapshots to verify dependency independence. + +```bash +export MSB_LIBKRUNFW_PATH=/private/tmp/msb-forked-build.8HN0Ie/lib/libkrunfw.5.dylib +export MSB_AGENTD_PATH=/private/tmp/msb-cow-8.PIhgYp/build/agentd + +python3 scripts/smoke/cli/snapshot-load-batch.py \ + --binary /private/tmp/msb-batch-load-final \ + --fixtures /private/tmp/sgpf \ + --image-home /private/tmp/sgpm/home \ + --output /private/tmp/batch-flat-new-run --live + +# Repeat with --fixtures /private/tmp/sgpm and a different output directory. +# Complete File archive coverage: add --file-only --file-standalone. +# Reproduce the producer limitation: add --file-only --fresh-file instead. +``` + +Raw JSON reports contain every command result, full output, timing, archive inventories, and cleanup results: + +| Run | Report | +| --- | --- | +| Final full flat | `/private/tmp/sblf-final/report.json` | +| Final full managed | `/private/tmp/sblm-final/report.json` | +| Final complete File flat | `/private/tmp/sblf-file-standalone-r2/report.json` | +| Final complete File managed | `/private/tmp/sblm-file-standalone-r2/report.json` | +| Fresh File delta producer failure, flat | `/private/tmp/sblf-fresh-file/report.json` | +| Fresh File delta producer failure, managed | `/private/tmp/sblm-fresh-file/report.json` | + +An earlier standalone-File harness attempt incorrectly expected the archive completeness spelling `complete`; the assertion was corrected to the actual `boot-complete` enum before the successful final File runs. No product assertion or dependency check was relaxed. + +## `.msb` extension follow-up + +The source, CLI help, SDK examples, and live harness now use `.msb`. This is only a naming convention: explicit paths are preserved, compressed/plain-tar decoding remains content-based, and the archive and descriptor schemas are unchanged. The direct-archive unit matrix passed all eight compression/filename combinations (`.msb`, `.tar.zst`, `.tar`, and extensionless, each compressed and plain). Snapshot library tests (75), artifact integration tests (55), CLI tests (12), TypeScript tests (14) and type checking, the Python stub test, Go unit tests, formatting, and diff checks passed again. + +The signed follow-up binary is `/private/tmp/msb-extension-final`, SHA-256 `3326644fcc666d55bb72c41b851b1f7f0c28b5bb06ca7095914421a0d28b8e24`. The same fixtures and runtime artifacts were used. Both full matrices and both complete File matrices passed again with newly exported `.msb` archives: 102 checked command outcomes, including actual `*.msb` shell expansion, dependent full imports, eager/forked full restore, and disk-only cold boot. All six test VMs were stopped. Linux and Windows were not rerun for this extension-only change. + +| Follow-up operation | Flat root | Managed root | +| --- | ---: | ---: | +| Load three full archives, reverse order | 3,442.06 ms | 2,441.31 ms | +| Restore imported full checkpoint, eager | 939.44 ms | 764.74 ms | +| Restore imported full checkpoint, forked | 955.37 ms | 789.54 ms | +| Load two complete disk-only archives | 803.58 ms | 477.47 ms | +| Cold boot imported disk-only snapshot | 430.29 ms | 336.07 ms | + +These remain development-build qualification timings, not isolated performance comparisons. Reports: `/private/tmp/sxbf2/report.json` (full flat), `/private/tmp/sxbm2/report.json` (full managed), `/private/tmp/sxbfd2/report.json` (File flat), and `/private/tmp/sxbmd2/report.json` (File managed). Initial attempts under the tool sandbox passed archive checks but were denied VM endpoint creation (`Operation not permitted`); their failed reports remain under `/private/tmp/sxbf`, `/private/tmp/sxbm`, `/private/tmp/sxbfd`, and `/private/tmp/sxbmd`. The passing reruns used host permissions without changing product code or relaxing assertions. The disk-only dependent-export limitation above remains unchanged. diff --git a/sdk/go/cow_lifecycle_test.go b/sdk/go/cow_lifecycle_test.go new file mode 100644 index 000000000..4e56f2526 --- /dev/null +++ b/sdk/go/cow_lifecycle_test.go @@ -0,0 +1,119 @@ +//go:build cow_live && microsandbox_ffi_path + +package microsandbox + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// This exercises the public SDK against a matching development runtime/kernel bundle. +func TestCowResidentCapture(t *testing.T) { + if os.Getenv("MSB_COW_LIVE") != "1" { + t.Skip("requires matching live bundle") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + name := fmt.Sprintf("cow8-go-%d", os.Getpid()) + source, err := CreateSandbox(ctx, name, WithImage("alpine"), WithRootDisk(RootDisk.Managed(512)), WithMemory(256)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := source.Stop(context.Background()); err != nil { + t.Error(err) + } + }) + if _, err := source.Exec(ctx, "sh", []string{"-c", "echo source > /dev/shm/sdk-marker"}); err != nil { + t.Fatal(err) + } + if err := source.Pause(ctx); err != nil { + t.Fatal(err) + } + paused, err := GetSandbox(ctx, name) + if err != nil { + t.Fatal(err) + } + if paused.Status() != SandboxStatusPaused { + t.Fatalf("got status %s", paused.Status()) + } + branched, err := paused.Branch(ctx, name+"-paused-branch") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := branched.Stop(context.Background()); err != nil { + t.Error(err) + } + branched.Close() + }) + branchResult, err := branched.Exec(ctx, "cat", []string{"/dev/shm/sdk-marker"}) + if err != nil { + t.Fatal(err) + } + if strings.TrimSpace(branchResult.Stdout()) != "source" { + t.Fatal("branch lost captured RAM") + } + snapshot, err := Snapshot.Create(ctx, SnapshotCreateOptions{Name: name + "-full", FromSandbox: name, Full: true}) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(snapshot.Path(), "snapshot.json")); err != nil { + t.Fatal(err) + } + if err := paused.Resume(ctx); err != nil { + t.Fatal(err) + } + // The returned artifact path selects the exact member in its snapshot group. + child, err := CreateSandbox(ctx, name+"-child", WithFromSnapshot(snapshot.Path()), WithForked()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := child.Stop(context.Background()); err != nil { + t.Error(err) + } + }) + result, err := child.Exec(ctx, "cat", []string{"/dev/shm/sdk-marker"}) + if err != nil { + t.Fatal(err) + } + if strings.TrimSpace(result.Stdout()) != "source" { + t.Fatal("child lost captured memory") + } + if _, err := child.Exec(ctx, "sh", []string{"-c", "echo child > /dev/shm/sdk-marker"}); err != nil { + t.Fatal(err) + } + result, err = source.Exec(ctx, "cat", []string{"/dev/shm/sdk-marker"}) + if err != nil { + t.Fatal(err) + } + if strings.TrimSpace(result.Stdout()) != "source" { + t.Fatal("child changed source memory") + } + if err := child.Pause(ctx); err != nil { + t.Fatal(err) + } + descendant, err := child.Branch(ctx, name+"-branch") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := descendant.Stop(context.Background()); err != nil { + t.Error(err) + } + descendant.Close() + }) + branchResult, err = descendant.Exec(ctx, "cat", []string{"/dev/shm/sdk-marker"}) + if err != nil { + t.Fatal(err) + } + if strings.TrimSpace(branchResult.Stdout()) != "child" { + t.Fatal("branch lost private writes") + } +} diff --git a/sdk/go/integration/snapshot_test.go b/sdk/go/integration/snapshot_test.go index 6fb00400d..071689c48 100644 --- a/sdk/go/integration/snapshot_test.go +++ b/sdk/go/integration/snapshot_test.go @@ -23,11 +23,12 @@ func TestSandboxHandleSnapshotAndWithFromSnapshotFork(t *testing.T) { baseName := uniqueIntegrationName(t, "go-sdk-snapshot-base") forkName := uniqueIntegrationName(t, "go-sdk-snapshot-fork") snapshotName := uniqueIntegrationName(t, "go-sdk-snapshot") + snapshotSelector := baseName + ":" + snapshotName t.Cleanup(func() { removeSandboxBestEffort(forkName) removeSandboxBestEffort(baseName) - removeSnapshotBestEffort(snapshotName) + removeSnapshotBestEffort(snapshotSelector) }) phaseStart := time.Now() @@ -79,7 +80,7 @@ func TestSandboxHandleSnapshotAndWithFromSnapshotFork(t *testing.T) { t.Fatalf("Verify returned incomplete report: %+v", report) } - handle, err := microsandbox.Snapshot.Get(ctx, snapshotName) + handle, err := microsandbox.Snapshot.Get(ctx, snapshotSelector) if err != nil { t.Fatalf("Snapshot.Get: %v", err) } @@ -114,7 +115,7 @@ func TestSandboxHandleSnapshotAndWithFromSnapshotFork(t *testing.T) { } phaseStart = time.Now() - fork, err := createSandbox(t, ctx, forkName, microsandbox.WithFromSnapshot(snapshotName)) + fork, err := createSandbox(t, ctx, forkName, microsandbox.WithFromSnapshot(snapshotSelector)) if err != nil { t.Fatalf("CreateSandbox with WithFromSnapshot: %v", err) } @@ -182,9 +183,18 @@ func TestSnapshotCreateAndSnapshotDirectoryOps(t *testing.T) { } logSnapshotPhase(t, "create snapshot", phaseStart) snapshotDir := artifact.Path() - if filepath.Base(snapshotDir) != snapshotName { - t.Fatalf("Snapshot.Create path = %q, want basename %q", snapshotDir, snapshotName) + if filepath.Base(snapshotDir) != artifact.ID() { + t.Fatalf("Snapshot.Create path = %q, want stable ID basename %q", snapshotDir, artifact.ID()) + } + update := artifact.HeadUpdate() + if update == nil || update.Group != baseName || update.Head != artifact.ID() { + t.Fatalf("Snapshot.Create missing group head outcome: %#v", update) } + head, err := microsandbox.Snapshot.GroupHead(ctx, baseName) + if err != nil || head.Head != artifact.ID() { + t.Fatalf("Snapshot.GroupHead = %#v, err = %v", head, err) + } + t.Cleanup(func() { removeSnapshotBestEffort(snapshotDir) }) opened, err := microsandbox.Snapshot.Open(ctx, snapshotDir) if err != nil { @@ -219,7 +229,7 @@ func TestSnapshotCreateAndSnapshotDirectoryOps(t *testing.T) { archivePath := filepath.Join(t.TempDir(), "snapshot.tar") phaseStart = time.Now() - if err := microsandbox.Snapshot.Save(ctx, snapshotName, archivePath, + if err := microsandbox.Snapshot.Save(ctx, snapshotDir, archivePath, microsandbox.SnapshotSaveOptions{PlainTar: true}); err != nil { t.Fatalf("Snapshot.Save: %v", err) } @@ -243,6 +253,44 @@ func TestSnapshotCreateAndSnapshotDirectoryOps(t *testing.T) { if imported.Digest() != artifact.Digest() { t.Fatalf("Snapshot.Load digest = %q, want %q", imported.Digest(), artifact.Digest()) } + + // Independent imports share an identity and digest. Handle operations must stay + // bound to one installed copy instead of re-resolving an ambiguous digest. + duplicate, err := microsandbox.Snapshot.Load(loadCtx, archivePath, importDir) + if err != nil { + t.Fatalf("Snapshot.Load duplicate: %v", err) + } + t.Cleanup(func() { removeSnapshotBestEffort(duplicate.Path()) }) + if duplicate.Path() == imported.Path() || duplicate.ID() != imported.ID() { + t.Fatalf("imports should be distinct copies of one snapshot: %q / %q", imported.Path(), duplicate.Path()) + } + if err := imported.Remove(loadCtx, false); err != nil { + t.Fatalf("SnapshotHandle.Remove with duplicate copies: %v", err) + } + if _, err := imported.Open(loadCtx); err == nil { + t.Fatal("removed handle unexpectedly reopened a different copy") + } + if _, err := duplicate.Open(loadCtx); err != nil { + t.Fatalf("removing first import affected second copy: %v", err) + } + if _, err := microsandbox.Snapshot.Open(loadCtx, snapshotDir); err != nil { + t.Fatalf("removing an import affected the original snapshot: %v", err) + } + + // A repeated archive still yields one result per input, but one batch group + // installs the identical snapshot only once. + batch, err := microsandbox.Snapshot.LoadMany(loadCtx, + []string{archivePath, archivePath}, microsandbox.SnapshotLoadOptions{Dest: importDir}) + if err != nil { + t.Fatalf("Snapshot.LoadMany: %v", err) + } + if len(batch) != 2 || batch[0].ID() != artifact.ID() || batch[1].ID() != artifact.ID() { + t.Fatalf("Snapshot.LoadMany returned unexpected handles: %#v", batch) + } + t.Cleanup(func() { removeSnapshotBestEffort(batch[0].Path()) }) + if batch[0].Path() != batch[1].Path() || batch[0].Group() == nil { + t.Fatal("batch should reuse one installed copy in one generated group") + } } func logSnapshotPhase(t *testing.T, phase string, started time.Time) { diff --git a/sdk/go/internal/ffi/ffi.go b/sdk/go/internal/ffi/ffi.go index 7805fdd92..9636aed59 100644 --- a/sdk/go/internal/ffi/ffi.go +++ b/sdk/go/internal/ffi/ffi.go @@ -122,6 +122,11 @@ typedef char *(*msb_sandbox_close_fn)(uint64_t cancel_id, uint64_t handle, uint8 typedef char *(*msb_sandbox_detach_fn)(uint64_t cancel_id, uint64_t handle, uint8_t *buf, size_t buf_len); typedef char *(*msb_sandbox_stop_fn)(uint64_t cancel_id, uint64_t handle, uint64_t timeout_ms, uint8_t *buf, size_t buf_len); typedef char *(*msb_sandbox_request_stop_fn)(uint64_t cancel_id, uint64_t handle, uint8_t *buf, size_t buf_len); +typedef char *(*msb_sandbox_pause_fn)(uint64_t cancel_id, uint64_t handle, uint8_t *buf, size_t buf_len); +typedef char *(*msb_sandbox_branch_fn)(uint64_t cancel_id, uint64_t handle, const char *source, const char *child, uint8_t *buf, size_t buf_len); +typedef char *(*msb_sandbox_resume_fn)(uint64_t cancel_id, uint64_t handle, uint8_t *buf, size_t buf_len); +typedef char *(*msb_sandbox_handle_pause_fn)(uint64_t cancel_id, const char *name, uint8_t *buf, size_t buf_len); +typedef char *(*msb_sandbox_handle_resume_fn)(uint64_t cancel_id, const char *name, uint8_t *buf, size_t buf_len); typedef char *(*msb_sandbox_kill_fn)(uint64_t cancel_id, uint64_t handle, uint64_t timeout_ms, uint8_t *buf, size_t buf_len); typedef char *(*msb_sandbox_request_kill_fn)(uint64_t cancel_id, uint64_t handle, uint8_t *buf, size_t buf_len); typedef char *(*msb_sandbox_list_fn)(uint64_t cancel_id, const char *filter_json, uint8_t *buf, size_t buf_len); @@ -228,6 +233,9 @@ typedef char *(*msb_snapshot_reindex_fn)(uint64_t cancel_id, const char *dir, ui typedef char *(*msb_snapshot_export_fn)(uint64_t cancel_id, const char *name_or_path, const char *out, const char *opts_json, uint8_t *buf, size_t buf_len); typedef char *(*msb_snapshot_import_fn)(uint64_t cancel_id, const char *archive, const char *dest, uint8_t *buf, size_t buf_len); typedef char *(*msb_snapshot_import_with_base_fn)(uint64_t cancel_id, const char *archive, const char *dest, const char *base, uint8_t *buf, size_t buf_len); +typedef char *(*msb_snapshot_import_with_options_fn)(uint64_t cancel_id, const char *archive, const char *opts_json, uint8_t *buf, size_t buf_len); +typedef char *(*msb_snapshot_import_many_fn)(uint64_t cancel_id, const char *archives_json, const char *opts_json, uint8_t *buf, size_t buf_len); +typedef char *(*msb_snapshot_group_head_fn)(uint64_t cancel_id, const char *selector, uint8_t *buf, size_t buf_len); typedef char *(*msb_sandbox_compact_fn)(uint64_t cancel_id, uint64_t handle, const char *name, const char *opts, uint8_t *buf, size_t buf_len); typedef char *(*msb_fs_read_stream_fn)(uint64_t cancel_id, uint64_t handle, const char *path, uint8_t *buf, size_t buf_len); @@ -275,6 +283,11 @@ static msb_sandbox_close_fn ptr_msb_sandbox_close = NULL; static msb_sandbox_detach_fn ptr_msb_sandbox_detach = NULL; static msb_sandbox_stop_fn ptr_msb_sandbox_stop = NULL; static msb_sandbox_request_stop_fn ptr_msb_sandbox_request_stop = NULL; +static msb_sandbox_pause_fn ptr_msb_sandbox_pause = NULL; +static msb_sandbox_branch_fn ptr_msb_sandbox_branch = NULL; +static msb_sandbox_resume_fn ptr_msb_sandbox_resume = NULL; +static msb_sandbox_handle_pause_fn ptr_msb_sandbox_handle_pause = NULL; +static msb_sandbox_handle_resume_fn ptr_msb_sandbox_handle_resume = NULL; static msb_sandbox_kill_fn ptr_msb_sandbox_kill = NULL; static msb_sandbox_request_kill_fn ptr_msb_sandbox_request_kill = NULL; static msb_sandbox_list_fn ptr_msb_sandbox_list = NULL; @@ -388,6 +401,9 @@ static msb_snapshot_reindex_fn ptr_msb_snapshot_reindex = NULL; static msb_snapshot_export_fn ptr_msb_snapshot_export = NULL; static msb_snapshot_import_fn ptr_msb_snapshot_import = NULL; static msb_snapshot_import_with_base_fn ptr_msb_snapshot_import_with_base = NULL; +static msb_snapshot_import_with_options_fn ptr_msb_snapshot_import_with_options = NULL; +static msb_snapshot_import_many_fn ptr_msb_snapshot_import_many = NULL; +static msb_snapshot_group_head_fn ptr_msb_snapshot_group_head = NULL; static msb_sandbox_compact_fn ptr_msb_sandbox_compact = NULL; // dlopen handle — set once by load_microsandbox, never closed. @@ -454,6 +470,11 @@ const char *load_microsandbox(const char *path) { RESOLVE(msb_sandbox_detach); RESOLVE(msb_sandbox_stop); RESOLVE(msb_sandbox_request_stop); + RESOLVE(msb_sandbox_pause); + RESOLVE(msb_sandbox_branch); + RESOLVE(msb_sandbox_resume); + RESOLVE(msb_sandbox_handle_pause); + RESOLVE(msb_sandbox_handle_resume); RESOLVE(msb_sandbox_kill); RESOLVE(msb_sandbox_request_kill); RESOLVE(msb_sandbox_list); @@ -567,6 +588,9 @@ const char *load_microsandbox(const char *path) { RESOLVE(msb_snapshot_export); RESOLVE(msb_snapshot_import); RESOLVE(msb_snapshot_import_with_base); + RESOLVE(msb_snapshot_import_with_options); + RESOLVE(msb_snapshot_import_many); + RESOLVE(msb_snapshot_group_head); RESOLVE(msb_sandbox_compact); return NULL; } @@ -654,6 +678,21 @@ char *call_msb_sandbox_stop(uint64_t cancel_id, uint64_t handle, uint64_t timeou char *call_msb_sandbox_request_stop(uint64_t cancel_id, uint64_t handle, uint8_t *buf, size_t buf_len) { return ptr_msb_sandbox_request_stop ? ptr_msb_sandbox_request_stop(cancel_id, handle, buf, buf_len) : NULL; } +char *call_msb_sandbox_pause(uint64_t cancel_id, uint64_t handle, uint8_t *buf, size_t buf_len) { + return ptr_msb_sandbox_pause ? ptr_msb_sandbox_pause(cancel_id, handle, buf, buf_len) : NULL; +} +char *call_msb_sandbox_branch(uint64_t cancel_id, uint64_t handle, const char *source, const char *child, uint8_t *buf, size_t buf_len) { + return ptr_msb_sandbox_branch ? ptr_msb_sandbox_branch(cancel_id, handle, source, child, buf, buf_len) : NULL; +} +char *call_msb_sandbox_resume(uint64_t cancel_id, uint64_t handle, uint8_t *buf, size_t buf_len) { + return ptr_msb_sandbox_resume ? ptr_msb_sandbox_resume(cancel_id, handle, buf, buf_len) : NULL; +} +char *call_msb_sandbox_handle_pause(uint64_t cancel_id, const char *name, uint8_t *buf, size_t buf_len) { + return ptr_msb_sandbox_handle_pause ? ptr_msb_sandbox_handle_pause(cancel_id, name, buf, buf_len) : NULL; +} +char *call_msb_sandbox_handle_resume(uint64_t cancel_id, const char *name, uint8_t *buf, size_t buf_len) { + return ptr_msb_sandbox_handle_resume ? ptr_msb_sandbox_handle_resume(cancel_id, name, buf, buf_len) : NULL; +} char *call_msb_sandbox_kill(uint64_t cancel_id, uint64_t handle, uint64_t timeout_ms, uint8_t *buf, size_t buf_len) { return ptr_msb_sandbox_kill ? ptr_msb_sandbox_kill(cancel_id, handle, timeout_ms, buf, buf_len) : NULL; } @@ -994,6 +1033,15 @@ char *call_msb_snapshot_import(uint64_t cancel_id, const char *archive, const ch char *call_msb_snapshot_import_with_base(uint64_t cancel_id, const char *archive, const char *dest, const char *base, uint8_t *buf, size_t buf_len) { return ptr_msb_snapshot_import_with_base ? ptr_msb_snapshot_import_with_base(cancel_id, archive, dest, base, buf, buf_len) : NULL; } +char *call_msb_snapshot_import_with_options(uint64_t cancel_id, const char *archive, const char *opts_json, uint8_t *buf, size_t buf_len) { + return ptr_msb_snapshot_import_with_options ? ptr_msb_snapshot_import_with_options(cancel_id, archive, opts_json, buf, buf_len) : NULL; +} +char *call_msb_snapshot_import_many(uint64_t cancel_id, const char *archives_json, const char *opts_json, uint8_t *buf, size_t buf_len) { + return ptr_msb_snapshot_import_many ? ptr_msb_snapshot_import_many(cancel_id, archives_json, opts_json, buf, buf_len) : NULL; +} +char *call_msb_snapshot_group_head(uint64_t cancel_id, const char *selector, uint8_t *buf, size_t buf_len) { + return ptr_msb_snapshot_group_head ? ptr_msb_snapshot_group_head(cancel_id, selector, buf, buf_len) : NULL; +} char *call_msb_sandbox_compact(uint64_t cancel_id, uint64_t handle, const char *name, const char *opts, uint8_t *buf, size_t buf_len) { return ptr_msb_sandbox_compact ? ptr_msb_sandbox_compact(cancel_id, handle, name, opts, buf, buf_len) : NULL; } @@ -1604,6 +1652,7 @@ type CreateOptions struct { CPUPlacement string `json:"cpu_placement,omitempty"` PlacementProfile string `json:"placement_profile,omitempty"` THP string `json:"thp,omitempty"` + Forked bool `json:"forked,omitempty"` Workdir string `json:"workdir,omitempty"` Shell string `json:"shell,omitempty"` SecurityProfile string `json:"security_profile,omitempty"` @@ -2252,6 +2301,32 @@ func RequestStopSandboxByName(ctx context.Context, name string) error { return err } +// PauseSandboxByName controls resident execution without an agent connection. +func PauseSandboxByName(ctx context.Context, name string) error { + if err := ensureLoaded(); err != nil { + return err + } + cName := C.CString(name) + defer C.free(unsafe.Pointer(cName)) + _, err := call(ctx, func(cancelID C.uint64_t, buf *C.uint8_t, bufLen C.size_t) *C.char { + return C.call_msb_sandbox_handle_pause(cancelID, cName, buf, bufLen) + }) + return err +} + +// ResumeSandboxByName controls resident execution without an agent connection. +func ResumeSandboxByName(ctx context.Context, name string) error { + if err := ensureLoaded(); err != nil { + return err + } + cName := C.CString(name) + defer C.free(unsafe.Pointer(cName)) + _, err := call(ctx, func(cancelID C.uint64_t, buf *C.uint8_t, bufLen C.size_t) *C.char { + return C.call_msb_sandbox_handle_resume(cancelID, cName, buf, bufLen) + }) + return err +} + // KillSandboxByName terminates a sandbox identified by name and waits for stopped observation. func KillSandboxByName(ctx context.Context, name string, timeoutMs uint64) error { if err := ensureLoaded(); err != nil { @@ -2464,6 +2539,66 @@ func (s *Sandbox) RequestStop(ctx context.Context) error { return err } +// Branch creates an independent local child through the host runtime. +func (s *Sandbox) Branch(ctx context.Context, name string) (*Sandbox, error) { + return branchSandbox(ctx, uint64(s.h()), s.name, name) +} + +// BranchSandboxByName branches execution without an agent connection to the source. +func BranchSandboxByName(ctx context.Context, source, name string) (*Sandbox, error) { + return branchSandbox(ctx, 0, source, name) +} + +func branchSandbox(ctx context.Context, handle uint64, source, name string) (*Sandbox, error) { + if err := ensureLoaded(); err != nil { + return nil, err + } + cSource, cName := C.CString(source), C.CString(name) + defer C.free(unsafe.Pointer(cSource)) + defer C.free(unsafe.Pointer(cName)) + out, err := call(ctx, func(cancelID C.uint64_t, buf *C.uint8_t, bufLen C.size_t) *C.char { + return C.call_msb_sandbox_branch(cancelID, C.uint64_t(handle), cSource, cName, buf, bufLen) + }) + if err != nil { + return nil, err + } + var resp struct { + Handle uint64 `json:"handle"` + BackendKind string `json:"backend_kind"` + } + if err := json.Unmarshal([]byte(out), &resp); err != nil { + if h := salvageHandle(out); h != 0 { + releaseHandle(h) + } + return nil, fmt.Errorf("parse branch response: %w", err) + } + s := &Sandbox{name: name, backendKind: resp.BackendKind} + s.handle.Store(resp.Handle) + return s, nil +} + +// Pause controls resident execution through the host runtime. +func (s *Sandbox) Pause(ctx context.Context) error { + if err := ensureLoaded(); err != nil { + return err + } + _, err := call(ctx, func(cancelID C.uint64_t, buf *C.uint8_t, bufLen C.size_t) *C.char { + return C.call_msb_sandbox_pause(cancelID, s.h(), buf, bufLen) + }) + return err +} + +// Resume controls resident execution through the host runtime. +func (s *Sandbox) Resume(ctx context.Context) error { + if err := ensureLoaded(); err != nil { + return err + } + _, err := call(ctx, func(cancelID C.uint64_t, buf *C.uint8_t, bufLen C.size_t) *C.char { + return C.call_msb_sandbox_resume(cancelID, s.h(), buf, bufLen) + }) + return err +} + // Kill terminates the sandbox and waits for stopped observation. func (s *Sandbox) Kill(ctx context.Context, timeoutMs uint64) error { if err := ensureLoaded(); err != nil { @@ -4806,28 +4941,29 @@ func ImageSave(ctx context.Context, references []string, outputPath string, form // --------------------------------------------------------------------------- type SnapshotInfo struct { - ID string `json:"id"` - Path string `json:"path"` - Digest string `json:"digest"` - SizeBytes *uint64 `json:"size_bytes"` - ImageRef string `json:"image_ref"` - ImageManifestDigest string `json:"image_manifest_digest"` - Scope string `json:"scope"` - StateKind string `json:"state_kind"` - Format *string `json:"format"` - Fstype *string `json:"fstype"` - UpperFile *string `json:"upper_file"` - UpperIntegrityAlgorithm *string `json:"upper_integrity_algorithm"` - UpperIntegrityDigest *string `json:"upper_integrity_digest"` - UpperIntegrityRoot *string `json:"upper_integrity_root"` - UpperIntegrityLogicalSize *uint64 `json:"upper_integrity_logical_size"` - UpperIntegrityLeafSize *uint32 `json:"upper_integrity_leaf_size"` - CheckpointID *string `json:"checkpoint_id"` - CheckpointManifestDigest *string `json:"checkpoint_manifest_digest"` - Parent *string `json:"parent"` - CreatedAt string `json:"created_at"` - Labels map[string]string `json:"labels"` - SourceSandbox *string `json:"source_sandbox"` + HeadUpdate *SnapshotHeadUpdate `json:"head_update"` + ID string `json:"id"` + Path string `json:"path"` + Digest string `json:"digest"` + SizeBytes *uint64 `json:"size_bytes"` + ImageRef string `json:"image_ref"` + ImageManifestDigest string `json:"image_manifest_digest"` + Scope string `json:"scope"` + StateKind string `json:"state_kind"` + Format *string `json:"format"` + Fstype *string `json:"fstype"` + UpperFile *string `json:"upper_file"` + UpperIntegrityAlgorithm *string `json:"upper_integrity_algorithm"` + UpperIntegrityDigest *string `json:"upper_integrity_digest"` + UpperIntegrityRoot *string `json:"upper_integrity_root"` + UpperIntegrityLogicalSize *uint64 `json:"upper_integrity_logical_size"` + UpperIntegrityLeafSize *uint32 `json:"upper_integrity_leaf_size"` + CheckpointID *string `json:"checkpoint_id"` + CheckpointManifestDigest *string `json:"checkpoint_manifest_digest"` + Parent *string `json:"parent"` + CreatedAt string `json:"created_at"` + Labels map[string]string `json:"labels"` + SourceSandbox *string `json:"source_sandbox"` } type SnapshotArchiveInfo struct { @@ -4837,23 +4973,25 @@ type SnapshotArchiveInfo struct { } type SnapshotHandleInfo struct { - ID string `json:"id"` - Digest string `json:"digest"` - Name *string `json:"name"` - ParentDigest *string `json:"parent_digest"` - ImageRef string `json:"image_ref"` - Scope string `json:"scope"` - StateKind string `json:"state_kind"` - Format *string `json:"format"` - Fstype *string `json:"fstype"` - CheckpointManifestDigest *string `json:"checkpoint_manifest_digest"` - SizeBytes *uint64 `json:"size_bytes"` - Locality string `json:"locality"` - Availability string `json:"availability"` - MigrationState string `json:"migration_state"` - MigrationErrorCode *string `json:"migration_error_code"` - CreatedAtUnix int64 `json:"created_at_unix"` - Path string `json:"path"` + Group *string `json:"group"` + HeadUpdate *SnapshotHeadUpdate `json:"head_update"` + ID string `json:"id"` + Digest string `json:"digest"` + Name *string `json:"name"` + ParentDigest *string `json:"parent_digest"` + ImageRef string `json:"image_ref"` + Scope string `json:"scope"` + StateKind string `json:"state_kind"` + Format *string `json:"format"` + Fstype *string `json:"fstype"` + CheckpointManifestDigest *string `json:"checkpoint_manifest_digest"` + SizeBytes *uint64 `json:"size_bytes"` + Locality string `json:"locality"` + Availability string `json:"availability"` + MigrationState string `json:"migration_state"` + MigrationErrorCode *string `json:"migration_error_code"` + CreatedAtUnix int64 `json:"created_at_unix"` + Path string `json:"path"` } type SnapshotVerifyReport struct { @@ -4872,6 +5010,7 @@ type SnapshotVerifyReport struct { type SnapshotCreateOptions struct { Name string `json:"name,omitempty"` + Group string `json:"group,omitempty"` DestDir string `json:"dest_dir,omitempty"` Labels map[string]string `json:"labels,omitempty"` Force bool `json:"force,omitempty"` @@ -4887,6 +5026,21 @@ type SnapshotSaveOptions struct { PlainTar bool `json:"plain_tar,omitempty"` } +type SnapshotLoadOptions struct { + Dest string `json:"dest,omitempty"` + Base string `json:"base,omitempty"` + Group string `json:"group,omitempty"` + SetHead bool `json:"set_head,omitempty"` +} + +type SnapshotHeadUpdate struct { + Group string `json:"group"` + Previous *string `json:"previous"` + Head string `json:"head"` + Reason string `json:"reason"` + Changed bool `json:"changed"` +} + func SandboxHandleSnapshot(ctx context.Context, sandboxName, snapshotName string) (*SnapshotInfo, error) { if err := ensureLoaded(); err != nil { return nil, err @@ -5147,3 +5301,78 @@ func SnapshotLoadWithBase(ctx context.Context, archive, dest, base string) (*Sna } return &info, nil } + +func SnapshotLoadWithOptions(ctx context.Context, archive string, opts SnapshotLoadOptions) (*SnapshotHandleInfo, error) { + if err := ensureLoaded(); err != nil { + return nil, err + } + payload, err := json.Marshal(opts) + if err != nil { + return nil, err + } + cArchive, cOpts := C.CString(archive), C.CString(string(payload)) + defer C.free(unsafe.Pointer(cArchive)) + defer C.free(unsafe.Pointer(cOpts)) + out, err := call(ctx, func(cancelID C.uint64_t, buf *C.uint8_t, bufLen C.size_t) *C.char { + return C.call_msb_snapshot_import_with_options(cancelID, cArchive, cOpts, buf, bufLen) + }) + if err != nil { + return nil, err + } + var info SnapshotHandleInfo + if err := json.Unmarshal([]byte(out), &info); err != nil { + return nil, fmt.Errorf("parse snapshot load: %w", err) + } + return &info, nil +} + +func SnapshotLoadMany(ctx context.Context, archives []string, opts SnapshotLoadOptions) ([]*SnapshotHandleInfo, error) { + if err := ensureLoaded(); err != nil { + return nil, err + } + // A nil slice is an empty batch, not JSON null; the core validates empty batches. + if archives == nil { + archives = []string{} + } + archivePayload, err := json.Marshal(archives) + if err != nil { + return nil, err + } + optsPayload, err := json.Marshal(opts) + if err != nil { + return nil, err + } + cArchives, cOpts := C.CString(string(archivePayload)), C.CString(string(optsPayload)) + defer C.free(unsafe.Pointer(cArchives)) + defer C.free(unsafe.Pointer(cOpts)) + out, err := call(ctx, func(cancelID C.uint64_t, buf *C.uint8_t, bufLen C.size_t) *C.char { + return C.call_msb_snapshot_import_many(cancelID, cArchives, cOpts, buf, bufLen) + }) + if err != nil { + return nil, err + } + var infos []*SnapshotHandleInfo + if err := json.Unmarshal([]byte(out), &infos); err != nil { + return nil, fmt.Errorf("parse snapshot batch load: %w", err) + } + return infos, nil +} + +func SnapshotGroupHead(ctx context.Context, selector string) (*SnapshotHeadUpdate, error) { + if err := ensureLoaded(); err != nil { + return nil, err + } + cSelector := C.CString(selector) + defer C.free(unsafe.Pointer(cSelector)) + out, err := call(ctx, func(cancelID C.uint64_t, buf *C.uint8_t, bufLen C.size_t) *C.char { + return C.call_msb_snapshot_group_head(cancelID, cSelector, buf, bufLen) + }) + if err != nil { + return nil, err + } + var update SnapshotHeadUpdate + if err := json.Unmarshal([]byte(out), &update); err != nil { + return nil, fmt.Errorf("parse snapshot group head: %w", err) + } + return &update, nil +} diff --git a/sdk/go/native/microsandbox_go_ffi.h b/sdk/go/native/microsandbox_go_ffi.h index 42a9b60a5..49d05c6df 100644 --- a/sdk/go/native/microsandbox_go_ffi.h +++ b/sdk/go/native/microsandbox_go_ffi.h @@ -93,6 +93,16 @@ char *msb_sandbox_handle_stop(uint64_t cancel_id, unsigned char *buf, uintptr_t buf_len); +char *msb_sandbox_handle_pause(uint64_t cancel_id, + const char *name, + unsigned char *buf, + uintptr_t buf_len); + +char *msb_sandbox_handle_resume(uint64_t cancel_id, + const char *name, + unsigned char *buf, + uintptr_t buf_len); + char *msb_sandbox_handle_request_stop(uint64_t cancel_id, const char *name, unsigned char *buf, @@ -160,6 +170,20 @@ char *msb_sandbox_stop(uint64_t cancel_id, unsigned char *buf, uintptr_t buf_len); +char *msb_sandbox_pause(uint64_t cancel_id, Handle handle, unsigned char *buf, uintptr_t buf_len); + +/** + * Branch by live handle, or by persisted name when handle is zero. + */ +char *msb_sandbox_branch(uint64_t cancel_id, + Handle handle, + const char *source, + const char *child, + unsigned char *buf, + uintptr_t buf_len); + +char *msb_sandbox_resume(uint64_t cancel_id, Handle handle, unsigned char *buf, uintptr_t buf_len); + char *msb_sandbox_request_stop(uint64_t cancel_id, Handle handle, unsigned char *buf, @@ -803,6 +827,32 @@ char *msb_snapshot_import_with_base(uint64_t cancel_id, unsigned char *buf, uintptr_t buf_len); +/** + * Import an archive with group selection without changing the existing import ABI. + */ +char *msb_snapshot_import_with_options(uint64_t cancel_id, + const char *archive, + const char *opts_json, + unsigned char *buf, + uintptr_t buf_len); + +/** + * Import archives together with dependencies resolved within the batch and destination group. + */ +char *msb_snapshot_import_many(uint64_t cancel_id, + const char *archives_json, + const char *opts_json, + unsigned char *buf, + uintptr_t buf_len); + +/** + * Read a group head, or select a `group:member` as its head. + */ +char *msb_snapshot_group_head(uint64_t cancel_id, + const char *selector, + unsigned char *buf, + uintptr_t buf_len); + /** * Open a streaming read from a guest file. * Returns `{"stream_handle":}`. diff --git a/sdk/go/native/src/lib.rs b/sdk/go/native/src/lib.rs index 760c7cdbb..31c716a68 100644 --- a/sdk/go/native/src/lib.rs +++ b/sdk/go/native/src/lib.rs @@ -1063,6 +1063,7 @@ struct SandboxCreateOpts { cpu_placement: Option, placement_profile: Option, thp: Option, + forked: Option, workdir: Option, shell: Option, env: Option>, @@ -1185,6 +1186,7 @@ struct LogStreamOpts { #[derive(serde::Deserialize, Default)] struct SnapshotCreateOpts { name: Option, + group: Option, dest_dir: Option, #[serde(default)] labels: HashMap, @@ -1208,6 +1210,15 @@ struct SnapshotSaveOptsJson { plain_tar: bool, } +#[derive(serde::Deserialize, Default)] +struct SnapshotLoadOptsJson { + dest: Option, + base: Option, + group: Option, + #[serde(default)] + set_head: bool, +} + #[derive(serde::Deserialize, Default)] struct MountSpec { bind: Option, @@ -2290,6 +2301,9 @@ pub unsafe extern "C" fn msb_sandbox_create( .map_err(FfiError::invalid_argument)?; builder = builder.thp(policy); } + if opts.forked.unwrap_or(false) { + builder = builder.forked(); + } if let Some(w) = opts.workdir { builder = builder.workdir(w); } @@ -2887,6 +2901,44 @@ pub unsafe extern "C" fn msb_sandbox_handle_stop( }) } +#[unsafe(no_mangle)] +pub unsafe extern "C" fn msb_sandbox_handle_pause( + cancel_id: u64, + name: *const c_char, + buf: *mut c_uchar, + buf_len: usize, +) -> *mut c_char { + run_c(cancel_id, buf, buf_len, || { + let name = unsafe { cstr(name) }?; + Ok(Box::pin(async move { + let sb = Sandbox::get_for_control(&name) + .await + .map_err(FfiError::from)?; + sb.pause().await.map_err(FfiError::from)?; + Ok(r#"{"ok":true}"#.into()) + })) + }) +} + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn msb_sandbox_handle_resume( + cancel_id: u64, + name: *const c_char, + buf: *mut c_uchar, + buf_len: usize, +) -> *mut c_char { + run_c(cancel_id, buf, buf_len, || { + let name = unsafe { cstr(name) }?; + Ok(Box::pin(async move { + let sb = Sandbox::get_for_control(&name) + .await + .map_err(FfiError::from)?; + sb.resume().await.map_err(FfiError::from)?; + Ok(r#"{"ok":true}"#.into()) + })) + }) +} + #[unsafe(no_mangle)] pub unsafe extern "C" fn msb_sandbox_handle_request_stop( cancel_id: u64, @@ -3165,6 +3217,74 @@ pub unsafe extern "C" fn msb_sandbox_stop( }) } +#[unsafe(no_mangle)] +pub unsafe extern "C" fn msb_sandbox_pause( + cancel_id: u64, + handle: Handle, + buf: *mut c_uchar, + buf_len: usize, +) -> *mut c_char { + run_c(cancel_id, buf, buf_len, || { + let sb = get(handle)?; + Ok(Box::pin(async move { + sb.pause().await.map_err(FfiError::from)?; + Ok(r#"{"ok":true}"#.into()) + })) + }) +} + +/// Branch by live handle, or by persisted name when handle is zero. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn msb_sandbox_branch( + cancel_id: u64, + handle: Handle, + source: *const c_char, + child: *const c_char, + buf: *mut c_uchar, + buf_len: usize, +) -> *mut c_char { + run_c(cancel_id, buf, buf_len, || { + let child = unsafe { cstr(child) }?; + let source = unsafe { cstr(source) }?; + let live = if handle == 0 { + None + } else { + Some(get(handle)?) + }; + Ok(Box::pin(async move { + let sb = if let Some(live) = live { + live.branch(child).await.map_err(FfiError::from)? + } else { + Sandbox::get(&source) + .await + .map_err(FfiError::from)? + .branch(child) + .await + .map_err(FfiError::from)? + }; + let backend_kind = sb.backend_kind().as_str(); + let handle = register(sb)?; + Ok(serde_json::json!({ "handle": handle, "backend_kind": backend_kind }).to_string()) + })) + }) +} + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn msb_sandbox_resume( + cancel_id: u64, + handle: Handle, + buf: *mut c_uchar, + buf_len: usize, +) -> *mut c_char { + run_c(cancel_id, buf, buf_len, || { + let sb = get(handle)?; + Ok(Box::pin(async move { + sb.resume().await.map_err(FfiError::from)?; + Ok(r#"{"ok":true}"#.into()) + })) + }) +} + #[unsafe(no_mangle)] pub unsafe extern "C" fn msb_sandbox_request_stop( cancel_id: u64, @@ -6059,6 +6179,7 @@ fn snapshot_json(s: &Snapshot) -> serde_json::Value { }; serde_json::json!({ "path": s.path().display().to_string(), + "head_update": s.head_update(), "id": s.id().as_str(), "digest": s.digest(), "size_bytes": s.size_bytes(), @@ -6088,6 +6209,8 @@ fn snapshot_handle_json(h: µsandbox::SnapshotHandle) -> serde_json::Value { "id": h.id(), "digest": h.digest(), "name": h.name(), + "group": h.group(), + "head_update": h.head_update(), "parent_digest": h.parent_digest(), "image_ref": h.image_ref(), "scope": snapshot_scope_str(h.scope()), @@ -6127,10 +6250,10 @@ fn snapshot_builder_from_opts( source_sandbox: String, opts: SnapshotCreateOpts, ) -> Result { - let Some(name) = opts.name else { - return Err(FfiError::invalid_argument("snapshot create requires name")); - }; - let mut builder = Snapshot::builder(name).from_sandbox(source_sandbox); + let mut builder = Snapshot::builder(opts.name.unwrap_or_default()).from_sandbox(source_sandbox); + if let Some(group) = opts.group { + builder = builder.group(group); + } if let Some(dest_dir) = opts.dest_dir { builder = builder.dest_dir(PathBuf::from(dest_dir)); } @@ -6434,6 +6557,91 @@ pub unsafe extern "C" fn msb_snapshot_import_with_base( }) } +/// Import an archive with group selection without changing the existing import ABI. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn msb_snapshot_import_with_options( + cancel_id: u64, + archive: *const c_char, + opts_json: *const c_char, + buf: *mut c_uchar, + buf_len: usize, +) -> *mut c_char { + run_c(cancel_id, buf, buf_len, || { + let archive = PathBuf::from(unsafe { cstr(archive) }?); + let opts_raw = unsafe { cstr(opts_json) }?; + let opts: SnapshotLoadOptsJson = serde_json::from_str(&opts_raw) + .map_err(|error| FfiError::invalid_argument(error.to_string()))?; + Ok(Box::pin(async move { + let h = Snapshot::load_with_options( + &archive, + microsandbox::snapshot::LoadOpts { + dest: opts.dest, + base: opts.base, + group: opts.group, + set_head: opts.set_head, + }, + ) + .await + .map_err(FfiError::from)?; + Ok(snapshot_handle_json(&h).to_string()) + })) + }) +} + +/// Import archives together with dependencies resolved within the batch and destination group. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn msb_snapshot_import_many( + cancel_id: u64, + archives_json: *const c_char, + opts_json: *const c_char, + buf: *mut c_uchar, + buf_len: usize, +) -> *mut c_char { + run_c(cancel_id, buf, buf_len, || { + let archives_raw = unsafe { cstr(archives_json) }?; + let archives: Vec = serde_json::from_str(&archives_raw) + .map_err(|error| FfiError::invalid_argument(error.to_string()))?; + let opts_raw = unsafe { cstr(opts_json) }?; + let opts: SnapshotLoadOptsJson = serde_json::from_str(&opts_raw) + .map_err(|error| FfiError::invalid_argument(error.to_string()))?; + Ok(Box::pin(async move { + let handles = Snapshot::load_many( + &archives, + microsandbox::snapshot::LoadOpts { + dest: opts.dest, + base: opts.base, + group: opts.group, + set_head: opts.set_head, + }, + ) + .await + .map_err(FfiError::from)?; + let values = handles.iter().map(snapshot_handle_json).collect::>(); + Ok(serde_json::Value::Array(values).to_string()) + })) + }) +} + +/// Read a group head, or select a `group:member` as its head. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn msb_snapshot_group_head( + cancel_id: u64, + selector: *const c_char, + buf: *mut c_uchar, + buf_len: usize, +) -> *mut c_char { + run_c(cancel_id, buf, buf_len, || { + let selector = unsafe { cstr(selector) }?; + Ok(Box::pin(async move { + let update = Snapshot::group_head(&selector) + .await + .map_err(FfiError::from)?; + serde_json::to_string(&update) + .map_err(|error| FfiError::invalid_argument(error.to_string())) + })) + }) +} + // --------------------------------------------------------------------------- // Filesystem streaming — FsReadStream / FsWriteSink // --------------------------------------------------------------------------- diff --git a/sdk/go/options.go b/sdk/go/options.go index 683b1f991..d1274462e 100644 --- a/sdk/go/options.go +++ b/sdk/go/options.go @@ -36,6 +36,7 @@ type SandboxConfig struct { CPUPlacement CPUPlacement PlacementProfile string THP THPPolicy + Forked bool Workdir string Shell string SecurityProfile SecurityProfile @@ -479,6 +480,12 @@ const ( // THPPolicy selects the guest transparent huge-page policy at boot. type THPPolicy string +// WithForked restores a full snapshot with private copy-on-write memory. +// It cannot be combined with a fresh boot or disk-only restore. +func WithForked() SandboxOption { + return func(o *SandboxConfig) { o.Forked = true } +} + const ( // THPAlways transparently uses huge pages for eligible anonymous mappings. THPAlways THPPolicy = "always" diff --git a/sdk/go/options_test.go b/sdk/go/options_test.go index 0eded18a4..66519f79a 100644 --- a/sdk/go/options_test.go +++ b/sdk/go/options_test.go @@ -15,6 +15,21 @@ func TestWithImage(t *testing.T) { } } +func TestForkedRestoreOption(t *testing.T) { + var config SandboxConfig + WithForked()(&config) + if !config.Forked { + t.Fatal("forked option was lost") + } + // Restore policy is construction-only, not a property of stopped sandbox disks. + if err := json.Unmarshal([]byte(`{"resources":{"cpus":1,"memory_mib":128}}`), &config); err != nil { + t.Fatal(err) + } + if config.Forked { + t.Fatal("forked option leaked into persisted configuration") + } +} + func TestWithRootDiskManaged(t *testing.T) { o := SandboxConfig{} WithRootDisk(RootDisk.Managed(8192))(&o) diff --git a/sdk/go/sandbox.go b/sdk/go/sandbox.go index 6abd455de..f1701a653 100644 --- a/sdk/go/sandbox.go +++ b/sdk/go/sandbox.go @@ -101,6 +101,7 @@ func buildFFICreateOptions(o SandboxConfig) ffi.CreateOptions { CPUPlacement: string(o.CPUPlacement), PlacementProfile: o.PlacementProfile, THP: string(o.THP), + Forked: o.Forked, Workdir: o.Workdir, Shell: o.Shell, SecurityProfile: string(o.SecurityProfile), @@ -922,6 +923,25 @@ func (h *SandboxHandle) RequestStop(ctx context.Context) error { return wrapFFI(ffi.SandboxHandleVoidLifecycle(ctx, h.name, h.id, "request_stop", ffi.SandboxHandleLifecycleOptions{})) } +// Branch creates an independent local CoW child without publishing a durable full snapshot. +func (h *SandboxHandle) Branch(ctx context.Context, name string) (*Sandbox, error) { + inner, err := ffi.BranchSandboxByName(ctx, h.name, name) + if err != nil { + return nil, wrapFFI(err) + } + return &Sandbox{inner: inner}, nil +} + +// Pause controls resident execution without creating a snapshot. +func (h *SandboxHandle) Pause(ctx context.Context) error { + return wrapFFI(ffi.PauseSandboxByName(ctx, h.name)) +} + +// Resume controls resident execution without creating a snapshot. +func (h *SandboxHandle) Resume(ctx context.Context) error { + return wrapFFI(ffi.ResumeSandboxByName(ctx, h.name)) +} + // Kill force-kills the sandbox and waits until stopped state is observed. func (h *SandboxHandle) Kill(ctx context.Context, opts ...KillOption) error { return wrapFFI(ffi.SandboxHandleVoidLifecycle(ctx, h.name, h.id, "kill", ffi.SandboxHandleLifecycleOptions{TimeoutMs: killTimeoutMillis(opts)})) @@ -994,7 +1014,7 @@ func (h *SandboxHandle) Destroy(ctx context.Context, opts ...DestroyOption) erro })) } -// Snapshot captures this stopped sandbox under a bare name in the default +// Snapshot captures this sandbox's disk under a bare name in the default // snapshots directory. func (h *SandboxHandle) Snapshot(ctx context.Context, name string) (*SnapshotArtifact, error) { info, err := ffi.SandboxHandleSnapshot(ctx, h.name, name) @@ -1028,6 +1048,25 @@ func (s *Sandbox) RequestStop(ctx context.Context) error { return wrapFFI(s.inner.RequestStop(ctx)) } +// Pause controls resident execution without creating a snapshot. +func (s *Sandbox) Pause(ctx context.Context) error { + return wrapFFI(s.inner.Pause(ctx)) +} + +// Branch creates an independent local CoW child without publishing a durable full snapshot. +func (s *Sandbox) Branch(ctx context.Context, name string) (*Sandbox, error) { + inner, err := s.inner.Branch(ctx, name) + if err != nil { + return nil, wrapFFI(err) + } + return &Sandbox{inner: inner}, nil +} + +// Resume controls resident execution without creating a snapshot. +func (s *Sandbox) Resume(ctx context.Context) error { + return wrapFFI(s.inner.Resume(ctx)) +} + // Kill force-kills the sandbox and waits until stopped state is observed. func (s *Sandbox) Kill(ctx context.Context, opts ...KillOption) error { return wrapFFI(s.inner.Kill(ctx, killTimeoutMillis(opts))) diff --git a/sdk/go/snapshot.go b/sdk/go/snapshot.go index 89255dfe4..807b1f411 100644 --- a/sdk/go/snapshot.go +++ b/sdk/go/snapshot.go @@ -14,13 +14,14 @@ type snapshotFactory struct{} // SnapshotCreateOptions configures Snapshot.Create. type SnapshotCreateOptions struct { - // Snapshot name, resolved under the default snapshots directory - // (or under DestDir when set). + // Snapshot member name; generated when empty. Name string - // Source sandbox to snapshot. Must be stopped. Required. + // Group to install the member in; defaults to the source sandbox's name. + Group string + // Source sandbox to snapshot. Disk capture preserves running/paused state. Required. FromSandbox string // Parent directory to create the artifact in; empty = the default - // snapshots directory. The artifact lands at DestDir/. + // snapshots directory. The group is created under this root. DestDir string Labels map[string]string Force bool @@ -30,7 +31,7 @@ type SnapshotCreateOptions struct { // SnapshotSaveOptions configures Snapshot.Save. type SnapshotSaveOptions struct { - // Since identifies an exact base snapshot or standalone base archive. + // Since omits disk layers and RAM objects supplied by a base snapshot or standalone archive. Since string // LastLayers includes the newest N sealed disk layers. Mutually exclusive with Since. LastLayers *uint32 @@ -39,6 +40,27 @@ type SnapshotSaveOptions struct { PlainTar bool } +// SnapshotLoadOptions configures importing one or more archives into a snapshot group. +type SnapshotLoadOptions struct { + // Parent directory containing snapshot groups; empty selects the default. + Dest string + // External snapshot or standalone archive for dependencies absent from the batch/group. + Base string + // Destination group; generated when empty. + Group string + // Select the unique imported tip even when it is not a fast-forward. + SetHead bool +} + +// SnapshotHeadUpdate reports the result of reading or selecting a group head. +type SnapshotHeadUpdate struct { + Group string + Previous *string + Head string + Reason string + Changed bool +} + // SnapshotArchiveOptions configures direct sandbox-to-archive capture. type SnapshotArchiveOptions struct { SnapshotCreateOptions @@ -117,6 +139,7 @@ type SnapshotIntegrity struct { // SnapshotArtifact is a snapshot artifact on disk. type SnapshotArtifact struct { + headUpdate *SnapshotHeadUpdate id string path string digest string @@ -133,6 +156,7 @@ type SnapshotArtifact struct { func snapshotFromInfo(info *ffi.SnapshotInfo) *SnapshotArtifact { return &SnapshotArtifact{ + headUpdate: snapshotHeadUpdateFromInfo(info.HeadUpdate), id: info.ID, path: info.Path, digest: info.Digest, @@ -173,6 +197,11 @@ func (s *SnapshotArtifact) CreatedAt() string { return s.createdAt } func (s *SnapshotArtifact) Labels() map[string]string { return cloneMap(s.labels) } func (s *SnapshotArtifact) SourceSandbox() *string { return cloneStringPtr(s.sourceSandbox) } +// HeadUpdate returns the group head outcome recorded by this capture, if any. +func (s *SnapshotArtifact) HeadUpdate() *SnapshotHeadUpdate { + return cloneSnapshotHeadUpdate(s.headUpdate) +} + // Verify recomputes recorded content integrity for the snapshot. func (s *SnapshotArtifact) Verify(ctx context.Context) (*SnapshotVerifyReport, error) { report, err := ffi.SnapshotVerify(ctx, s.path) @@ -184,6 +213,8 @@ func (s *SnapshotArtifact) Verify(ctx context.Context) (*SnapshotVerifyReport, e // SnapshotHandle is a lightweight handle backed by the snapshot index. type SnapshotHandle struct { + group *string + headUpdate *SnapshotHeadUpdate id string digest string name *string @@ -205,6 +236,8 @@ type SnapshotHandle struct { func snapshotHandleFromInfo(info *ffi.SnapshotHandleInfo) *SnapshotHandle { return &SnapshotHandle{ + group: info.Group, + headUpdate: snapshotHeadUpdateFromInfo(info.HeadUpdate), id: info.ID, digest: info.Digest, name: info.Name, @@ -225,9 +258,17 @@ func snapshotHandleFromInfo(info *ffi.SnapshotHandleInfo) *SnapshotHandle { } } -func (h *SnapshotHandle) ID() string { return h.id } -func (h *SnapshotHandle) Digest() string { return h.digest } -func (h *SnapshotHandle) Name() *string { return cloneStringPtr(h.name) } +func (h *SnapshotHandle) ID() string { return h.id } +func (h *SnapshotHandle) Digest() string { return h.digest } +func (h *SnapshotHandle) Name() *string { return cloneStringPtr(h.name) } + +// Group returns the local group containing this indexed snapshot. +func (h *SnapshotHandle) Group() *string { return cloneStringPtr(h.group) } + +// HeadUpdate returns the group head outcome recorded by this import, if any. +func (h *SnapshotHandle) HeadUpdate() *SnapshotHeadUpdate { + return cloneSnapshotHeadUpdate(h.headUpdate) +} func (h *SnapshotHandle) ParentDigest() *string { return cloneStringPtr(h.parentDigest) } func (h *SnapshotHandle) Scope() string { return h.scope } func (h *SnapshotHandle) ImageRef() string { return h.imageRef } @@ -250,18 +291,17 @@ func (h *SnapshotHandle) Open(ctx context.Context) (*SnapshotArtifact, error) { } func (h *SnapshotHandle) Remove(ctx context.Context, force bool) error { - return Snapshot.Remove(ctx, h.digest, force) + // Copies in different groups share a digest; the handle owns one exact artifact path. + return Snapshot.Remove(ctx, h.path, force) } func (snapshotFactory) Create(ctx context.Context, opts SnapshotCreateOptions) (*SnapshotArtifact, error) { - if opts.Name == "" { - return nil, &Error{Kind: ErrInvalidConfig, Message: "snapshot create requires a non-empty Name"} - } if opts.FromSandbox == "" { return nil, &Error{Kind: ErrInvalidConfig, Message: "snapshot create requires a source sandbox (FromSandbox)"} } info, err := ffi.SnapshotCreate(ctx, opts.FromSandbox, ffi.SnapshotCreateOptions{ Name: opts.Name, + Group: opts.Group, DestDir: opts.DestDir, Labels: opts.Labels, Force: opts.Force, @@ -277,9 +317,6 @@ func (snapshotFactory) Create(ctx context.Context, opts SnapshotCreateOptions) ( // CreateArchive captures a disk or full snapshot directly into one archive file. // It does not create an installed snapshot directory or index row. func (snapshotFactory) CreateArchive(ctx context.Context, opts SnapshotArchiveOptions) (*SnapshotArchive, error) { - if opts.Name == "" { - return nil, &Error{Kind: ErrInvalidConfig, Message: "snapshot archive create requires a non-empty Name"} - } if opts.FromSandbox == "" { return nil, &Error{Kind: ErrInvalidConfig, Message: "snapshot archive create requires a source sandbox (FromSandbox)"} } @@ -290,6 +327,7 @@ func (snapshotFactory) CreateArchive(ctx context.Context, opts SnapshotArchiveOp create.DestDir = "" info, err := ffi.SnapshotCreateArchive(ctx, opts.FromSandbox, opts.ArchivePath, ffi.SnapshotCreateOptions{ Name: create.Name, + Group: create.Group, Labels: create.Labels, Force: create.Force, RecordIntegrity: create.RecordIntegrity, @@ -381,6 +419,69 @@ func (snapshotFactory) LoadWithBase(ctx context.Context, archive, dest, base str return snapshotHandleFromInfo(info), nil } +// LoadWithOptions imports an archive into a selected or generated group. +func (snapshotFactory) LoadWithOptions(ctx context.Context, archive string, opts SnapshotLoadOptions) (*SnapshotHandle, error) { + info, err := ffi.SnapshotLoadWithOptions(ctx, archive, ffi.SnapshotLoadOptions{ + Dest: opts.Dest, + Base: opts.Base, + Group: opts.Group, + SetHead: opts.SetHead, + }) + if err != nil { + return nil, wrapFFI(err) + } + return snapshotHandleFromInfo(info), nil +} + +// LoadMany imports archives together into one group, resolving dependencies regardless of input order. +func (snapshotFactory) LoadMany(ctx context.Context, archives []string, opts SnapshotLoadOptions) ([]*SnapshotHandle, error) { + infos, err := ffi.SnapshotLoadMany(ctx, archives, ffi.SnapshotLoadOptions{ + Dest: opts.Dest, + Base: opts.Base, + Group: opts.Group, + SetHead: opts.SetHead, + }) + if err != nil { + return nil, wrapFFI(err) + } + handles := make([]*SnapshotHandle, len(infos)) + for index, info := range infos { + handles[index] = snapshotHandleFromInfo(info) + } + return handles, nil +} + +// GroupHead reads a group head, or selects a group:member as its head. +func (snapshotFactory) GroupHead(ctx context.Context, selector string) (*SnapshotHeadUpdate, error) { + update, err := ffi.SnapshotGroupHead(ctx, selector) + if err != nil { + return nil, wrapFFI(err) + } + return snapshotHeadUpdateFromInfo(update), nil +} + +func snapshotHeadUpdateFromInfo(update *ffi.SnapshotHeadUpdate) *SnapshotHeadUpdate { + if update == nil { + return nil + } + return &SnapshotHeadUpdate{ + Group: update.Group, + Previous: update.Previous, + Head: update.Head, + Reason: update.Reason, + Changed: update.Changed, + } +} + +func cloneSnapshotHeadUpdate(update *SnapshotHeadUpdate) *SnapshotHeadUpdate { + if update == nil { + return nil + } + copy := *update + copy.Previous = cloneStringPtr(update.Previous) + return © +} + func normalizeSnapshotScope(scope string) string { if scope == "" { return SnapshotScopeDisk diff --git a/sdk/go/snapshot_test.go b/sdk/go/snapshot_test.go index 3a89b195c..88d67d554 100644 --- a/sdk/go/snapshot_test.go +++ b/sdk/go/snapshot_test.go @@ -9,16 +9,6 @@ import ( "github.com/superradcompany/microsandbox/sdk/go/internal/ffi" ) -func TestSnapshotCreateEmptyName(t *testing.T) { - _, err := Snapshot.Create(context.Background(), SnapshotCreateOptions{FromSandbox: "baseline"}) - if !IsKind(err, ErrInvalidConfig) { - t.Fatalf("err = %v, want ErrInvalidConfig", err) - } - if !strings.Contains(err.Error(), "Name") { - t.Fatalf("error should name the missing field: %q", err.Error()) - } -} - func TestSnapshotCreateEmptyFromSandbox(t *testing.T) { _, err := Snapshot.Create(context.Background(), SnapshotCreateOptions{Name: "after-pip-install"}) if !IsKind(err, ErrInvalidConfig) { @@ -68,6 +58,42 @@ func TestFFIWireShape_SnapshotCreateDestDir(t *testing.T) { } } +func TestFFIWireShape_SnapshotGroupWithGeneratedName(t *testing.T) { + got := marshalSnapshotCreateOptions(t, ffi.SnapshotCreateOptions{Group: "work"}) + if got["group"] != "work" { + t.Fatalf("group = %v, want work", got["group"]) + } + if _, present := got["name"]; present { + t.Fatal("generated names must be omitted for the Rust builder to assign") + } +} + +func TestFFIWireShape_SnapshotLoadGroupOptions(t *testing.T) { + payload, err := json.Marshal(ffi.SnapshotLoadOptions{ + Dest: "/snapshots", Base: "work:baseline", Group: "work", SetHead: true, + }) + if err != nil { + t.Fatal(err) + } + var got map[string]any + if err := json.Unmarshal(payload, &got); err != nil { + t.Fatal(err) + } + if got["dest"] != "/snapshots" || got["base"] != "work:baseline" || got["group"] != "work" || got["set_head"] != true { + t.Fatalf("unexpected load options: %s", payload) + } +} + +func TestFFIWireShape_SnapshotHeadUpdate(t *testing.T) { + var update ffi.SnapshotHeadUpdate + if err := json.Unmarshal([]byte(`{"group":"work","previous":null,"head":"baseline","reason":"initialized","changed":true}`), &update); err != nil { + t.Fatal(err) + } + if update.Group != "work" || update.Previous != nil || update.Head != "baseline" || update.Reason != "initialized" || !update.Changed { + t.Fatalf("unexpected head update: %#v", update) + } +} + func TestSnapshotStateProjectionDistinguishesMissingAndMerkleIntegrity(t *testing.T) { format := "raw" fstype := "ext4" diff --git a/sdk/node-ts/native/fs.rs b/sdk/node-ts/native/fs.rs index ec81e7193..997791f38 100644 --- a/sdk/node-ts/native/fs.rs +++ b/sdk/node-ts/native/fs.rs @@ -9,6 +9,7 @@ use napi_derive::napi; use tokio::sync::Mutex; use crate::error::to_napi_error; +use crate::shared_handle::SharedHandle; use crate::types::*; //-------------------------------------------------------------------------------------------------- @@ -18,7 +19,7 @@ use crate::types::*; /// Filesystem operations on a running sandbox (via agent protocol). #[napi(js_name = "SandboxFsOps")] pub struct JsSandboxFsOps { - sandbox: Arc>>, + sandbox: Arc>, } /// A streaming reader for file data from the sandbox. @@ -46,7 +47,7 @@ pub struct JsFsWriteSink { //-------------------------------------------------------------------------------------------------- impl JsSandboxFsOps { - pub fn new(sandbox: Arc>>) -> Self { + pub(crate) fn new(sandbox: Arc>) -> Self { Self { sandbox } } } @@ -56,8 +57,7 @@ impl JsSandboxFsOps { /// Read a file as a Buffer. #[napi] pub async fn read(&self, path: String) -> Result { - let guard = self.sandbox.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.sandbox.get().await.ok_or_else(consumed_error)?; let data = sb.fs().read(&path).await.map_err(to_napi_error)?; Ok(data.to_vec().into()) } @@ -65,8 +65,7 @@ impl JsSandboxFsOps { /// Read a file as a UTF-8 string. #[napi] pub async fn read_string(&self, path: String) -> Result { - let guard = self.sandbox.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.sandbox.get().await.ok_or_else(consumed_error)?; sb.fs().read_to_string(&path).await.map_err(to_napi_error) } @@ -74,16 +73,14 @@ impl JsSandboxFsOps { #[napi] pub async fn write(&self, path: String, data: Buffer) -> Result<()> { let bytes: Vec = data.to_vec(); - let guard = self.sandbox.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.sandbox.get().await.ok_or_else(consumed_error)?; sb.fs().write(&path, &bytes).await.map_err(to_napi_error) } /// List directory contents. #[napi] pub async fn list(&self, path: String) -> Result> { - let guard = self.sandbox.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.sandbox.get().await.ok_or_else(consumed_error)?; let entries = sb.fs().list(&path).await.map_err(to_napi_error)?; Ok(entries.iter().map(fs_entry_to_js).collect()) } @@ -91,48 +88,42 @@ impl JsSandboxFsOps { /// Create a directory. #[napi] pub async fn mkdir(&self, path: String) -> Result<()> { - let guard = self.sandbox.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.sandbox.get().await.ok_or_else(consumed_error)?; sb.fs().mkdir(&path).await.map_err(to_napi_error) } /// Remove a directory. #[napi] pub async fn remove_dir(&self, path: String) -> Result<()> { - let guard = self.sandbox.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.sandbox.get().await.ok_or_else(consumed_error)?; sb.fs().remove_dir(&path).await.map_err(to_napi_error) } /// Remove a file. #[napi] pub async fn remove(&self, path: String) -> Result<()> { - let guard = self.sandbox.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.sandbox.get().await.ok_or_else(consumed_error)?; sb.fs().remove(&path).await.map_err(to_napi_error) } /// Copy a file within the sandbox. #[napi] pub async fn copy(&self, from: String, to: String) -> Result<()> { - let guard = self.sandbox.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.sandbox.get().await.ok_or_else(consumed_error)?; sb.fs().copy(&from, &to).await.map_err(to_napi_error) } /// Rename a file within the sandbox. #[napi] pub async fn rename(&self, from: String, to: String) -> Result<()> { - let guard = self.sandbox.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.sandbox.get().await.ok_or_else(consumed_error)?; sb.fs().rename(&from, &to).await.map_err(to_napi_error) } /// Get file or directory metadata. #[napi] pub async fn stat(&self, path: String) -> Result { - let guard = self.sandbox.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.sandbox.get().await.ok_or_else(consumed_error)?; let meta = sb.fs().stat(&path).await.map_err(to_napi_error)?; Ok(fs_metadata_to_js(&meta)) } @@ -140,16 +131,14 @@ impl JsSandboxFsOps { /// Check if a path exists. #[napi] pub async fn exists(&self, path: String) -> Result { - let guard = self.sandbox.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.sandbox.get().await.ok_or_else(consumed_error)?; sb.fs().exists(&path).await.map_err(to_napi_error) } /// Copy a file from the host into the sandbox. #[napi] pub async fn copy_from_host(&self, host_path: String, guest_path: String) -> Result<()> { - let guard = self.sandbox.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.sandbox.get().await.ok_or_else(consumed_error)?; sb.fs() .copy_from_host(&host_path, &guest_path) .await @@ -159,8 +148,7 @@ impl JsSandboxFsOps { /// Copy a file from the sandbox to the host. #[napi] pub async fn copy_to_host(&self, guest_path: String, host_path: String) -> Result<()> { - let guard = self.sandbox.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.sandbox.get().await.ok_or_else(consumed_error)?; sb.fs() .copy_to_host(&guest_path, &host_path) .await @@ -170,8 +158,7 @@ impl JsSandboxFsOps { /// Read a file with streaming (~3 MiB chunks). #[napi(js_name = "readStream")] pub async fn read_stream(&self, path: String) -> Result { - let guard = self.sandbox.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.sandbox.get().await.ok_or_else(consumed_error)?; let stream = sb.fs().read_stream(&path).await.map_err(to_napi_error)?; Ok(JsFsReadStream { inner: Arc::new(Mutex::new(stream)), @@ -181,8 +168,7 @@ impl JsSandboxFsOps { /// Write a file with streaming. Returns a sink the caller writes to. #[napi(js_name = "writeStream")] pub async fn write_stream(&self, path: String) -> Result { - let guard = self.sandbox.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.sandbox.get().await.ok_or_else(consumed_error)?; let sink = sb.fs().write_stream(&path).await.map_err(to_napi_error)?; Ok(JsFsWriteSink { inner: Arc::new(Mutex::new(Some(sink))), diff --git a/sdk/node-ts/native/index.d.ts b/sdk/node-ts/native/index.d.ts index b6608cecb..67592a24d 100644 --- a/sdk/node-ts/native/index.d.ts +++ b/sdk/node-ts/native/index.d.ts @@ -998,6 +998,12 @@ export declare class Sandbox { attachShell(): Promise /** Stop the sandbox gracefully and wait for it to exit. */ stop(): Promise + /** Create an independent local CoW child without a durable full snapshot. */ + branch(name: string): Promise + /** Explicit resident pause through host control. */ + pause(): Promise + /** Explicit resident resume through host control. */ + resume(): Promise /** Stop and wait for exit, returning the exit status. */ stopAndWait(): Promise /** Request graceful shutdown without waiting for observed exit. */ @@ -1088,7 +1094,7 @@ export declare class SandboxBuilder { * snapshot already pins the image reference and digest. */ fromSnapshot(pathOrName: string): this - /** Supply the exact base for a disk-dependent snapshot archive. */ + /** Supply the base for omitted disk layers and RAM objects in a snapshot archive. */ snapshotBase(base: string): this /** Cold-boot only the disk state carried by a full snapshot. */ diskOnly(): this @@ -1106,6 +1112,8 @@ export declare class SandboxBuilder { maxMemory(mib: number): this /** Guest transparent huge-page policy selected at boot. */ thp(policy: 'always' | 'madvise' | 'never'): this + /** Restore a full snapshot with private copy-on-write memory. */ + forked(): this /** Override log verbosity: `"trace" | "debug" | "info" | "warn" | "error"`. */ logLevel(level: string): this /** Suppress sandbox logs. */ @@ -1389,6 +1397,12 @@ export declare class SandboxHandle { * override with `stopWithTimeout(timeoutMs)`. */ stop(): Promise + /** Create an independent local CoW child without a durable full snapshot. */ + branch(name: string): Promise + /** Explicit resident pause through host control. */ + pause(): Promise + /** Explicit resident resume through host control. */ + resume(): Promise /** Request graceful shutdown without waiting. */ requestStop(): Promise /** @@ -1532,7 +1546,14 @@ export declare class Snapshot { */ static save(nameOrPath: string, out: string, opts?: SaveOpts | undefined | null): Promise static load(archive: string, dest?: string | undefined | null, base?: string | undefined | null): Promise + static loadWithOptions(archive: string, opts?: LoadOpts | undefined | null): Promise + /** Import archives together, resolving dependencies within the batch and destination group. */ + static loadMany(archives: Array, opts?: LoadOpts | undefined | null): Promise> + /** Read a group's head, or select `group:member` as its head. */ + static groupHead(selector: string): Promise get path(): string + /** Outcome of the group head update performed by this capture. */ + get headUpdate(): HeadUpdate | null get id(): string get digest(): string get sizeBytes(): bigint | null @@ -1573,14 +1594,16 @@ export declare class SnapshotBuilder { constructor(name: string) /** * Create the artifact under this parent directory instead of the - * default snapshots store. The artifact lands at `destDir/`. + * default snapshots store. The snapshot group is created under this root. */ destDir(destDir: string): this + /** Install the snapshot in this group (defaults to the source sandbox's name). */ + group(group: string): this /** Set the source sandbox to snapshot. Required. */ fromSandbox(sourceSandbox: string): this /** Attach a key-value label. May be called multiple times. */ label(key: string, value: string): this - /** Overwrite an existing artifact at the destination. */ + /** Overwrite an archive destination; installed group members are immutable. */ force(): this /** Compute and record content integrity at create time. */ recordIntegrity(): this @@ -1605,6 +1628,8 @@ export type JsSnapshotBuilder = SnapshotBuilder /** Lightweight snapshot handle from the local index. */ export declare class SnapshotHandle { + get group(): string | null + get headUpdate(): HeadUpdate | null get id(): string get digest(): string get name(): string | null @@ -1868,6 +1893,15 @@ export interface FsMetadata { created?: number } +/** Outcome of reading or selecting a snapshot group's head. */ +export interface HeadUpdate { + group: string + previous?: string + head: string + reason: string + changed: boolean +} + /** OCI config fields extracted from the database. */ export interface ImageConfigDetail { digest: string @@ -1980,6 +2014,18 @@ export interface JsSandboxPage { nextCursor?: string } +/** Options for importing one or more archives into a snapshot group. */ +export interface LoadOpts { + /** Parent directory containing snapshot groups. */ + dest?: string + /** External snapshot or standalone archive for dependencies absent from the batch/group. */ + base?: string + /** Destination group (generated when omitted). */ + group?: string + /** Select the unique imported tip even when it is not a fast-forward. */ + setHead?: boolean +} + /** One captured log entry from `exec.log`. */ export interface LogEntry { /** Wall-clock timestamp when the chunk was captured (ms since epoch). */ @@ -2404,6 +2450,7 @@ export declare function setRuntimeMsbPath(path: string): void /** Built snapshot configuration produced by `SnapshotBuilder.build()`. */ export interface SnapshotConfig { name: string + group?: string sourceSandbox?: string destDir?: string labels: Array @@ -2417,6 +2464,8 @@ export interface SnapshotInfo { id: string digest: string name?: string + group?: string + headUpdate?: HeadUpdate parentDigest?: string imageRef: string /** `"disk"` for file state or `"full"` for a complete VM checkpoint. */ diff --git a/sdk/node-ts/native/lib.rs b/sdk/node-ts/native/lib.rs index eb90be718..b01985c4f 100644 --- a/sdk/node-ts/native/lib.rs +++ b/sdk/node-ts/native/lib.rs @@ -32,6 +32,7 @@ mod sandbox_builder; mod sandbox_handle; mod secret_builder; mod setup; +mod shared_handle; mod snapshot; mod snapshot_builder; mod ssh; diff --git a/sdk/node-ts/native/sandbox.rs b/sdk/node-ts/native/sandbox.rs index 67263d285..89649f323 100644 --- a/sdk/node-ts/native/sandbox.rs +++ b/sdk/node-ts/native/sandbox.rs @@ -15,6 +15,7 @@ use crate::fs::JsSandboxFsOps; use crate::sandbox_handle::{ JsSandboxHandle, SandboxDestroyOptions, SandboxRestartOptions, destroy_options, restart_options, }; +use crate::shared_handle::SharedHandle; use crate::ssh::{JsSshClient, JsSshServer, apply_client_options, apply_server_options}; use crate::types::*; @@ -28,7 +29,7 @@ use crate::types::*; /// to the guest VM and can execute commands, access the filesystem, and query metrics. #[napi] pub struct Sandbox { - inner: Arc>>, + inner: Arc>, backend_kind: &'static str, id: String, owns_lifecycle: bool, @@ -79,7 +80,7 @@ impl Sandbox { let id = inner.id().to_string(); let owns_lifecycle = inner.owns_lifecycle(); Sandbox { - inner: Arc::new(Mutex::new(Some(inner))), + inner: Arc::new(SharedHandle::new(inner)), backend_kind, id, owns_lifecycle, @@ -183,8 +184,7 @@ impl Sandbox { /// Sandbox name. Names are limited to 128 UTF-8 bytes. #[napi(getter)] pub async fn name(&self) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; Ok(sb.name().to_string()) } @@ -205,8 +205,7 @@ impl Sandbox { /// The TS layer parses + camelCase-remaps this into a plain object. #[napi(js_name = "configJson")] pub async fn config_json(&self) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; serde_json::to_string(sb.config()) .map_err(|e| napi::Error::from_reason(format!("failed to serialize config: {e}"))) } @@ -218,8 +217,7 @@ impl Sandbox { /// Execute the sandbox's effective OCI entrypoint and CMD. #[napi] pub async fn exec_default(&self) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let output = sb.exec_default().await.map_err(to_napi_error)?; Ok(ExecOutput::from_rust(output)) } @@ -231,8 +229,7 @@ impl Sandbox { builder: &mut JsExecOptionsBuilder, ) -> Result { let opts_builder = builder.take_inner_builder()?; - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let output = sb .exec_default_with(|_default| opts_builder) .await @@ -243,8 +240,7 @@ impl Sandbox { /// Execute the sandbox's effective OCI entrypoint and CMD with streaming I/O. #[napi] pub async fn exec_default_stream(&self) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let handle = sb.exec_default_stream().await.map_err(to_napi_error)?; Ok(JsExecHandle::from_rust(handle)) } @@ -256,8 +252,7 @@ impl Sandbox { builder: &mut JsExecOptionsBuilder, ) -> Result { let opts_builder = builder.take_inner_builder()?; - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let handle = sb .exec_default_stream_with(|_default| opts_builder) .await @@ -268,8 +263,7 @@ impl Sandbox { /// Execute a command and wait for completion. #[napi] pub async fn exec(&self, cmd: String, args: Option>) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let args_owned = args.unwrap_or_default(); let output = sb.exec(&cmd, args_owned).await.map_err(to_napi_error)?; Ok(ExecOutput::from_rust(output)) @@ -284,8 +278,7 @@ impl Sandbox { builder: &mut JsExecOptionsBuilder, ) -> Result { let opts_builder = builder.take_inner_builder()?; - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let output = sb .exec_with(&cmd, |_default| opts_builder) .await @@ -300,8 +293,7 @@ impl Sandbox { cmd: String, args: Option>, ) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let args_owned = args.unwrap_or_default(); let handle = sb .exec_stream(&cmd, args_owned) @@ -321,8 +313,7 @@ impl Sandbox { builder: &mut JsExecOptionsBuilder, ) -> Result { let opts_builder = builder.take_inner_builder()?; - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let handle = sb .exec_stream_with(&cmd, |_default| opts_builder) .await @@ -333,8 +324,7 @@ impl Sandbox { /// Execute a shell command using the sandbox's configured shell. #[napi] pub async fn shell(&self, script: String) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let output = sb.shell(&script).await.map_err(to_napi_error)?; Ok(ExecOutput::from_rust(output)) } @@ -342,8 +332,7 @@ impl Sandbox { /// Execute a shell command with streaming I/O. #[napi] pub async fn shell_stream(&self, script: String) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let handle = sb.shell_stream(&script).await.map_err(to_napi_error)?; Ok(JsExecHandle::from_rust(handle)) } @@ -365,8 +354,7 @@ impl Sandbox { /// Connect a native in-process SSH client to this sandbox. #[napi(js_name = "sshConnect")] pub async fn ssh_connect(&self, options: Option) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let client = sb .ssh() .connect_with(|builder| apply_client_options(options, builder)) @@ -378,8 +366,7 @@ impl Sandbox { /// Prepare a reusable SSH server endpoint for this sandbox. #[napi(js_name = "sshServer")] pub async fn ssh_server(&self, options: Option) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let server = sb .ssh() .server_with(|builder| apply_server_options(options, builder)) @@ -395,8 +382,7 @@ impl Sandbox { /// Get point-in-time resource metrics. #[napi] pub async fn metrics(&self) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let m = sb.metrics().await.map_err(to_napi_error)?; Ok(metrics_to_js(&m)) } @@ -408,8 +394,7 @@ impl Sandbox { /// Check whether agentd is reachable without refreshing idle activity. #[napi] pub async fn ping(&self) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let result = sb.ping().await.map_err(to_napi_error)?; Ok(sandbox_ping_result_to_js(result)) } @@ -417,8 +402,7 @@ impl Sandbox { /// Explicitly refresh this sandbox's idle activity timer. #[napi] pub async fn touch(&self) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let result = sb.touch().await.map_err(to_napi_error)?; Ok(sandbox_touch_result_to_js(result)) } @@ -428,8 +412,7 @@ impl Sandbox { #[napi] pub async fn modify(&self, options: Option) -> Result { let builder = { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; configure_modify(sb.modify(), options.as_ref())? }; run_modify(builder, modify_dry_run(options.as_ref())).await @@ -438,18 +421,15 @@ impl Sandbox { /// Compact the immutable disk prefix; the count includes the base, not the writable head. #[napi] pub async fn compact(&self, layers: Option, dry_run: Option) -> Result { - let builder = { - let guard = self.inner.lock().await; - guard.as_ref().ok_or_else(consumed_error)?.compact() - }; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; + let builder = sb.compact(); run_compact(builder, layers, dry_run.unwrap_or(false)).await } /// Stream metrics snapshots at the requested interval (in milliseconds). #[napi] pub async fn metrics_stream(&self, interval_ms: f64) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let interval = Duration::from_millis(interval_ms as u64); let mut stream = Box::pin(sb.metrics_stream(interval)); @@ -475,8 +455,7 @@ impl Sandbox { /// Attach to the sandbox's effective OCI entrypoint and CMD. #[napi] pub async fn attach_default(&self) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; sb.attach_default().await.map_err(to_napi_error) } @@ -487,8 +466,7 @@ impl Sandbox { builder: &mut JsAttachOptionsBuilder, ) -> Result { let opts_builder = builder.take_inner_builder()?; - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; sb.attach_default_with(|_default| opts_builder) .await .map_err(to_napi_error) @@ -499,8 +477,7 @@ impl Sandbox { /// Bridges the host terminal to the guest process. Returns the exit code. #[napi] pub async fn attach(&self, cmd: String, args: Option>) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let args_owned = args.unwrap_or_default(); sb.attach(&cmd, args_owned).await.map_err(to_napi_error) } @@ -514,8 +491,7 @@ impl Sandbox { builder: &mut JsAttachOptionsBuilder, ) -> Result { let opts_builder = builder.take_inner_builder()?; - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; sb.attach_with(&cmd, |_default| opts_builder) .await .map_err(to_napi_error) @@ -524,8 +500,7 @@ impl Sandbox { /// Attach to the sandbox's default shell. #[napi] pub async fn attach_shell(&self) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; sb.attach_shell().await.map_err(to_napi_error) } @@ -536,16 +511,37 @@ impl Sandbox { /// Stop the sandbox gracefully and wait for it to exit. #[napi] pub async fn stop(&self) -> Result<()> { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; sb.stop().await.map_err(to_napi_error) } + /// Create an independent local CoW child without a durable full snapshot. + #[napi] + pub async fn branch(&self, name: String) -> Result { + let sb = self.inner.get().await.ok_or_else(consumed_error)?; + Ok(Sandbox::from_rust( + sb.branch(name).await.map_err(to_napi_error)?, + )) + } + + /// Explicit resident pause through host control. + #[napi] + pub async fn pause(&self) -> Result<()> { + let sb = self.inner.get().await.ok_or_else(consumed_error)?; + sb.pause().await.map_err(to_napi_error) + } + + /// Explicit resident resume through host control. + #[napi] + pub async fn resume(&self) -> Result<()> { + let sb = self.inner.get().await.ok_or_else(consumed_error)?; + sb.resume().await.map_err(to_napi_error) + } + /// Stop and wait for exit, returning the exit status. #[napi] pub async fn stop_and_wait(&self) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let status = sb.stop_and_wait().await.map_err(to_napi_error)?; Ok(exit_status_to_js(status)) } @@ -553,16 +549,14 @@ impl Sandbox { /// Request graceful shutdown without waiting for observed exit. #[napi] pub async fn request_stop(&self) -> Result<()> { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; sb.request_stop().await.map_err(to_napi_error) } /// Stop gracefully with an explicit timeout before escalating to SIGKILL. #[napi] pub async fn stop_with_timeout(&self, timeout_ms: u32) -> Result<()> { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let timeout = Duration::from_millis(timeout_ms.into()); sb.stop_with_timeout(timeout).await.map_err(to_napi_error) } @@ -570,24 +564,21 @@ impl Sandbox { /// Kill the sandbox immediately and wait for observed exit. #[napi] pub async fn kill(&self) -> Result<()> { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; sb.kill().await.map_err(to_napi_error) } /// Request force termination without waiting for observed exit. #[napi] pub async fn request_kill(&self) -> Result<()> { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; sb.request_kill().await.map_err(to_napi_error) } /// Force-kill the sandbox with an explicit observation timeout. #[napi] pub async fn kill_with_timeout(&self, timeout_ms: u32) -> Result<()> { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let timeout = Duration::from_millis(timeout_ms.into()); sb.kill_with_timeout(timeout).await.map_err(to_napi_error) } @@ -595,24 +586,21 @@ impl Sandbox { /// Graceful drain (SIGUSR1 — for load balancing). #[napi] pub async fn drain(&self) -> Result<()> { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; sb.drain().await.map_err(to_napi_error) } /// Request graceful drain without waiting for observed exit. #[napi] pub async fn request_drain(&self) -> Result<()> { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; sb.request_drain().await.map_err(to_napi_error) } /// Wait until this exact sandbox reaches the requested status. #[napi(js_name = "waitForStatus")] pub async fn wait_for_status(&self, status: String) -> Result { - let guard = self.inner.lock().await; - let sandbox = guard.as_ref().ok_or_else(consumed_error)?; + let sandbox = self.inner.get().await.ok_or_else(consumed_error)?; let status = match status.as_str() { "created" => microsandbox::sandbox::SandboxStatus::Created, "starting" => microsandbox::sandbox::SandboxStatus::Starting, @@ -637,8 +625,7 @@ impl Sandbox { /// Stop and start this exact sandbox. #[napi] pub async fn restart(&self, options: Option) -> Result { - let guard = self.inner.lock().await; - let sandbox = guard.as_ref().ok_or_else(consumed_error)?; + let sandbox = self.inner.get().await.ok_or_else(consumed_error)?; let restarted = sandbox .restart_with(restart_options(options)) .await @@ -649,8 +636,7 @@ impl Sandbox { /// Stop and remove this exact sandbox. #[napi] pub async fn destroy(&self, options: Option) -> Result<()> { - let guard = self.inner.lock().await; - let sandbox = guard.as_ref().ok_or_else(consumed_error)?; + let sandbox = self.inner.get().await.ok_or_else(consumed_error)?; sandbox .destroy_with(destroy_options(options)) .await @@ -660,8 +646,7 @@ impl Sandbox { /// Wait until the sandbox is observed in a terminal non-running state. #[napi] pub async fn wait_until_stopped(&self) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let result = sb.wait_until_stopped().await.map_err(to_napi_error)?; Ok(sandbox_stop_result_to_js(result)) } @@ -669,27 +654,29 @@ impl Sandbox { /// Wait for the sandbox process to exit. #[napi(js_name = "wait")] pub async fn wait_for_exit(&self) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let status = sb.wait().await.map_err(to_napi_error)?; Ok(exit_status_to_js(status)) } /// Detach from the sandbox — it will continue running after this handle is dropped. + /// New operations are rejected; already admitted operations retain their connection. #[napi] pub async fn detach(&self) -> Result<()> { - let mut guard = self.inner.lock().await; - if let Some(sb) = guard.take() { - sb.detach().await; + if let Some(sb) = self.inner.take().await { + // Only detach consumes the Rust value. Ordinary operations clone the Arc, + // not the sandbox configuration; admitted operations keep their reference. + Arc::unwrap_or_clone(sb).detach().await; } Ok(()) } /// Remove the persisted database record after stopping. + /// Consumes this wrapper even on failure. Already admitted operations may finish or + /// fail at the runtime boundary; removal does not wait for guest operations to drain. #[napi] pub async fn remove_persisted(&self) -> Result<()> { - let mut guard = self.inner.lock().await; - let sb = guard.take().ok_or_else(consumed_error)?; + let sb = self.inner.take().await.ok_or_else(consumed_error)?; sb.remove_persisted().await.map_err(to_napi_error) } @@ -700,8 +687,7 @@ impl Sandbox { /// protocol traffic. #[napi] pub async fn logs(&self, opts: Option) -> Result> { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let rust_opts = log_options_from_js(opts).map_err(napi::Error::from_reason)?; let entries = sb.logs(&rust_opts).await.map_err(to_napi_error)?; Ok(entries.into_iter().map(log_entry_to_js).collect()) @@ -715,8 +701,7 @@ impl Sandbox { /// entry. #[napi] pub async fn log_stream(&self, opts: Option) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let rust_opts = log_stream_options_from_js(opts).map_err(napi::Error::from_reason)?; let stream = sb.log_stream(&rust_opts).await.map_err(to_napi_error)?; spawn_log_stream_from_stream(stream).await diff --git a/sdk/node-ts/native/sandbox_builder.rs b/sdk/node-ts/native/sandbox_builder.rs index fb85d5cea..1796715e3 100644 --- a/sdk/node-ts/native/sandbox_builder.rs +++ b/sdk/node-ts/native/sandbox_builder.rs @@ -155,7 +155,7 @@ impl JsSandboxBuilder { self } - /// Supply the exact base for a disk-dependent snapshot archive. + /// Supply the base for omitted disk layers and RAM objects in a snapshot archive. #[napi] pub fn snapshot_base(&mut self, base: String) -> &Self { let prev = self.take_inner(); @@ -244,6 +244,17 @@ impl JsSandboxBuilder { Ok(self) } + /// Restore a full snapshot with private copy-on-write memory. + #[napi] + pub fn forked(&mut self) -> Result<&Self> { + let prev = self + .inner + .take() + .ok_or_else(|| napi::Error::from_reason("builder already consumed"))?; + self.inner = Some(prev.forked()); + Ok(self) + } + /// Override log verbosity: `"trace" | "debug" | "info" | "warn" | "error"`. #[napi(js_name = "logLevel")] pub fn log_level(&mut self, level: String) -> Result<&Self> { diff --git a/sdk/node-ts/native/sandbox_handle.rs b/sdk/node-ts/native/sandbox_handle.rs index 235d2c501..765bc3323 100644 --- a/sdk/node-ts/native/sandbox_handle.rs +++ b/sdk/node-ts/native/sandbox_handle.rs @@ -195,6 +195,26 @@ impl JsSandboxHandle { self.inner.stop().await.map_err(to_napi_error) } + /// Create an independent local CoW child without a durable full snapshot. + #[napi] + pub async fn branch(&self, name: String) -> Result { + Ok(crate::sandbox::Sandbox::from_rust( + self.inner.branch(name).await.map_err(to_napi_error)?, + )) + } + + /// Explicit resident pause through host control. + #[napi] + pub async fn pause(&self) -> Result<()> { + self.inner.pause().await.map_err(to_napi_error) + } + + /// Explicit resident resume through host control. + #[napi] + pub async fn resume(&self) -> Result<()> { + self.inner.resume().await.map_err(to_napi_error) + } + /// Request graceful shutdown without waiting. #[napi] pub async fn request_stop(&self) -> Result<()> { @@ -324,7 +344,7 @@ impl JsSandboxHandle { crate::sandbox::spawn_log_stream_from_stream(stream).await } - /// Snapshot this (stopped) sandbox under a bare name. + /// Snapshot this sandbox's disk under a bare name, preserving its running/paused state. /// /// Resolves under `~/.microsandbox/snapshots//`. Move /// artifacts with `Snapshot.save`/`Snapshot.load`. diff --git a/sdk/node-ts/native/shared_handle.rs b/sdk/node-ts/native/shared_handle.rs new file mode 100644 index 000000000..12f4fe7bc --- /dev/null +++ b/sdk/node-ts/native/shared_handle.rs @@ -0,0 +1,102 @@ +use std::sync::Arc; + +use tokio::sync::Mutex; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// A consumable wrapper whose admitted operations own their reference independently. +/// +/// The slot lock orders admission against consumption, not guest work against lifecycle work. +/// Holding it across guest I/O would let a paused guest prevent its own resume. +pub(crate) struct SharedHandle { + inner: Mutex>>, +} + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl SharedHandle { + pub(crate) fn new(value: T) -> Self { + Self { + inner: Mutex::new(Some(Arc::new(value))), + } + } + + /// Admit an operation without cloning the underlying sandbox configuration. + pub(crate) async fn get(&self) -> Option> { + self.inner.lock().await.clone() + } + + /// Reject future admissions; already admitted operations retain their references. + pub(crate) async fn take(&self) -> Option> { + self.inner.lock().await.take() + } +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + + #[tokio::test] + async fn pending_operation_does_not_block_admission_or_consumption() { + let handle = Arc::new(SharedHandle::new(String::from("sandbox"))); + let (admitted_tx, admitted_rx) = tokio::sync::oneshot::channel(); + let (finish_tx, finish_rx) = tokio::sync::oneshot::channel(); + let task_handle = handle.clone(); + let operation = tokio::spawn(async move { + let reference = task_handle.get().await.unwrap(); + admitted_tx.send(()).unwrap(); + finish_rx.await.unwrap(); + assert_eq!(reference.as_str(), "sandbox"); + }); + admitted_rx.await.unwrap(); + + tokio::time::timeout(Duration::from_secs(1), async { + let second = handle.get().await.unwrap(); + let consumed = handle.take().await.unwrap(); + assert!(Arc::ptr_eq(&second, &consumed)); + assert!(handle.get().await.is_none()); + assert!(handle.take().await.is_none()); + }) + .await + .expect("a pending operation must not hold the admission lock"); + finish_tx.send(()).unwrap(); + operation.await.unwrap(); + } + + #[tokio::test] + async fn concurrent_consumers_have_exactly_one_winner() { + let handle = SharedHandle::new(()); + let (first, second) = tokio::join!(handle.take(), handle.take()); + assert_ne!(first.is_some(), second.is_some()); + assert!(handle.get().await.is_none()); + } + + #[tokio::test] + async fn cancellation_releases_last_admitted_reference() { + let handle = Arc::new(SharedHandle::new(())); + let reference = handle.get().await.unwrap(); + let weak = Arc::downgrade(&reference); + let (admitted_tx, admitted_rx) = tokio::sync::oneshot::channel(); + let operation = tokio::spawn(async move { + let _reference = reference; + admitted_tx.send(()).unwrap(); + std::future::pending::<()>().await; + }); + admitted_rx.await.unwrap(); + drop(handle.take().await); + assert!(weak.upgrade().is_some()); + operation.abort(); + assert!(operation.await.unwrap_err().is_cancelled()); + assert!(weak.upgrade().is_none()); + } +} diff --git a/sdk/node-ts/native/snapshot.rs b/sdk/node-ts/native/snapshot.rs index beec3ecd5..5a7386878 100644 --- a/sdk/node-ts/native/snapshot.rs +++ b/sdk/node-ts/native/snapshot.rs @@ -1,7 +1,9 @@ use std::collections::HashMap; use std::path::PathBuf; -use microsandbox::snapshot::SaveOpts as RustSaveOpts; +use microsandbox::snapshot::{ + HeadUpdateReason, LoadOpts as RustLoadOpts, SaveOpts as RustSaveOpts, +}; use microsandbox::{ Snapshot as RustSnapshot, SnapshotArchive as RustSnapshotArchive, SnapshotFormat as RustSnapshotFormat, SnapshotHandle as RustSnapshotHandle, @@ -45,12 +47,36 @@ pub struct JsSaveOpts { pub with_image: Option, /// Skip zstd compression and write a plain `.tar`. pub plain_tar: Option, - /// Exact base snapshot or standalone archive for incremental disk export. + /// Base snapshot or standalone archive supplying reusable disk layers and RAM objects. pub since: Option, /// Newest N immutable disk layers to include. pub last_layers: Option, } +/// Options for importing one or more archives into a snapshot group. +#[derive(Default)] +#[napi(object, js_name = "LoadOpts")] +pub struct JsLoadOpts { + /// Parent directory containing snapshot groups. + pub dest: Option, + /// External snapshot or standalone archive for dependencies absent from the batch/group. + pub base: Option, + /// Destination group (generated when omitted). + pub group: Option, + /// Select the unique imported tip even when it is not a fast-forward. + pub set_head: Option, +} + +/// Outcome of reading or selecting a snapshot group's head. +#[napi(object, js_name = "HeadUpdate")] +pub struct JsHeadUpdate { + pub group: String, + pub previous: Option, + pub head: String, + pub reason: String, + pub changed: bool, +} + /// Result of `Snapshot.verify()`. /// /// `upperKind` is `"notRecorded"` when integrity is absent or `"verified"` @@ -79,6 +105,8 @@ pub struct JsSnapshotInfo { pub id: String, pub digest: String, pub name: Option, + pub group: Option, + pub head_update: Option, pub parent_digest: Option, pub image_ref: String, /// `"disk"` for file state or `"full"` for a complete VM checkpoint. @@ -199,6 +227,60 @@ impl JsSnapshot { Ok(JsSnapshotHandle::from_rust(h)) } + #[napi(js_name = "loadWithOptions")] + pub async fn load_with_options( + archive: String, + opts: Option, + ) -> Result { + let opts = opts.unwrap_or_default(); + let h = RustSnapshot::load_with_options( + &PathBuf::from(archive), + RustLoadOpts { + dest: opts.dest.map(PathBuf::from), + base: opts.base, + group: opts.group, + set_head: opts.set_head.unwrap_or(false), + }, + ) + .await + .map_err(to_napi_error)?; + Ok(JsSnapshotHandle::from_rust(h)) + } + + /// Import archives together, resolving dependencies within the batch and destination group. + #[napi(js_name = "loadMany")] + pub async fn load_many( + archives: Vec, + opts: Option, + ) -> Result> { + let opts = opts.unwrap_or_default(); + let paths = archives.into_iter().map(PathBuf::from).collect::>(); + let handles = RustSnapshot::load_many( + &paths, + RustLoadOpts { + dest: opts.dest.map(PathBuf::from), + base: opts.base, + group: opts.group, + set_head: opts.set_head.unwrap_or(false), + }, + ) + .await + .map_err(to_napi_error)?; + Ok(handles + .into_iter() + .map(JsSnapshotHandle::from_rust) + .collect()) + } + + /// Read a group's head, or select `group:member` as its head. + #[napi(js_name = "groupHead")] + pub async fn group_head(selector: String) -> Result { + let update = RustSnapshot::group_head(&selector) + .await + .map_err(to_napi_error)?; + Ok(head_update_to_js(&update)) + } + //---------------------------------------------------------------------------------------------- // Instance accessors (mirror PyVolume's getter style) //---------------------------------------------------------------------------------------------- @@ -208,6 +290,12 @@ impl JsSnapshot { self.inner.path().display().to_string() } + /// Outcome of the group head update performed by this capture. + #[napi(getter)] + pub fn head_update(&self) -> Option { + self.inner.head_update().map(head_update_to_js) + } + #[napi(getter)] pub fn id(&self) -> String { self.inner.id().to_string() @@ -410,6 +498,15 @@ impl JsSnapshot { #[napi] impl JsSnapshotHandle { + #[napi(getter)] + pub fn group(&self) -> Option { + self.inner.group().map(str::to_string) + } + + #[napi(getter)] + pub fn head_update(&self) -> Option { + self.inner.head_update().map(head_update_to_js) + } #[napi(getter)] pub fn id(&self) -> String { self.inner.id().to_string() @@ -537,6 +634,8 @@ fn snapshot_handle_to_info(h: &RustSnapshotHandle) -> JsSnapshotInfo { id: h.id().to_string(), digest: h.digest().to_string(), name: h.name().map(|s| s.to_string()), + group: h.group().map(str::to_string), + head_update: h.head_update().map(head_update_to_js), parent_digest: h.parent_digest().map(|s| s.to_string()), image_ref: h.image_ref().to_string(), scope: format_scope(h.scope()).into(), @@ -554,6 +653,26 @@ fn snapshot_handle_to_info(h: &RustSnapshotHandle) -> JsSnapshotInfo { } } +fn head_update_to_js(update: µsandbox::snapshot::HeadUpdate) -> JsHeadUpdate { + // Match the stable serde spelling without losing the closed reason variants. + let reason = match update.reason { + HeadUpdateReason::Initialized => "initialized", + HeadUpdateReason::FastForwarded => "fast_forwarded", + HeadUpdateReason::Selected => "selected", + HeadUpdateReason::Unchanged => "unchanged", + HeadUpdateReason::Diverged => "diverged", + HeadUpdateReason::UnknownAncestry => "unknown_ancestry", + HeadUpdateReason::AmbiguousCandidates => "ambiguous_candidates", + }; + JsHeadUpdate { + group: update.group.clone(), + previous: update.previous.clone(), + head: update.head.clone(), + reason: reason.into(), + changed: update.changed, + } +} + fn verify_report_to_js( report: microsandbox::snapshot::SnapshotVerifyReport, ) -> JsSnapshotVerifyReport { diff --git a/sdk/node-ts/native/snapshot_builder.rs b/sdk/node-ts/native/snapshot_builder.rs index 7361208d9..96d777fcd 100644 --- a/sdk/node-ts/native/snapshot_builder.rs +++ b/sdk/node-ts/native/snapshot_builder.rs @@ -14,6 +14,7 @@ use crate::snapshot::{JsSnapshot, JsSnapshotArchive}; #[napi(object, js_name = "SnapshotConfig")] pub struct JsSnapshotConfig { pub name: String, + pub group: Option, pub source_sandbox: Option, pub dest_dir: Option, // Keep the public name stable when napi-rs renders this renamed nested object. @@ -37,6 +38,7 @@ pub struct JsSnapshotLabel { pub struct JsSnapshotBuilder { inner: Option, name: String, + group: Option, source_sandbox: Option, dest_dir: Option, labels: Vec<(String, String)>, @@ -56,6 +58,7 @@ impl JsSnapshotBuilder { Self { inner: Some(RustSnapshot::builder(&name)), name, + group: None, source_sandbox: None, dest_dir: None, labels: Vec::new(), @@ -66,7 +69,7 @@ impl JsSnapshotBuilder { } /// Create the artifact under this parent directory instead of the - /// default snapshots store. The artifact lands at `destDir/`. + /// default snapshots store. The snapshot group is created under this root. #[napi(js_name = "destDir")] pub fn dest_dir(&mut self, dest_dir: String) -> &Self { let prev = self.take_inner(); @@ -75,6 +78,15 @@ impl JsSnapshotBuilder { self } + /// Install the snapshot in this group (defaults to the source sandbox's name). + #[napi] + pub fn group(&mut self, group: String) -> &Self { + let prev = self.take_inner(); + self.inner = Some(prev.group(&group)); + self.group = Some(group); + self + } + /// Set the source sandbox to snapshot. Required. // `from_*` normally takes no self, but napi setters mutate in place and // the JS-facing name `fromSandbox` is the contract. @@ -96,7 +108,7 @@ impl JsSnapshotBuilder { self } - /// Overwrite an existing artifact at the destination. + /// Overwrite an archive destination; installed group members are immutable. #[napi] pub fn force(&mut self) -> &Self { let prev = self.take_inner(); @@ -128,6 +140,7 @@ impl JsSnapshotBuilder { pub fn build(&self) -> JsSnapshotConfig { JsSnapshotConfig { name: self.name.clone(), + group: self.group.clone(), source_sandbox: self.source_sandbox.clone(), dest_dir: self.dest_dir.clone(), labels: self diff --git a/sdk/node-ts/src/index.ts b/sdk/node-ts/src/index.ts index 366bf3fc9..403a2d7ed 100644 --- a/sdk/node-ts/src/index.ts +++ b/sdk/node-ts/src/index.ts @@ -113,14 +113,16 @@ import { Snapshot as _Snapshot, type SnapshotBuilder as _SnapBT } from "./snapsh */ export const SnapshotBuilder = function SnapshotBuilder( this: unknown, - name: string, + name = "", ) { return _Snapshot.builder(name); -} as unknown as new (name: string) => _SnapBT; +} as unknown as new (name?: string) => _SnapBT; export type SnapshotBuilder = _SnapBT; export { SnapshotHandle } from "./snapshot-handle.js"; export type { SaveOpts, + LoadOpts, + HeadUpdate, SnapshotScope, SnapshotState, SnapshotVerifyReport, diff --git a/sdk/node-ts/src/internal/napi.ts b/sdk/node-ts/src/internal/napi.ts index db67863b5..cd0384e9d 100644 --- a/sdk/node-ts/src/internal/napi.ts +++ b/sdk/node-ts/src/internal/napi.ts @@ -189,6 +189,7 @@ export interface NapiSandboxBuilderSetters { memory(mib: number): this; maxMemory(mib: number): this; thp(policy: "always" | "madvise" | "never"): this; + forked(): this; logLevel(level: string): this; quietLogs(): this; detached(enabled: boolean): this; @@ -294,6 +295,9 @@ export interface NapiSandbox { attachWithBuilder(cmd: string, builder: NapiAttachOptionsBuilder): Promise; attachShell(): Promise; stop(): Promise; + branch(name: string): Promise; + pause(): Promise; + resume(): Promise; requestStop(): Promise; stopWithTimeout(timeoutMs: number): Promise; kill(): Promise; @@ -329,6 +333,9 @@ export interface NapiSandboxHandle { connectWithTimeout(timeoutMs: number): Promise; connectOrStart(detached?: boolean): Promise; stop(): Promise; + branch(name: string): Promise; + pause(): Promise; + resume(): Promise; requestStop(): Promise; stopWithTimeout(timeoutMs: number): Promise; kill(): Promise; @@ -581,6 +588,24 @@ export interface NapiSnapshotStatic { reindex(dir?: string): Promise; save(name: string, out: string, opts?: NapiSaveOpts): Promise; load(archive: string, dest?: string, base?: string): Promise; + loadWithOptions(archive: string, opts?: NapiLoadOpts): Promise; + loadMany(archives: string[], opts?: NapiLoadOpts): Promise; + groupHead(selector: string): Promise; +} + +export interface NapiLoadOpts { + dest?: string; + base?: string; + group?: string; + setHead?: boolean; +} + +export interface NapiHeadUpdate { + readonly group: string; + readonly previous: string | null | undefined; + readonly head: string; + readonly reason: string; + readonly changed: boolean; } export type NapiSnapshotBuilderCtor = new (name: string) => NapiSnapshotBuilder; @@ -588,6 +613,7 @@ export type NapiSnapshotBuilderCtor = new (name: string) => NapiSnapshotBuilder; export interface NapiSnapshotBuilderSetters { fromSandbox(sourceSandbox: string): this; destDir(destDir: string): this; + group(group: string): this; label(key: string, value: string): this; force(): this; recordIntegrity(): this; @@ -608,6 +634,7 @@ export interface NapiSnapshotArchive { export interface NapiSnapshot { readonly id: string; readonly path: string; + readonly headUpdate: NapiHeadUpdate | null | undefined; readonly digest: string; readonly sizeBytes: bigint | null | undefined; readonly imageRef: string; @@ -634,6 +661,8 @@ export interface NapiSnapshotHandle { readonly id: string; readonly digest: string; readonly name: string | null | undefined; + readonly group: string | null | undefined; + readonly headUpdate: NapiHeadUpdate | null | undefined; readonly parentDigest: string | null | undefined; readonly scope: string; // "disk" | "full" readonly imageRef: string; @@ -656,6 +685,8 @@ export interface NapiSnapshotInfo { readonly id: string; readonly digest: string; readonly name: string | null | undefined; + readonly group: string | null | undefined; + readonly headUpdate: NapiHeadUpdate | null | undefined; readonly parentDigest: string | null | undefined; readonly scope: string; // "disk" | "full" readonly imageRef: string; diff --git a/sdk/node-ts/src/sandbox-handle.ts b/sdk/node-ts/src/sandbox-handle.ts index 71e6a768c..3858bb72b 100644 --- a/sdk/node-ts/src/sandbox-handle.ts +++ b/sdk/node-ts/src/sandbox-handle.ts @@ -188,6 +188,22 @@ export class SandboxHandle { await withMappedErrors(() => this.inner.stop()); } + /** Create an independent local CoW child without a durable full snapshot. */ + async branch(name: string): Promise { + const child = await withMappedErrors(() => this.inner.branch(name)); + return new Sandbox(child, name, false); + } + + /** Suspend this resident VM without creating a snapshot. */ + async pause(): Promise { + await withMappedErrors(() => this.inner.pause()); + } + + /** Explicit resident resume; no snapshot is created. */ + async resume(): Promise { + await withMappedErrors(() => this.inner.resume()); + } + async requestStop(): Promise { await withMappedErrors(() => this.inner.requestStop()); } @@ -282,12 +298,12 @@ export class SandboxHandle { } /** - * Snapshot this (stopped) sandbox under a bare name. Resolves under + * Snapshot this sandbox's disk under a bare name. Resolves under * `~/.microsandbox/snapshots//`. For an explicit filesystem * destination, move the artifact with `Snapshot.save`/`Snapshot.load`. * - * The sandbox must be stopped (or crashed); running sandboxes are - * rejected with a `SnapshotSandboxRunning` error. + * Running and paused sources are supported. A live cut is crash-consistent + * and preserves the source's running/paused state. */ async snapshot(name: string): Promise { const raw = await withMappedErrors(() => this.inner.snapshot(name)); diff --git a/sdk/node-ts/src/sandbox.ts b/sdk/node-ts/src/sandbox.ts index ae8f3aa41..76cdb918c 100644 --- a/sdk/node-ts/src/sandbox.ts +++ b/sdk/node-ts/src/sandbox.ts @@ -519,6 +519,22 @@ export class Sandbox implements AsyncDisposable { await withMappedErrors(() => this.inner.stop()); } + /** Create an independent local CoW child without a durable full snapshot. */ + async branch(name: string): Promise { + const child = await withMappedErrors(() => this.inner.branch(name)); + return new Sandbox(child, name, false); + } + + /** Suspend this resident VM without creating a snapshot. */ + async pause(): Promise { + await withMappedErrors(() => this.inner.pause()); + } + + /** Explicit resident resume; no snapshot is created. */ + async resume(): Promise { + await withMappedErrors(() => this.inner.resume()); + } + async requestStop(): Promise { await withMappedErrors(() => this.inner.requestStop()); } @@ -581,6 +597,11 @@ export class Sandbox implements AsyncDisposable { ); } + /** + * Consume this handle without stopping the sandbox. New guest and filesystem + * operations on this handle are rejected; already admitted operations retain + * their connection. Await operations first when their completion matters. + */ async detach(): Promise { await withMappedErrors(() => this.inner.detach()); } diff --git a/sdk/node-ts/src/snapshot-handle.ts b/sdk/node-ts/src/snapshot-handle.ts index 1cff7c7c0..97ef2ceb4 100644 --- a/sdk/node-ts/src/snapshot-handle.ts +++ b/sdk/node-ts/src/snapshot-handle.ts @@ -3,7 +3,7 @@ import type { NapiSnapshotHandle, NapiSnapshotInfo, } from "./internal/napi.js"; -import { Snapshot, type SnapshotScope } from "./snapshot.js"; +import { Snapshot, type HeadUpdate, type SnapshotScope } from "./snapshot.js"; const READ_ONLY_MSG = "SnapshotHandle is read-only — fetch a live handle via Snapshot.get(name) for lifecycle methods."; @@ -23,6 +23,10 @@ export class SnapshotHandle { readonly digest: string; /** Convenience name. `null` for digest-only entries. */ readonly name: string | null; + /** Local group containing this indexed snapshot. */ + readonly group: string | null; + /** Outcome of the group head update performed by this import. */ + readonly headUpdate: HeadUpdate | null; /** Manifest digest of the parent snapshot, or `null` for a root. */ readonly parentDigest: string | null; /** Snapshot payload scope. */ @@ -58,6 +62,10 @@ export class SnapshotHandle { this.id = inner.id; this.digest = inner.digest; this.name = (inner.name ?? null) as string | null; + this.group = inner.group ?? null; + this.headUpdate = inner.headUpdate + ? { ...inner.headUpdate, previous: inner.headUpdate.previous ?? null } + : null; this.parentDigest = (inner.parentDigest ?? null) as string | null; this.scope = inner.scope as SnapshotScope; this.imageRef = inner.imageRef; diff --git a/sdk/node-ts/src/snapshot.ts b/sdk/node-ts/src/snapshot.ts index ee32daa4d..2f9c02100 100644 --- a/sdk/node-ts/src/snapshot.ts +++ b/sdk/node-ts/src/snapshot.ts @@ -52,7 +52,7 @@ export type SnapshotState = * Bundle options for `Snapshot.save`. */ export interface SaveOpts { - /** Exact base snapshot or standalone archive; mutually exclusive with lastLayers/withParents. */ + /** Omit disk layers and RAM objects supplied by this base; mutually exclusive with lastLayers/withParents. */ since?: string; /** Newest N sealed disk layers. Full snapshots still include all memory/device state. */ lastLayers?: number; @@ -64,6 +64,27 @@ export interface SaveOpts { plainTar?: boolean; } +/** Options for importing one or more archives into a snapshot group. */ +export interface LoadOpts { + /** Parent directory containing snapshot groups. */ + dest?: string; + /** External snapshot or standalone archive for dependencies absent from the batch/group. */ + base?: string; + /** Destination group; generated when omitted. */ + group?: string; + /** Select the unique imported tip even when it is not a fast-forward. */ + setHead?: boolean; +} + +/** Outcome of reading or selecting a snapshot group's head. */ +export interface HeadUpdate { + readonly group: string; + readonly previous: string | null; + readonly head: string; + readonly reason: string; + readonly changed: boolean; +} + /** Result of an explicit `Snapshot.verify()` call. */ export type SnapshotVerifyReport = | { @@ -133,23 +154,20 @@ export class Snapshot { } /** - * Begin building a snapshot named `name`, stored under the default - * snapshots directory. + * Begin building a snapshot member; an omitted name is generated. * * The source sandbox is required: * `Snapshot.builder("clean").fromSandbox("box").create()`. * - * Use `destDir(dir)` to create the artifact under a different parent - * directory instead; it lands at `destDir/`, and the name stays - * the snapshot's identity either way. + * Use `group(name)` to select a group and `destDir(dir)` to select its + * parent directory. The default group is the source sandbox's name. */ - static builder(name: string): SnapshotBuilder { + static builder(name = ""): SnapshotBuilder { return wrapBuilder(new napi.SnapshotBuilder(name)); } /** - * Open an existing snapshot artifact. Bare names resolve under the - * default snapshots directory; anything else is treated as a path. + * Open a snapshot by path, group head, or `group:member` selector. * * Cheap metadata validation only — does not read the upper file. * Use `verify()` for content checks. @@ -227,6 +245,24 @@ export class Snapshot { return new SnapshotHandle(raw); } + /** Import into a selected or generated group, with optional head selection. */ + static async loadWithOptions(archive: string, opts: LoadOpts = {}): Promise { + const raw = await withMappedErrors(() => napi.Snapshot.loadWithOptions(archive, opts)); + return new SnapshotHandle(raw); + } + + /** Import archives together into one group, resolving dependencies regardless of input order. */ + static async loadMany(archives: string[], opts: LoadOpts = {}): Promise { + const raw = await withMappedErrors(() => napi.Snapshot.loadMany(archives, opts)); + return raw.map((handle) => new SnapshotHandle(handle)); + } + + /** Read a group's head, or select `group:member` as its head. */ + static async groupHead(selector: string): Promise { + const update = await withMappedErrors(() => napi.Snapshot.groupHead(selector)); + return { ...update, previous: update.previous ?? null }; + } + //-------------------------------------------------------------------------- // Instance accessors //-------------------------------------------------------------------------- @@ -236,6 +272,12 @@ export class Snapshot { return this.inner.path; } + /** Outcome of the group head update performed by this capture. */ + get headUpdate(): HeadUpdate | null { + const update = this.inner.headUpdate; + return update ? { ...update, previous: update.previous ?? null } : null; + } + /** Canonical content digest (`sha256:hex`). The snapshot's identity. */ get id(): string { return this.inner.id; diff --git a/sdk/node-ts/tests/cow-lifecycle.test.ts b/sdk/node-ts/tests/cow-lifecycle.test.ts new file mode 100644 index 000000000..4f0402fea --- /dev/null +++ b/sdk/node-ts/tests/cow-lifecycle.test.ts @@ -0,0 +1,33 @@ +import { expect, it } from "vitest"; +import { Sandbox, Snapshot } from "../dist/index.js"; + +// Opt-in because this starts real VMs with a matching development runtime/kernel bundle. +it.skipIf(process.env.MSB_COW_LIVE !== "1")("captures a resident pause and restores private memory", async () => { + const name = `cow8-node-${process.pid}`; + const source = await Sandbox.builder(name).image("alpine").rootDisk(512).memory(256).create(); + let child: Sandbox | undefined; + const branches: Sandbox[] = []; + try { + await source.exec("sh", ["-c", "echo source > /dev/shm/sdk-marker"]); + await source.pause(); + const paused = await Sandbox.get(name); + expect(paused.status).toBe("paused"); + const branched = await paused.branch(`${name}-paused-branch`); + branches.push(branched); + expect((await branched.exec("cat", ["/dev/shm/sdk-marker"])).stdout().trim()).toBe("source"); + const snapshot = await Snapshot.builder(`${name}-full`).fromSandbox(name).full().create(); + await paused.resume(); + child = await Sandbox.builder(`${name}-child`).fromSnapshot(snapshot.path).forked().create(); + expect((await child.exec("cat", ["/dev/shm/sdk-marker"])).stdout().trim()).toBe("source"); + await child.exec("sh", ["-c", "echo child > /dev/shm/sdk-marker"]); + const descendant = await child.branch(`${name}-branch`); + branches.push(descendant); + expect((await descendant.exec("cat", ["/dev/shm/sdk-marker"])).stdout().trim()).toBe("child"); + expect((await source.exec("cat", ["/dev/shm/sdk-marker"])).stdout().trim()).toBe("source"); + await child.pause(); + } finally { + for (const branch of branches.reverse()) await branch.stop(); + await child?.stop(); + await source.stop(); + } +}, 120_000); diff --git a/sdk/node-ts/tests/lifecycle-concurrency.test.ts b/sdk/node-ts/tests/lifecycle-concurrency.test.ts new file mode 100644 index 000000000..283ce4afb --- /dev/null +++ b/sdk/node-ts/tests/lifecycle-concurrency.test.ts @@ -0,0 +1,215 @@ +import { createRequire } from "node:module"; +import { execFileSync } from "node:child_process"; +import { mkdtemp, open, rmdir, unlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { performance } from "node:perf_hooks"; +import { setTimeout as delay } from "node:timers/promises"; +import { describe, expect, it } from "vitest"; +import { Sandbox, SandboxNotFoundError } from "../dist/index.js"; + +const native: typeof import("../native/index.js") = createRequire(import.meta.url)("../native/index.cjs"); +const consumed = /Sandbox handle has been consumed/; + +async function bounded(promise: Promise, label: string, timeoutMs = 2_000): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`${label} timed out after ${timeoutMs} ms`)), timeoutMs); + }), + ]); + } finally { + clearTimeout(timer); + } +} + +async function fixture(label: string, run: (source: Sandbox, observer: Sandbox) => Promise) { + const name = `node-concurrency-${label}-${process.pid}`; + const source = await Sandbox.builder(name).image("alpine").rootDisk(512).memory(256).create(); + let observer: Sandbox | undefined; + try { + observer = await (await Sandbox.get(name)).connect(); + await run(source, observer); + } finally { + // A separate handle can recover a paused VM even if the tested wrapper is + // consumed or regresses to holding its lock while waiting for the guest. + const handle = await Sandbox.get(name).catch(error => { + if (error instanceof SandboxNotFoundError) return undefined; + throw error; + }); + if (handle) { + if (handle.status === "paused") await handle.resume(); + await handle.stop(); + await Sandbox.remove(name); + } + } +} + +async function gatedExec(source: Sandbox, observer: Sandbox) { + const pending = source.exec("sh", ["-c", "touch /dev/shm/started; while [ ! -e /dev/shm/release ]; do sleep 0.02; done; echo completed"]); + // Attach a rejection handler immediately so fixture cleanup cannot produce + // an unhandled rejection if a later assertion fails and stops the VM. + void pending.catch(() => undefined); + await bounded((async () => { + while (!(await observer.fs().exists("/dev/shm/started"))) await delay(10); + })(), "guest admission"); + return { pending }; +} + +describe.skipIf(process.env.MSB_NODE_LIFECYCLE_LIVE !== "1")("same-object lifecycle concurrency", () => { + it("pauses and resumes while exec is pending, then completes the original exec", async () => { + await fixture("exec", async (source, observer) => { + const { pending } = await gatedExec(source, observer); + let settled = false; + void pending.finally(() => { settled = true; }).catch(() => undefined); + const pauses: number[] = []; + const resumes: number[] = []; + for (let cycle = 0; cycle < 20; cycle++) { + let started = performance.now(); + await bounded(source.pause(), "pause during exec"); + pauses.push(performance.now() - started); + expect((await Sandbox.get(source.name)).status).toBe("paused"); + started = performance.now(); + await bounded(source.resume(), "resume during exec"); + resumes.push(performance.now() - started); + expect(settled).toBe(false); + } + await observer.fs().write("/dev/shm/release", "go"); + expect((await bounded(pending, "exec completion")).stdout().trim()).toBe("completed"); + const timings = { platform: process.platform, arch: process.arch, pauses_ms: pauses, resumes_ms: resumes }; + console.log(JSON.stringify(timings)); + if (process.env.MSB_NODE_TIMINGS) await writeFile(process.env.MSB_NODE_TIMINGS, JSON.stringify(timings, null, 2)); + }); + }); + + it("does not let a filesystem request to a paused guest block resume", async () => { + await fixture("fs", async (source) => { + const fs = source.fs(); + await fs.write("/dev/shm/payload", "retained"); + await source.pause(); + const read = fs.readToString("/dev/shm/payload"); + // Both a prompt paused-state refusal and a read waiting for resume are + // valid; neither is allowed to monopolize the wrapper lock. + const outcome = read.then(value => ({ value }), error => ({ error })); + await delay(50); + await bounded(source.resume(), "resume during filesystem read"); + const result = await bounded(outcome, "filesystem completion"); + if ("error" in result) expect(String(result.error)).toMatch(/paused/i); + else expect(result.value).toBe("retained"); + expect(await fs.readToString("/dev/shm/payload")).toBe("retained"); + }); + }); + + it("detach consumes new calls and existing filesystem facades without draining exec", async () => { + await fixture("detach", async (source, observer) => { + const fs = source.fs(); + const { pending } = await gatedExec(source, observer); + await bounded(source.detach(), "detach during exec"); + await expect(source.pause()).rejects.toThrow(consumed); + await expect(source.exec("true")).rejects.toThrow(consumed); + await expect(fs.exists("/dev/shm/started")).rejects.toThrow(consumed); + await source.detach(); // Detach remains idempotent. + await observer.fs().write("/dev/shm/release", "go"); + expect((await bounded(pending, "detached exec completion")).stdout().trim()).toBe("completed"); + expect((await observer.exec("echo", ["alive"])).stdout().trim()).toBe("alive"); + }); + }); + + it.skipIf(process.platform === "win32")("pause, resume and detach progress during a blocked filesystem upload", async () => { + await fixture("upload", async (source, observer) => { + const directory = await mkdtemp(join(tmpdir(), "msb-node-upload-")); + const fifo = join(directory, "input"); + execFileSync("mkfifo", [fifo]); + const fs = source.fs(); + const upload = fs.copyFromHost(fifo, "/dev/shm/uploaded"); + void upload.catch(() => undefined); + // Opening the writer proves the native filesystem operation has opened + // its reader. With no bytes or EOF, the upload cannot complete yet. + const writer = await open(fifo, "w"); + let closed = false; + try { + await writer.writeFile("initial "); + await bounded((async () => { + while (true) { + try { + if ((await observer.fs().stat("/dev/shm/uploaded")).size >= 8) break; + } catch (error) { + if (!String(error).includes("No such file")) throw error; + } + await delay(10); + } + })(), "guest upload admission"); + await bounded(source.pause(), "pause during upload"); + await bounded(source.resume(), "resume during upload"); + await bounded(source.detach(), "detach during upload"); + await expect(fs.exists("/tmp")).rejects.toThrow(consumed); + await writer.writeFile("private upload payload"); + await writer.close(); + closed = true; + await bounded(upload, "admitted upload after detach"); + expect(await observer.fs().readToString("/dev/shm/uploaded")).toBe("initial private upload payload"); + } finally { + if (!closed) await writer.close(); + await unlink(fifo); + await rmdir(directory); + } + }); + }); + + it("removal consumes its wrapper while exec is pending, independently of lifecycle locking", async () => { + await fixture("remove", async (source, observer) => { + const raw = await (await native.Sandbox.get(source.name)).connect(); + const fs = raw.fs(); + const pending = raw.exec("sh", ["-c", "touch /dev/shm/started; while [ ! -e /dev/shm/release ]; do sleep 0.02; done; echo completed"]); + void pending.catch(() => undefined); + await bounded((async () => { + while (!(await observer.fs().exists("/dev/shm/started"))) await delay(10); + })(), "native exec admission"); + const removal = raw.removePersisted().then(() => ({ removed: true }), error => ({ error })); + await delay(50); + await expect(bounded(raw.pause(), "consumed pause")).rejects.toThrow(consumed); + await expect(bounded(fs.exists("/dev/shm/started"), "consumed filesystem")).rejects.toThrow(consumed); + await expect(raw.removePersisted()).rejects.toThrow(consumed); + await observer.fs().write("/dev/shm/release", "go"); + expect((await bounded(pending, "exec after failed removal")).success).toBe(true); + // Runtime lifecycle locking can defer removal until shutdown or reject + // the running VM. Neither should keep the Node admission slot locked. + const stopError = await observer.stop().then(() => undefined, error => error); + const result = await bounded(removal, "removal after shutdown", 10_000); + if (stopError !== undefined) { + // Removal can win after the VM exits but before stop's final database + // observation. Accept a missing row only when this removal succeeded. + expect(stopError).toBeInstanceOf(SandboxNotFoundError); + expect("removed" in result).toBe(true); + } + if ("error" in result) expect(String(result.error)).toMatch(/running|stopped|lock|timed out/i); + }); + }); + + it("a terminal-state waiter does not block stopping and successful removal", async () => { + const name = `node-concurrency-stopped-${process.pid}`; + const raw = await new native.SandboxBuilder(name).image("alpine").rootDisk(512).memory(256).create(); + try { + const fs = raw.fs(); + const waiter = raw.waitUntilStopped(); + void waiter.catch(() => undefined); + await delay(50); + await bounded(raw.stop(), "stop during wait", 10_000); + await bounded(waiter, "terminal-state observation"); + await raw.removePersisted(); + await expect(raw.resume()).rejects.toThrow(consumed); + await expect(fs.exists("/tmp")).rejects.toThrow(consumed); + await expect(native.Sandbox.get(name)).rejects.toThrow(); + } finally { + // Successful removal already deleted the record; only recover a fixture + // that still exists. Unexpected cleanup errors must remain visible. + const handle = await Sandbox.get(name).catch(() => undefined); + if (handle) { + await handle.stop(); + await Sandbox.remove(name); + } + } + }); +}); diff --git a/sdk/node-ts/tests/unit/builders.test.ts b/sdk/node-ts/tests/unit/builders.test.ts index 36955f7f8..e5b2c6068 100644 --- a/sdk/node-ts/tests/unit/builders.test.ts +++ b/sdk/node-ts/tests/unit/builders.test.ts @@ -352,6 +352,11 @@ describe("PatchBuilder", () => { }); describe("SandboxBuilder.build", () => { + it("rejects forked for a fresh boot", async () => { + await expect(Sandbox.builder("forked-policy").image("alpine").forked().build()) + .rejects.toThrow("forked requires a full snapshot"); + }); + it("requires .image()", async () => { await expect(Sandbox.builder("x").build()).rejects.toThrow( InvalidConfigError, diff --git a/sdk/node-ts/tests/unit/native-contract.test.ts b/sdk/node-ts/tests/unit/native-contract.test.ts index 3ac4e76a4..39d9ad1a9 100644 --- a/sdk/node-ts/tests/unit/native-contract.test.ts +++ b/sdk/node-ts/tests/unit/native-contract.test.ts @@ -80,7 +80,22 @@ describe("native image cache contract", () => { }); describe("native snapshot contract", () => { + it("exports group creation, import, and head selection", () => { + expect(typeof napi.Snapshot.loadWithOptions).toBe("function"); + expect(typeof napi.Snapshot.loadMany).toBe("function"); + expect(typeof napi.Snapshot.groupHead).toBe("function"); + expect(typeof napi.SnapshotBuilder.prototype.group).toBe("function"); + const builder = new napi.SnapshotBuilder("").fromSandbox("source").group("work"); + const config = (builder as unknown as { build(): { name: string; group: string } }).build(); + expect(config.name).toBe(""); + expect(config.group).toBe("work"); + }); it("exports the direct archive result used by the TS wrapper", () => { expect(typeof napi.SnapshotArchive).toBe("function"); }); + it("passes empty batches to core validation", async () => { + await expect(napi.Snapshot.loadMany([])).rejects.toThrow( + "snapshot load requires at least one archive", + ); + }); }); diff --git a/sdk/node-ts/tests/unit/snapshot.test.ts b/sdk/node-ts/tests/unit/snapshot.test.ts index 2d2d5cfef..5c172cbd6 100644 --- a/sdk/node-ts/tests/unit/snapshot.test.ts +++ b/sdk/node-ts/tests/unit/snapshot.test.ts @@ -1,5 +1,10 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { Snapshot } from "../../dist/snapshot.js"; +import { napi } from "../../dist/internal/napi.js"; + +vi.mock("../../dist/internal/napi.js", () => ({ + napi: { Snapshot: { loadWithOptions: vi.fn(), loadMany: vi.fn(), groupHead: vi.fn() } }, +})); function projectedSnapshot( overrides: Record = {}, @@ -39,6 +44,64 @@ function projectedSnapshot( } describe("Snapshot native projections", () => { + it("exposes the create outcome with a nullable previous head", () => { + const snapshot = projectedSnapshot({ + headUpdate: { + group: "work", previous: undefined, head: "snapshot-1", reason: "initialized", changed: true, + }, + }); + expect(snapshot.headUpdate).toEqual({ + group: "work", previous: null, head: "snapshot-1", reason: "initialized", changed: true, + }); + expect(projectedSnapshot().headUpdate).toBeNull(); + }); + + it("forwards import options and preserves a retained-head outcome", async () => { + const headUpdate = { + group: "work", previous: "snapshot-1", head: "snapshot-1", reason: "diverged", changed: false, + }; + vi.mocked(napi.Snapshot.loadWithOptions).mockResolvedValue({ + id: "snapshot-2", digest: "sha256:two", group: "work", headUpdate, + name: "other", createdAt: 0, path: "/snapshots/work/snapshot-2", + } as never); + const options = { dest: "/snapshots", base: "work:base", group: "work", setHead: false }; + const handle = await Snapshot.loadWithOptions("other.msb", options); + expect(napi.Snapshot.loadWithOptions).toHaveBeenCalledWith("other.msb", options); + expect(handle.group).toBe("work"); + expect(handle.id).toBe("snapshot-2"); + expect(handle.headUpdate).toEqual(headUpdate); + }); + + it("forwards a member selector for explicit head selection", async () => { + vi.mocked(napi.Snapshot.groupHead).mockResolvedValue({ + group: "work", previous: "snapshot-2", head: "snapshot-1", reason: "selected", changed: true, + }); + expect(await Snapshot.groupHead("work:baseline")).toMatchObject({ reason: "selected", changed: true }); + expect(napi.Snapshot.groupHead).toHaveBeenCalledWith("work:baseline"); + }); + + it("loads a batch once and preserves input-order handles and headless outcomes", async () => { + vi.mocked(napi.Snapshot.loadMany).mockResolvedValue([ + { id: "snapshot-tip", digest: "sha256:tip", path: "/snapshots/received/tip", group: "received", createdAt: 0 }, + { id: "snapshot-base", digest: "sha256:base", path: "/snapshots/received/base", group: "received", createdAt: 0 }, + ] as never); + const archives = ["changes.msb", "base.msb"]; + const options = { group: "received", dest: "/snapshots" }; + const handles = await Snapshot.loadMany(archives, options); + expect(napi.Snapshot.loadMany).toHaveBeenCalledWith(archives, options); + expect(handles.map((handle) => handle.id)).toEqual(["snapshot-tip", "snapshot-base"]); + expect(handles.map((handle) => handle.group)).toEqual(["received", "received"]); + expect(handles.every((handle) => handle.headUpdate === null)).toBe(true); + }); + + it("passes explicit batch head selection to the native importer", async () => { + vi.mocked(napi.Snapshot.loadMany).mockResolvedValue([]); + await Snapshot.loadMany(["tip.msb", "base.msb"], { group: "received", base: "outside:base", setHead: true }); + expect(napi.Snapshot.loadMany).toHaveBeenLastCalledWith( + ["tip.msb", "base.msb"], { group: "received", base: "outside:base", setHead: true }, + ); + }); + it("returns complete file and checkpoint states", () => { expect(projectedSnapshot().state).toMatchObject({ kind: "file", diff --git a/sdk/python/integration/test_snapshots.py b/sdk/python/integration/test_snapshots.py index 285d4cb64..0a6020164 100644 --- a/sdk/python/integration/test_snapshots.py +++ b/sdk/python/integration/test_snapshots.py @@ -3,6 +3,7 @@ from __future__ import annotations from contextlib import suppress +from pathlib import Path import pytest @@ -17,14 +18,18 @@ @pytest.mark.asyncio -async def test_snapshot_create_open_list_and_boot(sandbox_name): +@pytest.mark.parametrize( + "source_kind", ["member", "group", "id", "directory", "pathlike", "archive"] +) +async def test_snapshot_create_open_list_and_boot(sandbox_name, tmp_path, source_kind): base_name = sandbox_name("py-sdk-snap-base") fork_name = sandbox_name("py-sdk-snap-fork") snapshot_name = sandbox_name("py-sdk-snap") await remove_sandbox(fork_name) await remove_sandbox(base_name) - await remove_snapshot(snapshot_name) + snapshot_selector = f"{base_name}:{snapshot_name}" + await remove_snapshot(snapshot_selector) base = await Sandbox.create(base_name, image=IMAGE, cpus=1, memory=512, replace=True) fork = None @@ -44,11 +49,18 @@ async def test_snapshot_create_open_list_and_boot(sandbox_name): verify_result = await snapshot.verify() assert isinstance(verify_result, dict) - handle = await Snapshot.get(snapshot_name) + handle = await Snapshot.get(snapshot_selector) assert handle.digest == snapshot.digest assert handle.state_kind is SnapshotStateKind.FILE assert handle.format is SnapshotFormat.RAW assert handle.scope is SnapshotScope.DISK + assert handle.group == base_name + assert snapshot.head_update["reason"] == "initialized" + head = await Snapshot.group_head(base_name) + assert head["head"] == snapshot.id + assert head["changed"] is False + selected = await Snapshot.group_head(snapshot_selector) + assert selected["head"] == snapshot.id opened = await handle.open() assert opened.digest == snapshot.digest assert opened.state_kind is SnapshotStateKind.FILE @@ -56,9 +68,25 @@ async def test_snapshot_create_open_list_and_boot(sandbox_name): snapshots = await Snapshot.list() assert any(item.digest == snapshot.digest for item in snapshots) + # All public selector forms must reach the same Rust resolver. In particular, a + # group:member selector is not a literal directory underneath MSB_HOME/snapshots. + sources = { + "member": snapshot_selector, + "group": base_name, + "id": snapshot.id, + "directory": snapshot.path, + "pathlike": Path(snapshot.path), + } + if source_kind == "archive": + archive = tmp_path / "saved.msb" + await Snapshot.save(snapshot_selector, str(archive)) + source = archive + else: + source = sources[source_kind] + fork = await Sandbox.create( fork_name, - from_snapshot=snapshot_name, + from_snapshot=source, cpus=1, memory=512, replace=True, @@ -72,4 +100,4 @@ async def test_snapshot_create_open_list_and_boot(sandbox_name): await fork.stop() await remove_sandbox(fork_name) await remove_sandbox(base_name) - await remove_snapshot(snapshot_name) + await remove_snapshot(snapshot_selector) diff --git a/sdk/python/microsandbox/_microsandbox.pyi b/sdk/python/microsandbox/_microsandbox.pyi index f4707d88d..b891779eb 100644 --- a/sdk/python/microsandbox/_microsandbox.pyi +++ b/sdk/python/microsandbox/_microsandbox.pyi @@ -94,6 +94,7 @@ class Sandbox: from_snapshot: str | os.PathLike[str] | None = None, disk_only: bool = False, snapshot_base: str | None = None, + forked: bool = False, memory: int | None = None, cpus: int | None = None, max_memory: int | None = None, @@ -136,6 +137,7 @@ class Sandbox: from_snapshot: str | os.PathLike[str] | None = None, disk_only: bool = False, snapshot_base: str | None = None, + forked: bool = False, memory: int | None = None, cpus: int | None = None, max_memory: int | None = None, @@ -193,6 +195,7 @@ class Sandbox: from_snapshot: str | os.PathLike[str] | None = None, disk_only: bool = False, snapshot_base: str | None = None, + forked: bool = False, memory: int | None = None, cpus: int | None = None, max_memory: int | None = None, @@ -366,6 +369,9 @@ class Sandbox: follow: bool = False, ) -> LogStream: ... async def stop(self, timeout: float | None = None) -> None: ... + async def branch(self, name: str) -> Sandbox: ... + async def pause(self) -> None: ... + async def resume(self) -> None: ... async def request_stop(self) -> None: ... async def kill(self, timeout: float | None = None) -> None: ... async def request_kill(self) -> None: ... @@ -475,6 +481,9 @@ class SandboxHandle: async def connect(self, timeout: float | None = None) -> Sandbox: ... async def connect_or_start(self, *, detached: bool = False) -> Sandbox: ... async def stop(self, timeout: float | None = None) -> None: ... + async def branch(self, name: str) -> Sandbox: ... + async def pause(self) -> None: ... + async def resume(self) -> None: ... async def request_stop(self) -> None: ... async def kill(self, timeout: float | None = None) -> None: ... async def request_kill(self) -> None: ... @@ -888,9 +897,10 @@ class ImagePruneReport: class Snapshot: @staticmethod async def create( - name: str, + name: str = "", *, from_sandbox: str, + group: str | None = None, dest_dir: str | os.PathLike[str] | None = None, labels: dict[str, str] | None = None, force: bool = False, @@ -903,6 +913,7 @@ class Snapshot: archive: str | os.PathLike[str], *, from_sandbox: str, + group: str | None = None, labels: dict[str, str] | None = None, force: bool = False, record_integrity: bool = False, @@ -938,7 +949,22 @@ class Snapshot: *, dest: str | os.PathLike[str] | None = None, base: str | None = None, + group: str | None = None, + set_head: bool = False, ) -> SnapshotHandle: ... + @staticmethod + async def load_many( + archives: Sequence[str | os.PathLike[str]], + *, + dest: str | os.PathLike[str] | None = None, + base: str | None = None, + group: str | None = None, + set_head: bool = False, + ) -> list[SnapshotHandle]: ... + @staticmethod + async def group_head(selector: str) -> dict[str, str | bool | None]: ... + @property + def head_update(self) -> dict[str, str | bool | None] | None: ... @property def id(self) -> str: ... @property @@ -982,6 +1008,10 @@ class SnapshotArchive: def path(self) -> str: ... class SnapshotHandle: + @property + def group(self) -> str | None: ... + @property + def head_update(self) -> dict[str, str | bool | None] | None: ... @property def id(self) -> str: ... @property diff --git a/sdk/python/src/error.rs b/sdk/python/src/error.rs index 3f488e5ae..0a04cf2e1 100644 --- a/sdk/python/src/error.rs +++ b/sdk/python/src/error.rs @@ -31,6 +31,12 @@ pub fn local_only(name: &str) -> PyErr { pub fn to_py_err(err: microsandbox::MicrosandboxError) -> PyErr { use microsandbox::MicrosandboxError::*; + // Missing snapshot selectors are resolved when the create future is awaited, like explicit + // artifact paths. Retain Python's useful missing-file exception without duplicating resolution. + if let SnapshotNotFound(_) = &err { + return pyo3::exceptions::PyFileNotFoundError::new_err(err.to_string()); + } + Python::with_gil(|py| { let errors_mod = match py.import("microsandbox.errors") { Ok(m) => m, diff --git a/sdk/python/src/helpers.rs b/sdk/python/src/helpers.rs index 45eeed11e..3de8ac1eb 100644 --- a/sdk/python/src/helpers.rs +++ b/sdk/python/src/helpers.rs @@ -25,6 +25,7 @@ const KNOWN_CREATE_KWARGS: &[&str] = &[ "cpu_placement", "placement_profile", "thp", + "forked", "workdir", "shell", "security", @@ -226,18 +227,8 @@ pub fn sandbox_builder_from_args( "from_snapshot must be str or os.PathLike", )); }; - // Preserve Python's established immediate missing-artifact error before constructing an - // awaitable. Descriptor parsing and disk/full admission remain deferred to the shared Rust - // resolver so installed directories and direct archives follow exactly the same path. - let snapshot_path = resolve_snapshot_path(&snap_str); - if !snapshot_path.exists() { - return Err(pyo3::exceptions::PyFileNotFoundError::new_err(format!( - "snapshot artifact not found: {}", - snapshot_path.display() - ))); - } - // Resolution stays deferred until the async build so installed and direct-archive sources - // share the same disk/full admission path. + // Resolve through the shared async builder: a group/member or snapshot identity is not + // a directory name. A Python-only existence check would reject these valid selectors. builder = builder.from_snapshot(snap_str); if let Some(base) = kwargs.get_item("snapshot_base")? && !base.is_none() @@ -337,6 +328,9 @@ pub fn sandbox_builder_from_args( .map_err(pyo3::exceptions::PyValueError::new_err)?; builder = builder.thp(policy); } + if extract_opt::(kwargs, "forked")?.unwrap_or(false) { + builder = builder.forked(); + } if let Some(workdir) = extract_opt::(kwargs, "workdir")? { builder = builder.workdir(workdir); } @@ -2016,37 +2010,3 @@ fn extract_required<'py, T: FromPyObject<'py>>( .ok_or_else(|| pyo3::exceptions::PyValueError::new_err(format!("{key} is required")))? .extract() } - -/// Resolve a snapshot reference only far enough to preserve synchronous Python path validation. -fn resolve_snapshot_path(reference: &str) -> std::path::PathBuf { - if snapshot_ref_looks_like_path(reference) { - std::path::PathBuf::from(reference) - } else { - microsandbox::backend::default_backend() - .as_local() - .map(|local| local.snapshots_dir().join(reference)) - .unwrap_or_else(|| std::path::PathBuf::from(reference)) - } -} - -/// Match the Rust snapshot resolver's bare-name versus filesystem-path boundary. -fn snapshot_ref_looks_like_path(reference: &str) -> bool { - if reference.contains('/') || reference.starts_with('.') || reference.starts_with('~') { - return true; - } - - #[cfg(windows)] - { - use typed_path::{Utf8WindowsComponent, Utf8WindowsPath}; - - reference.contains('\\') - || matches!( - Utf8WindowsPath::new(reference).components().next(), - Some(Utf8WindowsComponent::Prefix(_)) - ) - } - #[cfg(not(windows))] - { - false - } -} diff --git a/sdk/python/src/sandbox.rs b/sdk/python/src/sandbox.rs index 68923740a..96efe167a 100644 --- a/sdk/python/src/sandbox.rs +++ b/sdk/python/src/sandbox.rs @@ -992,6 +992,37 @@ impl PySandbox { }) } + /// Create an independent local CoW child without a durable full snapshot. + fn branch<'py>(&self, py: Python<'py>, name: String) -> PyResult> { + let inner = self.inner.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let sandbox = Self::clone_sandbox(&inner).await?; + Ok(PySandbox::from_rust( + sandbox.branch(name).await.map_err(to_py_err)?, + )) + }) + } + + /// Suspend this resident VM without releasing RAM. + fn pause<'py>(&self, py: Python<'py>) -> PyResult> { + let inner = self.inner.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let sandbox = Self::clone_sandbox(&inner).await?; + sandbox.pause().await.map_err(to_py_err)?; + Ok(()) + }) + } + + /// Resume the same resident VM and its workloads. + fn resume<'py>(&self, py: Python<'py>) -> PyResult> { + let inner = self.inner.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let sandbox = Self::clone_sandbox(&inner).await?; + sandbox.resume().await.map_err(to_py_err)?; + Ok(()) + }) + } + /// Request graceful shutdown without waiting. fn request_stop<'py>(&self, py: Python<'py>) -> PyResult> { let inner = self.inner.clone(); diff --git a/sdk/python/src/sandbox_handle.rs b/sdk/python/src/sandbox_handle.rs index ce91c5c2d..ebc008efd 100644 --- a/sdk/python/src/sandbox_handle.rs +++ b/sdk/python/src/sandbox_handle.rs @@ -394,6 +394,37 @@ impl PySandboxHandle { }) } + /// Create an independent local CoW child without a durable full snapshot. + fn branch<'py>(&self, py: Python<'py>, name: String) -> PyResult> { + let inner = self.inner.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let guard = inner.lock().await; + Ok(PySandbox::from_rust( + guard.branch(name).await.map_err(to_py_err)?, + )) + }) + } + + /// Suspend this resident VM without releasing RAM. + fn pause<'py>(&self, py: Python<'py>) -> PyResult> { + let inner = self.inner.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let guard = inner.lock().await; + guard.pause().await.map_err(to_py_err)?; + Ok(()) + }) + } + + /// Resume the same resident VM and its workloads. + fn resume<'py>(&self, py: Python<'py>) -> PyResult> { + let inner = self.inner.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let guard = inner.lock().await; + guard.resume().await.map_err(to_py_err)?; + Ok(()) + }) + } + /// Request graceful shutdown without waiting. fn request_stop<'py>(&self, py: Python<'py>) -> PyResult> { let inner = self.inner.clone(); @@ -525,7 +556,7 @@ impl PySandboxHandle { }) } - /// Snapshot this (stopped) sandbox under a bare name. Resolves + /// Snapshot this sandbox's disk under a bare name, preserving its running/paused state. Resolves /// under `~/.microsandbox/snapshots//`. Move artifacts with /// `Snapshot.save`/`Snapshot.load`. fn snapshot<'py>(&self, py: Python<'py>, name: String) -> PyResult> { diff --git a/sdk/python/src/snapshot.rs b/sdk/python/src/snapshot.rs index 1a3859341..207d959f4 100644 --- a/sdk/python/src/snapshot.rs +++ b/sdk/python/src/snapshot.rs @@ -4,7 +4,7 @@ use std::path::PathBuf; use pyo3::prelude::*; use pyo3::types::PyDict; -use microsandbox::snapshot::SaveOpts as RustSaveOpts; +use microsandbox::snapshot::{LoadOpts as RustLoadOpts, SaveOpts as RustSaveOpts}; use microsandbox::{ Snapshot as RustSnapshot, SnapshotArchive as RustSnapshotArchive, SnapshotFormat as RustSnapshotFormat, SnapshotHandle as RustSnapshotHandle, @@ -42,17 +42,19 @@ pub struct PySnapshotHandle { #[pymethods] impl PySnapshot { - /// Create a disk snapshot from a stopped sandbox or a full snapshot from a running one. + /// Create a disk snapshot, or include memory and execution state with full=True. /// - /// The artifact is created under `~/.microsandbox/snapshots//`, - /// or under `dest_dir=` when given; move artifacts with `save`/`load`. + /// The artifact is installed in a snapshot group under the default snapshots + /// directory or `dest_dir`. Omitted member names are generated; the group + /// defaults to the source sandbox's name. // PyO3 kwargs map one-to-one onto function parameters; the count is the contract. #[allow(clippy::too_many_arguments)] #[staticmethod] #[pyo3(signature = ( - name, + name = "".to_string(), *, from_sandbox, + group = None, dest_dir = None, labels = None, force = false, @@ -63,6 +65,7 @@ impl PySnapshot { py: Python<'py>, name: String, from_sandbox: String, + group: Option, dest_dir: Option, labels: Option>, force: bool, @@ -71,6 +74,9 @@ impl PySnapshot { ) -> PyResult> { pyo3_async_runtimes::tokio::future_into_py(py, async move { let mut builder = RustSnapshot::builder(name).from_sandbox(&from_sandbox); + if let Some(group) = group { + builder = builder.group(group); + } if let Some(dest_dir) = dest_dir { builder = builder.dest_dir(dest_dir); } @@ -101,6 +107,7 @@ impl PySnapshot { archive, *, from_sandbox, + group = None, labels = None, force = false, record_integrity = false, @@ -112,6 +119,7 @@ impl PySnapshot { name: String, archive: PathBuf, from_sandbox: String, + group: Option, labels: Option>, force: bool, record_integrity: bool, @@ -120,6 +128,9 @@ impl PySnapshot { ) -> PyResult> { pyo3_async_runtimes::tokio::future_into_py(py, async move { let mut builder = RustSnapshot::builder(name).from_sandbox(from_sandbox); + if let Some(group) = group { + builder = builder.group(group); + } if let Some(labels) = labels { for (key, value) in labels { builder = builder.label(key, value); @@ -142,7 +153,7 @@ impl PySnapshot { }) } - /// Open an existing snapshot artifact by path or bare name. + /// Open a snapshot by path, group head, or `group:member` selector. /// Cheap metadata validation only — does not read the upper file. #[staticmethod] fn open<'py>(py: Python<'py>, path_or_name: String) -> PyResult> { @@ -228,6 +239,9 @@ impl PySnapshot { /// Bundle a snapshot into a `.tar.zst` archive. /// + /// `since` omits disk layers and RAM objects supplied by the base; `last_layers` only + /// selects disk layers. Dependent archives require a base when loaded or restored. + /// /// The recorded manifest is archived as-is, so create the snapshot /// with `record_integrity=True` if receivers must verify content. // PyO3 kwargs map one-to-one onto function parameters; preserve the public keyword contract. @@ -272,24 +286,72 @@ impl PySnapshot { /// snapshots directory, preserving recorded integrity for explicit /// verification. #[staticmethod] - #[pyo3(signature = (archive, *, dest = None, base = None))] + #[pyo3(signature = (archive, *, dest = None, base = None, group = None, set_head = false))] fn load<'py>( py: Python<'py>, archive: PathBuf, dest: Option, base: Option, + group: Option, + set_head: bool, ) -> PyResult> { pyo3_async_runtimes::tokio::future_into_py(py, async move { - let h = if let Some(base) = base { - RustSnapshot::load_with_base(&archive, dest.as_deref(), &base).await - } else { - RustSnapshot::load(&archive, dest.as_deref()).await - } + let h = RustSnapshot::load_with_options( + &archive, + RustLoadOpts { + dest, + base, + group, + set_head, + }, + ) + .await .map_err(to_py_err)?; Ok(PySnapshotHandle::from_rust(h)) }) } + /// Import archives together, resolving dependencies within the batch and destination group. + #[staticmethod] + #[pyo3(signature = (archives, *, dest = None, base = None, group = None, set_head = false))] + fn load_many<'py>( + py: Python<'py>, + archives: Vec, + dest: Option, + base: Option, + group: Option, + set_head: bool, + ) -> PyResult> { + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let handles = RustSnapshot::load_many( + &archives, + RustLoadOpts { + dest, + base, + group, + set_head, + }, + ) + .await + .map_err(to_py_err)?; + Ok(handles + .into_iter() + .map(PySnapshotHandle::from_rust) + .collect::>()) + }) + } + + /// Read a group's head, or select `group:member` as its head. + #[staticmethod] + fn group_head<'py>(py: Python<'py>, selector: String) -> PyResult> { + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let update = RustSnapshot::group_head(&selector) + .await + .map_err(to_py_err)?; + Python::with_gil(|py| head_update_to_py(py, &update)) + }) + } + //---------------------------------------------------------------------------------------------- // Instance accessors //---------------------------------------------------------------------------------------------- @@ -300,6 +362,15 @@ impl PySnapshot { self.inner.path().display().to_string() } + /// Outcome of the group head update performed by this capture. + #[getter] + fn head_update(&self, py: Python<'_>) -> PyResult>> { + self.inner + .head_update() + .map(|update| head_update_to_py(py, update)) + .transpose() + } + /// Canonical content digest (`sha256:hex`). The snapshot's identity. #[getter] fn id(&self) -> &str { @@ -492,6 +563,20 @@ impl PySnapshot { #[pymethods] impl PySnapshotHandle { + /// Local group containing this indexed snapshot. + #[getter] + fn group(&self) -> Option<&str> { + self.inner.group() + } + + /// Outcome of the group head update performed by this import. + #[getter] + fn head_update(&self, py: Python<'_>) -> PyResult>> { + self.inner + .head_update() + .map(|update| head_update_to_py(py, update)) + .transpose() + } #[getter] fn id(&self) -> &str { self.inner.id() @@ -611,6 +696,25 @@ impl PySnapshotHandle { // Functions: Helpers //-------------------------------------------------------------------------------------------------- +fn head_update_to_py( + py: Python<'_>, + update: µsandbox::snapshot::HeadUpdate, +) -> PyResult> { + let result = PyDict::new(py); + result.set_item("group", &update.group)?; + result.set_item("previous", &update.previous)?; + result.set_item("head", &update.head)?; + // Preserve the stable reason spelling used by all serialized API surfaces. + let reason = serde_json::to_value(update.reason) + .map_err(|error| pyo3::exceptions::PyRuntimeError::new_err(error.to_string()))?; + let reason = reason.as_str().ok_or_else(|| { + pyo3::exceptions::PyRuntimeError::new_err("snapshot head reason is not a string") + })?; + result.set_item("reason", reason)?; + result.set_item("changed", update.changed)?; + Ok(result.unbind()) +} + fn format_str(f: RustSnapshotFormat) -> &'static str { match f { RustSnapshotFormat::Raw => "raw", diff --git a/sdk/python/tests/test_cow_lifecycle.py b/sdk/python/tests/test_cow_lifecycle.py new file mode 100644 index 000000000..77b6d1313 --- /dev/null +++ b/sdk/python/tests/test_cow_lifecycle.py @@ -0,0 +1,50 @@ +"""Opt-in live CoW lifecycle check using a matching runtime/kernel bundle.""" + +import os +from pathlib import Path + +import pytest + +from microsandbox import Sandbox, Snapshot + + +@pytest.mark.skipif(os.environ.get("MSB_COW_LIVE") != "1", reason="requires matching live bundle") +@pytest.mark.asyncio +async def test_cow_resident_capture_and_child_isolation(): + name = f"cow8-python-{os.getpid()}" + source = await Sandbox.create( + name, image="alpine", memory=256 + ) + child = None + branches = [] + try: + await source.exec("sh", ["-c", "echo source > /dev/shm/sdk-marker"]) + await source.pause() + paused = await Sandbox.get(name) + assert str(paused.status) == "paused" + branched = await paused.branch(f"{name}-paused-branch") + branches.append(branched) + assert (await branched.exec("cat", ["/dev/shm/sdk-marker"])).stdout_text.strip() == "source" + snapshot = await Snapshot.create(f"{name}-full", from_sandbox=name, full=True) + assert (Path(snapshot.path) / "snapshot.json").is_file() + await paused.resume() + # The returned artifact path selects the exact member in its snapshot group. + child = await Sandbox.create( + f"{name}-child", from_snapshot=snapshot.path, forked=True + ) + result = await child.exec("cat", ["/dev/shm/sdk-marker"]) + assert result.stdout_text.strip() == "source" + await child.exec("sh", ["-c", "echo child > /dev/shm/sdk-marker"]) + descendant = await child.branch(f"{name}-branch") + branches.append(descendant) + result = await descendant.exec("cat", ["/dev/shm/sdk-marker"]) + assert result.stdout_text.strip() == "child" + result = await source.exec("cat", ["/dev/shm/sdk-marker"]) + assert result.stdout_text.strip() == "source" + await child.pause() + finally: + for branched in reversed(branches): + await branched.stop() + if child is not None: + await child.stop() + await source.stop() diff --git a/sdk/python/tests/test_create_stub.py b/sdk/python/tests/test_create_stub.py index 2c65b546c..e6d390c48 100644 --- a/sdk/python/tests/test_create_stub.py +++ b/sdk/python/tests/test_create_stub.py @@ -12,6 +12,7 @@ "from_snapshot", "disk_only", "snapshot_base", + "forked", "memory", "cpus", "max_memory", @@ -86,6 +87,7 @@ def test_create_closed_values_are_precisely_typed() -> None: } assert annotations["security"] == "SecurityProfile | None" + assert annotations["forked"] == "bool" assert annotations["init"] == "str | InitConfig | InitOptions | None" assert annotations["pull_policy"] == "PullPolicy | None" assert annotations["log_level"] == "LogLevel | None" diff --git a/sdk/python/tests/test_snapshot_stub.py b/sdk/python/tests/test_snapshot_stub.py new file mode 100644 index 000000000..e23ca6a1d --- /dev/null +++ b/sdk/python/tests/test_snapshot_stub.py @@ -0,0 +1,28 @@ +"""Verify the typed single-archive and batch snapshot import contracts.""" + +from __future__ import annotations + +import ast +from pathlib import Path + + +def test_batch_load_preserves_single_load_options_and_returns_handles() -> None: + stub = Path(__file__).parent.parent / "microsandbox" / "_microsandbox.pyi" + tree = ast.parse(stub.read_text()) + snapshot = next( + node for node in tree.body if isinstance(node, ast.ClassDef) and node.name == "Snapshot" + ) + methods = { + node.name: node for node in snapshot.body if isinstance(node, ast.AsyncFunctionDef) + } + single, batch = methods["load"], methods["load_many"] + assert [arg.arg for arg in batch.args.args] == ["archives"] + assert ast.unparse(batch.args.args[0].annotation) == "Sequence[str | os.PathLike[str]]" + assert [arg.arg for arg in batch.args.kwonlyargs] == [ + arg.arg for arg in single.args.kwonlyargs + ] == ["dest", "base", "group", "set_head"] + assert [ast.dump(value) for value in batch.args.kw_defaults] == [ + ast.dump(value) for value in single.args.kw_defaults + ] + assert ast.unparse(single.returns) == "SnapshotHandle" + assert ast.unparse(batch.returns) == "list[SnapshotHandle]" diff --git a/sdk/python/tests/test_types_enums.py b/sdk/python/tests/test_types_enums.py index 4342a0838..e7ede0707 100644 --- a/sdk/python/tests/test_types_enums.py +++ b/sdk/python/tests/test_types_enums.py @@ -292,7 +292,8 @@ def test_sandbox_create_treats_explicit_none_as_omitted() -> None: with pytest.raises(ValueError, match="image= or from_snapshot= is required"): Sandbox.create("explicit-none-image", image=None) - with pytest.raises(FileNotFoundError, match="snapshot artifact not found"): + # Selector lookup is async; type/options validation still happens before making the future. + with pytest.raises(type(baseline.value)): Sandbox.create( "explicit-none-image-with-snapshot", image=None, @@ -300,6 +301,19 @@ def test_sandbox_create_treats_explicit_none_as_omitted() -> None: ) +@pytest.mark.asyncio +@pytest.mark.parametrize("selector", ["missing-group", "missing-group:missing-member"]) +async def test_missing_snapshot_selector_is_reported_when_awaited(selector: str) -> None: + with pytest.raises(FileNotFoundError): + await Sandbox.create("missing-snapshot-source", image=None, from_snapshot=selector) + + +@pytest.mark.asyncio +async def test_missing_snapshot_pathlike_is_reported_when_awaited(tmp_path) -> None: + with pytest.raises(FileNotFoundError): + await Sandbox.create("missing-snapshot-path", from_snapshot=tmp_path / "missing") + + def test_inactive_mount_enum_fields_are_still_validated() -> None: config = MountConfig( kind=MountKind.BIND, diff --git a/sdk/rust/lib/backend/cloud/sandbox.rs b/sdk/rust/lib/backend/cloud/sandbox.rs index 11ae7be33..11263548a 100644 --- a/sdk/rust/lib/backend/cloud/sandbox.rs +++ b/sdk/rust/lib/backend/cloud/sandbox.rs @@ -341,6 +341,12 @@ impl TryFrom for CloudCreateBody { /// Build the cloud create body from an SDK config, rejecting the /// create-time options the cloud does not accept. fn try_from(mut config: SandboxConfig) -> MicrosandboxResult { + if config.forked { + return Err(MicrosandboxError::unsupported( + Operation::SandboxCreate, + UnsupportedReason::ConfigField("forked"), + )); + } if config.replace_existing { return Err(MicrosandboxError::unsupported( Operation::SandboxCreate, diff --git a/sdk/rust/lib/backend/local/control_lookup.rs b/sdk/rust/lib/backend/local/control_lookup.rs new file mode 100644 index 000000000..18f9e70c8 --- /dev/null +++ b/sdk/rust/lib/backend/local/control_lookup.rs @@ -0,0 +1,388 @@ +//! Read-only lookup for a live control target, without unrelated snapshot reconciliation. + +use std::time::Duration; + +use microsandbox_db::DbReadConnection; +use microsandbox_migration::schema_metadata; +use sea_orm::{ColumnTrait, ConnectionTrait, DatabaseBackend, EntityTrait, QueryFilter, Statement}; + +use super::{ + LocalBackend, acquire_migration_lock, refuse_incomplete_self_downgrade, refuse_schema_ahead, +}; +use crate::db::entity::sandbox as sandbox_entity; +use crate::sandbox::SandboxStatus; +use crate::sandbox::identity::SandboxRunIdentity; +use crate::{MicrosandboxError, MicrosandboxResult}; + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl LocalBackend { + /// Select the exact live run while the caller owns the name's transition guard. + pub(crate) async fn control_run_identity( + &self, + name: &str, + expected_id: i32, + ) -> MicrosandboxResult { + if let Some((model, run)) = self.try_control_target(name).await? { + if model.id != expected_id { + return Err(MicrosandboxError::SandboxReplaced { + name: name.into(), + expected: format!("local:{expected_id}"), + actual: format!("local:{}", model.id), + }); + } + return Ok(run); + } + let (model, _) = self.sandbox_handle_state(name, Some(expected_id)).await?; + let run = Self::load_active_run(self.db().await?.read(), model.id).await?; + let run = run + .filter(|run| run.pid.is_some_and(Self::pid_is_alive)) + .ok_or_else(|| { + MicrosandboxError::SandboxNotRunning(format!( + "sandbox {name:?} has no live runtime" + )) + })?; + Ok(SandboxRunIdentity { + sandbox_id: model.id, + run_id: run.id, + pid: run.pid.expect("live run has a PID"), + }) + } + + /// A runtime restart must not redirect a control request selected for its predecessor. + pub(crate) async fn validate_control_run( + &self, + name: &str, + expected: SandboxRunIdentity, + ) -> MicrosandboxResult<()> { + let current = self.control_run_identity(name, expected.sandbox_id).await?; + if current != expected { + return Err(MicrosandboxError::Runtime(format!( + "sandbox {name:?} changed runtime during control operation" + ))); + } + Ok(()) + } + + /// Return a healthy live target from a current catalog. `None` requests the existing + /// migration/stale-runtime path; errors must never become an unvalidated fast path. + pub(crate) async fn try_control_handle_state( + &self, + name: &str, + ) -> MicrosandboxResult)>> { + Ok(self + .try_control_target(name) + .await? + .map(|(model, run)| (model, Some(run.pid)))) + } + + /// Keep authoritative control selection on the same read-only path as CLI lookup. + async fn try_control_target( + &self, + name: &str, + ) -> MicrosandboxResult> { + let db_dir = self.config().home().join(microsandbox_utils::DB_SUBDIR); + let db_path = db_dir.join(microsandbox_utils::DB_FILENAME); + match std::fs::metadata(&db_path) { + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.into()), + } + // Use the same installation coordination as the slow path. In particular, a rolled + // back database can look current while an incomplete downgrade still owns its files. + let _migration_lock = acquire_migration_lock(&db_dir).await?; + refuse_incomplete_self_downgrade(&db_dir)?; + let database = &self.config().database; + let read = if let Some(pools) = self.db.get() { + pools.read().clone() + } else { + DbReadConnection::open_read_only( + &db_path, + Duration::from_secs(database.connect_timeout_secs), + Duration::from_secs(database.busy_timeout_secs), + ) + .await + .map_err(|error| { + MicrosandboxError::Custom(format!( + "read control catalog {}: {error}", + db_path.display() + )) + })? + }; + microsandbox_runtime::maintenance::refuse_if_install_exclusive_held(&read) + .await + .map_err(|error| MicrosandboxError::Runtime(error.to_string()))?; + refuse_schema_ahead(read.inner()).await?; + let row = match read + .query_one_raw(Statement::from_string( + DatabaseBackend::Sqlite, + "SELECT COUNT(*) FROM seaql_migrations", + )) + .await + { + Ok(Some(row)) => row, + Ok(None) => return Ok(None), + Err(error) if super::is_missing_migrations_table(&error) => return Ok(None), + Err(error) => return Err(error.into()), + }; + if row.try_get_by_index::(0)? != schema_metadata::migration_ids().count() as i64 { + return Ok(None); + } + let model = sandbox_entity::Entity::find() + .filter(sandbox_entity::Column::Name.eq(name)) + .one(&read) + .await? + .ok_or_else(|| MicrosandboxError::SandboxNotFound(name.into()))?; + if !matches!( + model.status, + SandboxStatus::Running | SandboxStatus::Draining + ) { + return Ok(None); + } + let run = Self::load_active_run(&read, model.id).await?; + let pid = Self::pid_from_run(run.as_ref()); + // Do not clean up sockets from this read-only observation. The slow path rechecks + // the exact row/run under lifecycle ownership before touching stale artifacts. + if let (Some(run), Some(pid)) = (run, pid) { + let identity = SandboxRunIdentity { + sandbox_id: model.id, + run_id: run.id, + pid, + }; + Ok(Some((model, identity))) + } else { + Ok(None) + } + } +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + async fn fixture() -> (tempfile::TempDir, LocalBackend) { + let home = tempfile::tempdir().unwrap(); + let backend = LocalBackend::builder().home(home.path()).build_lazy(); + let pools = backend.db().await.unwrap(); + pools.write().execute_unprepared( + "INSERT INTO sandbox (id, name, config, status, ephemeral) VALUES (1, 'source', '{}', 'Running', 0)", + ).await.unwrap(); + pools + .write() + .execute_raw(Statement::from_sql_and_values( + DatabaseBackend::Sqlite, + "INSERT INTO run (sandbox_id, pid, status) VALUES (1, ?, 'Running')", + [i64::from(std::process::id()).into()], + )) + .await + .unwrap(); + (home, backend) + } + + #[tokio::test] + async fn healthy_lookup_does_not_initialize_writer_or_reconcile_snapshots() { + let (home, _) = fixture().await; + // A non-directory snapshot path would fail normal reconciliation. The live + // control path has no reason to inspect it or mutate the target's catalog. + let snapshots = home.path().join("snapshots"); + if snapshots.exists() { + std::fs::remove_dir(&snapshots).unwrap(); + } + std::fs::write(&snapshots, b"unrelated snapshot inventory").unwrap(); + let backend = LocalBackend::builder().home(home.path()).build_lazy(); + let (model, pid) = backend + .try_control_handle_state("source") + .await + .unwrap() + .unwrap(); + assert_eq!(model.id, 1); + assert_eq!(pid, Some(std::process::id() as i32)); + assert!(backend.db.get().is_none()); + let run = backend + .control_run_identity("source", model.id) + .await + .unwrap(); + backend.validate_control_run("source", run).await.unwrap(); + assert!( + backend.db.get().is_none(), + "control run validation must stay read-only" + ); + assert_eq!( + std::fs::read(snapshots).unwrap(), + b"unrelated snapshot inventory" + ); + } + + #[tokio::test] + async fn branch_refuses_a_restarted_source_before_writing_handoff_state() { + let (home, backend) = fixture().await; + let selected = backend.control_run_identity("source", 1).await.unwrap(); + let writer = backend.db().await.unwrap().write(); + writer + .execute_unprepared("UPDATE run SET status = 'Terminated' WHERE sandbox_id = 1") + .await + .unwrap(); + writer + .execute_raw(Statement::from_sql_and_values( + DatabaseBackend::Sqlite, + "INSERT INTO run (sandbox_id, pid, status) VALUES (1, ?, 'Running')", + [i64::from(std::process::id()).into()], + )) + .await + .unwrap(); + assert_ne!( + selected.run_id, + backend + .control_run_identity("source", 1) + .await + .unwrap() + .run_id + ); + let mut child = crate::sandbox::SandboxConfig::default(); + child.spec.name = "child".into(); + let child_dir = home.path().join("child"); + std::fs::create_dir(&child_dir).unwrap(); + let result = crate::sandbox::branch::capture_child( + &backend, + &mut child, + &crate::sandbox::identity::BranchSource { + name: "source".into(), + run: selected, + }, + &child_dir, + ) + .await; + assert!(result.unwrap_err().to_string().contains("changed runtime")); + assert_eq!(std::fs::read_dir(child_dir).unwrap().count(), 0); + } + + #[tokio::test] + async fn absent_catalog_and_terminal_target_request_slow_path() { + let home = tempfile::tempdir().unwrap(); + let empty = LocalBackend::builder().home(home.path()).build_lazy(); + assert!( + empty + .try_control_handle_state("source") + .await + .unwrap() + .is_none() + ); + assert!(!home.path().join("db").exists()); + let (_home, backend) = fixture().await; + backend + .db() + .await + .unwrap() + .write() + .execute_unprepared("UPDATE sandbox SET status = 'Stopped' WHERE id = 1") + .await + .unwrap(); + assert!( + backend + .try_control_handle_state("source") + .await + .unwrap() + .is_none() + ); + } + + #[tokio::test] + async fn schema_and_install_gates_are_not_bypassed() { + let (home, backend) = fixture().await; + let writer = backend.db().await.unwrap().write(); + writer.execute_unprepared( + "INSERT INTO seaql_migrations (version, applied_at) VALUES ('future_unknown_migration', 0)", + ).await.unwrap(); + assert!( + backend + .try_control_handle_state("source") + .await + .unwrap_err() + .to_string() + .contains("schema is newer") + ); + writer + .execute_unprepared( + "DELETE FROM seaql_migrations WHERE version = 'future_unknown_migration'", + ) + .await + .unwrap(); + writer.execute_raw(Statement::from_sql_and_values( + DatabaseBackend::Sqlite, + "INSERT OR REPLACE INTO maintenance_lease (name, holder_pid, lease_expires_at) VALUES ('install_exclusive', ?, ?)", + [(std::process::id() as i32).into(), (chrono::Utc::now().naive_utc() + chrono::Duration::minutes(10)).into()], + )).await.unwrap(); + assert!( + backend + .try_control_handle_state("source") + .await + .unwrap_err() + .to_string() + .contains("install operation in progress") + ); + writer + .execute_unprepared("DELETE FROM maintenance_lease WHERE name = 'install_exclusive'") + .await + .unwrap(); + let journal_dir = home.path().join("db/self-downgrade/test-operation"); + std::fs::create_dir_all(&journal_dir).unwrap(); + std::fs::write( + journal_dir.join("journal.json"), + br#"{"phase":"preparing"}"#, + ) + .unwrap(); + assert!( + backend + .try_control_handle_state("source") + .await + .unwrap_err() + .to_string() + .contains("self_downgrade_recovery_required") + ); + } + + #[tokio::test] + async fn older_schema_and_missing_active_run_fall_back_without_cleanup() { + let (home, backend) = fixture().await; + let writer = backend.db().await.unwrap().write(); + writer + .execute_unprepared("DELETE FROM run WHERE sandbox_id = 1") + .await + .unwrap(); + let runtime = home.path().join("sandboxes/source/runtime"); + std::fs::create_dir_all(&runtime).unwrap(); + let hint = runtime.join("runtime-boot-id"); + std::fs::write(&hint, b"do-not-clean-during-observation").unwrap(); + assert!( + backend + .try_control_handle_state("source") + .await + .unwrap() + .is_none() + ); + assert!(hint.exists()); + let last = schema_metadata::migration_ids().last().unwrap(); + writer + .execute_raw(Statement::from_sql_and_values( + DatabaseBackend::Sqlite, + "DELETE FROM seaql_migrations WHERE version = ?", + [last.into()], + )) + .await + .unwrap(); + assert!( + backend + .try_control_handle_state("source") + .await + .unwrap() + .is_none() + ); + assert!(hint.exists()); + } +} diff --git a/sdk/rust/lib/backend/local/mod.rs b/sdk/rust/lib/backend/local/mod.rs index 72b460ba2..c06674cfb 100644 --- a/sdk/rust/lib/backend/local/mod.rs +++ b/sdk/rust/lib/backend/local/mod.rs @@ -15,6 +15,7 @@ //! the bulk of the old global config singleton plus the SQLite pool, so multiple //! backends can hold different configurations for tests / migrations. +mod control_lookup; mod sandbox; use std::{ @@ -129,6 +130,15 @@ impl LocalBackend { profile: Option, ) -> Self { let config = load_persisted_config_or_default().unwrap_or_default(); + Self::lazy_with_config(config, selection_source, profile) + } + + /// Reuse the configuration document already read by ambient profile resolution. + pub(crate) fn lazy_with_config( + config: GlobalConfig, + selection_source: BackendSelectionSource, + profile: Option, + ) -> Self { Self { config: Arc::new(config), db: OnceCell::new(), diff --git a/sdk/rust/lib/backend/local/sandbox/create.rs b/sdk/rust/lib/backend/local/sandbox/create.rs index 7d62a68bf..9058537f4 100644 --- a/sdk/rust/lib/backend/local/sandbox/create.rs +++ b/sdk/rust/lib/backend/local/sandbox/create.rs @@ -17,7 +17,6 @@ use microsandbox_image::{ PullResult, Reference, Registry, ext4, tree, }; use sea_orm::{ColumnTrait, ConnectionTrait, EntityTrait, QueryFilter, Set, sea_query::Expr}; -use sha2::{Digest as _, Sha256}; use tokio::sync::Mutex; use super::LocalBackend; @@ -169,6 +168,18 @@ impl LocalBackend { Self::acquire_sandbox_transition_guard(&self.config().run_dir(), &config.spec.name) .await?; Self::prepare_create_target(db, &config, &sandbox_dir, &self.config().run_dir()).await?; + // Hold the existing lifecycle lock across reservation, capture and spawn. Recheck + // under the lock so two creates cannot both own the same child staging directory. + let lifecycle_guard = crate::runtime::acquire_sandbox_lifecycle_guard( + &self.config().run_dir(), + &config.spec.name, + std::time::Duration::from_secs(5), + ) + .await?; + let mut reserved_config = config.clone(); + reserved_config.replace_existing = false; + Self::prepare_create_target(db, &reserved_config, &sandbox_dir, &self.config().run_dir()) + .await?; let mut child_stage_guard = None; // Preserve only the installed-snapshot source that existed on entry. Direct archive // materialization below installs its checkpoint closure directly into child staging, so @@ -177,6 +188,16 @@ impl LocalBackend { let installed_checkpoint_restore = config.checkpoint_restore.take(); let installed_file_sources = std::mem::take(&mut config.snapshot_root_layer_sources); let installed_file_virtual_size = config.snapshot_root_virtual_size.take(); + let _branch_pin = if let Some(source) = config.branch_source.take() { + tokio::fs::create_dir(&sandbox_dir).await?; + child_stage_guard = Some(ChildStageGuard::new(sandbox_dir.clone())); + Some( + crate::sandbox::branch::capture_child(self, &mut config, &source, &sandbox_dir) + .await?, + ) + } else { + None + }; // A direct archive restore streams its layer into the ordinary child // staging location before image resolution. The archive supplies the @@ -195,6 +216,7 @@ impl LocalBackend { )) .await?; config.spec.image = RootfsSource::oci(materialized.manifest.image.reference.clone()); + config.snapshot_parent = Some(materialized.manifest.snapshot_id.to_string()); config.manifest_digest = Some(materialized.manifest.image.manifest_digest.clone()); crate::sandbox::apply_snapshot_root_layout( &mut config, @@ -224,7 +246,7 @@ impl LocalBackend { crate::sandbox::apply_checkpoint_restore_constraints( &mut config, state, - &closure, + closure.checkpoint(), overrides, )?; config.checkpoint_restore = Some(restore); @@ -276,6 +298,13 @@ impl LocalBackend { } } } + // Archive descriptors are resolved here, after the builder's initial validation. + // Do not let a disk archive turn an explicit CoW restore into a fresh boot. + if config.forked && config.checkpoint_restore.is_none() { + return Err(crate::MicrosandboxError::InvalidConfig( + "forked requires a full snapshot restore".into(), + )); + } if !installed_file_sources.is_empty() { child_stage_guard = Some(ChildStageGuard::new(sandbox_dir.clone())); let virtual_size = installed_file_virtual_size.ok_or_else(|| { @@ -593,7 +622,9 @@ impl LocalBackend { // Claim the persisted identity in Starting state. Running is published only after the // guest agent and all create-time validation are ready for callers. let write_db = db.write(); - let persisted_config = config.clone_for_persistence(); + let mut persisted_config = config.clone_for_persistence(); + // Keep restore intent until activation succeeds so an interrupted restore cannot boot cold. + persisted_config.checkpoint_restore = config.checkpoint_restore.clone(); let sandbox_id = match Self::insert_starting_sandbox_record(write_db, &persisted_config).await { Ok(sandbox_id) => sandbox_id, @@ -616,17 +647,8 @@ impl LocalBackend { .as_ref() .map(|restore| restore.closure.clone()); let created = self - .create_sandbox_inner(config, sandbox_id, mode, None) + .create_sandbox_inner(config, sandbox_id, mode, Some(lifecycle_guard)) .await; - if let Some(closure) = restore_closure - && let Err(error) = remove_dir_if_exists(&closure) - { - tracing::warn!( - error = %error, - path = %closure.display(), - "failed to remove consumed eager checkpoint closure" - ); - } let (local_state, mut returned_config) = match created { Ok(pair) => pair, Err(e) => { @@ -647,7 +669,7 @@ impl LocalBackend { }; returned_config.checkpoint_restore = None; returned_config.snapshot_upper_layers.clear(); - let sandbox = Sandbox::from_local(backend.clone(), local_state, returned_config); + let mut sandbox = Sandbox::from_local(backend.clone(), local_state, returned_config); // This is the readiness publication boundary: create_sandbox_inner returns only after // the relay is connected and agentd has accepted its readiness handshake. if !Self::compare_and_set_sandbox_status( @@ -658,7 +680,7 @@ impl LocalBackend { ) .await? { - let _ = sandbox.stop().await; + sandbox.terminate_creation_owner().await; return Err(crate::MicrosandboxError::Runtime(format!( "sandbox {:?} lost its Starting state before readiness publication", sandbox.name() @@ -671,7 +693,7 @@ impl LocalBackend { ) .await { - let _ = sandbox.stop().await; + sandbox.terminate_creation_owner().await; return Err(err); } @@ -681,7 +703,7 @@ impl LocalBackend { ) && let Err(err) = Self::persist_oci_manifest_pin(write_db, sandbox_id, manifest_digest).await { - let _ = sandbox.stop().await; + sandbox.terminate_creation_owner().await; if created_named_volumes.is_empty() { let _ = Self::update_sandbox_status(write_db, sandbox_id, SandboxStatus::Stopped).await; @@ -699,7 +721,7 @@ impl LocalBackend { match sandbox.fs().stat(workdir).await { Ok(metadata) if metadata.kind == FsEntryKind::Directory => {} Ok(_) => { - let _ = sandbox.stop().await; + sandbox.terminate_creation_owner().await; if created_named_volumes.is_empty() { let _ = Self::update_sandbox_status( write_db, @@ -716,7 +738,7 @@ impl LocalBackend { ))); } Err(_) => { - let _ = sandbox.stop().await; + sandbox.terminate_creation_owner().await; if created_named_volumes.is_empty() { let _ = Self::update_sandbox_status( write_db, @@ -735,9 +757,49 @@ impl LocalBackend { } } + if let Some(closure) = restore_closure { + // Do not lose the recovery discriminator if any preceding creation check failed. + // RAM/device state has been consumed and the runtime owns its disk chain and pins. + if let Err(error) = Self::complete_sandbox_restore(write_db, sandbox_id).await { + sandbox.terminate_creation_owner().await; + return Err(error); + } + if let Err(error) = remove_dir_if_exists(&closure) { + tracing::warn!(error = %error, path = %closure.display(), "failed to remove consumed checkpoint closure"); + } + } + if matches!(mode, SpawnMode::Detached) { + sandbox.finish_detached_creation().await?; + } Ok(sandbox) } + /// Clear only the pending construction intent, preserving any concurrent desired edits. + async fn complete_sandbox_restore( + db: &DbWriteConnection, + sandbox_id: i32, + ) -> MicrosandboxResult<()> { + sandbox_entity::Entity::update_many() + .col_expr( + sandbox_entity::Column::Config, + Expr::cust("json_remove(config, '$.checkpoint_restore')"), + ) + .filter(sandbox_entity::Column::Id.eq(sandbox_id)) + .exec(db) + .await?; + Ok(()) + } + + pub(crate) fn validate_completed_restore(config: &SandboxConfig) -> MicrosandboxResult<()> { + if config.checkpoint_restore.is_some() { + return Err(crate::MicrosandboxError::InvalidConfig(format!( + "sandbox {:?} has an incomplete restore; remove and recreate it from the snapshot; refusing a cold boot", + config.spec.name + ))); + } + Ok(()) + } + /// Inner local create logic separated for error-cleanup wrapper. Returns /// the local-variant state plus the (possibly mutated) config. pub(super) async fn create_sandbox_inner( @@ -764,12 +826,9 @@ impl LocalBackend { "sandbox ready", ); } - let handle = if matches!(mode, SpawnMode::Detached) { - handle.disarm(); - None - } else { - Some(Arc::new(Mutex::new(handle))) - }; + // Even detached launches remain creator-owned until catalog publication and validation + // finish. Cancellation or failure before that boundary must terminate this exact child. + let handle = Some(Arc::new(Mutex::new(handle))); Ok(( crate::backend::SandboxLocalState { @@ -952,6 +1011,7 @@ impl LocalBackend { pinned_digest, pull_policy, registry_overrides, + materialization, progress, ) .await @@ -964,6 +1024,7 @@ impl LocalBackend { pinned_digest: &str, pull_policy: PullPolicy, registry_overrides: RegistryOverrides, + materialization: microsandbox_image::RootfsMaterialization, progress: Option, ) -> MicrosandboxResult { let manifest_digest: Digest = pinned_digest.parse().map_err(|e| { @@ -974,8 +1035,19 @@ impl LocalBackend { let pinned_reference = Self::digest_pinned_reference(reference, pinned_digest)?; let cache = GlobalCache::new_async(&self.cache_dir()).await?; - if let Some((pull_result, metadata)) = - Registry::pull_cached_by_manifest_digest(&cache, &manifest_digest).await? + let pinned_ref: Reference = pinned_reference.parse().map_err(|e| { + crate::MicrosandboxError::InvalidConfig(format!("invalid pinned reference: {e}")) + })?; + let original_ref: Reference = reference.parse().map_err(|e| { + crate::MicrosandboxError::InvalidConfig(format!("invalid image reference: {e}")) + })?; + if let Some((pull_result, metadata)) = Registry::pull_snapshot_cached( + &cache, + &[pinned_ref.clone(), original_ref], + &manifest_digest, + materialization, + ) + .await? { Self::emit_cached_pull_progress(progress.as_ref(), reference, &metadata); return Ok(ResolvedOciImage { @@ -992,6 +1064,33 @@ impl LocalBackend { ))); } + if materialization == microsandbox_image::RootfsMaterialization::Flat { + // The snapshot supplies the complete disk. Fetch its pinned image + // defaults, not a second root filesystem that will never be used. + let global = self.config(); + let auth = match registry_overrides.auth { + Some(auth) => auth, + None => global.resolve_registry_auth(pinned_ref.registry())?, + }; + let mut ca_certs = global.resolve_ca_certs().await?; + ca_certs.extend(registry_overrides.ca_certs); + let mut insecure = global.insecure_registries(); + if registry_overrides.insecure { + insecure.push(pinned_ref.registry().to_string()); + } + let registry = Registry::builder(microsandbox_image::Platform::host_linux(), cache) + .auth(auth) + .extra_ca_certs(ca_certs) + .add_insecure_registries(insecure) + .build()?; + let pull_result = registry.pull_snapshot_metadata(&pinned_ref).await?; + return Ok(ResolvedOciImage { + pull_result, + metadata_reference: pinned_reference, + cached_metadata: None, + }); + } + // Pull by digest, never by the mutable source tag, when the exact // snapshot base is absent from the local cache. let pull_result = match self @@ -1222,10 +1321,11 @@ impl LocalBackend { sandbox_dir.display() )) })?; - let model = Self::reconcile_sandbox_runtime_state_with_paths( + let model = Self::reconcile_sandbox_runtime_state_owned( pools, model, Some((run_dir, sandboxes_dir)), + true, ) .await?; let active = matches!( @@ -1237,6 +1337,8 @@ impl LocalBackend { .await?; } + let _lineage = + crate::snapshot::lineage::lock_source(run_dir, &config.spec.name).await?; let _guard = crate::runtime::acquire_sandbox_lifecycle_guard( run_dir, &config.spec.name, @@ -1263,6 +1365,7 @@ impl LocalBackend { return Ok(()); } + let _lineage = crate::snapshot::lineage::lock_source(run_dir, &config.spec.name).await?; let _guard = crate::runtime::acquire_sandbox_lifecycle_guard( run_dir, &config.spec.name, @@ -1629,11 +1732,7 @@ fn snapshot_root_layout_from_config( /// Derive a stable, filesystem-safe transition-lock path for one sandbox name. fn sandbox_transition_lock_path(run_dir: &Path, name: &str) -> PathBuf { - let digest = Sha256::digest(name.as_bytes()); - // Keep the original on-disk namespace so mixed-version processes still contend on one lock. - run_dir - .join("creation-locks") - .join(format!("{}.lock", hex::encode(&digest[..16]))) + microsandbox_runtime::ipc::sandbox_transition_lock_path(run_dir, name) } /// Probe every backward-compatible Unix endpoint before recovering an @@ -1941,6 +2040,54 @@ mod tests { fs::remove_dir(path).unwrap(); } + #[tokio::test] + async fn incomplete_restore_survives_failure_until_explicit_completion() { + let temp = tempdir().unwrap(); + let pools = open_test_pools(&temp.path().join("test.db")).await; + let mut config = test_config_with_rootfs("pending", bind_rootfs(temp.path().to_path_buf())); + config.checkpoint_restore = Some(microsandbox_runtime::launch::CheckpointRestoreConfig { + local_branch: false, + forked: true, + closure: temp.path().join("checkpoint"), + checkpoint_root: "blake3:pending".into(), + checkpoint_id: "pending".into(), + }); + let id = LocalBackend::insert_sandbox_record(pools.write(), &config) + .await + .unwrap(); + LocalBackend::update_sandbox_status(pools.write(), id, SandboxStatus::Stopped) + .await + .unwrap(); + let model = sandbox_entity::Entity::find_by_id(id) + .one(pools.read()) + .await + .unwrap() + .unwrap(); + let pending: SandboxConfig = serde_json::from_str(&model.config).unwrap(); + assert!( + LocalBackend::validate_completed_restore(&pending) + .unwrap_err() + .to_string() + .contains("refusing a cold boot") + ); + assert!(pending.checkpoint_restore.as_ref().unwrap().forked); + + // Ordinary post-success/snapshot projections must not perpetuate one-shot restore input. + assert!(pending.clone_for_persistence().checkpoint_restore.is_none()); + LocalBackend::complete_sandbox_restore(pools.write(), id) + .await + .unwrap(); + let model = sandbox_entity::Entity::find_by_id(id) + .one(pools.read()) + .await + .unwrap() + .unwrap(); + let completed: SandboxConfig = serde_json::from_str(&model.config).unwrap(); + assert!(completed.checkpoint_restore.is_none()); + assert_eq!(completed.spec.name, "pending"); + LocalBackend::validate_completed_restore(&completed).unwrap(); + } + #[tokio::test] async fn test_persist_oci_manifest_pin_upserts_rootfs_record() { let temp = tempdir().unwrap(); diff --git a/sdk/rust/lib/backend/local/sandbox/mod.rs b/sdk/rust/lib/backend/local/sandbox/mod.rs index 7f06a07ba..f4631b88f 100644 --- a/sdk/rust/lib/backend/local/sandbox/mod.rs +++ b/sdk/rust/lib/backend/local/sandbox/mod.rs @@ -160,6 +160,9 @@ impl LocalBackend { } let mut config: SandboxConfig = serde_json::from_str(&model.config)?; + // A failed or interrupted first restore is not a stopped ordinary VM. In particular, + // its sealed base may be hard-linked to a snapshot and must never become a boot disk. + Self::validate_completed_restore(&config)?; self.apply_deployment_profile(&mut config); config.apply_runtime_defaults(); validate_hostname(config.spec.runtime.hostname.as_deref())?; @@ -204,7 +207,8 @@ impl LocalBackend { .await { Ok((local_state, returned_config)) => { - let sandbox = Sandbox::from_local(backend.clone(), local_state, returned_config); + let mut sandbox = + Sandbox::from_local(backend.clone(), local_state, returned_config); // Publish Running only after create_sandbox_inner has completed the agent // readiness handshake, so concurrent connectors cannot race endpoint creation. if !Self::compare_and_set_sandbox_status( @@ -215,7 +219,7 @@ impl LocalBackend { ) .await? { - let _ = sandbox.stop().await; + sandbox.terminate_creation_owner().await; return Err(crate::MicrosandboxError::Runtime(format!( "sandbox {name:?} lost its Starting state before readiness publication" ))); @@ -227,9 +231,12 @@ impl LocalBackend { ) .await { - let _ = sandbox.stop().await; + sandbox.terminate_creation_owner().await; return Err(err); } + if matches!(mode, SpawnMode::Detached) { + sandbox.finish_detached_creation().await?; + } Ok(sandbox) } Err(err) => { @@ -256,7 +263,11 @@ impl LocalBackend { /// /// No-op when the sandbox isn't Starting, Running, or Draining. async fn stop_sandbox(&self, name: &str, expected_id: Option) -> MicrosandboxResult<()> { - let (model, pid) = self.sandbox_handle_state(name, expected_id).await?; + let _transition = + Self::acquire_sandbox_transition_guard(&self.config().run_dir(), name).await?; + let (model, pid) = self + .sandbox_handle_state_owned(name, expected_id, true) + .await?; if !matches!( model.status, SandboxStatus::Starting | SandboxStatus::Running | SandboxStatus::Draining @@ -296,7 +307,11 @@ impl LocalBackend { /// libkrun PID, waits briefly for the process to exit, then marks the DB /// row Stopped if all signalled PIDs are confirmed dead. async fn kill_sandbox(&self, name: &str, expected_id: Option) -> MicrosandboxResult<()> { - let (model, pid) = self.sandbox_handle_state(name, expected_id).await?; + let _transition = + Self::acquire_sandbox_transition_guard(&self.config().run_dir(), name).await?; + let (model, pid) = self + .sandbox_handle_state_owned(name, expected_id, true) + .await?; if !matches!( model.status, SandboxStatus::Starting | SandboxStatus::Running | SandboxStatus::Draining @@ -340,7 +355,11 @@ impl LocalBackend { /// `core.shutdown` agent message so the guest can sync and power off /// without pretending a direct process termination is graceful. async fn drain_sandbox(&self, name: &str, expected_id: Option) -> MicrosandboxResult<()> { - let (model, pid) = self.sandbox_handle_state(name, expected_id).await?; + let _transition = + Self::acquire_sandbox_transition_guard(&self.config().run_dir(), name).await?; + let (model, pid) = self + .sandbox_handle_state_owned(name, expected_id, true) + .await?; if model.status != SandboxStatus::Running && model.status != SandboxStatus::Draining { return Ok(()); } @@ -394,10 +413,20 @@ impl LocalBackend { } /// Load the local DB row + active PID for a sandbox handle. - async fn sandbox_handle_state( + pub(crate) async fn sandbox_handle_state( &self, name: &str, expected_id: Option, + ) -> MicrosandboxResult<(sandbox_entity::Model, Option)> { + self.sandbox_handle_state_owned(name, expected_id, false) + .await + } + + async fn sandbox_handle_state_owned( + &self, + name: &str, + expected_id: Option, + transition_owned: bool, ) -> MicrosandboxResult<(sandbox_entity::Model, Option)> { let pools = self.db().await?; let model = sandbox_entity::Entity::find() @@ -406,7 +435,13 @@ impl LocalBackend { .await? .ok_or_else(|| crate::MicrosandboxError::SandboxNotFound(name.into()))?; ensure_local_identity(name, expected_id, model.id)?; - let model = self.reconcile_sandbox_runtime_state(pools, model).await?; + let model = Self::reconcile_sandbox_runtime_state_owned( + pools, + model, + Some((&self.config().run_dir(), &self.sandboxes_dir())), + transition_owned, + ) + .await?; let run = Self::load_active_run(pools.read(), model.id).await?; let pid = Self::pid_from_run(run.as_ref()); Ok((model, pid)) @@ -497,7 +532,13 @@ impl LocalBackend { ))); } - if let RootfsSource::Oci(_) = &config.spec.image + // Flat roots own their disk and never boot through the OCI VMDK. + // Metadata-only snapshot restores deliberately do not populate it. + if let RootfsSource::Oci(oci) = &config.spec.image + && !matches!( + oci.root_disk.as_ref(), + Some(crate::sandbox::RootDisk::Flat { .. }) + ) && let Some(ref digest_str) = config.manifest_digest { let cache_dir = self.cache_dir(); @@ -538,7 +579,13 @@ impl LocalBackend { name: &str, ) -> MicrosandboxResult { let sandbox = load_sandbox_record(pools.read(), name).await?; - self.reconcile_sandbox_runtime_state(pools, sandbox).await + Self::reconcile_sandbox_runtime_state_owned( + pools, + sandbox, + Some((&self.config().run_dir(), &self.sandboxes_dir())), + true, + ) + .await } /// Reconcile a Starting/Running/Draining row against the owning process's @@ -563,6 +610,16 @@ impl LocalBackend { pools: &DbPools, sandbox: sandbox_entity::Model, socket_roots: Option<(&Path, &Path)>, + ) -> MicrosandboxResult { + Self::reconcile_sandbox_runtime_state_owned(pools, sandbox, socket_roots, false).await + } + + /// `transition_owned` is used only by lifecycle callers already holding the name guard. + async fn reconcile_sandbox_runtime_state_owned( + pools: &DbPools, + sandbox: sandbox_entity::Model, + socket_roots: Option<(&Path, &Path)>, + transition_owned: bool, ) -> MicrosandboxResult { if !matches!( sandbox.status, @@ -583,6 +640,16 @@ impl LocalBackend { // A dead-PID snapshot is not sufficient: another process may already // have reconciled and restarted this name. Serialize on the runtime // ownership lock, then re-read the exact row/run before unlinking. + let _transition = if !transition_owned && let Some((run_dir, _)) = socket_roots { + let Some(guard) = + microsandbox_runtime::ipc::try_acquire_transition_guard(run_dir, &sandbox.name)? + else { + return Ok(sandbox); + }; + Some(guard) + } else { + None + }; let _guard = if let Some((run_dir, _)) = socket_roots { let Some(guard) = microsandbox_runtime::ipc::try_acquire_lifecycle_guard(run_dir, &sandbox.name)? @@ -607,11 +674,13 @@ impl LocalBackend { } let run = Self::load_active_run(pools.read(), sandbox.id).await?; - // No run record yet while Starting means the child has not inserted its PID. A Draining row with no - // active run, however, has already completed shutdown from the DB's point - // of view and should not keep stop callers polling forever. + // An unowned Starting claim with no run is an abandoned launcher. Both guards above + // prove there is no creator in the Windows lock handoff gap and no resident runtime. + // Without filesystem ownership information, retain the conservative observation. let Some(run) = run else { - if sandbox.status == SandboxStatus::Draining { + if sandbox.status == SandboxStatus::Draining + || (sandbox.status == SandboxStatus::Starting && socket_roots.is_some()) + { if let Some((run_dir, sandboxes_dir)) = socket_roots { crate::runtime::remove_sandbox_socket_artifacts_at( run_dir, @@ -722,7 +791,7 @@ impl LocalBackend { } /// Extract a live PID from a run record, if the process is still alive. - fn pid_from_run(run: Option<&run_entity::Model>) -> Option { + pub(super) fn pid_from_run(run: Option<&run_entity::Model>) -> Option { run.and_then(|model| model.pid) .filter(|pid| Self::pid_is_alive(*pid)) } @@ -896,7 +965,7 @@ impl LocalBackend { } /// Whether `pid` refers to a live process. - fn pid_is_alive(pid: i32) -> bool { + pub(super) fn pid_is_alive(pid: i32) -> bool { microsandbox_utils::process::pid_is_alive(pid) } @@ -1060,7 +1129,8 @@ impl SandboxBackend for LocalBackend { name: &'a str, ) -> BoxFuture<'a, MicrosandboxResult> { Box::pin(async move { - let (model, pid) = self.sandbox_handle_state(name, None).await?; + let (mut model, pid) = self.sandbox_handle_state(name, None).await?; + model.status = crate::sandbox::pause::projected_status(self, name, model.status).await; Ok(SandboxHandle::from_local_model(backend, model, pid)) }) } @@ -1072,10 +1142,22 @@ impl SandboxBackend for LocalBackend { ) -> BoxFuture<'a, MicrosandboxResult> { Box::pin(async move { let (rows, next_cursor) = self.list_sandbox_handle_state(&query).await?; - let sandboxes = rows - .into_iter() - .map(|(model, pid)| SandboxHandle::from_local_model(backend.clone(), model, pid)) - .collect(); + let sandboxes = stream::iter(rows) + .map(|(mut model, pid)| { + let backend = backend.clone(); + async move { + model.status = crate::sandbox::pause::projected_status( + self, + &model.name, + model.status, + ) + .await; + SandboxHandle::from_local_model(backend, model, pid) + } + }) + .buffered(16) + .collect() + .await; Ok(SandboxPage { sandboxes, next_cursor, @@ -1328,6 +1410,7 @@ mod tests { #[cfg(unix)] use std::process::Command; use std::sync::Arc; + use std::time::Duration; use futures::StreamExt; use microsandbox_db::entity::run as run_entity; @@ -1389,6 +1472,71 @@ mod tests { pid } + #[cfg(unix)] + #[tokio::test] + async fn control_lookup_skips_observation_but_get_and_list_still_project_pause() { + use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; + + let home = tempfile::tempdir_in("/tmp").unwrap(); + let backend = Arc::new( + LocalBackend::builder() + .home(home.path()) + .build() + .await + .unwrap(), + ); + let pools = backend.db().await.unwrap(); + let name = "resident"; + let id = LocalBackend::insert_sandbox_record(pools.write(), &test_config(name)) + .await + .unwrap(); + run_entity::Entity::insert(run_entity::ActiveModel { + sandbox_id: Set(id), + pid: Set(Some(std::process::id() as i32)), + status: Set(run_entity::RunStatus::Running), + ..Default::default() + }) + .exec(pools.write()) + .await + .unwrap(); + let agent = + crate::runtime::sandbox_agent_socket_path_candidates_for(&backend, name).remove(0); + let path = microsandbox_runtime::control::control_socket_path_for(&agent); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + let listener = tokio::net::UnixListener::bind(path).unwrap(); + let server = tokio::spawn(async move { + // The mutation arrives first. Ordinary observational APIs retain their projection. + for operation in ["pause", "pause_state", "pause_state"] { + let (stream, _) = listener.accept().await.unwrap(); + let mut stream = BufReader::new(stream); + let mut line = String::new(); + stream.read_line(&mut line).await.unwrap(); + assert_eq!(line, format!("{{\"op\":\"{operation}\"}}\n")); + stream + .get_mut() + .write_all( + b"{\"ok\":true,\"pause\":{\"paused\":true,\"recovery_required\":false}}\n", + ) + .await + .unwrap(); + } + }); + let backend_dyn: Arc = backend; + crate::backend::with_backend(backend_dyn, async { + let handle = crate::Sandbox::get_for_control(name).await.unwrap(); + handle.pause().await.unwrap(); + assert_eq!( + crate::Sandbox::get(name).await.unwrap().status_snapshot(), + SandboxStatus::Paused + ); + let page = crate::Sandbox::list().await.unwrap(); + assert_eq!(page.sandboxes.len(), 1); + assert_eq!(page.sandboxes[0].status_snapshot(), SandboxStatus::Paused); + }) + .await; + server.await.unwrap(); + } + #[tokio::test] async fn follow_logs_replays_filtered_history_then_streams_from_snapshot_cursor() { let temp = tempdir().unwrap(); @@ -1629,6 +1777,121 @@ mod tests { ); } + #[tokio::test] + async fn abandoned_start_recovers_only_after_creator_ownership_ends() { + #[cfg(unix)] + let home = tempfile::tempdir_in("/tmp").unwrap(); + #[cfg(not(unix))] + let home = tempdir().unwrap(); + let backend = LocalBackend::builder() + .home(home.path()) + .build() + .await + .unwrap(); + let pools = backend.db().await.unwrap(); + let mut config = test_config("abandoned"); + config.checkpoint_restore = Some(microsandbox_runtime::launch::CheckpointRestoreConfig { + local_branch: false, + forked: false, + closure: home.path().join("checkpoint"), + checkpoint_root: "pending".into(), + checkpoint_id: "pending".into(), + }); + let id = LocalBackend::insert_sandbox_record(pools.write(), &config) + .await + .unwrap(); + LocalBackend::update_sandbox_status(pools.write(), id, SandboxStatus::Starting) + .await + .unwrap(); + let transition = LocalBackend::acquire_sandbox_transition_guard( + &backend.config().run_dir(), + "abandoned", + ) + .await + .unwrap(); + // Deliberately no runtime guard: this also models the Windows handoff gap. + assert_eq!( + backend + .sandbox_handle_state("abandoned", Some(id)) + .await + .unwrap() + .0 + .status, + SandboxStatus::Starting + ); + drop(transition); + let (recovered, _) = backend + .sandbox_handle_state("abandoned", Some(id)) + .await + .unwrap(); + assert_eq!(recovered.status, SandboxStatus::Crashed); + let persisted: SandboxConfig = serde_json::from_str(&recovered.config).unwrap(); + assert!(LocalBackend::validate_completed_restore(&persisted).is_err()); + } + + #[cfg(unix)] + #[tokio::test] + async fn kill_waits_for_start_publication_and_terminates_the_created_run() { + let home = tempfile::tempdir_in("/tmp").unwrap(); + let backend = Arc::new( + LocalBackend::builder() + .home(home.path()) + .build() + .await + .unwrap(), + ); + let pools = backend.db().await.unwrap(); + let id = LocalBackend::insert_sandbox_record(pools.write(), &test_config("kill-start")) + .await + .unwrap(); + LocalBackend::update_sandbox_status(pools.write(), id, SandboxStatus::Starting) + .await + .unwrap(); + let transition = LocalBackend::acquire_sandbox_transition_guard( + &backend.config().run_dir(), + "kill-start", + ) + .await + .unwrap(); + let other = backend.clone(); + let mut kill = + tokio::spawn(async move { other.kill_sandbox("kill-start", Some(id)).await }); + assert!( + tokio::time::timeout(Duration::from_millis(50), &mut kill) + .await + .is_err() + ); + let mut child = Command::new("sleep").arg("30").spawn().unwrap(); + let pid = child.id() as i32; + run_entity::Entity::insert(run_entity::ActiveModel { + sandbox_id: Set(id), + pid: Set(Some(pid)), + status: Set(run_entity::RunStatus::Running), + ..Default::default() + }) + .exec(pools.write()) + .await + .unwrap(); + LocalBackend::update_sandbox_status(pools.write(), id, SandboxStatus::Running) + .await + .unwrap(); + drop(transition); + let result = tokio::time::timeout(Duration::from_secs(6), kill).await; + // Ensure assertion failures never leave the helper process behind. + let _ = child.kill(); + child.wait().unwrap(); + result.unwrap().unwrap().unwrap(); + assert_eq!( + backend + .sandbox_handle_state("kill-start", Some(id)) + .await + .unwrap() + .0 + .status, + SandboxStatus::Stopped + ); + } + #[tokio::test] async fn test_reconcile_sandbox_runtime_state_marks_dead_processes_crashed() { #[cfg(unix)] @@ -1870,6 +2133,42 @@ mod tests { let _ = backend.validate_start_state(&config, &sandbox_dir); } + #[tokio::test] + async fn flat_restart_does_not_require_layered_image_artifacts() { + let temp = tempdir().unwrap(); + let backend = LocalBackend::builder() + .home(temp.path()) + .build() + .await + .unwrap(); + let sandbox_dir = temp.path().join("persisted"); + fs::create_dir(&sandbox_dir).unwrap(); + let mut config = test_config_with_rootfs( + "persisted", + RootfsSource::Oci(OciRootfsSource { + reference: "alpine".into(), + root_disk: Some(crate::sandbox::RootDisk::Flat { + size_mib: Some(512), + fstype: None, + clone: microsandbox_types::FlatClone::Auto, + }), + }), + ); + config.manifest_digest = Some(format!("sha256:{}", "a".repeat(64))); + backend.validate_start_state(&config, &sandbox_dir).unwrap(); + let RootfsSource::Oci(oci) = &mut config.spec.image else { + unreachable!() + }; + oci.root_disk = None; + assert!( + backend + .validate_start_state(&config, &sandbox_dir) + .unwrap_err() + .to_string() + .contains("VMDK missing") + ); + } + /// Simulates the reaper sweep: queries all Starting/Running/Draining sandboxes and /// reconciles each. Verifies that only stale entries are reaped while /// live, stopped, and starting (no run record) sandboxes are left untouched. diff --git a/sdk/rust/lib/backend/profile.rs b/sdk/rust/lib/backend/profile.rs index 96b2a677b..c059f28d8 100644 --- a/sdk/rust/lib/backend/profile.rs +++ b/sdk/rust/lib/backend/profile.rs @@ -104,9 +104,15 @@ enum BackendSelection { /// Missing file → `Ok(SdkConfig::default())`. Malformed JSON → `Err`. /// Honours `MSB_CONFIG_PATH` env override for the file path. pub fn load_sdk_config() -> MicrosandboxResult { + load_sdk_config_document().map(|(config, _)| config) +} + +/// Keep the source document for the local half of ambient backend resolution. Local field +/// errors retain the lazy backend's existing default fallback; SDK profile errors remain fatal. +fn load_sdk_config_document() -> MicrosandboxResult<(SdkConfig, Option)> { let path = sdk_config_path(); if !path.exists() { - return Ok(SdkConfig::default()); + return Ok((SdkConfig::default(), None)); } let raw = fs::read_to_string(&path).map_err(|e| { MicrosandboxError::InvalidConfig(format!( @@ -123,7 +129,7 @@ pub fn load_sdk_config() -> MicrosandboxResult { path.display() )) })?; - Ok(cfg) + Ok((cfg, Some(raw))) } /// Resolve the default backend according to the Q1 precedence ladder. @@ -161,7 +167,16 @@ pub fn resolve_default_backend() -> MicrosandboxResult> { } } - let cfg = load_sdk_config()?; + let (cfg, document) = load_sdk_config_document()?; + #[cfg(not(feature = "local"))] + let _ = &document; + #[cfg(feature = "local")] + let local_config = || { + document + .as_deref() + .and_then(|raw| serde_json::from_str::(raw).ok()) + .unwrap_or_default() + }; let env_profile = std::env::var("MSB_PROFILE").ok(); let selection = select_backend( backend_kind.as_deref(), @@ -171,7 +186,16 @@ pub fn resolve_default_backend() -> MicrosandboxResult> { )?; match selection { - BackendSelection::Local => local_backend(BackendSelectionSource::Default, None), + BackendSelection::Local => { + #[cfg(not(feature = "local"))] + return Err(feature_disabled("local")); + #[cfg(feature = "local")] + Ok(Arc::new(LocalBackend::lazy_with_config( + local_config(), + BackendSelectionSource::Default, + None, + ))) + } BackendSelection::DirectCloud => { #[cfg(not(feature = "cloud"))] return Err(feature_disabled("cloud")); @@ -213,7 +237,18 @@ pub fn resolve_default_backend() -> MicrosandboxResult> { } else { BackendSelectionSource::ActiveProfile }; - backend_from_profile(&name, profile, source) + if profile.backend == ProfileBackend::Local { + #[cfg(not(feature = "local"))] + return Err(feature_disabled("local")); + #[cfg(feature = "local")] + Ok(Arc::new(LocalBackend::lazy_with_config( + local_config(), + source, + Some(name), + ))) + } else { + backend_from_profile(&name, profile, source) + } } } } diff --git a/sdk/rust/lib/error.rs b/sdk/rust/lib/error.rs index dd5c0524a..2ded67e56 100644 --- a/sdk/rust/lib/error.rs +++ b/sdk/rust/lib/error.rs @@ -324,6 +324,10 @@ pub enum Operation { SandboxStart, /// `Sandbox::stop`. SandboxStop, + /// `Sandbox::pause`. + SandboxPause, + /// `Sandbox::resume`. + SandboxResume, /// `Sandbox::remove`. SandboxRemove, /// `Sandbox::remove_persisted`. @@ -492,6 +496,8 @@ impl Operation { Operation::SandboxCreate => "Sandbox::create", Operation::SandboxStart => "Sandbox::start", Operation::SandboxStop => "Sandbox::stop", + Operation::SandboxPause => "Sandbox::pause", + Operation::SandboxResume => "Sandbox::resume", Operation::SandboxRemove => "Sandbox::remove", Operation::SandboxRemovePersisted => "Sandbox::remove_persisted", Operation::SandboxKill => "Sandbox::kill", diff --git a/sdk/rust/lib/lib.rs b/sdk/rust/lib/lib.rs index 75125d909..1ce82acb8 100644 --- a/sdk/rust/lib/lib.rs +++ b/sdk/rust/lib/lib.rs @@ -95,9 +95,9 @@ pub use sandbox::{ }; #[cfg(feature = "local")] pub use snapshot::{ - CheckpointSnapshotState, FileSnapshotState, SaveOpts, Snapshot, SnapshotArchive, - SnapshotBuilder, SnapshotConfig, SnapshotDescriptor, SnapshotFormat, SnapshotHandle, - SnapshotRootDisk, SnapshotScope, SnapshotSpec, SnapshotState, SnapshotVerifyReport, - UpperIntegrity, UpperVerifyStatus, + CheckpointSnapshotState, FileSnapshotState, HeadUpdate, HeadUpdateReason, LoadOpts, SaveOpts, + Snapshot, SnapshotArchive, SnapshotBuilder, SnapshotConfig, SnapshotDescriptor, SnapshotFormat, + SnapshotHandle, SnapshotRootDisk, SnapshotScope, SnapshotSpec, SnapshotState, + SnapshotVerifyReport, UpperIntegrity, UpperVerifyStatus, }; pub use volume::{Volume, VolumeConfig, VolumeHandle, VolumeKind, VolumeSpec}; diff --git a/sdk/rust/lib/runtime/spawn.rs b/sdk/rust/lib/runtime/spawn.rs index 982ffd8ae..0dbb6f923 100644 --- a/sdk/rust/lib/runtime/spawn.rs +++ b/sdk/rust/lib/runtime/spawn.rs @@ -354,9 +354,6 @@ pub async fn spawn_sandbox( } }; - #[cfg(not(unix))] - let _ = lifecycle_guard; - // Lifecycle callers prove any previous owner dead before reaching spawn. // With ownership now serialized, remove exact leftovers from that prior // generation so compatibility-link publication cannot be masked by them. @@ -674,16 +671,14 @@ pub async fn spawn_sandbox( ensure_sigchld_handler_uses_alt_stack_before_spawn().await?; - // Spawn the sandbox process. + // Spawn and Windows lock release form one handoff, before waiting for startup JSON. let mut child = { - #[cfg(windows)] - let _stdio_inherit_guard = if matches!(mode, SpawnMode::Detached) { - Some(StdioInheritGuard::new()?) - } else { - None - }; - - match cmd.spawn() { + match spawn_runtime_command( + &mut cmd, + mode, + #[cfg(not(unix))] + lifecycle_guard, + ) { Ok(child) => child, Err(err) => { release_metrics_reservation(config, metrics_reservation.as_ref()); @@ -804,6 +799,25 @@ pub async fn spawn_sandbox( Ok((handle, agent_sock_path)) } +/// Start the process after releasing ownership that cannot be inherited on Windows. +fn spawn_runtime_command( + cmd: &mut Command, + _mode: SpawnMode, + #[cfg(not(unix))] lifecycle_guard: Option, +) -> std::io::Result { + // The caller retains its transition guard through readiness; only the runtime ownership + // moves to the child. Keeping this handle during startup creates a parent/child deadlock. + #[cfg(not(unix))] + drop(lifecycle_guard); + #[cfg(windows)] + let _stdio_inherit_guard = if matches!(_mode, SpawnMode::Detached) { + Some(StdioInheritGuard::new()?) + } else { + None + }; + cmd.spawn() +} + fn block_writeback_policy( config: &RuntimeConfig, ) -> MicrosandboxResult<(Option, Option)> { @@ -2472,6 +2486,12 @@ fn sandbox_cli_args( // typed `LaunchConfig`, delivered over the config fd. See issue #997. let mut visible = vec![OsString::from("sandbox")]; + // An old binary might ignore unknown JSON fields, including the whole restore source. + // An explicit argv requirement instead fails in its command parser, before any VM exists. + if config.checkpoint_restore.is_some() { + visible.push(OsString::from("--restore")); + } + if let Some(log_level) = config.spec.runtime.log_level { visible.push(OsString::from(sandbox_log_level_cli_flag(log_level))); } @@ -2528,13 +2548,22 @@ fn sandbox_cli_args( agent_sock: agent_sock_path.to_path_buf(), libkrunfw_path: libkrunfw_path.to_path_buf(), thp: config.spec.resources.thp, + memory_cache_dir: Some(local.cache_dir().join("memory")), startup: startup_command(config), lifecycle: Lifecycle { max_duration_secs: config.spec.lifecycle.max_duration_secs, idle_timeout_secs: config.spec.lifecycle.idle_timeout_secs, }, vsock: config.spec.vsock.routes.clone(), - checkpoint_restore: config.checkpoint_restore.clone(), + execution: if config.checkpoint_restore.is_some() { + microsandbox_runtime::launch::ExecutionIntent::Restore + } else { + microsandbox_runtime::launch::ExecutionIntent::Boot + }, + checkpoint_restore: config.checkpoint_restore.clone().map(|mut restore| { + restore.forked = config.forked; + restore + }), #[cfg(feature = "net")] deployment_profile: config.spec.deployment_profile, bootstrap: GuestBootstrap { @@ -2937,6 +2966,64 @@ mod tests { volume::VolumeKind, }; + #[cfg(windows)] + #[test] + fn windows_lifecycle_handoff_child() { + use std::io::Write; + let Some(run_dir) = std::env::var_os("MSB_TEST_LIFECYCLE_RUN_DIR") else { + return; + }; + let _guard = microsandbox_runtime::ipc::acquire_lifecycle_guard( + std::path::Path::new(&run_dir), + "handoff", + ) + .unwrap(); + let pipe = std::env::var_os("MSB_TEST_LIFECYCLE_STARTUP_PIPE").unwrap(); + let mut writer = std::fs::OpenOptions::new().write(true).open(pipe).unwrap(); + writeln!(writer, "{{\"pid\":{}}}", std::process::id()).unwrap(); + } + + #[cfg(windows)] + #[tokio::test] + async fn windows_spawn_handoff_releases_lifecycle_before_startup_reply() { + let directory = tempfile::tempdir().unwrap(); + let run_dir = directory.path().join("run"); + let _transition = + microsandbox_runtime::ipc::try_acquire_transition_guard(&run_dir, "handoff") + .unwrap() + .unwrap(); + let guard = + microsandbox_runtime::ipc::acquire_lifecycle_guard(&run_dir, "handoff").unwrap(); + let pipe = super::create_startup_pipe("handoff", 1).unwrap(); + let mut command = tokio::process::Command::new(std::env::current_exe().unwrap()); + command + .args([ + "--exact", + "runtime::spawn::tests::windows_lifecycle_handoff_child", + "--nocapture", + ]) + .env("MSB_TEST_LIFECYCLE_RUN_DIR", &run_dir) + .env("MSB_TEST_LIFECYCLE_STARTUP_PIPE", &pipe.name) + .kill_on_drop(true); + let mut child = + super::spawn_runtime_command(&mut command, super::SpawnMode::Attached, Some(guard)) + .unwrap(); + let reply = tokio::time::timeout( + std::time::Duration::from_secs(5), + super::read_startup_line(&mut child, Some(pipe)), + ) + .await; + if reply.is_err() { + let _ = child.kill().await; + } + let reply = reply + .expect("child waited on a lifecycle lock retained by its parent") + .unwrap(); + let info: serde_json::Value = serde_json::from_str(&reply).unwrap(); + assert_eq!(info["pid"].as_u64(), child.id().map(u64::from)); + assert!(child.wait().await.unwrap().success()); + } + #[cfg(windows)] fn windows_handle_flags(handle: super::HANDLE) -> u32 { let mut flags = 0; @@ -4104,6 +4191,8 @@ mod tests { }, ]; config.checkpoint_restore = Some(CheckpointRestoreConfig { + local_branch: false, + forked: false, closure: PathBuf::from("/tmp/checkpoint"), checkpoint_root: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb".into(), @@ -4113,6 +4202,11 @@ mod tests { let launch = render_launch(&config); assert!(launch.rootfs.upper.is_none()); + assert_eq!( + launch.execution, + microsandbox_runtime::launch::ExecutionIntent::Restore + ); + assert!(render_args(&config).contains(&"--restore".to_string())); assert_eq!(launch.rootfs.upper_layers.len(), 2); assert_eq!(launch.rootfs.upper_layers[0].format, "raw"); assert_eq!(launch.rootfs.upper_layers[1].format, "qcow2"); diff --git a/sdk/rust/lib/sandbox/branch.rs b/sdk/rust/lib/sandbox/branch.rs new file mode 100644 index 000000000..8d9d3f077 --- /dev/null +++ b/sdk/rust/lib/sandbox/branch.rs @@ -0,0 +1,255 @@ +//! Direct local execution branching through the existing control and restore paths. + +use std::sync::Arc; +#[cfg(feature = "local")] +use std::{fs::File, path::Path}; + +#[cfg(feature = "local")] +use microsandbox_runtime::checkpoint::LocalBranchState; +#[cfg(feature = "local")] +use microsandbox_runtime::control::ControlRequest; +#[cfg(feature = "local")] +use microsandbox_runtime::launch::{CheckpointRestoreConfig, RootfsUpperLayerConfig}; + +use crate::backend::Backend; +#[cfg(feature = "local")] +use crate::backend::LocalBackend; +use crate::backend::sandbox::SandboxIdentity; +use crate::{MicrosandboxError, MicrosandboxResult}; + +use super::{Sandbox, SandboxHandle}; +#[cfg(feature = "local")] +use super::{SandboxConfig, SandboxStatus, modify}; + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl Sandbox { + /// Branch current execution into an independent local child using private CoW RAM. + /// The source keeps its running/paused state; no durable full snapshot is created. + pub async fn branch(&self, name: impl Into) -> MicrosandboxResult { + branch( + self.backend().clone(), + self.name(), + self.identity(), + name.into(), + ) + .await + } +} + +impl SandboxHandle { + /// Branch a running or user-paused local sandbox without connecting to its guest. + pub async fn branch(&self, name: impl Into) -> MicrosandboxResult { + branch( + self.backend.clone(), + self.name(), + self.identity(), + name.into(), + ) + .await + } +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +#[cfg(not(feature = "local"))] +async fn branch( + _backend: Arc, + _source: &str, + _identity: SandboxIdentity, + _name: String, +) -> MicrosandboxResult { + Err(MicrosandboxError::InvalidConfig( + "direct branching requires a local backend".into(), + )) +} + +#[cfg(feature = "local")] +async fn branch( + backend: Arc, + source: &str, + identity: SandboxIdentity, + name: String, +) -> MicrosandboxResult { + super::validate_sandbox_name(&name)?; + let local = backend.as_local().ok_or_else(|| { + MicrosandboxError::InvalidConfig("direct branching requires a local backend".into()) + })?; + let SandboxIdentity::Local(expected_id) = identity else { + return Err(MicrosandboxError::InvalidConfig( + "direct branching requires a local source".into(), + )); + }; + let run = { + let _transition = + LocalBackend::acquire_sandbox_transition_guard(&local.config().run_dir(), source) + .await?; + local.control_run_identity(source, expected_id).await? + }; + let handle = backend.sandboxes().get(backend.clone(), source).await?; + if handle.identity() != SandboxIdentity::Local(expected_id) { + return Err(MicrosandboxError::SandboxReplaced { + name: source.into(), + expected: format!("local:{expected_id}"), + actual: handle.id().to_string(), + }); + } + if !matches!( + handle.status_snapshot(), + SandboxStatus::Running | SandboxStatus::Paused + ) { + return Err(MicrosandboxError::InvalidConfig( + "branch requires a running or user-paused source".into(), + )); + } + let mut config = handle + .active_config()? + .unwrap_or(handle.config()?) + .clone_for_persistence(); + if !config.spec.network.ports.is_empty() { + return Err(MicrosandboxError::InvalidConfig( + "branch cannot inherit published host ports; remove port publications before branching" + .into(), + )); + } + let capabilities = + modify::control_request_for_run(local, source, run, "{\"op\":\"capabilities\"}\n".into()) + .await?; + if !capabilities.capabilities.is_some_and(|c| c.branch_create) { + return Err(MicrosandboxError::Runtime( + "source runtime does not support direct local branching".into(), + )); + } + config.spec.name = name; + config.replace_existing = false; + config.spec.patches.clear(); + config.branch_source = Some(super::identity::BranchSource { + name: source.into(), + run, + }); + config.suppress_launch_for_full_restore(); + backend + .sandboxes() + .create_detached(backend.clone(), config) + .await +} + +/// Called only after the ordinary create path reserves the child name and directory. +/// Retain this pin through spawn, until the runtime owns its independent mapping handle. +#[cfg(feature = "local")] +pub(crate) async fn capture_child( + local: &LocalBackend, + config: &mut SandboxConfig, + source: &super::identity::BranchSource, + child: &Path, +) -> MicrosandboxResult { + // Child reservation precedes capture; source transition ownership now excludes restart or + // replacement until the exact selected generation has handed off its state. + let _transition = + LocalBackend::acquire_sandbox_transition_guard(&local.config().run_dir(), &source.name) + .await?; + local.validate_control_run(&source.name, source.run).await?; + // Serialize with durable source captures so a child's ancestry describes its actual cut. + let lineage = crate::snapshot::lineage::begin(local, &source.name).await?; + config.snapshot_parent = lineage.parent.as_ref().map(ToString::to_string); + let id = format!("branch_{:032x}", rand::random::()); + // Acquired before publication: source exit or another capture cannot create an unpinned + // eviction window before this caller opens the completed memory file. + let _handoff = microsandbox_runtime::checkpoint::LocalMemory::reserve( + &local.cache_dir().join("memory"), + &id, + )?; + tokio::fs::write(child.join(".branch-reservation"), &id).await?; + let request = ControlRequest::BranchCreate { + branch_id: id.clone(), + child_name: config.spec.name.clone(), + memory_cache_dir: local.cache_dir().join("memory"), + }; + let response = modify::control_request_for_run( + local, + &source.name, + source.run, + format!("{}\n", serde_json::to_string(&request)?), + ) + .await?; + local.validate_control_run(&source.name, source.run).await?; + lineage.validate_source(local, &source.name).await?; + let closure = child.join(".branch-restore"); + if response.branch.as_ref() != Some(&closure) { + return Err(MicrosandboxError::Runtime( + "branch returned an unexpected handoff path".into(), + )); + } + let state = LocalBranchState::open(&closure)?; + if state.id != id { + return Err(MicrosandboxError::Runtime("branch identity differs".into())); + } + let pin = state.memory.pin()?; + config.spec.resources.cpus = state.vcpus; + config.spec.resources.max_cpus = state.max_cpus; + config.spec.resources.memory_mib = state.memory_mib; + config.spec.resources.max_memory_mib = state.max_memory_mib; + // The captured effective address wins over launch-time pools/defaults. Each user-mode + // network stack is isolated; host listeners were rejected before source mutation. + config.spec.network.interface = None; + super::builder::apply_capture_network(config, &state.resources)?; + let layout = match config.spec.image.oci_root_disk() { + Some(super::RootDisk::Flat { .. }) => crate::snapshot::SnapshotRootDisk::Flat, + Some(super::RootDisk::Tmpfs { size_mib }) => crate::snapshot::SnapshotRootDisk::Tmpfs { + size_mib: *size_mib, + }, + _ => crate::snapshot::SnapshotRootDisk::Managed, + }; + match state.disks.as_slice() { + [] if matches!(layout, crate::snapshot::SnapshotRootDisk::Tmpfs { .. }) => {} + [disk] => { + disk.to_canonical_bytes() + .map_err(|e| MicrosandboxError::SnapshotIntegrity(e.to_string()))?; + if disk.pause_generation != state.pause_generation { + return Err(MicrosandboxError::SnapshotIntegrity( + "branch disk epoch differs".into(), + )); + } + let sources = disk + .layers + .iter() + .map(|layer| RootfsUpperLayerConfig { + path: closure + .join("layers") + .join(format!("{}.{}", layer.layer_id, layer.format)), + format: layer.format.clone(), + }) + .collect::>(); + let size = disk + .layers + .last() + .ok_or_else(|| MicrosandboxError::SnapshotIntegrity("branch disk is empty".into()))? + .virtual_size; + let materialized = crate::snapshot::materialize_file_snapshot_for_child( + &sources, size, child, &layout, + ) + .await?; + config.snapshot_upper_layers = materialized.upper_layers; + } + _ => { + return Err(MicrosandboxError::SnapshotIntegrity( + "branch disk closure differs from root layout".into(), + )); + } + } + config.checkpoint_restore = Some(CheckpointRestoreConfig { + local_branch: true, + forked: true, + closure, + checkpoint_root: String::new(), + checkpoint_id: id, + }); + config.forked = true; + config.suppress_launch_for_full_restore(); + tokio::fs::remove_file(child.join(".branch-reservation")).await?; + Ok(pin) +} diff --git a/sdk/rust/lib/sandbox/builder.rs b/sdk/rust/lib/sandbox/builder.rs index 3b67d263d..b255e938f 100644 --- a/sdk/rust/lib/sandbox/builder.rs +++ b/sdk/rust/lib/sandbox/builder.rs @@ -391,6 +391,15 @@ impl SandboxBuilder { self } + /// Restore a full snapshot using private copy-on-write memory. + /// + /// Clean pages can be shared by children; writes remain private. This requires + /// a full snapshot and cannot be combined with a fresh boot or disk-only restore. + pub fn forked(mut self) -> Self { + self.config.forked = true; + self + } + /// Set the runtime log level for the sandbox process. /// /// This controls the verbosity of the `msb sandbox` process. @@ -1201,7 +1210,7 @@ impl SandboxBuilder { self } - /// Supply the exact base snapshot or standalone base archive for a disk-dependent archive. + /// Supply the base snapshot or standalone archive for omitted disk layers and RAM objects. pub fn snapshot_base(mut self, base: impl Into) -> Self { self.config.snapshot_base = Some(base.into()); self @@ -1287,6 +1296,7 @@ impl SandboxBuilder { } let snap = crate::snapshot::Snapshot::open(&snapshot_ref).await?; + self.config.snapshot_parent = Some(snap.id().to_string()); let unsupported = snap.manifest().unsupported_requires(); if !unsupported.is_empty() { return Err(crate::MicrosandboxError::unsupported( @@ -1315,27 +1325,22 @@ impl SandboxBuilder { ) .map_err(|error| crate::MicrosandboxError::SnapshotIntegrity(error.to_string()))?; let closure = snap.path().join(crate::snapshot::CHECKPOINT_DIRECTORY); - let opened = match self.config.snapshot_restore_mode { - SnapshotRestoreMode::Full => { - microsandbox_image::checkpoint::CheckpointClosure::open( - &closure, - Some(&expected), - ) - } - SnapshotRestoreMode::DiskOnly => { - microsandbox_image::checkpoint::CheckpointClosure::open_portable( - &closure, - Some(&expected), - ) - } - } + let opened = microsandbox_image::checkpoint::CheckpointClosure::inspect_manifest( + &closure, + Some(&expected), + ) .map_err(|error| crate::MicrosandboxError::SnapshotIntegrity(error.to_string()))?; - if opened.checkpoint().checkpoint_id != state.checkpoint_id { + if opened.checkpoint_id != state.checkpoint_id { return Err(crate::MicrosandboxError::SnapshotIntegrity( "snapshot and checkpoint closure identities differ".into(), )); } if self.config.snapshot_restore_mode == SnapshotRestoreMode::Full { + if opened.architecture != std::env::consts::ARCH { + return Err(crate::MicrosandboxError::SnapshotIntegrity( + "checkpoint architecture cannot restore on this host".into(), + )); + } let restore_overrides = self.restore_override_intent(); apply_checkpoint_restore_constraints( &mut self.config, @@ -1347,6 +1352,8 @@ impl SandboxBuilder { } self.config.checkpoint_restore = Some(microsandbox_runtime::launch::CheckpointRestoreConfig { + local_branch: false, + forked: false, closure, checkpoint_root: state.checkpoint_root.clone(), checkpoint_id: state.checkpoint_id.clone(), @@ -1650,6 +1657,17 @@ impl SandboxBuilder { )); } #[cfg(feature = "local")] + if self.config.forked + && (self.config.snapshot_restore_mode == SnapshotRestoreMode::DiskOnly + || (self.config.checkpoint_restore.is_none() + && self.config.snapshot_archive_source.is_none())) + { + return Err(crate::MicrosandboxError::InvalidConfig( + "forked requires a full snapshot restore and cannot be combined with disk_only" + .into(), + )); + } + #[cfg(feature = "local")] if self.config.checkpoint_restore.is_some() && !self.config.spec.patches.is_empty() { return Err(crate::MicrosandboxError::InvalidConfig( "patches cannot be combined with full snapshot restore".into(), @@ -1931,14 +1949,19 @@ fn validate_config_script_name(name: &str) -> Result<(), String> { pub(crate) fn apply_checkpoint_restore_constraints( config: &mut SandboxConfig, state: &crate::snapshot::CheckpointSnapshotState, - closure: µsandbox_image::checkpoint::CheckpointClosure, + checkpoint: µsandbox_image::checkpoint::CheckpointManifest, overrides: RestoreOverrideIntent, ) -> MicrosandboxResult<()> { apply_checkpoint_resources(config, state, overrides)?; + apply_capture_network(config, &checkpoint.resources) +} - let mut resources = closure - .checkpoint() - .resources +#[cfg(feature = "local")] +pub(crate) fn apply_capture_network( + config: &mut SandboxConfig, + captured_resources: &[microsandbox_image::checkpoint::ResourceDescriptor], +) -> MicrosandboxResult<()> { + let mut resources = captured_resources .iter() .filter(|resource| resource.kind == "network"); let Some(resource) = resources.next() else { @@ -3342,4 +3365,51 @@ mod tests { vec!["/workspace", "/workspace/persist"] ); } + #[tokio::test] + async fn forked_rejects_fresh_boot() { + let error = SandboxBuilder::new("forked-boot") + .image("alpine") + .forked() + .build() + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("forked requires a full snapshot") + ); + } + + #[tokio::test] + async fn forked_restore_is_transient_and_requires_execution() { + let mut builder = SandboxBuilder::new("forked-child").image("alpine").forked(); + builder.config.checkpoint_restore = + Some(microsandbox_runtime::launch::CheckpointRestoreConfig { + local_branch: false, + forked: false, + closure: "/owned/checkpoint".into(), + checkpoint_root: "blake3:captured-root".into(), + checkpoint_id: "captured".into(), + }); + builder.validate().unwrap(); + let config = builder.config.clone(); + assert!(config.forked); + assert!(!config.clone_for_persistence().forked); + assert!( + serde_json::to_value(&config) + .unwrap() + .get("forked") + .is_none() + ); + builder.config.snapshot_restore_mode = + crate::sandbox::config::SnapshotRestoreMode::DiskOnly; + assert!( + builder + .build() + .await + .unwrap_err() + .to_string() + .contains("forked") + ); + } } diff --git a/sdk/rust/lib/sandbox/compact.rs b/sdk/rust/lib/sandbox/compact.rs index b6cf7a75a..1edc0de98 100644 --- a/sdk/rust/lib/sandbox/compact.rs +++ b/sdk/rust/lib/sandbox/compact.rs @@ -78,6 +78,7 @@ impl DiskCompactionBuilder { .await? .ok_or_else(|| MicrosandboxError::SandboxNotFound(self.name.clone()))?; let config: SandboxConfig = serde_json::from_str(&model.config)?; + crate::LocalBackend::validate_completed_restore(&config)?; use microsandbox_types::RootDisk; if config.manifest_digest.is_none() || matches!( @@ -124,6 +125,8 @@ impl DiskCompactionBuilder { )); } let runtime_dir = local.sandboxes_dir().join(&self.name).join("runtime"); + let current_config: SandboxConfig = serde_json::from_str(¤t.config)?; + crate::LocalBackend::validate_completed_restore(¤t_config)?; tokio::task::spawn_blocking(move || { // Dropping an SDK future does not cancel spawn_blocking. Keep disk ownership in the // worker until it finishes, even when its caller disconnects or cancels the await. diff --git a/sdk/rust/lib/sandbox/config.rs b/sdk/rust/lib/sandbox/config.rs index b1aadf081..e15b964dc 100644 --- a/sdk/rust/lib/sandbox/config.rs +++ b/sdk/rust/lib/sandbox/config.rs @@ -217,14 +217,29 @@ pub struct SandboxConfig { #[serde(skip)] pub(crate) snapshot_base: Option, - /// Child-owned checkpoint closure used only for this process construction. + /// Snapshot from which this sandbox derives. Later captures retain their own local cursor. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) snapshot_parent: Option, + + /// Child-owned checkpoint closure for an unfinished restore construction. /// /// The builder initially points this at an installed snapshot. The local create path copies - /// the closure into child staging and rewrites the path before spawning the runtime. - #[serde(skip)] + /// the closure into child staging and rewrites the path before spawning the runtime. Local + /// creation persists this intent until activation succeeds; an interrupted restore must not + /// subsequently be interpreted as an ordinary cold boot. + #[serde(default, skip_serializing_if = "Option::is_none")] #[cfg(feature = "local")] pub(crate) checkpoint_restore: Option, + /// Source name for a one-shot direct local branch, consumed under child reservation. + #[serde(skip)] + #[cfg(feature = "local")] + pub(crate) branch_source: Option, + + /// Restore captured RAM through private CoW mappings; never a cold-boot policy. + #[serde(skip)] + pub(crate) forked: bool, + /// Transient checkpoint materialization policy selected by the caller. #[serde(skip)] pub(crate) snapshot_restore_mode: SnapshotRestoreMode, @@ -285,6 +300,8 @@ impl SandboxConfig { #[cfg(feature = "local")] { config.checkpoint_restore = None; + config.branch_source = None; + config.forked = false; } config.snapshot_restore_mode = SnapshotRestoreMode::Full; config.resumed_from_full_snapshot = false; @@ -766,9 +783,13 @@ impl Default for SandboxConfig { snapshot_root_layer_sources: Vec::new(), snapshot_root_virtual_size: None, snapshot_archive_source: None, + snapshot_parent: None, snapshot_base: None, #[cfg(feature = "local")] checkpoint_restore: None, + #[cfg(feature = "local")] + branch_source: None, + forked: false, snapshot_restore_mode: SnapshotRestoreMode::Full, resumed_from_full_snapshot: false, #[cfg(feature = "local")] @@ -1747,6 +1768,8 @@ mod tests { }, snapshot_restore_mode: restore_mode, checkpoint_restore: Some(CheckpointRestoreConfig { + local_branch: false, + forked: false, closure: PathBuf::from("/tmp/checkpoint"), checkpoint_root: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" diff --git a/sdk/rust/lib/sandbox/fs.rs b/sdk/rust/lib/sandbox/fs.rs index 41387da98..5e79fe3d6 100644 --- a/sdk/rust/lib/sandbox/fs.rs +++ b/sdk/rust/lib/sandbox/fs.rs @@ -653,10 +653,11 @@ impl FsReadStream { )) .await; } - MessageType::FsResponse => { - let resp: FsResponse = msg.payload()?; + MessageType::FsResponse | MessageType::CoreError => { + let response = filesystem_response(msg); let close_result = self.close_owned_handle().await; self.finished = true; + let resp = response?; if !resp.ok { return Err(MicrosandboxError::SandboxFsOps( resp.error.unwrap_or_else(|| "unknown error".into()), @@ -918,9 +919,22 @@ fn entry_info_to_metadata(info: &FsEntryInfo) -> FsMetadata { } } +/// Check the envelope before decoding: a paused runtime rejects new work with +/// `core.error`, which has no filesystem `ok` field. Do not hide that diagnostic +/// behind a CBOR decoding error (or silently discard it on a stream). +fn filesystem_response(msg: Message) -> MicrosandboxResult { + if msg.t != MessageType::FsResponse { + return Err(super::unexpected_agent_response( + "filesystem operation", + &msg, + )); + } + Ok(msg.payload()?) +} + /// Deserialize and check a simple ok/error `FsResponse`. fn check_response(msg: Message) -> MicrosandboxResult<()> { - let resp: FsResponse = msg.payload()?; + let resp = filesystem_response(msg)?; if resp.ok { Ok(()) } else { @@ -934,7 +948,9 @@ fn check_response(msg: Message) -> MicrosandboxResult<()> { async fn wait_for_ok_frame_response(rx: &mut mpsc::Receiver) -> MicrosandboxResult<()> { while let Some(frame) = rx.recv().await { match frame { - AgentFrame::Control(message) if message.t == MessageType::FsResponse => { + AgentFrame::Control(message) + if matches!(message.t, MessageType::FsResponse | MessageType::CoreError) => + { return check_response(message); } AgentFrame::Control(message) if message.t == MessageType::BulkCancel => { @@ -979,7 +995,9 @@ async fn apply_next_fs_write_credit( cancel.message ))); } - AgentFrame::Control(message) if message.t == MessageType::FsResponse => { + AgentFrame::Control(message) + if matches!(message.t, MessageType::FsResponse | MessageType::CoreError) => + { check_response(message)?; return Err(MicrosandboxError::SandboxFsOps( "filesystem write completed before its finish marker".into(), @@ -1015,7 +1033,11 @@ async fn receive_fs_bulk_acceptance( )) }); } - AgentFrame::Control(message) if message.t == MessageType::FsResponse => { + AgentFrame::Control(message) + if matches!(message.t, MessageType::FsResponse | MessageType::CoreError) => + { + // Host-side pause rejection can arrive before the guest accepts bulk mode. + // Keep that terminal diagnostic instead of waiting for a closed correlation. check_response(message)?; return Err(MicrosandboxError::SandboxFsOps( "filesystem stream completed before bulk acceptance".into(), @@ -1088,9 +1110,7 @@ pub(crate) mod agent { BULK_FLOW_MASK_GUEST_TO_HOST, BULK_FLOW_MASK_HOST_TO_GUEST, BulkFlow, BulkKind, BulkOffer, BulkReceiveState, BulkSendState, }, - fs::{ - FS_CHUNK_SIZE, FsOp, FsOpenOptions, FsRequest, FsResponse, FsResponseData, FsSetAttrs, - }, + fs::{FS_CHUNK_SIZE, FsOp, FsOpenOptions, FsRequest, FsResponseData, FsSetAttrs}, message::MessageType, }; use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -1099,8 +1119,8 @@ pub(crate) mod agent { use super::{ FsEntry, FsHandle, FsMetadata, FsReadStream, FsWriteSink, check_response, - entry_info_to_fs_entry, entry_info_to_metadata, receive_fs_bulk_acceptance, - should_offer_fs_write_bulk, + entry_info_to_fs_entry, entry_info_to_metadata, filesystem_response, + receive_fs_bulk_acceptance, should_offer_fs_write_bulk, }; /// Open a fresh agent connection for the named sandbox. @@ -1132,7 +1152,7 @@ pub(crate) mod agent { bulk: None, }; let resp_msg = client.request(MessageType::FsRequest, &req).await?; - let resp: FsResponse = resp_msg.payload()?; + let resp = filesystem_response(resp_msg)?; if !resp.ok { return Err(MicrosandboxError::SandboxFsOps( resp.error.unwrap_or_else(|| "unknown error".into()), @@ -1154,7 +1174,7 @@ pub(crate) mod agent { bulk: None, }; let resp_msg = client.request(MessageType::FsRequest, &req).await?; - let resp: FsResponse = resp_msg.payload()?; + let resp = filesystem_response(resp_msg)?; if !resp.ok { return Err(MicrosandboxError::SandboxFsOps( resp.error.unwrap_or_else(|| "unknown error".into()), @@ -1314,7 +1334,7 @@ pub(crate) mod agent { bulk: None, }; let resp_msg = client.request(MessageType::FsRequest, &req).await?; - let resp: FsResponse = resp_msg.payload()?; + let resp = filesystem_response(resp_msg)?; if !resp.ok { return Err(MicrosandboxError::SandboxFsOps( @@ -1339,7 +1359,7 @@ pub(crate) mod agent { bulk: None, }; let resp_msg = client.request(MessageType::FsRequest, &req).await?; - let resp: FsResponse = resp_msg.payload()?; + let resp = filesystem_response(resp_msg)?; if !resp.ok { return Err(MicrosandboxError::SandboxFsOps( @@ -1441,7 +1461,7 @@ pub(crate) mod agent { bulk: None, }; let resp_msg = client.request(MessageType::FsRequest, &req).await?; - let resp: FsResponse = resp_msg.payload()?; + let resp = filesystem_response(resp_msg)?; if !resp.ok { return Err(MicrosandboxError::SandboxFsOps( @@ -1572,7 +1592,7 @@ pub(crate) mod agent { bulk: None, }; let resp_msg = client.request(MessageType::FsRequest, &req).await?; - let resp: FsResponse = resp_msg.payload()?; + let resp = filesystem_response(resp_msg)?; if !resp.ok { return Err(MicrosandboxError::SandboxFsOps( @@ -1621,7 +1641,7 @@ pub(crate) mod agent { bulk: None, }; let resp_msg = client.request(MessageType::FsRequest, &req).await?; - let resp: FsResponse = resp_msg.payload()?; + let resp = filesystem_response(resp_msg)?; if !resp.ok { return Err(MicrosandboxError::SandboxFsOps( @@ -1668,7 +1688,7 @@ pub(crate) mod agent { bulk: None, }; let resp_msg = client.request(MessageType::FsRequest, &req).await?; - let resp: FsResponse = resp_msg.payload()?; + let resp = filesystem_response(resp_msg)?; if !resp.ok { return Err(MicrosandboxError::SandboxFsOps( @@ -2082,6 +2102,164 @@ mod tests { } } +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod pause_tests { + use microsandbox_protocol::core::{CoreError, CoreErrorKind}; + + use super::*; + + fn paused_response() -> Message { + Message::with_payload( + MessageType::CoreError, + 1, + &CoreError { + kind: CoreErrorKind::InvalidSession, + message: "sandbox is paused; resume it before starting guest work".into(), + offending_type: None, + workload_failure: None, + }, + ) + .unwrap() + } + + #[test] + fn filesystem_error_preserves_paused_diagnostic() { + let error = filesystem_response(paused_response()).unwrap_err(); + assert!(matches!(error, MicrosandboxError::Runtime(_))); + assert!(error.to_string().contains("sandbox is paused")); + } + + #[test] + fn filesystem_response_rejects_unexpected_envelope() { + let mut message = paused_response(); + message.t = MessageType::Pong; + let error = filesystem_response(message).unwrap_err(); + assert!(error.to_string().contains("agent returned")); + } + + #[test] + fn filesystem_response_retains_normal_success_and_failure() { + for ok in [true, false] { + let response = FsResponse { + ok, + error: (!ok).then(|| "permission denied".to_string()), + data: None, + }; + let message = Message::with_payload(MessageType::FsResponse, 1, &response).unwrap(); + let result = check_response(message); + if ok { + assert!(result.is_ok()); + } else { + assert!(matches!(result, Err(MicrosandboxError::SandboxFsOps(_)))); + } + } + } + + #[tokio::test] + async fn filesystem_read_stream_preserves_rejection_instead_of_eof() { + let (tx, rx) = mpsc::channel(1); + tx.send(AgentFrame::Control(paused_response())) + .await + .unwrap(); + let mut stream = FsReadStream { + id: 1, + rx, + client: None, + close_handle: None, + finished: false, + bulk: None, + bulk_finish_seen: false, + }; + let result = tokio::time::timeout(std::time::Duration::from_secs(1), stream.recv()) + .await + .expect("a read rejection must not be discarded"); + assert!( + result + .unwrap_err() + .to_string() + .contains("sandbox is paused") + ); + } + + #[tokio::test] + async fn filesystem_stream_rejection_does_not_wait_for_channel_close() { + let (tx, mut rx) = mpsc::channel(1); + tx.send(AgentFrame::Control(paused_response())) + .await + .unwrap(); + let result = tokio::time::timeout( + std::time::Duration::from_secs(1), + wait_for_ok_frame_response(&mut rx), + ) + .await + .expect("core.error must terminate the stream even with its sender alive"); + assert!( + result + .unwrap_err() + .to_string() + .contains("sandbox is paused") + ); + } + + #[tokio::test] + async fn filesystem_bulk_acceptance_preserves_paused_rejection() { + for (offer, flow) in [ + (BulkOffer::filesystem_read(), BulkFlow::GuestToHost), + (BulkOffer::filesystem_write(), BulkFlow::HostToGuest), + ] { + let (tx, mut rx) = mpsc::channel(1); + tx.send(AgentFrame::Control(paused_response())) + .await + .unwrap(); + // Keep the sender alive: the rejection, not a later channel close, is terminal. + let result = tokio::time::timeout( + std::time::Duration::from_secs(1), + receive_fs_bulk_acceptance(&mut rx, offer, flow.mask()), + ) + .await + .expect("bulk negotiation must not discard core.error"); + assert!( + result + .unwrap_err() + .to_string() + .contains("sandbox is paused") + ); + } + } + + #[tokio::test] + async fn filesystem_bulk_credit_preserves_terminal_rejection() { + let offer = BulkOffer::filesystem_write(); + let mut sender = BulkSendState::new( + BulkKind::Filesystem, + BulkFlow::HostToGuest, + offer.max_record_payload, + offer.max_record_payload as u64, + ) + .unwrap(); + let (tx, mut rx) = mpsc::channel(1); + tx.send(AgentFrame::Control(paused_response())) + .await + .unwrap(); + let result = tokio::time::timeout( + std::time::Duration::from_secs(1), + apply_next_fs_write_credit(&mut rx, &mut sender), + ) + .await + .expect("a terminal rejection must interrupt a credit wait"); + assert!( + result + .unwrap_err() + .to_string() + .contains("sandbox is paused") + ); + } +} + //-------------------------------------------------------------------------------------------------- // Re-Exports //-------------------------------------------------------------------------------------------------- diff --git a/sdk/rust/lib/sandbox/handle.rs b/sdk/rust/lib/sandbox/handle.rs index a5f47a9d6..3b3a6e68f 100644 --- a/sdk/rust/lib/sandbox/handle.rs +++ b/sdk/rust/lib/sandbox/handle.rs @@ -90,7 +90,7 @@ enum RestartAction { /// [`connect`](SandboxHandle::connect) when the sandbox is already running, or /// [`start`](SandboxHandle::start) to boot a stopped sandbox. pub struct SandboxHandle { - backend: Arc, + pub(super) backend: Arc, inner: SandboxHandleInner, name: String, } @@ -405,7 +405,10 @@ impl SandboxHandle { .local() .ok_or_else(|| MicrosandboxError::local_only(Operation::SandboxHandleMetrics))?; - if local.status != SandboxStatus::Running && local.status != SandboxStatus::Draining { + if !matches!( + local.status, + SandboxStatus::Running | SandboxStatus::Draining | SandboxStatus::Paused + ) { return Err(MicrosandboxError::SandboxNotRunning(format!( "'{}' is not running (status: {:?})", self.name, local.status @@ -647,9 +650,9 @@ impl SandboxHandle { /// Snapshot this sandbox to a bare name under the default snapshots /// directory (`~/.microsandbox/snapshots//`). /// - /// The sandbox must be stopped (or crashed); running sandboxes are - /// rejected with `MicrosandboxError::SnapshotSandboxRunning`. **Local - /// handles only** — cloud snapshot semantics are deferred. + /// Captures disk only, including running and paused sources. A live cut is + /// crash-consistent and preserves the source's running/paused state. + /// **Local handles only** — cloud snapshot semantics are deferred. #[cfg(feature = "local")] pub async fn snapshot( &self, diff --git a/sdk/rust/lib/sandbox/identity.rs b/sdk/rust/lib/sandbox/identity.rs index c5c31fbbf..4480a1df2 100644 --- a/sdk/rust/lib/sandbox/identity.rs +++ b/sdk/rust/lib/sandbox/identity.rs @@ -12,6 +12,23 @@ #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct SandboxId(pub(crate) String); +/// One local runtime generation selected before opening a name-addressed control endpoint. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[cfg(feature = "local")] +pub(crate) struct SandboxRunIdentity { + pub(crate) sandbox_id: i32, + pub(crate) run_id: i32, + pub(crate) pid: i32, +} + +/// Exact source selected for a direct branch before reserving its child. +#[derive(Clone, Debug)] +#[cfg(feature = "local")] +pub(crate) struct BranchSource { + pub(crate) name: String, + pub(crate) run: SandboxRunIdentity, +} + //-------------------------------------------------------------------------------------------------- // Methods //-------------------------------------------------------------------------------------------------- diff --git a/sdk/rust/lib/sandbox/mod.rs b/sdk/rust/lib/sandbox/mod.rs index 52e783aa3..6e3e5a2a4 100644 --- a/sdk/rust/lib/sandbox/mod.rs +++ b/sdk/rust/lib/sandbox/mod.rs @@ -6,6 +6,7 @@ //! for guest communication. pub(crate) mod attach; +pub(crate) mod branch; mod builder; mod compact; pub(crate) mod config; @@ -14,13 +15,15 @@ pub mod exec; pub(crate) mod flat_rootfs; pub mod fs; mod handle; -mod identity; +pub(crate) mod identity; pub mod init; pub(crate) mod metrics; #[cfg(feature = "local")] mod modify; #[cfg(feature = "local")] mod patch; +#[cfg(feature = "local")] +pub(crate) mod pause; #[cfg(all(feature = "local", windows))] mod reap; #[cfg(feature = "ssh")] @@ -104,6 +107,8 @@ pub(crate) use builder::{apply_checkpoint_restore_constraints, apply_snapshot_ro #[cfg(feature = "local")] pub(crate) use modify::control_checkpoint_create; #[cfg(feature = "local")] +pub(crate) use modify::control_disk_checkpoint_create; +#[cfg(feature = "local")] pub(crate) use patch::{apply_patches, build_flat_tree, build_upper_tree}; #[cfg(all(feature = "local", windows))] pub(crate) use reap::reap_leaked_runtime_process; @@ -151,6 +156,8 @@ pub use microsandbox_network::policy::{ }; #[cfg(feature = "net")] pub use microsandbox_network::{OutboundProxy, Socks5Credentials}; +#[cfg(feature = "local")] +pub use microsandbox_runtime::control::PauseControlState as SandboxPauseState; pub use microsandbox_types::SandboxLogLevel as LogLevel; pub use microsandbox_types::{CpuPlacement, PullPolicy}; #[cfg(feature = "net")] @@ -1089,6 +1096,36 @@ impl Sandbox { // ProcessHandle drops without sending SIGTERM. } + /// Keep the creator's process safety net armed until every creation check has succeeded. + #[cfg(feature = "local")] + pub(crate) async fn finish_detached_creation(&mut self) -> MicrosandboxResult<()> { + let inner = Arc::get_mut(&mut self.inner).ok_or_else(|| { + crate::MicrosandboxError::Runtime( + "creation owner was shared before finalization".into(), + ) + })?; + if let crate::backend::SandboxInner::Local(local) = inner + && let Some(handle) = local.handle.take() + { + handle.lock().await.disarm(); + } + Ok(()) + } + + /// Creation cleanup owns this exact process and must not reacquire its transition lock. + #[cfg(feature = "local")] + pub(crate) async fn terminate_creation_owner(&self) { + if let Some(local) = self.local() + && let Some(handle) = &local.handle + { + let mut handle = handle.lock().await; + if matches!(handle.try_wait(), Ok(None)) { + let _ = handle.kill(); + let _ = tokio::time::timeout(DEFAULT_KILL_TIMEOUT, handle.wait()).await; + } + } + } + fn is_local_ephemeral(&self) -> bool { #[cfg(feature = "local")] { @@ -1696,6 +1733,8 @@ pub(super) async fn remove_local_persisted_sandbox( let _transition_guard = LocalBackend::acquire_sandbox_transition_guard(&local_backend.config().run_dir(), name) .await?; + let _lineage_guard = + crate::snapshot::lineage::lock_source(&local_backend.config().run_dir(), name).await?; // Re-read after acquiring transition ownership. A stale `Sandbox` object must never delete a // newer sandbox that reused the same deterministic name, and an active identity must not be @@ -2133,4 +2172,51 @@ mod tests { .is_none() ); } + + #[tokio::test] + async fn persisted_removal_waits_for_snapshot_lineage_owner() { + let temp = tempdir().unwrap(); + let backend = std::sync::Arc::new( + LocalBackend::builder() + .home(temp.path().join("home")) + .build() + .await + .unwrap(), + ); + let pools = backend.db().await.unwrap(); + let current = super::sandbox_entity::ActiveModel { + name: Set("snapshot-source".to_string()), + config: Set("{}".to_string()), + status: Set(SandboxStatus::Stopped), + ephemeral: Set(false), + ..Default::default() + } + .insert(pools.write()) + .await + .unwrap(); + let sandbox_dir = backend.sandboxes_dir().join("snapshot-source"); + std::fs::create_dir_all(&sandbox_dir).unwrap(); + let lineage = + crate::snapshot::lineage::lock_source(&backend.config().run_dir(), "snapshot-source") + .await + .unwrap(); + let other = backend.clone(); + let mut removal = tokio::spawn(async move { + remove_local_persisted_sandbox(&other, "snapshot-source", current.id).await + }); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(50), &mut removal) + .await + .is_err() + ); + // The source must remain present while capture can still publish its cursor. + assert!(sandbox_dir.exists()); + drop(lineage); + tokio::time::timeout(std::time::Duration::from_secs(5), removal) + .await + .unwrap() + .unwrap() + .unwrap(); + assert!(!sandbox_dir.exists()); + } } diff --git a/sdk/rust/lib/sandbox/modify.rs b/sdk/rust/lib/sandbox/modify.rs index 0281d005a..e58e53aaa 100644 --- a/sdk/rust/lib/sandbox/modify.rs +++ b/sdk/rust/lib/sandbox/modify.rs @@ -7,11 +7,11 @@ use microsandbox_types::{ }; use sea_orm::{ActiveModelTrait, ColumnTrait, EntityTrait, QueryFilter, Set}; -use crate::MicrosandboxResult; use crate::backend::Backend; use crate::db::entity::{sandbox as sandbox_entity, sandbox_label as sandbox_label_entity}; use crate::error::{Operation, UnsupportedReason}; use crate::size::Mebibytes; +use crate::{MicrosandboxError, MicrosandboxResult}; use super::{SandboxConfig, SandboxStatus}; @@ -282,6 +282,9 @@ impl SandboxModificationBuilder { .await?; let status = handle.status_snapshot(); let mut config = handle.config()?; + // A failed restore can still own staged immutable lower layers. Do not let + // offline disk growth or a restart-backed modification bypass its launch gate. + crate::LocalBackend::validate_completed_restore(&config)?; let mut active = handle.active_config().ok().flatten(); let live = live_control(&self.name, status).await; let mut plan = build_plan( @@ -716,7 +719,7 @@ async fn live_control(name: &str, status: SandboxStatus) -> LiveControl { } /// Ask the sandbox process which live-control operations it serves. -async fn control_capabilities( +pub(super) async fn control_capabilities( name: &str, ) -> MicrosandboxResult { let response = control_request(name, "{\"op\":\"capabilities\"}\n".to_string()).await?; @@ -755,7 +758,7 @@ async fn connect_control_pipe( } /// Send one control request line and parse the reply. -async fn control_request( +pub(super) async fn control_request( name: &str, request: String, ) -> MicrosandboxResult { @@ -771,6 +774,136 @@ async fn control_request( Ok(response) } +/// Use the handle's local backend, never an ambient backend with a matching sandbox name. +pub(super) async fn control_request_for( + local: &crate::backend::LocalBackend, + name: &str, + request: String, +) -> MicrosandboxResult { + let response = control_request_raw_for(local, name, request).await?; + if !response.ok { + return Err(crate::MicrosandboxError::Runtime(format!( + "runtime control refused: {}", + response.error.unwrap_or_else(|| "unknown error".into()) + ))); + } + Ok(response) +} + +/// Bind the command to the selected process before sending any bytes on a reusable endpoint. +pub(super) async fn control_request_for_run( + local: &crate::backend::LocalBackend, + name: &str, + run: super::identity::SandboxRunIdentity, + request: String, +) -> MicrosandboxResult { + let candidates = crate::runtime::sandbox_agent_socket_path_candidates_for(local, name) + .into_iter() + .map(|path| microsandbox_runtime::control::control_socket_path_for(&path)); + #[cfg(unix)] + let stream = connect_control_socket(candidates).await?; + #[cfg(windows)] + let stream = connect_control_pipe( + &candidates + .into_iter() + .next() + .ok_or_else(|| MicrosandboxError::Runtime("no backend control endpoint".into()))?, + ) + .await?; + let peer_pid = control_peer_pid(&stream)?; + if peer_pid != run.pid { + return Err(MicrosandboxError::Runtime(format!( + "sandbox {name:?} control endpoint belongs to pid {peer_pid}, expected {}", + run.pid + ))); + } + local.validate_control_run(name, run).await?; + let response = control_request_over_stream(stream, &request).await?; + if !response.ok { + return Err(MicrosandboxError::Runtime(format!( + "runtime control refused: {}", + response.error.unwrap_or_else(|| "unknown error".into()) + ))); + } + Ok(response) +} + +#[cfg(target_os = "linux")] +fn control_peer_pid(stream: &tokio::net::UnixStream) -> std::io::Result { + stream.peer_cred()?.pid().ok_or_else(|| { + std::io::Error::other("control endpoint did not report its process identity") + }) +} + +#[cfg(target_os = "macos")] +fn control_peer_pid(stream: &tokio::net::UnixStream) -> std::io::Result { + use std::os::fd::AsRawFd; + let mut pid: libc::pid_t = 0; + let mut size = std::mem::size_of_val(&pid) as libc::socklen_t; + // LOCAL_PEERPID identifies the server attached to this connected socket, not a later + // process that reuses its filesystem pathname. getpeereid alone exposes only UID/GID. + let result = unsafe { + libc::getsockopt( + stream.as_raw_fd(), + libc::SOL_LOCAL, + libc::LOCAL_PEERPID, + (&mut pid as *mut libc::pid_t).cast(), + &mut size, + ) + }; + if result == -1 { + return Err(std::io::Error::last_os_error()); + } + if size as usize != std::mem::size_of_val(&pid) || pid <= 0 { + return Err(std::io::Error::other( + "invalid control endpoint process identity", + )); + } + Ok(pid) +} + +#[cfg(all(unix, not(any(target_os = "linux", target_os = "macos"))))] +fn control_peer_pid(_stream: &tokio::net::UnixStream) -> std::io::Result { + Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "control endpoint process verification is unsupported on this platform", + )) +} + +#[cfg(windows)] +fn control_peer_pid( + stream: &tokio::net::windows::named_pipe::NamedPipeClient, +) -> std::io::Result { + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::{Foundation::HANDLE, System::Pipes::GetNamedPipeServerProcessId}; + let mut pid = 0u32; + let result = unsafe { GetNamedPipeServerProcessId(stream.as_raw_handle() as HANDLE, &mut pid) }; + if result == 0 { + return Err(std::io::Error::last_os_error()); + } + i32::try_from(pid) + .map_err(|_| std::io::Error::other("control endpoint PID exceeds supported range")) +} + +async fn control_request_raw_for( + local: &crate::backend::LocalBackend, + name: &str, + request: String, +) -> MicrosandboxResult { + let candidates = crate::runtime::sandbox_agent_socket_path_candidates_for(local, name) + .into_iter() + .map(|path| microsandbox_runtime::control::control_socket_path_for(&path)); + #[cfg(unix)] + let stream = connect_control_socket(candidates).await?; + #[cfg(windows)] + let stream = + connect_control_pipe(&candidates.into_iter().next().ok_or_else(|| { + crate::MicrosandboxError::Runtime("no backend control endpoint".into()) + })?) + .await?; + control_request_over_stream(stream, &request).await +} + async fn control_request_raw( name: &str, request: String, @@ -868,12 +1001,17 @@ pub(crate) async fn control_disk_compact( /// A published checkpoint may coexist with failed source recovery. Preserve both facts so the /// snapshot caller can publish the artifact before reporting a typed partial failure. pub(crate) async fn control_checkpoint_create( + local: &crate::backend::LocalBackend, name: &str, checkpoint_id: String, ) -> MicrosandboxResult { - let capabilities = control_capabilities(name).await?; - if !capabilities.checkpoint_create { - return Err(crate::MicrosandboxError::unsupported( + let capabilities = + control_request_for(local, name, "{\"op\":\"capabilities\"}\n".into()).await?; + if !capabilities + .capabilities + .is_some_and(|capabilities| capabilities.checkpoint_create) + { + return Err(MicrosandboxError::unsupported( Operation::SnapshotOps, UnsupportedReason::NotAvailable( "this running sandbox does not support full checkpoint capture".into(), @@ -884,9 +1022,12 @@ pub(crate) async fn control_checkpoint_create( checkpoint_id, intent: microsandbox_runtime::control::CheckpointCaptureIntent::FullSnapshot, }; - let mut line = serde_json::to_string(&request)?; - line.push('\n'); - let response = control_request_raw(name, line).await?; + let response = control_request_raw_for( + local, + name, + format!("{}\n", serde_json::to_string(&request)?), + ) + .await?; checkpoint_response(response) } @@ -911,6 +1052,31 @@ fn checkpoint_response( ))) } +/// Request disk-only capture without falling back to full-state capture or a stopped copy. +pub(crate) async fn control_disk_checkpoint_create( + local: &crate::backend::LocalBackend, + name: &str, + checkpoint_id: String, +) -> MicrosandboxResult { + let capabilities = + control_request_for(local, name, "{\"op\":\"capabilities\"}\n".into()).await?; + if !capabilities + .capabilities + .is_some_and(|c| c.disk_checkpoint_create) + { + return Err(MicrosandboxError::unsupported(Operation::SnapshotOps, + UnsupportedReason::NotAvailable("this runtime does not support live disk-only snapshots; recreate the sandbox with the updated runtime".into()))); + } + let request = + microsandbox_runtime::control::ControlRequest::DiskCheckpointCreate { checkpoint_id }; + let mut line = serde_json::to_string(&request)?; + line.push('\n'); + let response = control_request_for(local, name, line).await?; + response.disk_checkpoint.ok_or_else(|| { + MicrosandboxError::Runtime("runtime omitted the disk-only capture result".into()) + }) +} + /// Send the value-bearing live secret batch to the sandbox process. The /// request travels only over the private per-sandbox control endpoint and is /// never logged; failures surface the runtime's error, which carries secret @@ -2495,6 +2661,76 @@ mod tests { use crate::backend::LocalBackend; use crate::size::SizeExt; + #[cfg(unix)] + #[tokio::test] + async fn full_checkpoint_uses_selected_backend_and_retains_post_publish_failure() { + use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; + + let first_home = tempfile::tempdir_in("/tmp").unwrap(); + let second_home = tempfile::tempdir_in("/tmp").unwrap(); + let first = LocalBackend::builder() + .home(first_home.path()) + .build() + .await + .unwrap(); + let second = LocalBackend::builder() + .home(second_home.path()) + .build() + .await + .unwrap(); + let mut servers = Vec::new(); + for (local, label, resume_ok) in [(&first, "first", true), (&second, "second", false)] { + let agent = + crate::runtime::sandbox_agent_socket_path_candidates_for(local, "worker").remove(0); + let path = microsandbox_runtime::control::control_socket_path_for(&agent); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + let listener = tokio::net::UnixListener::bind(path).unwrap(); + servers.push(tokio::spawn(async move { + for request_index in 0..2 { + let (stream, _) = listener.accept().await.unwrap(); + let mut stream = BufReader::new(stream); + let mut line = String::new(); + stream.read_line(&mut line).await.unwrap(); + let request: serde_json::Value = serde_json::from_str(&line).unwrap(); + let response = if request_index == 0 { + assert_eq!(request["op"], "capabilities"); + serde_json::json!({"ok":true,"capabilities":{"checkpoint_create":true,"cpu_resize":false,"memory_resize":false,"secrets_update":false}}) + } else { + assert_eq!(request["op"], "checkpoint_create"); + serde_json::json!({"ok":resume_ok,"error":"source resume failed","checkpoint":{ + "checkpoint_id":request["checkpoint_id"], "checkpoint_root":format!("sha256:{}", "a".repeat(64)), + "path":format!("/capture/{label}"), "memory_mode":"full", "memory_logical_bytes":4096, "memory_emitted_bytes":4096 + }}) + }; + stream.get_mut().write_all(format!("{response}\n").as_bytes()).await.unwrap(); + } + })); + } + let first_capture = control_checkpoint_create(&first, "worker", "first-checkpoint".into()) + .await + .unwrap(); + let second_capture = + control_checkpoint_create(&second, "worker", "second-checkpoint".into()) + .await + .unwrap(); + assert_eq!( + first_capture.checkpoint.path, + std::path::Path::new("/capture/first") + ); + assert!(first_capture.recovery_error.is_none()); + assert_eq!( + second_capture.checkpoint.path, + std::path::Path::new("/capture/second") + ); + assert_eq!( + second_capture.recovery_error.as_deref(), + Some("source resume failed") + ); + for server in servers { + server.await.unwrap(); + } + } + #[tokio::test] async fn size_setters_accept_bare_mib_and_typed_sizes() { let temp = tempdir().unwrap(); diff --git a/sdk/rust/lib/sandbox/pause.rs b/sdk/rust/lib/sandbox/pause.rs new file mode 100644 index 000000000..fff1a3ad7 --- /dev/null +++ b/sdk/rust/lib/sandbox/pause.rs @@ -0,0 +1,493 @@ +//! Resident pause/resume through the existing host control endpoint. + +use microsandbox_runtime::control::ControlRequest; + +use crate::backend::sandbox::SandboxIdentity; +use crate::backend::{Backend, LocalBackend}; +use crate::error::Operation; +use crate::{MicrosandboxError, MicrosandboxResult}; + +use super::{Sandbox, SandboxHandle, SandboxPauseState, modify}; + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl Sandbox { + /// Internal CLI lookup for an immediately following authoritative control mutation. + /// + /// Keep database/runtime reconciliation, but skip the pause observation used by ordinary + /// `get`/`list`: that observation is already stale by the time the mutation executes. + #[doc(hidden)] + pub async fn get_for_control(name: &str) -> MicrosandboxResult { + let backend = crate::backend::default_backend(); + if let Some(local) = backend.as_local() { + let (model, pid) = match local.try_control_handle_state(name).await? { + Some(target) => target, + None => local.sandbox_handle_state(name, None).await?, + }; + return Ok(SandboxHandle::from_local_model(backend, model, pid)); + } + backend.sandboxes().get(backend.clone(), name).await + } + + /// Suspend this resident VM without creating a snapshot or releasing RAM. + pub async fn pause(&self) -> MicrosandboxResult<()> { + lifecycle( + self.name(), + self.identity(), + self.backend().as_ref(), + ControlRequest::Pause, + ) + .await + .map(|_| ()) + } + + /// Resume the same VM and processes, correcting wall clock before thawing workloads. + pub async fn resume(&self) -> MicrosandboxResult<()> { + lifecycle( + self.name(), + self.identity(), + self.backend().as_ref(), + ControlRequest::Resume, + ) + .await + .map(|_| ()) + } + + /// Inspect the host-confirmed pause state without contacting the suspended guest. + pub async fn pause_state(&self) -> MicrosandboxResult { + lifecycle( + self.name(), + self.identity(), + self.backend().as_ref(), + ControlRequest::PauseState, + ) + .await + } +} + +impl SandboxHandle { + /// Suspend an existing resident sandbox without connecting to its guest. + pub async fn pause(&self) -> MicrosandboxResult<()> { + lifecycle( + self.name(), + self.identity(), + self.backend.as_ref(), + ControlRequest::Pause, + ) + .await + .map(|_| ()) + } + + /// Resume an existing user-paused sandbox through host control. + pub async fn resume(&self) -> MicrosandboxResult<()> { + lifecycle( + self.name(), + self.identity(), + self.backend.as_ref(), + ControlRequest::Resume, + ) + .await + .map(|_| ()) + } + + /// Inspect resident suspension without opening an agent connection. + pub async fn pause_state(&self) -> MicrosandboxResult { + lifecycle( + self.name(), + self.identity(), + self.backend.as_ref(), + ControlRequest::PauseState, + ) + .await + } +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +/// Overlay resident suspension on database lifecycle without persisting a stale pause on crash. +pub(crate) async fn projected_status( + local: &LocalBackend, + name: &str, + status: super::SandboxStatus, +) -> super::SandboxStatus { + if status != super::SandboxStatus::Running { + return status; + } + // Old runtimes have no pause endpoint. Bound observation so a busy or unavailable host + // never makes ordinary list/get wait for an entire checkpoint operation. + let request = modify::control_request_for(local, name, "{\"op\":\"pause_state\"}\n".into()); + match tokio::time::timeout(std::time::Duration::from_millis(250), request).await { + Ok(Ok(response)) + if response + .pause + .as_ref() + .is_some_and(|state| state.paused || state.recovery_required) => + { + super::SandboxStatus::Paused + } + _ => status, + } +} + +async fn lifecycle( + name: &str, + identity: SandboxIdentity, + backend: &dyn Backend, + request: ControlRequest, +) -> MicrosandboxResult { + let operation = if matches!(request, ControlRequest::Resume) { + Operation::SandboxResume + } else { + Operation::SandboxPause + }; + let local = backend + .as_local() + .ok_or_else(|| MicrosandboxError::local_only(operation))?; + let SandboxIdentity::Local(expected_id) = identity else { + return Err(MicrosandboxError::local_only(operation)); + }; + let _transition = + LocalBackend::acquire_sandbox_transition_guard(&local.config().run_dir(), name).await?; + let run = local.control_run_identity(name, expected_id).await?; + // The mutation itself is authoritative. Unknown operations fail on older runtimes, and + // successful replies must carry pause state; neither case can silently become a no-op. + let line = format!("{}\n", serde_json::to_string(&request)?); + let response = modify::control_request_for_run(local, name, run, line).await?; + let state = response + .pause + .ok_or_else(|| MicrosandboxError::Runtime("control response omitted pause state".into()))?; + // An acknowledgement must confirm the requested transition, not just contain some + // observation. State inspection itself must still be able to report recovery required. + let expected = match request { + ControlRequest::Pause => Some(true), + ControlRequest::Resume => Some(false), + _ => None, + }; + if expected.is_some_and(|paused| state.paused != paused || state.recovery_required) { + return Err(MicrosandboxError::Runtime( + "control response did not confirm the requested pause transition".into(), + )); + } + Ok(state) +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(all(test, unix))] +mod tests { + use std::sync::Arc; + + use sea_orm::{EntityTrait, Set}; + use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; + + use super::*; + use crate::backend::with_backend; + + async fn seed_run(local: &LocalBackend, name: &str) -> i32 { + use crate::db::entity::{run, sandbox}; + let db = local.db().await.unwrap(); + let id = sandbox::Entity::insert(sandbox::ActiveModel { + name: Set(name.into()), + config: Set("{}".into()), + status: Set(super::super::SandboxStatus::Running), + ephemeral: Set(false), + ..Default::default() + }) + .exec(db.write()) + .await + .unwrap() + .last_insert_id; + run::Entity::insert(run::ActiveModel { + sandbox_id: Set(id), + pid: Set(Some(std::process::id() as i32)), + status: Set(run::RunStatus::Running), + ..Default::default() + }) + .exec(db.write()) + .await + .unwrap(); + id + } + + #[tokio::test] + async fn pause_observation_uses_bound_backend_outside_its_ambient_scope() { + // macOS's per-user TMPDIR may already consume most of the Unix socket path limit. + let ambient_home = tempfile::tempdir_in("/tmp").unwrap(); + let bound_home = tempfile::tempdir_in("/tmp").unwrap(); + let ambient: Arc = Arc::new( + LocalBackend::builder() + .home(ambient_home.path()) + .build() + .await + .unwrap(), + ); + let bound = LocalBackend::builder() + .home(bound_home.path()) + .build() + .await + .unwrap(); + let id = seed_run(&bound, "same-name").await; + let agent = + crate::runtime::sandbox_agent_socket_path_candidates_for(&bound, "same-name").remove(0); + let path = microsandbox_runtime::control::control_socket_path_for(&agent); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + let listener = tokio::net::UnixListener::bind(path).unwrap(); + let server = tokio::spawn(async move { + // Each observation is one exchange; it needs no capabilities preflight. + for _ in 0..2 { + let (stream, _) = listener.accept().await.unwrap(); + let mut stream = BufReader::new(stream); + let mut line = String::new(); + stream.read_line(&mut line).await.unwrap(); + assert_eq!(line, "{\"op\":\"pause_state\"}\n"); + let response = "{\"ok\":true,\"pause\":{\"paused\":true,\"recovery_required\":false,\"capture_unavailable\":null}}\n"; + stream + .get_mut() + .write_all(response.as_bytes()) + .await + .unwrap(); + } + }); + with_backend(ambient, async { + assert_eq!( + projected_status(&bound, "same-name", super::super::SandboxStatus::Running).await, + super::super::SandboxStatus::Paused + ); + assert!( + lifecycle( + "same-name", + SandboxIdentity::Local(id), + &bound, + ControlRequest::PauseState + ) + .await + .unwrap() + .paused + ); + }) + .await; + server.await.unwrap(); + } + + #[tokio::test] + async fn lifecycle_sends_one_mutation_and_requires_an_authoritative_reply() { + for (operation, response, accepted) in [ + ( + "pause", + "{\"ok\":true,\"pause\":{\"paused\":true,\"recovery_required\":false}}\n", + true, + ), + ( + "resume", + "{\"ok\":true,\"pause\":{\"paused\":false,\"recovery_required\":false}}\n", + true, + ), + // Old runtime unknown-operation errors and unsupported current kernels must fail. + ( + "pause", + "{\"ok\":false,\"error\":\"unknown variant pause\"}\n", + false, + ), + ( + "resume", + "{\"ok\":false,\"error\":\"pause/resume unavailable\"}\n", + false, + ), + ("resume", "{\"ok\":true}\n", false), + ( + "pause", + "{\"ok\":true,\"pause\":{\"paused\":false,\"recovery_required\":false}}\n", + false, + ), + ( + "resume", + "{\"ok\":true,\"pause\":{\"paused\":true,\"recovery_required\":false}}\n", + false, + ), + ( + "pause", + "{\"ok\":true,\"pause\":{\"paused\":true,\"recovery_required\":true}}\n", + false, + ), + ] { + let home = tempfile::tempdir_in("/tmp").unwrap(); + let backend = LocalBackend::builder() + .home(home.path()) + .build() + .await + .unwrap(); + let id = seed_run(&backend, "source").await; + let agent = + crate::runtime::sandbox_agent_socket_path_candidates_for(&backend, "source") + .remove(0); + let path = microsandbox_runtime::control::control_socket_path_for(&agent); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + let listener = tokio::net::UnixListener::bind(path).unwrap(); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut stream = BufReader::new(stream); + let mut line = String::new(); + stream.read_line(&mut line).await.unwrap(); + assert_eq!(line, format!("{{\"op\":\"{operation}\"}}\n")); + stream + .get_mut() + .write_all(response.as_bytes()) + .await + .unwrap(); + }); + let request = if operation == "pause" { + ControlRequest::Pause + } else { + ControlRequest::Resume + }; + let result = tokio::time::timeout( + std::time::Duration::from_secs(2), + lifecycle("source", SandboxIdentity::Local(id), &backend, request), + ) + .await + .unwrap(); + assert_eq!(result.is_ok(), accepted, "{operation}: {response}"); + server.await.unwrap(); + } + } + + #[tokio::test] + async fn stale_receiver_refuses_pause_resume_and_branch_before_control_connect() { + use crate::db::entity::sandbox; + let home = tempfile::tempdir_in("/tmp").unwrap(); + let backend = Arc::new( + LocalBackend::builder() + .home(home.path()) + .build() + .await + .unwrap(), + ); + let old_id = seed_run(&backend, "reused").await; + let model = sandbox::Entity::find_by_id(old_id) + .one(backend.db().await.unwrap().read()) + .await + .unwrap() + .unwrap(); + let stale = SandboxHandle::from_local_model( + backend.clone(), + model, + Some(std::process::id() as i32), + ); + let (client_io, mut server_io) = tokio::io::duplex(4096); + let handshake = tokio::spawn(async move { + use microsandbox_protocol::{ + codec, + core::Ready, + message::{Message, MessageType}, + }; + server_io.write_all(&1u32.to_be_bytes()).await.unwrap(); + server_io.write_all(&1024u32.to_be_bytes()).await.unwrap(); + codec::write_message( + &mut server_io, + &Message::with_payload(MessageType::Ready, 0, &Ready::default()).unwrap(), + ) + .await + .unwrap(); + }); + let client = crate::agent::AgentClient::connect_stream_with_timeout( + client_io, + std::time::Duration::from_secs(1), + ) + .await + .unwrap(); + handshake.await.unwrap(); + let mut config = super::super::SandboxConfig::default(); + config.spec.name = "reused".into(); + let live = Sandbox::from_local( + backend.clone(), + crate::backend::SandboxLocalState { + db_id: old_id, + handle: None, + client: Arc::new(client), + }, + config, + ); + sandbox::Entity::delete_by_id(old_id) + .exec(backend.db().await.unwrap().write()) + .await + .unwrap(); + let replacement_id = seed_run(&backend, "reused").await; + assert_ne!(old_id, replacement_id); + for result in [ + stale.pause().await, + stale.resume().await, + stale.pause_state().await.map(|_| ()), + ] { + assert!(matches!( + result, + Err(MicrosandboxError::SandboxReplaced { .. }) + )); + } + assert!(matches!( + stale.branch("child").await, + Err(MicrosandboxError::SandboxReplaced { .. }) + )); + for result in [ + live.pause().await, + live.resume().await, + live.pause_state().await.map(|_| ()), + ] { + assert!(matches!( + result, + Err(MicrosandboxError::SandboxReplaced { .. }) + )); + } + assert!(matches!( + live.branch("child").await, + Err(MicrosandboxError::SandboxReplaced { .. }) + )); + assert!(!backend.sandboxes_dir().join("child").exists()); + } + + #[tokio::test] + async fn control_peer_mismatch_sends_no_mutation() { + let home = tempfile::tempdir_in("/tmp").unwrap(); + let backend = LocalBackend::builder() + .home(home.path()) + .build() + .await + .unwrap(); + let agent = + crate::runtime::sandbox_agent_socket_path_candidates_for(&backend, "source").remove(0); + let path = microsandbox_runtime::control::control_socket_path_for(&agent); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + let listener = tokio::net::UnixListener::bind(path).unwrap(); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut line = String::new(); + assert_eq!( + BufReader::new(stream).read_line(&mut line).await.unwrap(), + 0 + ); + }); + let result = modify::control_request_for_run( + &backend, + "source", + super::super::identity::SandboxRunIdentity { + sandbox_id: 1, + run_id: 1, + pid: std::process::id() as i32 + 1, + }, + "{\"op\":\"pause\"}\n".into(), + ) + .await; + assert!( + result + .unwrap_err() + .to_string() + .contains("control endpoint belongs to pid") + ); + server.await.unwrap(); + } +} diff --git a/sdk/rust/lib/snapshot/archive.rs b/sdk/rust/lib/snapshot/archive.rs index c15e1dd83..596b67d37 100644 --- a/sdk/rust/lib/snapshot/archive.rs +++ b/sdk/rust/lib/snapshot/archive.rs @@ -1,4 +1,5 @@ -//! Snapshot save / load via `.tar.zst` bundles. +//! Snapshot save / load via `.msb` bundles (tar + zstd, or explicit plain tar). +//! Encoding is detected from contents; legacy suffixes and extensionless inputs remain valid. //! //! Default archive format is zstd-compressed tar. Regular files with holes, notably the sparse `upper.ext4` whose logical size is the configured upper cap rather than the data //! written, are stored as old-GNU sparse entries (type `S`): only allocated extents are read and archived, so save cost scales with the data a sandbox actually wrote instead of @@ -8,6 +9,7 @@ //! depths, produced by our own save path), and owning the walk lets sparse entries be restored map-driven: data runs copied straight off the wire, holes never written and kept //! unallocated per platform ([`extent::mark_sparse`] on NTFS, [`extent::punch_hole_aligned`] on APFS). `tokio_tar` remains the header codec and the dense-entry writer. +mod batch; mod delta; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; @@ -59,6 +61,19 @@ const GNU_EXT_SPARSE_SLOTS: usize = 21; // Types //-------------------------------------------------------------------------------------------------- +/// Options for installing an archive in a local snapshot group. +#[derive(Debug, Clone, Default)] +pub struct LoadOpts { + /// Group-store root; defaults to the configured snapshots directory. + pub dest: Option, + /// Explicit base selector for omitted disk layers and RAM objects. + pub base: Option, + /// Existing/new destination group, or a freshly generated group when omitted. + pub group: Option, + /// Select the imported target even when it is not a fast-forward. + pub set_head: bool, +} + /// Options for [`super::Snapshot::save`]. #[derive(Debug, Clone, Default)] pub struct SaveOpts { @@ -69,7 +84,8 @@ pub struct SaveOpts { pub with_image: bool, /// Skip zstd compression and write a plain `.tar`. Default: zstd. pub plain_tar: bool, - /// Export only disk layers after this exact base snapshot (name, directory, or archive). + /// Omit disk layers and RAM objects supplied by this base (name, directory, or archive). + /// The base must be an exact physical disk prefix; full-snapshot metadata stays complete. /// Mutually exclusive with `last_layers` and `with_parents`. pub since: Option, /// Export the newest N sealed disk layers, requiring an explicit base when loading omissions. @@ -289,13 +305,25 @@ pub(super) async fn save_snapshot( let mut parents: Vec = Vec::new(); if opts.with_parents { - let mut current = head.manifest().parent.clone(); - while let Some(parent_id) = current { - let parent_path = resolve_parent_artifact(local, parent_id.as_str()).await?; + let mut current = head.clone(); + let mut visited = HashSet::from([head.id().to_string()]); + while let Some(parent_id) = current.manifest().parent.clone() { + if !visited.insert(parent_id.to_string()) { + return Err(MicrosandboxError::SnapshotIntegrity(format!( + "snapshot parent chain contains a cycle at {parent_id}" + ))); + } + let parent_path = resolve_parent_artifact(local, ¤t, parent_id.as_str()).await?; let parent = store::open_snapshot(local, parent_path.to_string_lossy().as_ref()).await?; + if parent.id() != &parent_id { + return Err(MicrosandboxError::SnapshotIntegrity(format!( + "snapshot parent path contains {}, expected {parent_id}", + parent.id() + ))); + } parents.push(parent.clone()); - current = parent.manifest().parent.clone(); + current = parent; } } parents.reverse(); @@ -920,167 +948,34 @@ pub(super) async fn load_snapshot_with_base( dest: Option<&Path>, base: Option<&str>, ) -> MicrosandboxResult { - let total_started = Instant::now(); - let snapshots_dir = match dest { - Some(d) => d.to_path_buf(), - None => local.snapshots_dir(), - }; - tokio::fs::create_dir_all(&snapshots_dir).await?; - let cache_dir = local.cache_dir(); - tokio::fs::create_dir_all(&cache_dir).await?; - - let snapshot_stage = tempfile::Builder::new() - .prefix(".msb-snapshot-import-") - .tempdir_in(&snapshots_dir)?; - let cache_tmp_dir = cache_dir.join("tmp"); - tokio::fs::create_dir_all(&cache_tmp_dir).await?; - let cache_stage = tempfile::Builder::new() - .prefix("snapshot-import-") - .tempdir_in(&cache_tmp_dir)?; - - // Stream rather than slurp — archives carry the full upper layer and are - // routinely multi-GB. - let file = tokio::fs::File::open(archive).await?; - let mut buf = BufReader::new(file); - let is_zstd = { - let bytes = buf.fill_buf().await?; - bytes.starts_with(&[0x28, 0xb5, 0x2f, 0xfd]) - }; + load_snapshot_with_options( + local, + archive, + LoadOpts { + dest: dest.map(Path::to_path_buf), + base: base.map(str::to_string), + ..Default::default() + }, + ) + .await +} - let unpack_started = Instant::now(); - let unpacked = if is_zstd { - let decoder = ZstdDecoder::new(buf); - // The decoder and archive walker both carry sizeable buffers across - // await points. Keep their combined future off Tokio's worker stack. - Box::pin(unpack_archive( - decoder, - snapshot_stage.path(), - cache_stage.path(), - )) - .await? - } else { - Box::pin(unpack_archive( - buf, - snapshot_stage.path(), - cache_stage.path(), - )) - .await? - }; - let unpack_us = unpack_started.elapsed().as_micros(); +pub(super) async fn load_snapshot_with_options( + local: &LocalBackend, + archive: &Path, + opts: LoadOpts, +) -> MicrosandboxResult { + let mut loaded = batch::load(local, &[archive.to_path_buf()], opts).await?; + Ok(loaded.remove(0)) +} - let validate_started = Instant::now(); - if unpacked.inventory.is_none() { - super::migration::normalize_staged(local.db().await?, &unpacked.manifest_dirs).await?; - } else if let Some(inventory) = unpacked.inventory.as_ref() { - delta::resolve( - local, - inventory, - snapshot_stage.path(), - cache_stage.path(), - base, - ) - .await?; - materialize_inventory_layers(inventory, snapshot_stage.path()).await?; - } - let imported = verify_imported_snapshots(local, &unpacked.manifest_dirs).await?; - for snapshot in &imported { - super::metadata::write(snapshot.path(), snapshot.labels()).await?; - } - if let Some(inventory) = unpacked.inventory.as_ref() { - validate_inventory_snapshot_bindings(inventory, &imported)?; - } - let head_index = match unpacked.head.as_deref() { - Some(head) => imported - .iter() - .position(|snapshot| snapshot.id().as_str() == head) - .ok_or_else(|| { - MicrosandboxError::Custom(format!("archive inventory head {head} was not imported")) - })?, - None => select_head_snapshot(&imported)?, - }; - let head_stage_path = imported[head_index].path().to_path_buf(); - let head_relative = head_stage_path - .strip_prefix(snapshot_stage.path()) - .map_err(|_| MicrosandboxError::Custom("imported snapshot escaped staging dir".into()))? - .to_path_buf(); - let head_manifest = imported[head_index].manifest().clone(); - let head_path = snapshots_dir.join(&head_relative); - let validate_us = validate_started.elapsed().as_micros(); - - let promote_started = Instant::now(); - ensure_promote_targets_available(snapshot_stage.path(), &snapshots_dir).await?; - // Cache installation carries hashing buffers across await points. Keep - // that future on the heap so the archive loader remains within Windows' - // smaller default worker-thread stack. - Box::pin(install_staged_cache( - cache_stage.path(), - &cache_dir, - &head_manifest, - )) - .await?; - promote_stage(snapshot_stage.path(), &snapshots_dir).await?; - - let snap = store::open_snapshot(local, head_path.to_string_lossy().as_ref()).await?; - - // Index this and any sibling artifacts that landed in the dest dir. - let _ = store::reindex_dir(local, &snapshots_dir).await; - let promote_index_us = promote_started.elapsed().as_micros(); - - let (state_kind, format, fstype, checkpoint_manifest_digest, size_bytes) = - match &snap.manifest().state { - SnapshotState::File(state) => ( - "file".to_string(), - Some(state.disk_format), - Some(state.filesystem.clone()), - None, - Some(state.virtual_size), - ), - SnapshotState::Checkpoint(state) => ( - "checkpoint".to_string(), - None, - None, - Some(state.checkpoint_root.clone()), - None, - ), - }; - let handle = SnapshotHandle { - snapshot_id: snap.id().to_string(), - digest: snap.digest().to_string(), - name: snap - .path() - .file_name() - .and_then(|s| s.to_str()) - .map(|s| s.to_string()), - parent_digest: snap.manifest().parent.as_ref().map(ToString::to_string), - scope: snap.manifest().scope, - image_ref: snap.manifest().image.reference.clone(), - state_kind, - format, - fstype, - checkpoint_manifest_digest, - size_bytes, - locality: "embedded".into(), - availability: "ready".into(), - migration_state: "canonical".into(), - migration_error_code: None, - created_at: chrono::DateTime::parse_from_rfc3339(&snap.manifest().capture.created_at) - .map(|d| d.naive_utc()) - .unwrap_or_else(|_| chrono::Utc::now().naive_utc()), - artifact_path: snap.path().to_path_buf(), - }; - let archive_bytes = tokio::fs::metadata(archive).await?.len(); - tracing::info!( - target: "microsandbox_checkpoint_timing", - operation = "snapshot_load_archive", - zstd = is_zstd, - archive_bytes, - total_us = total_started.elapsed().as_micros(), - unpack_us, - validate_us, - promote_index_us, - "snapshot archive load timing" - ); - Ok(handle) +/// Resolve all supplied archives together, publishing their members into one group. +pub(super) async fn load_snapshots( + local: &LocalBackend, + archives: &[PathBuf], + opts: LoadOpts, +) -> MicrosandboxResult> { + batch::load(local, archives, opts).await } /// Consume a current archive directly into a child sandbox's staging directory. @@ -1114,7 +1009,7 @@ pub(crate) async fn materialize_archive_for_child_with_base( .tempdir_in(&cache_tmp_dir)?; let file = tokio::fs::File::open(archive).await?; - let mut buffered = BufReader::new(file); + let mut buffered = BufReader::with_capacity(1024 * 1024, file); let is_zstd = buffered .fill_buf() .await? @@ -1326,7 +1221,7 @@ async fn write_archive_entries( cache_files: &[(PathBuf, String)], head: &Snapshot, opts: &SaveOpts, - dependencies: Option<&delta::DiskDependencies>, + dependencies: Option<&delta::Dependencies>, ) -> MicrosandboxResult<()> where W: tokio::io::AsyncWrite + Unpin + Send, @@ -1460,6 +1355,30 @@ where Ok(()) } +async fn normalize_imported_descriptor(snapshot: &Snapshot) -> MicrosandboxResult<()> { + let path = snapshot.path().join(DESCRIPTOR_FILENAME); + let canonical = snapshot + .manifest() + .to_canonical_bytes() + .map_err(|error| MicrosandboxError::SnapshotIntegrity(error.to_string()))?; + if tokio::fs::read(&path).await? == canonical { + return Ok(()); + } + if let SnapshotState::File(file) = &snapshot.manifest().state { + for layer in &file.layers { + let source = snapshot.layer_path(layer); + let destination = snapshot.path().join(file.layer_path(layer)); + if source != destination { + tokio::fs::create_dir_all(destination.parent().expect("layer has parent")).await?; + tokio::fs::rename(source, destination).await?; + } + } + } + tokio::fs::write(&path, canonical).await?; + tokio::fs::File::open(path).await?.sync_all().await?; + Ok(()) +} + async fn build_archive_inventory( snapshots: &[Snapshot], cache_files: &[(PathBuf, String)], @@ -1586,12 +1505,28 @@ async fn build_archive_inventory( snapshot_members.sort_by(|left, right| left.snapshot_id.cmp(&right.snapshot_id)); entries.sort_by(|left, right| left.path.as_bytes().cmp(right.path.as_bytes())); - let suggested_name = head - .path() - .file_name() - .and_then(|name| name.to_str()) - .filter(|name| !name.is_empty() && name.len() <= 255) - .map(str::to_string); + // Names are local aliases, not descriptor identity. Carry them as optional + // archive metadata so importing a group preserves its useful selectors. + let mut member_names = BTreeMap::new(); + for snapshot in snapshots { + if let Some(name) = super::group::member_name(snapshot.path())? { + member_names.insert(snapshot.id().to_string(), name); + } + } + let suggested_name = member_names.get(head.id().as_str()).cloned().or_else(|| { + head.path() + .file_name() + .and_then(|name| name.to_str()) + .filter(|name| !name.is_empty() && name.len() <= 255) + .map(str::to_string) + }); + let mut extensions = BTreeMap::new(); + if !member_names.is_empty() { + extensions.insert( + "msb-snapshot-member-names".into(), + serde_json::to_value(member_names)?, + ); + } let encoded_bytes = entries.iter().map(|entry| entry.encoded_size).sum(); let apparent_bytes = entries.iter().map(|entry| entry.apparent_size).sum(); Ok(ArchiveInventory { @@ -1606,7 +1541,7 @@ async fn build_archive_inventory( apparent_bytes, }, entries, - extensions: BTreeMap::new(), + extensions, requires: vec![ARCHIVE_MEMBER_TRANSPORT_ALGORITHM.into()], }) } @@ -1932,8 +1867,23 @@ where let mut header = Header::new_gnu(); header.set_metadata_in_mode(&meta, HeaderMode::Complete); if header.set_path(name).is_err() { - // Needs a GNU long-name entry; the dense path emits one. - return Ok(None); + // GNU long-name records apply to sparse members too. Canonical qcow2 + // checkpoint paths exceed the fixed name field by one byte. + let mut long = Header::new_gnu(); + // set_path normalizes away the leading dots; use the exact GNU + // marker emitted by the existing dense writer and accepted by readers. + long.as_gnu_mut().expect("GNU header").name[..13].copy_from_slice(b"././@LongLink"); + long.set_entry_type(EntryType::GNULongName); + long.set_mode(0o644); + long.set_size(name.len() as u64 + 1); + long.set_cksum(); + let dst = builder.get_mut(); + dst.write_all(long.as_bytes()).await?; + dst.write_all(name.as_bytes()).await?; + dst.write_all(&[0]).await?; + let padding = tar_pad(name.len() as u64 + 1) as usize; + dst.write_all(&[0u8; TAR_BLOCK as usize][..padding]).await?; + header.set_path("sparse-member")?; } header.set_entry_type(EntryType::GNUSparse); header.set_size(map.archived); @@ -2447,6 +2397,29 @@ fn tar_pad(size: u64) -> u64 { (TAR_BLOCK - size % TAR_BLOCK) % TAR_BLOCK } +/// Amortize async filesystem dispatch while preserving the caller's bounded +/// reader and transport hashing. Reuse the same buffer across sparse extents. +async fn copy_archive_payload( + reader: &mut R, + writer: &mut W, + buffer: &mut [u8], +) -> std::io::Result +where + R: tokio::io::AsyncRead + Unpin, + W: tokio::io::AsyncWrite + Unpin, +{ + use tokio::io::AsyncWriteExt; + let mut copied = 0; + loop { + let read = reader.read(buffer).await?; + if read == 0 { + return Ok(copied); + } + writer.write_all(&buffer[..read]).await?; + copied += read as u64; + } +} + /// Stream a dense entry's bytes into `target`. async fn unpack_dense_entry( reader: &mut R, @@ -2466,7 +2439,8 @@ where hasher: archive_transport_hasher(kind, archive_path, size, size, &[]), bytes_read: 0, }; - let copied = tokio::io::copy(&mut source, &mut file).await?; + let mut buffer = vec![0u8; 1024 * 1024]; + let copied = copy_archive_payload(&mut source, &mut file, &mut buffer).await?; if copied != size { return Err(MicrosandboxError::Custom( "archive truncated mid-entry".into(), @@ -2586,6 +2560,7 @@ where std_file.set_len(realsize)?; let mut file = tokio::fs::File::from_std(std_file); let mut transport = archive_transport_hasher(kind, archive_path, archived, realsize, &map); + let mut buffer = vec![0u8; 1024 * 1024]; for (offset, numbytes) in &map { if *numbytes == 0 { @@ -2597,7 +2572,7 @@ where hasher: transport, bytes_read: 0, }; - let copied = tokio::io::copy(&mut source, &mut file).await?; + let copied = copy_archive_payload(&mut source, &mut file, &mut buffer).await?; transport = source.hasher; if copied != *numbytes { return Err(MicrosandboxError::Custom( @@ -2823,7 +2798,7 @@ async fn validate_archive_inventory( } if !matches!( inventory.completeness.as_str(), - "boot-complete" | "disk-dependent" + "boot-complete" | "dependent" ) { return Err(MicrosandboxError::unsupported( Operation::SnapshotOps, @@ -3570,28 +3545,6 @@ fn select_head_snapshot(snapshots: &[Snapshot]) -> MicrosandboxResult { } } -async fn ensure_promote_targets_available(stage: &Path, dest: &Path) -> MicrosandboxResult<()> { - let mut entries = tokio::fs::read_dir(stage).await?; - while let Some(entry) = entries.next_entry().await? { - let target = dest.join(entry.file_name()); - if tokio::fs::symlink_metadata(&target).await.is_ok() { - return Err(MicrosandboxError::SnapshotAlreadyExists( - target.display().to_string(), - )); - } - } - Ok(()) -} - -async fn promote_stage(stage: &Path, dest: &Path) -> MicrosandboxResult<()> { - let mut entries = tokio::fs::read_dir(stage).await?; - while let Some(entry) = entries.next_entry().await? { - let target = dest.join(entry.file_name()); - tokio::fs::rename(entry.path(), target).await?; - } - Ok(()) -} - async fn install_staged_cache( cache_stage: &Path, cache_dir: &Path, @@ -3965,8 +3918,17 @@ fn file_name_str(p: &Path) -> MicrosandboxResult { async fn resolve_parent_artifact( local: &LocalBackend, + child: &Snapshot, parent_id: &str, ) -> MicrosandboxResult { + // An archive may be installed repeatedly in independent groups. Follow local siblings + // before consulting the global identity index, where multiple copies are ambiguous. + if let Some(directory) = super::group::group_path(child.path()) { + let sibling = directory.join(parent_id); + if tokio::fs::try_exists(&sibling).await? { + return Ok(sibling); + } + } if let Some(handle) = store::lookup_by_digest(local, parent_id).await? { return Ok(handle.artifact_path); } @@ -4011,6 +3973,155 @@ mod tests { use super::*; + fn grouped_archive_manifest(id: u128, parent: Option<&Manifest>) -> Manifest { + let layer_id = DiskLayerId::new(format!("layer_{id:032x}")).unwrap(); + Manifest { + schema: SCHEMA.into(), + snapshot_id: SnapshotId::new(format!("snap_{id:032x}")).unwrap(), + scope: SnapshotScope::Disk, + state: SnapshotState::File(FileSnapshotState { + disk_format: SnapshotFormat::Raw, + filesystem: "ext4".into(), + virtual_size: 4096, + head: layer_id.clone(), + layers: vec![DiskLayer { + layer_id, + format: SnapshotFormat::Raw, + virtual_size: 4096, + backing: None, + payload: LayerPayload { + file_kind: LayerFileKind::Regular, + integrity: None, + }, + }], + }), + capture: SnapshotCapture { + created_at: "2026-09-10T00:00:00Z".into(), + source_lineage: None, + source_checkpoint: None, + consistency: SnapshotConsistency::CrashConsistent, + }, + image: ImageRef { + reference: "docker.io/library/alpine:3.20".into(), + manifest_digest: format!("sha256:{}", "a".repeat(64)), + }, + root_disk: SnapshotRootDisk::Managed, + parent: parent.map(|parent| parent.snapshot_id.clone()), + extensions: BTreeMap::new(), + requires: Vec::new(), + } + } + + fn write_grouped_archive_fixture(path: &Path, manifest: &Manifest) { + std::fs::create_dir_all(path).unwrap(); + std::fs::write( + path.join(DESCRIPTOR_FILENAME), + manifest.to_canonical_bytes().unwrap(), + ) + .unwrap(); + let SnapshotState::File(state) = &manifest.state else { + unreachable!() + }; + for layer in &state.layers { + let payload = path.join(state.layer_path(layer)); + std::fs::create_dir_all(payload.parent().unwrap()).unwrap(); + std::fs::write(payload, vec![42; layer.virtual_size as usize]).unwrap(); + } + } + + #[tokio::test] + async fn with_parents_prefers_group_members_when_global_identities_repeat() { + let home = tempfile::tempdir().unwrap(); + let local = LocalBackend::builder() + .home(home.path()) + .build() + .await + .unwrap(); + let parent = grouped_archive_manifest(1, None); + let child = grouped_archive_manifest(2, Some(&parent)); + for name in ["first", "second"] { + let group = super::super::group::ensure(&local.snapshots_dir(), Some(name)) + .await + .unwrap(); + let stage = tempfile::tempdir().unwrap(); + for manifest in [&parent, &child] { + write_grouped_archive_fixture( + &stage.path().join(manifest.snapshot_id.as_str()), + manifest, + ); + } + let aliases = BTreeMap::from([ + (parent.snapshot_id.to_string(), "base".into()), + (child.snapshot_id.to_string(), "child".into()), + ]); + super::super::group::publish(&group, stage.path(), &aliases, &child.snapshot_id, false) + .await + .unwrap(); + } + store::reindex_dir(&local, &local.snapshots_dir()) + .await + .unwrap(); + assert!( + store::lookup_by_digest(&local, parent.snapshot_id.as_str()) + .await + .is_err() + ); + let archive = home.path().join("group.msb"); + save_snapshot( + &local, + "first:child", + &archive, + SaveOpts { + with_parents: true, + plain_tar: true, + ..Default::default() + }, + ) + .await + .unwrap(); + let loaded = load_snapshot(&local, &archive, None).await.unwrap(); + let loaded_group = loaded.group().unwrap(); + assert_eq!( + store::get_handle(&local, &format!("{loaded_group}:base")) + .await + .unwrap() + .id(), + parent.snapshot_id.as_str() + ); + assert_eq!( + store::get_handle(&local, &format!("{loaded_group}:child")) + .await + .unwrap() + .id(), + child.snapshot_id.as_str() + ); + } + + #[tokio::test] + async fn legacy_suggested_name_that_is_not_a_group_alias_does_not_block_import() { + let home = tempfile::tempdir().unwrap(); + let local = LocalBackend::builder() + .home(home.path()) + .build() + .await + .unwrap(); + let manifest = grouped_archive_manifest(1, None); + let artifact = home.path().join("legacy name with spaces"); + write_grouped_archive_fixture(&artifact, &manifest); + let archive = home.path().join("legacy.msb"); + save_snapshot( + &local, + artifact.to_str().unwrap(), + &archive, + SaveOpts::default(), + ) + .await + .unwrap(); + let loaded = load_snapshot(&local, &archive, None).await.unwrap(); + assert_eq!(loaded.id(), manifest.snapshot_id.as_str()); + assert!(loaded.group().is_some()); + } + #[test] fn digest_hex_rejects_uppercase_identity() { let uppercase = format!("sha256:{}", "A".repeat(64)); @@ -4101,7 +4212,6 @@ mod tests { let directory = tempfile::tempdir().unwrap(); let home = directory.path().join("home"); let source = directory.path().join("upper.ext4"); - let archive = directory.path().join("snapshot.tar.zst"); let child_stage = directory.path().join("child"); let mut payload = b"direct archive payload".to_vec(); payload.resize(4096, 0); @@ -4147,28 +4257,42 @@ mod tests { }; let local = LocalBackend::builder().home(&home).build().await.unwrap(); - save_direct_file_snapshot( - &manifest, - &BTreeMap::new(), - "test-snapshot", - std::slice::from_ref(&source), - &archive, - false, - false, - ) - .await - .unwrap(); - let restored = materialize_archive_for_child(&local, &archive, &child_stage, false) - .await - .unwrap(); - - assert_eq!(restored.manifest.snapshot_id, snapshot_id); - assert_eq!( - std::fs::read(child_stage.join("upper.ext4")).unwrap(), - payload - ); - assert!(!child_stage.join(snapshot_id.as_str()).exists()); - assert!(!home.join("snapshots").join(snapshot_id.as_str()).exists()); + // The suffix is only a user-facing convention, never the encoding discriminator. + // Exercise compressed and plain tar under both conventional and misleading names. + for plain_tar in [false, true] { + let archive_dir = directory.path().join(plain_tar.to_string()); + std::fs::create_dir(&archive_dir).unwrap(); + for name in [ + "snapshot.msb", + "snapshot.tar.zst", + "snapshot.tar", + "snapshot", + ] { + let archive = archive_dir.join(name); + save_direct_file_snapshot( + &manifest, + &BTreeMap::new(), + "test-snapshot", + std::slice::from_ref(&source), + &archive, + plain_tar, + false, + ) + .await + .unwrap(); + let child_stage = child_stage.join(format!("{plain_tar}-{name}")); + let restored = materialize_archive_for_child(&local, &archive, &child_stage, false) + .await + .unwrap(); + assert_eq!(restored.manifest.snapshot_id, snapshot_id); + assert_eq!( + std::fs::read(child_stage.join("upper.ext4")).unwrap(), + payload + ); + assert!(!child_stage.join(snapshot_id.as_str()).exists()); + assert!(!home.join("snapshots").join(snapshot_id.as_str()).exists()); + } + } } #[tokio::test] @@ -4327,14 +4451,14 @@ mod tests { drop(layer_file); // The canonical checkpoint qcow member is one byte too long for the - // fixed GNU header path field. It therefore exercises dense long-name - // fallback even though the source itself has a sparse extent map. + // fixed GNU header path field. It must retain sparse encoding even + // when a GNU long-name record precedes the sparse header. let archive_layer_path = format!("checkpoints/snap_00000000000000000000000000000002/layers/{layer_id}.qcow2"); assert_eq!(archive_layer_path.len(), 101); assert!( archive_encoded_size(&source_layer).await.unwrap() < 4 * 1024 * 1024, - "test source must remain sparse so dense fallback changes the encoded size" + "test source must remain sparse" ); let layer_integrity = sparse_file_integrity(&source_layer).unwrap(); let disk = DiskGenerationManifest { @@ -4416,6 +4540,21 @@ mod tests { ) .await .unwrap(); + // Inspect the actual transport, not just same-reader roundtrip results. + let compressed = tokio::fs::File::open(&archive).await.unwrap(); + let decoder = ZstdDecoder::new(tokio::io::BufReader::new(compressed)); + let mut tar = tokio_tar::Archive::new(decoder); + let mut entries = tar.entries().unwrap(); + let mut found_sparse = false; + while let Some(entry) = futures::StreamExt::next(&mut entries).await { + let entry = entry.unwrap(); + if entry.path().unwrap() == Path::new(&archive_layer_path) { + assert!(entry.header().entry_type().is_gnu_sparse()); + assert!(entry.header().entry_size().unwrap() < 4 * 1024 * 1024); + found_sparse = true; + } + } + assert!(found_sparse); std::fs::remove_dir_all(&source).unwrap(); let restored = materialize_archive_for_child(&local, &archive, &child_stage, false) .await diff --git a/sdk/rust/lib/snapshot/archive/batch.rs b/sdk/rust/lib/snapshot/archive/batch.rs new file mode 100644 index 000000000..dce629fe6 --- /dev/null +++ b/sdk/rust/lib/snapshot/archive/batch.rs @@ -0,0 +1,444 @@ +//! One-pass archive staging and dependency resolution for a local group import. + +use super::*; +use microsandbox_image::snapshot::{Manifest, SnapshotId}; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +struct StagedArchive { + source: PathBuf, + snapshots: tempfile::TempDir, + cache: tempfile::TempDir, + unpacked: UnpackedArchive, +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +pub(super) async fn load( + local: &LocalBackend, + archives: &[PathBuf], + opts: LoadOpts, +) -> MicrosandboxResult> { + if archives.is_empty() { + return Err(MicrosandboxError::InvalidConfig( + "snapshot load requires at least one archive".into(), + )); + } + let started = Instant::now(); + let snapshots_dir = opts.dest.clone().unwrap_or_else(|| local.snapshots_dir()); + // Validate the requested namespace before doing expensive I/O. This read does not create a + // group; a bad or incomplete batch must not publish any of its snapshot members. + let existing = match opts.group.as_deref() { + Some(group) => super::super::group::dependency_members(&snapshots_dir, group).await?, + None => Vec::new(), + }; + tokio::fs::create_dir_all(&snapshots_dir).await?; + let cache_dir = local.cache_dir(); + let cache_tmp = cache_dir.join("tmp"); + tokio::fs::create_dir_all(&cache_tmp).await?; + + let mut staged = Vec::with_capacity(archives.len()); + for archive in archives { + staged.push(unpack(archive, &snapshots_dir, &cache_tmp).await?); + } + let unpack_us = started.elapsed().as_micros(); + let has_dependencies = staged.iter().try_fold(false, |found, item| { + Ok::<_, MicrosandboxError>( + found + | match item.unpacked.inventory.as_ref() { + Some(inventory) => delta::validate(inventory)?.is_some(), + None => false, + }, + ) + })?; + + let mut sources = delta::Sources::default(); + // Legacy archives have no payload omissions. Translate only owned staging, as in a single + // import, before offering their canonical layers as potential sources for another archive. + for item in &staged { + if item.unpacked.inventory.is_none() { + super::super::migration::normalize_staged( + local.db().await?, + &item.unpacked.manifest_dirs, + ) + .await?; + for snapshot in verify_imported_snapshots(local, &item.unpacked.manifest_dirs).await? { + super::super::metadata::write(snapshot.path(), snapshot.labels()).await?; + normalize_imported_descriptor(&snapshot).await?; + } + } + if has_dependencies { + for directory in &item.unpacked.manifest_dirs { + let manifest = read_manifest(directory).await?; + let shared = item + .unpacked + .inventory + .as_ref() + .map(|_| item.snapshots.path()); + sources.add(&manifest, directory, shared).await?; + } + } + } + + // No ambient/global search: only the explicitly selected group and optional external base + // augment the supplied batch. Keep an unpacked base alive until all copies are destination-owned. + let complete = |sources: &delta::Sources| { + staged.iter().all(|item| { + item.unpacked + .inventory + .as_ref() + .is_none_or(|inventory| sources.require(inventory).is_ok()) + }) + }; + let external = if has_dependencies && !complete(&sources) { + for directory in &existing { + if complete(&sources) { + break; + } + // An unrelated damaged checkpoint must not block a load whose dependencies are + // available elsewhere. A missing required identity is still reported below, and any + // chosen payload is verified after copying into this operation's owned staging. + let inspected = async { + let manifest = read_manifest(directory).await?; + sources.add(&manifest, directory, None).await + } + .await; + if let Err(error) = inspected { + tracing::debug!(path = %directory.display(), %error, "skipping unavailable group dependency source"); + } + } + match opts.base.as_deref().filter(|_| !complete(&sources)) { + Some(base) => { + let opened = Box::pin(delta::open_base(local, base)).await?; + sources + .add(opened.snapshot.manifest(), opened.snapshot.path(), None) + .await?; + Some(opened) + } + None => None, + } + } else { + None + }; + // Plan every omission before copying any of them. Missing dependencies identify their target + // archive and payload, rather than guessing ancestry or requiring a particular filename. + for item in &staged { + if let Some(inventory) = &item.unpacked.inventory { + sources + .require(inventory) + .map_err(|error| archive_error(&item.source, error))?; + } + } + + for item in &staged { + if let Some(inventory) = &item.unpacked.inventory { + delta::resolve_sources( + local, + inventory, + item.snapshots.path(), + item.cache.path(), + &sources, + ) + .await + .map_err(|error| archive_error(&item.source, error))?; + } + } + // Keep all original source paths intact until every borrowing read is done. Materializing + // file snapshots below consumes their archive-shared layer directories. + let mut imported = Vec::new(); + let mut candidates = Vec::with_capacity(archives.len()); + let mut aliases = BTreeMap::new(); + let mut identities = BTreeMap::new(); + for item in &staged { + if let Some(inventory) = &item.unpacked.inventory { + materialize_inventory_layers(inventory, item.snapshots.path()).await?; + } + let snapshots = verify_imported_snapshots(local, &item.unpacked.manifest_dirs).await?; + if let Some(inventory) = &item.unpacked.inventory { + validate_inventory_snapshot_bindings(inventory, &snapshots)?; + } + let head_index = match item.unpacked.head.as_deref() { + Some(head) => snapshots + .iter() + .position(|snapshot| snapshot.id().as_str() == head) + .ok_or_else(|| { + MicrosandboxError::SnapshotIntegrity(format!( + "archive {} head {head} is not an imported member", + item.source.display() + )) + })?, + None => select_head_snapshot(&snapshots)?, + }; + let head = snapshots[head_index].id().clone(); + if let Some(inventory) = &item.unpacked.inventory { + merge_aliases(&mut aliases, inventory, &head)?; + } + candidates.push(head); + for snapshot in &snapshots { + if let Some((digest, labels)) = identities.insert( + snapshot.id().clone(), + (snapshot.digest().to_string(), snapshot.labels().clone()), + ) { + if digest != snapshot.digest() { + return Err(MicrosandboxError::SnapshotIntegrity(format!( + "snapshot ID {} has conflicting descriptors in this batch", + snapshot.id() + ))); + } + if &labels != snapshot.labels() { + return Err(MicrosandboxError::InvalidConfig(format!( + "snapshot {} has conflicting labels in this batch", + snapshot.id() + ))); + } + } + super::super::metadata::write(snapshot.path(), snapshot.labels()).await?; + normalize_imported_descriptor(snapshot).await?; + } + imported.push(snapshots); + } + drop(external); + let validate_us = started.elapsed().as_micros() - unpack_us; + + let publication = tempfile::Builder::new() + .prefix(".msb-snapshot-batch-") + .tempdir_in(&snapshots_dir)?; + for ((item, snapshots), candidate) in staged.iter().zip(&imported).zip(&candidates) { + let head = snapshots + .iter() + .find(|snapshot| snapshot.id() == candidate) + .expect("validated archive head"); + Box::pin(install_staged_cache( + item.cache.path(), + &cache_dir, + head.manifest(), + )) + .await?; + } + // Resolve all cross-archive reads before moving any source directory. Same-ID/same-descriptor + // duplicates were independently validated above and are published only once. + for snapshots in &imported { + for snapshot in snapshots { + let target = publication.path().join(snapshot.id().as_str()); + if !target.exists() { + tokio::fs::rename(snapshot.path(), target).await?; + } + } + } + let group_dir = super::super::group::ensure(&snapshots_dir, opts.group.as_deref()).await?; + let update = super::super::group::publish_batch( + &group_dir, + publication.path(), + &aliases, + &candidates, + opts.set_head, + ) + .await?; + let group = group_dir + .file_name() + .and_then(|name| name.to_str()) + .expect("validated group name"); + let mut handles = Vec::with_capacity(candidates.len()); + for id in &candidates { + let path = group_dir.join(id.as_str()); + let snapshot = store::open_snapshot(local, path.to_string_lossy().as_ref()).await?; + handles.push(handle(&snapshot, group, update.clone())?); + } + let _ = store::reindex_dir(local, &group_dir).await; + tracing::info!( + target: "microsandbox_checkpoint_timing", + operation = "snapshot_load_batch", + archives = archives.len(), + total_us = started.elapsed().as_micros(), + unpack_us, + validate_us, + "snapshot batch load timing" + ); + Ok(handles) +} + +async fn unpack( + archive: &Path, + root: &Path, + cache_tmp: &Path, +) -> MicrosandboxResult { + let snapshots = tempfile::Builder::new() + .prefix(".msb-snapshot-import-") + .tempdir_in(root)?; + let cache = tempfile::Builder::new() + .prefix("snapshot-import-") + .tempdir_in(cache_tmp)?; + let file = tokio::fs::File::open(archive).await?; + let mut reader = BufReader::with_capacity(1024 * 1024, file); + let compressed = reader + .fill_buf() + .await? + .starts_with(&[0x28, 0xb5, 0x2f, 0xfd]); + let unpacked = if compressed { + Box::pin(unpack_archive( + ZstdDecoder::new(reader), + snapshots.path(), + cache.path(), + )) + .await? + } else { + Box::pin(unpack_archive(reader, snapshots.path(), cache.path())).await? + }; + Ok(StagedArchive { + source: archive.to_path_buf(), + snapshots, + cache, + unpacked, + }) +} + +async fn read_manifest(directory: &Path) -> MicrosandboxResult { + let bytes = tokio::fs::read(directory.join(DESCRIPTOR_FILENAME)).await?; + Manifest::from_bytes(&bytes) + .map_err(|error| MicrosandboxError::SnapshotIntegrity(error.to_string())) +} + +fn archive_error(archive: &Path, error: MicrosandboxError) -> MicrosandboxError { + MicrosandboxError::SnapshotIntegrity(format!("archive {}: {error}", archive.display())) +} + +fn merge_aliases( + aliases: &mut BTreeMap, + inventory: &ArchiveInventory, + head: &SnapshotId, +) -> MicrosandboxResult<()> { + let mut incoming: BTreeMap = inventory + .extensions + .get("msb-snapshot-member-names") + .map(|value| serde_json::from_value(value.clone())) + .transpose()? + .unwrap_or_default(); + if let Some(name) = inventory + .suggested_name + .as_ref() + .filter(|name| super::super::group::validate_alias(name).is_ok()) + { + incoming + .entry(head.to_string()) + .or_insert_with(|| name.clone()); + } + for (id, name) in incoming { + if !inventory + .members + .iter() + .any(|member| member.snapshot_id == id) + { + return Err(MicrosandboxError::SnapshotIntegrity(format!( + "archive member name refers to snapshot {id} outside that archive" + ))); + } + if let Some(previous) = aliases.get(&id) + && previous != &name + { + return Err(MicrosandboxError::InvalidConfig(format!( + "snapshot {id} has conflicting batch member names '{previous}' and '{name}'" + ))); + } + aliases.insert(id, name); + } + Ok(()) +} + +fn handle( + snap: &Snapshot, + group: &str, + head_update: Option, +) -> MicrosandboxResult { + let (state_kind, format, fstype, checkpoint_manifest_digest, size_bytes) = + match &snap.manifest().state { + SnapshotState::File(state) => ( + "file", + Some(state.disk_format), + Some(state.filesystem.clone()), + None, + Some(state.virtual_size), + ), + SnapshotState::Checkpoint(state) => ( + "checkpoint", + None, + None, + Some(state.checkpoint_root.clone()), + None, + ), + }; + Ok(SnapshotHandle { + group: Some(group.into()), + head_update, + snapshot_id: snap.id().to_string(), + digest: snap.digest().to_string(), + name: super::super::group::member_name(snap.path())?, + parent_digest: snap.manifest().parent.as_ref().map(ToString::to_string), + scope: snap.manifest().scope, + image_ref: snap.manifest().image.reference.clone(), + state_kind: state_kind.into(), + format, + fstype, + checkpoint_manifest_digest, + size_bytes, + locality: "embedded".into(), + availability: "ready".into(), + migration_state: "canonical".into(), + migration_error_code: None, + created_at: chrono::DateTime::parse_from_rfc3339(&snap.manifest().capture.created_at) + .map(|date| date.naive_utc()) + .unwrap_or_else(|_| chrono::Utc::now().naive_utc()), + artifact_path: snap.path().to_path_buf(), + }) +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn member_alias_cannot_refer_to_snapshot_outside_its_archive() { + let existing = SnapshotId::new(format!("snap_{:032x}", 1)).unwrap(); + let incoming = SnapshotId::new(format!("snap_{:032x}", 2)).unwrap(); + let inventory = ArchiveInventory { + schema: "microsandbox.snapshot-archive/1".into(), + head: incoming.to_string(), + suggested_name: None, + completeness: "boot-complete".into(), + members: vec![ArchiveSnapshot { + snapshot_id: incoming.to_string(), + descriptor_path: format!("snapshots/{incoming}/{DESCRIPTOR_FILENAME}"), + descriptor_digest: format!("sha256:{}", "0".repeat(64)), + }], + entries: Vec::new(), + limits: ArchiveLimits { + entry_count: 0, + encoded_bytes: 0, + apparent_bytes: 0, + }, + extensions: BTreeMap::from([( + "msb-snapshot-member-names".into(), + serde_json::to_value(BTreeMap::from([(existing.to_string(), "renamed-existing")])) + .unwrap(), + )]), + requires: Vec::new(), + }; + // Knowing this ID from the destination or another supplied archive does not authorize + // this inventory to rename it. Membership is checked before merging local aliases. + let original = BTreeMap::from([(existing.to_string(), "existing-name".into())]); + let mut aliases = original.clone(); + let error = merge_aliases(&mut aliases, &inventory, &incoming).unwrap_err(); + assert!( + error.to_string().contains("outside that archive"), + "{error}" + ); + assert_eq!(aliases, original); + } +} diff --git a/sdk/rust/lib/snapshot/archive/delta.rs b/sdk/rust/lib/snapshot/archive/delta.rs index acb537562..8f12705b2 100644 --- a/sdk/rust/lib/snapshot/archive/delta.rs +++ b/sdk/rust/lib/snapshot/archive/delta.rs @@ -1,6 +1,8 @@ -//! Exact physical-prefix dependencies for explicitly incremental disk exports. +//! Explicit disk-prefix and immutable RAM-object dependencies for incremental exports. -use microsandbox_image::checkpoint::{DiskLayerExportPlan, DiskLayerRef}; +use microsandbox_image::checkpoint::{ + DiskGenerationManifest, DiskLayerExportPlan, DiskLayerRef, MemoryManifest, +}; use microsandbox_image::snapshot::{DiskLayer, Manifest}; use super::*; @@ -9,7 +11,8 @@ use super::*; // Constants //-------------------------------------------------------------------------------------------------- -pub(super) const REQUIREMENT: &str = "msb-disk-layer-dependencies-v1"; +pub(super) const REQUIREMENT: &str = "msb-snapshot-dependencies-v1"; +const MAX_METADATA: u64 = 8 * 1024 * 1024; //-------------------------------------------------------------------------------------------------- // Types @@ -36,8 +39,9 @@ struct RequiredLayer { #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] -pub(super) struct DiskDependencies { - required: Vec, +pub(super) struct Dependencies { + disks: Vec, + memory: Vec, } struct PhysicalLayer { @@ -45,78 +49,325 @@ struct PhysicalLayer { source: PathBuf, } -struct BaseSnapshot { - snapshot: Snapshot, - // Keep archive staging alive until all required layers have been copied to the destination. +pub(super) struct BaseSnapshot { + pub(super) snapshot: Snapshot, + // Keep archive staging alive until all required payloads belong to the destination. _stage: Option, } +/// Sources are scoped to this load, never persisted or discovered through global path scans. +/// Payload identity, not parentage, connects archives that can reconstruct one another. +#[derive(Default)] +pub(super) struct Sources { + disks: BTreeMap, + memory: BTreeMap, +} + +type SourceIndex = (Vec<(String, PathBuf)>, Vec<(ObjectId, PathBuf)>); + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl Sources { + pub(super) async fn add( + &mut self, + manifest: &Manifest, + directory: &Path, + archive_stage: Option<&Path>, + ) -> MicrosandboxResult<()> { + let manifest = manifest.clone(); + let directory = directory.to_path_buf(); + let archive_stage = archive_stage.map(Path::to_path_buf); + let (disks, memory) = tokio::task::spawn_blocking(move || { + inspect_sources(&manifest, &directory, archive_stage.as_deref()) + }) + .await + .map_err(|error| { + MicrosandboxError::Runtime(format!("snapshot source inspection: {error}")) + })??; + // Prefer batch bytes over destination-group copies. Every borrowed payload is checked in + // its owned destination, so this index is a location hint, not an integrity receipt. + for (identity, path) in disks { + self.disks.entry(identity).or_insert(path); + } + for (identity, path) in memory { + self.memory.entry(identity).or_insert(path); + } + Ok(()) + } + + pub(super) fn require(&self, inventory: &ArchiveInventory) -> MicrosandboxResult<()> { + let Some(dependencies) = validate(inventory)? else { + return Ok(()); + }; + let mut missing = Vec::new(); + for layer in &dependencies.disks { + if !self + .disks + .contains_key(&serde_json::to_string(&layer.identity)?) + { + missing.push(format!("disk layer {}", layer.path)); + } + } + for object in &dependencies.memory { + if !self.memory.contains_key(object) { + missing.push(format!("RAM object {object}")); + } + } + if !missing.is_empty() { + return Err(MicrosandboxError::SnapshotIntegrity(format!( + "snapshot {} is missing dependencies: {}; supply the missing archives or an external --base", + inventory.head, + missing.join(", ") + ))); + } + Ok(()) + } +} + //-------------------------------------------------------------------------------------------------- // Functions //-------------------------------------------------------------------------------------------------- +/// Inspect only descriptors and small identity-verified manifests. Missing payloads in dependent +/// archives are expected; admission still happens after filling their complete target closure. +fn inspect_sources( + manifest: &Manifest, + directory: &Path, + archive_stage: Option<&Path>, +) -> MicrosandboxResult { + let mut disks = Vec::new(); + let mut memory = Vec::new(); + match &manifest.state { + SnapshotState::File(file) => { + for layer in &file.layers { + let relative = file.layer_path(layer); + let path = match archive_stage { + Some(root) => root + .join(".archive-layers") + .join(relative.file_name().expect("canonical layer filename")), + None => directory.join(relative), + }; + if regular_source(&path)? { + disks.push(( + serde_json::to_string(&LayerIdentity::File(layer.clone()))?, + path, + )); + } + } + } + SnapshotState::Checkpoint(state) => { + let root = directory.join(CHECKPOINT_DIRECTORY); + let expected = ObjectId::new(&state.checkpoint_root).map_err(source_error)?; + let checkpoint = CheckpointClosure::inspect_manifest(&root, Some(&expected)) + .map_err(source_error)?; + let ram = MemoryManifest::from_bytes(&read_metadata_object(&root, &checkpoint.memory)?) + .map_err(source_error)?; + let mut metadata = BTreeSet::from([ + checkpoint.memory.clone(), + checkpoint.execution_state.clone(), + ]); + metadata.extend(checkpoint.disks.iter().cloned()); + metadata.extend(checkpoint.devices.iter().map(|device| device.state.clone())); + let objects: BTreeSet<_> = ram + .extents + .iter() + .filter_map(|extent| match &extent.content { + MemoryExtentContent::Object(content) if !metadata.contains(&content.object) => { + Some(content.object.clone()) + } + _ => None, + }) + .collect(); + for object in objects { + let path = checkpoint_object_path(&root, &object); + if regular_source(&path)? { + memory.push((object, path)); + } + } + for disk in &checkpoint.disks { + let disk = DiskGenerationManifest::from_bytes(&read_metadata_object(&root, disk)?) + .map_err(source_error)?; + for layer in disk.layers { + let path = root + .join("layers") + .join(format!("{}.{}", layer.layer_id, layer.format)); + if regular_source(&path)? { + disks.push(( + serde_json::to_string(&LayerIdentity::Checkpoint(layer))?, + path, + )); + } + } + } + } + } + Ok((disks, memory)) +} + +fn regular_source(path: &Path) -> MicrosandboxResult { + match std::fs::symlink_metadata(path) { + Ok(metadata) if metadata.file_type().is_file() => Ok(true), + Ok(_) => Err(MicrosandboxError::SnapshotIntegrity(format!( + "snapshot payload is not a regular file: {}", + path.display() + ))), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error.into()), + } +} + +fn read_metadata_object(root: &Path, id: &ObjectId) -> MicrosandboxResult> { + use std::io::Read; + let path = checkpoint_object_path(root, id); + if !regular_source(&path)? { + return Err(MicrosandboxError::SnapshotIntegrity(format!( + "missing checkpoint metadata {id}" + ))); + } + // Match the checkpoint resolver's 8 MiB metadata limit; a changing file cannot cause an + // unbounded allocation. This is not a RAM-payload read or a new admission format. + let mut bytes = Vec::new(); + std::fs::File::open(path)? + .take(MAX_METADATA + 1) + .read_to_end(&mut bytes)?; + if bytes.len() as u64 > MAX_METADATA + || &ObjectId::from_bytes(&bytes).map_err(source_error)? != id + { + return Err(MicrosandboxError::SnapshotIntegrity(format!( + "invalid checkpoint metadata {id}" + ))); + } + Ok(bytes) +} + +fn source_error(error: impl std::fmt::Display) -> MicrosandboxError { + MicrosandboxError::SnapshotIntegrity(error.to_string()) +} + +pub(super) async fn resolve_sources( + local: &LocalBackend, + inventory: &ArchiveInventory, + snapshots_dir: &Path, + cache_dir: &Path, + sources: &Sources, +) -> MicrosandboxResult<()> { + let Some(dependencies) = validate(inventory)? else { + return Ok(()); + }; + sources.require(inventory)?; + for layer in &dependencies.disks { + let source = &sources.disks[&serde_json::to_string(&layer.identity)?]; + let target = inventory_entry_target(&layer.path, snapshots_dir, cache_dir)?; + copy_dependency(source, &target).await?; + if let LayerIdentity::File(layer) = &layer.identity { + super::super::verify::verify_file_payload(&target, layer.payload.integrity.as_ref()) + .await?; + } + } + for id in &dependencies.memory { + let target = inventory_entry_target( + &memory_archive_path(&inventory.head, id), + snapshots_dir, + cache_dir, + )?; + copy_dependency(&sources.memory[id], &target).await?; + let actual = format!( + "sha256:{}", + hex::encode(Box::pin(file_sha256(&target)).await?) + ); + if actual != id.as_str() { + return Err(MicrosandboxError::SnapshotIntegrity(format!( + "borrowed RAM object content does not match {id}" + ))); + } + } + validate_resolved(local, inventory, snapshots_dir, cache_dir, &dependencies).await +} + pub(super) async fn selection( local: &LocalBackend, head: &Snapshot, opts: &SaveOpts, -) -> MicrosandboxResult> { +) -> MicrosandboxResult> { if opts.since.is_none() && opts.last_layers.is_none() { return Ok(None); } if opts.with_parents || (opts.since.is_some() && opts.last_layers.is_some()) { return Err(MicrosandboxError::InvalidConfig( - "disk-layer export takes either since or last_layers, without with_parents".into(), + "incremental export takes either since or last_layers, without with_parents".into(), )); } let layers = physical_layers(head.manifest(), head.path())?; - let plan = if let Some(base) = &opts.since { - let base = open_base(local, base).await?; + let mut memory = Vec::new(); + let required = if let Some(base) = &opts.since { + // Base archives carry buffered decoder/verification futures; keep them off the caller's + // stack, including when this planner is nested inside a direct restore or SDK call. + let base = Box::pin(open_base(local, base)).await?; let baseline = physical_layers(base.snapshot.manifest(), base.snapshot.path())?; - DiskLayerExportPlan::since( - &layers - .iter() - .map(|layer| &layer.required.identity) - .collect::>(), - &baseline - .iter() - .map(|layer| &layer.required.identity) - .collect::>(), - ) + let available = memory_objects(&base.snapshot)?; + memory = memory_objects(head)? + .intersection(&available) + .cloned() + .collect(); + // Tmpfs-root full snapshots have no disks, but may still depend on RAM objects. + // Do not let disk completeness suppress an independent memory dependency. + if layers.is_empty() && baseline.is_empty() { + 0..0 + } else { + DiskLayerExportPlan::since( + &layers + .iter() + .map(|layer| &layer.required.identity) + .collect::>(), + &baseline + .iter() + .map(|layer| &layer.required.identity) + .collect::>(), + ) + .map_err(|error| MicrosandboxError::InvalidConfig(error.to_string()))? + .required() + } } else { DiskLayerExportPlan::last(layers.len(), opts.last_layers.expect("selector checked")) - } - .map_err(|error| MicrosandboxError::InvalidConfig(error.to_string()))?; - if plan.is_disk_complete() { + .map_err(|error| MicrosandboxError::InvalidConfig(error.to_string()))? + .required() + }; + if required.is_empty() && memory.is_empty() { return Ok(None); } - Ok(Some(DiskDependencies { - required: layers[plan.required()] + Ok(Some(Dependencies { + disks: layers[required] .iter() .map(|layer| layer.required.clone()) .collect(), + memory, })) } pub(super) fn apply( inventory: &mut ArchiveInventory, - dependencies: &DiskDependencies, + dependencies: &Dependencies, ) -> MicrosandboxResult<()> { - for required in &dependencies.required { - let entry = inventory - .entries - .iter_mut() - .find(|entry| entry.path == required.path) - .ok_or_else(|| { - MicrosandboxError::SnapshotIntegrity( - "required disk layer is absent from archive inventory".into(), - ) - })?; + let paths = dependency_paths(&inventory.head, dependencies); + let mut found = 0; + for entry in &mut inventory.entries { + if !paths.contains(entry.path.as_str()) { + continue; + } + found += 1; entry.included = false; entry.encoded_size = 0; entry.sparse_ranges.clear(); entry.transport_integrity = None; } - inventory.completeness = "disk-dependent".into(); + if found != paths.len() { + return Err(MicrosandboxError::SnapshotIntegrity( + "required payload is absent from archive inventory".into(), + )); + } + inventory.completeness = "dependent".into(); inventory.requires.push(REQUIREMENT.into()); inventory.requires.sort(); inventory @@ -142,14 +393,9 @@ pub(super) fn apply( Ok(()) } -pub(super) fn validate( - inventory: &ArchiveInventory, -) -> MicrosandboxResult> { +pub(super) fn validate(inventory: &ArchiveInventory) -> MicrosandboxResult> { let extension = inventory.extensions.get(REQUIREMENT); - let required = inventory - .requires - .iter() - .any(|requirement| requirement == REQUIREMENT); + let required = inventory.requires.iter().any(|value| value == REQUIREMENT); if inventory.completeness == "boot-complete" && !required && extension.is_none() { if inventory.entries.iter().any(|entry| !entry.included) { return Err(MicrosandboxError::SnapshotIntegrity( @@ -158,41 +404,60 @@ pub(super) fn validate( } return Ok(None); } - if inventory.completeness != "disk-dependent" || !required || extension.is_none() { + if inventory.completeness != "dependent" || !required || extension.is_none() { return Err(MicrosandboxError::SnapshotIntegrity( - "invalid disk dependency capability/completeness binding".into(), + "invalid snapshot dependency capability/completeness binding".into(), )); } - let dependencies: DiskDependencies = serde_json::from_value(extension.unwrap().clone())?; - if dependencies.required.is_empty() || dependencies.required.len() > 256 { + let dependencies: Dependencies = serde_json::from_value(extension.unwrap().clone())?; + if (dependencies.disks.is_empty() && dependencies.memory.is_empty()) + || dependencies.disks.len() > 256 + || dependencies.memory.len() > inventory.entries.len() + || dependencies + .memory + .windows(2) + .any(|pair| pair[0] >= pair[1]) + { return Err(MicrosandboxError::SnapshotIntegrity( - "invalid disk dependency count".into(), + "invalid snapshot dependency count or object ordering".into(), )); } - let mut paths = HashSet::new(); - for layer in &dependencies.required { - let entry = inventory - .entries - .iter() - .find(|entry| entry.path == layer.path) - .ok_or_else(|| { - MicrosandboxError::SnapshotIntegrity( - "disk dependency lacks an inventory entry".into(), - ) - })?; - if entry.included - || !matches!( + let entries: HashMap<_, _> = inventory + .entries + .iter() + .map(|entry| (entry.path.as_str(), entry)) + .collect(); + let paths = dependency_paths(&inventory.head, &dependencies); + if paths.len() != dependencies.disks.len() + dependencies.memory.len() { + return Err(MicrosandboxError::SnapshotIntegrity( + "duplicate snapshot dependency".into(), + )); + } + for path in &paths { + let entry = entries.get(path.as_str()).ok_or_else(|| { + MicrosandboxError::SnapshotIntegrity("dependency lacks an inventory entry".into()) + })?; + let is_memory = dependencies + .memory + .binary_search_by(|id| memory_archive_path(&inventory.head, id).cmp(path)) + .is_ok(); + let valid_kind = if is_memory { + entry.kind == "checkpoint-object" + } else { + matches!( entry.kind.as_str(), "file-payload" | "checkpoint-disk-layer" ) + }; + if entry.included + || !valid_kind || entry.owner_snapshot.as_deref() != Some(inventory.head.as_str()) || entry.encoded_size != 0 || !entry.sparse_ranges.is_empty() || entry.transport_integrity.is_some() - || !paths.insert(&layer.path) { return Err(MicrosandboxError::SnapshotIntegrity( - "invalid omitted disk-layer binding".into(), + "invalid omitted payload binding".into(), )); } } @@ -204,13 +469,13 @@ pub(super) fn validate( != paths.len() { return Err(MicrosandboxError::SnapshotIntegrity( - "archive omits a non-disk dependency".into(), + "archive omits an undeclared dependency".into(), )); } Ok(Some(dependencies)) } -/// Resolve only a caller-supplied base; never search ambient directories or qcow backing paths. +/// Resolve only a caller-supplied base; never search ambient directories or backing paths. pub(super) async fn resolve( local: &LocalBackend, inventory: &ArchiveInventory, @@ -221,56 +486,213 @@ pub(super) async fn resolve( let Some(dependencies) = validate(inventory)? else { return Ok(()); }; - let base = base.ok_or_else(|| MicrosandboxError::InvalidConfig( - "this disk-dependent archive requires an explicit base snapshot or standalone base archive".into(), - ))?; - let base = open_base(local, base).await?; + let base = base.ok_or_else(|| { + MicrosandboxError::InvalidConfig( + "this dependent archive requires an explicit base snapshot or standalone base archive" + .into(), + ) + })?; + let base = Box::pin(open_base(local, base)).await?; let available = physical_layers(base.snapshot.manifest(), base.snapshot.path())?; - if available.len() != dependencies.required.len() - || available - .iter() - .zip(&dependencies.required) - .any(|(layer, required)| layer.required.identity != required.identity) + if !dependencies.disks.is_empty() + && (available.len() != dependencies.disks.len() + || available + .iter() + .zip(&dependencies.disks) + .any(|(layer, required)| layer.required.identity != required.identity)) { return Err(MicrosandboxError::SnapshotIntegrity( "supplied base is not the exact required physical disk prefix".into(), )); } - // Copy into operation-owned staging. Imported artifacts must survive deleting the supplied - // base and must never inherit a writable hardlink into another sandbox. - for (source, required) in available.iter().zip(&dependencies.required) { + let available_memory = memory_objects(&base.snapshot)?; + if dependencies + .memory + .iter() + .any(|id| !available_memory.contains(id)) + { + return Err(MicrosandboxError::SnapshotIntegrity( + "supplied base does not contain the required RAM objects".into(), + )); + } + + // Every dependency is copied into operation-owned staging. In particular, a restored child + // must not inherit a writable hardlink into the base; deleting the base must be harmless. + for (source, required) in available.iter().zip(&dependencies.disks) { let target = inventory_entry_target(&required.path, snapshots_dir, cache_dir)?; - if target.exists() { - return Err(MicrosandboxError::SnapshotIntegrity( - "dependency collides with an extracted member".into(), - )); - } - if let Some(parent) = target.parent() { - tokio::fs::create_dir_all(parent).await?; + copy_dependency(&source.source, &target).await?; + } + for id in &dependencies.memory { + let source = checkpoint_object_path(&base.snapshot.path().join(CHECKPOINT_DIRECTORY), id); + let target = inventory_entry_target( + &memory_archive_path(&inventory.head, id), + snapshots_dir, + cache_dir, + )?; + copy_dependency(&source, &target).await?; + // Verify only the objects actually borrowed, in the destination-owned copy. Export + // selection is metadata-only for RAM; it must not scan the base's entire guest memory. + // This reader owns a 64 KiB buffer; boxing prevents every enclosing archive/SDK + // future from embedding another copy of that buffer in its own stack frame. + let actual = format!( + "sha256:{}", + hex::encode(Box::pin(file_sha256(&target)).await?) + ); + if actual != id.as_str() { + return Err(MicrosandboxError::SnapshotIntegrity(format!( + "base RAM object content does not match {id}" + ))); } - let source = source.source.clone(); - tokio::task::spawn_blocking(move || microsandbox_utils::copy::fast_copy(&source, &target)) - .await - .map_err(|error| MicrosandboxError::Runtime(format!("base layer copy: {error}")))??; } + + validate_resolved(local, inventory, snapshots_dir, cache_dir, &dependencies).await +} + +async fn validate_resolved( + local: &LocalBackend, + inventory: &ArchiveInventory, + snapshots_dir: &Path, + cache_dir: &Path, + dependencies: &Dependencies, +) -> MicrosandboxResult<()> { + // Open the complete target only after filling omissions. This retains its normal metadata, + // range, epoch and disk-integrity validation instead of introducing a partial-closure mode. let artifact = snapshots_dir.join(&inventory.head); let manifest = Manifest::from_bytes(&tokio::fs::read(artifact.join(DESCRIPTOR_FILENAME)).await?) .map_err(|error| MicrosandboxError::SnapshotIntegrity(error.to_string()))?; let target = physical_layers(&manifest, &artifact)?; - if target.len() < dependencies.required.len() + if target.len() < dependencies.disks.len() || target .iter() - .zip(&dependencies.required) + .zip(&dependencies.disks) .any(|(layer, required)| &layer.required != required) { return Err(MicrosandboxError::SnapshotIntegrity( "dependency list is not the target descriptor's exact disk prefix".into(), )); } + let target_memory = if dependencies.memory.is_empty() { + BTreeSet::new() + } else { + let snapshot = store::open_snapshot(local, artifact.to_string_lossy().as_ref()).await?; + memory_objects(&snapshot)? + }; + if dependencies + .memory + .iter() + .any(|id| !target_memory.contains(id)) + { + return Err(MicrosandboxError::SnapshotIntegrity( + "omitted object is not a target RAM payload; metadata must remain included".into(), + )); + } + let entries: HashMap<_, _> = inventory + .entries + .iter() + .map(|entry| (entry.path.as_str(), entry)) + .collect(); + for id in &dependencies.memory { + let path = memory_archive_path(&inventory.head, id); + let entry = entries + .get(path.as_str()) + .expect("dependency inventory was validated"); + let target = inventory_entry_target(&path, snapshots_dir, cache_dir)?; + if tokio::fs::metadata(target).await?.len() != entry.apparent_size { + return Err(MicrosandboxError::SnapshotIntegrity( + "resolved RAM object size differs from inventory".into(), + )); + } + } + Ok(()) +} + +async fn copy_dependency(source: &Path, target: &Path) -> MicrosandboxResult<()> { + if tokio::fs::symlink_metadata(target).await.is_ok() { + return Err(MicrosandboxError::SnapshotIntegrity( + "dependency collides with an extracted member".into(), + )); + } + if let Some(parent) = target.parent() { + tokio::fs::create_dir_all(parent).await?; + } + let source = source.to_path_buf(); + let target = target.to_path_buf(); + tokio::task::spawn_blocking(move || microsandbox_utils::copy::fast_copy(&source, &target)) + .await + .map_err(|error| MicrosandboxError::Runtime(format!("base payload copy: {error}")))??; Ok(()) } +fn memory_archive_path(snapshot_id: &str, id: &ObjectId) -> String { + let hash = id + .as_str() + .strip_prefix("sha256:") + .expect("validated ObjectId"); + format!( + "checkpoints/{snapshot_id}/objects/sha256/{}/{hash}", + &hash[..2] + ) +} + +fn checkpoint_object_path(root: &Path, id: &ObjectId) -> PathBuf { + let hash = id + .as_str() + .strip_prefix("sha256:") + .expect("validated ObjectId"); + root.join("objects") + .join("sha256") + .join(&hash[..2]) + .join(hash) +} + +fn dependency_paths(head: &str, dependencies: &Dependencies) -> BTreeSet { + dependencies + .disks + .iter() + .map(|layer| layer.path.clone()) + .chain( + dependencies + .memory + .iter() + .map(|id| memory_archive_path(head, id)), + ) + .collect() +} + +/// Return reusable RAM payload IDs, never metadata objects, even if bytes happen to coincide. +fn memory_objects(snapshot: &Snapshot) -> MicrosandboxResult> { + let SnapshotState::Checkpoint(state) = &snapshot.manifest().state else { + return Ok(BTreeSet::new()); + }; + let expected = ObjectId::new(&state.checkpoint_root) + .map_err(|error| MicrosandboxError::SnapshotIntegrity(error.to_string()))?; + let closure = CheckpointClosure::open_portable( + snapshot.path().join(CHECKPOINT_DIRECTORY), + Some(&expected), + ) + .map_err(|error| MicrosandboxError::SnapshotIntegrity(error.to_string()))?; + let checkpoint = closure.checkpoint(); + let mut objects: BTreeSet<_> = closure + .memory() + .extents + .iter() + .filter_map(|extent| match &extent.content { + MemoryExtentContent::Object(content) => Some(content.object.clone()), + MemoryExtentContent::Zero => None, + }) + .collect(); + objects.remove(&checkpoint.memory); + objects.remove(&checkpoint.execution_state); + for id in &checkpoint.disks { + objects.remove(id); + } + for device in &checkpoint.devices { + objects.remove(&device.state); + } + Ok(objects) +} + fn physical_layers( manifest: &Manifest, directory: &Path, @@ -295,14 +717,15 @@ fn physical_layers( .map_err(|error| MicrosandboxError::SnapshotIntegrity(error.to_string()))?; let closure = CheckpointClosure::open_portable(&root, Some(&expected)) .map_err(|error| MicrosandboxError::SnapshotIntegrity(error.to_string()))?; - if closure.disks().len() != 1 { + if closure.disks().len() > 1 { return Err(MicrosandboxError::InvalidConfig( - "disk-layer selection requires exactly one checkpoint disk".into(), + "disk-layer selection supports at most one checkpoint disk".into(), )); } - Ok(closure.disks()[0] - .layers + Ok(closure + .disks() .iter() + .flat_map(|disk| &disk.layers) .map(|layer| PhysicalLayer { required: RequiredLayer { path: format!( @@ -318,11 +741,16 @@ fn physical_layers( } } -async fn open_base(local: &LocalBackend, input: &str) -> MicrosandboxResult { +pub(super) async fn open_base( + local: &LocalBackend, + input: &str, +) -> MicrosandboxResult { let path = Path::new(input); if !path.is_file() { let snapshot = store::open_snapshot(local, input).await?; - snapshot.verify().await?; + if matches!(snapshot.manifest().state, SnapshotState::File(_)) { + Box::pin(snapshot.verify()).await?; + } return Ok(BaseSnapshot { snapshot, _stage: None, @@ -370,6 +798,9 @@ async fn open_base(local: &LocalBackend, input: &str) -> MicrosandboxResult MicrosandboxResult, parent| Manifest { + schema: "microsandbox.snapshot/1".into(), + snapshot_id: SnapshotId::new(format!("snap_{value:032x}")).unwrap(), + scope: SnapshotScope::Disk, + root_disk: SnapshotRootDisk::Managed, + state: SnapshotState::File(FileSnapshotState { + disk_format: layers.last().unwrap().format, + filesystem: "ext4".into(), + virtual_size: 65536, + head: layers.last().unwrap().layer_id.clone(), + layers, + }), + capture: SnapshotCapture { + created_at: "2026-09-10T00:00:00Z".into(), + source_lineage: None, + source_checkpoint: None, + consistency: SnapshotConsistency::CrashConsistent, + }, + image: ImageRef { + reference: "docker.io/library/alpine:3.20".into(), + manifest_digest: format!("sha256:{}", "0".repeat(64)), + }, + parent, + extensions: BTreeMap::new(), + requires: Vec::new(), + }; + let base = descriptor(1, vec![base_layer.clone()], None); + let child = descriptor(2, vec![base_layer, top], Some(base.snapshot_id.clone())); + for (directory, manifest) in [(&base_dir, &base), (&child_dir, &child)] { + std::fs::write( + directory.join(DESCRIPTOR_FILENAME), + manifest.to_canonical_bytes().unwrap(), + ) + .unwrap(); + } + let base_archive = temp.path().join("base.msb"); + save_snapshot( + &local, + base_dir.to_str().unwrap(), + &base_archive, + SaveOpts::default(), + ) + .await + .unwrap(); + let child_archive = temp.path().join("child.msb"); + save_snapshot( + &local, + child_dir.to_str().unwrap(), + &child_archive, + SaveOpts { + since: Some(base_dir.to_string_lossy().into_owned()), + ..Default::default() + }, + ) + .await + .unwrap(); + let options = || LoadOpts { + group: Some("corrupt-source".into()), + ..Default::default() + }; + let installed = load_snapshots(&local, &[base_archive], options()) + .await + .unwrap(); + let group_dir = installed[0].path().parent().unwrap().to_path_buf(); + let previous_head = std::fs::read(group_dir.join("group.json")).unwrap(); + // Metadata and length still agree: only payload verification can detect this change. + std::fs::write(installed[0].path().join(&base_path), vec![92u8; 65536]).unwrap(); + let error = load_snapshots(&local, &[child_archive], options()) + .await + .unwrap_err(); + assert!(error.to_string().contains("integrity mismatch"), "{error}"); + assert!(!group_dir.join(child.snapshot_id.as_str()).exists()); + assert_eq!( + std::fs::read(group_dir.join("group.json")).unwrap(), + previous_head + ); + assert_eq!( + super::super::super::group::dependency_members( + &local.snapshots_dir(), + "corrupt-source", + ) + .await + .unwrap(), + vec![installed[0].path().to_path_buf()] + ); + } + #[tokio::test] async fn delta_load_and_direct_restore_require_exact_base_and_own_their_closure() { let temp = tempfile::tempdir().unwrap(); @@ -493,11 +1065,12 @@ mod tests { .await .unwrap(); assert!(load_snapshot(&local, &archive, None).await.is_err()); - assert!( - load_snapshot_with_base(&local, &archive, None, Some(head_name)) - .await - .is_err() - ); + // Loading now resolves exact payload identities from a source pool. A complete newer + // snapshot can supply the required prefix even when it contains additional layers. + let supplied_by_newer = load_snapshot_with_base(&local, &archive, None, Some(head_name)) + .await + .unwrap(); + assert!(supplied_by_newer.path().join(&base_path).exists()); let loaded = load_snapshot_with_base(&local, &archive, None, Some(base_name)) .await .unwrap(); @@ -547,6 +1120,29 @@ mod tests { ) .await .unwrap(); + // File-state archives use a shared layers directory, unlike full checkpoint payloads. + // Both input orders must finish all borrowing reads before consuming those directories. + for (group, inputs) in [ + ("file-reverse", vec![archive.clone(), base_archive.clone()]), + ("file-forward", vec![base_archive.clone(), archive.clone()]), + ] { + let batch = load_snapshots( + &local, + &inputs, + LoadOpts { + group: Some(group.into()), + ..Default::default() + }, + ) + .await + .unwrap(); + for snapshot in &batch { + assert_eq!( + std::fs::read(snapshot.path().join(&base_path)).unwrap(), + vec![91u8; 65536] + ); + } + } std::fs::remove_dir_all(&base_dir).unwrap(); assert_eq!( std::fs::read(loaded.path().join(&base_path)).unwrap(), diff --git a/sdk/rust/lib/snapshot/archive/delta_tests.rs b/sdk/rust/lib/snapshot/archive/delta_tests.rs new file mode 100644 index 000000000..1733af9bd --- /dev/null +++ b/sdk/rust/lib/snapshot/archive/delta_tests.rs @@ -0,0 +1,800 @@ +//! Archive-level fixtures exercise transport and staging, not hypervisor execution codecs. + +use super::*; +use microsandbox_image::checkpoint::{ + CaptureIntent, CheckpointManifest, ContentRef, DeviceStateRef, DiskGenerationManifest, + LocalObjectStore, MemoryCaptureMode, MemoryExtent, MemoryManifest, sparse_file_integrity, +}; +use microsandbox_image::snapshot::{ + CheckpointSnapshotState, ImageRef, SnapshotCapture, SnapshotConsistency, SnapshotId, + SnapshotRootDisk, SnapshotScope, +}; + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +async fn fixture( + local: &LocalBackend, + path: &Path, + generation: u64, + previous: Option<&Snapshot>, + disk: bool, +) -> Snapshot { + let root = path.join(CHECKPOINT_DIRECTORY); + let store = LocalObjectStore::open(&root).unwrap(); + // Packed objects deliberately contain bytes not selected by the map. Export is object-based. + let original = store.put_bytes(&vec![0xa5; 8192]).unwrap(); + let changed = store + .put_bytes(&vec![(generation / 3) as u8; 8192]) + .unwrap(); + let memory = MemoryManifest { + schema: "microsandbox.memory/1".into(), + architecture: std::env::consts::ARCH.into(), + guest_page_size: 4096, + topology_generation: 1, + generation, + capture_mode: if generation == 1 { + MemoryCaptureMode::Full + } else { + MemoryCaptureMode::Incremental + }, + pause_generation: generation, + extents: vec![ + MemoryExtent { + start: 0, + length: 4096, + content: MemoryExtentContent::Object(ContentRef { + object: original, + object_offset: 4096, + }), + }, + MemoryExtent { + start: 4096, + length: 4096, + content: MemoryExtentContent::Object(ContentRef { + object: changed, + object_offset: 4096, + }), + }, + MemoryExtent { + start: 8192, + length: 4096, + content: MemoryExtentContent::Zero, + }, + ], + }; + let mut disks = Vec::new(); + if disk { + std::fs::create_dir_all(root.join("layers")).unwrap(); + let mut layers = Vec::new(); + if let Some(previous) = previous { + for old in physical_layers(previous.manifest(), previous.path()).unwrap() { + let LayerIdentity::Checkpoint(layer) = old.required.identity else { + unreachable!() + }; + let dest = root + .join("layers") + .join(format!("{}.{}", layer.layer_id, layer.format)); + microsandbox_utils::copy::fast_copy(&old.source, &dest).unwrap(); + layers.push(layer); + } + } + let layer_id = format!("layer_{generation:032x}"); + let format = if layers.is_empty() { "raw" } else { "qcow2" }; + let layer_path = root.join("layers").join(format!("{layer_id}.{format}")); + if let Some(base) = layers.last() { + microsandbox_image::checkpoint::create_qcow2_overlay( + &layer_path, + 65536, + &root + .join("layers") + .join(format!("{}.{}", base.layer_id, base.format)), + &base.format, + ) + .await + .unwrap(); + } else { + std::fs::write(&layer_path, vec![17u8; 65536]).unwrap(); + } + let layer = DiskLayerRef { + layer_id: layer_id.clone(), + format: format.into(), + virtual_size: 65536, + predecessor: layers.last().map(|layer| layer.layer_id.clone()), + integrity_root: sparse_file_integrity(&layer_path).unwrap().root, + }; + layers.push(layer); + let disk = DiskGenerationManifest { + schema: "microsandbox.disk-generation/1".into(), + volume_id: "vol_test".into(), + device_id: "vdb".into(), + generation, + layers, + head: layer_id, + pause_generation: generation, + }; + disks.push( + store + .put_bytes(&disk.to_canonical_bytes().unwrap()) + .unwrap(), + ); + } + let checkpoint = CheckpointManifest { + schema: "microsandbox.checkpoint/1".into(), + checkpoint_id: format!("checkpoint_{generation}"), + capture_intent: CaptureIntent::FullSnapshot, + architecture: std::env::consts::ARCH.into(), + pause_generation: generation, + execution_state: store + .put_bytes(format!("execution-{generation}").as_bytes()) + .unwrap(), + memory: store + .put_bytes(&memory.to_canonical_bytes().unwrap()) + .unwrap(), + disks, + devices: vec![DeviceStateRef { + device_type: 4, + device_id: "rng".into(), + state: store.put_bytes(b"unchanged-device-state").unwrap(), + }], + resources: Vec::new(), + requires: Vec::new(), + }; + let bytes = checkpoint.to_canonical_bytes().unwrap(); + let root_id = ObjectId::from_bytes(&bytes).unwrap(); + std::fs::write(root.join("checkpoint.json"), bytes).unwrap(); + let manifest = Manifest { + schema: "microsandbox.snapshot/1".into(), + snapshot_id: SnapshotId::new(format!("snap_{generation:032x}")).unwrap(), + scope: SnapshotScope::Full, + root_disk: if disk { + SnapshotRootDisk::Managed + } else { + SnapshotRootDisk::Tmpfs { size_mib: None } + }, + state: SnapshotState::Checkpoint(CheckpointSnapshotState { + checkpoint_id: checkpoint.checkpoint_id, + checkpoint_root: root_id.to_string(), + restore_intents: vec!["clone".into(), "resume".into()], + requirements_summary: BTreeMap::from([ + ("vcpus".into(), 1.into()), + ("max_vcpus".into(), 1.into()), + ("memory_mib".into(), 128.into()), + ("max_memory_mib".into(), 128.into()), + ]), + }), + capture: SnapshotCapture { + created_at: "2026-09-10T00:00:00Z".into(), + source_lineage: None, + source_checkpoint: None, + consistency: SnapshotConsistency::ApplicationConsistent, + }, + image: ImageRef { + reference: "docker.io/library/alpine:3.20".into(), + manifest_digest: format!("sha256:{}", "0".repeat(64)), + }, + parent: previous.map(|snapshot| snapshot.id().clone()), + extensions: BTreeMap::new(), + requires: Vec::new(), + }; + std::fs::write( + path.join(DESCRIPTOR_FILENAME), + manifest.to_canonical_bytes().unwrap(), + ) + .unwrap(); + store::open_snapshot(local, path.to_str().unwrap()) + .await + .unwrap() +} + +fn assert_ram(path: &Path, generation: u64) { + let closure = CheckpointClosure::open_portable(path.join(CHECKPOINT_DIRECTORY), None).unwrap(); + closure.verify_memory_objects().unwrap(); + let mut ram = Vec::new(); + for extent in &closure.memory().extents { + match &extent.content { + MemoryExtentContent::Zero => ram.extend(vec![0; extent.length as usize]), + MemoryExtentContent::Object(content) => { + let object = closure.read_object(&content.object, 8192).unwrap(); + let start = content.object_offset as usize; + ram.extend_from_slice(&object[start..start + extent.length as usize]); + } + } + } + assert_eq!(&ram[..4096], &vec![0xa5; 4096]); + assert_eq!(&ram[4096..8192], &vec![(generation / 3) as u8; 4096]); + assert_eq!(&ram[8192..], &vec![0; 4096]); + assert_eq!( + closure + .read_object(&closure.checkpoint().execution_state, 128) + .unwrap(), + format!("execution-{generation}").as_bytes() + ); +} + +async fn unpack(path: &Path, stage: &Path) -> ArchiveInventory { + let file = BufReader::new(tokio::fs::File::open(path).await.unwrap()); + let cache = stage.join("cache"); + tokio::fs::create_dir_all(&cache).await.unwrap(); + // Tests use plain tar to inspect exactly which payloads were physically transported. + unpack_archive(file, stage, &cache) + .await + .unwrap() + .inventory + .unwrap() +} + +async fn chain(disk: bool) { + let temp = tempfile::tempdir().unwrap(); + let local = LocalBackend::builder() + .home(temp.path().join("home")) + .build() + .await + .unwrap(); + let mut previous = None; + let mut loaded: Option = None; + for generation in 1..=12 { + let source = fixture( + &local, + &temp.path().join(format!("source-{generation}")), + generation, + previous.as_ref(), + disk, + ) + .await; + let archive = temp.path().join(format!("cp{generation:02}.msb")); + save_snapshot( + &local, + source.path().to_str().unwrap(), + &archive, + SaveOpts { + since: previous + .as_ref() + .map(|snapshot: &Snapshot| snapshot.path().to_string_lossy().into_owned()), + plain_tar: true, + ..Default::default() + }, + ) + .await + .unwrap(); + let stage = temp.path().join(format!("unpacked-{generation}")); + let inventory = unpack(&archive, &stage).await; + if generation > 1 { + let dependencies = validate(&inventory).unwrap().unwrap(); + assert_eq!( + dependencies.disks.len(), + if disk { generation as usize - 1 } else { 0 } + ); + assert!(!dependencies.memory.is_empty()); + for id in &dependencies.memory { + let path = memory_archive_path(source.id().as_str(), id); + assert!( + !inventory_entry_target(&path, &stage, &stage.join("cache")) + .unwrap() + .exists() + ); + } + let closure = + CheckpointClosure::open_portable(source.path().join(CHECKPOINT_DIRECTORY), None) + .unwrap(); + for id in [ + &closure.checkpoint().memory, + &closure.checkpoint().execution_state, + &closure.checkpoint().devices[0].state, + ] { + assert!(inventory.entries.iter().any(|entry| entry.path + == memory_archive_path(source.id().as_str(), id) + && entry.included)); + } + assert!(load_snapshot(&local, &archive, None).await.is_err()); + } else { + assert!(validate(&inventory).unwrap().is_none()); + } + let base = loaded + .as_ref() + .map(|snapshot| snapshot.path().to_str().unwrap()); + if generation == 12 { + // Direct archive restore must use the same dependency resolver, without installation. + let child = temp.path().join("child"); + let result = + materialize_archive_for_child_with_base(&local, &archive, &child, false, base) + .await + .unwrap(); + assert!(result.checkpoint_restore.is_some()); + let closure = + CheckpointClosure::open_portable(child.join(".checkpoint-restore"), None).unwrap(); + closure.verify_memory_objects().unwrap(); + assert!(!local.snapshots_dir().join(source.id().as_str()).exists()); + } + let current = load_snapshot_with_base(&local, &archive, None, base) + .await + .unwrap(); + assert_ram(current.path(), generation); + if let Some(old) = loaded.take() { + // These are exclusively test-owned artifacts; later loads cannot depend on their paths. + std::fs::remove_dir_all(old.path()).unwrap(); + assert_ram(current.path(), generation); + } + loaded = Some(current); + previous = Some(source); + } + let final_snapshot = loaded.unwrap(); + let standalone = temp.path().join("standalone.msb"); + save_snapshot( + &local, + final_snapshot.path().to_str().unwrap(), + &standalone, + SaveOpts::default(), + ) + .await + .unwrap(); + let other = load_snapshot(&local, &standalone, Some(&temp.path().join("other-host"))) + .await + .unwrap(); + assert_ram(other.path(), 12); +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[tokio::test] +async fn unordered_batch_resolves_disk_and_ram_from_all_supplied_archives() { + for disk in [false, true] { + let temp = tempfile::tempdir().unwrap(); + let local = LocalBackend::builder() + .home(temp.path().join("home")) + .build() + .await + .unwrap(); + let mut previous = None; + let mut archives = Vec::new(); + for generation in 1..=6 { + let source = fixture( + &local, + &temp.path().join(format!("source-{generation}")), + generation, + previous.as_ref(), + disk, + ) + .await; + let archive = temp.path().join(format!("cp{generation}.msb")); + save_snapshot( + &local, + source.path().to_str().unwrap(), + &archive, + SaveOpts { + since: previous + .as_ref() + .map(|snapshot: &Snapshot| snapshot.path().to_string_lossy().into_owned()), + plain_tar: generation % 2 == 0, + ..Default::default() + }, + ) + .await + .unwrap(); + archives.push(archive); + previous = Some(source); + } + archives.reverse(); + let loaded = load_snapshots( + &local, + &archives, + LoadOpts { + group: Some("received".into()), + ..Default::default() + }, + ) + .await + .unwrap(); + for (offset, snapshot) in loaded.iter().enumerate() { + assert_ram(snapshot.path(), 6 - offset as u64); + } + assert_eq!(loaded[0].head_update().unwrap().head, loaded[0].snapshot_id); + // Neither source files, archive bytes, nor other installed generations are needed by + // the final snapshot after the batch has reconstructed destination-owned closures. + for generation in 1..=6 { + std::fs::remove_dir_all(temp.path().join(format!("source-{generation}"))).unwrap(); + } + for archive in &archives { + std::fs::remove_file(archive).unwrap(); + } + for snapshot in &loaded[1..] { + std::fs::remove_dir_all(snapshot.path()).unwrap(); + } + assert_ram(loaded[0].path(), 6); + } +} + +#[tokio::test] +async fn automatic_group_sources_fill_ram_and_disks_without_base_flag() { + for disk in [false, true] { + let temp = tempfile::tempdir().unwrap(); + let local = LocalBackend::builder() + .home(temp.path().join("home")) + .build() + .await + .unwrap(); + let base = fixture(&local, &temp.path().join("base"), 1, None, disk).await; + let target = fixture(&local, &temp.path().join("target"), 3, Some(&base), disk).await; + let baseline = temp.path().join("base.msb"); + let delta = temp.path().join("delta.msb"); + save_snapshot( + &local, + base.path().to_str().unwrap(), + &baseline, + SaveOpts::default(), + ) + .await + .unwrap(); + save_snapshot( + &local, + target.path().to_str().unwrap(), + &delta, + SaveOpts { + since: Some(base.path().to_string_lossy().into_owned()), + ..Default::default() + }, + ) + .await + .unwrap(); + let opts = LoadOpts { + group: Some("received".into()), + ..Default::default() + }; + let installed_base = load_snapshot_with_options(&local, &baseline, opts.clone()) + .await + .unwrap(); + // The same dependent archive cannot search a different, unnamed group implicitly. + let error = load_snapshot_with_options(&local, &delta, LoadOpts::default()) + .await + .unwrap_err(); + assert!( + error.to_string().contains("missing dependencies"), + "{error}" + ); + assert!(!local.snapshots_dir().join("elsewhere").exists()); + let imported = load_snapshot_with_options(&local, &delta, opts) + .await + .unwrap(); + assert_ram(imported.path(), 3); + std::fs::remove_dir_all(installed_base.path()).unwrap(); + assert_ram(imported.path(), 3); + } +} + +#[tokio::test] +async fn missing_or_corrupt_borrowed_ram_never_publishes_target() { + let temp = tempfile::tempdir().unwrap(); + let local = LocalBackend::builder() + .home(temp.path().join("home")) + .build() + .await + .unwrap(); + let base = fixture(&local, &temp.path().join("base"), 1, None, false).await; + let target = fixture(&local, &temp.path().join("target"), 3, Some(&base), false).await; + let baseline = temp.path().join("base.msb"); + let delta = temp.path().join("delta.msb"); + save_snapshot( + &local, + base.path().to_str().unwrap(), + &baseline, + SaveOpts::default(), + ) + .await + .unwrap(); + save_snapshot( + &local, + target.path().to_str().unwrap(), + &delta, + SaveOpts { + since: Some(base.path().to_string_lossy().into_owned()), + ..Default::default() + }, + ) + .await + .unwrap(); + let opts = LoadOpts { + group: Some("received".into()), + ..Default::default() + }; + let installed = load_snapshot_with_options(&local, &baseline, opts.clone()) + .await + .unwrap(); + let id = memory_objects(&base) + .unwrap() + .intersection(&memory_objects(&target).unwrap()) + .next() + .unwrap() + .clone(); + let path = checkpoint_object_path(&installed.path().join(CHECKPOINT_DIRECTORY), &id); + let original = std::fs::read(&path).unwrap(); + std::fs::remove_file(&path).unwrap(); + let missing = load_snapshot_with_options(&local, &delta, opts.clone()) + .await + .unwrap_err(); + assert!( + missing.to_string().contains("missing dependencies"), + "{missing}" + ); + std::fs::write(&path, vec![0x42; original.len()]).unwrap(); + let corrupt = load_snapshot_with_options(&local, &delta, opts) + .await + .unwrap_err(); + assert!( + corrupt + .to_string() + .contains("RAM object content does not match"), + "{corrupt}" + ); + assert!( + !installed + .path() + .parent() + .unwrap() + .join(target.id().as_str()) + .exists() + ); + let head = super::super::super::group::select(&local.snapshots_dir(), "received") + .await + .unwrap(); + assert_eq!(head.head, base.id().as_str()); +} + +#[tokio::test] +async fn twelve_ram_only_archives_resolve_without_intermediate_vms() { + chain(false).await; +} + +#[tokio::test] +async fn twelve_disk_and_ram_archives_resolve_without_intermediate_vms() { + chain(true).await; +} + +#[tokio::test] +async fn last_layers_keeps_ram_complete_and_wrong_ram_base_fails() { + let temp = tempfile::tempdir().unwrap(); + let local = LocalBackend::builder() + .home(temp.path().join("home")) + .build() + .await + .unwrap(); + let base = fixture(&local, &temp.path().join("base"), 1, None, true).await; + let target = fixture(&local, &temp.path().join("target"), 2, Some(&base), true).await; + let selection = selection( + &local, + &target, + &SaveOpts { + last_layers: Some(1), + ..Default::default() + }, + ) + .await + .unwrap() + .unwrap(); + assert!(selection.memory.is_empty()); + let archive = temp.path().join("delta.msb"); + save_snapshot( + &local, + target.path().to_str().unwrap(), + &archive, + SaveOpts { + since: Some(base.path().to_string_lossy().into_owned()), + plain_tar: true, + ..Default::default() + }, + ) + .await + .unwrap(); + let ids = memory_objects(&base).unwrap(); + let missing = checkpoint_object_path( + &base.path().join(CHECKPOINT_DIRECTORY), + ids.first().unwrap(), + ); + let saved = std::fs::read(&missing).unwrap(); + std::fs::remove_file(&missing).unwrap(); + assert!( + load_snapshot_with_base(&local, &archive, None, Some(base.path().to_str().unwrap())) + .await + .is_err() + ); + assert!(!local.snapshots_dir().join(target.id().as_str()).exists()); + std::fs::write(&missing, vec![0x33; saved.len()]).unwrap(); + assert!( + load_snapshot_with_base(&local, &archive, None, Some(base.path().to_str().unwrap())) + .await + .is_err() + ); + std::fs::write(&missing, saved).unwrap(); + let loaded = + load_snapshot_with_base(&local, &archive, None, Some(base.path().to_str().unwrap())) + .await + .unwrap(); + assert_ram(loaded.path(), 2); +} + +#[tokio::test] +async fn memory_dependency_validation_rejects_incomplete_and_misbound_inventories() { + let temp = tempfile::tempdir().unwrap(); + let local = LocalBackend::builder() + .home(temp.path().join("home")) + .build() + .await + .unwrap(); + let base = fixture(&local, &temp.path().join("base"), 1, None, false).await; + let target = fixture(&local, &temp.path().join("target"), 4, None, false).await; + let archive = temp.path().join("delta.tar"); + save_snapshot( + &local, + target.path().to_str().unwrap(), + &archive, + SaveOpts { + since: Some(base.path().to_string_lossy().into_owned()), + plain_tar: true, + ..Default::default() + }, + ) + .await + .unwrap(); + let stage = temp.path().join("stage"); + let inventory = unpack(&archive, &stage).await; + assert!(validate(&inventory).unwrap().is_some()); + let clone = || { + serde_json::from_value::(serde_json::to_value(&inventory).unwrap()) + .unwrap() + }; + let mut missing_requirement = clone(); + missing_requirement + .requires + .retain(|name| name != REQUIREMENT); + assert!(validate(&missing_requirement).is_err()); + let mut no_dependencies = clone(); + no_dependencies.extensions.insert( + REQUIREMENT.into(), + serde_json::json!({"disks": [], "memory": []}), + ); + assert!(validate(&no_dependencies).is_err()); + let mut duplicate = clone(); + let mut deps = validate(&duplicate).unwrap().unwrap(); + deps.memory.push(deps.memory[0].clone()); + duplicate + .extensions + .insert(REQUIREMENT.into(), serde_json::to_value(&deps).unwrap()); + assert!(validate(&duplicate).is_err()); + for kind in ["snapshot-descriptor", "checkpoint-root", "image-object"] { + let mut wrong_kind = clone(); + wrong_kind + .entries + .iter_mut() + .find(|entry| !entry.included) + .unwrap() + .kind = kind.into(); + assert!(validate(&wrong_kind).is_err()); + } + let mut wrong_owner = clone(); + wrong_owner + .entries + .iter_mut() + .find(|entry| !entry.included) + .unwrap() + .owner_snapshot = Some(base.id().to_string()); + assert!(validate(&wrong_owner).is_err()); + let mut undeclared = clone(); + undeclared + .entries + .iter_mut() + .find(|entry| entry.kind == "checkpoint-root") + .unwrap() + .included = false; + assert!(validate(&undeclared).is_err()); + + // An inventory can describe an existing base object that the target does not reference. + // Structural inventory validation alone is insufficient: resolve must check the target map. + let mut unreferenced = clone(); + let extra = memory_objects(&base) + .unwrap() + .difference(&memory_objects(&target).unwrap()) + .next() + .unwrap() + .clone(); + let mut deps = validate(&unreferenced).unwrap().unwrap(); + deps.memory.push(extra.clone()); + deps.memory.sort(); + unreferenced + .extensions + .insert(REQUIREMENT.into(), serde_json::to_value(&deps).unwrap()); + let mut entry = serde_json::from_value::( + serde_json::to_value( + unreferenced + .entries + .iter() + .find(|entry| !entry.included) + .unwrap(), + ) + .unwrap(), + ) + .unwrap(); + entry.path = memory_archive_path(target.id().as_str(), &extra); + unreferenced.entries.push(entry); + assert!( + resolve( + &local, + &unreferenced, + &stage, + &stage.join("cache"), + Some(base.path().to_str().unwrap()) + ) + .await + .is_err() + ); + assert!(!local.snapshots_dir().join(target.id().as_str()).exists()); + + let truncated = temp.path().join("truncated.msb"); + let bytes = std::fs::read(&archive).unwrap(); + std::fs::write(&truncated, &bytes[..bytes.len() / 2]).unwrap(); + assert!( + load_snapshot_with_base( + &local, + &truncated, + None, + Some(base.path().to_str().unwrap()) + ) + .await + .is_err() + ); + assert!(!local.snapshots_dir().join(target.id().as_str()).exists()); +} + +#[tokio::test] +async fn standalone_base_archive_resolves_ram_but_dependent_base_archive_is_refused() { + let temp = tempfile::tempdir().unwrap(); + let local = LocalBackend::builder() + .home(temp.path().join("home")) + .build() + .await + .unwrap(); + let base = fixture(&local, &temp.path().join("base"), 1, None, false).await; + let target = fixture(&local, &temp.path().join("target"), 4, None, false).await; + let base_archive = temp.path().join("base.msb"); + save_snapshot( + &local, + base.path().to_str().unwrap(), + &base_archive, + SaveOpts::default(), + ) + .await + .unwrap(); + let delta = temp.path().join("delta.msb"); + save_snapshot( + &local, + target.path().to_str().unwrap(), + &delta, + SaveOpts { + since: Some(base_archive.to_string_lossy().into_owned()), + ..Default::default() + }, + ) + .await + .unwrap(); + assert!(open_base(&local, delta.to_str().unwrap()).await.is_err()); + let loaded = + load_snapshot_with_base(&local, &delta, None, Some(base_archive.to_str().unwrap())) + .await + .unwrap(); + assert_ram(loaded.path(), 4); + let child = temp.path().join("child"); + assert!( + materialize_archive_for_child_with_base( + &local, + &delta, + &child, + false, + Some(base_archive.to_str().unwrap()) + ) + .await + .unwrap() + .checkpoint_restore + .is_some() + ); +} diff --git a/sdk/rust/lib/snapshot/create.rs b/sdk/rust/lib/snapshot/create.rs index 81d488309..5ff31a6b5 100644 --- a/sdk/rust/lib/snapshot/create.rs +++ b/sdk/rust/lib/snapshot/create.rs @@ -1,4 +1,4 @@ -//! Snapshot creation from a stopped sandbox. +//! Disk-only and full snapshot creation with source lifecycle preservation. use std::collections::BTreeMap; use std::path::{Path, PathBuf}; @@ -42,6 +42,13 @@ struct CapturedFullSnapshot { source_recovery: Option, } +/// A complete operation-owned artifact, not yet a durable group member. +#[derive(Debug)] +struct StagedSnapshot { + snapshot: Snapshot, + source_recovery: Option, +} + /// Non-identity publication options shared by the full-capture entry point. #[derive(Clone, Copy)] struct SnapshotDestination<'a> { @@ -67,6 +74,22 @@ struct SnapshotDiskSource { struct SnapshotDiskClosure { sources: Vec, virtual_size: u64, + /// A live capture owns immutable runtime staging until artifact publication completes. + capture_root: Option, +} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +impl Drop for SnapshotDiskClosure { + fn drop(&mut self) { + if let Some(path) = &self.capture_root + && let Err(error) = std::fs::remove_dir_all(path) + { + tracing::warn!(%error, "failed to remove consumed disk-only capture staging"); + } + } } //-------------------------------------------------------------------------------------------------- @@ -75,11 +98,144 @@ struct SnapshotDiskClosure { pub(super) async fn create_snapshot( local: &LocalBackend, - config: SnapshotConfig, + mut config: SnapshotConfig, ) -> MicrosandboxResult { + if config.force { + return Err(MicrosandboxError::InvalidConfig( + "grouped snapshots are immutable; choose another member name or remove the existing member explicitly".into(), + )); + } + let generated_name = config.name.is_empty(); + if generated_name { + config.name = format!("msb-{:08x}", rand::random::()); + } + validate_snapshot_name(&config.name)?; + let lineage = super::lineage::begin(local, &config.source_sandbox).await?; + let root = config + .dest_dir + .take() + .unwrap_or_else(|| local.snapshots_dir()); + let group_name = config + .group + .take() + .unwrap_or_else(|| config.source_sandbox.clone()); + let group_dir = super::group::ensure(&root, Some(&group_name)).await?; + let staging = tempfile::Builder::new() + .prefix(".capture-") + .tempdir_in(&group_dir)?; + let name = config.name.clone(); + let source_sandbox = config.source_sandbox.clone(); + config.dest_dir = Some(staging.path().to_path_buf()); + let captured = capture_installed(local, config, lineage.sandbox_id()).await?; + publish_snapshot_group( + local, + captured, + staging, + lineage, + name, + generated_name, + &source_sandbox, + ) + .await +} + +/// Source failure is reported only after the outer group and ancestry commit. Staging never +/// becomes the artifact locator, and source recovery does not cause a second capture or thaw. +async fn publish_snapshot_group( + local: &LocalBackend, + captured: StagedSnapshot, + staging: tempfile::TempDir, + lineage: super::lineage::CaptureLineage, + name: String, + generated_name: bool, + source_sandbox: &str, +) -> MicrosandboxResult { + let StagedSnapshot { + snapshot: mut captured, + source_recovery, + } = captured; + let group_dir = staging + .path() + .parent() + .expect("group staging has a parent") + .to_path_buf(); + let published = async { + lineage.validate_source(local, source_sandbox).await?; + // Ancestry belongs to the immutable descriptor, not to the group head or export base. + captured.manifest.parent = lineage.parent.clone(); + captured.digest = captured + .manifest + .digest() + .map_err(|error| MicrosandboxError::SnapshotIntegrity(error.to_string()))?; + let descriptor = captured + .manifest + .to_canonical_bytes() + .map_err(|error| MicrosandboxError::SnapshotIntegrity(error.to_string()))?; + write_descriptor(captured.path(), &descriptor).await?; + // Publication owns its staging and ancestry sequencer. Dropping an SDK future must not + // release the source lock while a blocking group commit is still running in the background. + let captured = tokio::spawn(async move { + let update = publish_with_name_retry( + &group_dir, + staging.path(), + captured.id(), + name, + generated_name, + || format!("msb-{:08x}", rand::random::()), + ).await?; + captured.path = group_dir.join(captured.id().as_str()); + lineage.commit(captured.id()).await?; + tracing::info!(group = %update.group, head = %update.head, reason = ?update.reason, "snapshot group publication"); + captured.head_update = Some(update); + Ok::<_, MicrosandboxError>(captured) + }).await.map_err(|error| MicrosandboxError::Runtime(format!("snapshot publication task: {error}")))??; + if let Err(error) = index_upsert( + local, + captured.path(), + captured.digest(), + captured.manifest(), + ) + .await + { + tracing::warn!(%error, "snapshot index update failed after group publication"); + } + Ok(captured) + }.await; + finish_capture(published, source_recovery, installed_artifact) +} + +/// Retry generated local names against the same captured artifact; explicit names remain strict. +pub(super) async fn publish_with_name_retry( + group_dir: &Path, + staged: &Path, + snapshot_id: &SnapshotId, + mut name: String, + generated_name: bool, + mut next_name: impl FnMut() -> String, +) -> MicrosandboxResult { + loop { + let aliases = BTreeMap::from([(snapshot_id.to_string(), name)]); + match super::group::publish(group_dir, staged, &aliases, snapshot_id, false).await { + Err(MicrosandboxError::SnapshotAlreadyExists(_)) if generated_name => { + // Alias conflicts are preflight errors: no staged payload was moved and the + // descriptor's identity/ancestry remain unchanged, so no recapture is needed. + name = next_name(); + } + result => return result, + } + } +} + +/// Build a complete artifact in operation-owned staging; group publication happens afterward. +async fn capture_installed( + local: &LocalBackend, + config: SnapshotConfig, + expected_source_id: i32, +) -> MicrosandboxResult { let total_started = Instant::now(); let SnapshotConfig { name, + group: _, dest_dir, source_sandbox, labels, @@ -106,6 +262,11 @@ pub(super) async fn create_snapshot( .await? .ok_or_else(|| MicrosandboxError::SandboxNotFound(source_sandbox.clone()))?; + if model.id != expected_source_id { + return Err(MicrosandboxError::InvalidConfig( + "source sandbox changed before snapshot capture".into(), + )); + } if full { return create_full_snapshot( local, @@ -121,33 +282,39 @@ pub(super) async fn create_snapshot( .await; } - if matches!( - model.status, - SandboxStatus::Running | SandboxStatus::Draining | SandboxStatus::Paused - ) { + if model.status == SandboxStatus::Draining { return Err(MicrosandboxError::SnapshotSandboxRunning( source_sandbox.clone(), )); } - // Reuse the runtime's existing lifecycle ownership lock so start, - // replacement, and removal cannot race the upper copy. - let _lifecycle_guard = crate::runtime::acquire_sandbox_lifecycle_guard( - &local.config().run_dir(), - &source_sandbox, - std::time::Duration::from_secs(5), - ) - .await?; + // Resident runtimes own the lifecycle lock and serialize the disk cut through control. + // Stopped copies acquire it here; the SDK never reads a live writable head. + let live = matches!(model.status, SandboxStatus::Running | SandboxStatus::Paused); + let _lifecycle_guard = if live { + None + } else { + Some( + crate::runtime::acquire_sandbox_lifecycle_guard( + &local.config().run_dir(), + &source_sandbox, + std::time::Duration::from_secs(5), + ) + .await?, + ) + }; let current = sandbox_entity::Entity::find() .filter(sandbox_entity::Column::Name.eq(&source_sandbox)) .one(local.db().await?.read()) .await? .ok_or_else(|| MicrosandboxError::SandboxNotFound(source_sandbox.clone()))?; if current.id != model.id - || matches!( - current.status, - SandboxStatus::Running | SandboxStatus::Draining | SandboxStatus::Paused - ) + || current.status == SandboxStatus::Draining + || live + != matches!( + current.status, + SandboxStatus::Running | SandboxStatus::Paused + ) { return Err(MicrosandboxError::SnapshotSandboxRunning( source_sandbox.clone(), @@ -155,6 +322,7 @@ pub(super) async fn create_snapshot( } let sandbox_config: SandboxConfig = serde_json::from_str(¤t.config)?; + LocalBackend::validate_completed_restore(&sandbox_config)?; // Only OCI-rooted sandboxes can be snapshotted today; non-OCI // rootfs (passthrough, disk-image-rootfs) are out of scope. @@ -173,7 +341,15 @@ pub(super) async fn create_snapshot( } let sandbox_dir = local.sandboxes_dir().join(&source_sandbox); - let disk = snapshot_disk_closure(&sandbox_dir, &root_disk)?; + let disk = capture_disk_source( + local, + &sandbox_dir, + &source_sandbox, + current.id, + current.status, + &root_disk, + ) + .await?; // Stage the artifact in a sibling directory, so a failed create never // leaves a partial artifact at the destination (which would poison @@ -220,25 +396,20 @@ pub(super) async fn create_snapshot( promote_snapshot_directory(&staging_dir, &dest_dir, force).await?; let promote_us = promote_started.elapsed().as_micros(); - // Best-effort index upsert. Failures are logged, not propagated — - // the artifact on disk is the source of truth. - let index_started = Instant::now(); - if let Err(e) = index_upsert(local, &dest_dir, &digest, &manifest).await { - tracing::warn!(error = %e, snapshot = %digest, "snapshot_index upsert failed"); - } - let index_us = index_started.elapsed().as_micros(); tracing::info!( target: "microsandbox_checkpoint_timing", - operation = "snapshot_create_installed_stopped", + operation = "snapshot_create_installed_disk", source_sandbox, total_us = total_started.elapsed().as_micros(), artifact_build_us, promote_us, - index_us, - "stopped snapshot creation timing" + "disk snapshot creation timing" ); - Ok(Snapshot::from_parts(dest_dir, digest, manifest, labels)) + Ok(StagedSnapshot { + snapshot: Snapshot::from_parts(dest_dir, digest, manifest, labels), + source_recovery: None, + }) } /// Capture one running sandbox into an installed composite-checkpoint snapshot. @@ -248,7 +419,7 @@ async fn create_full_snapshot( source_sandbox: &str, labels: Vec<(String, String)>, model: sandbox_entity::Model, -) -> MicrosandboxResult { +) -> MicrosandboxResult { let dest_dir = destination.path; let total_started = Instant::now(); let parent_dir = dest_dir @@ -265,10 +436,9 @@ async fn create_full_snapshot( // The runtime owns capture and recovery even if this client disappears. Do not allocate an // artifact staging directory while waiting for it: there is nothing to stage until capture // succeeds. The guard also removes partial materialization on ordinary errors/cancellation. - let captured = capture_full_snapshot(source_sandbox, labels, model).await?; + let captured = capture_full_snapshot(local, source_sandbox, labels, model).await?; let capture_us = capture_started.elapsed().as_micros(); - publish_full_snapshot( - local, + stage_full_snapshot( destination, source_sandbox, captured, @@ -279,25 +449,23 @@ async fn create_full_snapshot( .await } -/// Publish a validated capture independently from the running source. Keeping this boundary -/// separate also lets failure tests exercise real materialization without starting a VM. -async fn publish_full_snapshot( - local: &LocalBackend, +/// Materialize a validated capture without exposing its operation-owned path as publication. +async fn stage_full_snapshot( destination: SnapshotDestination<'_>, source_sandbox: &str, mut captured: CapturedFullSnapshot, parent_dir: PathBuf, total_started: Instant, capture_us: u128, -) -> MicrosandboxResult { +) -> MicrosandboxResult { let SnapshotDestination { name, path: dest_dir, force, } = destination; let source_recovery = captured.source_recovery.take(); - // Recovery belongs to the source, not to the immutable artifact. Complete publication before - // returning the diagnostic; all intermediate errors must retain it as well. + // Recovery belongs to the source, not to the immutable artifact. Carry it through staging + // until the outer publisher owns the final group member and its ancestry cursor. let published = async { let staging = tempfile::Builder::new() .prefix(&format!(".{name}.")) @@ -341,11 +509,6 @@ async fn publish_full_snapshot( let promote_started = Instant::now(); promote_snapshot_directory(&staging_dir, dest_dir, force).await?; let promote_us = promote_started.elapsed().as_micros(); - let index_started = Instant::now(); - if let Err(error) = index_upsert(local, dest_dir, &digest, &captured.manifest).await { - tracing::warn!(error = %error, snapshot = %digest, "snapshot_index upsert failed"); - } - let index_us = index_started.elapsed().as_micros(); tracing::info!( target: "microsandbox_checkpoint_timing", operation = "snapshot_create_installed_full", @@ -356,7 +519,6 @@ async fn publish_full_snapshot( closure_verify_us, metadata_descriptor_us, promote_us, - index_us, "installed full snapshot creation timing" ); Ok(Snapshot::from_parts( @@ -367,17 +529,16 @@ async fn publish_full_snapshot( )) } .await; - finish_capture(published, source_recovery, |snapshot| { - PublishedSnapshotArtifact { - kind: SnapshotArtifactKind::Installed, - path: snapshot.path().to_path_buf(), - snapshot_id: snapshot.id().to_string(), - digest: snapshot.digest().to_string(), - } - }) + match published { + Ok(snapshot) => Ok(StagedSnapshot { + snapshot, + source_recovery, + }), + Err(error) => Err(capture_publication_failure(error, source_recovery)), + } } -/// Capture directly from a stopped sandbox into an archive without creating +/// Capture a disk or full snapshot directly into an archive without creating /// an installed artifact directory or index row. pub(super) async fn create_snapshot_archive( local: &LocalBackend, @@ -387,7 +548,8 @@ pub(super) async fn create_snapshot_archive( ) -> MicrosandboxResult { let total_started = Instant::now(); let SnapshotConfig { - name, + mut name, + group, dest_dir, source_sandbox, labels, @@ -395,21 +557,35 @@ pub(super) async fn create_snapshot_archive( record_integrity, full, } = config; - if dest_dir.is_some() { + if dest_dir.is_some() || group.is_some() { return Err(MicrosandboxError::InvalidConfig( - "direct archive capture is mutually exclusive with dest_dir".into(), + "direct archive capture does not install a group; omit group and dest_dir".into(), )); } + if name.is_empty() { + name = format!("msb-{:08x}", rand::random::()); + } validate_snapshot_name(&name)?; + let lineage = super::lineage::begin(local, &source_sandbox).await?; let db = local.db().await?.read(); let model = sandbox_entity::Entity::find() .filter(sandbox_entity::Column::Name.eq(&source_sandbox)) .one(db) .await? .ok_or_else(|| MicrosandboxError::SandboxNotFound(source_sandbox.clone()))?; + if model.id != lineage.sandbox_id() { + return Err(MicrosandboxError::InvalidConfig( + "source sandbox changed before snapshot capture".into(), + )); + } if full { let capture_started = Instant::now(); - let captured = capture_full_snapshot(&source_sandbox, labels, model).await?; + let mut captured = capture_full_snapshot(local, &source_sandbox, labels, model).await?; + lineage + .validate_source(local, &source_sandbox) + .await + .map_err(|error| capture_publication_failure(error, captured.source_recovery.take()))?; + captured.manifest.parent = lineage.parent.clone(); let capture_us = capture_started.elapsed().as_micros(); return publish_full_archive( SnapshotDestination { @@ -419,38 +595,46 @@ pub(super) async fn create_snapshot_archive( }, &source_sandbox, captured, + lineage, plain_tar, total_started, capture_us, ) .await; } - if matches!( - model.status, - SandboxStatus::Running | SandboxStatus::Draining | SandboxStatus::Paused - ) { + if model.status == SandboxStatus::Draining { return Err(MicrosandboxError::SnapshotSandboxRunning(source_sandbox)); } - let _lifecycle_guard = crate::runtime::acquire_sandbox_lifecycle_guard( - &local.config().run_dir(), - &source_sandbox, - std::time::Duration::from_secs(5), - ) - .await?; + let live = matches!(model.status, SandboxStatus::Running | SandboxStatus::Paused); + let _lifecycle_guard = if live { + None + } else { + Some( + crate::runtime::acquire_sandbox_lifecycle_guard( + &local.config().run_dir(), + &source_sandbox, + std::time::Duration::from_secs(5), + ) + .await?, + ) + }; let current = sandbox_entity::Entity::find() .filter(sandbox_entity::Column::Name.eq(&source_sandbox)) .one(local.db().await?.read()) .await? .ok_or_else(|| MicrosandboxError::SandboxNotFound(source_sandbox.clone()))?; if current.id != model.id - || matches!( - current.status, - SandboxStatus::Running | SandboxStatus::Draining | SandboxStatus::Paused - ) + || current.status == SandboxStatus::Draining + || live + != matches!( + current.status, + SandboxStatus::Running | SandboxStatus::Paused + ) { return Err(MicrosandboxError::SnapshotSandboxRunning(source_sandbox)); } let sandbox_config: SandboxConfig = serde_json::from_str(¤t.config)?; + LocalBackend::validate_completed_restore(&sandbox_config)?; let manifest_digest = sandbox_config.manifest_digest.clone().ok_or_else(|| { MicrosandboxError::InvalidConfig( "only OCI-rooted sandboxes with a pinned image can be snapshotted".into(), @@ -464,7 +648,16 @@ pub(super) async fn create_snapshot_archive( ))); } let sandbox_dir = local.sandboxes_dir().join(&source_sandbox); - let disk = snapshot_disk_closure(&sandbox_dir, &root_disk)?; + let disk = capture_disk_source( + local, + &sandbox_dir, + &source_sandbox, + current.id, + current.status, + &root_disk, + ) + .await?; + lineage.validate_source(local, &source_sandbox).await?; let integrity_started = Instant::now(); let integrities = vec![None; disk.sources.len()]; let labels: BTreeMap<_, _> = labels.into_iter().collect(); @@ -476,6 +669,7 @@ pub(super) async fn create_snapshot_archive( &source_sandbox, root_disk, )?; + manifest.parent = lineage.parent.clone(); if record_integrity && let SnapshotState::File(file) = &mut manifest.state { for index in 0..file.layers.len() { let source = &disk.sources[index].path; @@ -502,28 +696,42 @@ pub(super) async fn create_snapshot_archive( .iter() .map(|source| source.path.clone()) .collect::>(); - super::archive::save_direct_file_snapshot( - &manifest, - &labels, - &name, - &source_paths, - out, - plain_tar, - force, - ) - .await?; + let owned_out = out.to_path_buf(); + let logical_bytes = disk.virtual_size; + let (manifest, labels) = tokio::spawn(async move { + // A stopped disk remains locked and a live immutable cut remains pinned until the + // background writer finishes, even if the caller stops awaiting this operation. + let _disk = disk; + let _lifecycle_guard = _lifecycle_guard; + super::archive::save_direct_file_snapshot( + &manifest, + &labels, + &name, + &source_paths, + &owned_out, + plain_tar, + force, + ) + .await?; + lineage.commit(&manifest.snapshot_id).await?; + Ok::<_, MicrosandboxError>((manifest, labels)) + }) + .await + .map_err(|error| { + MicrosandboxError::Runtime(format!("snapshot archive publication: {error}")) + })??; let archive_us = archive_started.elapsed().as_micros(); tracing::info!( target: "microsandbox_checkpoint_timing", - operation = "snapshot_create_archive_stopped", + operation = "snapshot_create_archive_disk", source_sandbox, plain_tar, record_integrity, - logical_bytes = disk.virtual_size, + logical_bytes, total_us = total_started.elapsed().as_micros(), integrity_us, archive_us, - "direct stopped snapshot archive timing" + "direct disk snapshot archive timing" ); Ok(SnapshotArchive::from_parts( out.to_path_buf(), @@ -538,6 +746,7 @@ async fn publish_full_archive( destination: SnapshotDestination<'_>, source_sandbox: &str, mut captured: CapturedFullSnapshot, + lineage: super::lineage::CaptureLineage, plain_tar: bool, total_started: Instant, capture_us: u128, @@ -554,16 +763,28 @@ async fn publish_full_archive( .digest() .map_err(|error| MicrosandboxError::SnapshotIntegrity(error.to_string()))?; let archive_started = Instant::now(); - super::archive::save_direct_checkpoint_snapshot( - &captured.manifest, - &captured.labels, - name, - &captured.checkpoint_path, - out, - plain_tar, - force, - ) - .await?; + let name = name.to_owned(); + let owned_out = out.to_path_buf(); + // Keep both the immutable input and source sequencer alive when the caller cancels its + // wait. The archive writer and cursor publication still complete in their original order. + let captured = tokio::spawn(async move { + super::archive::save_direct_checkpoint_snapshot( + &captured.manifest, + &captured.labels, + &name, + &captured.checkpoint_path, + &owned_out, + plain_tar, + force, + ) + .await?; + lineage.commit(&captured.manifest.snapshot_id).await?; + Ok::<_, MicrosandboxError>(captured) + }) + .await + .map_err(|error| { + MicrosandboxError::Runtime(format!("snapshot archive publication: {error}")) + })??; let archive_us = archive_started.elapsed().as_micros(); tracing::info!( target: "microsandbox_checkpoint_timing", @@ -594,6 +815,7 @@ async fn publish_full_archive( /// Installed snapshots and direct archives share this boundary so both publish byte-for-byte the /// same descriptor and checkpoint closure. async fn capture_full_snapshot( + local: &LocalBackend, source_sandbox: &str, labels: Vec<(String, String)>, model: sandbox_entity::Model, @@ -605,6 +827,7 @@ async fn capture_full_snapshot( )); } let sandbox_config: SandboxConfig = serde_json::from_str(&model.config)?; + LocalBackend::validate_completed_restore(&sandbox_config)?; let manifest_digest = sandbox_config.manifest_digest.clone().ok_or_else(|| { MicrosandboxError::InvalidConfig(format!( "sandbox '{source_sandbox}' has no OCI image pinned; full snapshots require an OCI root" @@ -615,7 +838,8 @@ async fn capture_full_snapshot( let checkpoint_id = format!("checkpoint_{:032x}", rand::random::()); let outcome = - crate::sandbox::control_checkpoint_create(source_sandbox, checkpoint_id.clone()).await?; + crate::sandbox::control_checkpoint_create(local, source_sandbox, checkpoint_id.clone()) + .await?; let checkpoint = outcome.checkpoint; let validated = (|| { if checkpoint.checkpoint_id != checkpoint_id { @@ -745,6 +969,28 @@ fn finish_capture( Err(MicrosandboxError::SnapshotSourceRecovery(Box::new(failure))) } +fn installed_artifact(snapshot: &Snapshot) -> PublishedSnapshotArtifact { + PublishedSnapshotArtifact { + kind: SnapshotArtifactKind::Installed, + path: snapshot.path().to_path_buf(), + snapshot_id: snapshot.id().to_string(), + digest: snapshot.digest().to_string(), + } +} + +fn capture_publication_failure( + error: MicrosandboxError, + source_recovery: Option, +) -> MicrosandboxError { + match source_recovery { + Some(mut failure) => { + failure.publication_error = Some(error.to_string()); + MicrosandboxError::SnapshotSourceRecovery(Box::new(failure)) + } + None => error, + } +} + /// Build the artifact contents (upper copy, integrity, descriptor) into /// `dir`. Pure staging: the caller promotes or discards the directory. async fn build_artifact( @@ -866,7 +1112,7 @@ async fn build_artifact( payload_sync_us, integrity_us, descriptor_us, - "stopped snapshot artifact build timing" + "disk snapshot artifact build timing" ); Ok((digest, manifest)) @@ -1011,6 +1257,90 @@ fn snapshot_root_disk( } } +/// Resident runtimes return a sealed closure while retaining their lifecycle lock. +/// Stopped callers own that lock themselves. Packaging never reads a live writable head. +async fn capture_disk_source( + local: &LocalBackend, + sandbox_dir: &Path, + source: &str, + source_id: i32, + status: SandboxStatus, + root_disk: &SnapshotRootDisk, +) -> MicrosandboxResult { + if !matches!(status, SandboxStatus::Running | SandboxStatus::Paused) { + return snapshot_disk_closure(sandbox_dir, root_disk); + } + let id = format!("disk_{:032x}", rand::random::()); + let captured = + crate::sandbox::control_disk_checkpoint_create(local, source, id.clone()).await?; + let expected_path = sandbox_dir.join("runtime").join("checkpoints").join(&id); + let expected_device = match root_disk { + SnapshotRootDisk::Flat => "vda", + SnapshotRootDisk::Managed => "vdb", + SnapshotRootDisk::Tmpfs { .. } => { + return Err(MicrosandboxError::InvalidConfig( + "tmpfs requires a full snapshot".into(), + )); + } + }; + if captured.checkpoint_id != id + || captured.path != expected_path + || captured.disk.device_id != expected_device + { + return Err(MicrosandboxError::SnapshotIntegrity( + "disk capture identity, path, or root device mismatch".into(), + )); + } + captured + .disk + .validate() + .map_err(|e| MicrosandboxError::SnapshotIntegrity(e.to_string()))?; + let sources = captured + .disk + .layers + .iter() + .map(|layer| { + let format = match layer.format.as_str() { + "raw" => SnapshotFormat::Raw, + "qcow2" => SnapshotFormat::Qcow2, + other => { + return Err(MicrosandboxError::SnapshotIntegrity(format!( + "unsupported live disk format {other}" + ))); + } + }; + Ok(SnapshotDiskSource { + path: captured + .path + .join("layers") + .join(format!("{}.{}", layer.layer_id, layer.format)), + format, + }) + }) + .collect::>>()?; + let size = captured + .disk + .layers + .last() + .ok_or_else(|| MicrosandboxError::SnapshotIntegrity("empty disk capture".into()))? + .virtual_size; + let mut disk = validate_snapshot_disk_sources(sources, size)?; + disk.capture_root = Some(expected_path); + // A live runtime owns the lifecycle lock, not this SDK call. If the source was replaced + // between lookup and capture, never publish its disk under the original image/config. + let current = sandbox_entity::Entity::find() + .filter(sandbox_entity::Column::Name.eq(source)) + .one(local.db().await?.read()) + .await?; + if !current.is_some_and(|model| model.id == source_id) { + return Err(MicrosandboxError::Runtime( + "snapshot source was replaced during disk capture; retry with the current sandbox" + .into(), + )); + } + Ok(disk) +} + fn snapshot_disk_closure( sandbox_dir: &Path, root_disk: &SnapshotRootDisk, @@ -1105,6 +1435,7 @@ fn validate_snapshot_disk_sources( Ok(SnapshotDiskClosure { sources, virtual_size, + capture_root: None, }) } @@ -1162,6 +1493,20 @@ fn validate_snapshot_name(name: &str) -> MicrosandboxResult<()> { pub(crate) fn materialize_checkpoint_closure( source: &Path, destination: &Path, +) -> std::io::Result<()> { + materialize_checkpoint_tree(source, destination, true) +} + +/// Construction-only closure: retain independent links, but do not make disposable staging +/// durable. Persistent disk successors are published separately before guest activation. +pub(crate) fn stage_checkpoint_closure(source: &Path, destination: &Path) -> std::io::Result<()> { + materialize_checkpoint_tree(source, destination, false) +} + +fn materialize_checkpoint_tree( + source: &Path, + destination: &Path, + durable: bool, ) -> std::io::Result<()> { let source_metadata = std::fs::symlink_metadata(source)?; if !source_metadata.file_type().is_dir() { @@ -1176,7 +1521,7 @@ pub(crate) fn materialize_checkpoint_closure( let source_member = source.join(member); match std::fs::symlink_metadata(&source_member) { Ok(metadata) if metadata.file_type().is_dir() => { - copy_checkpoint_directory(&source_member, &destination.join(member))?; + copy_checkpoint_directory(&source_member, &destination.join(member), durable)?; } Ok(_) => { return Err(std::io::Error::new( @@ -1193,10 +1538,17 @@ pub(crate) fn materialize_checkpoint_closure( &source.join("checkpoint.json"), &destination.join("checkpoint.json"), )?; - sync_directory(destination) + if durable { + sync_directory(destination)?; + } + Ok(()) } -fn copy_checkpoint_directory(source: &Path, destination: &Path) -> std::io::Result<()> { +fn copy_checkpoint_directory( + source: &Path, + destination: &Path, + durable: bool, +) -> std::io::Result<()> { std::fs::create_dir(destination)?; for entry in std::fs::read_dir(source)? { let entry = entry?; @@ -1204,7 +1556,7 @@ fn copy_checkpoint_directory(source: &Path, destination: &Path) -> std::io::Resu let destination_path = destination.join(entry.file_name()); let metadata = std::fs::symlink_metadata(&source_path)?; if metadata.file_type().is_dir() { - copy_checkpoint_directory(&source_path, &destination_path)?; + copy_checkpoint_directory(&source_path, &destination_path, durable)?; } else if metadata.file_type().is_file() { copy_checkpoint_file(&source_path, &destination_path)?; } else { @@ -1217,7 +1569,10 @@ fn copy_checkpoint_directory(source: &Path, destination: &Path) -> std::io::Resu )); } } - sync_directory(destination) + if durable { + sync_directory(destination)?; + } + Ok(()) } pub(crate) fn copy_checkpoint_file(source: &Path, destination: &Path) -> std::io::Result<()> { @@ -1371,9 +1726,26 @@ mod tests { use std::path::PathBuf; use microsandbox_types::DiskImageFormat; + use sea_orm::{ActiveModelTrait, ActiveValue::Set}; use super::*; + async fn fixture_source(local: &LocalBackend) { + let mut config = SandboxConfig::default(); + config.spec.name = "box".into(); + std::fs::create_dir_all(local.sandboxes_dir().join("box")).unwrap(); + sandbox_entity::ActiveModel { + name: Set("box".into()), + config: Set(serde_json::to_string(&config).unwrap()), + status: Set(SandboxStatus::Crashed), + ephemeral: Set(false), + ..Default::default() + } + .insert(local.db().await.unwrap().write()) + .await + .unwrap(); + } + fn file_metadata(root_disk: SnapshotRootDisk) -> FileSnapshotMetadata<'static> { FileSnapshotMetadata { image_reference: "docker.io/library/alpine:3.20".into(), @@ -1497,22 +1869,50 @@ mod tests { let captured = captured_fixture(&temp.path().join("checkpoint"), Some(detail)); let canonical = captured.manifest.to_canonical_bytes().unwrap(); let root = captured.checkpoint_root.clone(); - let destination = temp.path().join("snapshot"); - let error = publish_full_snapshot( - &local, + let snapshot_id = captured.manifest.snapshot_id.clone(); + fixture_source(&local).await; + let lineage = super::super::lineage::begin(&local, "box").await.unwrap(); + let group = super::super::group::ensure(&local.snapshots_dir(), Some("box")) + .await + .unwrap(); + let staging = tempfile::Builder::new() + .prefix(".capture-") + .tempdir_in(&group) + .unwrap(); + let staged_path = staging.path().join("snapshot"); + let staged = stage_full_snapshot( SnapshotDestination { name: "snapshot", - path: &destination, + path: &staged_path, force: false, }, "box", captured, - temp.path().to_path_buf(), + staging.path().to_path_buf(), Instant::now(), 0, ) .await + .unwrap(); + assert!(staged.source_recovery.is_some()); + assert!( + super::super::store::list_indexed(&local) + .await + .unwrap() + .is_empty() + ); + let error = publish_snapshot_group( + &local, + staged, + staging, + lineage, + "snapshot".into(), + false, + "box", + ) + .await .unwrap_err(); + let destination = group.join(snapshot_id.as_str()); let MicrosandboxError::SnapshotSourceRecovery(failure) = error else { panic!("expected partial failure") }; @@ -1521,6 +1921,27 @@ mod tests { let artifact = failure.artifact.unwrap(); assert_eq!(artifact.kind, SnapshotArtifactKind::Installed); assert_eq!(artifact.path, destination); + assert_ne!(artifact.path, staged_path); + assert!(!staged_path.exists()); + let indexed = super::super::store::list_indexed(&local).await.unwrap(); + assert_eq!(indexed.len(), 1); + assert_eq!( + indexed[0].artifact_path, + destination.canonicalize().unwrap() + ); + assert_eq!( + super::super::group::resolve(&local.snapshots_dir(), "box") + .await + .unwrap(), + destination + ); + assert_eq!( + super::super::lineage::begin(&local, "box") + .await + .unwrap() + .parent, + Some(snapshot_id) + ); assert_eq!( std::fs::read(destination.join(DESCRIPTOR_FILENAME)).unwrap(), canonical @@ -1550,6 +1971,8 @@ mod tests { ); let expected = captured.manifest.to_canonical_bytes().unwrap(); let out = temp.path().join("snapshot.tar"); + fixture_source(&local).await; + let lineage = super::super::lineage::begin(&local, "box").await.unwrap(); let error = publish_full_archive( SnapshotDestination { name: "snapshot", @@ -1558,6 +1981,7 @@ mod tests { }, "box", captured, + lineage, true, Instant::now(), 0, @@ -1592,6 +2016,7 @@ mod tests { .build() .await .unwrap(); + fixture_source(&local).await; for archive in [false, true] { let captured = captured_fixture( &temp.path().join(if archive { @@ -1611,12 +2036,12 @@ mod tests { force: false, }; let error = if archive { - publish_full_archive(target, "box", captured, true, Instant::now(), 0) + let lineage = super::super::lineage::begin(&local, "box").await.unwrap(); + publish_full_archive(target, "box", captured, lineage, true, Instant::now(), 0) .await .unwrap_err() } else { - publish_full_snapshot( - &local, + stage_full_snapshot( target, "box", captured, @@ -1647,6 +2072,13 @@ mod tests { #[tokio::test] async fn successful_source_recovery_keeps_existing_success_result() { let temp = tempfile::tempdir().unwrap(); + let local = LocalBackend::builder() + .home(temp.path().join("home")) + .build() + .await + .unwrap(); + fixture_source(&local).await; + let lineage = super::super::lineage::begin(&local, "box").await.unwrap(); let captured = captured_fixture(&temp.path().join("checkpoint"), None); let out = temp.path().join("snapshot.tar"); let archive = publish_full_archive( @@ -1657,6 +2089,7 @@ mod tests { }, "box", captured, + lineage, true, Instant::now(), 0, @@ -1729,6 +2162,7 @@ mod tests { let source = temp.path().join("source.ext4"); std::fs::write(&source, b"snapshot payload").unwrap(); let disk = SnapshotDiskClosure { + capture_root: None, sources: vec![SnapshotDiskSource { path: source, format: SnapshotFormat::Raw, @@ -1780,6 +2214,7 @@ mod tests { .await .unwrap(); let disk = SnapshotDiskClosure { + capture_root: None, sources: vec![ SnapshotDiskSource { path: raw, diff --git a/sdk/rust/lib/snapshot/downgrade.rs b/sdk/rust/lib/snapshot/downgrade.rs index 2107cbb4f..b7b96d42c 100644 --- a/sdk/rust/lib/snapshot/downgrade.rs +++ b/sdk/rust/lib/snapshot/downgrade.rs @@ -1753,7 +1753,8 @@ mod tests { "reverse_complete" ); - Migrator::down(pools.write().inner(), Some(1)) + // Reverse both the empty group projection and stable-identity projection. + Migrator::down(pools.write().inner(), Some(2)) .await .unwrap(); let count = pools diff --git a/sdk/rust/lib/snapshot/group.rs b/sdk/rust/lib/snapshot/group.rs new file mode 100644 index 000000000..9998f81e6 --- /dev/null +++ b/sdk/rust/lib/snapshot/group.rs @@ -0,0 +1,938 @@ +//! Durable local snapshot namespaces and their explicitly selected heads. +//! +//! Group membership is represented by installed artifact directories. Only the head and each +//! member's optional local alias need metadata; immutable descriptors remain authoritative for +//! ancestry. All group operations share one process-held lock, acquired off the async executor. + +use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Write}; +#[cfg(unix)] +use std::os::unix::fs::OpenOptionsExt; +use std::path::{Path, PathBuf}; + +use microsandbox_image::snapshot::{ + DESCRIPTOR_FILENAME, MAX_DESCRIPTOR_BYTES, Manifest, SnapshotId, +}; +use microsandbox_utils::process_lock; +use serde::{Deserialize, Serialize}; + +use crate::{MicrosandboxError, MicrosandboxResult}; + +//-------------------------------------------------------------------------------------------------- +// Constants +//-------------------------------------------------------------------------------------------------- + +pub(crate) const GROUP_FILENAME: &str = "group.json"; +const GROUP_SCHEMA: &str = "microsandbox.snapshot-group/1"; +const MEMBER_FILENAME: &str = "group-member.json"; +const MEMBER_SCHEMA: &str = "microsandbox.snapshot-group-member/1"; +const MAX_METADATA_BYTES: usize = 4096; +const MAX_NAME_BYTES: usize = 128; +const MAX_ANCESTRY_DEPTH: usize = 65536; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// Outcome of publishing snapshots into a local group or explicitly selecting its head. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct HeadUpdate { + /// Local group name. + pub group: String, + /// Previously selected stable snapshot ID, if the group had a head. + pub previous: Option, + /// Stable snapshot ID selected after the operation. + pub head: String, + /// Explanation for advancing or retaining the selected head. + pub reason: HeadUpdateReason, + /// Whether the selected head changed. + pub changed: bool, +} + +/// Why a snapshot group's head advanced or remained selected. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum HeadUpdateReason { + /// The first candidate initialized an empty group. + Initialized, + /// The candidate is a proven descendant of the current head. + FastForwarded, + /// The caller explicitly selected an installed member. + Selected, + /// The candidate was already selected, or the caller only read the head. + Unchanged, + /// The candidate is not a descendant of the current head. + Diverged, + /// Missing history prevents proving that the candidate descends from the head. + UnknownAncestry, + /// Supplied archive heads have multiple tips that known ancestry cannot order. + AmbiguousCandidates, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct GroupState { + schema: String, + head: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct MemberMetadata { + schema: String, + name: String, +} + +#[derive(Debug)] +struct Member { + path: PathBuf, + digest: String, + parent: Option, + name: Option, +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +/// Resolve a group head or a qualified group member; explicit paths belong to the caller. +pub(super) async fn resolve(root: &Path, selector: &str) -> MicrosandboxResult { + let root = root.to_path_buf(); + let selector = selector.to_owned(); + blocking(move || { + let (name, member) = parse_selector(&selector)?; + let directory = root.join(name); + let _lock = lock_group(&directory).map_err(|error| match error { + // A selector lookup has the same not-found contract as an explicit snapshot path. + MicrosandboxError::Io(ref io) if io.kind() == std::io::ErrorKind::NotFound => { + MicrosandboxError::SnapshotNotFound(selector.clone()) + } + other => other, + })?; + let state = read_group(&directory)?; + let (id, _) = resolve_selected(&directory, &state, member)?; + Ok(directory.join(id)) + }) + .await +} + +/// Open or create an explicitly named group, or create a fresh generated local group. +pub(super) async fn ensure(root: &Path, name: Option<&str>) -> MicrosandboxResult { + let root = root.to_path_buf(); + let name = name.map(str::to_owned); + blocking(move || { + if let Some(name) = &name { + validate_group_name(name)?; + } + fs::create_dir_all(&root)?; + require_directory(&root)?; + // Serialize creation separately: the group lock does not exist until publication. + let creation_lock = process_lock::open_lock_file(&root.join(".groups.lock"))?; + process_lock::lock_exclusive(&creation_lock)?; + let name = match name { + Some(name) => name, + None => loop { + let candidate = format!("msb-{:08x}", rand::random::()); + if !path_exists(&root.join(&candidate))? { + break candidate; + } + }, + }; + let directory = root.join(&name); + if path_exists(&directory)? { + require_directory(&directory)?; + if !path_exists(&directory.join(GROUP_FILENAME))? { + return Err(MicrosandboxError::InvalidConfig(format!( + "'{name}' already names a snapshot directory, not a snapshot group; choose another group name or open the existing snapshot by its explicit path" + ))); + } + read_group(&directory)?; + return Ok(directory); + } + + // The initial head and lock become visible together with the new group directory. + let staging = tempfile::Builder::new() + .prefix(".group-new-") + .tempdir_in(&root)?; + write_group(staging.path(), None)?; + process_lock::create_new_lock_file(&staging.path().join(".group.lock"))?.sync_all()?; + sync_directory(staging.path())?; + fs::rename(staging.path(), &directory)?; + sync_directory(&root)?; + Ok(directory) + }) + .await +} + +/// Publish complete staged artifacts and atomically decide the group's next head. +/// +/// `staged` contains immediate child artifact directories. The caller prepares, validates, and +/// flushes their payloads before calling this function. All descriptor and alias conflicts are +/// checked before publication; existing identical artifacts are never overwritten. +pub(super) async fn publish( + group_dir: &Path, + staged: &Path, + aliases: &BTreeMap, + candidate: &SnapshotId, + set_head: bool, +) -> MicrosandboxResult { + publish_batch( + group_dir, + staged, + aliases, + std::slice::from_ref(candidate), + set_head, + ) + .await? + .ok_or_else(|| integrity("single snapshot publication did not choose a head".into())) +} + +/// Publish a validated batch under one lock, selecting a head only when supplied candidates +/// have one tip that is a known descendant of every other candidate. +pub(super) async fn publish_batch( + group_dir: &Path, + staged: &Path, + aliases: &BTreeMap, + candidates: &[SnapshotId], + set_head: bool, +) -> MicrosandboxResult> { + if candidates.is_empty() { + return Err(MicrosandboxError::InvalidConfig( + "snapshot batch must contain at least one candidate head".into(), + )); + } + let group_dir = group_dir.to_path_buf(); + let staged = staged.to_path_buf(); + let aliases = aliases.clone(); + let candidates: BTreeSet = candidates.iter().map(ToString::to_string).collect(); + blocking(move || { + require_directory(&staged)?; + if fs::canonicalize(&group_dir)?.starts_with(fs::canonicalize(&staged)?) { + return Err(MicrosandboxError::InvalidConfig( + "snapshot staging must not contain the destination group".into(), + )); + } + let incoming = read_members(&staged, false)?; + let _lock = lock_group(&group_dir)?; + let state = read_group(&group_dir)?; + let mut members = read_members(&group_dir, true)?; + validate_head(&state, &members)?; + + for (id, member) in &incoming { + if let Some(existing) = members.get(id) { + if existing.digest != member.digest { + return Err(integrity(format!( + "snapshot ID {id} already exists in this group with a different descriptor" + ))); + } + } else { + // Even an unrecognized file at the destination must not be overwritten. + if path_exists(&group_dir.join(id))? { + return Err(integrity(format!( + "snapshot destination already exists: {}", + group_dir.join(id).display() + ))); + } + members.insert( + id.clone(), + Member { + path: member.path.clone(), + digest: member.digest.clone(), + parent: member.parent.clone(), + name: None, + }, + ); + } + } + for candidate in &candidates { + if !members.contains_key(candidate) { + return Err(MicrosandboxError::SnapshotNotFound(candidate.clone())); + } + } + apply_aliases(&mut members, &aliases)?; + validate_ancestry(&members)?; + let update = batch_head_update( + &group_dir, + state.head.as_deref(), + &candidates, + &members, + set_head, + )?; + + // Complete members are durable before publishing the head. A crash or I/O error can + // leave additional complete members, but never a head pointing at a half-written one. + for (id, member) in &incoming { + let destination = group_dir.join(id); + if !path_exists(&destination)? { + write_member_name(&member.path, members[id].name.as_deref())?; + sync_directory(&member.path)?; + fs::rename(&member.path, destination)?; + } + } + for id in aliases.keys() { + write_member_name(&group_dir.join(id), members[id].name.as_deref())?; + } + sync_directory(&group_dir)?; + if let Some(update) = &update + && update.changed + { + write_group(&group_dir, Some(update.head.clone()))?; + } + Ok(update) + }) + .await +} + +/// List installed members available as dependency sources without creating a destination group. +/// Callers must still validate the physical payloads they borrow from these artifact paths. +pub(super) async fn dependency_members( + root: &Path, + name: &str, +) -> MicrosandboxResult> { + let root = root.to_path_buf(); + let name = name.to_owned(); + blocking(move || { + validate_group_name(&name)?; + if !path_exists(&root)? { + return Ok(Vec::new()); + } + require_directory(&root)?; + let directory = root.join(name); + if !path_exists(&directory)? { + return Ok(Vec::new()); + } + require_directory(&directory)?; + read_group(&directory)?; + // Preflight must not create even a lock file in an existing malformed namespace. + let lock = process_lock::open_existing_lock_file(&directory.join(".group.lock"))?; + process_lock::lock_exclusive(&lock)?; + let state = read_group(&directory)?; + let members = read_members(&directory, true)?; + validate_head(&state, &members)?; + validate_ancestry(&members)?; + Ok(members.into_values().map(|member| member.path).collect()) + }) + .await +} + +/// Read a bare group's head, or explicitly select a qualified member as its head. +pub(super) async fn select(root: &Path, selector: &str) -> MicrosandboxResult { + let root = root.to_path_buf(); + let selector = selector.to_owned(); + blocking(move || { + let (name, selected) = parse_selector(&selector)?; + let directory = root.join(name); + let _lock = lock_group(&directory)?; + let state = read_group(&directory)?; + let (candidate, member) = resolve_selected(&directory, &state, selected)?; + let members = BTreeMap::from([(candidate.clone(), member)]); + let update = head_update( + &directory, + state.head.as_deref(), + &candidate, + &members, + selected.is_some(), + )?; + if update.changed { + write_group(&directory, Some(update.head.clone()))?; + } + Ok(update) + }) + .await +} + +/// Read a member's optional local friendly name without altering its immutable descriptor. +pub(super) fn member_name(path: &Path) -> MicrosandboxResult> { + let metadata_path = path.join(MEMBER_FILENAME); + if !path_exists(&metadata_path)? { + return Ok(None); + } + let metadata: MemberMetadata = + serde_json::from_slice(&read_regular(&metadata_path, MAX_METADATA_BYTES)?)?; + if metadata.schema != MEMBER_SCHEMA { + return Err(integrity(format!( + "unsupported snapshot member metadata schema: {}", + metadata.schema + ))); + } + validate_alias(&metadata.name)?; + Ok(Some(metadata.name)) +} + +/// Return the containing group when a member's parent has regular group metadata. +pub(super) fn group_path(path: &Path) -> Option { + let parent = path.parent()?; + let metadata = fs::symlink_metadata(parent.join(GROUP_FILENAME)).ok()?; + metadata.file_type().is_file().then(|| parent.to_path_buf()) +} + +/// Remove a grouped member under its publication lock, returning false for ungrouped paths. +pub(super) async fn remove_member(path: &Path) -> MicrosandboxResult { + let path = path.to_path_buf(); + blocking(move || { + let Some(directory) = group_path(&path) else { + return Ok(false); + }; + let _lock = lock_group(&directory)?; + let state = read_group(&directory)?; + let members = read_members(&directory, true)?; + validate_head(&state, &members)?; + let id = path.file_name().and_then(|name| name.to_str()).ok_or_else(|| { + MicrosandboxError::InvalidConfig("snapshot member path has no stable ID".into()) + })?; + if !members.contains_key(id) { + return Err(MicrosandboxError::SnapshotNotFound(id.into())); + } + if state.head.as_deref() == Some(id) { + if members.len() > 1 { + return Err(MicrosandboxError::InvalidConfig(format!( + "cannot remove current head {id}; first select another snapshot with 'msb snapshot head {}:'", + directory.file_name().unwrap_or_default().to_string_lossy() + ))); + } + // Clear first so an interrupted recursive removal cannot strand a dangling head. + // A failed removal is recoverable by explicitly selecting the surviving member. + write_group(&directory, None)?; + } + fs::remove_dir_all(&path).map_err(|error| { + MicrosandboxError::Custom(format!( + "could not fully remove snapshot {}: {error}; inspect the group before retrying", + path.display() + )) + })?; + sync_directory(&directory)?; + Ok(true) + }) + .await +} + +//-------------------------------------------------------------------------------------------------- +// Functions: Helpers +//-------------------------------------------------------------------------------------------------- + +async fn blocking( + work: impl FnOnce() -> MicrosandboxResult + Send + 'static, +) -> MicrosandboxResult { + tokio::task::spawn_blocking(work) + .await + .map_err(|error| MicrosandboxError::Custom(format!("snapshot group operation: {error}")))? +} + +fn parse_selector(selector: &str) -> MicrosandboxResult<(&str, Option<&str>)> { + let (group, member) = match selector.split_once(':') { + Some((group, member)) => (group, Some(member)), + None => (selector, None), + }; + validate_group_name(group)?; + if let Some(member) = member { + validate_name(member, "snapshot selector")?; + } + Ok((group, member)) +} + +fn validate_name(name: &str, kind: &str) -> MicrosandboxResult<()> { + let first = name.as_bytes().first().copied(); + if name.len() > MAX_NAME_BYTES + || !first.is_some_and(|byte| byte.is_ascii_alphanumeric()) + || name.ends_with('.') + || !name + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"-_.".contains(&byte)) + { + return Err(MicrosandboxError::InvalidConfig(format!( + "invalid {kind} '{name}': use 1–{MAX_NAME_BYTES} ASCII letters, digits, '-', '_' or '.', start with a letter or digit, and do not end with '.'" + ))); + } + // Reject device names even on Unix so local selectors remain portable to Windows. + let stem = name.split('.').next().unwrap_or(name).to_ascii_uppercase(); + if matches!(stem.as_str(), "CON" | "PRN" | "AUX" | "NUL") + || (stem.len() == 4 + && (stem.starts_with("COM") || stem.starts_with("LPT")) + && matches!(stem.as_bytes()[3], b'1'..=b'9')) + { + return Err(MicrosandboxError::InvalidConfig(format!( + "invalid {kind} '{name}': reserved device name" + ))); + } + Ok(()) +} + +pub(super) fn validate_alias(name: &str) -> MicrosandboxResult<()> { + validate_name(name, "snapshot name")?; + if SnapshotId::new(name).is_ok() { + return Err(MicrosandboxError::InvalidConfig( + "a snapshot's friendly name must not be a stable snapshot ID".into(), + )); + } + Ok(()) +} + +fn validate_group_name(name: &str) -> MicrosandboxResult<()> { + validate_name(name, "group")?; + if matches!(name, "sha256" | "sha512") || SnapshotId::new(name).is_ok() { + return Err(MicrosandboxError::InvalidConfig(format!( + "invalid group name '{name}': reserved snapshot identifier namespace" + ))); + } + Ok(()) +} + +fn lock_group(directory: &Path) -> MicrosandboxResult { + require_directory(directory)?; + read_group(directory)?; + let lock = process_lock::open_lock_file(&directory.join(".group.lock"))?; + process_lock::lock_exclusive(&lock)?; + Ok(lock) +} + +fn read_group(directory: &Path) -> MicrosandboxResult { + let path = directory.join(GROUP_FILENAME); + if !path_exists(&path)? { + return Err(MicrosandboxError::SnapshotNotFound(format!( + "snapshot group {}", + directory.display() + ))); + } + let state: GroupState = serde_json::from_slice(&read_regular(&path, MAX_METADATA_BYTES)?)?; + if state.schema != GROUP_SCHEMA { + return Err(integrity(format!( + "unsupported snapshot group schema: {}", + state.schema + ))); + } + if let Some(head) = &state.head { + SnapshotId::new(head).map_err(|error| integrity(error.to_string()))?; + } + Ok(state) +} + +fn write_group(directory: &Path, head: Option) -> MicrosandboxResult<()> { + write_json( + directory, + GROUP_FILENAME, + &GroupState { + schema: GROUP_SCHEMA.into(), + head, + }, + ) +} + +fn read_members(directory: &Path, installed: bool) -> MicrosandboxResult> { + let mut members = BTreeMap::new(); + for entry in fs::read_dir(directory)? { + let entry = entry?; + let filename = entry.file_name(); + let name = filename + .to_str() + .ok_or_else(|| integrity("snapshot member directory name is not valid UTF-8".into()))?; + if installed && !name.starts_with("snap_") { + continue; + } + if !entry.file_type()?.is_dir() { + if installed { + return Err(integrity(format!( + "snapshot member is not a regular directory: {}", + entry.path().display() + ))); + } + return Err(integrity(format!( + "snapshot staging contains a non-directory member: {}", + entry.path().display() + ))); + } + let (id, member) = read_member(&entry.path(), installed)?; + if installed && id != name { + return Err(integrity(format!( + "snapshot member directory {name} does not match descriptor ID {id}" + ))); + } + if members.insert(id.clone(), member).is_some() { + return Err(integrity(format!( + "snapshot staging contains duplicate stable ID {id}" + ))); + } + } + Ok(members) +} + +fn apply_aliases( + members: &mut BTreeMap, + aliases: &BTreeMap, +) -> MicrosandboxResult<()> { + for (id, name) in aliases { + validate_alias(name)?; + let member = members + .get_mut(id) + .ok_or_else(|| MicrosandboxError::SnapshotNotFound(id.clone()))?; + if let Some(existing) = &member.name + && existing != name + { + return Err(MicrosandboxError::SnapshotAlreadyExists(format!( + "snapshot {id} already has local name '{existing}', not '{name}'" + ))); + } + member.name = Some(name.clone()); + } + let mut names = BTreeMap::new(); + for (id, member) in members { + if let Some(name) = &member.name + && let Some(previous) = names.insert(name.clone(), id.clone()) + { + return Err(MicrosandboxError::SnapshotAlreadyExists(format!( + "snapshot name '{name}' conflicts between {previous} and {id} in this group" + ))); + } + } + Ok(()) +} + +fn read_member(path: &Path, installed: bool) -> MicrosandboxResult<(String, Member)> { + require_directory(path)?; + let manifest = Manifest::from_bytes(&read_regular( + &path.join(DESCRIPTOR_FILENAME), + MAX_DESCRIPTOR_BYTES, + )?) + .map_err(|error| integrity(error.to_string()))?; + let id = manifest.snapshot_id.to_string(); + let member = Member { + digest: manifest + .digest() + .map_err(|error| integrity(error.to_string()))?, + parent: manifest.parent.map(|parent| parent.to_string()), + name: if installed { member_name(path)? } else { None }, + path: path.to_path_buf(), + }; + Ok((id, member)) +} + +fn resolve_selected( + directory: &Path, + state: &GroupState, + selected: Option<&str>, +) -> MicrosandboxResult<(String, Member)> { + // ID and head lookup touch only the selected descriptor. Large histories do not make the + // normal open path progressively slower, and unrelated artifacts need not be reopened. + let selected_id = match selected { + None => Some(state.head.clone().ok_or_else(|| { + MicrosandboxError::SnapshotNotFound(format!( + "snapshot group {} has no head selected; choose an installed member with 'msb snapshot head {}:'", + directory.display(), + directory.file_name().unwrap_or_default().to_string_lossy() + )) + })?), + Some(selected) if SnapshotId::new(selected).is_ok() => Some(selected.to_owned()), + Some(_) => None, + }; + let expected = match selected_id { + Some(id) => id, + None => { + let selected = selected.unwrap(); + let mut matched = None; + for entry in fs::read_dir(directory)? { + let entry = entry?; + let name = entry.file_name(); + let Some(name) = name.to_str().filter(|name| name.starts_with("snap_")) else { + continue; + }; + require_directory(&entry.path())?; + if member_name(&entry.path())?.as_deref() == Some(selected) { + if matched.is_some() { + return Err(integrity(format!( + "snapshot name '{selected}' is ambiguous in this group" + ))); + } + matched = Some(name.to_owned()); + } + } + matched.ok_or_else(|| { + MicrosandboxError::SnapshotNotFound(format!( + "{}:{selected}", + directory.file_name().unwrap_or_default().to_string_lossy() + )) + })? + } + }; + let path = directory.join(&expected); + if !path_exists(&path)? { + return Err(MicrosandboxError::SnapshotNotFound(format!( + "snapshot group member {} is missing", + path.display() + ))); + } + let (id, member) = read_member(&path, true)?; + if id != expected { + return Err(integrity(format!( + "snapshot member directory {expected} does not match descriptor ID {id}" + ))); + } + Ok((id, member)) +} + +fn validate_head(state: &GroupState, members: &BTreeMap) -> MicrosandboxResult<()> { + if let Some(head) = &state.head + && !members.contains_key(head) + { + return Err(integrity(format!( + "snapshot group head {head} is missing; explicitly select an installed member to repair the head" + ))); + } + Ok(()) +} + +fn head_update( + directory: &Path, + previous: Option<&str>, + candidate: &str, + members: &BTreeMap, + explicit: bool, +) -> MicrosandboxResult { + let reason = match previous { + Some(head) if head == candidate => HeadUpdateReason::Unchanged, + _ if explicit => HeadUpdateReason::Selected, + None => HeadUpdateReason::Initialized, + Some(head) => ancestry_reason(candidate, head, members)?, + }; + let changed = matches!( + reason, + HeadUpdateReason::Initialized + | HeadUpdateReason::FastForwarded + | HeadUpdateReason::Selected + ); + Ok(HeadUpdate { + group: directory + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| integrity("snapshot group directory has no valid name".into()))? + .into(), + previous: previous.map(str::to_owned), + head: if changed { + candidate.into() + } else { + previous.unwrap_or(candidate).into() + }, + reason, + changed, + }) +} + +fn batch_head_update( + directory: &Path, + previous: Option<&str>, + candidates: &BTreeSet, + members: &BTreeMap, + explicit: bool, +) -> MicrosandboxResult> { + // Remove supplied heads that are proven ancestors of another supplied head. Shared paths + // need be traversed only once: their candidate ancestors were already marked on first visit. + let mut ancestors = HashSet::new(); + let mut visited = HashSet::new(); + for candidate in candidates { + let mut current = members[candidate].parent.as_deref(); + while let Some(parent) = current { + if !visited.insert(parent) { + break; + } + if candidates.contains(parent) { + ancestors.insert(parent); + } + current = members + .get(parent) + .and_then(|member| member.parent.as_deref()); + } + } + let mut tips = candidates + .iter() + .filter(|candidate| !ancestors.contains(candidate.as_str())); + let candidate = tips + .next() + .ok_or_else(|| integrity("snapshot batch has no candidate tip".into()))?; + if tips.next().is_none() { + return head_update(directory, previous, candidate, members, explicit).map(Some); + } + if explicit { + return Err(MicrosandboxError::InvalidConfig( + "--set-head cannot choose between multiple snapshot archive heads with incomparable or unknown ancestry; load without --set-head, then use 'msb snapshot head :'".into(), + )); + } + // A fresh group may contain several branches without claiming that one is current. + previous + .map(|head| { + let mut update = head_update(directory, Some(head), head, members, false)?; + update.reason = HeadUpdateReason::AmbiguousCandidates; + Ok(update) + }) + .transpose() +} + +fn ancestry_reason( + candidate: &str, + head: &str, + members: &BTreeMap, +) -> MicrosandboxResult { + let mut current = candidate; + let mut visited = HashSet::new(); + while visited.len() < MAX_ANCESTRY_DEPTH { + if !visited.insert(current) { + return Err(integrity("snapshot ancestry contains a cycle".into())); + } + let Some(member) = members.get(current) else { + return Ok(HeadUpdateReason::UnknownAncestry); + }; + let Some(parent) = member.parent.as_deref() else { + return Ok(HeadUpdateReason::Diverged); + }; + if parent == head { + return Ok(HeadUpdateReason::FastForwarded); + } + current = parent; + } + Err(integrity(format!( + "snapshot ancestry exceeds the {MAX_ANCESTRY_DEPTH}-member traversal limit" + ))) +} + +fn validate_ancestry(members: &BTreeMap) -> MicrosandboxResult<()> { + let mut complete = HashSet::new(); + for id in members.keys() { + let mut current = id.as_str(); + let mut visiting = HashSet::new(); + while !complete.contains(current) { + if !visiting.insert(current) { + return Err(integrity("snapshot ancestry contains a cycle".into())); + } + if visiting.len() > MAX_ANCESTRY_DEPTH { + return Err(integrity(format!( + "snapshot ancestry exceeds the {MAX_ANCESTRY_DEPTH}-member traversal limit" + ))); + } + let Some(parent) = members + .get(current) + .and_then(|member| member.parent.as_deref()) + else { + break; + }; + current = parent; + } + complete.extend(visiting); + } + Ok(()) +} + +fn write_member_name(directory: &Path, name: Option<&str>) -> MicrosandboxResult<()> { + let path = directory.join(MEMBER_FILENAME); + match name { + Some(name) => write_json( + directory, + MEMBER_FILENAME, + &MemberMetadata { + schema: MEMBER_SCHEMA.into(), + name: name.into(), + }, + ), + None => { + // Imported local metadata does not choose names in the receiving namespace. + if path_exists(&path)? { + if !fs::symlink_metadata(&path)?.file_type().is_file() { + return Err(integrity(format!( + "snapshot member metadata is not a regular file: {}", + path.display() + ))); + } + fs::remove_file(path)?; + sync_directory(directory)?; + } + Ok(()) + } + } +} + +fn write_json(directory: &Path, filename: &str, value: &impl Serialize) -> MicrosandboxResult<()> { + let bytes = serde_json::to_vec(value)?; + if bytes.len() > MAX_METADATA_BYTES { + return Err(integrity( + "snapshot group metadata exceeds its size limit".into(), + )); + } + let mut temporary = tempfile::Builder::new() + .prefix(".group-write-") + .tempfile_in(directory)?; + temporary.write_all(&bytes)?; + temporary.as_file().sync_all()?; + temporary + .persist(directory.join(filename)) + .map_err(|error| MicrosandboxError::from(error.error))?; + sync_directory(directory)?; + Ok(()) +} + +fn read_regular(path: &Path, maximum: usize) -> MicrosandboxResult> { + let metadata = fs::symlink_metadata(path)?; + if !metadata.file_type().is_file() || metadata.len() > maximum as u64 { + return Err(integrity(format!( + "snapshot metadata is not a bounded regular file: {}", + path.display() + ))); + } + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + options.custom_flags(libc::O_NOFOLLOW); + let file = options.open(path)?; + if !file.metadata()?.is_file() { + return Err(integrity(format!( + "snapshot metadata is not a regular file: {}", + path.display() + ))); + } + let mut bytes = Vec::new(); + file.take(maximum as u64 + 1).read_to_end(&mut bytes)?; + if bytes.len() > maximum { + return Err(integrity(format!( + "snapshot metadata exceeds its size limit: {}", + path.display() + ))); + } + Ok(bytes) +} + +fn require_directory(path: &Path) -> MicrosandboxResult<()> { + if !fs::symlink_metadata(path)?.file_type().is_dir() { + return Err(integrity(format!( + "snapshot group path is not a regular directory: {}", + path.display() + ))); + } + Ok(()) +} + +fn path_exists(path: &Path) -> MicrosandboxResult { + match fs::symlink_metadata(path) { + Ok(_) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error.into()), + } +} + +fn integrity(message: String) -> MicrosandboxError { + MicrosandboxError::SnapshotIntegrity(message) +} + +#[cfg(unix)] +fn sync_directory(path: &Path) -> std::io::Result<()> { + File::open(path)?.sync_all() +} + +#[cfg(windows)] +fn sync_directory(_path: &Path) -> std::io::Result<()> { + // Match artifact publication: payloads and metadata are flushed, directory rename is atomic. + Ok(()) +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +#[path = "group_tests.rs"] +mod tests; diff --git a/sdk/rust/lib/snapshot/group_tests.rs b/sdk/rust/lib/snapshot/group_tests.rs new file mode 100644 index 000000000..fb6680af8 --- /dev/null +++ b/sdk/rust/lib/snapshot/group_tests.rs @@ -0,0 +1,840 @@ +//! Group publication and selector tests using small, complete file-state artifacts. + +use microsandbox_image::snapshot::{ + DiskLayer, DiskLayerId, FileSnapshotState, ImageRef, LayerFileKind, LayerPayload, SCHEMA, + SnapshotCapture, SnapshotConsistency, SnapshotFormat, SnapshotRootDisk, SnapshotScope, + SnapshotState, +}; + +use super::*; + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +fn id(value: u128) -> SnapshotId { + SnapshotId::new(format!("snap_{value:032x}")).unwrap() +} + +fn descriptor(value: u128, parent: Option) -> Manifest { + let layer_id = DiskLayerId::new(format!("layer_{value:032x}")).unwrap(); + Manifest { + schema: SCHEMA.into(), + snapshot_id: id(value), + scope: SnapshotScope::Disk, + state: SnapshotState::File(FileSnapshotState { + disk_format: SnapshotFormat::Raw, + filesystem: "ext4".into(), + virtual_size: 4, + head: layer_id.clone(), + layers: vec![DiskLayer { + layer_id, + format: SnapshotFormat::Raw, + virtual_size: 4, + backing: None, + payload: LayerPayload { + file_kind: LayerFileKind::Regular, + integrity: None, + }, + }], + }), + capture: SnapshotCapture { + created_at: "2026-09-10T00:00:00Z".into(), + source_lineage: Some("source".into()), + source_checkpoint: None, + consistency: SnapshotConsistency::CrashConsistent, + }, + image: ImageRef { + reference: "docker.io/library/alpine:latest".into(), + manifest_digest: format!("sha256:{}", "a".repeat(64)), + }, + root_disk: SnapshotRootDisk::Managed, + parent: parent.map(id), + requires: Vec::new(), + extensions: BTreeMap::new(), + } +} + +fn stage(root: &Path, manifests: &[Manifest]) -> tempfile::TempDir { + let staging = tempfile::Builder::new() + .prefix(".stage-") + .tempdir_in(root) + .unwrap(); + for manifest in manifests { + let directory = staging.path().join(manifest.snapshot_id.as_str()); + fs::create_dir(&directory).unwrap(); + fs::write( + directory.join(DESCRIPTOR_FILENAME), + manifest.to_canonical_bytes().unwrap(), + ) + .unwrap(); + let SnapshotState::File(state) = &manifest.state else { + unreachable!(); + }; + let layer = directory.join(state.layer_path(&state.layers[0])); + fs::create_dir_all(layer.parent().unwrap()).unwrap(); + fs::write(layer, [0u8; 4]).unwrap(); + } + staging +} + +async fn add(group: &Path, value: u128, parent: Option) -> HeadUpdate { + let staging = stage(group.parent().unwrap(), &[descriptor(value, parent)]); + publish(group, staging.path(), &BTreeMap::new(), &id(value), false) + .await + .unwrap() +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[tokio::test] +async fn batch_head_is_independent_of_archive_and_staging_order() { + let root = tempfile::tempdir().unwrap(); + for (index, order) in [ + [1, 2, 3], + [1, 3, 2], + [2, 1, 3], + [2, 3, 1], + [3, 1, 2], + [3, 2, 1], + ] + .into_iter() + .enumerate() + { + let group = ensure(root.path(), Some(&format!("order-{index}"))) + .await + .unwrap(); + let manifests = order + .iter() + .map(|value| descriptor(*value, (*value > 1).then_some(*value - 1))) + .collect::>(); + let staged = stage(root.path(), &manifests); + let candidates = order.into_iter().map(id).collect::>(); + let update = publish_batch(&group, staged.path(), &BTreeMap::new(), &candidates, false) + .await + .unwrap() + .unwrap(); + assert_eq!(update.head, id(3).as_str()); + assert_eq!(update.reason, HeadUpdateReason::Initialized); + assert_eq!(read_members(&group, true).unwrap().len(), 3); + } +} + +#[tokio::test] +async fn batch_uses_known_destination_intermediates_to_prove_one_tip() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("intermediate")).await.unwrap(); + add(&group, 1, None).await; + add(&group, 2, Some(1)).await; + let staged = stage(root.path(), &[descriptor(3, Some(2))]); + let update = publish_batch( + &group, + staged.path(), + &BTreeMap::new(), + &[id(3), id(1), id(3)], + false, + ) + .await + .unwrap() + .unwrap(); + assert_eq!(update.reason, HeadUpdateReason::FastForwarded); + assert_eq!(update.previous.as_deref(), Some(id(2).as_str())); + assert_eq!(update.head, id(3).as_str()); +} + +#[tokio::test] +async fn batch_unique_tip_still_respects_existing_head_ancestry() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("retained-head")).await.unwrap(); + add(&group, 1, None).await; + for (parent, reason) in [ + (None, HeadUpdateReason::Diverged), + (Some(9), HeadUpdateReason::UnknownAncestry), + ] { + let first = if parent.is_none() { 2 } else { 4 }; + let staged = stage( + root.path(), + &[ + descriptor(first, parent), + descriptor(first + 1, Some(first)), + ], + ); + let update = publish_batch( + &group, + staged.path(), + &BTreeMap::new(), + &[id(first), id(first + 1)], + false, + ) + .await + .unwrap() + .unwrap(); + assert_eq!(update.reason, reason); + assert!(!update.changed); + assert_eq!(update.head, id(1).as_str()); + assert!(group.join(id(first + 1).as_str()).is_dir()); + } +} + +#[tokio::test] +async fn batch_branches_preserve_existing_head_or_leave_new_group_unselected() { + let root = tempfile::tempdir().unwrap(); + for existing in [false, true] { + for order in [[2, 3], [3, 2]] { + let name = format!("branches-{existing}-{}", order[0]); + let group = ensure(root.path(), Some(&name)).await.unwrap(); + if existing { + add(&group, 1, None).await; + } + let staged = stage( + root.path(), + &[ + descriptor(1, None), + descriptor(2, Some(1)), + descriptor(3, Some(1)), + ], + ); + let candidates = order.map(id); + let update = publish_batch(&group, staged.path(), &BTreeMap::new(), &candidates, false) + .await + .unwrap(); + if existing { + let update = update.unwrap(); + assert_eq!(update.reason, HeadUpdateReason::AmbiguousCandidates); + assert!(!update.changed); + assert_eq!(update.head, id(1).as_str()); + } else { + assert_eq!(update, None); + assert_eq!(read_group(&group).unwrap().head, None); + let error = resolve(root.path(), &name).await.unwrap_err().to_string(); + assert!(error.contains("no head selected")); + assert!(error.contains("msb snapshot head")); + assert_eq!( + resolve(root.path(), &format!("{name}:{}", id(3))) + .await + .unwrap(), + group.join(id(3).as_str()) + ); + } + assert_eq!(read_members(&group, true).unwrap().len(), 3); + } + } +} + +#[tokio::test] +async fn batch_unknown_history_does_not_guess_a_candidate_order() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("holes")).await.unwrap(); + add(&group, 1, None).await; + // Snapshot 3 may descend from 1, but absent snapshot 2 prevents proving the relationship. + let staged = stage(root.path(), &[descriptor(3, Some(2))]); + let update = publish_batch( + &group, + staged.path(), + &BTreeMap::new(), + &[id(3), id(1)], + false, + ) + .await + .unwrap() + .unwrap(); + assert_eq!(update.reason, HeadUpdateReason::AmbiguousCandidates); + assert_eq!(update.head, id(1).as_str()); + let staged = stage(root.path(), &[descriptor(2, Some(1))]); + // Repeating the same candidates is now conclusive, even though their intermediate is not + // itself a supplied archive head. + let update = publish_batch( + &group, + staged.path(), + &BTreeMap::new(), + &[id(1), id(3)], + false, + ) + .await + .unwrap() + .unwrap(); + assert_eq!(update.reason, HeadUpdateReason::FastForwarded); + assert_eq!(update.head, id(3).as_str()); +} + +#[tokio::test] +async fn batch_set_head_requires_one_candidate_tip_before_any_publication() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("explicit-batch")).await.unwrap(); + add(&group, 1, None).await; + let staged = stage( + root.path(), + &[descriptor(2, Some(1)), descriptor(3, Some(1))], + ); + let error = publish_batch( + &group, + staged.path(), + &BTreeMap::new(), + &[id(2), id(3)], + true, + ) + .await + .unwrap_err(); + assert!(error.to_string().contains("--set-head cannot choose")); + assert!(error.to_string().contains("msb snapshot head")); + for candidate in [2, 3] { + assert!(!group.join(id(candidate).as_str()).exists()); + assert!(staged.path().join(id(candidate).as_str()).is_dir()); + } + assert_eq!( + read_group(&group).unwrap().head.as_deref(), + Some(id(1).as_str()) + ); + let staged = stage(root.path(), &[descriptor(4, None), descriptor(5, Some(4))]); + let update = publish_batch( + &group, + staged.path(), + &BTreeMap::new(), + &[id(5), id(4)], + true, + ) + .await + .unwrap() + .unwrap(); + assert_eq!(update.reason, HeadUpdateReason::Selected); + assert_eq!(update.head, id(5).as_str()); +} + +#[tokio::test] +async fn batch_descriptor_and_alias_conflicts_do_not_partly_publish() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("batch-conflicts")).await.unwrap(); + let staged = stage(root.path(), &[descriptor(1, None)]); + publish( + &group, + staged.path(), + &BTreeMap::from([(id(1).to_string(), "base".into())]), + &id(1), + false, + ) + .await + .unwrap(); + let mut conflict = descriptor(1, None); + conflict.capture.source_lineage = Some("different-source".into()); + let staged = stage(root.path(), &[descriptor(2, Some(1)), conflict]); + assert!( + publish_batch( + &group, + staged.path(), + &BTreeMap::new(), + &[id(2), id(1)], + false + ) + .await + .unwrap_err() + .to_string() + .contains("different descriptor") + ); + assert!(!group.join(id(2).as_str()).exists()); + assert!(staged.path().join(id(2).as_str()).is_dir()); + let staged = stage( + root.path(), + &[descriptor(2, Some(1)), descriptor(3, Some(2))], + ); + let aliases = BTreeMap::from([ + (id(2).to_string(), "other".into()), + (id(3).to_string(), "base".into()), + ]); + assert!( + publish_batch(&group, staged.path(), &aliases, &[id(2), id(3)], false) + .await + .unwrap_err() + .to_string() + .contains("conflicts") + ); + for candidate in [2, 3] { + assert!(!group.join(id(candidate).as_str()).exists()); + assert!(staged.path().join(id(candidate).as_str()).is_dir()); + } + assert_eq!( + read_group(&group).unwrap().head.as_deref(), + Some(id(1).as_str()) + ); +} + +#[tokio::test] +async fn dependency_lookup_reads_only_existing_installed_group_members() { + let root = tempfile::tempdir().unwrap(); + let missing_root = root.path().join("missing-root"); + assert!( + dependency_members(&missing_root, "work") + .await + .unwrap() + .is_empty() + ); + assert!(!missing_root.exists()); + assert!( + dependency_members(root.path(), "work") + .await + .unwrap() + .is_empty() + ); + assert!(!root.path().join("work").exists()); + assert!(!root.path().join(".groups.lock").exists()); + assert!( + dependency_members(&missing_root, "../escape") + .await + .is_err() + ); + let group = ensure(root.path(), Some("work")).await.unwrap(); + add(&group, 1, None).await; + let _incomplete = stage(&group, &[descriptor(2, Some(1))]); + assert_eq!( + dependency_members(root.path(), "work").await.unwrap(), + vec![group.join(id(1).as_str())] + ); + let malformed = root.path().join("malformed"); + fs::create_dir(&malformed).unwrap(); + write_group(&malformed, None).unwrap(); + assert!(dependency_members(root.path(), "malformed").await.is_err()); + assert!(!malformed.join(".group.lock").exists()); +} + +#[cfg(unix)] +#[tokio::test] +async fn dependency_lookup_rejects_symlinked_roots_and_groups() { + use std::os::unix::fs::symlink; + + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("work")).await.unwrap(); + symlink(&group, root.path().join("redirect")).unwrap(); + assert!(dependency_members(root.path(), "redirect").await.is_err()); + symlink(root.path(), root.path().join("root-link")).unwrap(); + assert!( + dependency_members(&root.path().join("root-link"), "work") + .await + .is_err() + ); +} + +#[tokio::test] +async fn initializes_and_fast_forwards_through_multiple_imported_ancestors() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("work")).await.unwrap(); + let first = add(&group, 10, None).await; + assert_eq!(first.reason, HeadUpdateReason::Initialized); + assert_eq!(first.previous, None); + + // Directory order puts the tip first. The designated candidate determines the head. + let staging = stage( + root.path(), + &[descriptor(5, Some(20)), descriptor(20, Some(10))], + ); + let update = publish(&group, staging.path(), &BTreeMap::new(), &id(5), false) + .await + .unwrap(); + assert_eq!(update.reason, HeadUpdateReason::FastForwarded); + assert_eq!(update.previous.as_deref(), Some(id(10).as_str())); + assert_eq!( + resolve(root.path(), "work").await.unwrap(), + group.join(id(5).as_str()) + ); + assert!(group.join(id(20).as_str()).is_dir()); +} + +#[tokio::test] +async fn resolved_identity_stays_fixed_after_the_group_head_advances() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("stable")).await.unwrap(); + add(&group, 1, None).await; + let selected = resolve(root.path(), "stable").await.unwrap(); + add(&group, 2, Some(1)).await; + assert_eq!(selected, group.join(id(1).as_str())); + assert_eq!(read_member(&selected, true).unwrap().0, id(1).as_str()); + assert_eq!( + resolve(root.path(), "stable").await.unwrap(), + group.join(id(2).as_str()) + ); +} + +#[tokio::test] +async fn head_and_id_lookup_do_not_scan_unrelated_descriptors() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("lookup")).await.unwrap(); + add(&group, 1, None).await; + let unrelated = group.join(id(2).as_str()); + fs::create_dir(&unrelated).unwrap(); + fs::write( + unrelated.join(DESCRIPTOR_FILENAME), + "broken unrelated descriptor", + ) + .unwrap(); + assert_eq!( + resolve(root.path(), "lookup").await.unwrap(), + group.join(id(1).as_str()) + ); + assert_eq!( + resolve(root.path(), &format!("lookup:{}", id(1))) + .await + .unwrap(), + group.join(id(1).as_str()), + ); + assert_eq!( + select(root.path(), "lookup").await.unwrap().head, + id(1).as_str() + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn concurrent_siblings_keep_both_artifacts_and_only_one_advances() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("race")).await.unwrap(); + add(&group, 1, None).await; + let left = stage(root.path(), &[descriptor(2, Some(1))]); + let right = stage(root.path(), &[descriptor(3, Some(1))]); + let aliases = BTreeMap::new(); + let left_id = id(2); + let right_id = id(3); + let (left_result, right_result) = tokio::join!( + publish(&group, left.path(), &aliases, &left_id, false), + publish(&group, right.path(), &aliases, &right_id, false), + ); + let left_result = left_result.unwrap(); + let right_result = right_result.unwrap(); + assert_ne!(left_result.changed, right_result.changed); + let (winner, retained) = if left_result.changed { + (left_result, right_result) + } else { + (right_result, left_result) + }; + assert_eq!(winner.reason, HeadUpdateReason::FastForwarded); + assert_eq!(retained.reason, HeadUpdateReason::Diverged); + assert_eq!(retained.head, winner.head); + assert_eq!(select(root.path(), "race").await.unwrap().head, winner.head); + assert!(group.join(id(2).as_str()).is_dir()); + assert!(group.join(id(3).as_str()).is_dir()); +} + +#[tokio::test] +async fn unknown_history_is_retained_without_retroactively_selecting_a_tip() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("history")).await.unwrap(); + add(&group, 1, None).await; + let unknown = add(&group, 3, Some(2)).await; + assert_eq!(unknown.reason, HeadUpdateReason::UnknownAncestry); + assert_eq!(unknown.head, id(1).as_str()); + assert!(group.join(id(3).as_str()).is_dir()); + let intermediate = add(&group, 2, Some(1)).await; + assert_eq!(intermediate.head, id(2).as_str()); + assert_eq!( + select(root.path(), "history").await.unwrap().head, + id(2).as_str() + ); + + let staging = stage(root.path(), &[]); + let retried = publish(&group, staging.path(), &BTreeMap::new(), &id(3), false) + .await + .unwrap(); + assert_eq!(retried.reason, HeadUpdateReason::FastForwarded); + assert_eq!(retried.head, id(3).as_str()); +} + +#[tokio::test] +async fn identical_ids_reuse_members_and_conflicts_fail_before_publication() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("duplicates")).await.unwrap(); + add(&group, 1, None).await; + fs::write(group.join(id(1).as_str()).join("keep"), "unchanged").unwrap(); + let duplicate = add(&group, 1, None).await; + assert!(!duplicate.changed); + assert_eq!(duplicate.reason, HeadUpdateReason::Unchanged); + assert_eq!( + fs::read_to_string(group.join(id(1).as_str()).join("keep")).unwrap(), + "unchanged" + ); + + let mut conflicting = descriptor(1, None); + conflicting.capture.source_lineage = Some("another-source".into()); + let staging = stage(root.path(), &[descriptor(2, Some(1)), conflicting]); + let error = publish(&group, staging.path(), &BTreeMap::new(), &id(2), false) + .await + .unwrap_err(); + assert!(error.to_string().contains("different descriptor")); + assert!(!group.join(id(2).as_str()).exists()); + assert!(staging.path().join(id(2).as_str()).is_dir()); + assert_eq!( + select(root.path(), "duplicates").await.unwrap().head, + id(1).as_str() + ); +} + +#[tokio::test] +async fn aliases_are_local_and_all_conflicts_are_checked_before_moving_members() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("aliases")).await.unwrap(); + let staging = stage(root.path(), &[descriptor(1, None)]); + let aliases = BTreeMap::from([(id(1).to_string(), "clean".into())]); + publish(&group, staging.path(), &aliases, &id(1), false) + .await + .unwrap(); + assert_eq!( + resolve(root.path(), "aliases:clean").await.unwrap(), + group.join(id(1).as_str()) + ); + assert_eq!( + member_name(&group.join(id(1).as_str())).unwrap().as_deref(), + Some("clean") + ); + assert_eq!(group_path(&group.join(id(1).as_str())), Some(group.clone())); + + let staging = stage( + root.path(), + &[descriptor(2, Some(1)), descriptor(3, Some(1))], + ); + let aliases = BTreeMap::from([ + (id(2).to_string(), "other".into()), + (id(3).to_string(), "clean".into()), + ]); + let error = publish(&group, staging.path(), &aliases, &id(2), false) + .await + .unwrap_err(); + assert!(error.to_string().contains("conflicts")); + assert!(!group.join(id(2).as_str()).exists()); + assert!(!group.join(id(3).as_str()).exists()); + assert!(staging.path().join(id(2).as_str()).is_dir()); +} + +#[tokio::test] +async fn generated_names_retry_publication_without_recapturing_the_artifact() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("generated")).await.unwrap(); + let first = stage(root.path(), &[descriptor(1, None)]); + let aliases = BTreeMap::from([(id(1).to_string(), "msb-00000001".into())]); + publish(&group, first.path(), &aliases, &id(1), false) + .await + .unwrap(); + + let captured = stage(root.path(), &[descriptor(2, Some(1))]); + let staged_member = captured.path().join(id(2).as_str()); + let original_descriptor = fs::read(staged_member.join(DESCRIPTOR_FILENAME)).unwrap(); + fs::write(staged_member.join("capture-marker"), b"same capture").unwrap(); + let mut retries = 0; + let update = super::super::create::publish_with_name_retry( + &group, + captured.path(), + &id(2), + "msb-00000001".into(), + true, + || { + retries += 1; + // The conflict was detected before moving or rewriting any captured state. + assert_eq!( + fs::read(staged_member.join(DESCRIPTOR_FILENAME)).unwrap(), + original_descriptor + ); + assert_eq!( + fs::read(staged_member.join("capture-marker")).unwrap(), + b"same capture" + ); + "msb-00000002".into() + }, + ) + .await + .unwrap(); + assert_eq!(retries, 1); + assert_eq!(update.reason, HeadUpdateReason::FastForwarded); + let installed = group.join(id(2).as_str()); + assert_eq!( + member_name(&installed).unwrap().as_deref(), + Some("msb-00000002") + ); + assert_eq!( + fs::read(installed.join(DESCRIPTOR_FILENAME)).unwrap(), + original_descriptor + ); + assert_eq!( + fs::read(installed.join("capture-marker")).unwrap(), + b"same capture" + ); + assert_eq!( + member_name(&group.join(id(1).as_str())).unwrap().as_deref(), + Some("msb-00000001") + ); +} + +#[tokio::test] +async fn explicit_names_report_collision_without_retry_or_staging_changes() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("explicit")).await.unwrap(); + let first = stage(root.path(), &[descriptor(1, None)]); + let aliases = BTreeMap::from([(id(1).to_string(), "chosen".into())]); + publish(&group, first.path(), &aliases, &id(1), false) + .await + .unwrap(); + let captured = stage(root.path(), &[descriptor(2, Some(1))]); + let error = super::super::create::publish_with_name_retry( + &group, + captured.path(), + &id(2), + "chosen".into(), + false, + || panic!("explicit names must not be regenerated"), + ) + .await + .unwrap_err(); + assert!(matches!(error, MicrosandboxError::SnapshotAlreadyExists(_))); + assert!( + captured + .path() + .join(id(2).as_str()) + .join(DESCRIPTOR_FILENAME) + .is_file() + ); + assert!(!group.join(id(2).as_str()).exists()); + assert_eq!( + read_group(&group).unwrap().head.as_deref(), + Some(id(1).as_str()) + ); +} + +#[tokio::test] +async fn explicit_selection_can_choose_a_retained_branch_or_an_older_snapshot() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("selection")).await.unwrap(); + add(&group, 1, None).await; + add(&group, 2, Some(1)).await; + assert_eq!( + add(&group, 3, Some(1)).await.reason, + HeadUpdateReason::Diverged + ); + let side = select(root.path(), &format!("selection:{}", id(3))) + .await + .unwrap(); + assert_eq!(side.reason, HeadUpdateReason::Selected); + assert_eq!(side.previous.as_deref(), Some(id(2).as_str())); + assert_eq!(side.head, id(3).as_str()); + let old = select(root.path(), &format!("selection:{}", id(1))) + .await + .unwrap(); + assert_eq!(old.head, id(1).as_str()); + assert_eq!( + select(root.path(), "selection").await.unwrap().reason, + HeadUpdateReason::Unchanged + ); +} + +#[tokio::test] +async fn removing_a_head_requires_selection_unless_it_is_the_final_member() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("removal")).await.unwrap(); + add(&group, 1, None).await; + add(&group, 2, Some(1)).await; + let error = remove_member(&group.join(id(2).as_str())) + .await + .unwrap_err(); + assert!(error.to_string().contains("first select another")); + assert!(group.join(id(2).as_str()).is_dir()); + assert!(remove_member(&group.join(id(1).as_str())).await.unwrap()); + assert!(remove_member(&group.join(id(2).as_str())).await.unwrap()); + assert_eq!(read_group(&group).unwrap().head, None); + assert!( + resolve(root.path(), "removal") + .await + .unwrap_err() + .to_string() + .contains("has no head") + ); + assert_eq!( + add(&group, 3, None).await.reason, + HeadUpdateReason::Initialized + ); +} + +#[tokio::test] +async fn cycles_are_rejected_even_when_a_group_has_no_head() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("cycle")).await.unwrap(); + let staging = stage( + root.path(), + &[descriptor(1, Some(2)), descriptor(2, Some(1))], + ); + let error = publish(&group, staging.path(), &BTreeMap::new(), &id(1), false) + .await + .unwrap_err(); + assert!(error.to_string().contains("cycle")); + assert_eq!(read_group(&group).unwrap().head, None); + assert!(!group.join(id(1).as_str()).exists()); +} + +#[tokio::test] +async fn generated_groups_are_fresh_and_flat_directories_are_never_migrated() { + let root = tempfile::tempdir().unwrap(); + let (first, second) = tokio::join!(ensure(root.path(), None), ensure(root.path(), None)); + let first = first.unwrap(); + let second = second.unwrap(); + assert_ne!(first, second); + assert!( + first + .file_name() + .unwrap() + .to_str() + .unwrap() + .starts_with("msb-") + ); + let flat = root.path().join("flat"); + fs::create_dir(&flat).unwrap(); + fs::write(flat.join("keep"), "untouched").unwrap(); + let error = ensure(root.path(), Some("flat")).await.unwrap_err(); + assert!(error.to_string().contains("explicit path")); + assert!(!flat.join(GROUP_FILENAME).exists()); + assert_eq!(fs::read_to_string(flat.join("keep")).unwrap(), "untouched"); +} + +#[tokio::test] +async fn selectors_and_group_names_cannot_escape_the_store() { + let root = tempfile::tempdir().unwrap(); + for name in [ + "", + ".", + "..", + "../escape", + "a:b", + "a\\b", + "con", + "trailing.", + ] { + assert!( + ensure(root.path(), Some(name)).await.is_err(), + "accepted {name}" + ); + } + for selector in ["valid:", "valid:../escape", "valid:a:b", "../escape:name"] { + assert!( + resolve(root.path(), selector).await.is_err(), + "accepted {selector}" + ); + } +} + +#[cfg(unix)] +#[tokio::test] +async fn symlinked_group_and_descriptor_paths_are_rejected() { + use std::os::unix::fs::symlink; + + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("real")).await.unwrap(); + symlink(&group, root.path().join("redirect")).unwrap(); + assert!(ensure(root.path(), Some("redirect")).await.is_err()); + let staging = stage(root.path(), &[descriptor(1, None)]); + let path = staging + .path() + .join(id(1).as_str()) + .join(DESCRIPTOR_FILENAME); + let external = root.path().join("external.json"); + fs::rename(&path, &external).unwrap(); + symlink(&external, &path).unwrap(); + assert!( + publish(&group, staging.path(), &BTreeMap::new(), &id(1), false) + .await + .is_err() + ); + assert!(!group.join(id(1).as_str()).exists()); + assert!(external.is_file()); +} diff --git a/sdk/rust/lib/snapshot/lineage.rs b/sdk/rust/lib/snapshot/lineage.rs new file mode 100644 index 000000000..d95ba6417 --- /dev/null +++ b/sdk/rust/lib/snapshot/lineage.rs @@ -0,0 +1,415 @@ +//! Capture ancestry independent of group names, dirty tracking, and export dependencies. + +use std::fs::File; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use sea_orm::{ColumnTrait, EntityTrait, QueryFilter}; +use serde::{Deserialize, Serialize}; + +use crate::backend::LocalBackend; +use crate::db::entity::sandbox; +use crate::sandbox::SandboxConfig; +use crate::{MicrosandboxError, MicrosandboxResult}; + +use super::SnapshotId; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Cursor { + sandbox_id: i32, + snapshot_id: String, +} + +/// Holds the per-source capture sequencer without holding a VM pause or database transaction. +pub(crate) struct CaptureLineage { + _lock: Arc, + path: PathBuf, + sandbox_id: i32, + pub(crate) parent: Option, +} + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl CaptureLineage { + /// Source incarnation whose ancestry is protected by this sequencer. + pub(crate) fn sandbox_id(&self) -> i32 { + self.sandbox_id + } + + /// Refuse a completed capture if its named source was removed or replaced meanwhile. + pub(crate) async fn validate_source( + &self, + local: &LocalBackend, + name: &str, + ) -> MicrosandboxResult<()> { + let current = sandbox::Entity::find() + .filter(sandbox::Column::Name.eq(name)) + .one(local.db().await?.read()) + .await?; + if current + .as_ref() + .is_none_or(|model| model.id != self.sandbox_id) + { + return Err(MicrosandboxError::InvalidConfig( + "source sandbox changed during snapshot capture".into(), + )); + } + Ok(()) + } + + /// Advance only after the artifact/archive has been successfully published. + pub(crate) async fn commit(&self, snapshot_id: &SnapshotId) -> MicrosandboxResult<()> { + let path = self.path.clone(); + // A cancelled awaiting task must not release the source sequencer while its blocking + // publication still runs, otherwise an older cursor could replace a newer capture. + let lock = Arc::clone(&self._lock); + let cursor = Cursor { + sandbox_id: self.sandbox_id, + snapshot_id: snapshot_id.to_string(), + }; + tokio::task::spawn_blocking(move || -> MicrosandboxResult<()> { + let _lock = lock; + let parent = path.parent().expect("cursor has a sandbox directory"); + let mut staged = tempfile::NamedTempFile::new_in(parent)?; + staged.write_all(&serde_json::to_vec(&cursor)?)?; + staged.as_file().sync_all()?; + staged + .persist(&path) + .map_err(|error| MicrosandboxError::Io(error.error))?; + #[cfg(unix)] + File::open(parent)?.sync_all()?; + Ok(()) + }) + .await + .map_err(|error| { + MicrosandboxError::Runtime(format!("snapshot ancestry publication: {error}")) + })? + } +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +/// Serialize capture publication with removal/replacement of the same named source. The lock +/// lives outside its removable directory and must never be unlinked. Callers that also own a +/// transition or lifecycle guard acquire transition, then lineage, then lifecycle ownership. +pub(crate) async fn lock_source(run_dir: &Path, name: &str) -> MicrosandboxResult { + let path = microsandbox_runtime::ipc::snapshot_lineage_lock_path(run_dir, name); + tokio::fs::create_dir_all(path.parent().expect("lineage lock has a parent")).await?; + let lock = microsandbox_utils::process_lock::open_lock_file(&path)?; + // Waiting asynchronously keeps cancellation bounded and avoids occupying a blocking-pool + // thread for each capture queued behind a large archive publication. + loop { + if microsandbox_utils::process_lock::try_lock_exclusive(&lock)? { + return Ok(lock); + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } +} + +pub(crate) async fn begin(local: &LocalBackend, name: &str) -> MicrosandboxResult { + let model = sandbox::Entity::find() + .filter(sandbox::Column::Name.eq(name)) + .one(local.db().await?.read()) + .await? + .ok_or_else(|| MicrosandboxError::SandboxNotFound(name.into()))?; + let expected_id = model.id; + let lock = lock_source(&local.config().run_dir(), name).await?; + let directory = local.sandboxes_dir().join(name); + let (lock, cursor) = tokio::task::spawn_blocking(move || -> MicrosandboxResult<_> { + // Removal/replacement owns the same stable lock, so this path remains bound to the + // checked source until publication commits. Never recreate a missing source directory. + if !std::fs::symlink_metadata(&directory)?.is_dir() { + return Err(MicrosandboxError::SnapshotIntegrity( + "snapshot source directory is not a directory".into(), + )); + } + let path = directory.join("snapshot-lineage.json"); + let cursor = match std::fs::symlink_metadata(&path) { + Ok(meta) if meta.is_file() && meta.len() <= 4096 => { + Some(serde_json::from_slice::(&std::fs::read(&path)?)?) + } + Ok(_) => { + return Err(MicrosandboxError::SnapshotIntegrity( + "invalid snapshot ancestry cursor".into(), + )); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(error.into()), + }; + Ok((lock, (path, cursor))) + }) + .await + .map_err(|error| MicrosandboxError::Runtime(format!("snapshot ancestry lock: {error}")))??; + let current = sandbox::Entity::find() + .filter(sandbox::Column::Name.eq(name)) + .one(local.db().await?.read()) + .await? + .ok_or_else(|| MicrosandboxError::SandboxNotFound(name.into()))?; + if current.id != expected_id { + return Err(MicrosandboxError::InvalidConfig( + "source sandbox changed while waiting for capture".into(), + )); + } + let config: SandboxConfig = + serde_json::from_str(current.active_config.as_deref().unwrap_or(¤t.config))?; + let parent = match cursor.1 { + Some(cursor) if cursor.sandbox_id == current.id => Some(cursor.snapshot_id), + Some(_) => { + return Err(MicrosandboxError::SnapshotIntegrity( + "snapshot ancestry belongs to another sandbox instance".into(), + )); + } + None => config.snapshot_parent, + } + .map(SnapshotId::new) + .transpose() + .map_err(|error| MicrosandboxError::SnapshotIntegrity(error.to_string()))?; + Ok(CaptureLineage { + _lock: Arc::new(lock), + path: cursor.0, + sandbox_id: current.id, + parent, + }) +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use sea_orm::{ActiveModelTrait, ActiveValue::Set}; + + use super::*; + + async fn source(local: &LocalBackend, name: &str, origin: Option<&SnapshotId>) -> i32 { + let mut config = SandboxConfig::default(); + config.spec.name = name.into(); + config.snapshot_parent = origin.map(ToString::to_string); + std::fs::create_dir_all(local.sandboxes_dir().join(name)).unwrap(); + sandbox::ActiveModel { + name: Set(name.into()), + config: Set(serde_json::to_string(&config).unwrap()), + status: Set(sandbox::SandboxStatus::Stopped), + ephemeral: Set(false), + ..Default::default() + } + .insert(local.db().await.unwrap().write()) + .await + .unwrap() + .id + } + + fn id(value: u128) -> SnapshotId { + SnapshotId::new(format!("snap_{value:032x}")).unwrap() + } + + #[test] + fn cancelled_cursor_wait_retains_lock_until_blocking_publication_finishes() { + // Hold the sole blocking worker so cancellation deterministically lands after commit + // queues publication but before that publication can touch the cursor. + let runtime = tokio::runtime::Builder::new_current_thread() + .max_blocking_threads(1) + .enable_all() + .build() + .unwrap(); + runtime.block_on(async { + let directory = tempfile::tempdir().unwrap(); + let lock_path = directory.path().join(".snapshot-lineage.lock"); + let lock = microsandbox_utils::process_lock::open_lock_file(&lock_path).unwrap(); + microsandbox_utils::process_lock::lock_exclusive(&lock).unwrap(); + let lineage = CaptureLineage { + _lock: Arc::new(lock), + path: directory.path().join("snapshot-lineage.json"), + sandbox_id: 1, + parent: None, + }; + let (release, wait_release) = std::sync::mpsc::channel(); + let (started, wait_started) = tokio::sync::oneshot::channel(); + let blocker = tokio::task::spawn_blocking(move || { + started.send(()).unwrap(); + wait_release.recv().unwrap(); + }); + wait_started.await.unwrap(); + let snapshot = id(4); + let mut publication = Box::pin(lineage.commit(&snapshot)); + assert!(futures::poll!(&mut publication).is_pending()); + drop(publication); + drop(lineage); + let observer = + microsandbox_utils::process_lock::open_existing_lock_file(&lock_path).unwrap(); + let retained = + !microsandbox_utils::process_lock::try_lock_exclusive(&observer).unwrap(); + // Release before asserting so a failed test cannot strand its runtime worker. + release.send(()).unwrap(); + blocker.await.unwrap(); + assert!( + retained, + "cancelled await released an in-flight cursor publication lock" + ); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + while !microsandbox_utils::process_lock::try_lock_exclusive(&observer).unwrap() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let cursor: Cursor = serde_json::from_slice( + &std::fs::read(directory.path().join("snapshot-lineage.json")).unwrap(), + ) + .unwrap(); + assert_eq!(cursor.snapshot_id, snapshot.as_str()); + }); + } + + #[tokio::test] + async fn restored_origin_advances_only_after_successful_publication() { + let home = tempfile::tempdir().unwrap(); + let local = LocalBackend::builder() + .home(home.path()) + .build() + .await + .unwrap(); + let origin = id(1); + let captured = id(2); + let source_id = source(&local, "worker", Some(&origin)).await; + let failed = begin(&local, "worker").await.unwrap(); + assert_eq!(failed.sandbox_id(), source_id); + assert_eq!(failed.parent.as_ref(), Some(&origin)); + drop(failed); + let successful = begin(&local, "worker").await.unwrap(); + assert_eq!(successful.parent.as_ref(), Some(&origin)); + successful.commit(&captured).await.unwrap(); + drop(successful); + assert_eq!( + begin(&local, "worker").await.unwrap().parent, + Some(captured) + ); + } + + #[tokio::test] + async fn replacement_waits_for_cursor_publication_and_gets_independent_ancestry() { + let home = tempfile::tempdir().unwrap(); + let local = LocalBackend::builder() + .home(home.path()) + .build() + .await + .unwrap(); + let original = source(&local, "worker", Some(&id(1))).await; + let capture = begin(&local, "worker").await.unwrap(); + let run_dir = local.config().run_dir(); + let mut replacement = Box::pin(lock_source(&run_dir, "worker")); + assert!(futures::poll!(&mut replacement).is_pending()); + + // Publication may be delayed arbitrarily after the source check; removal still cannot + // change the directory receiving this cursor while the capture owns its lineage pin. + capture.commit(&id(2)).await.unwrap(); + assert!(futures::poll!(&mut replacement).is_pending()); + drop(capture); + let replacement = replacement.await.unwrap(); + std::fs::remove_dir_all(local.sandboxes_dir().join("worker")).unwrap(); + sandbox::Entity::delete_by_id(original) + .exec(local.db().await.unwrap().write()) + .await + .unwrap(); + source(&local, "worker", Some(&id(3))).await; + drop(replacement); + + let next = begin(&local, "worker").await.unwrap(); + assert_eq!(next.parent, Some(id(3))); + assert!( + !local + .sandboxes_dir() + .join("worker/snapshot-lineage.json") + .exists() + ); + } + + #[tokio::test] + async fn same_named_sources_in_different_backends_keep_separate_ancestry() { + let first_home = tempfile::tempdir().unwrap(); + let second_home = tempfile::tempdir().unwrap(); + let first = LocalBackend::builder() + .home(first_home.path()) + .build() + .await + .unwrap(); + let second = LocalBackend::builder() + .home(second_home.path()) + .build() + .await + .unwrap(); + source(&first, "worker", Some(&id(1))).await; + source(&second, "worker", Some(&id(2))).await; + let capture = begin(&first, "worker").await.unwrap(); + capture.commit(&id(3)).await.unwrap(); + drop(capture); + assert_eq!(begin(&first, "worker").await.unwrap().parent, Some(id(3))); + assert_eq!(begin(&second, "worker").await.unwrap().parent, Some(id(2))); + } + + #[tokio::test] + async fn cursor_from_a_different_source_incarnation_is_rejected() { + let home = tempfile::tempdir().unwrap(); + let local = LocalBackend::builder() + .home(home.path()) + .build() + .await + .unwrap(); + let source_id = source(&local, "worker", None).await; + std::fs::write( + local.sandboxes_dir().join("worker/snapshot-lineage.json"), + serde_json::to_vec(&Cursor { + sandbox_id: source_id + 1, + snapshot_id: id(1).to_string(), + }) + .unwrap(), + ) + .unwrap(); + let error = begin(&local, "worker") + .await + .err() + .expect("wrong incarnation must fail"); + assert!(error.to_string().contains("another sandbox instance")); + } + + #[tokio::test] + async fn completed_capture_accepts_status_changes_but_rejects_replacement() { + let home = tempfile::tempdir().unwrap(); + let local = LocalBackend::builder() + .home(home.path()) + .build() + .await + .unwrap(); + let original = source(&local, "worker", None).await; + let lineage = begin(&local, "worker").await.unwrap(); + sandbox::Entity::update_many() + .col_expr( + sandbox::Column::Status, + sea_orm::sea_query::Expr::value("Crashed"), + ) + .filter(sandbox::Column::Id.eq(original)) + .exec(local.db().await.unwrap().write()) + .await + .unwrap(); + lineage.validate_source(&local, "worker").await.unwrap(); + sandbox::Entity::delete_by_id(original) + .exec(local.db().await.unwrap().write()) + .await + .unwrap(); + assert!(lineage.validate_source(&local, "worker").await.is_err()); + let replacement = source(&local, "worker", None).await; + assert_ne!(original, replacement); + assert!(lineage.validate_source(&local, "worker").await.is_err()); + } +} diff --git a/sdk/rust/lib/snapshot/migration.rs b/sdk/rust/lib/snapshot/migration.rs index 6cacb04b7..83f2ddff5 100644 --- a/sdk/rust/lib/snapshot/migration.rs +++ b/sdk/rust/lib/snapshot/migration.rs @@ -831,17 +831,8 @@ async fn publish_index_component( transaction .execute_raw(Statement::from_sql_and_values( DatabaseBackend::Sqlite, - "DELETE FROM snapshot_index WHERE digest = ? OR artifact_path = ?", - [ - candidate - .inspected - .pinned - .source - .source_digest - .clone() - .into(), - path.clone().into(), - ], + "DELETE FROM snapshot_index WHERE artifact_path = ?", + [path.clone().into()], )) .await?; insert_canonical_index_row(&transaction, candidate).await?; @@ -859,7 +850,7 @@ async fn publish_index_component( } transaction .execute_unprepared( - "UPDATE snapshot_index SET child_count = (SELECT COUNT(*) FROM snapshot_index child WHERE child.parent_digest = snapshot_index.snapshot_id)", + "UPDATE snapshot_index SET child_count = (SELECT COUNT(DISTINCT COALESCE(child.snapshot_id, child.digest)) FROM snapshot_index child WHERE child.parent_digest = snapshot_index.snapshot_id)", ) .await?; transaction.commit().await?; diff --git a/sdk/rust/lib/snapshot/mod.rs b/sdk/rust/lib/snapshot/mod.rs index 5dcd4faae..b91b52152 100644 --- a/sdk/rust/lib/snapshot/mod.rs +++ b/sdk/rust/lib/snapshot/mod.rs @@ -9,6 +9,8 @@ mod archive; mod create; #[doc(hidden)] pub mod downgrade; +pub(crate) mod group; +pub(crate) mod lineage; mod metadata; pub(crate) mod migration; mod restore; @@ -36,6 +38,7 @@ pub struct Snapshot { digest: String, manifest: Manifest, labels: BTreeMap, + head_update: Option, } /// Result of direct sandbox-to-archive capture. @@ -56,6 +59,7 @@ pub struct SnapshotArchive { /// [`from_sandbox`](Self::from_sandbox) and is required. pub struct SnapshotBuilder { name: String, + group: Option, source_sandbox: Option, dest_dir: Option, labels: Vec<(String, String)>, @@ -84,6 +88,7 @@ impl Snapshot { pub fn builder(name: impl Into) -> SnapshotBuilder { SnapshotBuilder { name: name.into(), + group: None, source_sandbox: None, dest_dir: None, labels: Vec::new(), @@ -95,8 +100,9 @@ impl Snapshot { /// Create an installed disk or full snapshot artifact. /// - /// Disk capture requires a stopped or crashed sandbox. A builder configured with - /// [`full`](SnapshotBuilder::full) captures a running sandbox's checkpoint closure. + /// Disk capture supports resident and stopped sources, briefly quiescing a running root + /// without capturing RAM. A user-paused source remains paused. A builder configured with + /// [`full`](SnapshotBuilder::full) also captures memory and execution state. /// Publication is atomic and the local index remains a rebuildable cache. pub async fn create(config: SnapshotConfig) -> MicrosandboxResult { let backend = crate::backend::default_backend(); @@ -115,10 +121,10 @@ impl Snapshot { create::create_snapshot_archive(local, config, out.as_ref(), plain_tar).await } - /// Open an existing snapshot artifact by path or bare name. + /// Open an existing snapshot by explicit path, group head, or `group:member`. /// - /// Bare names (no path separator) resolve under the default - /// snapshots directory; anything else is treated as a path. + /// Bare names select the group's head under the default snapshots directory. + /// An exact member selector or explicit path remains fixed if that head advances. /// This is a fast metadata operation: it verifies the manifest /// structure, recomputes the manifest digest, and checks that the /// upper file exists with the recorded size. It does not read the @@ -159,6 +165,11 @@ impl Snapshot { &self.labels } + /// Group publication outcome, present on a newly captured snapshot. + pub fn head_update(&self) -> Option<&HeadUpdate> { + self.head_update.as_ref() + } + /// Apparent size of a file-state upper layer in bytes. pub fn size_bytes(&self) -> Option { self.manifest @@ -239,7 +250,8 @@ impl Snapshot { store::reindex_dir(local, dir.as_ref()).await } - /// Bundle a snapshot into a `.tar.zst` archive. + /// Bundle a snapshot into a `.msb` archive (tar + zstd by default). + /// The explicit output path is preserved; legacy suffixes remain supported. pub async fn save( name_or_path: &str, out: &Path, @@ -250,7 +262,7 @@ impl Snapshot { archive::save_snapshot(local, name_or_path, out, opts).await } - /// Unpack a snapshot archive (`.tar.zst` or `.tar`) into the + /// Unpack a snapshot archive (`.msb`, `.tar.zst`, or `.tar`) into the /// snapshots dir, registering anything found in the index. pub async fn load( archive_path: &Path, @@ -261,7 +273,7 @@ impl Snapshot { archive::load_snapshot(local, archive_path, dest).await } - /// Load a disk-dependent archive using its exact base snapshot or standalone base archive. + /// Load a dependent archive using its base snapshot or standalone base archive. /// The imported snapshot owns a complete local closure after this call. pub async fn load_with_base( archive_path: &Path, @@ -272,6 +284,37 @@ impl Snapshot { let local = backend.as_local().ok_or_else(snapshots_require_local)?; archive::load_snapshot_with_base(local, archive_path, dest, Some(base)).await } + + /// Import into a selected or newly generated group with explicit dependency/head policy. + pub async fn load_with_options( + archive_path: &Path, + opts: LoadOpts, + ) -> MicrosandboxResult { + let backend = crate::backend::default_backend(); + let local = backend.as_local().ok_or_else(snapshots_require_local)?; + archive::load_snapshot_with_options(local, archive_path, opts).await + } + + /// Load archives together, resolving omitted payloads from the batch, destination group, + /// and optional external base. Input order never chooses the group's head. + /// + /// Returns one handle per input archive head, in input order. Repeated snapshots are + /// installed once; all inputs are validated before publishing any snapshot members. + pub async fn load_many( + archive_paths: &[std::path::PathBuf], + opts: LoadOpts, + ) -> MicrosandboxResult> { + let backend = crate::backend::default_backend(); + let local = backend.as_local().ok_or_else(snapshots_require_local)?; + archive::load_snapshots(local, archive_paths, opts).await + } + + /// Read a group's head, or explicitly select a qualified `group:member`. + pub async fn group_head(selector: &str) -> MicrosandboxResult { + let backend = crate::backend::default_backend(); + let local = backend.as_local().ok_or_else(snapshots_require_local)?; + group::select(&local.snapshots_dir(), selector).await + } } impl SnapshotArchive { @@ -333,6 +376,8 @@ pub(crate) use create::CHECKPOINT_DIRECTORY; /// content verification. #[derive(Debug, Clone)] pub struct SnapshotHandle { + pub(crate) group: Option, + pub(crate) head_update: Option, pub(crate) snapshot_id: String, pub(crate) digest: String, pub(crate) name: Option, @@ -353,6 +398,14 @@ pub struct SnapshotHandle { } impl SnapshotHandle { + /// Group publication outcome, present when this handle was returned by import. + pub fn head_update(&self) -> Option<&HeadUpdate> { + self.head_update.as_ref() + } + /// Local group containing this installed copy, if any. + pub fn group(&self) -> Option<&str> { + self.group.as_deref() + } /// Stable opaque snapshot identity. pub fn id(&self) -> &str { &self.snapshot_id @@ -445,20 +498,25 @@ impl SnapshotHandle { /// Remove this snapshot. See [`Snapshot::remove`]. pub async fn remove(&self, force: bool) -> MicrosandboxResult<()> { - Snapshot::remove(&self.digest, force).await + // A handle denotes this installed copy, not every copy of its portable identity. + Snapshot::remove(self.artifact_path.to_string_lossy().as_ref(), force).await } } impl SnapshotBuilder { + /// Place the new member in this local snapshot group. + pub fn group(mut self, group: impl Into) -> Self { + self.group = Some(group.into()); + self + } /// Set the source sandbox to snapshot. Required. pub fn from_sandbox(mut self, source_sandbox: impl Into) -> Self { self.source_sandbox = Some(source_sandbox.into()); self } - /// Create the artifact under this parent directory instead of the - /// default snapshots store. The artifact directory is - /// `dest_dir/`; the name stays the snapshot's identity. + /// Use this group-store root instead of the default snapshots directory. + /// The artifact is installed at `dest_dir//`. pub fn dest_dir(mut self, dest_dir: impl Into) -> Self { self.dest_dir = Some(dest_dir.into()); self @@ -470,7 +528,7 @@ impl SnapshotBuilder { self } - /// Overwrite an existing artifact at the destination. + /// Overwrite a direct archive destination. Installed group members are immutable. pub fn force(mut self) -> Self { self.force = true; self @@ -501,6 +559,7 @@ impl SnapshotBuilder { })?; Ok(SnapshotConfig { name: self.name, + group: self.group, dest_dir: self.dest_dir, source_sandbox, labels: self.labels, @@ -529,9 +588,10 @@ impl SnapshotBuilder { // Re-Exports //-------------------------------------------------------------------------------------------------- -pub use archive::SaveOpts; #[cfg(feature = "fuzzing")] pub use archive::fuzz_unpack_archive; +pub use archive::{LoadOpts, SaveOpts}; +pub use group::{HeadUpdate, HeadUpdateReason}; pub use microsandbox_image::snapshot::{ CheckpointSnapshotState, DESCRIPTOR_FILENAME, DiskLayer, DiskLayerId, FileSnapshotState, ImageRef, LayerFileKind, LayerPayload, Manifest, SnapshotCapture, SnapshotConsistency, @@ -557,6 +617,7 @@ impl Snapshot { digest, manifest, labels, + head_update: None, } } } diff --git a/sdk/rust/lib/snapshot/restore.rs b/sdk/rust/lib/snapshot/restore.rs index dda0a75af..5b2cfc2d8 100644 --- a/sdk/rust/lib/snapshot/restore.rs +++ b/sdk/rust/lib/snapshot/restore.rs @@ -8,7 +8,7 @@ use microsandbox_runtime::launch::{CheckpointRestoreConfig, RootfsUpperLayerConf use crate::{MicrosandboxError, MicrosandboxResult, Operation, UnsupportedReason}; -use super::create::{copy_checkpoint_file, materialize_checkpoint_closure}; +use super::create::{copy_checkpoint_file, stage_checkpoint_closure}; //-------------------------------------------------------------------------------------------------- // Constants @@ -44,23 +44,14 @@ pub(crate) async fn materialize_checkpoint_for_child( child_stage: &Path, root_disk: &SnapshotRootDisk, ) -> MicrosandboxResult { - let expected = ObjectId::new(&source.checkpoint_root) - .map_err(|error| MicrosandboxError::SnapshotIntegrity(error.to_string()))?; - let source_closure = CheckpointClosure::open(&source.closure, Some(&expected)) - .map_err(|error| MicrosandboxError::SnapshotIntegrity(error.to_string()))?; - if source_closure.checkpoint().checkpoint_id != source.checkpoint_id { - return Err(MicrosandboxError::SnapshotIntegrity( - "checkpoint restore source has another identity".into(), - )); - } - validate_root_disk_closure(&source_closure, root_disk, false)?; - + // Validate once after obtaining child-owned files. Validating the source first neither + // protects against a later source mutation nor substitutes for validation of the child. tokio::fs::create_dir_all(child_stage).await?; let closure_destination = child_stage.join(CHILD_CHECKPOINT_DIRECTORY); let source_path = source.closure.clone(); let destination_for_copy = closure_destination.clone(); tokio::task::spawn_blocking(move || { - materialize_checkpoint_closure(&source_path, &destination_for_copy) + stage_checkpoint_closure(&source_path, &destination_for_copy) }) .await .map_err(|error| MicrosandboxError::Custom(format!("checkpoint child copy task: {error}")))??; @@ -119,6 +110,8 @@ pub(crate) async fn materialize_checkpoint_child_state( Ok(CheckpointChildMaterialization { restore: CheckpointRestoreConfig { + local_branch: false, + forked: false, closure: closure_destination.to_path_buf(), checkpoint_root: checkpoint_root.to_string(), checkpoint_id: checkpoint_id.to_string(), @@ -456,6 +449,8 @@ mod tests { let checkpoint_root = ObjectId::from_bytes(&checkpoint_bytes).unwrap(); std::fs::write(source.join("checkpoint.json"), checkpoint_bytes).unwrap(); let restore = CheckpointRestoreConfig { + local_branch: false, + forked: false, closure: source.clone(), checkpoint_root: checkpoint_root.to_string(), checkpoint_id: checkpoint.checkpoint_id, diff --git a/sdk/rust/lib/snapshot/store.rs b/sdk/rust/lib/snapshot/store.rs index ac918cca8..62988fe62 100644 --- a/sdk/rust/lib/snapshot/store.rs +++ b/sdk/rust/lib/snapshot/store.rs @@ -24,9 +24,8 @@ use super::{Snapshot, SnapshotFormat, SnapshotHandle, SnapshotScope, UpperIntegr /// Open and validate snapshot artifact metadata. /// -/// `path_or_name` is treated as a path if it contains `/` or starts -/// with `.` or `~`; otherwise as a bare name resolved under the -/// passed-in `local` backend's snapshots directory. +/// Explicit paths remain valid. Bare selectors resolve a group's head; qualified selectors +/// resolve a group member. Global portable identities must identify exactly one local copy. pub(super) async fn open_snapshot( local: &LocalBackend, path_or_name: &str, @@ -37,11 +36,7 @@ pub(super) async fn open_snapshot( )); } - let dir = if looks_like_path(path_or_name) { - PathBuf::from(path_or_name) - } else { - local.snapshots_dir().join(path_or_name) - }; + let dir = resolve_path(local, path_or_name).await?; if !dir.exists() { return Err(MicrosandboxError::SnapshotNotFound( @@ -129,14 +124,22 @@ pub(super) async fn open_snapshot( let labels = super::metadata::read(&dir, &manifest, translated_labels).await?; let snap = Snapshot::from_parts(dir.clone(), digest.clone(), manifest, labels); - // Opportunistic auto-reindex: if the artifact lives under the - // configured snapshots dir but its digest isn't in the local - // index, insert it. Keeps the cache aligned with reality without - // forcing the user to think about it. Best-effort — errors are - // logged, not propagated. + // Published managed members and explicitly opened flat artifacts remain discoverable for + // parent traversal. Archive/capture staging must never replace durable index entries. let snapshots_dir = local.snapshots_dir(); - if dir.parent() == Some(snapshots_dir.as_path()) - && let Ok(None) = lookup_by_digest(local, &digest).await + let managed = dir + .strip_prefix(&snapshots_dir) + .ok() + .is_some_and(|relative| { + relative + .components() + .all(|part| !part.as_os_str().to_string_lossy().starts_with('.')) + }); + if managed + && (super::group::group_path(&dir).is_some() + || dir.parent() == Some(snapshots_dir.as_path())) + && let Ok(existing) = indexed_path(local, &dir).await + && existing.as_ref().is_none_or(|row| row.digest != digest) && let Err(e) = index_upsert(local, snap.path(), snap.digest(), snap.manifest()).await { tracing::debug!(error = %e, snapshot = %digest, "auto-reindex skipped"); @@ -159,42 +162,31 @@ pub(super) async fn index_upsert( .unwrap_or_else(|_| Utc::now().naive_utc()); let indexed_at = Utc::now().naive_utc(); + let artifact_path = canonical_path(artifact_path); let artifact_path_str = artifact_path.display().to_string(); - let artifact_name = artifact_path - .file_name() - .and_then(|s| s.to_str()) - .map(|s| s.to_string()); - - // Delete any prior row for this digest, name, or path, then insert. - // This keeps the rebuildable index aligned when an artifact is - // replaced in-place or when a manifest rewrite changes its digest. - // The superseded rows' parent edges disappear with them, so their - // parents' child_count must come down first; the fresh insert re-adds - // its own edge below. + let group_path = super::group::group_path(&artifact_path); + let group_name = group_path + .as_ref() + .and_then(|path| path.file_name()) + .map(|name| name.to_string_lossy().into_owned()); + let artifact_name = super::group::member_name(&artifact_path)?.or_else(|| { + artifact_path + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + }); + let group_path = group_path.map(|path| path.display().to_string()); + + // Portable identities may occur in multiple groups. Replace only this local address, + // never another copy that happens to share descriptor bytes, identity, or member name. let mut supersede = sea_orm::Condition::any() - .add(snapshot_entity::Column::Digest.eq(digest.to_string())) - .add(snapshot_entity::Column::SnapshotId.eq(manifest.snapshot_id.to_string())) .add(snapshot_entity::Column::ArtifactPath.eq(artifact_path_str.clone())); - if let Some(name) = artifact_name.as_ref() { - supersede = supersede.add(snapshot_entity::Column::Name.eq(name.clone())); + if let (Some(group), Some(name)) = (&group_path, &artifact_name) { + supersede = supersede.add( + sea_orm::Condition::all() + .add(snapshot_entity::Column::GroupPath.eq(group.clone())) + .add(snapshot_entity::Column::Name.eq(name.clone())), + ); } - let superseded = snapshot_entity::Entity::find() - .filter(supersede.clone()) - .all(db) - .await?; - for row in &superseded { - if let Some(parent) = row.parent_digest.as_ref() { - db.execute_unprepared(&format!( - "UPDATE snapshot_index SET child_count = MAX(0, child_count - 1) WHERE snapshot_id = '{}'", - parent.replace('\'', "''") - )) - .await?; - } - } - snapshot_entity::Entity::delete_many() - .filter(supersede) - .exec(db) - .await?; let (state_kind, format, fstype, checkpoint_manifest_digest, size_bytes) = match &manifest.state { @@ -233,6 +225,8 @@ pub(super) async fn index_upsert( snapshot_id: Set(Some(manifest.snapshot_id.to_string())), descriptor_digest: Set(Some(digest.to_string())), name: Set(artifact_name), + group_name: Set(group_name), + group_path: Set(group_path), parent_digest: Set(manifest.parent.as_ref().map(ToString::to_string)), scope: Set(scope_str.into()), state_kind: Set(state_kind.into()), @@ -252,17 +246,20 @@ pub(super) async fn index_upsert( indexed_at: Set(indexed_at), child_count: Set(0), }; - row.insert(db).await?; - - // If this snapshot has a parent, bump the parent's child_count. - if let Some(parent) = manifest.parent.as_ref() { - use sea_orm::ConnectionTrait; - db.execute_unprepared(&format!( - "UPDATE snapshot_index SET child_count = child_count + 1 WHERE snapshot_id = '{}'", - parent.as_str().replace('\'', "''") - )) - .await?; - } + db.transaction::<_, _, _, sea_orm::DbErr>(|transaction| { + let row = row.clone(); + let supersede = supersede.clone(); + async move { + snapshot_entity::Entity::delete_many() + .filter(supersede) + .exec(&transaction) + .await?; + row.insert(&transaction).await?; + recompute_children(&transaction).await?; + Ok((transaction, ())) + } + }) + .await?; Ok(()) } @@ -308,11 +305,11 @@ pub(super) async fn list_dir( if !dir.exists() { return Ok(Vec::new()); } - let mut out = Vec::new(); + let mut candidates = Vec::new(); let mut entries = tokio::fs::read_dir(dir).await?; while let Some(entry) = entries.next_entry().await? { let path = entry.path(); - if !path.is_dir() { + if !entry.file_type().await?.is_dir() { continue; } // Dot-prefixed directories are never artifacts; create() stages @@ -325,6 +322,23 @@ pub(super) async fn list_dir( { continue; } + if path.join(super::group::GROUP_FILENAME).is_file() { + // Groups are exactly one level deep. Do not recursively walk arbitrary folders, + // symlink trees, checkpoint stores, or failed staging directories. + let mut members = tokio::fs::read_dir(&path).await?; + while let Some(member) = members.next_entry().await? { + if member.file_type().await?.is_dir() + && !member.file_name().to_string_lossy().starts_with('.') + { + candidates.push(member.path()); + } + } + } else { + candidates.push(path); + } + } + let mut out = Vec::new(); + for path in candidates { if !path.join(DESCRIPTOR_FILENAME).exists() && !path.join(V066_DESCRIPTOR_FILENAME).exists() { continue; @@ -348,41 +362,12 @@ pub(super) async fn remove_snapshot( let read_db = pools.read(); let write_db = pools.write(); - // Resolve the target row. Accept digest, name, or path. - let (digest, artifact_path) = if path_or_name.starts_with("snap_") { - let row = snapshot_entity::Entity::find() - .filter(snapshot_entity::Column::SnapshotId.eq(path_or_name.to_string())) - .one(read_db) - .await? - .ok_or_else(|| MicrosandboxError::SnapshotNotFound(path_or_name.into()))?; - (row.digest.clone(), PathBuf::from(row.artifact_path)) - } else if path_or_name.starts_with("sha256:") || path_or_name.starts_with("sha512:") { - let row = snapshot_entity::Entity::find_by_id(path_or_name.to_string()) - .one(read_db) - .await? - .ok_or_else(|| MicrosandboxError::SnapshotNotFound(path_or_name.into()))?; - (row.digest.clone(), PathBuf::from(row.artifact_path)) - } else if looks_like_path(path_or_name) { - // Path: open to read the digest, then drop both row and dir. - let snap = open_snapshot(local, path_or_name).await?; - (snap.digest.clone(), snap.path.clone()) - } else { - // Bare name: prefer the index lookup; fall back to default-dir resolution. - let row = snapshot_entity::Entity::find() - .filter(snapshot_entity::Column::Name.eq(path_or_name.to_string())) - .one(read_db) - .await?; - if let Some(row) = row { - (row.digest.clone(), PathBuf::from(row.artifact_path)) - } else { - let dir = local.snapshots_dir().join(path_or_name); - let snap = open_snapshot(local, dir.to_string_lossy().as_ref()).await?; - (snap.digest.clone(), snap.path.clone()) - } - }; + let snapshot = open_snapshot(local, path_or_name).await?; + let artifact_path = canonical_path(snapshot.path()); + let artifact_key = artifact_path.display().to_string(); // Check children unless --force. - let row = snapshot_entity::Entity::find_by_id(digest.clone()) + let row = snapshot_entity::Entity::find_by_id(artifact_key.clone()) .one(read_db) .await?; if let Some(ref row) = row @@ -391,28 +376,20 @@ pub(super) async fn remove_snapshot( { return Err(MicrosandboxError::Custom(format!( "snapshot {} has {} indexed child snapshot(s); pass --force to remove anyway", - digest, row.child_count + snapshot.id(), + row.child_count ))); } - // Drop the index row and decrement parent's child_count if any. - let parent = row.as_ref().and_then(|r| r.parent_digest.clone()); - snapshot_entity::Entity::delete_by_id(digest.clone()) - .exec(write_db) - .await?; - if let Some(p) = parent { - write_db - .execute_unprepared(&format!( - "UPDATE snapshot_index SET child_count = MAX(0, child_count - 1) WHERE snapshot_id = '{}'", - p.replace('\'', "''") - )) - .await?; - } - - // Delete the artifact directory. - if artifact_path.exists() { + // The group helper validates head removal and removes the member under its publication + // lock. Even --force must not leave a group's head dangling while other members remain. + if !super::group::remove_member(&artifact_path).await? && artifact_path.exists() { tokio::fs::remove_dir_all(&artifact_path).await?; } + snapshot_entity::Entity::delete_by_id(artifact_key) + .exec(write_db) + .await?; + recompute_children(write_db).await?; Ok(()) } @@ -429,47 +406,41 @@ pub(super) async fn reindex_dir(local: &LocalBackend, dir: &Path) -> Microsandbo // After upserts, recompute child_count from parent edges in one pass // to keep the cache honest about the current set of artifacts. let db = local.db().await?.write(); - db.execute_unprepared( - "UPDATE snapshot_index SET child_count = (\ - SELECT COUNT(*) FROM snapshot_index AS c \ - WHERE c.parent_digest = snapshot_index.snapshot_id)", - ) - .await?; + recompute_children(db).await?; Ok(indexed) } -/// Look up a snapshot by digest, name, or path in the local index. +/// Resolve a local address and refresh its rebuildable index row before returning a handle. pub(super) async fn get_handle( local: &LocalBackend, needle: &str, ) -> MicrosandboxResult { - let db = local.db().await?.read(); - - let row = if needle.starts_with("snap_") { - snapshot_entity::Entity::find() - .filter(snapshot_entity::Column::SnapshotId.eq(needle.to_string())) - .one(db) - .await? - } else if needle.starts_with("sha256:") || needle.starts_with("sha512:") { - snapshot_entity::Entity::find_by_id(needle.to_string()) - .one(db) - .await? - } else if looks_like_path(needle) { - // Path lookup: match by artifact_path. - let canon = std::fs::canonicalize(needle) - .map(|p| p.to_string_lossy().to_string()) - .unwrap_or_else(|_| needle.to_string()); - snapshot_entity::Entity::find() - .filter(snapshot_entity::Column::ArtifactPath.eq(canon)) - .one(db) - .await? - } else { - snapshot_entity::Entity::find() - .filter(snapshot_entity::Column::Name.eq(needle.to_string())) - .one(db) - .await? - }; - + let snapshot = open_snapshot(local, needle).await?; + let artifact_path = canonical_path(snapshot.path()); + let alias = super::group::member_name(&artifact_path)?.or_else(|| { + artifact_path + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + }); + if let Some(row) = indexed_path(local, &artifact_path).await? + && row.digest == snapshot.digest() + && row.name == alias + && row.group_path + == super::group::group_path(&artifact_path).map(|path| path.display().to_string()) + { + return Ok(handle_from_model(row)); + } + index_upsert( + local, + snapshot.path(), + snapshot.digest(), + snapshot.manifest(), + ) + .await?; + let row = + snapshot_entity::Entity::find_by_id(canonical_path(snapshot.path()).display().to_string()) + .one(local.db().await?.read()) + .await?; row.map(handle_from_model) .ok_or_else(|| MicrosandboxError::SnapshotNotFound(needle.into())) } @@ -480,15 +451,78 @@ pub(super) async fn lookup_by_digest( digest: &str, ) -> MicrosandboxResult> { let db = local.db().await?.read(); - let row = snapshot_entity::Entity::find() + let rows = snapshot_entity::Entity::find() .filter( sea_orm::Condition::any() .add(snapshot_entity::Column::Digest.eq(digest.to_string())) .add(snapshot_entity::Column::SnapshotId.eq(digest.to_string())), ) - .one(db) + .all(db) .await?; - Ok(row.map(handle_from_model)) + unique_identity_match(rows, digest).map(|row| row.map(handle_from_model)) +} + +async fn resolve_path(local: &LocalBackend, selector: &str) -> MicrosandboxResult { + if looks_like_path(selector) { + return Ok(PathBuf::from(selector)); + } + if microsandbox_image::snapshot::SnapshotId::new(selector).is_ok() + || selector.starts_with("sha256:") + || selector.starts_with("sha512:") + { + return lookup_by_digest(local, selector) + .await? + .map(|handle| handle.artifact_path) + .ok_or_else(|| MicrosandboxError::SnapshotNotFound(selector.into())); + } + if !selector.contains(':') { + let flat = local.snapshots_dir().join(selector); + if flat.join(DESCRIPTOR_FILENAME).is_file() || flat.join(V066_DESCRIPTOR_FILENAME).is_file() + { + return Err(MicrosandboxError::InvalidConfig(format!( + "'{selector}' is an ungrouped snapshot; bare names now select group heads, so open this artifact by its explicit path: {}", + flat.display() + ))); + } + } + super::group::resolve(&local.snapshots_dir(), selector).await +} + +fn canonical_path(path: &Path) -> PathBuf { + std::fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()) +} + +async fn indexed_path( + local: &LocalBackend, + path: &Path, +) -> MicrosandboxResult> { + Ok( + snapshot_entity::Entity::find_by_id(canonical_path(path).display().to_string()) + .one(local.db().await?.read()) + .await?, + ) +} + +fn unique_identity_match( + mut rows: Vec, + identity: &str, +) -> MicrosandboxResult> { + if rows.len() > 1 { + return Err(MicrosandboxError::InvalidConfig(format!( + "snapshot identity {identity} has {} local copies; use group:member or an explicit artifact path", + rows.len() + ))); + } + Ok(rows.pop()) +} + +async fn recompute_children(db: &C) -> Result<(), sea_orm::DbErr> { + // Repeated imports are instances, not additional lineage edges. Apply the same number of + // distinct child identities to every local copy of a parent. + db.execute_unprepared( + "UPDATE snapshot_index SET child_count = (SELECT COUNT(DISTINCT COALESCE(c.snapshot_id, c.digest)) FROM snapshot_index c WHERE c.parent_digest = snapshot_index.snapshot_id)", + ).await?; + Ok(()) } fn handle_from_model(m: snapshot_entity::Model) -> SnapshotHandle { @@ -510,6 +544,8 @@ fn handle_from_model(m: snapshot_entity::Model) -> SnapshotHandle { snapshot_id: m.snapshot_id.unwrap_or_else(|| m.digest.clone()), digest: m.digest, name: m.name, + group: m.group_name, + head_update: None, parent_digest: m.parent_digest, scope, image_ref: m.image_ref, @@ -533,7 +569,180 @@ fn handle_from_model(m: snapshot_entity::Model) -> SnapshotHandle { #[cfg(test)] mod tests { - use super::looks_like_path; + use std::collections::BTreeMap; + + use microsandbox_image::snapshot::{ + CheckpointSnapshotState, ImageRef, SCHEMA, SnapshotCapture, SnapshotConsistency, + SnapshotId, SnapshotRootDisk, + }; + + use super::*; + + fn manifest(id: u128, parent: Option<&Manifest>) -> Manifest { + Manifest { + schema: SCHEMA.into(), + snapshot_id: SnapshotId::new(format!("snap_{id:032x}")).unwrap(), + scope: SnapshotScope::Full, + // These tests exercise addressing and indexing, not checkpoint restoration. + state: SnapshotState::Checkpoint(CheckpointSnapshotState { + checkpoint_id: "checkpoint_test".into(), + checkpoint_root: format!("sha256:{}", "a".repeat(64)), + restore_intents: vec!["resume".into()], + requirements_summary: BTreeMap::new(), + }), + capture: SnapshotCapture { + created_at: "2026-09-10T00:00:00Z".into(), + source_lineage: None, + source_checkpoint: None, + consistency: SnapshotConsistency::CrashConsistent, + }, + image: ImageRef { + reference: "docker.io/library/alpine:3.20".into(), + manifest_digest: format!("sha256:{}", "b".repeat(64)), + }, + root_disk: SnapshotRootDisk::Managed, + parent: parent.map(|parent| parent.snapshot_id.clone()), + extensions: BTreeMap::new(), + requires: Vec::new(), + } + } + + async fn install( + local: &LocalBackend, + group: &str, + name: &str, + manifest: &Manifest, + ) -> PathBuf { + let directory = super::super::group::ensure(&local.snapshots_dir(), Some(group)) + .await + .unwrap(); + let stage = tempfile::tempdir().unwrap(); + let artifact = stage.path().join("member"); + std::fs::create_dir(&artifact).unwrap(); + std::fs::write( + artifact.join(DESCRIPTOR_FILENAME), + manifest.to_canonical_bytes().unwrap(), + ) + .unwrap(); + super::super::group::publish( + &directory, + stage.path(), + &BTreeMap::from([(manifest.snapshot_id.to_string(), name.into())]), + &manifest.snapshot_id, + false, + ) + .await + .unwrap(); + directory.join(manifest.snapshot_id.as_str()) + } + + #[tokio::test] + async fn duplicate_identities_keep_group_addresses_and_remove_only_selected_copy() { + let home = tempfile::tempdir().unwrap(); + let local = LocalBackend::builder() + .home(home.path()) + .build() + .await + .unwrap(); + let snapshot = manifest(1, None); + let first = install(&local, "first", "baseline", &snapshot).await; + let second = install(&local, "second", "baseline", &snapshot).await; + assert_eq!( + reindex_dir(&local, &local.snapshots_dir()).await.unwrap(), + 2 + ); + let first_handle = get_handle(&local, "first:baseline").await.unwrap(); + let second_handle = get_handle(&local, "second").await.unwrap(); + assert_eq!(first_handle.digest(), second_handle.digest()); + assert_eq!(first_handle.group(), Some("first")); + assert_eq!(second_handle.group(), Some("second")); + assert!( + get_handle(&local, snapshot.snapshot_id.as_str()) + .await + .unwrap_err() + .to_string() + .contains("local copies") + ); + assert!( + get_handle(&local, first_handle.digest()) + .await + .unwrap_err() + .to_string() + .contains("local copies") + ); + remove_snapshot(&local, "first:baseline", false) + .await + .unwrap(); + assert!(!first.exists()); + assert!(second.exists()); + assert_eq!(list_indexed(&local).await.unwrap().len(), 1); + assert_eq!( + get_handle(&local, "second").await.unwrap().digest(), + snapshot.digest().unwrap() + ); + } + + #[tokio::test] + async fn distinct_child_counts_and_head_guard_survive_reindex() { + let home = tempfile::tempdir().unwrap(); + let local = LocalBackend::builder() + .home(home.path()) + .build() + .await + .unwrap(); + let parent = manifest(1, None); + let child = manifest(2, Some(&parent)); + for group in ["first", "second"] { + install(&local, group, "base", &parent).await; + install(&local, group, "child", &child).await; + } + reindex_dir(&local, &local.snapshots_dir()).await.unwrap(); + reindex_dir(&local, &local.snapshots_dir()).await.unwrap(); + let rows = snapshot_entity::Entity::find() + .filter(snapshot_entity::Column::SnapshotId.eq(parent.snapshot_id.as_str())) + .all(local.db().await.unwrap().read()) + .await + .unwrap(); + assert_eq!(rows.len(), 2); + assert!(rows.iter().all(|row| row.child_count == 1)); + assert!( + remove_snapshot(&local, "first:child", true) + .await + .unwrap_err() + .to_string() + .contains("current head") + ); + assert_eq!( + get_handle(&local, "first").await.unwrap().id(), + child.snapshot_id.as_str() + ); + } + + #[tokio::test] + async fn flat_artifact_requires_explicit_path() { + let home = tempfile::tempdir().unwrap(); + let local = LocalBackend::builder() + .home(home.path()) + .build() + .await + .unwrap(); + let artifact = local.snapshots_dir().join("flat"); + std::fs::create_dir_all(&artifact).unwrap(); + let snapshot = manifest(1, None); + std::fs::write( + artifact.join(DESCRIPTOR_FILENAME), + snapshot.to_canonical_bytes().unwrap(), + ) + .unwrap(); + assert!(open_snapshot(&local, "flat").await.is_err()); + assert_eq!( + open_snapshot(&local, artifact.to_str().unwrap()) + .await + .unwrap() + .id(), + &snapshot.snapshot_id + ); + } #[test] fn bare_names_are_not_paths() { diff --git a/sdk/rust/lib/snapshot/verify.rs b/sdk/rust/lib/snapshot/verify.rs index d80fe62f7..229ee3b68 100644 --- a/sdk/rust/lib/snapshot/verify.rs +++ b/sdk/rust/lib/snapshot/verify.rs @@ -164,12 +164,19 @@ async fn verify_file_layer( snap: &Snapshot, layer: µsandbox_image::snapshot::DiskLayer, ) -> MicrosandboxResult { - let Some(expected) = layer.payload.integrity.as_ref() else { + verify_file_payload(&snap.layer_path(layer), layer.payload.integrity.as_ref()).await +} + +/// Verify an owned imported layer with the same codecs used by explicit snapshot verification. +pub(super) async fn verify_file_payload( + upper_path: &Path, + expected: Option<&UpperIntegrity>, +) -> MicrosandboxResult { + let Some(expected) = expected else { return Ok(UpperVerifyStatus::NotRecorded); }; - let upper_path = snap.layer_path(layer); - let payload = open_verification_source(&upper_path)?; + let payload = open_verification_source(upper_path)?; let before = verification_source_identity(&payload.metadata()?); let actual = match expected { UpperIntegrity::Sha256 { .. } => { @@ -182,7 +189,7 @@ async fn verify_file_layer( compute_merkle_integrity_from_file(payload.try_clone()?).await? } }; - ensure_verification_source_unchanged(&payload, &upper_path, &before)?; + ensure_verification_source_unchanged(&payload, upper_path, &before)?; if actual != *expected { return Err(MicrosandboxError::SnapshotIntegrity(format!( diff --git a/sdk/rust/tests/plain_http_secret.rs b/sdk/rust/tests/plain_http_secret.rs index 4e17b25ef..83f2a5d84 100644 --- a/sdk/rust/tests/plain_http_secret.rs +++ b/sdk/rust/tests/plain_http_secret.rs @@ -11,7 +11,8 @@ use tokio::task::JoinHandle; // Constants -const ALPINE_IMAGE: &str = "alpine"; +// Match the mirrored fixture used by the other SDK integration tests. +const ALPINE_IMAGE: &str = "mirror.gcr.io/library/alpine:latest"; const REAL_SECRET: &str = "real-secret-plain-http"; /// Placeholder the guest sees for the `API_KEY` secret: the env var name with /// the `MSB_` prefix the runtime injects. diff --git a/sdk/rust/tests/snapshot_artifact.rs b/sdk/rust/tests/snapshot_artifact.rs index 88bbfe4d5..e7c8068b1 100644 --- a/sdk/rust/tests/snapshot_artifact.rs +++ b/sdk/rust/tests/snapshot_artifact.rs @@ -7,7 +7,7 @@ use std::collections::BTreeMap; use std::io::Cursor; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::Arc; use microsandbox::Snapshot; @@ -532,6 +532,52 @@ async fn isolated_backend(home: &Path) -> Arc { Arc::new(LocalBackend::builder().home(home).build().await.unwrap()) } +/// Export complete synthetic artifacts independently: their parent edges describe history, +/// not a requirement to have every ancestor present merely to read the disk payload. +async fn save_batch_fixtures(parent: &Path, artifacts: &[PathBuf]) -> Vec { + let mut archives = Vec::new(); + for (index, artifact) in artifacts.iter().enumerate() { + let archive = parent.join(format!("batch-{index}.msb")); + Snapshot::save( + artifact.to_string_lossy().as_ref(), + &archive, + microsandbox::snapshot::SaveOpts { + plain_tar: true, + ..Default::default() + }, + ) + .await + .unwrap(); + archives.push(archive); + } + archives +} + +fn batch_group_options(group: &str) -> microsandbox::snapshot::LoadOpts { + microsandbox::snapshot::LoadOpts { + group: Some(group.into()), + ..Default::default() + } +} + +/// A failed batch may leave an empty group/staging directory, but no immutable member +/// may become visible before every input archive and publication conflict is checked. +fn assert_no_batch_members(root: &Path) { + let mut pending = vec![root.to_path_buf()]; + while let Some(path) = pending.pop() { + if !path.exists() { + continue; + } + for entry in std::fs::read_dir(path).unwrap() { + let entry = entry.unwrap(); + assert_ne!(entry.file_name(), DESCRIPTOR_FILENAME); + if entry.file_type().unwrap().is_dir() { + pending.push(entry.path()); + } + } + } +} + //-------------------------------------------------------------------------------------------------- // Tests //-------------------------------------------------------------------------------------------------- @@ -833,6 +879,478 @@ async fn save_then_load_round_trips_via_plain_tar() { assert_eq!(handle.digest(), original_digest); } +#[tokio::test] +async fn repeated_loads_preserve_ids_and_resolve_local_group_names() { + let tmp = TempDir::new().unwrap(); + let home = tmp.path().join("home"); + let backend = isolated_backend(&home).await; + let (source, digest) = make_artifact(tmp.path(), "clean", b"group payload"); + let snapshot_id = artifact_id(&source); + let archive = tmp.path().join("group.msb"); + let reexport = tmp.path().join("renamed.msb"); + + microsandbox::with_backend(backend, async { + Snapshot::save( + source.to_string_lossy().as_ref(), + &archive, + microsandbox::snapshot::SaveOpts::default(), + ) + .await + .unwrap(); + let options = microsandbox::snapshot::LoadOpts { + group: Some("work".into()), + ..Default::default() + }; + let first = Snapshot::load_with_options(&archive, options.clone()) + .await + .unwrap(); + assert_eq!(first.group(), Some("work")); + assert_eq!(first.name(), Some("clean")); + assert_eq!(first.path(), home.join("snapshots/work").join(&snapshot_id)); + assert_eq!( + first.head_update().unwrap().reason, + microsandbox::snapshot::HeadUpdateReason::Initialized + ); + + // Reimporting the same identity into the same group is idempotent. + let repeated = Snapshot::load_with_options(&archive, options) + .await + .unwrap(); + assert_eq!(repeated.path(), first.path()); + assert_eq!(repeated.id(), snapshot_id); + assert_eq!( + repeated.head_update().unwrap().reason, + microsandbox::snapshot::HeadUpdateReason::Unchanged + ); + assert_eq!(Snapshot::list().await.unwrap().len(), 1); + assert_eq!(Snapshot::open("work").await.unwrap().digest(), digest); + assert_eq!( + Snapshot::open("work:clean").await.unwrap().id().as_str(), + snapshot_id + ); + assert_eq!( + Snapshot::open(format!("work:{snapshot_id}")) + .await + .unwrap() + .digest(), + digest + ); + + // A default import always gets its own local namespace, even for identical bytes. + let fresh = Snapshot::load(&archive, None).await.unwrap(); + let another = Snapshot::load(&archive, None).await.unwrap(); + assert_ne!(fresh.group(), another.group()); + assert_ne!(fresh.group(), Some("work")); + assert_eq!(fresh.id(), first.id()); + assert_eq!(another.id(), first.id()); + assert_eq!(Snapshot::list().await.unwrap().len(), 3); + + Snapshot::save( + "work:clean", + &reexport, + microsandbox::snapshot::SaveOpts::default(), + ) + .await + .unwrap(); + let renamed = Snapshot::load_with_options( + &reexport, + microsandbox::snapshot::LoadOpts { + group: Some("renamed".into()), + ..Default::default() + }, + ) + .await + .unwrap(); + assert_eq!(renamed.name(), Some("clean")); + assert_eq!(renamed.id(), snapshot_id); + assert_eq!( + Snapshot::group_head("renamed").await.unwrap().head, + snapshot_id + ); + + // Removing one installed copy does not erase another group's membership or payload. + Snapshot::remove(&format!("{}:clean", fresh.group().unwrap()), false) + .await + .unwrap(); + assert!(!fresh.path().exists()); + assert!(first.path().is_dir()); + assert!(another.path().is_dir()); + assert!(renamed.path().is_dir()); + assert_eq!(Snapshot::list().await.unwrap().len(), 3); + assert_eq!(Snapshot::open("work:clean").await.unwrap().digest(), digest); + }) + .await; +} + +#[tokio::test] +async fn group_alias_collision_keeps_the_installed_snapshot_and_head() { + let tmp = TempDir::new().unwrap(); + let home = tmp.path().join("home"); + let backend = isolated_backend(&home).await; + let (first, first_digest) = make_artifact(&tmp.path().join("first"), "clean", b"first"); + let (second, _) = make_artifact(&tmp.path().join("second"), "clean", b"second"); + let original_id = artifact_id(&first); + let competing_id = artifact_id(&second); + let archive = tmp.path().join("first.msb"); + let competing = tmp.path().join("second.msb"); + microsandbox::with_backend(backend, async { + for (source, destination) in [(&first, &archive), (&second, &competing)] { + Snapshot::save( + source.to_string_lossy().as_ref(), + destination, + microsandbox::snapshot::SaveOpts::default(), + ) + .await + .unwrap(); + } + let options = microsandbox::snapshot::LoadOpts { + group: Some("work".into()), + ..Default::default() + }; + let installed = Snapshot::load_with_options(&archive, options.clone()) + .await + .unwrap(); + let error = Snapshot::load_with_options(&competing, options) + .await + .unwrap_err(); + assert!( + error.to_string().contains("conflicts"), + "unexpected error: {error}" + ); + assert_eq!( + Snapshot::group_head("work").await.unwrap().head, + original_id + ); + assert_eq!( + Snapshot::open("work:clean").await.unwrap().digest(), + first_digest + ); + assert_eq!( + std::fs::read(artifact_payload_path(installed.path())).unwrap(), + b"first" + ); + assert!(!home.join("snapshots/work").join(competing_id).exists()); + assert_eq!(Snapshot::list().await.unwrap().len(), 1); + }) + .await; +} + +#[tokio::test] +async fn load_many_selects_lineage_tip_independently_of_input_order() { + let tmp = TempDir::new().unwrap(); + let home = tmp.path().join("home"); + let backend = isolated_backend(&home).await; + let source = tmp.path().join("source-artifacts"); + let (first, _) = make_artifact(&source, "cp01", b"first disk"); + let first_id = artifact_id(&first); + let (second, _) = + make_artifact_with_parent(&source, "cp02", b"second disk", Some(first_id.clone())); + let second_id = artifact_id(&second); + let (third, _) = + make_artifact_with_parent(&source, "cp03", b"third disk", Some(second_id.clone())); + let third_id = artifact_id(&third); + microsandbox::with_backend(backend, async { + let archives = save_batch_fixtures(tmp.path(), &[first, second, third]).await; + for (group, order) in [("reverse", [2, 1, 0]), ("shuffled", [1, 0, 2])] { + let input = order.map(|index| archives[index].clone()); + let handles = Snapshot::load_many(&input, batch_group_options(group)) + .await + .unwrap(); + let expected_ids = [&first_id, &second_id, &third_id]; + assert_eq!(handles.len(), input.len()); + for (handle, index) in handles.iter().zip(order) { + assert_eq!(handle.id(), expected_ids[index]); + assert_eq!(handle.group(), Some(group)); + } + assert_eq!(Snapshot::group_head(group).await.unwrap().head, third_id); + assert_eq!( + Snapshot::open(format!("{group}:cp01")) + .await + .unwrap() + .id() + .as_str(), + first_id + ); + } + // Loading owns the reconstructed artifacts, never a path into an input archive + // or the sender's snapshot directory. + std::fs::remove_dir_all(&source).unwrap(); + for archive in archives { + std::fs::remove_file(archive).unwrap(); + } + for (group, name, expected) in [ + ("reverse", "cp01", b"first disk".as_slice()), + ("reverse", "cp02", b"second disk".as_slice()), + ("shuffled", "cp03", b"third disk".as_slice()), + ] { + let artifact = Snapshot::open(format!("{group}:{name}")).await.unwrap(); + assert_eq!( + std::fs::read(artifact_payload_path(artifact.path())).unwrap(), + expected + ); + } + }) + .await; +} + +#[tokio::test] +async fn load_many_duplicate_inputs_return_input_heads_but_install_once() { + let tmp = TempDir::new().unwrap(); + let home = tmp.path().join("home"); + let backend = isolated_backend(&home).await; + let (source, _) = make_artifact(tmp.path(), "baseline", b"owned bytes"); + microsandbox::with_backend(backend, async { + let archive = save_batch_fixtures(tmp.path(), &[source]).await.remove(0); + let copied_archive = tmp.path().join("identical-copy.msb"); + std::fs::copy(&archive, &copied_archive).unwrap(); + let handles = Snapshot::load_many( + &[archive.clone(), copied_archive, archive], + batch_group_options("work"), + ) + .await + .unwrap(); + assert_eq!(handles.len(), 3); + assert!( + handles + .iter() + .all(|handle| handle.path() == handles[0].path()) + ); + assert_eq!(Snapshot::list().await.unwrap().len(), 1); + }) + .await; +} + +#[tokio::test] +async fn load_many_sibling_batch_preserves_existing_head_and_leaves_new_group_headless() { + let tmp = TempDir::new().unwrap(); + let home = tmp.path().join("home"); + let backend = isolated_backend(&home).await; + let (base, _) = make_artifact(tmp.path(), "base", b"base"); + let base_id = artifact_id(&base); + let (left, _) = make_artifact_with_parent(tmp.path(), "left", b"left", Some(base_id.clone())); + let left_id = artifact_id(&left); + let (right, _) = + make_artifact_with_parent(tmp.path(), "right", b"right", Some(base_id.clone())); + microsandbox::with_backend(backend, async { + let archives = save_batch_fixtures(tmp.path(), &[base, left, right]).await; + Snapshot::load_with_options(&archives[0], batch_group_options("existing")) + .await + .unwrap(); + let siblings = [archives[2].clone(), archives[1].clone()]; + Snapshot::load_many(&siblings, batch_group_options("existing")) + .await + .unwrap(); + assert_eq!( + Snapshot::group_head("existing").await.unwrap().head, + base_id + ); + let handles = Snapshot::load_many(&siblings, batch_group_options("fresh")) + .await + .unwrap(); + assert_eq!(handles.len(), 2); + assert!(handles.iter().all(|handle| handle.head_update().is_none())); + assert!(Snapshot::open("fresh").await.is_err()); + assert_eq!( + Snapshot::open("fresh:left").await.unwrap().id().as_str(), + left_id + ); + assert_eq!( + Snapshot::group_head("fresh:left").await.unwrap().head, + left_id + ); + assert_eq!( + Snapshot::open("fresh").await.unwrap().id().as_str(), + left_id + ); + }) + .await; +} + +#[tokio::test] +async fn load_many_ambiguous_set_head_rejects_before_publishing_members() { + let tmp = TempDir::new().unwrap(); + let home = tmp.path().join("home"); + let backend = isolated_backend(&home).await; + let (left, _) = make_artifact(tmp.path(), "left", b"left"); + let (right, _) = make_artifact(tmp.path(), "right", b"right"); + microsandbox::with_backend(backend, async { + let archives = save_batch_fixtures(tmp.path(), &[left, right]).await; + let mut options = batch_group_options("ambiguous"); + options.set_head = true; + assert!(Snapshot::load_many(&archives, options).await.is_err()); + assert_no_batch_members(&home.join("snapshots/ambiguous")); + assert!(Snapshot::list().await.unwrap().is_empty()); + }) + .await; +} + +#[tokio::test] +async fn load_many_accepts_complete_payload_with_missing_historical_parent() { + let tmp = TempDir::new().unwrap(); + let home = tmp.path().join("home"); + let backend = isolated_backend(&home).await; + let missing = format!("snap_{:032x}", 7); + let (source, _) = + make_artifact_with_parent(tmp.path(), "complete", b"complete payload", Some(missing)); + let id = artifact_id(&source); + microsandbox::with_backend(backend, async { + let archives = save_batch_fixtures(tmp.path(), &[source]).await; + let handles = Snapshot::load_many(&archives, batch_group_options("work")) + .await + .unwrap(); + assert_eq!(handles[0].id(), id); + assert_eq!(Snapshot::group_head("work").await.unwrap().head, id); + assert_eq!( + std::fs::read(artifact_payload_path(handles[0].path())).unwrap(), + b"complete payload" + ); + }) + .await; +} + +#[tokio::test] +async fn load_many_conflicting_aliases_rejects_before_any_member_is_published() { + let tmp = TempDir::new().unwrap(); + let home = tmp.path().join("home"); + let backend = isolated_backend(&home).await; + let (first, _) = make_artifact(&tmp.path().join("one"), "same-name", b"one"); + let (second, _) = make_artifact(&tmp.path().join("two"), "same-name", b"two"); + microsandbox::with_backend(backend, async { + let archives = save_batch_fixtures(tmp.path(), &[first, second]).await; + let error = Snapshot::load_many(&archives, batch_group_options("work")) + .await + .unwrap_err(); + assert!( + error.to_string().contains("conflict"), + "unexpected error: {error}" + ); + assert_no_batch_members(&home.join("snapshots/work")); + }) + .await; +} + +#[tokio::test] +async fn load_many_duplicate_ids_with_conflicting_labels_rejects_before_publication() { + let tmp = TempDir::new().unwrap(); + let home = tmp.path().join("home"); + let backend = isolated_backend(&home).await; + // Keep both descriptor bytes and suggested aliases identical. Labels are the only conflict, + // and reversing the archive order must not silently choose either local metadata sidecar. + let (first, digest) = make_artifact(&tmp.path().join("one"), "same", b"same disk"); + let (second, _) = make_artifact(&tmp.path().join("two"), "same", b"same disk"); + std::fs::copy( + first.join(DESCRIPTOR_FILENAME), + second.join(DESCRIPTOR_FILENAME), + ) + .unwrap(); + for (artifact, label) in [(&first, "first"), (&second, "second")] { + std::fs::write( + artifact.join("metadata.json"), + serde_json::to_vec(&serde_json::json!({ + "schema": "microsandbox.snapshot-metadata/1", + "labels": {"stage": label}, + })) + .unwrap(), + ) + .unwrap(); + } + microsandbox::with_backend(backend, async { + for artifact in [&first, &second] { + assert_eq!( + Snapshot::open(artifact.to_string_lossy().as_ref()) + .await + .unwrap() + .digest(), + digest + ); + } + let archives = save_batch_fixtures(tmp.path(), &[first, second]).await; + for (group, order) in [("forward", [0, 1]), ("reverse", [1, 0])] { + let inputs = order.map(|index| archives[index].clone()); + let error = Snapshot::load_many(&inputs, batch_group_options(group)) + .await + .unwrap_err(); + assert!(error.to_string().contains("conflicting labels"), "{error}"); + assert_no_batch_members(&home.join("snapshots").join(group)); + } + assert!(Snapshot::list().await.unwrap().is_empty()); + }) + .await; +} + +#[tokio::test] +async fn load_many_conflicting_ids_rejects_before_any_member_is_published() { + let tmp = TempDir::new().unwrap(); + let home = tmp.path().join("home"); + let backend = isolated_backend(&home).await; + let (first, _) = make_artifact(tmp.path(), "first", b"one"); + let (second, _) = make_artifact(tmp.path(), "second", b"two"); + let mut descriptor = + Manifest::from_bytes(&std::fs::read(second.join(DESCRIPTOR_FILENAME)).unwrap()).unwrap(); + descriptor.snapshot_id = SnapshotId::new(artifact_id(&first)).unwrap(); + std::fs::write( + second.join(DESCRIPTOR_FILENAME), + descriptor.to_canonical_bytes().unwrap(), + ) + .unwrap(); + microsandbox::with_backend(backend, async { + let archives = save_batch_fixtures(tmp.path(), &[first, second]).await; + assert!( + Snapshot::load_many(&archives, batch_group_options("work")) + .await + .is_err() + ); + assert_no_batch_members(&home.join("snapshots/work")); + }) + .await; +} + +#[tokio::test] +async fn load_many_corrupt_later_archive_never_publishes_valid_earlier_member() { + let tmp = TempDir::new().unwrap(); + let home = tmp.path().join("home"); + let backend = isolated_backend(&home).await; + let (first, _) = make_artifact(tmp.path(), "first", b"one"); + let (second, _) = make_artifact(tmp.path(), "second", b"two"); + microsandbox::with_backend(backend, async { + let archives = save_batch_fixtures(tmp.path(), &[first, second]).await; + corrupt_dense_tar_member(&archives[1], ".raw"); + let error = Snapshot::load_many(&archives, batch_group_options("work")) + .await + .unwrap_err(); + assert!( + error.to_string().contains("integrity"), + "unexpected error: {error}" + ); + assert_no_batch_members(&home.join("snapshots/work")); + }) + .await; +} + +#[tokio::test] +async fn load_many_single_legacy_archive_preserves_single_load_compatibility() { + let tmp = TempDir::new().unwrap(); + let home = tmp.path().join("home"); + let backend = isolated_backend(&home).await; + let archive = tmp.path().join("legacy.tar"); + write_v066_archive(&archive, "sha256-0123456789abcdef", b"legacy disk"); + microsandbox::with_backend(backend, async { + let batch = Snapshot::load_many(&[archive.clone()], batch_group_options("batch")) + .await + .unwrap(); + let single = Snapshot::load_with_options(&archive, batch_group_options("single")) + .await + .unwrap(); + assert_eq!(batch.len(), 1); + assert_eq!(batch[0].id(), single.id()); + assert_eq!( + std::fs::read(artifact_payload_path(batch[0].path())).unwrap(), + b"legacy disk" + ); + }) + .await; +} + #[tokio::test] async fn save_sparse_upper_round_trips_and_preserves_holes() { let tmp = TempDir::new().unwrap(); @@ -1408,7 +1926,7 @@ async fn load_selects_child_head_when_parents_are_present() { let (parent_dir, _) = make_artifact(&snapshots_dir, "parent", b"parent"); let parent_id = artifact_id(&parent_dir); let (child_dir, child_digest) = - make_artifact_with_parent(&snapshots_dir, "child", b"child", Some(parent_id)); + make_artifact_with_parent(&snapshots_dir, "child", b"child", Some(parent_id.clone())); let child_id = artifact_id(&child_dir); let archive = tmp.path().join("chain.tar"); let dest = tmp.path().join("imported-chain"); @@ -1438,7 +1956,17 @@ async fn load_selects_child_head_when_parents_are_present() { .await; assert_eq!(handle.digest(), child_digest); assert_eq!(handle.id(), child_id); - assert_eq!(handle.path(), dest.join(child_id)); + let imported_group = dest.join(handle.group().expect("load creates a local group")); + assert_eq!(handle.path(), imported_group.join(&child_id)); + assert_eq!(handle.head_update().unwrap().head, child_id); + assert_eq!(handle.head_update().unwrap().previous, None); + assert!( + imported_group + .join(parent_id) + .join(DESCRIPTOR_FILENAME) + .is_file() + ); + assert_eq!(Snapshot::list_dir(&imported_group).await.unwrap().len(), 2); } #[tokio::test] @@ -1531,8 +2059,8 @@ async fn failed_load_with_conflicting_cache_target_does_not_install_cache_entrie .await; assert!( - !dest.join("src-cache-conflict").exists(), - "failed import promoted staged snapshot" + Snapshot::list_dir(&dest).await.unwrap().is_empty(), + "failed import promoted a grouped snapshot" ); assert_eq!( std::fs::read(&conflicting_metadata).unwrap(), @@ -1602,7 +2130,7 @@ async fn create_full_resolves_source_before_touching_anything() { }) .await; - assert!(!home.join("snapshots").join("warm").exists()); + assert!(!home.join("snapshots").join("box").exists()); } #[tokio::test] @@ -1673,10 +2201,14 @@ async fn replacing_child_in_place_does_not_inflate_parent_child_count() { b"child v2 with different size", Some(parent_id), ); - Snapshot::open("child").await.unwrap(); + Snapshot::open(cdir.to_string_lossy().as_ref()) + .await + .unwrap(); - Snapshot::remove("child", false).await.unwrap(); - Snapshot::remove("parent", false) + Snapshot::remove(cdir.to_string_lossy().as_ref(), false) + .await + .unwrap(); + Snapshot::remove(pdir.to_string_lossy().as_ref(), false) .await .expect("parent should be removable once its only child is gone"); }) diff --git a/vendor/libkrunfw b/vendor/libkrunfw index 4b334c292..6cca413ac 160000 --- a/vendor/libkrunfw +++ b/vendor/libkrunfw @@ -1 +1 @@ -Subproject commit 4b334c292ebae7364d9413d32cd57ce510a99e2d +Subproject commit 6cca413ac248f63e65d4ea4748b3bc36cd1b22f3