From af9bfa43e348dc533e5c09b7255a8fe21654e5b6 Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Mon, 7 Sep 2026 03:23:54 +0100 Subject: [PATCH 01/29] feat(snapshot): prepare immutable memory cache and msnap naming Use published msb_krun 0.1.33 in place of the superseded stack override. Add atomic, pinned, read-only memory cache realizations keyed by complete manifests. Cover warm reuse, concurrent publication, failed construction, alignment, eviction, and unlink lifetime without exposing CoW policy yet. Prefer .msnap in CLI help and SDK documentation while preserving explicit filenames and content-detected legacy archives. Exercise eight encoding and suffix combinations through direct capture and child staging. This is stack-8 groundwork; public CoW and pause/resume integration and end-to-end qualification remain pending. --- Cargo.lock | 55 +- Cargo.toml | 18 +- crates/cli/lib/commands/snapshot.rs | 6 +- crates/runtime/lib/checkpoint/memory_cache.rs | 471 ++++++++++++++++++ crates/runtime/lib/checkpoint/mod.rs | 2 + docs/sandboxes/snapshots.mdx | 36 +- docs/sdk/go/snapshots.mdx | 16 +- docs/sdk/python/snapshots.mdx | 22 +- docs/sdk/rust/snapshots.mdx | 18 +- docs/sdk/typescript/snapshots.mdx | 16 +- sdk/rust/lib/snapshot/archive.rs | 62 ++- sdk/rust/lib/snapshot/mod.rs | 5 +- 12 files changed, 607 insertions(+), 120 deletions(-) create mode 100644 crates/runtime/lib/checkpoint/memory_cache.rs diff --git a/Cargo.lock b/Cargo.lock index 92ac978cd..61e33bfe2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4352,8 +4352,9 @@ dependencies = [ [[package]] name = "msb_krun" -version = "0.1.32" -source = "git+https://github.com/superradcompany/libkrun?rev=ec9f119dcc144f1c011fe62ebd224d660c2191a3#ec9f119dcc144f1c011fe62ebd224d660c2191a3" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb69e940b9aea22169f0882108415dd309a1344803943e466652916002f7f680" dependencies = [ "crossbeam-channel", "kvm-bindings", @@ -4371,8 +4372,9 @@ dependencies = [ [[package]] name = "msb_krun_arch" -version = "0.1.32" -source = "git+https://github.com/superradcompany/libkrun?rev=ec9f119dcc144f1c011fe62ebd224d660c2191a3#ec9f119dcc144f1c011fe62ebd224d660c2191a3" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc8b11bc3e2829adcbe00eea6bee312101a987d8b800fe6c30d19f4fb4d9f18e" dependencies = [ "kvm-bindings", "kvm-ioctls", @@ -4385,13 +4387,15 @@ dependencies = [ [[package]] name = "msb_krun_arch_gen" -version = "0.1.32" -source = "git+https://github.com/superradcompany/libkrun?rev=ec9f119dcc144f1c011fe62ebd224d660c2191a3#ec9f119dcc144f1c011fe62ebd224d660c2191a3" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d935e248ac8ee4e527c82188cb043c9333f58cce8e5259d4b4367d30a4585649" [[package]] name = "msb_krun_cpuid" -version = "0.1.32" -source = "git+https://github.com/superradcompany/libkrun?rev=ec9f119dcc144f1c011fe62ebd224d660c2191a3#ec9f119dcc144f1c011fe62ebd224d660c2191a3" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75d8b5f47e7e11381832d7d1a1c803a27e8cef2e2dc551e2a172339f23d9d414" dependencies = [ "kvm-bindings", "kvm-ioctls", @@ -4400,8 +4404,9 @@ dependencies = [ [[package]] name = "msb_krun_devices" -version = "0.1.32" -source = "git+https://github.com/superradcompany/libkrun?rev=ec9f119dcc144f1c011fe62ebd224d660c2191a3#ec9f119dcc144f1c011fe62ebd224d660c2191a3" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e20026edb2f255659010d659c08ed8a84b67d549440ef81d5a745f1453fd6a" dependencies = [ "bincode", "bitflags 1.3.2", @@ -4431,8 +4436,9 @@ dependencies = [ [[package]] name = "msb_krun_hvf" -version = "0.1.32" -source = "git+https://github.com/superradcompany/libkrun?rev=ec9f119dcc144f1c011fe62ebd224d660c2191a3#ec9f119dcc144f1c011fe62ebd224d660c2191a3" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8de9d80864e34ee1845155c24ab95205907f71a587c562f603d9719d1aed697" dependencies = [ "crossbeam-channel", "libloading 0.8.9", @@ -4443,8 +4449,9 @@ dependencies = [ [[package]] name = "msb_krun_kernel" -version = "0.1.32" -source = "git+https://github.com/superradcompany/libkrun?rev=ec9f119dcc144f1c011fe62ebd224d660c2191a3#ec9f119dcc144f1c011fe62ebd224d660c2191a3" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2afad41ea9e1719499bb5c375f8b7ee1ad24d0dd6f2a54b494fdd442a95ccc9b" dependencies = [ "msb-vm-memory", "msb_krun_utils", @@ -4452,8 +4459,9 @@ dependencies = [ [[package]] name = "msb_krun_polly" -version = "0.1.32" -source = "git+https://github.com/superradcompany/libkrun?rev=ec9f119dcc144f1c011fe62ebd224d660c2191a3#ec9f119dcc144f1c011fe62ebd224d660c2191a3" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40ddcf4dfce736d7c1103ea38a24c5de57a76f06b84357347ec87f487d3f7be3" dependencies = [ "libc", "msb_krun_utils", @@ -4461,16 +4469,18 @@ dependencies = [ [[package]] name = "msb_krun_smbios" -version = "0.1.32" -source = "git+https://github.com/superradcompany/libkrun?rev=ec9f119dcc144f1c011fe62ebd224d660c2191a3#ec9f119dcc144f1c011fe62ebd224d660c2191a3" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22d46149e9eeb80c379b7c0419946688861da8e76ace11a0f387a3adfde1f3ff" dependencies = [ "msb-vm-memory", ] [[package]] name = "msb_krun_utils" -version = "0.1.32" -source = "git+https://github.com/superradcompany/libkrun?rev=ec9f119dcc144f1c011fe62ebd224d660c2191a3#ec9f119dcc144f1c011fe62ebd224d660c2191a3" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98b8923a8109908e51a17a7cec55056f58272f63058a62543f01104317fbaa25" dependencies = [ "bitflags 1.3.2", "crossbeam-channel", @@ -4484,8 +4494,9 @@ dependencies = [ [[package]] name = "msb_krun_vmm" -version = "0.1.32" -source = "git+https://github.com/superradcompany/libkrun?rev=ec9f119dcc144f1c011fe62ebd224d660c2191a3#ec9f119dcc144f1c011fe62ebd224d660c2191a3" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15c27ec272f4686597c41e452f618610e59bbdc84a7073c7f93cbcf721b33b33" dependencies = [ "bincode", "bzip2", diff --git a/Cargo.toml b/Cargo.toml index bcaae55ea..8176dcbdf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -88,8 +88,8 @@ microsandbox-protocol = { version = "=0.6.15", path = "crates/protocol" } microsandbox-runtime = { version = "=0.6.15", path = "crates/runtime", default-features = false } microsandbox-utils = { version = "=0.6.15", path = "crates/utils" } microsandbox-vsock = { version = "=0.6.15", path = "crates/vsock" } -msb_krun = "=0.1.32" -msb_krun_utils = "=0.1.32" +msb_krun = "=0.1.33" +msb_krun_utils = "=0.1.33" test-macros = { path = "crates/testing/macros" } test-utils = { path = "crates/testing/utils" } @@ -205,17 +205,3 @@ parking_lot = "0.12" rpassword = "7" russh = "0.62.4" russh-sftp = "2.3.0" - -# Keep this stack reproducible until the block-capacity API is released. -[patch.crates-io] -msb_krun = { git = "https://github.com/superradcompany/libkrun", rev = "ec9f119dcc144f1c011fe62ebd224d660c2191a3" } -msb_krun_arch = { git = "https://github.com/superradcompany/libkrun", rev = "ec9f119dcc144f1c011fe62ebd224d660c2191a3" } -msb_krun_arch_gen = { git = "https://github.com/superradcompany/libkrun", rev = "ec9f119dcc144f1c011fe62ebd224d660c2191a3" } -msb_krun_cpuid = { git = "https://github.com/superradcompany/libkrun", rev = "ec9f119dcc144f1c011fe62ebd224d660c2191a3" } -msb_krun_devices = { git = "https://github.com/superradcompany/libkrun", rev = "ec9f119dcc144f1c011fe62ebd224d660c2191a3" } -msb_krun_hvf = { git = "https://github.com/superradcompany/libkrun", rev = "ec9f119dcc144f1c011fe62ebd224d660c2191a3" } -msb_krun_kernel = { git = "https://github.com/superradcompany/libkrun", rev = "ec9f119dcc144f1c011fe62ebd224d660c2191a3" } -msb_krun_polly = { git = "https://github.com/superradcompany/libkrun", rev = "ec9f119dcc144f1c011fe62ebd224d660c2191a3" } -msb_krun_smbios = { git = "https://github.com/superradcompany/libkrun", rev = "ec9f119dcc144f1c011fe62ebd224d660c2191a3" } -msb_krun_utils = { git = "https://github.com/superradcompany/libkrun", rev = "ec9f119dcc144f1c011fe62ebd224d660c2191a3" } -msb_krun_vmm = { git = "https://github.com/superradcompany/libkrun", rev = "ec9f119dcc144f1c011fe62ebd224d660c2191a3" } diff --git a/crates/cli/lib/commands/snapshot.rs b/crates/cli/lib/commands/snapshot.rs index 8c0d1fd77..4e5fdb9c4 100644 --- a/crates/cli/lib/commands/snapshot.rs +++ b/crates/cli/lib/commands/snapshot.rs @@ -40,7 +40,7 @@ pub enum SnapshotCommands { /// Rebuild the local index from artifacts on disk. Reindex(SnapshotReindexArgs), - /// Save a snapshot into a `.tar.zst` archive. + /// Save a snapshot into a `.msnap` archive (tar + zstd). Save(SnapshotSaveArgs), /// Load a snapshot archive into the snapshots directory. @@ -151,7 +151,7 @@ pub struct SnapshotSaveArgs { /// Snapshot to save (path, name, or digest). pub snapshot: String, - /// Output archive path (`.tar.zst` recommended). + /// Output archive path (`.msnap` recommended; explicit filenames are preserved). pub out: std::path::PathBuf, /// Walk the parent chain and include each ancestor in the archive. @@ -163,7 +163,7 @@ pub struct SnapshotSaveArgs { #[arg(long)] pub with_image: bool, - /// Write a plain `.tar` instead of `.tar.zst`. Tradeoff: smaller + /// Write plain tar instead of zstd-compressed tar. Tradeoff: smaller /// CPU but much larger file for sparse uppers. #[arg(long)] pub plain_tar: bool, diff --git a/crates/runtime/lib/checkpoint/memory_cache.rs b/crates/runtime/lib/checkpoint/memory_cache.rs new file mode 100644 index 000000000..3d90158b7 --- /dev/null +++ b/crates/runtime/lib/checkpoint/memory_cache.rs @@ -0,0 +1,471 @@ +//! Immutable local realizations of complete portable memory manifests. +//! +//! The cache is trusted host storage, not a second portable snapshot format. Publication is +//! atomic; readers retain read-only handles, so removing a snapshot never invalidates live RAM. + +use std::collections::BTreeMap; +use std::fs::{File, OpenOptions}; +use std::io::{self, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; +use std::time::Instant; + +use microsandbox_image::checkpoint::{MemoryExtentContent, MemoryManifest, ObjectId}; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// One native-aligned, contiguous guest address span in a flat cache file. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CachedMemoryRegion { + /// Start of the guest physical span. + pub guest_address: u64, + /// Length of the span in bytes. + pub length: u64, + /// Byte offset in the immutable cache file. + pub file_offset: u64, +} + +/// An opened realization pinned against cooperative eviction until the handle is dropped. +pub struct CachedMemory { + /// Read-only backing ownership. Transfer this handle to the VMM, not merely its pathname. + pub file: File, + /// Exact guest coverage, with address holes omitted from physical storage. + pub regions: Vec, + /// Whether existing verified bytes were reused without rereading portable objects. + pub cache_hit: bool, + /// Time spent resolving or constructing this backing, in microseconds. + pub prepare_us: u128, +} + +/// Host-local, immutable memory cache. No entry is ever modified in place. +pub struct MemoryCache { + root: PathBuf, + page_size: u64, +} + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl MemoryCache { + /// Open a dedicated cache directory using this host's native mapping alignment. + pub fn open(root: impl Into) -> io::Result { + #[cfg(unix)] + { + let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + if page_size <= 0 { + return Err(io::Error::last_os_error()); + } + let root = root.into(); + std::fs::create_dir_all(&root)?; + Ok(Self { + root, + page_size: page_size as u64, + }) + } + #[cfg(not(unix))] + { + let _ = root; + Err(io::Error::new( + io::ErrorKind::Unsupported, + "private memory cache is not qualified on this backend", + )) + } + } + + /// Resolve a complete memory image, verifying portable objects only on a cache miss. + /// + /// `read_object` must return identity-verified bytes. It is called once per distinct packed + /// object, not once per extent. The complete canonical manifest identity names the cache; + /// neither an unverified partial delta nor a mutable file may be published under that name. + pub fn materialize( + &self, + manifest: &MemoryManifest, + identity: &ObjectId, + mut read_object: impl FnMut(&ObjectId) -> io::Result>, + ) -> io::Result { + let started = Instant::now(); + let canonical = manifest.to_canonical_bytes().map_err(io::Error::other)?; + if ObjectId::from_bytes(&canonical).map_err(io::Error::other)? != *identity { + return Err(invalid( + "memory cache identity does not match its complete manifest", + )); + } + let regions = memory_regions(manifest, self.page_size)?; + let length = regions + .last() + .and_then(|r| r.file_offset.checked_add(r.length)) + .ok_or_else(|| invalid("empty or overflowing memory topology"))?; + let path = self.entry_path(identity); + if let Some(file) = open_pinned(&path, length)? { + return Ok(CachedMemory { + file, + regions, + cache_hit: true, + prepare_us: started.elapsed().as_micros(), + }); + } + + let mut staging = tempfile::Builder::new() + .prefix(".memory-") + .tempfile_in(&self.root)?; + // A fresh sparse file supplies all zero extents without allocating or writing RAM-sized + // buffers. Only immutable nonzero object slices are copied into it. + staging.as_file().set_len(length)?; + let mut objects = BTreeMap::>::new(); + let mut region_index = 0; + for extent in &manifest.extents { + while extent.start >= regions[region_index].guest_address + regions[region_index].length + { + region_index += 1; + } + if let MemoryExtentContent::Object(content) = &extent.content { + let region = ®ions[region_index]; + let offset = region.file_offset + (extent.start - region.guest_address); + objects.entry(content.object.clone()).or_default().push(( + offset, + content.object_offset, + extent.length, + )); + } + } + for (id, slices) in objects { + let bytes = read_object(&id)?; + for (target, offset, count) in slices { + let start = usize::try_from(offset) + .map_err(|_| invalid("memory object offset overflows"))?; + let count = usize::try_from(count) + .map_err(|_| invalid("memory object length overflows"))?; + let end = start + .checked_add(count) + .ok_or_else(|| invalid("memory object slice overflows"))?; + let bytes = bytes + .get(start..end) + .ok_or_else(|| invalid("memory object slice exceeds verified bytes"))?; + staging.seek(SeekFrom::Start(target))?; + staging.write_all(bytes)?; + } + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + staging + .as_file() + .set_permissions(std::fs::Permissions::from_mode(0o400))?; + } + staging.as_file().sync_all()?; + // Publish the inode without replacement. Concurrent builders may do duplicate work, but + // no winner can overwrite backing another VM has already pinned or mapped. + match staging.persist_noclobber(&path) { + Ok(file) => drop(file), + Err(error) if error.error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error.error), + } + #[cfg(unix)] + File::open(&self.root)?.sync_all()?; + let file = open_pinned(&path, length)?.ok_or_else(|| { + io::Error::new( + io::ErrorKind::NotFound, + "memory cache was evicted before it could be pinned; retry restore", + ) + })?; + Ok(CachedMemory { + file, + regions, + cache_hit: false, + prepare_us: started.elapsed().as_micros(), + }) + } + + /// Remove an unpinned immutable entry. `false` means absent or still owned by a VM. + /// + /// Never truncate or hole-punch a live entry. POSIX open-handle lifetime also protects a + /// reader that opened the inode immediately before an eviction acquired its exclusive lock. + pub fn evict(&self, identity: &ObjectId) -> io::Result { + let path = self.entry_path(identity); + let file = match open_readonly(&path) { + Ok(file) => file, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error), + }; + if !microsandbox_utils::process_lock::try_lock_exclusive(&file)? { + return Ok(false); + } + // A competing evictor can have removed this same inode while we waited to acquire it. + // Do not unlink a new realization published at the old name in the meantime. + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let opened = file.metadata()?; + let current = match std::fs::symlink_metadata(&path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error), + }; + if (opened.dev(), opened.ino()) != (current.dev(), current.ino()) { + return Ok(false); + } + } + std::fs::remove_file(path)?; + Ok(true) + } + + fn entry_path(&self, identity: &ObjectId) -> PathBuf { + // Geometry lives in the identity-bearing manifest. Native alignment is local realization + // policy, so a cache prepared on a different page-size host must not collide with it. + self.root.join(format!( + "{}-{}.ram", + identity.as_str().replace(':', "-"), + self.page_size + )) + } +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +fn memory_regions( + manifest: &MemoryManifest, + page_size: u64, +) -> io::Result> { + let mut regions: Vec = Vec::new(); + let mut file_length = 0u64; + for extent in &manifest.extents { + if let Some(last) = regions.last_mut() + && last.guest_address.checked_add(last.length) == Some(extent.start) + { + last.length = last + .length + .checked_add(extent.length) + .ok_or_else(|| invalid("memory topology overflows"))?; + } else { + regions.push(CachedMemoryRegion { + guest_address: extent.start, + length: extent.length, + file_offset: file_length, + }); + } + file_length = file_length + .checked_add(extent.length) + .ok_or_else(|| invalid("memory cache size overflows"))?; + } + for region in ®ions { + if !region.guest_address.is_multiple_of(page_size) + || !region.length.is_multiple_of(page_size) + || !region.file_offset.is_multiple_of(page_size) + { + return Err(invalid( + "guest memory topology is not aligned for private mappings on this host", + )); + } + } + Ok(regions) +} + +fn open_readonly(path: &Path) -> io::Result { + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK); + } + options.open(path) +} + +fn open_pinned(path: &Path, length: u64) -> io::Result> { + let file = match open_readonly(path) { + Ok(file) => file, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + let metadata = file.metadata()?; + if !metadata.is_file() || metadata.len() != length { + return Err(invalid( + "memory cache entry has invalid type or length; evict and rebuild it", + )); + } + #[cfg(unix)] + { + use std::os::fd::AsRawFd; + loop { + if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_SH) } == 0 { + break; + } + let error = io::Error::last_os_error(); + if error.kind() != io::ErrorKind::Interrupted { + return Err(error); + } + } + } + Ok(Some(file)) +} + +fn invalid(message: &str) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, message) +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(all(test, unix))] +mod tests { + use super::*; + use microsandbox_image::checkpoint::{ContentRef, MemoryCaptureMode, MemoryExtent}; + use std::os::unix::fs::FileExt; + + fn fixture(page: u64) -> (MemoryManifest, ObjectId, Vec) { + let bytes = vec![0x5a; page as usize]; + let object = ObjectId::from_bytes(&bytes).unwrap(); + let manifest = MemoryManifest { + schema: "microsandbox.memory/1".into(), + architecture: std::env::consts::ARCH.into(), + guest_page_size: 4096, + topology_generation: 1, + generation: 1, + capture_mode: MemoryCaptureMode::Full, + pause_generation: 1, + extents: vec![ + MemoryExtent { + start: 0, + length: page, + content: MemoryExtentContent::Object(ContentRef { + object: object.clone(), + object_offset: 0, + }), + }, + MemoryExtent { + start: page, + length: page, + content: MemoryExtentContent::Zero, + }, + MemoryExtent { + start: page * 4, + length: page, + content: MemoryExtentContent::Object(ContentRef { + object, + object_offset: 0, + }), + }, + ], + }; + let id = ObjectId::from_bytes(&manifest.to_canonical_bytes().unwrap()).unwrap(); + (manifest, id, bytes) + } + + #[test] + fn materialize_once_reuse_pinned_bytes_and_evict_after_last_owner() { + let directory = tempfile::tempdir().unwrap(); + let cache = MemoryCache::open(directory.path()).unwrap(); + let (manifest, id, bytes) = fixture(cache.page_size); + let mut reads = 0; + let first = cache + .materialize(&manifest, &id, |_| { + reads += 1; + Ok(bytes.clone()) + }) + .unwrap(); + assert_eq!(reads, 1); + assert!(!first.cache_hit); + assert_eq!(first.regions.len(), 2); + assert_eq!(first.file.metadata().unwrap().len(), cache.page_size * 3); + let mut zero = vec![1; cache.page_size as usize]; + first + .file + .read_exact_at(&mut zero, cache.page_size) + .unwrap(); + assert!(zero.iter().all(|byte| *byte == 0)); + let second = cache + .materialize(&manifest, &id, |_| { + panic!("warm cache reread a portable object") + }) + .unwrap(); + assert!(second.cache_hit); + assert!(!cache.evict(&id).unwrap()); + drop(first); + assert!(!cache.evict(&id).unwrap()); + drop(second); + assert!(cache.evict(&id).unwrap()); + assert!(!cache.evict(&id).unwrap()); + } + + #[test] + fn failed_materialization_does_not_publish_or_leave_staging() { + let directory = tempfile::tempdir().unwrap(); + let cache = MemoryCache::open(directory.path()).unwrap(); + let (manifest, id, _) = fixture(cache.page_size); + let failure = cache.materialize(&manifest, &id, |_| { + Err(io::Error::other("injected object read failure")) + }); + assert!(failure.is_err()); + assert_eq!(std::fs::read_dir(directory.path()).unwrap().count(), 0); + assert!(cache.materialize(&manifest, &id, |_| Ok(vec![])).is_err()); + assert_eq!(std::fs::read_dir(directory.path()).unwrap().count(), 0); + } + + #[test] + fn reject_wrong_manifest_identity_and_host_alignment() { + let directory = tempfile::tempdir().unwrap(); + let cache = MemoryCache::open(directory.path()).unwrap(); + let (mut manifest, id, _) = fixture(cache.page_size); + manifest.generation += 1; + assert!( + cache + .materialize(&manifest, &id, |_| panic!( + "identity rejection must precede reads" + )) + .is_err() + ); + let (mut manifest, _, _) = fixture(4096); + manifest.extents.truncate(1); + assert!(memory_regions(&manifest, 16384).is_err()); + } + + #[test] + fn concurrent_builders_publish_one_immutable_inode() { + use std::os::unix::fs::MetadataExt; + let directory = tempfile::tempdir().unwrap(); + let cache = MemoryCache::open(directory.path()).unwrap(); + let (manifest, id, bytes) = fixture(cache.page_size); + let barrier = std::sync::Barrier::new(2); + std::thread::scope(|scope| { + let run = || { + cache + .materialize(&manifest, &id, |_| { + barrier.wait(); + Ok(bytes.clone()) + }) + .unwrap() + }; + let first = scope.spawn(run); + let second = scope.spawn(run); + let first = first.join().unwrap(); + let second = second.join().unwrap(); + assert_eq!( + first.file.metadata().unwrap().ino(), + second.file.metadata().unwrap().ino() + ); + }); + assert_eq!(std::fs::read_dir(directory.path()).unwrap().count(), 1); + } + + #[test] + fn unlinked_backing_survives_without_snapshot_paths() { + let directory = tempfile::tempdir().unwrap(); + let cache = MemoryCache::open(directory.path()).unwrap(); + let (manifest, id, bytes) = fixture(cache.page_size); + let memory = cache + .materialize(&manifest, &id, |_| Ok(bytes.clone())) + .unwrap(); + std::fs::remove_file(cache.entry_path(&id)).unwrap(); + let mut actual = vec![0; bytes.len()]; + memory + .file + .read_exact_at(&mut actual, cache.page_size * 2) + .unwrap(); + assert_eq!(actual, bytes); + } +} diff --git a/crates/runtime/lib/checkpoint/mod.rs b/crates/runtime/lib/checkpoint/mod.rs index a7b947bce..acd28fc7d 100644 --- a/crates/runtime/lib/checkpoint/mod.rs +++ b/crates/runtime/lib/checkpoint/mod.rs @@ -2,6 +2,7 @@ mod coordinator; mod disk; +mod memory_cache; mod restore; //-------------------------------------------------------------------------------------------------- @@ -14,6 +15,7 @@ pub use disk::{ DiskCompactionResult, RuntimeOwnedRootChain, RuntimeOwnedRootLayer, compact_stopped_root, grow_stopped_root, load_runtime_owned_root_chain, recover_stopped_root_growth, }; +pub use memory_cache::{CachedMemory, CachedMemoryRegion, MemoryCache}; pub(crate) use restore::{PreparedCheckpointRestore, RestoredAgentState}; //-------------------------------------------------------------------------------------------------- diff --git a/docs/sandboxes/snapshots.mdx b/docs/sandboxes/snapshots.mdx index 26af4df6b..af8bba3a2 100644 --- a/docs/sandboxes/snapshots.mdx +++ b/docs/sandboxes/snapshots.mdx @@ -6,7 +6,7 @@ icon: "code-branch" Local-only -A snapshot is a portable artifact that can hold either a sandbox's writable disk state or a full checkpoint of a running sandbox. Managed and flat OCI roots are supported; the descriptor preserves the root layout so a flat `rootfs.raw` is never mistaken for a managed OverlayFS upper. Move it with `scp`, archive it as `.tar.zst`, or create a child sandbox from it. +A snapshot is a portable artifact that can hold either a sandbox's writable disk state or a full checkpoint of a running sandbox. Managed and flat OCI roots are supported; the descriptor preserves the root layout so a flat `rootfs.raw` is never mistaken for a managed OverlayFS upper. Move it with `scp`, archive it as `.msnap`, or create a child sandbox from it. Disk snapshots capture stopped or crashed sandboxes. Full snapshots use `--full` and capture a running sandbox without requiring the user to pause it first. @@ -14,6 +14,8 @@ Disk snapshots capture stopped or crashed sandboxes. Full snapshots use `--full` ## What gets captured +`.msnap` is a tar + zstd snapshot archive. Older `.tar.zst` and `.tar` names still work, and explicit output filenames are kept as given. Disk-only, full, and incremental exports use the same extension; the archive records what it contains. + | Mode | Source | Captured state | Restore behavior | | ---- | ------ | -------------- | ---------------- | | Disk (default) | Stopped or crashed sandbox | Writable disk closure and pinned image | Cold-boots a fresh VM | @@ -156,7 +158,7 @@ Repeated full checkpoints add disk layers. You choose when to export changes and ```bash msb snapshot create checkpoint-b --from worker --full -msb snapshot save checkpoint-b changes.tar.zst --since checkpoint-a +msb snapshot save checkpoint-b changes.msnap --since checkpoint-a msb modify worker --compact --layers 3 --dry-run msb modify worker --compact --layers 3 ``` @@ -166,8 +168,8 @@ msb modify worker --compact --layers 3 `--since` requires an exact physical-prefix base. Alternatively, `--last-layers 2` includes the newest two sealed checkpoint layers. A full checkpoint export still includes all required memory and device state. Unlike ordinary standalone exports, these smaller archives require the omitted base when loading or restoring: ```bash -msb snapshot load changes.tar.zst --base checkpoint-a -msb create --name child --from-snapshot changes.tar.zst --snapshot-base checkpoint-a +msb snapshot load changes.msnap --base checkpoint-a +msb create --name child --from-snapshot changes.msnap --snapshot-base checkpoint-a ``` The base can also be a standalone snapshot archive. Restore copies the required closure into child-owned storage, without installing an intermediate snapshot. Add `--disk-only` to cold-boot only disk state. After compaction, export a new standalone baseline before resuming incremental exports: the old physical prefix no longer matches. Do not combine compaction with unrelated `modify` options, or incremental export with `--with-parents`. @@ -182,7 +184,7 @@ use microsandbox::Snapshot; let archive = Snapshot::builder("after-pip-install") .from_sandbox("baseline") - .create_archive("/tmp/after-pip-install.tar.zst", false) + .create_archive("/tmp/after-pip-install.msnap", false) .await?; ``` @@ -191,7 +193,7 @@ import { Snapshot } from "microsandbox"; const archive = await Snapshot.builder("after-pip-install") .fromSandbox("baseline") - .createArchive("/tmp/after-pip-install.tar.zst"); + .createArchive("/tmp/after-pip-install.msnap"); ``` ```python Python @@ -199,7 +201,7 @@ from microsandbox import Snapshot archive = await Snapshot.create_archive( "after-pip-install", - "/tmp/after-pip-install.tar.zst", + "/tmp/after-pip-install.msnap", from_sandbox="baseline", ) ``` @@ -210,14 +212,14 @@ archive, err := m.Snapshot.CreateArchive(ctx, m.SnapshotArchiveOptions{ Name: "after-pip-install", FromSandbox: "baseline", }, - ArchivePath: "/tmp/after-pip-install.tar.zst", + ArchivePath: "/tmp/after-pip-install.msnap", }) ``` ```bash CLI msb snapshot create after-pip-install \ --from baseline \ - --archive /tmp/after-pip-install.tar.zst + --archive /tmp/after-pip-install.msnap ``` @@ -226,7 +228,7 @@ Direct capture publishes only the archive and returns its snapshot ID and path. An explicit archive path can also be used directly as the source of a new sandbox: ```bash -msb run --name worker --from-snapshot ./after-pip-install.tar.zst -- python -V +msb run --name worker --from-snapshot ./after-pip-install.msnap -- python -V ``` The same archive path works with the SDK restore methods shown below. The archive is unpacked into child-owned staging, so no intermediate installed snapshot is loaded into `~/.microsandbox/snapshots`. @@ -395,17 +397,17 @@ The snapshot directory is the whole artifact; there is no hidden daemon state. C scp -r ~/.microsandbox/snapshots/after-pip-install \ other-host:~/.microsandbox/snapshots/ -# Bundle into a .tar.zst, transport, then load -msb snapshot save after-pip-install /tmp/snap.tar.zst -scp /tmp/snap.tar.zst other-host: -ssh other-host msb snapshot load /tmp/snap.tar.zst +# Bundle into a .msnap, transport, then load +msb snapshot save after-pip-install /tmp/snap.msnap +scp /tmp/snap.msnap other-host: +ssh other-host msb snapshot load /tmp/snap.msnap # Fully offline: include the OCI image cache so the target needs no network -msb snapshot save after-pip-install /tmp/snap.tar.zst --with-image -ssh other-host msb snapshot load /tmp/snap.tar.zst +msb snapshot save after-pip-install /tmp/snap.msnap --with-image +ssh other-host msb snapshot load /tmp/snap.msnap ``` -Archives default to `.tar.zst`. Pass `--plain-tar` for a plain `.tar`. SDKs expose the same save and load operations as the CLI. +Archives use tar + zstd by default; `.msnap` is the recommended filename extension. Pass `--plain-tar` for uncompressed tar. SDKs expose the same save and load operations as the CLI. ## Artifact identity and layout diff --git a/docs/sdk/go/snapshots.mdx b/docs/sdk/go/snapshots.mdx index bcabe305f..f723978cb 100644 --- a/docs/sdk/go/snapshots.mdx +++ b/docs/sdk/go/snapshots.mdx @@ -15,12 +15,12 @@ if err != nil { return err } layers := uint32(3) result, err := worker.Compact(ctx, m.DiskCompactionOptions{Layers: &layers}) if err != nil { return err } -err = m.Snapshot.Save(ctx, "checkpoint-b", "changes.tar.zst", m.SnapshotSaveOptions{Since: "checkpoint-a"}) +err = m.Snapshot.Save(ctx, "checkpoint-b", "changes.msnap", m.SnapshotSaveOptions{Since: "checkpoint-a"}) if err != nil { return err } -snapshot, err := m.Snapshot.LoadWithBase(ctx, "changes.tar.zst", "", "checkpoint-a") +snapshot, err := m.Snapshot.LoadWithBase(ctx, "changes.msnap", "", "checkpoint-a") ``` -The count includes the oldest base but excludes the writable head. A nil `Layers` selects all sealed layers; `DryRun: true` only resolves the plan. `LastLayers` is an alternative to `Since` for export. For direct restore combine `WithFromSnapshot("changes.tar.zst")` with `WithSnapshotBase("checkpoint-a")`. See [disk-chain maintenance](/sandboxes/snapshots#export-changes-and-compact-a-disk-chain) for dependency and retention rules. +The count includes the oldest base but excludes the writable head. A nil `Layers` selects all sealed layers; `DryRun: true` only resolves the plan. `LastLayers` is an alternative to `Since` for export. For direct restore combine `WithFromSnapshot("changes.msnap")` with `WithSnapshotBase("checkpoint-a")`. See [disk-chain maintenance](/sandboxes/snapshots#export-changes-and-compact-a-disk-chain) for dependency and retention rules. ## Snapshot @@ -85,7 +85,7 @@ archive, err := m.Snapshot.CreateArchive(ctx, m.SnapshotArchiveOptions{ Name: "after-pip-install", FromSandbox: "baseline", }, - ArchivePath: "/tmp/after-pip-install.tar.zst", + ArchivePath: "/tmp/after-pip-install.msnap", }) ``` @@ -306,7 +306,7 @@ Walk `dir` and rebuild the local index from the artifacts it finds. func (snapshotFactory) Save(ctx context.Context, nameOrPath, outPath string, opts SnapshotSaveOptions) error ``` -Bundle a snapshot into a `.tar.zst` archive at `outPath`. Set [`SnapshotSaveOptions.PlainTar`](#snapshotsaveoptionsstruct) to skip compression. +Bundle a snapshot into a `.msnap` archive at `outPath`. Set [`SnapshotSaveOptions.PlainTar`](#snapshotsaveoptionsstruct) to skip compression.

Parameters

@@ -332,7 +332,7 @@ Bundle a snapshot into a `.tar.zst` archive at `outPath`. Set [`SnapshotSaveOpti ```go -err := m.Snapshot.Save(ctx, "after-pip-install", "/tmp/snap.tar.zst", +err := m.Snapshot.Save(ctx, "after-pip-install", "/tmp/snap.msnap", m.SnapshotSaveOptions{WithParents: true}, ) ``` @@ -379,7 +379,7 @@ Unpack a snapshot archive into the snapshots directory or an explicit `dest` dir ```go -h, err := m.Snapshot.Load(ctx, "/tmp/snap.tar.zst", "") +h, err := m.Snapshot.Load(ctx, "/tmp/snap.msnap", "") ``` @@ -769,7 +769,7 @@ Configures [`Snapshot.Save`](#snapshot-save). |-------|------|-------------| | WithParents | `bool` | Include the snapshot's parent chain in the archive | | WithImage | `bool` | Include the base OCI image in the archive | -| PlainTar | `bool` | Write an uncompressed `.tar` instead of `.tar.zst` | +| PlainTar | `bool` | Write an uncompressed `.tar` instead of `.msnap` | ### SnapshotVerifyReportstruct diff --git a/docs/sdk/python/snapshots.mdx b/docs/sdk/python/snapshots.mdx index 89379972a..462a0f1d6 100644 --- a/docs/sdk/python/snapshots.mdx +++ b/docs/sdk/python/snapshots.mdx @@ -105,9 +105,9 @@ sb = await Sandbox.create("worker", image="python:3.12") worker = await Sandbox.get("worker") plan = await worker.compact(layers=3, dry_run=True) result = await worker.compact(layers=3) -await Snapshot.save("checkpoint-b", "changes.tar.zst", since="checkpoint-a") -await Snapshot.load("changes.tar.zst", base="checkpoint-a") -child = await Sandbox.create("child", from_snapshot="changes.tar.zst", snapshot_base="checkpoint-a") +await Snapshot.save("checkpoint-b", "changes.msnap", since="checkpoint-a") +await Snapshot.load("changes.msnap", base="checkpoint-a") +child = await Sandbox.create("child", from_snapshot="changes.msnap", snapshot_base="checkpoint-a") ``` The count includes the oldest base but excludes the writable head. Omit `layers` to compact all sealed layers. Use `last_layers=n` instead of `since` to export the newest N sealed layers. Results are dictionaries with `input_layers`, `selected_layers`, `output_layers`, `materialized_bytes`, `total_us`, `pause_us`, and `dry_run`. See [disk-chain maintenance](/sandboxes/snapshots#export-changes-and-compact-a-disk-chain) for dependency and retention rules. @@ -313,7 +313,7 @@ Capture directly into an archive without installing a snapshot directory or inde ```python archive = await Snapshot.create_archive( "after-pip-install", - "/tmp/after-pip-install.tar.zst", + "/tmp/after-pip-install.msnap", from_sandbox="baseline", ) ``` @@ -543,14 +543,14 @@ async def save( ```python await Snapshot.save( "after-pip-install", - "/tmp/after-pip-install.tar.zst", + "/tmp/after-pip-install.msnap", with_parents=True, ) ``` -Bundle a snapshot into a `.tar.zst` archive. The existing snapshot manifest is archived as-is; create the snapshot with recorded integrity when the archive will cross a trust boundary. +Bundle a snapshot into a `.msnap` archive. The existing snapshot manifest is archived as-is; create the snapshot with recorded integrity when the archive will cross a trust boundary.

Parameters

@@ -573,7 +573,7 @@ Bundle a snapshot into a `.tar.zst` archive. The existing snapshot manifest is a
plain_tarbool
-
Write an uncompressed .tar instead of .tar.zst. Default False.
+
Write an uncompressed .tar instead of .msnap. Default False.
@@ -582,7 +582,7 @@ Bundle a snapshot into a `.tar.zst` archive. The existing snapshot manifest is a ```python await Snapshot.save( "after-pip-install", - "/tmp/after-pip-install.tar.zst", + "/tmp/after-pip-install.msnap", with_parents=True, ) ``` @@ -607,14 +607,14 @@ async def load( ) -> SnapshotHandle ``` -Unpack a snapshot archive (`.tar.zst` or `.tar`) into the snapshots directory. Structural and archive-entry checks run during import; recorded payload integrity is preserved for explicit [`verify()`](#snap-verify). Compression is detected from magic bytes. +Unpack a snapshot archive (`.msnap` or `.tar`) into the snapshots directory. Structural and archive-entry checks run during import; recorded payload integrity is preserved for explicit [`verify()`](#snap-verify). Compression is detected from magic bytes.

Parameters

archivestr | os.PathLike
-
Archive path (.tar.zst or .tar).
+
Archive path (.msnap or .tar).
deststr | os.PathLike | None
@@ -634,7 +634,7 @@ Unpack a snapshot archive (`.tar.zst` or `.tar`) into the snapshots directory. S ```python -h = await Snapshot.load("/tmp/after-pip-install.tar.zst") +h = await Snapshot.load("/tmp/after-pip-install.msnap") print(h.path) ``` diff --git a/docs/sdk/rust/snapshots.mdx b/docs/sdk/rust/snapshots.mdx index 9b348351e..f53af1560 100644 --- a/docs/sdk/rust/snapshots.mdx +++ b/docs/sdk/rust/snapshots.mdx @@ -13,14 +13,14 @@ Create disk snapshots of stopped sandboxes and full checkpoints of running sandb let worker = Sandbox::get("worker").await?; let plan = worker.compact().layers(3).dry_run().await?; let result = worker.compact().layers(3).apply().await?; -Snapshot::save("checkpoint-b", Path::new("changes.tar.zst"), SaveOpts { +Snapshot::save("checkpoint-b", Path::new("changes.msnap"), SaveOpts { since: Some("checkpoint-a".into()), ..Default::default() }).await?; -Snapshot::load_with_base(Path::new("changes.tar.zst"), None, "checkpoint-a").await?; +Snapshot::load_with_base(Path::new("changes.msnap"), None, "checkpoint-a").await?; ``` -`layers` counts the oldest physical layers including the base, excluding the writable head. Omit it to compact all sealed layers. `last_layers: Some(n)` is an alternative to `since`; they cannot be combined with each other or `with_parents`. For direct restore, use `Sandbox::builder("child").from_snapshot("changes.tar.zst").snapshot_base("checkpoint-a")`. See [disk-chain maintenance](/sandboxes/snapshots#export-changes-and-compact-a-disk-chain) for dependency and retention rules. +`layers` counts the oldest physical layers including the base, excluding the writable head. Omit it to compact all sealed layers. `last_layers: Some(n)` is an alternative to `since`; they cannot be combined with each other or `with_parents`. For direct restore, use `Sandbox::builder("child").from_snapshot("changes.msnap").snapshot_base("checkpoint-a")`. See [disk-chain maintenance](/sandboxes/snapshots#export-changes-and-compact-a-disk-chain) for dependency and retention rules. ## Snapshot @@ -116,7 +116,7 @@ Capture a disk or full snapshot directly into an archive. The operation creates ```rust let archive = Snapshot::create_archive( Snapshot::builder("baseline").from_sandbox("api").build()?, - "/tmp/baseline.tar.zst", + "/tmp/baseline.msnap", false, ).await?; println!("{} {}", archive.id(), archive.path().display()); @@ -329,7 +329,7 @@ println!("indexed {n} snapshots"); async fn save(name_or_path: &str, out: &Path, opts: SaveOpts) -> MicrosandboxResult<()> ``` -Bundle a snapshot into a `.tar.zst` archive (or plain `.tar`) at `out`. Recorded payload integrity is preserved but not executed implicitly; call [`verify()`](#snap-verify) when an independent content scan is part of your workflow. See [`SaveOpts`](#saveopts) to also include ancestors and the OCI image cache. +Bundle a snapshot into a `.msnap` archive (or plain `.tar`) at `out`. Recorded payload integrity is preserved but not executed implicitly; call [`verify()`](#snap-verify) when an independent content scan is part of your workflow. See [`SaveOpts`](#saveopts) to also include ancestors and the OCI image cache.

Parameters

@@ -356,7 +356,7 @@ use std::path::Path; Snapshot::save( "baseline", - Path::new("/tmp/baseline.tar.zst"), + Path::new("/tmp/baseline.msnap"), SaveOpts { with_parents: true, with_image: true, ..Default::default() }, ).await?; ``` @@ -377,13 +377,13 @@ async fn load(archive_path: &Path, dest: Option<&Path>) -> MicrosandboxResult -Unpack a snapshot archive (`.tar.zst` or `.tar`, detected from magic bytes) into the snapshots directory (or `dest`), routing any bundled image-cache entries into the global cache and registering everything found in the index. Structural and archive-entry checks remain mandatory, while recorded payload integrity is preserved for explicit [`verify()`](#snap-verify). Returns a handle for the head snapshot. +Unpack a snapshot archive (`.msnap` or `.tar`, detected from magic bytes) into the snapshots directory (or `dest`), routing any bundled image-cache entries into the global cache and registering everything found in the index. Structural and archive-entry checks remain mandatory, while recorded payload integrity is preserved for explicit [`verify()`](#snap-verify). Returns a handle for the head snapshot.

Parameters

@@ -412,7 +412,7 @@ Unpack a snapshot archive (`.tar.zst` or `.tar`, detected from magic bytes) into ```rust use std::path::Path; -let h = Snapshot::load(Path::new("/tmp/baseline.tar.zst"), None).await?; +let h = Snapshot::load(Path::new("/tmp/baseline.msnap"), None).await?; println!("loaded {}", h.digest()); ``` diff --git a/docs/sdk/typescript/snapshots.mdx b/docs/sdk/typescript/snapshots.mdx index 9ea4028ac..8ba0755ed 100644 --- a/docs/sdk/typescript/snapshots.mdx +++ b/docs/sdk/typescript/snapshots.mdx @@ -95,10 +95,10 @@ const snap = await h.snapshot("after-pip-install"); const worker = await Sandbox.get("worker"); const plan = await worker.compact({ layers: 3, dryRun: true }); const result = await worker.compact({ layers: 3 }); -await Snapshot.save("checkpoint-b", "changes.tar.zst", { since: "checkpoint-a" }); -await Snapshot.load("changes.tar.zst", undefined, "checkpoint-a"); +await Snapshot.save("checkpoint-b", "changes.msnap", { since: "checkpoint-a" }); +await Snapshot.load("changes.msnap", undefined, "checkpoint-a"); const child = await Sandbox.builder("child") - .fromSnapshot("changes.tar.zst").snapshotBase("checkpoint-a").create(); + .fromSnapshot("changes.msnap").snapshotBase("checkpoint-a").create(); ``` The count includes the oldest base but excludes the writable head. Omit `layers` to compact all sealed layers. `lastLayers` selects the newest N sealed export layers instead of `since`. Results expose physical counts, `materializedBytes`, `totalUs`, and `pauseUs`; materialized bytes are not reclaimed space. See [disk-chain maintenance](/sandboxes/snapshots#export-changes-and-compact-a-disk-chain) for dependency and retention rules. @@ -267,7 +267,7 @@ Capture directly into an archive without installing a snapshot directory or inde ```typescript const archive = await Snapshot.builder("after-pip-install") .fromSandbox("baseline") - .createArchive("/tmp/after-pip-install.tar.zst"); + .createArchive("/tmp/after-pip-install.msnap"); ``` --- @@ -468,7 +468,7 @@ Walk the snapshots directory (default: the configured snapshots dir) and rebuild static save(nameOrPath: string, out: string, opts?: SaveOpts): Promise ``` -Bundle a snapshot into a `.tar.zst` archive. The recorded manifest is archived as-is, so create the snapshot with [`recordIntegrity()`](#recordintegrity) if receivers must verify content. See [`SaveOpts`](#saveopts-interface) for bundling options. +Bundle a snapshot into a `.msnap` archive. The recorded manifest is archived as-is, so create the snapshot with [`recordIntegrity()`](#recordintegrity) if receivers must verify content. See [`SaveOpts`](#saveopts-interface) for bundling options.

Parameters

@@ -490,7 +490,7 @@ Bundle a snapshot into a `.tar.zst` archive. The recorded manifest is archived a ```typescript -await Snapshot.save("after-pip-install", "./baseline.tar.zst", { +await Snapshot.save("after-pip-install", "./baseline.msnap", { withImage: true, }); ``` @@ -506,7 +506,7 @@ await Snapshot.save("after-pip-install", "./baseline.tar.zst", { static load(archive: string, dest?: string): Promise ``` -Unpack a snapshot archive (`.tar.zst` or `.tar`) into the snapshots directory. Structural and archive-entry checks run during import; recorded payload integrity is preserved for explicit [`verify()`](#snap-verify). Compression is detected from magic bytes. +Unpack a snapshot archive (`.msnap` or `.tar`) into the snapshots directory. Structural and archive-entry checks run during import; recorded payload integrity is preserved for explicit [`verify()`](#snap-verify). Compression is detected from magic bytes.

Parameters

@@ -533,7 +533,7 @@ Unpack a snapshot archive (`.tar.zst` or `.tar`) into the snapshots directory. S ```typescript -const h = await Snapshot.load("./baseline.tar.zst"); +const h = await Snapshot.load("./baseline.msnap"); console.log("loaded", h.digest); ``` diff --git a/sdk/rust/lib/snapshot/archive.rs b/sdk/rust/lib/snapshot/archive.rs index c15e1dd83..be3d84522 100644 --- a/sdk/rust/lib/snapshot/archive.rs +++ b/sdk/rust/lib/snapshot/archive.rs @@ -1,4 +1,5 @@ -//! Snapshot save / load via `.tar.zst` bundles. +//! Snapshot save / load via `.msnap` bundles (tar + zstd, or explicit plain tar). +//! Encoding is detected from contents; legacy suffixes and extensionless inputs remain valid. //! //! Default archive format is zstd-compressed tar. Regular files with holes, notably the sparse `upper.ext4` whose logical size is the configured upper cap rather than the data //! written, are stored as old-GNU sparse entries (type `S`): only allocated extents are read and archived, so save cost scales with the data a sandbox actually wrote instead of @@ -4101,7 +4102,6 @@ mod tests { let directory = tempfile::tempdir().unwrap(); let home = directory.path().join("home"); let source = directory.path().join("upper.ext4"); - let archive = directory.path().join("snapshot.tar.zst"); let child_stage = directory.path().join("child"); let mut payload = b"direct archive payload".to_vec(); payload.resize(4096, 0); @@ -4147,28 +4147,42 @@ mod tests { }; let local = LocalBackend::builder().home(&home).build().await.unwrap(); - save_direct_file_snapshot( - &manifest, - &BTreeMap::new(), - "test-snapshot", - std::slice::from_ref(&source), - &archive, - false, - false, - ) - .await - .unwrap(); - let restored = materialize_archive_for_child(&local, &archive, &child_stage, false) - .await - .unwrap(); - - assert_eq!(restored.manifest.snapshot_id, snapshot_id); - assert_eq!( - std::fs::read(child_stage.join("upper.ext4")).unwrap(), - payload - ); - assert!(!child_stage.join(snapshot_id.as_str()).exists()); - assert!(!home.join("snapshots").join(snapshot_id.as_str()).exists()); + // The suffix is only a user-facing convention, never the encoding discriminator. + // Exercise compressed and plain tar under both conventional and misleading names. + for plain_tar in [false, true] { + let archive_dir = directory.path().join(plain_tar.to_string()); + std::fs::create_dir(&archive_dir).unwrap(); + for name in [ + "snapshot.msnap", + "snapshot.tar.zst", + "snapshot.tar", + "snapshot", + ] { + let archive = archive_dir.join(name); + save_direct_file_snapshot( + &manifest, + &BTreeMap::new(), + "test-snapshot", + std::slice::from_ref(&source), + &archive, + plain_tar, + false, + ) + .await + .unwrap(); + let child_stage = child_stage.join(format!("{plain_tar}-{name}")); + let restored = materialize_archive_for_child(&local, &archive, &child_stage, false) + .await + .unwrap(); + assert_eq!(restored.manifest.snapshot_id, snapshot_id); + assert_eq!( + std::fs::read(child_stage.join("upper.ext4")).unwrap(), + payload + ); + assert!(!child_stage.join(snapshot_id.as_str()).exists()); + assert!(!home.join("snapshots").join(snapshot_id.as_str()).exists()); + } + } } #[tokio::test] diff --git a/sdk/rust/lib/snapshot/mod.rs b/sdk/rust/lib/snapshot/mod.rs index 5dcd4faae..d6e60e0a2 100644 --- a/sdk/rust/lib/snapshot/mod.rs +++ b/sdk/rust/lib/snapshot/mod.rs @@ -239,7 +239,8 @@ impl Snapshot { store::reindex_dir(local, dir.as_ref()).await } - /// Bundle a snapshot into a `.tar.zst` archive. + /// Bundle a snapshot into a `.msnap` archive (tar + zstd by default). + /// The explicit output path is preserved; legacy suffixes remain supported. pub async fn save( name_or_path: &str, out: &Path, @@ -250,7 +251,7 @@ impl Snapshot { archive::save_snapshot(local, name_or_path, out, opts).await } - /// Unpack a snapshot archive (`.tar.zst` or `.tar`) into the + /// Unpack a snapshot archive (`.msnap`, `.tar.zst`, or `.tar`) into the /// snapshots dir, registering anything found in the index. pub async fn load( archive_path: &Path, From afba6f57a4e021a6e16806740898043d50c05098 Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Mon, 7 Sep 2026 03:31:23 +0100 Subject: [PATCH 02/29] refactor(checkpoint): prepare capture for retained user pauses Let capture borrow an executor-owned pause generation and workload latch, validate the execution boundary, and leave that source paused on success or failure. Keep the existing running-source call path unchanged. Public pause ownership is not wired yet: agent freezer capability and partial-failure replies must be disambiguated before basic-pause fallback can be exposed safely. Existing checkpoint tests and runtime checks pass; the borrowed-pause path still needs end-to-end qualification. --- crates/runtime/lib/checkpoint/coordinator.rs | 73 ++++++++++++++++---- crates/runtime/lib/control/executor.rs | 1 + 2 files changed, 61 insertions(+), 13 deletions(-) diff --git a/crates/runtime/lib/checkpoint/coordinator.rs b/crates/runtime/lib/checkpoint/coordinator.rs index 06534b70b..0992ed683 100644 --- a/crates/runtime/lib/checkpoint/coordinator.rs +++ b/crates/runtime/lib/checkpoint/coordinator.rs @@ -107,6 +107,13 @@ struct FrozenWorkload { ready: Ready, } +/// Executor-owned resident pause. A recovery pause never acquires this public resume authority. +pub(crate) struct UserPause { + generation: msb_krun::VmPauseGeneration, + workload: Option, + pub(crate) capture_unavailable: Option, +} + struct MemoryObjectSink<'a> { store: &'a LocalObjectStore, updates: Vec, @@ -256,13 +263,22 @@ impl CheckpointCoordinator { }) } - /// Capture and publish one complete same-epoch checkpoint, then restore source execution. + /// Capture a same-epoch checkpoint while preserving the caller's prior execution state. pub(crate) fn capture( &mut self, vm: &msb_krun::VmControl, checkpoint_id: &str, intent: CaptureIntent, + user_pause: Option<&UserPause>, ) -> Result { + if let Some(paused) = user_pause { + paused + .validate(vm) + .map_err(CheckpointFailure::before_pause)?; + if let Some(reason) = &paused.capture_unavailable { + return Err(CheckpointFailure::before_pause(reason)); + } + } if self .root_disk .as_ref() @@ -301,13 +317,22 @@ impl CheckpointCoordinator { // The guest latch is acquired while vCPUs can still service agentd. // It remains held in captured guest memory so a restored child cannot // run application code before VM Generation ID activation completes. + // An already-paused source borrows its original latch and token: even a brief resume + // here would invalidate the user's paused boundary and require another guest handshake. let workload_unavailable_started = Instant::now(); let freeze_started = Instant::now(); - let workload = match self.freeze_workload(checkpoint_id) { - Ok(workload) => workload, - Err(error) => { - let _ = std::fs::remove_dir_all(&staging); - return Err(CheckpointFailure::before_pause(error)); + let acquired_workload; + let workload = match user_pause { + Some(paused) => paused.workload.as_ref().expect("validated workload latch"), + None => { + acquired_workload = match self.freeze_workload(checkpoint_id) { + Ok(workload) => workload, + Err(error) => { + let _ = std::fs::remove_dir_all(&staging); + return Err(CheckpointFailure::before_pause(error)); + } + }; + &acquired_workload } }; let freeze_us = freeze_started.elapsed().as_micros(); @@ -315,11 +340,14 @@ impl CheckpointCoordinator { let vm_pause_window_started = Instant::now(); let pause_started = Instant::now(); - let pause = match vm.pause() { + let pause = match user_pause + .map(|paused| Ok(paused.generation)) + .unwrap_or_else(|| vm.pause()) + { Ok(pause) => pause, Err(error) => { let _ = std::fs::remove_dir_all(&staging); - return match self.thaw_workload(&workload) { + return match self.thaw_workload(workload) { Ok(()) => Err(CheckpointFailure::before_pause(error)), Err(thaw_error) => Err(CheckpointFailure::paused(format!( "VM pause failed: {error}; workload thaw failed: {thaw_error}" @@ -343,13 +371,15 @@ impl CheckpointCoordinator { let captured = match paused { Ok(captured) => captured, Err(mut failure) => { - if !failure.keep_paused + if user_pause.is_none() + && !failure.keep_paused && let Err(error) = vm.resume(pause) { failure.keep_paused = true; failure.message = format!("{}; source resume failed: {error}", failure.message); - } else if !failure.keep_paused - && let Err(error) = self.thaw_workload(&workload) + } else if user_pause.is_none() + && !failure.keep_paused + && let Err(error) = self.thaw_workload(workload) { failure.keep_paused = true; failure.message = format!("{}; workload thaw failed: {error}", failure.message); @@ -378,7 +408,9 @@ impl CheckpointCoordinator { }; let baseline_publish_us = baseline_started.elapsed().as_micros(); let resume_started = Instant::now(); - if let Err(error) = vm.resume(pause) { + if user_pause.is_none() + && let Err(error) = vm.resume(pause) + { return Err(CheckpointFailure { message: format!("checkpoint published but source resume failed: {error}"), keep_paused: true, @@ -388,7 +420,9 @@ impl CheckpointCoordinator { let resume_us = resume_started.elapsed().as_micros(); let vm_pause_window_us = vm_pause_window_started.elapsed().as_micros(); let thaw_started = Instant::now(); - if let Err(error) = self.thaw_workload(&workload) { + if user_pause.is_none() + && let Err(error) = self.thaw_workload(workload) + { let repause = vm.pause().err(); let message = match repause { Some(pause_error) => format!( @@ -412,6 +446,7 @@ impl CheckpointCoordinator { tracing::info!( target: "microsandbox_checkpoint_timing", operation = "capture", + source_already_paused = user_pause.is_some(), checkpoint_id, memory_mode = ?captured.result.memory_mode, memory_logical_bytes = captured.result.memory_logical_bytes, @@ -794,6 +829,18 @@ impl CheckpointCoordinator { } } +impl UserPause { + fn validate(&self, vm: &msb_krun::VmControl) -> Result<(), String> { + if vm.execution_state() != Some(msb_krun::VmExecutionState::Paused(self.generation)) { + return Err("user pause no longer owns the current VM execution boundary".into()); + } + if self.workload.is_none() && self.capture_unavailable.is_none() { + return Err("user pause has no prepared workload latch for full capture".into()); + } + Ok(()) + } +} + impl CheckpointFailure { fn before_pause(error: impl fmt::Display) -> Self { Self { diff --git a/crates/runtime/lib/control/executor.rs b/crates/runtime/lib/control/executor.rs index 33f8ae06b..254778aba 100644 --- a/crates/runtime/lib/control/executor.rs +++ b/crates/runtime/lib/control/executor.rs @@ -315,6 +315,7 @@ impl RuntimeControlExecutor { microsandbox_image::checkpoint::CaptureIntent::TransparentTransfer } }, + None, ) { Ok(result) => { state.lifecycle = RuntimeLifecycle::Running; From 36e11cc293e3bb0c35fee8ba035f474aa31b4ac5 Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Mon, 7 Sep 2026 03:49:14 +0100 Subject: [PATCH 03/29] fix(checkpoint): retain ownership across uncertain freezer transitions Record latch ownership before freezer writes and retain it after failed freeze or thaw acknowledgement. Only confirmed thaw releases the latch. Add optional attempt-scoped error details without a protocol generation bump. Treat older ambiguous errors and transport failures as requiring recovery, and fence mutations if acknowledged recovery fails. Cover failure injection, retry ownership, protocol compatibility, and host recovery disposition. Public pause/resume wiring remains pending. --- crates/agentd/lib/agent.rs | 43 +++-- crates/agentd/lib/workload.rs | 87 ++++++++- crates/protocol/lib/core.rs | 59 ++++++ crates/runtime/lib/checkpoint/coordinator.rs | 179 ++++++++++++++++--- 4 files changed, 330 insertions(+), 38 deletions(-) diff --git a/crates/agentd/lib/agent.rs b/crates/agentd/lib/agent.rs index 38ad96a25..88b21c0ea 100644 --- a/crates/agentd/lib/agent.rs +++ b/crates/agentd/lib/agent.rs @@ -18,8 +18,8 @@ use microsandbox_protocol::bootstrap::GuestBootstrap; use microsandbox_protocol::codec::{self, MAX_FRAME_SIZE}; use microsandbox_protocol::core::{ ClockSync, CoreError, CoreErrorKind, InitAck, InitResolved, Ping, Pong, Ready, - RelayClientDisconnected, ResolvedUser, Touch, Touched, WorkloadFreeze, WorkloadFrozen, - WorkloadThaw, WorkloadThawed, + RelayClientDisconnected, ResolvedUser, Touch, Touched, WorkloadFailure, + WorkloadFailureDisposition, WorkloadFreeze, WorkloadFrozen, WorkloadThaw, WorkloadThawed, }; use microsandbox_protocol::exec::{ ExecExited, ExecFailed, ExecFailureKind, ExecRequest, ExecResize, ExecSignal, ExecStarted, @@ -512,6 +512,7 @@ async fn handle_message( kind: CoreErrorKind::CapabilityUnavailable, message, offending_type: Some(msg.t.as_str().into()), + workload_failure: None, }, ), } @@ -565,6 +566,7 @@ async fn handle_message( }; encode_workload_error( &msg, + &request.attempt_id, WorkloadLatchError::Io(std::io::Error::other(message)), out_buf, )?; @@ -588,7 +590,7 @@ async fn handle_message( })?; } } - Err(error) => encode_workload_error(&msg, error, out_buf)?, + Err(error) => encode_workload_error(&msg, &request.attempt_id, error, out_buf)?, } } @@ -615,7 +617,7 @@ async fn handle_message( AgentdError::ExecSession(format!("encode workload-thawed frame: {error}")) })?; } - Err(error) => encode_workload_error(&msg, error, out_buf)?, + Err(error) => encode_workload_error(&msg, &request.attempt_id, error, out_buf)?, } } @@ -1185,6 +1187,7 @@ fn encode_core_error( kind, message, offending_type, + workload_failure: None, }, ) .map_err(|e| AgentdError::ExecSession(format!("encode core error: {e}")))?; @@ -1195,6 +1198,7 @@ fn encode_core_error( fn encode_workload_error( source: &Message, + attempt_id: &str, error: WorkloadLatchError, out_buf: &mut Vec, ) -> AgentdResult<()> { @@ -1205,14 +1209,33 @@ fn encode_workload_error( WorkloadLatchError::InvalidAttempt(_) => CoreErrorKind::InvalidPayload, WorkloadLatchError::Conflict(_) => CoreErrorKind::InvalidSession, }; - encode_core_error_if_supported( - source, + // Keep the existing error category readable by older hosts. Only the additive, + // attempt-scoped detail proves that a basic-pause fallback is safe. + let disposition = match &error { + WorkloadLatchError::Unavailable(_) => WorkloadFailureDisposition::Unavailable, + _ => WorkloadFailureDisposition::RecoveryRequired, + }; + if !MessageType::CoreError.is_available_at(source.v) { + return Err(AgentdError::ExecSession( + "peer cannot receive workload errors".into(), + )); + } + let reply = Message::with_payload( + MessageType::CoreError, source.id, - kind, - error.to_string(), - Some(source.t.as_str().to_string()), - out_buf, + &CoreError { + kind, + message: error.to_string(), + offending_type: Some(source.t.as_str().to_string()), + workload_failure: Some(WorkloadFailure { + attempt_id: attempt_id.to_string(), + disposition, + }), + }, ) + .map_err(|error| AgentdError::ExecSession(format!("encode workload error: {error}")))?; + codec::encode_to_buf(&reply, out_buf) + .map_err(|error| AgentdError::ExecSession(format!("encode workload error frame: {error}"))) } fn encode_exec_failed(id: u32, payload: ExecFailed, out_buf: &mut Vec) -> AgentdResult<()> { diff --git a/crates/agentd/lib/workload.rs b/crates/agentd/lib/workload.rs index 6134ab292..b1daadbde 100644 --- a/crates/agentd/lib/workload.rs +++ b/crates/agentd/lib/workload.rs @@ -33,6 +33,7 @@ pub(crate) struct WorkloadLatch { enum LatchState { Running { last_thawed: Option }, Frozen { attempt_id: String }, + RecoveryRequired { attempt_id: String }, } trait FreezerControl: Send { @@ -116,9 +117,9 @@ impl WorkloadLatch { .transpose() } - /// Whether a checkpoint attempt currently holds the workload frozen. + /// Whether an attempt blocks new work, including an uncertain freezer transition. pub(crate) fn is_frozen(&self) -> bool { - matches!(self.state, LatchState::Frozen { .. }) + !matches!(self.state, LatchState::Running { .. }) } /// Freeze every process in the agentd-managed workload cgroup. @@ -130,14 +131,23 @@ impl WorkloadLatch { } if current == attempt_id => return Ok(()), LatchState::Frozen { attempt_id: current, + } + | LatchState::RecoveryRequired { + attempt_id: current, } => { return Err(WorkloadLatchError::Conflict(format!( - "attempt {current:?} already owns the freeze" + "attempt {current:?} owns the latch; thaw it before another freeze" ))); } LatchState::Running { .. } => {} } + self.freezer()?; + // Record ownership before writing: an error may follow a successful cgroup write. + // Only a confirmed thaw can release an uncertain transition. + self.state = LatchState::RecoveryRequired { + attempt_id: attempt_id.to_string(), + }; self.freezer()?.set_frozen(true)?; // Agentd itself remains outside the workload cgroup, so it can flush every mounted // filesystem after user processes stop mutating them and before the host pauses the VM. @@ -163,14 +173,21 @@ impl WorkloadLatch { } LatchState::Frozen { attempt_id: current, + } + | LatchState::RecoveryRequired { + attempt_id: current, } if current != attempt_id => { return Err(WorkloadLatchError::Conflict(format!( "attempt {current:?} owns the freeze" ))); } - LatchState::Frozen { .. } => {} + LatchState::Frozen { .. } | LatchState::RecoveryRequired { .. } => {} } + // A failed thaw must not leave a state that freeze retries can acknowledge as frozen. + self.state = LatchState::RecoveryRequired { + attempt_id: attempt_id.to_string(), + }; self.freezer()?.set_frozen(false)?; self.state = LatchState::Running { last_thawed: Some(attempt_id.to_string()), @@ -324,6 +341,7 @@ fn parse_frozen_event(events: &str) -> Option { #[cfg(test)] mod tests { + use std::collections::VecDeque; use std::sync::{Arc, Mutex}; use super::*; @@ -332,6 +350,67 @@ mod tests { states: Arc>>, } + struct FailingFreezer { + outcomes: Mutex>, + } + + impl FreezerControl for FailingFreezer { + fn placement(&self) -> io::Result { + unreachable!() + } + + fn set_frozen(&self, _frozen: bool) -> io::Result<()> { + // Model either a failed write or a write that succeeded before acknowledgement failed. + if self.outcomes.lock().unwrap().pop_front().unwrap() { + Ok(()) + } else { + Err(io::Error::other("injected freezer transition failure")) + } + } + } + + #[test] + fn failed_freeze_retains_ownership_until_confirmed_thaw() { + let mut latch = WorkloadLatch::with_freezer(Box::new(FailingFreezer { + outcomes: Mutex::new(VecDeque::from([false, false, true])), + })); + assert!(latch.freeze("a").is_err()); + assert!(latch.is_frozen()); + assert!(latch.freeze("a").is_err()); + assert!(latch.freeze("b").is_err()); + assert!(latch.thaw("b").is_err()); + assert!(latch.thaw("a").is_err()); + assert!(latch.is_frozen()); + latch.thaw("a").unwrap(); + assert!(!latch.is_frozen()); + latch.thaw("a").unwrap(); + } + + #[test] + fn failed_thaw_never_acknowledges_a_freeze_retry() { + let mut latch = WorkloadLatch::with_freezer(Box::new(FailingFreezer { + outcomes: Mutex::new(VecDeque::from([true, false, true])), + })); + latch.freeze("a").unwrap(); + assert!(latch.thaw("a").is_err()); + assert!(latch.is_frozen()); + assert!(latch.freeze("a").is_err()); + latch.thaw("a").unwrap(); + assert!(!latch.is_frozen()); + } + + #[test] + fn known_unavailable_never_takes_ownership() { + let mut latch = WorkloadLatch::unavailable("no cgroup freezer"); + for attempt in ["a", "b"] { + assert!(matches!( + latch.freeze(attempt), + Err(WorkloadLatchError::Unavailable(_)) + )); + assert!(!latch.is_frozen()); + } + } + impl FreezerControl for FakeFreezer { fn placement(&self) -> io::Result { Err(io::Error::new(io::ErrorKind::Unsupported, "not needed")) diff --git a/crates/protocol/lib/core.rs b/crates/protocol/lib/core.rs index 644781809..48f620cc7 100644 --- a/crates/protocol/lib/core.rs +++ b/crates/protocol/lib/core.rs @@ -139,6 +139,32 @@ pub struct CoreError { /// Wire message type involved in the error, when it could be determined. #[serde(default, skip_serializing_if = "Option::is_none")] pub offending_type: Option, + + /// Attempt-scoped freezer disposition. Absence is ambiguous, not proof that no work froze. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workload_failure: Option, +} + +/// Additional recovery information for a workload control error. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WorkloadFailure { + /// Attempt whose request failed. + pub attempt_id: String, + /// Whether a freeze was rejected before any freezer operation or needs recovery. + pub disposition: WorkloadFailureDisposition, +} + +/// Freezer failure dispositions; unknown future values never authorize a fallback. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WorkloadFailureDisposition { + /// No freezer exists and no freeze was attempted. + Unavailable, + /// The caller must obtain a confirmed thaw before treating the workload as running. + RecoveryRequired, + /// Unrecognized additional information from a newer agent. + #[serde(other)] + Unknown, } /// Machine-readable `core.error` categories. @@ -208,3 +234,36 @@ pub struct RelayClientDisconnected { /// Exclusive upper bound of the disconnected client's ID range. pub id_end_exclusive: u32, } + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn freezer_error_details_are_additive_and_unknown_details_are_not_unavailable() { + let old = serde_json::json!({"kind":"capability_unavailable", "message":"freezer failed"}); + let decoded: CoreError = serde_json::from_value(old.clone()).unwrap(); + assert!(decoded.workload_failure.is_none()); + let mut new = old; + new["workload_failure"] = + serde_json::json!({"attempt_id":"a", "disposition":"future_state"}); + let decoded: CoreError = serde_json::from_value(new.clone()).unwrap(); + assert_eq!( + decoded.workload_failure.unwrap().disposition, + WorkloadFailureDisposition::Unknown + ); + + #[derive(Deserialize)] + struct OldCoreError { + kind: CoreErrorKind, + message: String, + } + let old_reader: OldCoreError = serde_json::from_value(new).unwrap(); + assert_eq!(old_reader.kind, CoreErrorKind::CapabilityUnavailable); + assert_eq!(old_reader.message, "freezer failed"); + } +} diff --git a/crates/runtime/lib/checkpoint/coordinator.rs b/crates/runtime/lib/checkpoint/coordinator.rs index 0992ed683..3c1da5a3a 100644 --- a/crates/runtime/lib/checkpoint/coordinator.rs +++ b/crates/runtime/lib/checkpoint/coordinator.rs @@ -14,7 +14,8 @@ use microsandbox_image::checkpoint::{ }; use microsandbox_protocol::bootstrap::GuestBootstrap; use microsandbox_protocol::core::{ - CoreError, Ready, WorkloadFreeze, WorkloadFrozen, WorkloadThaw, WorkloadThawed, + CoreError, CoreErrorKind, Ready, WorkloadFailureDisposition, WorkloadFreeze, WorkloadFrozen, + WorkloadThaw, WorkloadThawed, }; use microsandbox_protocol::message::{Message, MessageType}; use msb_krun::{ @@ -325,11 +326,11 @@ impl CheckpointCoordinator { let workload = match user_pause { Some(paused) => paused.workload.as_ref().expect("validated workload latch"), None => { - acquired_workload = match self.freeze_workload(checkpoint_id) { + acquired_workload = match self.freeze_workload(vm, checkpoint_id) { Ok(workload) => workload, Err(error) => { let _ = std::fs::remove_dir_all(&staging); - return Err(CheckpointFailure::before_pause(error)); + return Err(error); } }; &acquired_workload @@ -475,14 +476,33 @@ impl CheckpointCoordinator { Ok(captured.result) } - fn freeze_workload(&self, attempt_id: &str) -> Result { + fn freeze_workload( + &self, + vm: &msb_krun::VmControl, + attempt_id: &str, + ) -> Result { let client = self .runtime .block_on(AgentClient::connect_with_timeout( &self.agent_sock, WORKLOAD_CONTROL_TIMEOUT, )) - .map_err(|error| format!("connect workload latch: {error}"))?; + .map_err(|error| { + CheckpointFailure::before_pause(format!("connect workload latch: {error}")) + })?; + // Gather identity before sending anything with side effects. From the first freeze + // request onward, a transport error is ambiguous and requires an acknowledged thaw. + let protocol_generation = client.negotiated_version(); + let ready = client.ready().map_err(CheckpointFailure::before_pause)?; + client + .ensure_version_compat(MessageType::WorkloadFreeze) + .map_err(CheckpointFailure::before_pause)?; + let workload = FrozenWorkload { + client, + attempt_id: attempt_id.to_string(), + protocol_generation, + ready, + }; let request = WorkloadFreeze { attempt_id: attempt_id.to_string(), }; @@ -491,28 +511,36 @@ impl CheckpointCoordinator { .block_on(async { tokio::time::timeout( WORKLOAD_CONTROL_TIMEOUT, - client.request(MessageType::WorkloadFreeze, &request), + workload + .client + .request(MessageType::WorkloadFreeze, &request), ) .await }) - .map_err(|_| "workload freeze timed out".to_string())? - .map_err(|error| format!("request workload freeze: {error}"))?; - validate_workload_reply::( - reply, - MessageType::WorkloadFrozen, - attempt_id, - |payload| &payload.attempt_id, - )?; - let protocol_generation = client.negotiated_version(); - let ready = client - .ready() - .map_err(|error| format!("read workload-agent identity: {error}"))?; - Ok(FrozenWorkload { - client, - attempt_id: attempt_id.to_string(), - protocol_generation, - ready, - }) + .map_err(|_| "workload freeze timed out".to_string()) + .and_then(|reply| reply.map_err(|error| format!("request workload freeze: {error}"))); + if let Ok(reply) = &reply + && let Some(reason) = unavailable_freezer_reason(reply, attempt_id) + { + return Err(CheckpointFailure::before_pause(reason)); + } + let result = reply.and_then(|reply| { + validate_workload_reply::( + reply, + MessageType::WorkloadFrozen, + attempt_id, + |payload| &payload.attempt_id, + ) + }); + if let Err(error) = result { + return Err(recover_failed_freeze( + attempt_id, + error, + || self.thaw_workload(&workload), + || vm.pause().map(|_| ()).map_err(|error| error.to_string()), + )); + } + Ok(workload) } fn thaw_workload(&self, workload: &FrozenWorkload) -> Result<(), String> { @@ -1004,6 +1032,42 @@ async fn root_growth_request( reply.payload().map_err(|e| e.to_string()) } +/// Only new, scoped evidence of no attempted freeze permits a capability fallback. +fn unavailable_freezer_reason(reply: &Message, attempt_id: &str) -> Option { + if reply.t != MessageType::CoreError { + return None; + } + let error = reply.payload::().ok()?; + let detail = error.workload_failure?; + (error.kind == CoreErrorKind::CapabilityUnavailable + && error.offending_type.as_deref() == Some(MessageType::WorkloadFreeze.as_str()) + && detail.attempt_id == attempt_id + && detail.disposition == WorkloadFailureDisposition::Unavailable) + .then_some(error.message) +} + +fn recover_failed_freeze( + attempt_id: &str, + error: String, + thaw: impl FnOnce() -> Result<(), String>, + pause: impl FnOnce() -> Result<(), String>, +) -> CheckpointFailure { + match thaw() { + Ok(()) => CheckpointFailure::before_pause(error), + Err(thaw_error) => { + // Stop further guest progress if possible, and fence host mutations even if the + // hypervisor pause itself fails. Never turn uncertainty into a running disposition. + let pause_status = match pause() { + Ok(()) => "VM paused".to_string(), + Err(error) => format!("VM pause also failed: {error}"), + }; + CheckpointFailure::paused(format!( + "attempt {attempt_id}: {error}; workload recovery required: {thaw_error}; {pause_status}" + )) + } + } +} + fn validate_workload_reply( reply: Message, expected_type: MessageType, @@ -1412,6 +1476,72 @@ mod tests { use microsandbox_protocol::message::{Message, MessageType}; use msb_krun::{GuestMemoryRange, MemoryCaptureSink}; + #[test] + fn unavailable_freezer_requires_explicit_matching_evidence() { + use microsandbox_protocol::core::{WorkloadFailure, WorkloadFailureDisposition}; + let mut error = CoreError { + kind: CoreErrorKind::CapabilityUnavailable, + message: "missing freezer".into(), + offending_type: Some(MessageType::WorkloadFreeze.as_str().into()), + workload_failure: None, + }; + let check = |error: &CoreError| { + let reply = Message::with_payload(MessageType::CoreError, 7, error).unwrap(); + super::unavailable_freezer_reason(&reply, "a").is_some() + }; + assert!(!check(&error), "older agent errors are ambiguous"); + for disposition in [ + WorkloadFailureDisposition::RecoveryRequired, + WorkloadFailureDisposition::Unknown, + ] { + error.workload_failure = Some(WorkloadFailure { + attempt_id: "a".into(), + disposition, + }); + assert!(!check(&error)); + } + error.workload_failure.as_mut().unwrap().disposition = + WorkloadFailureDisposition::Unavailable; + assert!(check(&error)); + error.workload_failure.as_mut().unwrap().attempt_id = "b".into(); + assert!(!check(&error)); + error.workload_failure.as_mut().unwrap().attempt_id = "a".into(); + error.offending_type = Some(MessageType::WorkloadThaw.as_str().into()); + assert!(!check(&error)); + error.offending_type = Some(MessageType::WorkloadFreeze.as_str().into()); + error.kind = CoreErrorKind::InvalidSession; + assert!(!check(&error)); + } + + #[test] + fn failed_freeze_returns_running_only_after_confirmed_recovery() { + let failure = super::recover_failed_freeze( + "a", + "lost reply".into(), + || Ok(()), + || panic!("must not pause after thaw"), + ); + assert!(!failure.keep_paused); + for pause_fails in [false, true] { + let failure = super::recover_failed_freeze( + "a", + "lost reply".into(), + || Err("thaw failed".into()), + || { + if pause_fails { + Err("pause failed".into()) + } else { + Ok(()) + } + }, + ); + assert!(failure.keep_paused); + assert!(failure.message.contains("attempt a")); + assert!(failure.message.contains("recovery required")); + assert_eq!(failure.message.contains("pause also failed"), pause_fails); + } + } + #[test] fn incremental_updates_split_and_reuse_unchanged_object_ranges() { let original = ObjectId::from_bytes(b"original").unwrap(); @@ -1642,6 +1772,7 @@ mod tests { kind: CoreErrorKind::CapabilityUnavailable, message: "freezer unavailable".into(), offending_type: Some(MessageType::WorkloadFreeze.as_str().into()), + workload_failure: None, }, ) .unwrap(); From 25dda36f7faa70c8dbae86e54853634104764cb7 Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Mon, 7 Sep 2026 12:32:37 +0100 Subject: [PATCH 04/29] feat(snapshot): integrate cow memory and resident pause lifecycle Wire explicit memory policy and pause/resume through the CLI and Rust, Python, Node, and Go SDKs. Preserve user pause ownership across full captures and correct the guest clock before releasing workloads. Restore from pinned immutable memory caches and prepare repeated capture caches from immutable generations, reusing filesystem reflinks where available. Resolve control endpoints and cache paths through the owning local backend, and pin the matching runtime and firmware companions. Add live lifecycle smoke matrices and record Linux/macOS results and remaining qualification gaps. Windows CoW remains unsupported; private zeroing on unplug does not yet reclaim host RAM. This commit does not claim full stack-8 qualification. --- Cargo.lock | 33 +-- Cargo.toml | 14 ++ crates/cli/bin/main.rs | 11 +- crates/cli/lib/commands/common.rs | 9 + crates/cli/lib/commands/inspect.rs | 20 ++ crates/cli/lib/commands/mod.rs | 1 + crates/cli/lib/commands/pause.rs | 38 ++++ crates/cli/lib/commands/snapshot.rs | 2 +- crates/cli/lib/sandbox_cmd.rs | 2 + crates/runtime/lib/checkpoint/coordinator.rs | 137 ++++++++++++- crates/runtime/lib/checkpoint/memory_cache.rs | 173 ++++++++++++++-- crates/runtime/lib/checkpoint/mod.rs | 2 +- crates/runtime/lib/checkpoint/restore.rs | 45 +++- crates/runtime/lib/console.rs | 3 + crates/runtime/lib/control.rs | 24 +++ crates/runtime/lib/control/executor.rs | 116 ++++++++++- crates/runtime/lib/launch.rs | 8 + crates/runtime/lib/relay.rs | 27 +++ crates/runtime/lib/vm.rs | 52 ++++- docs/sandboxes/lifecycle.mdx | 40 ++++ docs/sandboxes/snapshots.mdx | 19 +- packages/microsandbox-types/rust/lib/cloud.rs | 1 + .../microsandbox-types/rust/lib/domain.rs | 41 ++++ packages/microsandbox-types/rust/lib/lib.rs | 20 +- scripts/smoke/cli/cow-memory-lifecycle.py | 90 ++++++++ .../cow-memory-lifecycle-2026-09-07.md | 44 ++++ sdk/go/cow_lifecycle_test.go | 79 +++++++ sdk/go/internal/ffi/ffi.go | 73 +++++++ sdk/go/native/microsandbox_go_ffi.h | 14 ++ sdk/go/native/src/lib.rs | 70 +++++++ sdk/go/options.go | 37 +++- sdk/go/options_test.go | 22 ++ sdk/go/sandbox.go | 21 ++ sdk/node-ts/native/index.d.ts | 10 + sdk/node-ts/native/sandbox.rs | 16 ++ sdk/node-ts/native/sandbox_builder.rs | 13 ++ sdk/node-ts/native/sandbox_handle.rs | 12 ++ sdk/node-ts/src/internal/napi.ts | 5 + sdk/node-ts/src/sandbox-handle.ts | 10 + sdk/node-ts/src/sandbox-status.ts | 3 +- sdk/node-ts/src/sandbox.ts | 10 + sdk/node-ts/tests/cow-lifecycle.test.ts | 25 +++ sdk/node-ts/tests/unit/builders.test.ts | 10 + sdk/python/microsandbox/__init__.py | 2 + sdk/python/microsandbox/_microsandbox.pyi | 7 + sdk/python/microsandbox/types.py | 7 + sdk/python/src/helpers.rs | 6 + sdk/python/src/sandbox.rs | 20 ++ sdk/python/src/sandbox_handle.rs | 20 ++ sdk/python/tests/test_cow_lifecycle.py | 37 ++++ sdk/python/tests/test_create_stub.py | 3 + sdk/rust/lib/backend/cloud/sandbox.rs | 6 + sdk/rust/lib/backend/local/sandbox/mod.rs | 23 ++- sdk/rust/lib/error.rs | 6 + sdk/rust/lib/runtime/spawn.rs | 4 + sdk/rust/lib/sandbox/builder.rs | 6 + sdk/rust/lib/sandbox/config.rs | 2 + sdk/rust/lib/sandbox/config_patch.rs | 11 + sdk/rust/lib/sandbox/handle.rs | 7 +- sdk/rust/lib/sandbox/mod.rs | 4 +- sdk/rust/lib/sandbox/modify.rs | 31 ++- sdk/rust/lib/sandbox/pause.rs | 194 ++++++++++++++++++ vendor/libkrunfw | 2 +- 63 files changed, 1716 insertions(+), 84 deletions(-) create mode 100644 crates/cli/lib/commands/pause.rs create mode 100644 scripts/smoke/cli/cow-memory-lifecycle.py create mode 100644 scripts/smoke/reports/cow-memory-lifecycle-2026-09-07.md create mode 100644 sdk/go/cow_lifecycle_test.go create mode 100644 sdk/node-ts/tests/cow-lifecycle.test.ts create mode 100644 sdk/python/tests/test_cow_lifecycle.py create mode 100644 sdk/rust/lib/sandbox/pause.rs diff --git a/Cargo.lock b/Cargo.lock index 61e33bfe2..88caeb69c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4353,8 +4353,7 @@ dependencies = [ [[package]] name = "msb_krun" version = "0.1.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb69e940b9aea22169f0882108415dd309a1344803943e466652916002f7f680" +source = "git+https://github.com/superradcompany/libkrun?rev=94d680b21bf7ea7c2bed5262ed211833bd4379bd#94d680b21bf7ea7c2bed5262ed211833bd4379bd" dependencies = [ "crossbeam-channel", "kvm-bindings", @@ -4373,8 +4372,7 @@ dependencies = [ [[package]] name = "msb_krun_arch" version = "0.1.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc8b11bc3e2829adcbe00eea6bee312101a987d8b800fe6c30d19f4fb4d9f18e" +source = "git+https://github.com/superradcompany/libkrun?rev=94d680b21bf7ea7c2bed5262ed211833bd4379bd#94d680b21bf7ea7c2bed5262ed211833bd4379bd" dependencies = [ "kvm-bindings", "kvm-ioctls", @@ -4388,14 +4386,12 @@ dependencies = [ [[package]] name = "msb_krun_arch_gen" version = "0.1.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d935e248ac8ee4e527c82188cb043c9333f58cce8e5259d4b4367d30a4585649" +source = "git+https://github.com/superradcompany/libkrun?rev=94d680b21bf7ea7c2bed5262ed211833bd4379bd#94d680b21bf7ea7c2bed5262ed211833bd4379bd" [[package]] name = "msb_krun_cpuid" version = "0.1.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75d8b5f47e7e11381832d7d1a1c803a27e8cef2e2dc551e2a172339f23d9d414" +source = "git+https://github.com/superradcompany/libkrun?rev=94d680b21bf7ea7c2bed5262ed211833bd4379bd#94d680b21bf7ea7c2bed5262ed211833bd4379bd" dependencies = [ "kvm-bindings", "kvm-ioctls", @@ -4405,8 +4401,7 @@ dependencies = [ [[package]] name = "msb_krun_devices" version = "0.1.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e20026edb2f255659010d659c08ed8a84b67d549440ef81d5a745f1453fd6a" +source = "git+https://github.com/superradcompany/libkrun?rev=94d680b21bf7ea7c2bed5262ed211833bd4379bd#94d680b21bf7ea7c2bed5262ed211833bd4379bd" dependencies = [ "bincode", "bitflags 1.3.2", @@ -4437,8 +4432,7 @@ dependencies = [ [[package]] name = "msb_krun_hvf" version = "0.1.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8de9d80864e34ee1845155c24ab95205907f71a587c562f603d9719d1aed697" +source = "git+https://github.com/superradcompany/libkrun?rev=94d680b21bf7ea7c2bed5262ed211833bd4379bd#94d680b21bf7ea7c2bed5262ed211833bd4379bd" dependencies = [ "crossbeam-channel", "libloading 0.8.9", @@ -4450,8 +4444,7 @@ dependencies = [ [[package]] name = "msb_krun_kernel" version = "0.1.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2afad41ea9e1719499bb5c375f8b7ee1ad24d0dd6f2a54b494fdd442a95ccc9b" +source = "git+https://github.com/superradcompany/libkrun?rev=94d680b21bf7ea7c2bed5262ed211833bd4379bd#94d680b21bf7ea7c2bed5262ed211833bd4379bd" dependencies = [ "msb-vm-memory", "msb_krun_utils", @@ -4460,8 +4453,7 @@ dependencies = [ [[package]] name = "msb_krun_polly" version = "0.1.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ddcf4dfce736d7c1103ea38a24c5de57a76f06b84357347ec87f487d3f7be3" +source = "git+https://github.com/superradcompany/libkrun?rev=94d680b21bf7ea7c2bed5262ed211833bd4379bd#94d680b21bf7ea7c2bed5262ed211833bd4379bd" dependencies = [ "libc", "msb_krun_utils", @@ -4470,8 +4462,7 @@ dependencies = [ [[package]] name = "msb_krun_smbios" version = "0.1.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22d46149e9eeb80c379b7c0419946688861da8e76ace11a0f387a3adfde1f3ff" +source = "git+https://github.com/superradcompany/libkrun?rev=94d680b21bf7ea7c2bed5262ed211833bd4379bd#94d680b21bf7ea7c2bed5262ed211833bd4379bd" dependencies = [ "msb-vm-memory", ] @@ -4479,8 +4470,7 @@ dependencies = [ [[package]] name = "msb_krun_utils" version = "0.1.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98b8923a8109908e51a17a7cec55056f58272f63058a62543f01104317fbaa25" +source = "git+https://github.com/superradcompany/libkrun?rev=94d680b21bf7ea7c2bed5262ed211833bd4379bd#94d680b21bf7ea7c2bed5262ed211833bd4379bd" dependencies = [ "bitflags 1.3.2", "crossbeam-channel", @@ -4495,8 +4485,7 @@ dependencies = [ [[package]] name = "msb_krun_vmm" version = "0.1.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15c27ec272f4686597c41e452f618610e59bbdc84a7073c7f93cbcf721b33b33" +source = "git+https://github.com/superradcompany/libkrun?rev=94d680b21bf7ea7c2bed5262ed211833bd4379bd#94d680b21bf7ea7c2bed5262ed211833bd4379bd" dependencies = [ "bincode", "bzip2", diff --git a/Cargo.toml b/Cargo.toml index 8176dcbdf..0308b7983 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -205,3 +205,17 @@ parking_lot = "0.12" rpassword = "7" russh = "0.62.4" russh-sftp = "2.3.0" + +# #8 construction-time private memory and identity-preserving resume support. +[patch.crates-io] +msb_krun = { git = "https://github.com/superradcompany/libkrun", rev = "94d680b21bf7ea7c2bed5262ed211833bd4379bd" } +msb_krun_utils = { git = "https://github.com/superradcompany/libkrun", rev = "94d680b21bf7ea7c2bed5262ed211833bd4379bd" } +msb_krun_vmm = { git = "https://github.com/superradcompany/libkrun", rev = "94d680b21bf7ea7c2bed5262ed211833bd4379bd" } +msb_krun_devices = { git = "https://github.com/superradcompany/libkrun", rev = "94d680b21bf7ea7c2bed5262ed211833bd4379bd" } +msb_krun_arch = { git = "https://github.com/superradcompany/libkrun", rev = "94d680b21bf7ea7c2bed5262ed211833bd4379bd" } +msb_krun_arch_gen = { git = "https://github.com/superradcompany/libkrun", rev = "94d680b21bf7ea7c2bed5262ed211833bd4379bd" } +msb_krun_cpuid = { git = "https://github.com/superradcompany/libkrun", rev = "94d680b21bf7ea7c2bed5262ed211833bd4379bd" } +msb_krun_hvf = { git = "https://github.com/superradcompany/libkrun", rev = "94d680b21bf7ea7c2bed5262ed211833bd4379bd" } +msb_krun_kernel = { git = "https://github.com/superradcompany/libkrun", rev = "94d680b21bf7ea7c2bed5262ed211833bd4379bd" } +msb_krun_polly = { git = "https://github.com/superradcompany/libkrun", rev = "94d680b21bf7ea7c2bed5262ed211833bd4379bd" } +msb_krun_smbios = { git = "https://github.com/superradcompany/libkrun", rev = "94d680b21bf7ea7c2bed5262ed211833bd4379bd" } diff --git a/crates/cli/bin/main.rs b/crates/cli/bin/main.rs index 26fee23e6..0e17ea395 100644 --- a/crates/cli/bin/main.rs +++ b/crates/cli/bin/main.rs @@ -22,8 +22,9 @@ const TOP_LEVEL_COMMAND_GROUPS: &[CommandGroup] = &[ CommandGroup { heading: "Sandboxes", commands: &[ - "run", "create", "modify", "start", "stop", "restart", "ping", "touch", "list", - "status", "metrics", "remove", "exec", "copy", "logs", "ssh", "inspect", + "run", "create", "modify", "start", "stop", "pause", "resume", "restart", "ping", + "touch", "list", "status", "metrics", "remove", "exec", "copy", "logs", "ssh", + "inspect", ], }, CommandGroup { @@ -107,6 +108,10 @@ enum Commands { /// Stop one or more running sandboxes. Stop(stop::StopArgs), + /// Suspend a resident sandbox without creating a snapshot. + Pause(microsandbox_cli::commands::pause::PauseArgs), + /// Resume a user-paused resident sandbox. + Resume(microsandbox_cli::commands::pause::PauseArgs), /// Restart one or more sandboxes. Restart(restart::RestartArgs), @@ -666,6 +671,8 @@ fn run_async_command_anyhow( Commands::Modify(args) => modify::run(args).await, Commands::Start(args) => start::run(args).await, Commands::Stop(args) => stop::run(args).await, + Commands::Pause(args) => microsandbox_cli::commands::pause::run(args, false).await, + Commands::Resume(args) => microsandbox_cli::commands::pause::run(args, true).await, Commands::Restart(args) => restart::run(args).await, Commands::Ping(args) => ping::run(args).await, Commands::Touch(args) => touch::run(args).await, diff --git a/crates/cli/lib/commands/common.rs b/crates/cli/lib/commands/common.rs index cc5e47ef0..227d0bddc 100644 --- a/crates/cli/lib/commands/common.rs +++ b/crates/cli/lib/commands/common.rs @@ -123,6 +123,10 @@ pub struct SandboxOpts { #[arg(long, value_name = "POLICY", value_parser = ["always", "madvise", "never"])] pub thp: Option, + /// Memory snapshot representation; CoW is explicit and snapshots remain manual. + #[arg(long, value_name = "MODE", value_parser = ["standard", "cow"])] + pub memory_snapshot: Option, + /// Mount a host path or named volume into the sandbox (`SOURCE:DEST[:OPTIONS]`). /// OPTIONS may include paired `uid=,gid=` for directory-backed mounts. #[arg(short, long)] @@ -613,6 +617,7 @@ impl SandboxOpts { || self.memory.is_some() || self.max_memory.is_some() || self.thp.is_some() + || self.memory_snapshot.is_some() || !self.volume.is_empty() || !self.mount_dir.is_empty() || !self.mount_file.is_empty() @@ -883,6 +888,10 @@ fn apply_sandbox_opts_inner( .map_err(anyhow::Error::msg)?; builder = builder.thp(policy); } + if let Some(ref mode) = opts.memory_snapshot { + let mode = serde_json::from_value(serde_json::Value::String(mode.clone()))?; + builder = builder.memory_snapshot(mode); + } if let Some(ref workdir) = opts.workdir { builder = builder.workdir(workdir); } diff --git a/crates/cli/lib/commands/inspect.rs b/crates/cli/lib/commands/inspect.rs index 5da068159..24359c95d 100644 --- a/crates/cli/lib/commands/inspect.rs +++ b/crates/cli/lib/commands/inspect.rs @@ -86,6 +86,17 @@ pub async fn run(args: InspectArgs) -> anyhow::Result<()> { let handle = Sandbox::get(&args.name).await?; let desired_config = handle.config().ok(); let active_config = handle.active_config().ok().flatten(); + let pause_state = if matches!( + handle.status_snapshot(), + SandboxStatus::Running | SandboxStatus::Paused + ) { + tokio::time::timeout(std::time::Duration::from_millis(250), handle.pause_state()) + .await + .ok() + .and_then(Result::ok) + } else { + None + }; let pending_changes = pending_config_changes( handle.status_snapshot(), desired_config.as_ref(), @@ -98,6 +109,7 @@ pub async fn run(args: InspectArgs) -> anyhow::Result<()> { let mut json = serde_json::json!({ "name": handle.name(), "status": format!("{:?}", handle.status_snapshot()), + "pause": pause_state, "config": config, "created_at": handle.created_at().map(|dt| ui::format_json_datetime(&dt)), "updated_at": handle.updated_at().map(|dt| ui::format_json_datetime(&dt)), @@ -116,6 +128,14 @@ pub async fn run(args: InspectArgs) -> anyhow::Result<()> { ui::detail_kv("Name", handle.name()); ui::detail_kv("Status", &ui::format_status(&status)); + if let Some(state) = &pause_state { + if state.recovery_required { + ui::detail_kv("Recovery", "Required; ordinary resume is fenced"); + } + if let Some(reason) = &state.capture_unavailable { + ui::detail_kv("Full snapshot", reason); + } + } if let Some(dt) = handle.created_at() { ui::detail_kv("Created", &ui::format_datetime(&dt)); diff --git a/crates/cli/lib/commands/mod.rs b/crates/cli/lib/commands/mod.rs index d57878ed7..96db57957 100644 --- a/crates/cli/lib/commands/mod.rs +++ b/crates/cli/lib/commands/mod.rs @@ -21,6 +21,7 @@ pub mod list; pub mod logs; pub mod metrics; pub mod modify; +pub mod pause; pub mod ping; pub mod ps; pub mod pull; diff --git a/crates/cli/lib/commands/pause.rs b/crates/cli/lib/commands/pause.rs new file mode 100644 index 000000000..f68c33ddc --- /dev/null +++ b/crates/cli/lib/commands/pause.rs @@ -0,0 +1,38 @@ +//! Explicit resident pause and resume. + +use clap::Args; +use microsandbox::Sandbox; + +use crate::ui; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// Arguments shared by resident pause and resume. +#[derive(Args)] +pub struct PauseArgs { + /// Sandbox name. + pub name: String, + /// Suppress progress output. + #[arg(short, long)] + pub quiet: bool, +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +/// Change resident execution state through host control, without opening the guest agent. +pub async fn run(args: PauseArgs, resume: bool) -> anyhow::Result<()> { + let sandbox = Sandbox::get(&args.name).await?; + if resume { + sandbox.resume().await?; + } else { + sandbox.pause().await?; + } + if !args.quiet { + ui::success(if resume { "Resumed" } else { "Paused" }, &args.name); + } + Ok(()) +} diff --git a/crates/cli/lib/commands/snapshot.rs b/crates/cli/lib/commands/snapshot.rs index 4e5fdb9c4..689ee06d9 100644 --- a/crates/cli/lib/commands/snapshot.rs +++ b/crates/cli/lib/commands/snapshot.rs @@ -20,7 +20,7 @@ pub struct SnapshotArgs { /// Snapshot subcommands. #[derive(Debug, Subcommand)] pub enum SnapshotCommands { - /// Create a disk snapshot from a stopped sandbox or a full snapshot from a running one. + /// Create a disk snapshot from a stopped sandbox or a full snapshot from a running or paused one. Create(SnapshotCreateArgs), /// List indexed snapshots. diff --git a/crates/cli/lib/sandbox_cmd.rs b/crates/cli/lib/sandbox_cmd.rs index 2fc88a456..f177fcc5d 100644 --- a/crates/cli/lib/sandbox_cmd.rs +++ b/crates/cli/lib/sandbox_cmd.rs @@ -268,6 +268,8 @@ pub fn run(args: SandboxArgs) -> ! { let vm_config = VmConfig { libkrunfw_path: launch.libkrunfw_path, thp: launch.thp, + memory_snapshot: launch.memory_snapshot, + memory_cache_dir: launch.memory_cache_dir, vcpus: args.vcpus, memory_mib: args.memory_mib, max_cpus: args.max_vcpus.unwrap_or(args.vcpus).max(args.vcpus), diff --git a/crates/runtime/lib/checkpoint/coordinator.rs b/crates/runtime/lib/checkpoint/coordinator.rs index 3c1da5a3a..703c9dafb 100644 --- a/crates/runtime/lib/checkpoint/coordinator.rs +++ b/crates/runtime/lib/checkpoint/coordinator.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::fmt; -use std::io; +use std::io::{self, Read}; use std::path::{Path, PathBuf}; use std::time::{Duration, Instant}; @@ -56,6 +56,8 @@ pub(crate) struct CheckpointCoordinator { fs_resource_bindings: BTreeMap>, network_resource_binding: Option, previous_memory: Option, + memory_cache: Option, + cached_baseline: Option<(MemoryManifest, super::CachedMemory)>, } /// Published checkpoint identity returned to the control executor. @@ -73,6 +75,7 @@ pub(crate) struct CheckpointResult { #[derive(Debug)] pub(crate) struct CheckpointFailure { message: String, + freezer_unavailable: bool, pub(crate) keep_paused: bool, pub(crate) published: Option>, } @@ -139,6 +142,72 @@ struct PendingDeviceState { //-------------------------------------------------------------------------------------------------- impl CheckpointCoordinator { + /// Establish a resident, user-owned pause without requiring snapshot resource admission. + pub(crate) fn pause_user( + &self, + vm: &msb_krun::VmControl, + attempt_id: &str, + ) -> Result { + if !vm.clock_sync_supported() { + return Err(CheckpointFailure::before_pause( + "guest kernel lacks clock-only resume support", + )); + } + let (workload, capture_unavailable) = match self.freeze_workload(vm, attempt_id) { + Ok(workload) => (Some(workload), None), + Err(error) if error.freezer_unavailable => (None, Some(error.to_string())), + Err(error) => return Err(error), + }; + match vm.pause() { + Ok(generation) => Ok(UserPause { + generation, + workload, + capture_unavailable, + }), + Err(error) => { + if let Some(workload) = workload { + return Err(recover_failed_freeze( + attempt_id, + error.to_string(), + || self.thaw_workload(&workload), + || vm.pause().map(|_| ()).map_err(|error| error.to_string()), + )); + } + Err(CheckpointFailure::before_pause(error)) + } + } + } + + /// Resume this exact resident VM, processing clock correction before releasing workloads. + pub(crate) fn resume_user( + &self, + vm: &msb_krun::VmControl, + paused: &UserPause, + ) -> Result<(), CheckpointFailure> { + paused.validate(vm).map_err(CheckpointFailure::paused)?; + let request = vm + .request_clock_sync() + .ok_or_else(|| CheckpointFailure::paused("clock-only resume request unavailable"))?; + vm.resume(paused.generation) + .map_err(CheckpointFailure::paused)?; + let result = if vm.wait_vm_generation_processed(request, WORKLOAD_CONTROL_TIMEOUT) + == Some(msb_krun::VmGenerationWaitOutcome::Processed) + { + match &paused.workload { + Some(workload) => self.thaw_workload(workload), + None => Ok(()), + } + } else { + Err("guest did not acknowledge resident resume clock correction".into()) + }; + result.map_err(|error| { + let pause_error = vm.pause().err(); + CheckpointFailure::paused(format!( + "resume recovery required: {error}; pause error: {pause_error:?}" + )) + }) + } + pub(crate) fn compact( &mut self, vm: &msb_krun::VmControl, @@ -261,6 +330,17 @@ impl CheckpointCoordinator { fs_resource_bindings, network_resource_binding, previous_memory: None, + memory_cache: if vm.memory_snapshot == microsandbox_types::MemorySnapshotMode::Cow { + Some( + super::MemoryCache::open(vm.memory_cache_dir.as_ref().ok_or_else(|| { + "CoW memory requires its backend-resolved cache directory".to_string() + })?) + .map_err(|error| error.to_string())?, + ) + } else { + None + }, + cached_baseline: None, }) } @@ -413,6 +493,7 @@ impl CheckpointCoordinator { && let Err(error) = vm.resume(pause) { return Err(CheckpointFailure { + freezer_unavailable: false, message: format!("checkpoint published but source resume failed: {error}"), keep_paused: true, published: Some(Box::new(captured.result)), @@ -432,6 +513,7 @@ impl CheckpointCoordinator { None => format!("checkpoint published but workload thaw failed: {error}"), }; return Err(CheckpointFailure { + freezer_unavailable: false, message, keep_paused: true, published: Some(Box::new(captured.result)), @@ -439,6 +521,51 @@ impl CheckpointCoordinator { } let thaw_us = thaw_started.elapsed().as_micros(); let workload_unavailable_us = workload_unavailable_started.elapsed().as_micros(); + if let Some(cache) = &self.memory_cache { + // Source execution has resumed (unless explicitly user-paused). Read only the + // completed immutable capture, never live RAM, while preparing child acceleration. + let prepared = (|| -> Result { + let bytes = captured + .memory_manifest + .to_canonical_bytes() + .map_err(|e| e.to_string())?; + let identity = ObjectId::from_bytes(&bytes).map_err(|e| e.to_string())?; + cache + .materialize_with_baseline( + &captured.memory_manifest, + &identity, + self.cached_baseline + .as_ref() + .map(|(manifest, cached)| (manifest, cached)), + |id| { + let mut bytes = Vec::new(); + std::fs::File::open(self.store.object_path(id))? + .take(MEMORY_OBJECT_PACK_SIZE as u64 + 1) + .read_to_end(&mut bytes)?; + if bytes.len() > MEMORY_OBJECT_PACK_SIZE + || ObjectId::from_bytes(&bytes).map_err(io::Error::other)? != *id + { + return Err(io::Error::other( + "memory object failed size/identity validation", + )); + } + Ok(bytes) + }, + ) + .map_err(|e| e.to_string()) + })(); + match prepared { + Ok(cached) => { + tracing::info!(target: "microsandbox_checkpoint_timing", operation = "memory_cache", prepare_us = cached.prepare_us, cache_hit = cached.cache_hit, reflink = cached.reflink, "prepared immutable capture cache"); + self.cached_baseline = Some((captured.memory_manifest.clone(), cached)); + } + Err(error) => { + // Publication already succeeded. Losing optional acceleration does not + // erase the artifact or turn its successful capture into a false failure. + tracing::warn!(%error, "checkpoint published without memory cache acceleration"); + } + } + } if baseline_published { self.previous_memory = Some(captured.memory_manifest); } else { @@ -522,7 +649,9 @@ impl CheckpointCoordinator { if let Ok(reply) = &reply && let Some(reason) = unavailable_freezer_reason(reply, attempt_id) { - return Err(CheckpointFailure::before_pause(reason)); + let mut error = CheckpointFailure::before_pause(reason); + error.freezer_unavailable = true; + return Err(error); } let result = reply.and_then(|reply| { validate_workload_reply::( @@ -616,6 +745,7 @@ impl CheckpointCoordinator { let rollover = disk .rollover(vm, &self.runtime, staging, pause_generation) .map_err(|error| CheckpointFailure { + freezer_unavailable: false, message: error.to_string(), keep_paused: error.keep_paused, published: None, @@ -873,6 +1003,7 @@ impl CheckpointFailure { fn before_pause(error: impl fmt::Display) -> Self { Self { message: error.to_string(), + freezer_unavailable: false, keep_paused: false, published: None, } @@ -881,6 +1012,7 @@ impl CheckpointFailure { fn paused(error: impl fmt::Display) -> Self { Self { message: error.to_string(), + freezer_unavailable: false, keep_paused: true, published: None, } @@ -898,6 +1030,7 @@ impl FrozenWorkload { kind: "agent".into(), treatment: ResourceTreatment::Serialize, binding: BTreeMap::from([ + ("attempt_id".into(), self.attempt_id.clone()), ( "protocol_generation".into(), self.protocol_generation.to_string(), diff --git a/crates/runtime/lib/checkpoint/memory_cache.rs b/crates/runtime/lib/checkpoint/memory_cache.rs index 3d90158b7..fefad72dd 100644 --- a/crates/runtime/lib/checkpoint/memory_cache.rs +++ b/crates/runtime/lib/checkpoint/memory_cache.rs @@ -28,12 +28,16 @@ pub struct CachedMemoryRegion { /// An opened realization pinned against cooperative eviction until the handle is dropped. pub struct CachedMemory { + path: PathBuf, + identity: ObjectId, /// Read-only backing ownership. Transfer this handle to the VMM, not merely its pathname. pub file: File, /// Exact guest coverage, with address holes omitted from physical storage. pub regions: Vec, /// Whether existing verified bytes were reused without rereading portable objects. pub cache_hit: bool, + /// Whether this construction cloned its baseline using a filesystem reflink. + pub reflink: bool, /// Time spent resolving or constructing this backing, in microseconds. pub prepare_us: u128, } @@ -59,6 +63,10 @@ impl MemoryCache { } let root = root.into(); std::fs::create_dir_all(&root)?; + // Cache contents are guest RAM, not public image data. Restrict traversal even + // when the caller's umask permits other local users to read ordinary cache files. + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700))?; Ok(Self { root, page_size: page_size as u64, @@ -83,6 +91,18 @@ impl MemoryCache { &self, manifest: &MemoryManifest, identity: &ObjectId, + read_object: impl FnMut(&ObjectId) -> io::Result>, + ) -> io::Result { + self.materialize_with_baseline(manifest, identity, None, read_object) + } + + /// Reuse a pinned complete baseline before overlaying immutable changed object slices. + /// The source VM is never read or remapped here; both inputs are completed captures. + pub fn materialize_with_baseline( + &self, + manifest: &MemoryManifest, + identity: &ObjectId, + baseline: Option<(&MemoryManifest, &CachedMemory)>, mut read_object: impl FnMut(&ObjectId) -> io::Result>, ) -> io::Result { let started = Instant::now(); @@ -100,19 +120,53 @@ impl MemoryCache { let path = self.entry_path(identity); if let Some(file) = open_pinned(&path, length)? { return Ok(CachedMemory { + path, + identity: identity.clone(), file, regions, cache_hit: true, + reflink: false, prepare_us: started.elapsed().as_micros(), }); } - let mut staging = tempfile::Builder::new() + let staging_dir = tempfile::Builder::new() .prefix(".memory-") - .tempfile_in(&self.root)?; + .tempdir_in(&self.root)?; + let staging_path = staging_dir.path().join("memory"); + let baseline = baseline.filter(|(_, cached)| cached.regions == regions); + if let Some((previous, cached)) = baseline { + let bytes = previous.to_canonical_bytes().map_err(io::Error::other)?; + if ObjectId::from_bytes(&bytes).map_err(io::Error::other)? != cached.identity { + return Err(invalid("cache baseline does not match its pinned manifest")); + } + } + let mut reflink = false; + if let Some((_, cached)) = baseline { + let (_, strategy) = + microsandbox_utils::copy::fast_copy_with_strategy(&cached.path, &staging_path)?; + reflink = strategy == microsandbox_utils::copy::FastCopyStrategy::Reflink; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&staging_path, std::fs::Permissions::from_mode(0o600))?; + } + } + let mut staging = OpenOptions::new() + .write(true) + .create(true) + .truncate(false) + .open(&staging_path)?; // A fresh sparse file supplies all zero extents without allocating or writing RAM-sized // buffers. Only immutable nonzero object slices are copied into it. - staging.as_file().set_len(length)?; + staging.set_len(length)?; + let previous = baseline.map(|(manifest, _)| { + manifest + .extents + .iter() + .map(|extent| (extent.start, extent)) + .collect::>() + }); let mut objects = BTreeMap::>::new(); let mut region_index = 0; for extent in &manifest.extents { @@ -120,14 +174,28 @@ impl MemoryCache { { region_index += 1; } + if previous.as_ref().and_then(|map| map.get(&extent.start)) == Some(&extent) { + continue; + } + let region = ®ions[region_index]; + let offset = region.file_offset + (extent.start - region.guest_address); if let MemoryExtentContent::Object(content) = &extent.content { - let region = ®ions[region_index]; - let offset = region.file_offset + (extent.start - region.guest_address); objects.entry(content.object.clone()).or_default().push(( offset, content.object_offset, extent.length, )); + } else if baseline.is_some() { + // A newly zero range must overwrite the cloned bytes, never resurrect them. + // Bound the temporary allocation independently of guest RAM size. + staging.seek(SeekFrom::Start(offset))?; + let zeros = [0u8; 64 * 1024]; + let mut remaining = extent.length; + while remaining > 0 { + let count = remaining.min(zeros.len() as u64) as usize; + staging.write_all(&zeros[..count])?; + remaining -= count as u64; + } } } for (id, slices) in objects { @@ -150,17 +218,15 @@ impl MemoryCache { #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; - staging - .as_file() - .set_permissions(std::fs::Permissions::from_mode(0o400))?; + staging.set_permissions(std::fs::Permissions::from_mode(0o400))?; } - staging.as_file().sync_all()?; + staging.sync_all()?; // Publish the inode without replacement. Concurrent builders may do duplicate work, but // no winner can overwrite backing another VM has already pinned or mapped. - match staging.persist_noclobber(&path) { - Ok(file) => drop(file), - Err(error) if error.error.kind() == io::ErrorKind::AlreadyExists => {} - Err(error) => return Err(error.error), + match std::fs::hard_link(&staging_path, &path) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error), } #[cfg(unix)] File::open(&self.root)?.sync_all()?; @@ -171,9 +237,12 @@ impl MemoryCache { ) })?; Ok(CachedMemory { + path, + identity: identity.clone(), file, regions, cache_hit: false, + reflink, prepare_us: started.elapsed().as_micros(), }) } @@ -392,6 +461,84 @@ mod tests { assert!(!cache.evict(&id).unwrap()); } + #[test] + fn descendant_reuses_unchanged_objects_and_clears_new_zero_ranges() { + let directory = tempfile::tempdir().unwrap(); + let cache = MemoryCache::open(directory.path()).unwrap(); + let (manifest, id, bytes) = fixture(cache.page_size); + let baseline = cache + .materialize(&manifest, &id, |_| Ok(bytes.clone())) + .unwrap(); + let mut descendant = manifest.clone(); + descendant.generation += 1; + descendant.extents[0].content = MemoryExtentContent::Zero; + let changed = vec![0x7c; cache.page_size as usize]; + let changed_id = ObjectId::from_bytes(&changed).unwrap(); + descendant.extents[1].content = MemoryExtentContent::Object(ContentRef { + object: changed_id.clone(), + object_offset: 0, + }); + let descendant_id = + ObjectId::from_bytes(&descendant.to_canonical_bytes().unwrap()).unwrap(); + let mut reads = 0; + let child = cache + .materialize_with_baseline( + &descendant, + &descendant_id, + Some((&manifest, &baseline)), + |id| { + assert_eq!(id, &changed_id, "unchanged object was reread"); + reads += 1; + Ok(changed.clone()) + }, + ) + .unwrap(); + assert_eq!(reads, 1); + let mut result = vec![0; cache.page_size as usize * 3]; + child.file.read_exact_at(&mut result, 0).unwrap(); + assert!( + result[..cache.page_size as usize] + .iter() + .all(|byte| *byte == 0) + ); + assert_eq!( + &result[cache.page_size as usize..cache.page_size as usize * 2], + &changed + ); + assert_eq!(&result[cache.page_size as usize * 2..], &bytes); + baseline + .file + .read_exact_at(&mut result[..cache.page_size as usize], 0) + .unwrap(); + assert_eq!( + &result[..cache.page_size as usize], + &bytes, + "baseline was mutated" + ); + assert!(!cache.evict(&id).unwrap()); + } + + #[test] + fn reject_a_manifest_paired_with_the_wrong_baseline() { + let directory = tempfile::tempdir().unwrap(); + let cache = MemoryCache::open(directory.path()).unwrap(); + let (manifest, id, bytes) = fixture(cache.page_size); + let baseline = cache + .materialize(&manifest, &id, |_| Ok(bytes.clone())) + .unwrap(); + let mut wrong = manifest.clone(); + wrong.generation += 1; + let target = ObjectId::from_bytes(&wrong.to_canonical_bytes().unwrap()).unwrap(); + assert!( + cache + .materialize_with_baseline(&wrong, &target, Some((&wrong, &baseline)), |_| panic!( + "must reject before reads" + )) + .is_err() + ); + assert_eq!(std::fs::read_dir(directory.path()).unwrap().count(), 1); + } + #[test] fn failed_materialization_does_not_publish_or_leave_staging() { let directory = tempfile::tempdir().unwrap(); diff --git a/crates/runtime/lib/checkpoint/mod.rs b/crates/runtime/lib/checkpoint/mod.rs index acd28fc7d..5a42018cd 100644 --- a/crates/runtime/lib/checkpoint/mod.rs +++ b/crates/runtime/lib/checkpoint/mod.rs @@ -9,7 +9,7 @@ mod restore; // Re-Exports //-------------------------------------------------------------------------------------------------- -pub(crate) use coordinator::{CheckpointCoordinator, CheckpointResult}; +pub(crate) use coordinator::{CheckpointCoordinator, CheckpointResult, UserPause}; pub(crate) use disk::recover_runtime_owned_root; pub use disk::{ DiskCompactionResult, RuntimeOwnedRootChain, RuntimeOwnedRootLayer, compact_stopped_root, diff --git a/crates/runtime/lib/checkpoint/restore.rs b/crates/runtime/lib/checkpoint/restore.rs index 27fd5068d..6e9131ff9 100644 --- a/crates/runtime/lib/checkpoint/restore.rs +++ b/crates/runtime/lib/checkpoint/restore.rs @@ -157,9 +157,42 @@ impl PreparedCheckpointRestore { } /// Install all restore sources and leave the VM at an explicit activation gate. - pub(crate) fn install(self, vm: &mut msb_krun::Vm) -> RestoredAgentState { + pub(crate) fn install( + self, + vm: &mut msb_krun::Vm, + cache_root: Option, + ) -> Result { vm.set_execution_restore(self.execution); - vm.set_memory_restore(self.memory); + if let Some(root) = cache_root { + let closure = &self.memory.closure; + let cache = super::MemoryCache::open(root).map_err(|e| e.to_string())?; + let cached = cache + .materialize(closure.memory(), &closure.checkpoint().memory, |id| { + closure + .read_object(id, MAX_MEMORY_OBJECT_BYTES) + .map_err(io::Error::other) + }) + .map_err(|e| e.to_string())?; + tracing::info!( + cache_hit = cached.cache_hit, + prepare_us = cached.prepare_us, + "prepared private memory backing" + ); + let regions = cached + .regions + .into_iter() + .map(|region| msb_krun::PrivateMemoryRegion { + guest_address: region.guest_address, + length: region.length, + file_offset: region.file_offset, + }) + .collect(); + let backing = msb_krun::PrivateMemoryBacking::new(cached.file, regions) + .map_err(|e| e.to_string())?; + vm.set_private_memory_backing(backing); + } else { + vm.set_memory_restore(self.memory); + } for device in self.devices { match device { PreparedDeviceRestore::Block { device_id, state } => { @@ -169,7 +202,7 @@ impl PreparedCheckpointRestore { } } vm.set_start_paused(true); - self.agent + Ok(self.agent) } } @@ -317,7 +350,11 @@ fn parse_restored_agent_resource( ready_time_ns: parse_u64("ready_time_ns")?, agent_version: value("agent_version")?.clone(), }, - attempt_id: checkpoint_id.into(), + attempt_id: resource + .binding + .get("attempt_id") + .cloned() + .unwrap_or_else(|| checkpoint_id.into()), }) } diff --git a/crates/runtime/lib/console.rs b/crates/runtime/lib/console.rs index 5d91e5374..f9655f9b1 100644 --- a/crates/runtime/lib/console.rs +++ b/crates/runtime/lib/console.rs @@ -54,6 +54,8 @@ const NAMED_PIPE_BRIDGE_TX_POLL_INTERVAL: Duration = Duration::from_millis(1); /// transmitted by the guest agent", `rx_ring` = "bytes received by the guest /// agent". pub struct ConsoleSharedState { + /// User pause or recovery fence; checked only for new guest operations and idle policy. + pub resident_paused: Arc, /// Guest → Host: console TX thread pushes byte chunks, relay pops them. pub tx_ring: ArrayQueue>, @@ -102,6 +104,7 @@ impl ConsoleSharedState { /// Create shared state with a specific queue capacity. pub fn with_capacity(capacity: usize) -> Self { Self { + resident_paused: Arc::new(std::sync::atomic::AtomicBool::new(false)), tx_ring: ArrayQueue::new(capacity), rx_ring: ArrayQueue::new(capacity), tx_wake: WakePipe::new(), diff --git a/crates/runtime/lib/control.rs b/crates/runtime/lib/control.rs index 78dfda8ad..7400c7982 100644 --- a/crates/runtime/lib/control.rs +++ b/crates/runtime/lib/control.rs @@ -42,6 +42,12 @@ pub const CONTROL_SOCKET_EXTENSION: &str = "control.sock"; #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(tag = "op", rename_all = "snake_case")] pub enum ControlRequest { + /// Retain a resident pause until an explicit resume or stop. + Pause, + /// Resume a user-owned resident pause. + Resume, + /// Inspect user pause and full-capture availability without entering the guest. + PauseState, /// Grow the owned root disk and mounted ext4 filesystem without rebooting. RootDiskGrow { /// Target capacity in bytes. @@ -145,6 +151,9 @@ pub struct SecretValue(pub String); /// The reply to any control request. #[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct ControlResponse { + /// Resident pause status for lifecycle operations. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pause: Option, /// Guest-observed root capacities after successful filesystem expansion. #[serde(default, skip_serializing_if = "Option::is_none")] pub root_disk: Option, @@ -218,6 +227,9 @@ pub struct RootDiskGrowthResult { /// resize-capable and secrets-incapable. #[derive(Debug, Default, Clone, Copy, Serialize, Deserialize)] pub struct ControlCapabilities { + /// Resident pause/resume with identity-preserving clock correction. + #[serde(default)] + pub pause_resume: bool, /// Host control supports root growth; guest capability is checked before mutation. #[serde(default)] pub root_disk_grow: bool, @@ -238,6 +250,17 @@ pub struct ControlCapabilities { pub checkpoint_create: bool, } +/// Host-confirmed resident suspension state. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PauseControlState { + /// Whether a user pause is currently held. + pub paused: bool, + /// Whether a failed operation has fenced ordinary resume and mutations. + pub recovery_required: bool, + /// Why full capture cannot use this pause, if guest preparation is unavailable. + pub capture_unavailable: Option, +} + /// Memory sizing carried in [`ControlResponse`], all in MiB. #[derive(Debug, Default, Clone, Copy, Serialize, Deserialize)] pub struct MemoryControlState { @@ -582,6 +605,7 @@ mod tests { let response = ControlResponse { ok: true, capabilities: Some(ControlCapabilities { + pause_resume: true, root_disk_grow: true, disk_compact: true, cpu_resize: true, diff --git a/crates/runtime/lib/control/executor.rs b/crates/runtime/lib/control/executor.rs index 254778aba..d89329b99 100644 --- a/crates/runtime/lib/control/executor.rs +++ b/crates/runtime/lib/control/executor.rs @@ -17,7 +17,7 @@ use super::{ CheckpointCaptureIntent, CheckpointControlState, ControlCapabilities, ControlRequest, ControlResponse, CpuControlState, MemoryControlState, SecretLiveChange, }; -use crate::checkpoint::{CheckpointCoordinator, CheckpointResult}; +use crate::checkpoint::{CheckpointCoordinator, CheckpointResult, UserPause}; use crate::vm::VmConfig; use microsandbox_protocol::bootstrap::GuestBootstrap; @@ -93,6 +93,8 @@ pub struct ControlEnvelopeResponse { /// One in-process authority for all host-owned runtime mutations. pub struct RuntimeControlExecutor { + pause_observation: std::sync::RwLock, + resident_paused: std::sync::Arc, vm: msb_krun::VmControl, #[cfg(feature = "net")] secrets: Option, @@ -106,6 +108,7 @@ struct ExecutorState { dedup: BTreeMap, dedup_order: VecDeque, checkpoint: CheckpointCoordinator, + user_pause: Option, } #[derive(Clone)] @@ -130,6 +133,7 @@ impl RuntimeControlExecutor { guest_bootstrap: &GuestBootstrap, runtime: tokio::runtime::Handle, agent_sock: &Path, + resident_paused: std::sync::Arc, ) -> Result { let runtime_boot_id = new_runtime_boot_id(); persist_runtime_boot_id(runtime_dir, &runtime_boot_id) @@ -142,6 +146,16 @@ impl RuntimeControlExecutor { agent_sock, )?; Ok(Self { + pause_observation: std::sync::RwLock::new(ControlResponse { + ok: true, + pause: Some(super::PauseControlState { + paused: false, + recovery_required: false, + capture_unavailable: None, + }), + ..Default::default() + }), + resident_paused, vm, #[cfg(feature = "net")] secrets, @@ -152,14 +166,22 @@ impl RuntimeControlExecutor { dedup: BTreeMap::new(), dedup_order: VecDeque::new(), checkpoint, + user_pause: None, }), }) } /// Execute a legacy command through the same exclusive mutation path. pub fn execute_legacy(&self, command: ControlRequest) -> ControlResponse { + // Observation remains available during a long capture. It describes the last completed + // lifecycle transition; it neither borrows nor releases mutation/pause authority. + if matches!(command, ControlRequest::PauseState) { + return self.pause_observation.read().unwrap().clone(); + } let mut state = self.state.lock().unwrap(); - self.execute_locked(&mut state, command) + let response = self.execute_locked(&mut state, command); + *self.pause_observation.write().unwrap() = pause_response(&state); + response } /// Execute a fenced, idempotent control request. @@ -223,6 +245,7 @@ impl RuntimeControlExecutor { let request_id = envelope.request_id; let response = self.execute_locked(&mut state, envelope.command); + *self.pause_observation.write().unwrap() = pause_response(&state); let response = ControlEnvelopeResponse { request_id: request_id.clone(), runtime: snapshot_state(&state), @@ -250,8 +273,17 @@ impl RuntimeControlExecutor { | ControlRequest::SecretsUpdate { .. } | ControlRequest::CheckpointCreate { .. } | ControlRequest::DiskCompact { dry_run: false, .. } + | ControlRequest::Pause + | ControlRequest::Resume ); - if mutation && state.lifecycle != RuntimeLifecycle::Running { + let resident_operation = state.user_pause.is_some() + && matches!( + request, + ControlRequest::Pause + | ControlRequest::Resume + | ControlRequest::CheckpointCreate { .. } + ); + if mutation && state.lifecycle != RuntimeLifecycle::Running && !resident_operation { return control_error( "runtime_busy", "runtime lifecycle does not currently admit mutations", @@ -259,6 +291,45 @@ impl RuntimeControlExecutor { } let response = match request { + ControlRequest::Pause => { + if state.user_pause.is_none() { + self.resident_paused + .store(true, std::sync::atomic::Ordering::Release); + let attempt = format!("pause-{}-{}", state.runtime_boot_id, state.revision); + state.lifecycle = RuntimeLifecycle::Quiescing; + match state.checkpoint.pause_user(&self.vm, &attempt) { + Ok(paused) => { + state.user_pause = Some(paused); + state.lifecycle = RuntimeLifecycle::Quiesced; + } + Err(error) => { + state.lifecycle = if error.keep_paused { + RuntimeLifecycle::Quiesced + } else { + RuntimeLifecycle::Running + }; + self.resident_paused + .store(error.keep_paused, std::sync::atomic::Ordering::Release); + return control_error("pause_failed", error.to_string()); + } + } + } + pause_response(state) + } + ControlRequest::Resume => { + if let Some(paused) = state.user_pause.take() { + if let Err(error) = state.checkpoint.resume_user(&self.vm, &paused) { + // A failed resume becomes recovery-owned, never a public resume token. + state.lifecycle = RuntimeLifecycle::Quiesced; + return control_error("resume_recovery_required", error.to_string()); + } + state.lifecycle = RuntimeLifecycle::Running; + self.resident_paused + .store(false, std::sync::atomic::Ordering::Release); + } + pause_response(state) + } + ControlRequest::PauseState => pause_response(state), ControlRequest::RootDiskGrow { size_bytes } => { match state.checkpoint.grow_root(&self.vm, size_bytes) { Ok(root_disk) => ControlResponse { @@ -315,14 +386,23 @@ impl RuntimeControlExecutor { microsandbox_image::checkpoint::CaptureIntent::TransparentTransfer } }, - None, + state.user_pause.as_ref(), ) { Ok(result) => { - state.lifecycle = RuntimeLifecycle::Running; + state.lifecycle = if state.user_pause.is_some() { + RuntimeLifecycle::Quiesced + } else { + RuntimeLifecycle::Running + }; checkpoint_response(Some(result), true, None, None) } Err(error) => { - state.lifecycle = if error.keep_paused { + // A failed rebind can invalidate the user's original pause authority. + // Recovery-owned suspension must never be released by ordinary resume. + if error.keep_paused { + state.user_pause = None; + } + state.lifecycle = if error.keep_paused || state.user_pause.is_some() { RuntimeLifecycle::Quiesced } else { RuntimeLifecycle::Running @@ -338,6 +418,10 @@ impl RuntimeControlExecutor { } request => self.handle_request(request), }; + self.resident_paused.store( + state.lifecycle != RuntimeLifecycle::Running, + std::sync::atomic::Ordering::Release, + ); if mutation && response.ok { match state.revision.checked_add(1) { Some(revision) => state.revision = revision, @@ -397,6 +481,7 @@ impl RuntimeControlExecutor { checkpoint_create: true, disk_compact: true, root_disk_grow: true, + pause_resume: self.vm.clock_sync_supported(), }), ..Default::default() }, @@ -416,6 +501,9 @@ impl RuntimeControlExecutor { ControlRequest::CpuState => cpu(self.vm.cpu_state()), ControlRequest::SecretsUpdate { changes } => self.handle_secrets_update(changes), ControlRequest::CheckpointCreate { .. } + | ControlRequest::Pause + | ControlRequest::Resume + | ControlRequest::PauseState | ControlRequest::DiskCompact { .. } | ControlRequest::RootDiskGrow { .. } => { unreachable!("checkpoint requests are handled by the executor lifecycle path") @@ -484,6 +572,22 @@ fn new_runtime_boot_id() -> String { format!("boot_{}", hex::encode(bytes)) } +fn pause_response(state: &ExecutorState) -> ControlResponse { + ControlResponse { + ok: true, + pause: Some(super::PauseControlState { + paused: state.user_pause.is_some(), + recovery_required: state.lifecycle == RuntimeLifecycle::Quiesced + && state.user_pause.is_none(), + capture_unavailable: state + .user_pause + .as_ref() + .and_then(|paused| paused.capture_unavailable.clone()), + }), + ..Default::default() + } +} + fn checkpoint_response( result: Option, ok: bool, diff --git a/crates/runtime/lib/launch.rs b/crates/runtime/lib/launch.rs index 06f165e86..deb72eb13 100644 --- a/crates/runtime/lib/launch.rs +++ b/crates/runtime/lib/launch.rs @@ -74,6 +74,14 @@ pub struct LaunchConfig { #[serde(default)] pub thp: TransparentHugePagePolicy, + /// Explicit memory representation selected at construction. + #[serde(default)] + pub memory_snapshot: microsandbox_types::MemorySnapshotMode, + + /// Backend-resolved protected cache; required only for explicit CoW construction. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub memory_cache_dir: Option, + /// Per-writable-raw-disk hard budget for buffered host dirty data. #[serde(default, skip_serializing_if = "Option::is_none")] pub block_writeback_limit_bytes: Option, diff --git a/crates/runtime/lib/relay.rs b/crates/runtime/lib/relay.rs index 3c3a2d267..199a2f216 100644 --- a/crates/runtime/lib/relay.rs +++ b/crates/runtime/lib/relay.rs @@ -800,6 +800,7 @@ impl AgentRelay { next_id_clone, id_start, id_end_exclusive, + Arc::clone(&self.shared.resident_paused), )); } Err(e) => { @@ -1225,6 +1226,7 @@ async fn client_reader_task( next_session_id: Arc, id_start: u32, id_end_exclusive: u32, + resident_paused: Arc, ) { loop { let frame = match read_raw_frame(&mut reader).await { @@ -1250,6 +1252,31 @@ async fn client_reader_task( break; } + // Reject new work on the host: a paused guest cannot return its own error. + // Existing stream data retains the normal bounded backpressure path. + if is_session_start && resident_paused.load(Ordering::Acquire) { + let error = CoreError { + kind: microsandbox_protocol::core::CoreErrorKind::InvalidSession, + message: "sandbox is paused; resume it before starting guest work".into(), + offending_type: None, + workload_failure: None, + }; + if let Ok(reply) = Message::with_payload(MessageType::CoreError, frame.id, &error) { + let mut bytes = Vec::new(); + if codec::encode_to_buf(&reply, &mut bytes).is_ok() { + let writer = clients + .lock() + .await + .get(&slot) + .map(|client| client.write_tx.clone()); + if let Some(writer) = writer { + let _ = writer.send(Bytes::from(bytes)).await; + } + } + } + continue; + } + // Forward shutdown to agentd (via the agent_tx send below) so the // guest can sync filesystems and power off cleanly. Also notify the // caller so it can start the flush-grace fallback timer — if the diff --git a/crates/runtime/lib/vm.rs b/crates/runtime/lib/vm.rs index 17634912d..eaa623024 100644 --- a/crates/runtime/lib/vm.rs +++ b/crates/runtime/lib/vm.rs @@ -280,6 +280,12 @@ pub struct VmConfig { /// Guest transparent huge-page policy selected at boot. pub thp: microsandbox_types::TransparentHugePagePolicy, + /// Explicit construction-time memory representation. + pub memory_snapshot: microsandbox_types::MemorySnapshotMode, + + /// Protected memory cache resolved by the sandbox's owning local backend. + pub memory_cache_dir: Option, + /// Number of virtual CPUs online at boot. pub vcpus: u8, @@ -1012,6 +1018,7 @@ fn run(mut config: Config) -> RuntimeResult { &resolved_bootstrap, tokio_rt.handle().clone(), &config.agent_sock_path, + Arc::clone(&shared.resident_paused), ); let context = crate::control::ControlContext { executor: match executor { @@ -1251,6 +1258,7 @@ fn run(mut config: Config) -> RuntimeResult { { let shutdown_exit_handle = exit_handle.clone(); let shutdown_reason = Arc::clone(&exit_reason); + let shutdown_paused = Arc::clone(&shared.resident_paused); tokio_rt.spawn(async move { if relay_drain_rx.recv().await.is_some() { shutdown_reason.store( @@ -1260,7 +1268,9 @@ fn run(mut config: Config) -> RuntimeResult { tracing::info!( "core.shutdown forwarded to agentd, allowing flush window before host fallback" ); - tokio::time::sleep(shutdown_flush_timeout).await; + if !shutdown_paused.load(std::sync::atomic::Ordering::Acquire) { + tokio::time::sleep(shutdown_flush_timeout).await; + } tracing::info!("flush window elapsed, triggering host exit"); shutdown_exit_handle.trigger(); } @@ -1312,6 +1322,13 @@ fn run(mut config: Config) -> RuntimeResult { } } + if startup_shared + .resident_paused + .load(std::sync::atomic::Ordering::Acquire) + { + startup_exit_handle.trigger(); + return; + } match request_guest_shutdown(&startup_shared) { Ok(()) => { tokio::time::sleep(startup_shutdown_flush_timeout).await; @@ -1343,6 +1360,12 @@ fn run(mut config: Config) -> RuntimeResult { let mut interval = tokio::time::interval(Duration::from_secs(1)); loop { interval.tick().await; + if heartbeat_shared + .resident_paused + .load(std::sync::atomic::Ordering::Acquire) + { + continue; + } let decision = heartbeat_reader.check(idle_timeout); match decision { @@ -2070,12 +2093,28 @@ fn build_vm( &restore.checkpoint_root, ) .map_err(|error| RuntimeError::Custom(format!("prepare checkpoint restore: {error}")))?; - Some(prepared.install(&mut vm)) + let cache_root = (config.vm.memory_snapshot == microsandbox_types::MemorySnapshotMode::Cow) + .then(|| { + config.vm.memory_cache_dir.clone().ok_or_else(|| { + RuntimeError::Custom( + "CoW memory requires its backend-resolved cache directory".into(), + ) + }) + }) + .transpose()?; + Some( + prepared + .install(&mut vm, cache_root) + .map_err(RuntimeError::Custom)?, + ) } else { None }; let bootstrap_frame = if restored_agent.is_none() { + if config.vm.memory_snapshot == microsandbox_types::MemorySnapshotMode::Cow { + vm.set_private_memory_boot(true); + } Some(encode_bootstrap_frame(&bootstrap)?) } else { None @@ -2625,6 +2664,15 @@ fn spawn_parent_watchdog( Ok(ParentWatchdogSignal::ParentExited) => { tracing::info!("creator process exited; stopping attached sandbox"); exit_reason.store(EXIT_REASON_PARENT_EXIT, std::sync::atomic::Ordering::SeqCst); + // A suspended guest cannot process shutdown. Release the resident VM + // directly without thawing user workloads merely to stop them. + if shared + .resident_paused + .load(std::sync::atomic::Ordering::Acquire) + { + exit_handle.trigger(); + return; + } if let Err(err) = request_guest_shutdown(&shared) { tracing::warn!(error = %err, "parent-watch shutdown request failed"); } else { diff --git a/docs/sandboxes/lifecycle.mdx b/docs/sandboxes/lifecycle.mdx index 703d74824..bec17de40 100644 --- a/docs/sandboxes/lifecycle.mdx +++ b/docs/sandboxes/lifecycle.mdx @@ -27,6 +27,7 @@ stateDiagram-v2 |--------|-------------| | **Creating** | The VM is booting. The kernel is loaded, the filesystem is mounted, and the guest agent is initializing (configuring network, setting up the environment). | | **Running** | The guest agent is ready. You can call `exec`, `shell`, and `fs`. | +| **Paused** | The VM is resident but suspended. Use `resume` to continue its processes, or `stop` to shut it down. | | **Draining** | Graceful shutdown in progress. Existing commands run to completion, but new `exec` calls are rejected. Transitions to Stopped when all commands finish. | | **Stopped** | The VM has shut down. Sandbox configuration and state are persisted to the database and can be restarted. | | **Crashed** | The VM exited unexpectedly (e.g., kernel panic, OOM kill). | @@ -91,6 +92,45 @@ msb run -d python --name worker +## Pause and resume + +Pause keeps the local VM and its RAM allocated without creating a snapshot. Resume continues the same processes and corrects guest wall clock before releasing prepared workloads. New guest commands fail while paused; host inspection and stop remain available. Network peers may time out during a long pause. + + +```bash CLI +msb pause worker +msb snapshot create paused-state --from worker --full +msb resume worker +``` + +```rust Rust +let worker = Sandbox::get("worker").await?; +worker.pause().await?; +worker.resume().await?; +``` + +```python Python +worker = await Sandbox.get("worker") +await worker.pause() +await worker.resume() +``` + +```typescript TypeScript +const worker = await Sandbox.get("worker"); +await worker.pause(); +await worker.resume(); +``` + +```go Go +worker, err := m.GetSandbox(ctx, "worker") +if err != nil { return err } +if err := worker.Pause(ctx); err != nil { return err } +if err := worker.Resume(ctx); err != nil { return err } +``` + + +Pausing an already-paused VM and resuming an already-running VM are no-ops. Full snapshots taken while paused leave it paused. A stopped VM has no resident execution state: use `start` to boot it again. Pause/resume requires a matching runtime and guest kernel; cloud sandboxes are not supported. + ## Stop and restart Stopping gracefully terminates guest processes and shuts down the VM. The sandbox moves to `Stopped` and can be restarted later with all its configuration preserved. diff --git a/docs/sandboxes/snapshots.mdx b/docs/sandboxes/snapshots.mdx index af8bba3a2..2427109a7 100644 --- a/docs/sandboxes/snapshots.mdx +++ b/docs/sandboxes/snapshots.mdx @@ -9,7 +9,7 @@ icon: "code-branch" A snapshot is a portable artifact that can hold either a sandbox's writable disk state or a full checkpoint of a running sandbox. Managed and flat OCI roots are supported; the descriptor preserves the root layout so a flat `rootfs.raw` is never mistaken for a managed OverlayFS upper. Move it with `scp`, archive it as `.msnap`, or create a child sandbox from it. -Disk snapshots capture stopped or crashed sandboxes. Full snapshots use `--full` and capture a running sandbox without requiring the user to pause it first. +Disk snapshots capture stopped or crashed sandboxes. Full snapshots use `--full` and capture a running or user-paused sandbox. You do not need to pause it first; if you do, capture leaves it paused. ## What gets captured @@ -19,10 +19,25 @@ Disk snapshots capture stopped or crashed sandboxes. Full snapshots use `--full` | Mode | Source | Captured state | Restore behavior | | ---- | ------ | -------------- | ---------------- | | Disk (default) | Stopped or crashed sandbox | Writable disk closure and pinned image | Cold-boots a fresh VM | -| Full | Running sandbox | Disk, memory, vCPU, device, and admitted resource state | Eagerly resumes the captured execution in a child VM | +| Full | Running or user-paused sandbox | Disk, memory, vCPU, device, and admitted resource state | Resumes the captured execution in a child VM | Both modes produce the same schema-1 `snapshot.json` descriptor. Its closed `state.kind` is either `file` or `checkpoint`, and `root_disk.layout` records `managed`, `flat`, or `tmpfs`. Checkpoint payloads live in a content-addressed closure rather than a standalone disk file. A tmpfs root has no stopped disk snapshot because its writable state exists only in memory, but it is included in a running full snapshot; such a snapshot must be resumed and cannot use `--disk-only`. +## Share restored memory with CoW + +On Linux and macOS, request `--memory-snapshot cow` when creating a sandbox to use private file-backed memory. Children restored from the same full snapshot can share clean memory pages; writes stay private to each child. Each child must opt in explicitly. The default, `standard`, keeps eager anonymous memory. + +```bash +msb create alpine --name baseline --root-disk flat:1G --memory-snapshot cow +msb snapshot create ready --from baseline --full +msb create --name worker-a --from-snapshot ready --memory-snapshot cow +msb create --name worker-b --from-snapshot ready --memory-snapshot cow +``` + +The SDK creation options are Rust `.memory_snapshot(MemorySnapshotMode::Cow)`, Python `memory_snapshot=MemorySnapshotMode.COW`, TypeScript `.memorySnapshot("cow")`, and Go `WithMemorySnapshot(MemorySnapshotCow)`. + +Snapshots are still manual. CoW uses a protected local memory cache; the first uncached restore must build it, while later children reuse it. Repeated captures use filesystem reflinks when available. Removing the input archive does not invalidate a running child's backing. Windows CoW and explicit NUMA placement with CoW are not supported yet; requests fail instead of silently switching modes. + ## Quick start You'll usually reach for the CLI first: diff --git a/packages/microsandbox-types/rust/lib/cloud.rs b/packages/microsandbox-types/rust/lib/cloud.rs index d69516a8b..941ca70aa 100644 --- a/packages/microsandbox-types/rust/lib/cloud.rs +++ b/packages/microsandbox-types/rust/lib/cloud.rs @@ -972,6 +972,7 @@ impl TryFrom for SandboxSpec { }; let resources = SandboxResources { + memory_snapshot: crate::MemorySnapshotMode::Standard, cpus: spec.resources.vcpus, memory_mib: spec.resources.memory_mib, // The cloud wire type has no boot-capacity fields yet; treat the diff --git a/packages/microsandbox-types/rust/lib/domain.rs b/packages/microsandbox-types/rust/lib/domain.rs index 686c370e9..39ca7f978 100644 --- a/packages/microsandbox-types/rust/lib/domain.rs +++ b/packages/microsandbox-types/rust/lib/domain.rs @@ -832,6 +832,9 @@ pub struct SandboxSpec { #[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] #[cfg_attr(feature = "ts", derive(ts_rs::TS))] pub struct SandboxResources { + /// Explicit construction-time memory representation; does not create automatic snapshots. + #[serde(default, skip_serializing_if = "MemorySnapshotMode::is_standard")] + pub memory_snapshot: MemorySnapshotMode, /// Number of virtual CPUs currently presented to the guest at boot. pub cpus: u8, @@ -857,6 +860,19 @@ pub struct SandboxResources { pub thp: TransparentHugePagePolicy, } +/// Memory representation selected when constructing a sandbox. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +#[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub enum MemorySnapshotMode { + /// Anonymous memory with eager full restore. + #[default] + Standard, + /// Private file-backed memory, sharing immutable pages between restored children. + Cow, +} + /// Controls how Microsandbox places vCPU threads on host processors. #[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] @@ -1525,9 +1541,17 @@ impl Default for RootfsSource { } } +impl MemorySnapshotMode { + /// Whether the default anonymous representation was selected. + pub fn is_standard(&self) -> bool { + *self == Self::Standard + } +} + impl Default for SandboxResources { fn default() -> Self { Self { + memory_snapshot: MemorySnapshotMode::Standard, cpus: DEFAULT_SANDBOX_CPUS, memory_mib: DEFAULT_SANDBOX_MEMORY_MIB, max_cpus: DEFAULT_SANDBOX_CPUS, @@ -1558,6 +1582,8 @@ impl<'de> Deserialize<'de> for SandboxResources { placement_profile: Option, #[serde(default)] thp: TransparentHugePagePolicy, + #[serde(default)] + memory_snapshot: MemorySnapshotMode, } let raw = RawResources::deserialize(deserializer)?; @@ -1572,6 +1598,7 @@ impl<'de> Deserialize<'de> for SandboxResources { cpu_placement: raw.cpu_placement, placement_profile: raw.placement_profile, thp: raw.thp, + memory_snapshot: raw.memory_snapshot, }) } } @@ -2859,6 +2886,20 @@ impl fmt::Display for NetworkRateLimitDirection { mod tests { use super::*; + #[test] + fn memory_snapshot_policy_is_explicit_and_old_resources_default_to_standard() { + let standard = serde_json::to_value(SandboxResources::default()).unwrap(); + assert!(standard.get("memory_snapshot").is_none()); + let decoded: SandboxResources = serde_json::from_value(standard.clone()).unwrap(); + assert_eq!(decoded.memory_snapshot, MemorySnapshotMode::Standard); + let mut cow = standard; + cow["memory_snapshot"] = serde_json::json!("cow"); + let decoded: SandboxResources = serde_json::from_value(cow.clone()).unwrap(); + assert_eq!(decoded.memory_snapshot, MemorySnapshotMode::Cow); + cow["memory_snapshot"] = serde_json::json!("automatic"); + assert!(serde_json::from_value::(cow).is_err()); + } + fn tmpfs_mount(guest: &str) -> VolumeMount { VolumeMount::Tmpfs { guest: guest.to_owned(), diff --git a/packages/microsandbox-types/rust/lib/lib.rs b/packages/microsandbox-types/rust/lib/lib.rs index 99f691511..9af47f799 100644 --- a/packages/microsandbox-types/rust/lib/lib.rs +++ b/packages/microsandbox-types/rust/lib/lib.rs @@ -31,16 +31,16 @@ pub use domain::{ DEFAULT_SANDBOX_CPUS, DEFAULT_SANDBOX_MEMORY_MIB, DeploymentProfile, Destination, DestinationGroup, Direction, DiskImageFormat, DnsConfig, EnvVar, FlatClone, HandoffInit, HostPattern, HostPermissions, InterceptCaConfig, InterfaceOverrides, LogSource, - MAX_SECRET_PLACEHOLDER_BYTES, MemoryPlacement, MountOptions, NamedVolumeCreate, - NamedVolumeMode, NetworkPolicy, NetworkRateLimitDirection, NetworkRateLimiterConfig, - NetworkSpec, NumaPlacement, OciRootfsSource, Patch, PlacementProfile, PortProtocol, PortRange, - Protocol, PublishedPortSpec, PullPolicy, RateLimitConfigError, RateLimiterConfig, Rlimit, - RlimitResource, RootDisk, RootfsSource, Rule, SandboxLogLevel, SandboxPolicy, SandboxResources, - SandboxRuntimeOptions, SandboxSpec, ScopedUpstreamCaCert, ScopedVerifyUpstream, - SecretConfigError, SecretEntry, SecretInjection, SecretsConfig, SecurityProfile, SnapshotSpec, - StatVirtualization, TlsConfig, TokenBucketConfig, TransparentHugePagePolicy, ViolationAction, - VolumeKind, VolumeMount, VolumeSpec, VsockRouteSpec, VsockSocketType, VsockSpec, - canonicalize_volume_mounts, + MAX_SECRET_PLACEHOLDER_BYTES, MemoryPlacement, MemorySnapshotMode, MountOptions, + NamedVolumeCreate, NamedVolumeMode, NetworkPolicy, NetworkRateLimitDirection, + NetworkRateLimiterConfig, NetworkSpec, NumaPlacement, OciRootfsSource, Patch, PlacementProfile, + PortProtocol, PortRange, Protocol, PublishedPortSpec, PullPolicy, RateLimitConfigError, + RateLimiterConfig, Rlimit, RlimitResource, RootDisk, RootfsSource, Rule, SandboxLogLevel, + SandboxPolicy, SandboxResources, SandboxRuntimeOptions, SandboxSpec, ScopedUpstreamCaCert, + ScopedVerifyUpstream, SecretConfigError, SecretEntry, SecretInjection, SecretsConfig, + SecurityProfile, SnapshotSpec, StatVirtualization, TlsConfig, TokenBucketConfig, + TransparentHugePagePolicy, ViolationAction, VolumeKind, VolumeMount, VolumeSpec, + VsockRouteSpec, VsockSocketType, VsockSpec, canonicalize_volume_mounts, }; pub use error::{TypesError, TypesResult}; pub use modify::{ diff --git a/scripts/smoke/cli/cow-memory-lifecycle.py b/scripts/smoke/cli/cow-memory-lifecycle.py new file mode 100644 index 000000000..100b6b023 --- /dev/null +++ b/scripts/smoke/cli/cow-memory-lifecycle.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Isolated #8 live smoke matrix; every started sandbox is stopped in finally.""" +import json +import os +from pathlib import Path +import subprocess +import time + +binary = os.environ["MSB_PATH"] +root = Path(os.environ["STACK8_OUT"]) +root.mkdir(parents=True, exist_ok=True) +prefix = os.environ.get("STACK8_PREFIX", "cow8") +mode = os.environ.get("STACK8_MODE", "cow") +layout = os.environ.get("STACK8_LAYOUT", "flat:512M") +rows = [] +names = [] + +def run(label, *args, expected=0, timeout=120): + started = time.perf_counter() + result = subprocess.run([binary, *args], text=True, capture_output=True, timeout=timeout) + elapsed = (time.perf_counter() - started) * 1000 + (root / (label + ".stdout")).write_text(result.stdout) + (root / (label + ".stderr")).write_text(result.stderr) + row = {"case": label, "ms": round(elapsed, 2), "exit": result.returncode} + rows.append(row) + print(json.dumps(row), flush=True) + if expected is not None and result.returncode != expected: + raise RuntimeError(f"{label}: {result.stderr[-3000:]}") + return result + +try: + source = prefix + "-source" + names.append(source) + run("fresh-" + mode, "run", "-d", "-n", source, "--memory-snapshot", mode, + "--root-disk", layout, "--memory", "256M", "--cpus", "2", "alpine", + "--", "sh", "-c", "mkdir -p /dev/shm; echo captured > /dev/shm/cow-marker; i=0; while :; do echo $i > /tmp/cow-counter; i=$((i+1)); sleep 0.05; done") + run("marker-source", "exec", source, "--", "cat", "/dev/shm/cow-marker") + boot_id = run("boot-id-before", "exec", source, "--", "cat", "/proc/sys/kernel/random/boot_id").stdout.strip() + process = run("process-before", "exec", source, "--", "sh", "-c", "for p in /proc/[0-9]*/cmdline; do tr '\\0' ' ' < $p; echo; done").stdout + assert "cow-counter" in process + snap = prefix + "-full" + run("first-full", "snapshot", "create", snap, "--from", source, "--full", "--info") + run("pause", "pause", source) + run("pause-idempotent", "pause", source) + inspected = run("paused-inspect", "inspect", source, "--format", "json") + assert json.loads(inspected.stdout)["status"] == "Paused" + refusal = run("paused-exec", "exec", source, "--", "true", expected=None, timeout=10) + assert refusal.returncode != 0, "paused exec must fail promptly" + run("paused-full-1", "snapshot", "create", prefix + "-paused1", "--from", source, "--full", "--info") + run("paused-full-2", "snapshot", "create", prefix + "-paused2", "--from", source, "--full", "--info") + time.sleep(float(os.environ.get("STACK8_PAUSE_SECONDS", "5"))) + run("resume", "resume", source) + run("resume-idempotent", "resume", source) + assert run("boot-id-after", "exec", source, "--", "cat", "/proc/sys/kernel/random/boot_id").stdout.strip() == boot_id + guest_time = run("wall-clock-after", "exec", source, "--", "date", "+%s").stdout.strip() + assert abs(time.time() - int(guest_time)) < 3, f"guest wall clock stale: {guest_time}" + first_counter = run("counter-after", "exec", source, "--", "cat", "/tmp/cow-counter").stdout.strip() + time.sleep(0.2) + next_counter = run("counter-progress", "exec", source, "--", "cat", "/tmp/cow-counter").stdout.strip() + assert int(next_counter) > int(first_counter), "original workload must continue after resume" + run("marker-after-resume", "exec", source, "--", "cat", "/dev/shm/cow-marker") + for suffix in ("a", "b"): + child = prefix + "-" + suffix + names.append(child) + run("restore-" + suffix, "create", "-n", child, "--from-snapshot", snap, + "--memory-snapshot", mode, "--info") + result = run("marker-" + suffix, "exec", child, "--", "cat", "/dev/shm/cow-marker") + assert result.stdout.strip() == "captured" + run("mutate-a", "exec", prefix + "-a", "--", "sh", "-c", "echo private-a > /dev/shm/cow-marker") + assert run("isolation-b", "exec", prefix + "-b", "--", "cat", "/dev/shm/cow-marker").stdout.strip() == "captured" + assert run("isolation-source", "exec", source, "--", "cat", "/dev/shm/cow-marker").stdout.strip() == "captured" + archive = str(root / "direct.msnap") + run("direct-full", "snapshot", "create", prefix + "-direct", "--from", source, + "--full", "--archive", archive, "--info") + child = prefix + "-archive" + names.append(child) + run("direct-restore", "create", "-n", child, "--from-snapshot", archive, + "--memory-snapshot", mode, "--info") + Path(archive).unlink() + run("archive-unlink-survival", "exec", child, "--", "cat", "/dev/shm/cow-marker") + run("pause-for-stop", "pause", source) + run("stop-paused", "stop", source, timeout=20) + run("child-after-source-stop", "exec", prefix + "-a", "--", "cat", "/dev/shm/cow-marker") +finally: + for name in reversed(names): + try: + run("cleanup-" + name, "stop", name, expected=None, timeout=20) + except Exception as error: + rows.append({"case": "cleanup-" + name, "error": str(error)}) + (root / "results.json").write_text(json.dumps(rows, indent=2)) diff --git a/scripts/smoke/reports/cow-memory-lifecycle-2026-09-07.md b/scripts/smoke/reports/cow-memory-lifecycle-2026-09-07.md new file mode 100644 index 000000000..6cb3b6507 --- /dev/null +++ b/scripts/smoke/reports/cow-memory-lifecycle-2026-09-07.md @@ -0,0 +1,44 @@ +# CoW memory and resident lifecycle — 2026-09-07 + +Status: development integration and live smoke coverage, not full platform or performance qualification. Microsandbox #8 remains stacked directly on #7 `ce04099b`, with libkrun `94d680b21bf7ea7c2bed5262ed211833bd4379bd` and firmware `6cca413ac248f63e65d4ea4748b3bc36cd1b22f3`. The kernel and agentd used below were built from matching development sources on the authorized OVH host, including the ARM64 guest artifacts used on macOS and Windows. + +## Reproduce + +Use `scripts/smoke/cli/cow-memory-lifecycle.py` with `MSB_PATH`, an isolated `MSB_HOME`, matching `MSB_LIBKRUNFW_PATH`, an output directory in `STACK8_OUT`, and a fresh `STACK8_PREFIX`. Select `STACK8_LAYOUT=flat:512M`, `512M`, or `tmpfs`, `STACK8_MODE=cow` or `standard`, and optionally `STACK8_PAUSE_SECONDS=10`. The workload uses Alpine, 256 MiB RAM, and two vCPUs. The runner records each command's elapsed wall time, stdout, stderr, and exit code, and attempts to stop every sandbox it starts in `finally`. + +The opt-in language SDK tests are `sdk/python/tests/test_cow_lifecycle.py`, `sdk/node-ts/tests/cow-lifecycle.test.ts`, and `sdk/go/cow_lifecycle_test.go`. Set `MSB_COW_LIVE=1`; Go also requires tags `cow_live microsandbox_ffi_path` and `MICROSANDBOX_FFI_PATH` pointing to its matching native library. + +## Observed coverage + +Linux/KVM x86-64 passed CoW flat, managed, and tmpfs roots, plus a standard-memory flat-root baseline. macOS/HVF ARM64 passed the same root/memory variants. The checks exercise fresh construction, running full capture, idempotent pause/resume, host-observed Paused status, prompt rejection of new guest exec while paused, two successive full captures while retaining pause, installed-snapshot restore into two children, private child writes, direct full `.msnap` capture/restore, survival after input-archive unlink, and stop from paused. Completed runs stopped their test VMs; retained snapshot/cache artifacts remain in the isolated test homes for inspection. + +The later Linux flat/managed/tmpfs/standard runs and macOS tmpfs run additionally assert unchanged Linux boot ID across ordinary pause/resume, resumed progress of the original counter workload, and guest wall clock within three seconds of the host after a ten-second pause. These checks do not constitute host-suspend, every clock-failure, or VM Generation ID notification testing. + +The Python, Node, and Go live SDK checks passed on macOS: create with explicit CoW, pause, capture while paused, resume through a handle, restore a child, verify tmpfs contents and source/child write isolation, and stop a paused child. Windows ARM64 firmware build/export/load checks passed; native runtime compilation and live lifecycle coverage are still in progress. Windows explicit CoW requests remain unsupported and must fail without an eager fallback. + +## Individual debug-build timings + +These are single observations, not p50/p95, release benchmarks, or claims of speedup. Build activity and cache warmth varied. CLI wall time includes client/process/setup work and must not be quoted as stop-the-world duration. + +| Operation | Linux managed CoW (ms) | macOS managed CoW (ms) | macOS tmpfs CoW (ms) | +| --- | ---: | ---: | ---: | +| First running full capture | 2389.55 | 2000.17 | 1741.27 | +| Resident pause | 9.26 | 19.24 | 14.58 | +| First capture while user-paused | 2679.02 | 2132.41 | 1039.84 | +| Second capture while user-paused | 471.60 | 743.66 | 452.17 | +| Resident resume | 10.46 | 16.22 | 35.71 | +| Restore child A | 158.56 | 572.77 | 259.46 | +| Restore child B | 155.39 | 590.81 | 241.84 | +| Direct full archive capture | 1968.17 | 3050.15 | 2925.98 | +| Direct full archive restore | 396.31 | 1502.71 | 990.42 | +| Stop user-paused source | 111.19 | 119.05 | 116.72 | + +The macOS flat run logged APFS reflink reuse for repeated cache construction, including approximately 8 ms for one unchanged paused generation. This is cache preparation only, not total snapshot latency. A warm cache lookup logged 124 microseconds in one restore; that is not end-to-end restore latency or a physical-sharing measurement. + +## Other validation and remaining work + +Rust SDK library tests: 668 passed, three ignored. Runtime tests: 179 passed. CLI library tests: 315 passed, plus three enabled CLI integration checks; platform-dependent ignored tests remain ignored. Node: 137 unit tests and typecheck passed. Go unit and native-FFI smoke tests passed. Python's focused API/stub tests passed (33 tests). The new backend-binding regression test verifies that pause observation and lifecycle requests use the handle's local backend rather than an ambient backend with the same sandbox name. CoW cache location is also passed from the owning backend at launch. + +CoW virtio-mem unplug currently writes private zeros to prevent old backing bytes from reappearing, but does not reclaim those pages' host RAM. NUMA plus CoW is explicitly rejected. These are outstanding integration/performance limitations, not completed acceptance items. Further work includes backing-aware physical reclamation, resize/balloon live invariants, real shared/private resident-memory measurements, cache eviction and publication failure races, cancellation and lifecycle/maintenance concurrency, unsupported guest preparation, recovery/resume failures, Windows standard lifecycle qualification, and repeated release-build performance distributions. Public cache inspection/eviction workflow and comprehensive archive compatibility variants also remain to be completed. Do not mark #8 complete from these smoke results. + +Evidence locations: OVH `/home/ubuntu/msb-stack8.ElfKzf/`; macOS `/private/tmp/msb-stack8-mac-{results,managed-results,tmpfs-results,standard-results}/`; Windows isolated worktree `C:\Users\Stephen\AppData\Local\Temp\msb-stack8-20260907`. These are development outputs, not shipped artifacts. diff --git a/sdk/go/cow_lifecycle_test.go b/sdk/go/cow_lifecycle_test.go new file mode 100644 index 000000000..c79b24ca7 --- /dev/null +++ b/sdk/go/cow_lifecycle_test.go @@ -0,0 +1,79 @@ +//go:build cow_live && microsandbox_ffi_path + +package microsandbox + +import ( + "context" + "fmt" + "os" + "strings" + "testing" + "time" +) + +// This exercises the public SDK against a matching development runtime/kernel bundle. +func TestCowResidentCapture(t *testing.T) { + if os.Getenv("MSB_COW_LIVE") != "1" { + t.Skip("requires matching live bundle") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + name := fmt.Sprintf("cow8-go-%d", os.Getpid()) + source, err := CreateSandbox(ctx, name, WithImage("alpine"), WithRootDisk(RootDisk.Managed(512)), WithMemory(256), WithMemorySnapshot(MemorySnapshotCow)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := source.Stop(context.Background()); err != nil { + t.Error(err) + } + }) + if _, err := source.Exec(ctx, "sh", []string{"-c", "echo source > /dev/shm/sdk-marker"}); err != nil { + t.Fatal(err) + } + if err := source.Pause(ctx); err != nil { + t.Fatal(err) + } + paused, err := GetSandbox(ctx, name) + if err != nil { + t.Fatal(err) + } + if paused.Status() != SandboxStatusPaused { + t.Fatalf("got status %s", paused.Status()) + } + if _, err := Snapshot.Create(ctx, SnapshotCreateOptions{Name: name + "-full", FromSandbox: name, Full: true}); err != nil { + t.Fatal(err) + } + if err := paused.Resume(ctx); err != nil { + t.Fatal(err) + } + child, err := CreateSandbox(ctx, name+"-child", WithFromSnapshot(name+"-full"), WithMemorySnapshot(MemorySnapshotCow)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := child.Stop(context.Background()); err != nil { + t.Error(err) + } + }) + result, err := child.Exec(ctx, "cat", []string{"/dev/shm/sdk-marker"}) + if err != nil { + t.Fatal(err) + } + if strings.TrimSpace(result.Stdout()) != "source" { + t.Fatal("child lost captured memory") + } + if _, err := child.Exec(ctx, "sh", []string{"-c", "echo child > /dev/shm/sdk-marker"}); err != nil { + t.Fatal(err) + } + result, err = source.Exec(ctx, "cat", []string{"/dev/shm/sdk-marker"}) + if err != nil { + t.Fatal(err) + } + if strings.TrimSpace(result.Stdout()) != "source" { + t.Fatal("child changed source memory") + } + if err := child.Pause(ctx); err != nil { + t.Fatal(err) + } +} diff --git a/sdk/go/internal/ffi/ffi.go b/sdk/go/internal/ffi/ffi.go index 00b543d21..26a81f973 100644 --- a/sdk/go/internal/ffi/ffi.go +++ b/sdk/go/internal/ffi/ffi.go @@ -121,6 +121,10 @@ typedef char *(*msb_sandbox_close_fn)(uint64_t cancel_id, uint64_t handle, uint8 typedef char *(*msb_sandbox_detach_fn)(uint64_t cancel_id, uint64_t handle, uint8_t *buf, size_t buf_len); typedef char *(*msb_sandbox_stop_fn)(uint64_t cancel_id, uint64_t handle, uint64_t timeout_ms, uint8_t *buf, size_t buf_len); typedef char *(*msb_sandbox_request_stop_fn)(uint64_t cancel_id, uint64_t handle, uint8_t *buf, size_t buf_len); +typedef char *(*msb_sandbox_pause_fn)(uint64_t cancel_id, uint64_t handle, uint8_t *buf, size_t buf_len); +typedef char *(*msb_sandbox_resume_fn)(uint64_t cancel_id, uint64_t handle, uint8_t *buf, size_t buf_len); +typedef char *(*msb_sandbox_handle_pause_fn)(uint64_t cancel_id, const char *name, uint8_t *buf, size_t buf_len); +typedef char *(*msb_sandbox_handle_resume_fn)(uint64_t cancel_id, const char *name, uint8_t *buf, size_t buf_len); typedef char *(*msb_sandbox_kill_fn)(uint64_t cancel_id, uint64_t handle, uint64_t timeout_ms, uint8_t *buf, size_t buf_len); typedef char *(*msb_sandbox_request_kill_fn)(uint64_t cancel_id, uint64_t handle, uint8_t *buf, size_t buf_len); typedef char *(*msb_sandbox_list_fn)(uint64_t cancel_id, const char *filter_json, uint8_t *buf, size_t buf_len); @@ -273,6 +277,10 @@ static msb_sandbox_close_fn ptr_msb_sandbox_close = NULL; static msb_sandbox_detach_fn ptr_msb_sandbox_detach = NULL; static msb_sandbox_stop_fn ptr_msb_sandbox_stop = NULL; static msb_sandbox_request_stop_fn ptr_msb_sandbox_request_stop = NULL; +static msb_sandbox_pause_fn ptr_msb_sandbox_pause = NULL; +static msb_sandbox_resume_fn ptr_msb_sandbox_resume = NULL; +static msb_sandbox_handle_pause_fn ptr_msb_sandbox_handle_pause = NULL; +static msb_sandbox_handle_resume_fn ptr_msb_sandbox_handle_resume = NULL; static msb_sandbox_kill_fn ptr_msb_sandbox_kill = NULL; static msb_sandbox_request_kill_fn ptr_msb_sandbox_request_kill = NULL; static msb_sandbox_list_fn ptr_msb_sandbox_list = NULL; @@ -451,6 +459,10 @@ const char *load_microsandbox(const char *path) { RESOLVE(msb_sandbox_detach); RESOLVE(msb_sandbox_stop); RESOLVE(msb_sandbox_request_stop); + RESOLVE(msb_sandbox_pause); + RESOLVE(msb_sandbox_resume); + RESOLVE(msb_sandbox_handle_pause); + RESOLVE(msb_sandbox_handle_resume); RESOLVE(msb_sandbox_kill); RESOLVE(msb_sandbox_request_kill); RESOLVE(msb_sandbox_list); @@ -648,6 +660,18 @@ char *call_msb_sandbox_stop(uint64_t cancel_id, uint64_t handle, uint64_t timeou char *call_msb_sandbox_request_stop(uint64_t cancel_id, uint64_t handle, uint8_t *buf, size_t buf_len) { return ptr_msb_sandbox_request_stop ? ptr_msb_sandbox_request_stop(cancel_id, handle, buf, buf_len) : NULL; } +char *call_msb_sandbox_pause(uint64_t cancel_id, uint64_t handle, uint8_t *buf, size_t buf_len) { + return ptr_msb_sandbox_pause ? ptr_msb_sandbox_pause(cancel_id, handle, buf, buf_len) : NULL; +} +char *call_msb_sandbox_resume(uint64_t cancel_id, uint64_t handle, uint8_t *buf, size_t buf_len) { + return ptr_msb_sandbox_resume ? ptr_msb_sandbox_resume(cancel_id, handle, buf, buf_len) : NULL; +} +char *call_msb_sandbox_handle_pause(uint64_t cancel_id, const char *name, uint8_t *buf, size_t buf_len) { + return ptr_msb_sandbox_handle_pause ? ptr_msb_sandbox_handle_pause(cancel_id, name, buf, buf_len) : NULL; +} +char *call_msb_sandbox_handle_resume(uint64_t cancel_id, const char *name, uint8_t *buf, size_t buf_len) { + return ptr_msb_sandbox_handle_resume ? ptr_msb_sandbox_handle_resume(cancel_id, name, buf, buf_len) : NULL; +} char *call_msb_sandbox_kill(uint64_t cancel_id, uint64_t handle, uint64_t timeout_ms, uint8_t *buf, size_t buf_len) { return ptr_msb_sandbox_kill ? ptr_msb_sandbox_kill(cancel_id, handle, timeout_ms, buf, buf_len) : NULL; } @@ -1572,6 +1596,7 @@ type CreateOptions struct { CPUPlacement string `json:"cpu_placement,omitempty"` PlacementProfile string `json:"placement_profile,omitempty"` THP string `json:"thp,omitempty"` + MemorySnapshot string `json:"memory_snapshot,omitempty"` Workdir string `json:"workdir,omitempty"` Shell string `json:"shell,omitempty"` SecurityProfile string `json:"security_profile,omitempty"` @@ -2055,6 +2080,32 @@ func RequestStopSandboxByName(ctx context.Context, name string) error { return err } +// PauseSandboxByName controls resident execution without an agent connection. +func PauseSandboxByName(ctx context.Context, name string) error { + if err := ensureLoaded(); err != nil { + return err + } + cName := C.CString(name) + defer C.free(unsafe.Pointer(cName)) + _, err := call(ctx, func(cancelID C.uint64_t, buf *C.uint8_t, bufLen C.size_t) *C.char { + return C.call_msb_sandbox_handle_pause(cancelID, cName, buf, bufLen) + }) + return err +} + +// ResumeSandboxByName controls resident execution without an agent connection. +func ResumeSandboxByName(ctx context.Context, name string) error { + if err := ensureLoaded(); err != nil { + return err + } + cName := C.CString(name) + defer C.free(unsafe.Pointer(cName)) + _, err := call(ctx, func(cancelID C.uint64_t, buf *C.uint8_t, bufLen C.size_t) *C.char { + return C.call_msb_sandbox_handle_resume(cancelID, cName, buf, bufLen) + }) + return err +} + // KillSandboxByName terminates a sandbox identified by name and waits for stopped observation. func KillSandboxByName(ctx context.Context, name string, timeoutMs uint64) error { if err := ensureLoaded(); err != nil { @@ -2267,6 +2318,28 @@ func (s *Sandbox) RequestStop(ctx context.Context) error { return err } +// Pause controls resident execution through the host runtime. +func (s *Sandbox) Pause(ctx context.Context) error { + if err := ensureLoaded(); err != nil { + return err + } + _, err := call(ctx, func(cancelID C.uint64_t, buf *C.uint8_t, bufLen C.size_t) *C.char { + return C.call_msb_sandbox_pause(cancelID, s.h(), buf, bufLen) + }) + return err +} + +// Resume controls resident execution through the host runtime. +func (s *Sandbox) Resume(ctx context.Context) error { + if err := ensureLoaded(); err != nil { + return err + } + _, err := call(ctx, func(cancelID C.uint64_t, buf *C.uint8_t, bufLen C.size_t) *C.char { + return C.call_msb_sandbox_resume(cancelID, s.h(), buf, bufLen) + }) + return err +} + // Kill terminates the sandbox and waits for stopped observation. func (s *Sandbox) Kill(ctx context.Context, timeoutMs uint64) error { if err := ensureLoaded(); err != nil { diff --git a/sdk/go/native/microsandbox_go_ffi.h b/sdk/go/native/microsandbox_go_ffi.h index 298c0bb94..ff28b9a95 100644 --- a/sdk/go/native/microsandbox_go_ffi.h +++ b/sdk/go/native/microsandbox_go_ffi.h @@ -81,6 +81,16 @@ char *msb_sandbox_handle_stop(uint64_t cancel_id, unsigned char *buf, uintptr_t buf_len); +char *msb_sandbox_handle_pause(uint64_t cancel_id, + const char *name, + unsigned char *buf, + uintptr_t buf_len); + +char *msb_sandbox_handle_resume(uint64_t cancel_id, + const char *name, + unsigned char *buf, + uintptr_t buf_len); + char *msb_sandbox_handle_request_stop(uint64_t cancel_id, const char *name, unsigned char *buf, @@ -148,6 +158,10 @@ char *msb_sandbox_stop(uint64_t cancel_id, unsigned char *buf, uintptr_t buf_len); +char *msb_sandbox_pause(uint64_t cancel_id, Handle handle, unsigned char *buf, uintptr_t buf_len); + +char *msb_sandbox_resume(uint64_t cancel_id, Handle handle, unsigned char *buf, uintptr_t buf_len); + char *msb_sandbox_request_stop(uint64_t cancel_id, Handle handle, unsigned char *buf, diff --git a/sdk/go/native/src/lib.rs b/sdk/go/native/src/lib.rs index c144c4905..bb85824f1 100644 --- a/sdk/go/native/src/lib.rs +++ b/sdk/go/native/src/lib.rs @@ -1016,6 +1016,7 @@ struct SandboxCreateOpts { cpu_placement: Option, placement_profile: Option, thp: Option, + memory_snapshot: Option, workdir: Option, shell: Option, env: Option>, @@ -2226,6 +2227,9 @@ pub unsafe extern "C" fn msb_sandbox_create( .map_err(FfiError::invalid_argument)?; builder = builder.thp(policy); } + if let Some(mode) = opts.memory_snapshot { + builder = builder.memory_snapshot(mode); + } if let Some(w) = opts.workdir { builder = builder.workdir(w); } @@ -2601,6 +2605,40 @@ pub unsafe extern "C" fn msb_sandbox_handle_stop( }) } +#[unsafe(no_mangle)] +pub unsafe extern "C" fn msb_sandbox_handle_pause( + cancel_id: u64, + name: *const c_char, + buf: *mut c_uchar, + buf_len: usize, +) -> *mut c_char { + run_c(cancel_id, buf, buf_len, || { + let name = unsafe { cstr(name) }?; + Ok(Box::pin(async move { + let sb = Sandbox::get(&name).await.map_err(FfiError::from)?; + sb.pause().await.map_err(FfiError::from)?; + Ok(r#"{"ok":true}"#.into()) + })) + }) +} + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn msb_sandbox_handle_resume( + cancel_id: u64, + name: *const c_char, + buf: *mut c_uchar, + buf_len: usize, +) -> *mut c_char { + run_c(cancel_id, buf, buf_len, || { + let name = unsafe { cstr(name) }?; + Ok(Box::pin(async move { + let sb = Sandbox::get(&name).await.map_err(FfiError::from)?; + sb.resume().await.map_err(FfiError::from)?; + Ok(r#"{"ok":true}"#.into()) + })) + }) +} + #[unsafe(no_mangle)] pub unsafe extern "C" fn msb_sandbox_handle_request_stop( cancel_id: u64, @@ -2879,6 +2917,38 @@ pub unsafe extern "C" fn msb_sandbox_stop( }) } +#[unsafe(no_mangle)] +pub unsafe extern "C" fn msb_sandbox_pause( + cancel_id: u64, + handle: Handle, + buf: *mut c_uchar, + buf_len: usize, +) -> *mut c_char { + run_c(cancel_id, buf, buf_len, || { + let sb = get(handle)?; + Ok(Box::pin(async move { + sb.pause().await.map_err(FfiError::from)?; + Ok(r#"{"ok":true}"#.into()) + })) + }) +} + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn msb_sandbox_resume( + cancel_id: u64, + handle: Handle, + buf: *mut c_uchar, + buf_len: usize, +) -> *mut c_char { + run_c(cancel_id, buf, buf_len, || { + let sb = get(handle)?; + Ok(Box::pin(async move { + sb.resume().await.map_err(FfiError::from)?; + Ok(r#"{"ok":true}"#.into()) + })) + }) +} + #[unsafe(no_mangle)] pub unsafe extern "C" fn msb_sandbox_request_stop( cancel_id: u64, diff --git a/sdk/go/options.go b/sdk/go/options.go index 143d08764..bb1f53e9a 100644 --- a/sdk/go/options.go +++ b/sdk/go/options.go @@ -36,6 +36,7 @@ type SandboxConfig struct { CPUPlacement CPUPlacement PlacementProfile string THP THPPolicy + MemorySnapshot MemorySnapshotMode Workdir string Shell string SecurityProfile SecurityProfile @@ -133,13 +134,14 @@ type persistedInitConfig struct { } type persistedResources struct { - CPUs uint8 `json:"cpus"` - MemoryMiB uint32 `json:"memory_mib"` - MaxCPUs uint8 `json:"max_cpus"` - MaxMemoryMiB uint32 `json:"max_memory_mib"` - CPUPlacement CPUPlacement `json:"cpu_placement"` - PlacementProfile string `json:"placement_profile"` - THP THPPolicy `json:"thp"` + MemorySnapshot MemorySnapshotMode `json:"memory_snapshot"` + CPUs uint8 `json:"cpus"` + MemoryMiB uint32 `json:"memory_mib"` + MaxCPUs uint8 `json:"max_cpus"` + MaxMemoryMiB uint32 `json:"max_memory_mib"` + CPUPlacement CPUPlacement `json:"cpu_placement"` + PlacementProfile string `json:"placement_profile"` + THP THPPolicy `json:"thp"` } type persistedRuntime struct { @@ -211,6 +213,7 @@ func (c *SandboxConfig) UnmarshalJSON(data []byte) error { OCIUpperSizeMiB: upperSizeMiB, ociUpperSizeSet: upperSizeSet, MemoryMiB: raw.memoryMiB(), + MemorySnapshot: raw.memorySnapshot(), CPUs: raw.cpus(), MaxMemoryMiB: raw.maxMemoryMiB(), MaxCPUs: raw.maxCPUs(), @@ -267,6 +270,13 @@ func (c persistedSandboxConfig) maxCPUs() uint8 { return c.CPUs } +func (c persistedSandboxConfig) memorySnapshot() MemorySnapshotMode { + if c.Resources != nil && c.Resources.MemorySnapshot != "" { + return c.Resources.MemorySnapshot + } + return MemorySnapshotStandard +} + func (c persistedSandboxConfig) maxMemoryMiB() uint32 { if c.Resources != nil { if c.Resources.MaxMemoryMiB != 0 { @@ -478,6 +488,19 @@ const ( // THPPolicy selects the guest transparent huge-page policy at boot. type THPPolicy string +// MemorySnapshotMode selects anonymous or explicit private file-backed memory. +type MemorySnapshotMode string + +const ( + MemorySnapshotStandard MemorySnapshotMode = "standard" + MemorySnapshotCow MemorySnapshotMode = "cow" +) + +// WithMemorySnapshot selects memory representation; snapshots remain manual. +func WithMemorySnapshot(mode MemorySnapshotMode) SandboxOption { + return func(o *SandboxConfig) { o.MemorySnapshot = mode } +} + const ( // THPAlways transparently uses huge pages for eligible anonymous mappings. THPAlways THPPolicy = "always" diff --git a/sdk/go/options_test.go b/sdk/go/options_test.go index 2d9d99bf0..8b42690fd 100644 --- a/sdk/go/options_test.go +++ b/sdk/go/options_test.go @@ -15,6 +15,28 @@ func TestWithImage(t *testing.T) { } } +func TestMemorySnapshotPolicy(t *testing.T) { + var config SandboxConfig + WithMemorySnapshot(MemorySnapshotCow)(&config) + if config.MemorySnapshot != MemorySnapshotCow { + t.Fatal("CoW option was lost") + } + for _, tc := range []struct { + json string + want MemorySnapshotMode + }{ + {`{"resources":{"cpus":1,"memory_mib":128}}`, MemorySnapshotStandard}, + {`{"resources":{"cpus":1,"memory_mib":128,"memory_snapshot":"cow"}}`, MemorySnapshotCow}, + } { + if err := json.Unmarshal([]byte(tc.json), &config); err != nil { + t.Fatal(err) + } + if config.MemorySnapshot != tc.want { + t.Fatalf("got %q, want %q", config.MemorySnapshot, tc.want) + } + } +} + func TestWithRootDiskManaged(t *testing.T) { o := SandboxConfig{} WithRootDisk(RootDisk.Managed(8192))(&o) diff --git a/sdk/go/sandbox.go b/sdk/go/sandbox.go index 9bf8d25cc..fde33b788 100644 --- a/sdk/go/sandbox.go +++ b/sdk/go/sandbox.go @@ -83,6 +83,7 @@ func buildFFICreateOptions(o SandboxConfig) ffi.CreateOptions { CPUPlacement: string(o.CPUPlacement), PlacementProfile: o.PlacementProfile, THP: string(o.THP), + MemorySnapshot: string(o.MemorySnapshot), Workdir: o.Workdir, Shell: o.Shell, SecurityProfile: string(o.SecurityProfile), @@ -771,6 +772,16 @@ func (h *SandboxHandle) RequestStop(ctx context.Context) error { return wrapFFI(ffi.RequestStopSandboxByName(ctx, h.name)) } +// Pause controls resident execution without creating a snapshot. +func (h *SandboxHandle) Pause(ctx context.Context) error { + return wrapFFI(ffi.PauseSandboxByName(ctx, h.name)) +} + +// Resume controls resident execution without creating a snapshot. +func (h *SandboxHandle) Resume(ctx context.Context) error { + return wrapFFI(ffi.ResumeSandboxByName(ctx, h.name)) +} + // Kill force-kills the sandbox and waits until stopped state is observed. func (h *SandboxHandle) Kill(ctx context.Context, opts ...KillOption) error { return wrapFFI(ffi.KillSandboxByName(ctx, h.name, killTimeoutMillis(opts))) @@ -824,6 +835,16 @@ func (s *Sandbox) RequestStop(ctx context.Context) error { return wrapFFI(s.inner.RequestStop(ctx)) } +// Pause controls resident execution without creating a snapshot. +func (s *Sandbox) Pause(ctx context.Context) error { + return wrapFFI(s.inner.Pause(ctx)) +} + +// Resume controls resident execution without creating a snapshot. +func (s *Sandbox) Resume(ctx context.Context) error { + return wrapFFI(s.inner.Resume(ctx)) +} + // Kill force-kills the sandbox and waits until stopped state is observed. func (s *Sandbox) Kill(ctx context.Context, opts ...KillOption) error { return wrapFFI(s.inner.Kill(ctx, killTimeoutMillis(opts))) diff --git a/sdk/node-ts/native/index.d.ts b/sdk/node-ts/native/index.d.ts index 2f30c19df..7510cf12d 100644 --- a/sdk/node-ts/native/index.d.ts +++ b/sdk/node-ts/native/index.d.ts @@ -984,6 +984,10 @@ export declare class Sandbox { attachShell(): Promise /** Stop the sandbox gracefully and wait for it to exit. */ stop(): Promise + /** Explicit resident pause through host control. */ + pause(): Promise + /** Explicit resident resume through host control. */ + resume(): Promise /** Stop and wait for exit, returning the exit status. */ stopAndWait(): Promise /** Request graceful shutdown without waiting for observed exit. */ @@ -1086,6 +1090,8 @@ export declare class SandboxBuilder { maxMemory(mib: number): this /** Guest transparent huge-page policy selected at boot. */ thp(policy: 'always' | 'madvise' | 'never'): this + /** Select explicit private file-backed memory or standard anonymous memory. */ + memorySnapshot(mode: 'standard' | 'cow'): this /** Override log verbosity: `"trace" | "debug" | "info" | "warn" | "error"`. */ logLevel(level: string): this /** Suppress sandbox logs. */ @@ -1356,6 +1362,10 @@ export declare class SandboxHandle { * override with `stopWithTimeout(timeoutMs)`. */ stop(): Promise + /** Explicit resident pause through host control. */ + pause(): Promise + /** Explicit resident resume through host control. */ + resume(): Promise /** Request graceful shutdown without waiting. */ requestStop(): Promise /** diff --git a/sdk/node-ts/native/sandbox.rs b/sdk/node-ts/native/sandbox.rs index d99fa9f6e..eb9b4f907 100644 --- a/sdk/node-ts/native/sandbox.rs +++ b/sdk/node-ts/native/sandbox.rs @@ -529,6 +529,22 @@ impl Sandbox { sb.stop().await.map_err(to_napi_error) } + /// Explicit resident pause through host control. + #[napi] + pub async fn pause(&self) -> Result<()> { + let guard = self.inner.lock().await; + let sb = guard.as_ref().ok_or_else(consumed_error)?; + sb.pause().await.map_err(to_napi_error) + } + + /// Explicit resident resume through host control. + #[napi] + pub async fn resume(&self) -> Result<()> { + let guard = self.inner.lock().await; + let sb = guard.as_ref().ok_or_else(consumed_error)?; + sb.resume().await.map_err(to_napi_error) + } + /// Stop and wait for exit, returning the exit status. #[napi] pub async fn stop_and_wait(&self) -> Result { diff --git a/sdk/node-ts/native/sandbox_builder.rs b/sdk/node-ts/native/sandbox_builder.rs index 0f5f7eab7..facad0632 100644 --- a/sdk/node-ts/native/sandbox_builder.rs +++ b/sdk/node-ts/native/sandbox_builder.rs @@ -231,6 +231,19 @@ impl JsSandboxBuilder { Ok(self) } + /// Select explicit private file-backed memory or standard anonymous memory. + #[napi(ts_args_type = "mode: 'standard' | 'cow'")] + pub fn memory_snapshot(&mut self, mode: String) -> Result<&Self> { + let mode = serde_json::from_value(serde_json::Value::String(mode)) + .map_err(|e| napi::Error::from_reason(e.to_string()))?; + let prev = self + .inner + .take() + .ok_or_else(|| napi::Error::from_reason("builder already consumed"))?; + self.inner = Some(prev.memory_snapshot(mode)); + Ok(self) + } + /// Override log verbosity: `"trace" | "debug" | "info" | "warn" | "error"`. #[napi(js_name = "logLevel")] pub fn log_level(&mut self, level: String) -> Result<&Self> { diff --git a/sdk/node-ts/native/sandbox_handle.rs b/sdk/node-ts/native/sandbox_handle.rs index 245208fb6..57f0ca089 100644 --- a/sdk/node-ts/native/sandbox_handle.rs +++ b/sdk/node-ts/native/sandbox_handle.rs @@ -162,6 +162,18 @@ impl JsSandboxHandle { self.inner.stop().await.map_err(to_napi_error) } + /// Explicit resident pause through host control. + #[napi] + pub async fn pause(&self) -> Result<()> { + self.inner.pause().await.map_err(to_napi_error) + } + + /// Explicit resident resume through host control. + #[napi] + pub async fn resume(&self) -> Result<()> { + self.inner.resume().await.map_err(to_napi_error) + } + /// Request graceful shutdown without waiting. #[napi] pub async fn request_stop(&self) -> Result<()> { diff --git a/sdk/node-ts/src/internal/napi.ts b/sdk/node-ts/src/internal/napi.ts index 4c1046dac..1dcaebe82 100644 --- a/sdk/node-ts/src/internal/napi.ts +++ b/sdk/node-ts/src/internal/napi.ts @@ -187,6 +187,7 @@ export interface NapiSandboxBuilderSetters { memory(mib: number): this; maxMemory(mib: number): this; thp(policy: "always" | "madvise" | "never"): this; + memorySnapshot(mode: "standard" | "cow"): this; logLevel(level: string): this; quietLogs(): this; detached(enabled: boolean): this; @@ -273,6 +274,8 @@ export interface NapiSandbox { attachWithBuilder(cmd: string, builder: NapiAttachOptionsBuilder): Promise; attachShell(): Promise; stop(): Promise; + pause(): Promise; + resume(): Promise; requestStop(): Promise; stopWithTimeout(timeoutMs: number): Promise; kill(): Promise; @@ -303,6 +306,8 @@ export interface NapiSandboxHandle { connect(): Promise; connectWithTimeout(timeoutMs: number): Promise; stop(): Promise; + pause(): Promise; + resume(): Promise; requestStop(): Promise; stopWithTimeout(timeoutMs: number): Promise; kill(): Promise; diff --git a/sdk/node-ts/src/sandbox-handle.ts b/sdk/node-ts/src/sandbox-handle.ts index b302a5c0b..db26b2f8f 100644 --- a/sdk/node-ts/src/sandbox-handle.ts +++ b/sdk/node-ts/src/sandbox-handle.ts @@ -160,6 +160,16 @@ export class SandboxHandle { await withMappedErrors(() => this.inner.stop()); } + /** Explicit resident pause; no snapshot is created. */ + async pause(): Promise { + await withMappedErrors(() => this.inner.pause()); + } + + /** Explicit resident resume; no snapshot is created. */ + async resume(): Promise { + await withMappedErrors(() => this.inner.resume()); + } + async requestStop(): Promise { await withMappedErrors(() => this.inner.requestStop()); } diff --git a/sdk/node-ts/src/sandbox-status.ts b/sdk/node-ts/src/sandbox-status.ts index c07929c9e..6b5d84fc6 100644 --- a/sdk/node-ts/src/sandbox-status.ts +++ b/sdk/node-ts/src/sandbox-status.ts @@ -1,7 +1,8 @@ -export type SandboxStatus = "running" | "stopped" | "crashed" | "draining"; +export type SandboxStatus = "running" | "paused" | "stopped" | "crashed" | "draining"; export const SandboxStatuses: readonly SandboxStatus[] = [ "running", + "paused", "stopped", "crashed", "draining", diff --git a/sdk/node-ts/src/sandbox.ts b/sdk/node-ts/src/sandbox.ts index 0bb88efe7..2a50cd536 100644 --- a/sdk/node-ts/src/sandbox.ts +++ b/sdk/node-ts/src/sandbox.ts @@ -491,6 +491,16 @@ export class Sandbox implements AsyncDisposable { await withMappedErrors(() => this.inner.stop()); } + /** Explicit resident pause; no snapshot is created. */ + async pause(): Promise { + await withMappedErrors(() => this.inner.pause()); + } + + /** Explicit resident resume; no snapshot is created. */ + async resume(): Promise { + await withMappedErrors(() => this.inner.resume()); + } + async requestStop(): Promise { await withMappedErrors(() => this.inner.requestStop()); } diff --git a/sdk/node-ts/tests/cow-lifecycle.test.ts b/sdk/node-ts/tests/cow-lifecycle.test.ts new file mode 100644 index 000000000..b171b4fff --- /dev/null +++ b/sdk/node-ts/tests/cow-lifecycle.test.ts @@ -0,0 +1,25 @@ +import { expect, it } from "vitest"; +import { Sandbox, Snapshot } from "../dist/index.js"; + +// Opt-in because this starts real VMs with a matching development runtime/kernel bundle. +it.skipIf(process.env.MSB_COW_LIVE !== "1")("captures a resident pause and restores private memory", async () => { + const name = `cow8-node-${process.pid}`; + const source = await Sandbox.builder(name).image("alpine").rootDisk(512).memory(256).memorySnapshot("cow").create(); + let child: Sandbox | undefined; + try { + await source.exec("sh", ["-c", "echo source > /dev/shm/sdk-marker"]); + await source.pause(); + const paused = await Sandbox.get(name); + expect(paused.status).toBe("paused"); + await Snapshot.builder(`${name}-full`).fromSandbox(name).full().create(); + await paused.resume(); + child = await Sandbox.builder(`${name}-child`).fromSnapshot(`${name}-full`).memorySnapshot("cow").create(); + expect((await child.exec("cat", ["/dev/shm/sdk-marker"])).stdout().trim()).toBe("source"); + await child.exec("sh", ["-c", "echo child > /dev/shm/sdk-marker"]); + expect((await source.exec("cat", ["/dev/shm/sdk-marker"])).stdout().trim()).toBe("source"); + await child.pause(); + } finally { + await child?.stop(); + await source.stop(); + } +}, 120_000); diff --git a/sdk/node-ts/tests/unit/builders.test.ts b/sdk/node-ts/tests/unit/builders.test.ts index 587bb4b64..3cabe7ef5 100644 --- a/sdk/node-ts/tests/unit/builders.test.ts +++ b/sdk/node-ts/tests/unit/builders.test.ts @@ -351,6 +351,16 @@ describe("PatchBuilder", () => { }); describe("SandboxBuilder.build", () => { + it("makes CoW memory an explicit creation policy", async () => { + const config = await Sandbox.builder("cow-policy") + .image("alpine") + .memorySnapshot("cow") + .build(); + expect((config.resources as { memorySnapshot: string }).memorySnapshot).toBe("cow"); + const standard = await Sandbox.builder("standard-policy").image("alpine").build(); + expect((standard.resources as { memorySnapshot?: string }).memorySnapshot).toBeUndefined(); + }); + it("requires .image()", async () => { await expect(Sandbox.builder("x").build()).rejects.toThrow( InvalidConfigError, diff --git a/sdk/python/microsandbox/__init__.py b/sdk/python/microsandbox/__init__.py index eb6a6dc6c..c1d593f3d 100644 --- a/sdk/python/microsandbox/__init__.py +++ b/sdk/python/microsandbox/__init__.py @@ -117,6 +117,7 @@ LogLevel, LogReadSource, LogSource, + MemorySnapshotMode, MiB, ModificationConflict, ModificationDisposition, @@ -312,6 +313,7 @@ "FlatClone", "PullPolicy", "CpuPlacement", + "MemorySnapshotMode", "RegistryAuth", "LogLevel", "DeploymentProfile", diff --git a/sdk/python/microsandbox/_microsandbox.pyi b/sdk/python/microsandbox/_microsandbox.pyi index 9f522e527..3f8d6b336 100644 --- a/sdk/python/microsandbox/_microsandbox.pyi +++ b/sdk/python/microsandbox/_microsandbox.pyi @@ -20,6 +20,7 @@ from microsandbox.types import ( LogLevel, LogReadSource, LogSource, + MemorySnapshotMode, ModificationPolicy, MountConfig, NamedVolumeMode, @@ -95,6 +96,7 @@ class Sandbox: from_snapshot: str | os.PathLike[str] | None = None, disk_only: bool = False, snapshot_base: str | None = None, + memory_snapshot: MemorySnapshotMode | None = None, memory: int | None = None, cpus: int | None = None, max_memory: int | None = None, @@ -152,6 +154,7 @@ class Sandbox: from_snapshot: str | os.PathLike[str] | None = None, disk_only: bool = False, snapshot_base: str | None = None, + memory_snapshot: MemorySnapshotMode | None = None, memory: int | None = None, cpus: int | None = None, max_memory: int | None = None, @@ -322,6 +325,8 @@ class Sandbox: follow: bool = False, ) -> LogStream: ... async def stop(self, timeout: float | None = None) -> None: ... + async def pause(self) -> None: ... + async def resume(self) -> None: ... async def request_stop(self) -> None: ... async def kill(self, timeout: float | None = None) -> None: ... async def request_kill(self) -> None: ... @@ -417,6 +422,8 @@ class SandboxHandle: async def refresh(self) -> SandboxHandle: ... async def connect(self, timeout: float | None = None) -> Sandbox: ... async def stop(self, timeout: float | None = None) -> None: ... + async def pause(self) -> None: ... + async def resume(self) -> None: ... async def request_stop(self) -> None: ... async def kill(self, timeout: float | None = None) -> None: ... async def request_kill(self) -> None: ... diff --git a/sdk/python/microsandbox/types.py b/sdk/python/microsandbox/types.py index 40c0923a0..bab28d76a 100644 --- a/sdk/python/microsandbox/types.py +++ b/sdk/python/microsandbox/types.py @@ -41,6 +41,13 @@ class PullPolicy(StrEnum): NEVER = "never" +class MemorySnapshotMode(StrEnum): + """Explicit memory representation; snapshots are still created manually.""" + + STANDARD = "standard" + COW = "cow" + + class CpuPlacement(StrEnum): """Host placement policy for sandbox vCPU threads.""" diff --git a/sdk/python/src/helpers.rs b/sdk/python/src/helpers.rs index 304238be0..d24bd87b6 100644 --- a/sdk/python/src/helpers.rs +++ b/sdk/python/src/helpers.rs @@ -25,6 +25,7 @@ const KNOWN_CREATE_KWARGS: &[&str] = &[ "cpu_placement", "placement_profile", "thp", + "memory_snapshot", "workdir", "shell", "security", @@ -336,6 +337,11 @@ pub fn sandbox_builder_from_args( .map_err(pyo3::exceptions::PyValueError::new_err)?; builder = builder.thp(policy); } + if let Some(mode) = extract_opt::(kwargs, "memory_snapshot")? { + let mode = serde_json::from_value(serde_json::Value::String(mode)) + .map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))?; + builder = builder.memory_snapshot(mode); + } if let Some(workdir) = extract_opt::(kwargs, "workdir")? { builder = builder.workdir(workdir); } diff --git a/sdk/python/src/sandbox.rs b/sdk/python/src/sandbox.rs index b735017ee..6b47dea7a 100644 --- a/sdk/python/src/sandbox.rs +++ b/sdk/python/src/sandbox.rs @@ -963,6 +963,26 @@ impl PySandbox { }) } + /// Suspend this resident VM without releasing RAM. + fn pause<'py>(&self, py: Python<'py>) -> PyResult> { + let inner = self.inner.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let sandbox = Self::clone_sandbox(&inner).await?; + sandbox.pause().await.map_err(to_py_err)?; + Ok(()) + }) + } + + /// Resume the same resident VM and its workloads. + fn resume<'py>(&self, py: Python<'py>) -> PyResult> { + let inner = self.inner.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let sandbox = Self::clone_sandbox(&inner).await?; + sandbox.resume().await.map_err(to_py_err)?; + Ok(()) + }) + } + /// Request graceful shutdown without waiting. fn request_stop<'py>(&self, py: Python<'py>) -> PyResult> { let inner = self.inner.clone(); diff --git a/sdk/python/src/sandbox_handle.rs b/sdk/python/src/sandbox_handle.rs index 4590e4d7d..2446150cb 100644 --- a/sdk/python/src/sandbox_handle.rs +++ b/sdk/python/src/sandbox_handle.rs @@ -360,6 +360,26 @@ impl PySandboxHandle { }) } + /// Suspend this resident VM without releasing RAM. + fn pause<'py>(&self, py: Python<'py>) -> PyResult> { + let inner = self.inner.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let guard = inner.lock().await; + guard.pause().await.map_err(to_py_err)?; + Ok(()) + }) + } + + /// Resume the same resident VM and its workloads. + fn resume<'py>(&self, py: Python<'py>) -> PyResult> { + let inner = self.inner.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let guard = inner.lock().await; + guard.resume().await.map_err(to_py_err)?; + Ok(()) + }) + } + /// Request graceful shutdown without waiting. fn request_stop<'py>(&self, py: Python<'py>) -> PyResult> { let inner = self.inner.clone(); diff --git a/sdk/python/tests/test_cow_lifecycle.py b/sdk/python/tests/test_cow_lifecycle.py new file mode 100644 index 000000000..8ce67c960 --- /dev/null +++ b/sdk/python/tests/test_cow_lifecycle.py @@ -0,0 +1,37 @@ +"""Opt-in live CoW lifecycle check using a matching runtime/kernel bundle.""" + +import os + +import pytest + +from microsandbox import MemorySnapshotMode, Sandbox, Snapshot + + +@pytest.mark.skipif(os.environ.get("MSB_COW_LIVE") != "1", reason="requires matching live bundle") +@pytest.mark.asyncio +async def test_cow_resident_capture_and_child_isolation(): + name = f"cow8-python-{os.getpid()}" + source = await Sandbox.create( + name, image="alpine", memory=256, memory_snapshot=MemorySnapshotMode.COW + ) + child = None + try: + await source.exec("sh", ["-c", "echo source > /dev/shm/sdk-marker"]) + await source.pause() + paused = await Sandbox.get(name) + assert str(paused.status) == "paused" + await Snapshot.create(f"{name}-full", from_sandbox=name, full=True) + await paused.resume() + child = await Sandbox.create( + f"{name}-child", from_snapshot=f"{name}-full", memory_snapshot=MemorySnapshotMode.COW + ) + result = await child.exec("cat", ["/dev/shm/sdk-marker"]) + assert result.stdout_text.strip() == "source" + await child.exec("sh", ["-c", "echo child > /dev/shm/sdk-marker"]) + result = await source.exec("cat", ["/dev/shm/sdk-marker"]) + assert result.stdout_text.strip() == "source" + await child.pause() + finally: + if child is not None: + await child.stop() + await source.stop() diff --git a/sdk/python/tests/test_create_stub.py b/sdk/python/tests/test_create_stub.py index b539aa2b4..a9c678f98 100644 --- a/sdk/python/tests/test_create_stub.py +++ b/sdk/python/tests/test_create_stub.py @@ -11,6 +11,8 @@ "image", "from_snapshot", "disk_only", + "snapshot_base", + "memory_snapshot", "memory", "cpus", "max_memory", @@ -83,6 +85,7 @@ def test_create_closed_values_are_precisely_typed() -> None: } assert annotations["security"] == "SecurityProfile | None" + assert annotations["memory_snapshot"] == "MemorySnapshotMode | None" assert annotations["init"] == "str | InitConfig | InitOptions | None" assert annotations["pull_policy"] == "PullPolicy | None" assert annotations["log_level"] == "LogLevel | None" diff --git a/sdk/rust/lib/backend/cloud/sandbox.rs b/sdk/rust/lib/backend/cloud/sandbox.rs index 445d4d895..2cc1ceaa5 100644 --- a/sdk/rust/lib/backend/cloud/sandbox.rs +++ b/sdk/rust/lib/backend/cloud/sandbox.rs @@ -292,6 +292,12 @@ impl TryFrom for CloudCreateBody { /// Build the cloud create body from an SDK config, rejecting the /// create-time options the cloud does not accept. fn try_from(mut config: SandboxConfig) -> MicrosandboxResult { + if !config.spec.resources.memory_snapshot.is_standard() { + return Err(MicrosandboxError::unsupported( + Operation::SandboxCreate, + UnsupportedReason::ConfigField("memory_snapshot"), + )); + } if config.replace_existing { return Err(MicrosandboxError::unsupported( Operation::SandboxCreate, diff --git a/sdk/rust/lib/backend/local/sandbox/mod.rs b/sdk/rust/lib/backend/local/sandbox/mod.rs index c8c0cf01a..eb65289df 100644 --- a/sdk/rust/lib/backend/local/sandbox/mod.rs +++ b/sdk/rust/lib/backend/local/sandbox/mod.rs @@ -915,7 +915,8 @@ impl SandboxBackend for LocalBackend { name: &'a str, ) -> BoxFuture<'a, MicrosandboxResult> { Box::pin(async move { - let (model, pid) = self.sandbox_handle_state(name).await?; + let (mut model, pid) = self.sandbox_handle_state(name).await?; + model.status = crate::sandbox::pause::projected_status(self, name, model.status).await; Ok(SandboxHandle::from_local_model(backend, model, pid)) }) } @@ -927,10 +928,22 @@ impl SandboxBackend for LocalBackend { ) -> BoxFuture<'a, MicrosandboxResult> { Box::pin(async move { let (rows, next_cursor) = self.list_sandbox_handle_state(&query).await?; - let sandboxes = rows - .into_iter() - .map(|(model, pid)| SandboxHandle::from_local_model(backend.clone(), model, pid)) - .collect(); + let sandboxes = stream::iter(rows) + .map(|(mut model, pid)| { + let backend = backend.clone(); + async move { + model.status = crate::sandbox::pause::projected_status( + self, + &model.name, + model.status, + ) + .await; + SandboxHandle::from_local_model(backend, model, pid) + } + }) + .buffered(16) + .collect() + .await; Ok(SandboxPage { sandboxes, next_cursor, diff --git a/sdk/rust/lib/error.rs b/sdk/rust/lib/error.rs index d9e7b05c8..e9b817bda 100644 --- a/sdk/rust/lib/error.rs +++ b/sdk/rust/lib/error.rs @@ -243,6 +243,10 @@ pub enum Operation { SandboxStart, /// `Sandbox::stop`. SandboxStop, + /// `Sandbox::pause`. + SandboxPause, + /// `Sandbox::resume`. + SandboxResume, /// `Sandbox::remove`. SandboxRemove, /// `Sandbox::remove_persisted`. @@ -411,6 +415,8 @@ impl Operation { Operation::SandboxCreate => "Sandbox::create", Operation::SandboxStart => "Sandbox::start", Operation::SandboxStop => "Sandbox::stop", + Operation::SandboxPause => "Sandbox::pause", + Operation::SandboxResume => "Sandbox::resume", Operation::SandboxRemove => "Sandbox::remove", Operation::SandboxRemovePersisted => "Sandbox::remove_persisted", Operation::SandboxKill => "Sandbox::kill", diff --git a/sdk/rust/lib/runtime/spawn.rs b/sdk/rust/lib/runtime/spawn.rs index 25099c21b..9057a578a 100644 --- a/sdk/rust/lib/runtime/spawn.rs +++ b/sdk/rust/lib/runtime/spawn.rs @@ -2548,6 +2548,10 @@ fn sandbox_cli_args( agent_sock: agent_sock_path.to_path_buf(), libkrunfw_path: libkrunfw_path.to_path_buf(), thp: config.spec.resources.thp, + memory_snapshot: config.spec.resources.memory_snapshot, + memory_cache_dir: (config.spec.resources.memory_snapshot + == microsandbox_types::MemorySnapshotMode::Cow) + .then(|| local.cache_dir().join("memory")), startup: startup_command(config), lifecycle: Lifecycle { max_duration_secs: config.spec.lifecycle.max_duration_secs, diff --git a/sdk/rust/lib/sandbox/builder.rs b/sdk/rust/lib/sandbox/builder.rs index e204a7faa..880360886 100644 --- a/sdk/rust/lib/sandbox/builder.rs +++ b/sdk/rust/lib/sandbox/builder.rs @@ -378,6 +378,12 @@ impl SandboxBuilder { self } + /// Select explicit private file-backed memory or the default anonymous representation. + pub fn memory_snapshot(mut self, mode: microsandbox_types::MemorySnapshotMode) -> Self { + self.config.spec.resources.memory_snapshot = mode; + self + } + /// Set the runtime log level for the sandbox process. /// /// This controls the verbosity of the `msb sandbox` process. diff --git a/sdk/rust/lib/sandbox/config.rs b/sdk/rust/lib/sandbox/config.rs index 531501369..5373a9212 100644 --- a/sdk/rust/lib/sandbox/config.rs +++ b/sdk/rust/lib/sandbox/config.rs @@ -779,6 +779,7 @@ impl Default for SandboxConfig { Self { spec: SandboxSpec { resources: SandboxResources { + memory_snapshot: Default::default(), cpus: default_cpus(), memory_mib: default_memory_mib(), max_cpus: default_cpus(), @@ -1535,6 +1536,7 @@ mod tests { resources: SandboxResources { cpus: 2, memory_mib: 1024, + memory_snapshot: Default::default(), max_cpus: 2, max_memory_mib: 1024, cpu_placement: Default::default(), diff --git a/sdk/rust/lib/sandbox/config_patch.rs b/sdk/rust/lib/sandbox/config_patch.rs index 07a6e1e4e..aca44f834 100644 --- a/sdk/rust/lib/sandbox/config_patch.rs +++ b/sdk/rust/lib/sandbox/config_patch.rs @@ -67,6 +67,7 @@ pub enum SandboxImagePatch { /// Sparse resource and lifecycle limits. #[derive(Debug, Clone, Default)] pub struct ResourceConfigPatch { + memory_snapshot: Option, cpus: Option, memory_mib: Option, max_duration_secs: Option, @@ -360,6 +361,12 @@ impl ResourceConfigPatch { Self::default() } + /// Select memory representation for the next VM construction. + pub fn memory_snapshot(mut self, mode: super::MemorySnapshotMode) -> Self { + self.memory_snapshot = Some(mode); + self + } + /// Set the initial vCPU count. pub fn cpus(mut self, cpus: u8) -> Self { self.cpus = Some(cpus); @@ -393,6 +400,7 @@ impl ResourceConfigPatch { /// Overlay another resource patch. pub fn overlay(mut self, higher: Self) -> Self { replace(&mut self.cpus, higher.cpus); + replace(&mut self.memory_snapshot, higher.memory_snapshot); replace(&mut self.memory_mib, higher.memory_mib); replace(&mut self.max_duration_secs, higher.max_duration_secs); replace(&mut self.idle_timeout_secs, higher.idle_timeout_secs); @@ -401,6 +409,9 @@ impl ResourceConfigPatch { } fn apply_to(self, mut builder: SandboxBuilder) -> SandboxBuilder { + if let Some(mode) = self.memory_snapshot { + builder = builder.memory_snapshot(mode); + } if let Some(cpus) = self.cpus { builder = builder.cpus(cpus); } diff --git a/sdk/rust/lib/sandbox/handle.rs b/sdk/rust/lib/sandbox/handle.rs index 1b931ae2f..c130f292e 100644 --- a/sdk/rust/lib/sandbox/handle.rs +++ b/sdk/rust/lib/sandbox/handle.rs @@ -48,7 +48,7 @@ pub const DEFAULT_KILL_TIMEOUT: std::time::Duration = std::time::Duration::from_ /// [`connect`](SandboxHandle::connect) when the sandbox is already running, or /// [`start`](SandboxHandle::start) to boot a stopped sandbox. pub struct SandboxHandle { - backend: Arc, + pub(super) backend: Arc, inner: SandboxHandleInner, name: String, } @@ -321,7 +321,10 @@ impl SandboxHandle { .local() .ok_or_else(|| MicrosandboxError::local_only(Operation::SandboxHandleMetrics))?; - if local.status != SandboxStatus::Running && local.status != SandboxStatus::Draining { + if !matches!( + local.status, + SandboxStatus::Running | SandboxStatus::Draining | SandboxStatus::Paused + ) { return Err(MicrosandboxError::SandboxNotRunning(format!( "'{}' is not running (status: {:?})", self.name, local.status diff --git a/sdk/rust/lib/sandbox/mod.rs b/sdk/rust/lib/sandbox/mod.rs index 009dfab14..357ad3b73 100644 --- a/sdk/rust/lib/sandbox/mod.rs +++ b/sdk/rust/lib/sandbox/mod.rs @@ -18,6 +18,7 @@ pub mod init; pub(crate) mod metrics; mod modify; mod patch; +pub(crate) mod pause; #[cfg(windows)] mod reap; #[cfg(feature = "ssh")] @@ -137,8 +138,9 @@ pub use microsandbox_network::dns::Nameserver; pub use microsandbox_network::policy::{ Action as NetworkAction, NetworkPolicy, NetworkProfile, Rule as NetworkRule, }; +pub use microsandbox_runtime::control::PauseControlState as SandboxPauseState; pub use microsandbox_runtime::logging::LogLevel; -pub use microsandbox_types::{CpuPlacement, PullPolicy}; +pub use microsandbox_types::{CpuPlacement, MemorySnapshotMode, PullPolicy}; pub use microsandbox_types::{ EnvVar, MAX_HOSTNAME_BYTES, MAX_SANDBOX_NAME_BYTES, NetworkSpec, PortProtocol, PublishedPortSpec, SandboxLogLevel, SandboxResources, SandboxRuntimeOptions, SandboxSpec, diff --git a/sdk/rust/lib/sandbox/modify.rs b/sdk/rust/lib/sandbox/modify.rs index 16da7b806..d0a28cf5b 100644 --- a/sdk/rust/lib/sandbox/modify.rs +++ b/sdk/rust/lib/sandbox/modify.rs @@ -684,7 +684,7 @@ async fn live_control(name: &str, status: SandboxStatus) -> LiveControl { } /// Ask the sandbox process which live-control operations it serves. -async fn control_capabilities( +pub(super) async fn control_capabilities( name: &str, ) -> MicrosandboxResult { let response = control_request(name, "{\"op\":\"capabilities\"}\n".to_string()).await?; @@ -723,7 +723,7 @@ async fn connect_control_pipe( } /// Send one control request line and parse the reply. -async fn control_request( +pub(super) async fn control_request( name: &str, request: String, ) -> MicrosandboxResult { @@ -739,6 +739,33 @@ async fn control_request( Ok(response) } +/// Use the handle's local backend, never an ambient backend with a matching sandbox name. +pub(super) async fn control_request_for( + local: &crate::backend::LocalBackend, + name: &str, + request: String, +) -> MicrosandboxResult { + let candidates = crate::runtime::sandbox_agent_socket_path_candidates_for(local, name) + .into_iter() + .map(|path| microsandbox_runtime::control::control_socket_path_for(&path)); + #[cfg(unix)] + let stream = connect_control_socket(candidates).await?; + #[cfg(windows)] + let stream = + connect_control_pipe(&candidates.into_iter().next().ok_or_else(|| { + crate::MicrosandboxError::Runtime("no backend control endpoint".into()) + })?) + .await?; + let response = control_request_over_stream(stream, &request).await?; + if !response.ok { + return Err(crate::MicrosandboxError::Runtime(format!( + "runtime control refused: {}", + response.error.unwrap_or_else(|| "unknown error".into()) + ))); + } + Ok(response) +} + async fn control_request_raw( name: &str, request: String, diff --git a/sdk/rust/lib/sandbox/pause.rs b/sdk/rust/lib/sandbox/pause.rs new file mode 100644 index 000000000..b2736088e --- /dev/null +++ b/sdk/rust/lib/sandbox/pause.rs @@ -0,0 +1,194 @@ +//! Resident pause/resume through the existing host control endpoint. + +use microsandbox_runtime::control::ControlRequest; + +use crate::backend::{Backend, LocalBackend}; +use crate::error::Operation; +use crate::{MicrosandboxError, MicrosandboxResult}; + +use super::{Sandbox, SandboxHandle, SandboxPauseState, modify}; + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl Sandbox { + /// Suspend this resident VM without creating a snapshot or releasing RAM. + pub async fn pause(&self) -> MicrosandboxResult<()> { + lifecycle(self.name(), self.backend().as_ref(), ControlRequest::Pause) + .await + .map(|_| ()) + } + + /// Resume the same VM and processes, correcting wall clock before thawing workloads. + pub async fn resume(&self) -> MicrosandboxResult<()> { + lifecycle(self.name(), self.backend().as_ref(), ControlRequest::Resume) + .await + .map(|_| ()) + } + + /// Inspect the host-confirmed pause state without contacting the suspended guest. + pub async fn pause_state(&self) -> MicrosandboxResult { + lifecycle( + self.name(), + self.backend().as_ref(), + ControlRequest::PauseState, + ) + .await + } +} + +impl SandboxHandle { + /// Suspend an existing resident sandbox without connecting to its guest. + pub async fn pause(&self) -> MicrosandboxResult<()> { + lifecycle(self.name(), self.backend.as_ref(), ControlRequest::Pause) + .await + .map(|_| ()) + } + + /// Resume an existing user-paused sandbox through host control. + pub async fn resume(&self) -> MicrosandboxResult<()> { + lifecycle(self.name(), self.backend.as_ref(), ControlRequest::Resume) + .await + .map(|_| ()) + } + + /// Inspect resident suspension without opening an agent connection. + pub async fn pause_state(&self) -> MicrosandboxResult { + lifecycle( + self.name(), + self.backend.as_ref(), + ControlRequest::PauseState, + ) + .await + } +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +/// Overlay resident suspension on database lifecycle without persisting a stale pause on crash. +pub(crate) async fn projected_status( + local: &LocalBackend, + name: &str, + status: super::SandboxStatus, +) -> super::SandboxStatus { + if status != super::SandboxStatus::Running { + return status; + } + // Old runtimes have no pause endpoint. Bound observation so a busy or unavailable host + // never makes ordinary list/get wait for an entire checkpoint operation. + let request = modify::control_request_for(local, name, "{\"op\":\"pause_state\"}\n".into()); + match tokio::time::timeout(std::time::Duration::from_millis(250), request).await { + Ok(Ok(response)) + if response + .pause + .as_ref() + .is_some_and(|state| state.paused || state.recovery_required) => + { + super::SandboxStatus::Paused + } + _ => status, + } +} + +async fn lifecycle( + name: &str, + backend: &dyn Backend, + request: ControlRequest, +) -> MicrosandboxResult { + let operation = if matches!(request, ControlRequest::Resume) { + Operation::SandboxResume + } else { + Operation::SandboxPause + }; + let local = backend + .as_local() + .ok_or_else(|| MicrosandboxError::local_only(operation))?; + // Do not send a new operation to an old runtime that cannot implement its semantics. + let capabilities = + modify::control_request_for(local, name, "{\"op\":\"capabilities\"}\n".into()).await?; + if !capabilities + .capabilities + .is_some_and(|caps| caps.pause_resume) + { + return Err(MicrosandboxError::Runtime("resident pause/resume requires a runtime and guest kernel with clock-only resume support".into())); + } + let line = format!("{}\n", serde_json::to_string(&request)?); + let response = modify::control_request_for(local, name, line).await?; + response + .pause + .ok_or_else(|| MicrosandboxError::Runtime("control response omitted pause state".into())) +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(all(test, unix))] +mod tests { + use std::sync::Arc; + + use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; + + use super::*; + use crate::backend::with_backend; + + #[tokio::test] + async fn pause_observation_uses_bound_backend_outside_its_ambient_scope() { + // macOS's per-user TMPDIR may already consume most of the Unix socket path limit. + let ambient_home = tempfile::tempdir_in("/tmp").unwrap(); + let bound_home = tempfile::tempdir_in("/tmp").unwrap(); + let ambient: Arc = Arc::new( + LocalBackend::builder() + .home(ambient_home.path()) + .build() + .await + .unwrap(), + ); + let bound = LocalBackend::builder() + .home(bound_home.path()) + .build() + .await + .unwrap(); + let agent = + crate::runtime::sandbox_agent_socket_path_candidates_for(&bound, "same-name").remove(0); + let path = microsandbox_runtime::control::control_socket_path_for(&agent); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + let listener = tokio::net::UnixListener::bind(path).unwrap(); + let server = tokio::spawn(async move { + // One observation plus the capability-gated public lifecycle exchange. + for _ in 0..3 { + let (stream, _) = listener.accept().await.unwrap(); + let mut stream = BufReader::new(stream); + let mut line = String::new(); + stream.read_line(&mut line).await.unwrap(); + let response = if line.contains("capabilities") { + "{\"ok\":true,\"capabilities\":{\"pause_resume\":true,\"cpu_resize\":false,\"memory_resize\":false,\"secrets_update\":false}}\n" + } else { + "{\"ok\":true,\"pause\":{\"paused\":true,\"recovery_required\":false,\"capture_unavailable\":null}}\n" + }; + stream + .get_mut() + .write_all(response.as_bytes()) + .await + .unwrap(); + } + }); + with_backend(ambient, async { + assert_eq!( + projected_status(&bound, "same-name", super::super::SandboxStatus::Running).await, + super::super::SandboxStatus::Paused + ); + assert!( + lifecycle("same-name", &bound, ControlRequest::PauseState) + .await + .unwrap() + .paused + ); + }) + .await; + server.await.unwrap(); + } +} diff --git a/vendor/libkrunfw b/vendor/libkrunfw index 4b334c292..6cca413ac 160000 --- a/vendor/libkrunfw +++ b/vendor/libkrunfw @@ -1 +1 @@ -Subproject commit 4b334c292ebae7364d9413d32cd57ce510a99e2d +Subproject commit 6cca413ac248f63e65d4ea4748b3bc36cd1b22f3 From 776e92f69c40a3e7d8bc20673fc2bdaa1d5f977f Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Mon, 7 Sep 2026 15:59:22 +0100 Subject: [PATCH 05/29] test(snapshot): record resize coverage and windows restore failures Extend the live lifecycle runner with optional memory resizing, unsupported CoW admission checks, archive retention diagnostics, and explicit timeout evidence. Assert retained child contents after archive removal and source shutdown. Record passing Linux capacity convergence and Windows post-restore command timeouts without treating activation latency as usable restore performance. Keep incomplete qualification and implementation work explicit in the report. --- scripts/smoke/cli/cow-memory-lifecycle.py | 55 +++++++++++++++++-- .../cow-memory-lifecycle-2026-09-07.md | 16 +++++- 2 files changed, 64 insertions(+), 7 deletions(-) diff --git a/scripts/smoke/cli/cow-memory-lifecycle.py b/scripts/smoke/cli/cow-memory-lifecycle.py index 100b6b023..ba935cc97 100644 --- a/scripts/smoke/cli/cow-memory-lifecycle.py +++ b/scripts/smoke/cli/cow-memory-lifecycle.py @@ -12,12 +12,27 @@ prefix = os.environ.get("STACK8_PREFIX", "cow8") mode = os.environ.get("STACK8_MODE", "cow") layout = os.environ.get("STACK8_LAYOUT", "flat:512M") +resize = os.environ.get("STACK8_LIVE_RESIZE") == "1" rows = [] names = [] def run(label, *args, expected=0, timeout=120): started = time.perf_counter() - result = subprocess.run([binary, *args], text=True, capture_output=True, timeout=timeout) + try: + result = subprocess.run([binary, *args], text=True, capture_output=True, timeout=timeout) + except subprocess.TimeoutExpired as error: + # TimeoutExpired can carry bytes even when text=True. Preserve the failed + # command in the evidence instead of recording only subsequent cleanup. + for stream in ("stdout", "stderr"): + captured = getattr(error, stream) or b"" + if isinstance(captured, bytes): + captured = captured.decode("utf-8", errors="replace") + (root / (label + "." + stream)).write_text(captured) + row = {"case": label, "ms": round((time.perf_counter() - started) * 1000, 2), + "exit": "timeout", "timeout_seconds": timeout} + rows.append(row) + print(json.dumps(row), flush=True) + raise elapsed = (time.perf_counter() - started) * 1000 (root / (label + ".stdout")).write_text(result.stdout) (root / (label + ".stderr")).write_text(result.stderr) @@ -29,15 +44,44 @@ def run(label, *args, expected=0, timeout=120): return result try: + if os.environ.get("STACK8_CHECK_COW_REJECTION") == "1": + refused = prefix + "-cow-refused" + names.append(refused) + result = run("cow-unsupported", "create", "alpine", "-n", refused, + "--memory", "256M", "--memory-snapshot", "cow", expected=None) + assert result.returncode != 0, "unsupported CoW must not silently start eagerly" + assert "not qualified" in result.stderr or "not qualified" in result.stdout + inspected = run("cow-refused-inspect", "inspect", refused, "--format", "json", expected=None) + if inspected.returncode == 0: + assert json.loads(inspected.stdout)["status"] not in ("Running", "Paused") source = prefix + "-source" names.append(source) run("fresh-" + mode, "run", "-d", "-n", source, "--memory-snapshot", mode, - "--root-disk", layout, "--memory", "256M", "--cpus", "2", "alpine", + "--root-disk", layout, "--memory", "256M", "--cpus", "2", + *(["--max-memory", "512M"] if resize else []), "alpine", "--", "sh", "-c", "mkdir -p /dev/shm; echo captured > /dev/shm/cow-marker; i=0; while :; do echo $i > /tmp/cow-counter; i=$((i+1)); sleep 0.05; done") run("marker-source", "exec", source, "--", "cat", "/dev/shm/cow-marker") boot_id = run("boot-id-before", "exec", source, "--", "cat", "/proc/sys/kernel/random/boot_id").stdout.strip() process = run("process-before", "exec", source, "--", "sh", "-c", "for p in /proc/[0-9]*/cmdline; do tr '\\0' ' ' < $p; echo; done").stdout assert "cow-counter" in process + if resize: + baseline = int(run("memory-baseline", "exec", source, "--", "sh", "-c", + "awk '/MemTotal/ {print $2}' /proc/meminfo").stdout.strip()) + for step, target in enumerate((384, 256, 512, 256)): + run(f"memory-target-{step}", "modify", source, "--memory", f"{target}M", "--format", "json") + deadline = time.monotonic() + 30 + sample = 0 + while True: + observed = int(run(f"memory-convergence-{step}-{sample}", "exec", source, + "--", "sh", "-c", "awk '/MemTotal/ {print $2}' /proc/meminfo").stdout.strip()) + # Hotplug metadata consumes some newly onlined pages. Check actual guest + # capacity within 4 MiB, not just an accepted host target/configuration. + if abs(observed - (baseline + (target - 256) * 1024)) <= 4096: + break + assert time.monotonic() < deadline, f"memory target {target} did not converge: {observed} KiB" + sample += 1 + time.sleep(0.1) + assert run(f"memory-marker-{step}", "exec", source, "--", "cat", "/dev/shm/cow-marker").stdout.strip() == "captured" snap = prefix + "-full" run("first-full", "snapshot", "create", snap, "--from", source, "--full", "--info") run("pause", "pause", source) @@ -76,11 +120,12 @@ def run(label, *args, expected=0, timeout=120): names.append(child) run("direct-restore", "create", "-n", child, "--from-snapshot", archive, "--memory-snapshot", mode, "--info") - Path(archive).unlink() - run("archive-unlink-survival", "exec", child, "--", "cat", "/dev/shm/cow-marker") + if os.environ.get("STACK8_KEEP_ARCHIVE") != "1": + Path(archive).unlink() + assert run("archive-child-exec", "exec", child, "--", "cat", "/dev/shm/cow-marker").stdout.strip() == "captured" run("pause-for-stop", "pause", source) run("stop-paused", "stop", source, timeout=20) - run("child-after-source-stop", "exec", prefix + "-a", "--", "cat", "/dev/shm/cow-marker") + assert run("child-after-source-stop", "exec", prefix + "-a", "--", "cat", "/dev/shm/cow-marker").stdout.strip() == "private-a" finally: for name in reversed(names): try: diff --git a/scripts/smoke/reports/cow-memory-lifecycle-2026-09-07.md b/scripts/smoke/reports/cow-memory-lifecycle-2026-09-07.md index 6cb3b6507..bdcd8fc0e 100644 --- a/scripts/smoke/reports/cow-memory-lifecycle-2026-09-07.md +++ b/scripts/smoke/reports/cow-memory-lifecycle-2026-09-07.md @@ -6,6 +6,8 @@ Status: development integration and live smoke coverage, not full platform or pe Use `scripts/smoke/cli/cow-memory-lifecycle.py` with `MSB_PATH`, an isolated `MSB_HOME`, matching `MSB_LIBKRUNFW_PATH`, an output directory in `STACK8_OUT`, and a fresh `STACK8_PREFIX`. Select `STACK8_LAYOUT=flat:512M`, `512M`, or `tmpfs`, `STACK8_MODE=cow` or `standard`, and optionally `STACK8_PAUSE_SECONDS=10`. The workload uses Alpine, 256 MiB RAM, and two vCPUs. The runner records each command's elapsed wall time, stdout, stderr, and exit code, and attempts to stop every sandbox it starts in `finally`. +Optional `STACK8_LIVE_RESIZE=1` adds a 512 MiB ceiling and exercises targets 384, 256, 512, and 256 MiB before capture. It checks actual guest MemTotal convergence relative to the baseline within 4 MiB and preserves a tmpfs marker. `STACK8_CHECK_COW_REJECTION=1` checks unsupported-platform rejection without a running/paused fallback. `STACK8_KEEP_ARCHIVE=1` retains the direct archive for diagnosis instead of testing unlink survival. Timed-out commands are recorded explicitly, with partial output, before cleanup runs. + The opt-in language SDK tests are `sdk/python/tests/test_cow_lifecycle.py`, `sdk/node-ts/tests/cow-lifecycle.test.ts`, and `sdk/go/cow_lifecycle_test.go`. Set `MSB_COW_LIVE=1`; Go also requires tags `cow_live microsandbox_ffi_path` and `MICROSANDBOX_FFI_PATH` pointing to its matching native library. ## Observed coverage @@ -14,7 +16,17 @@ Linux/KVM x86-64 passed CoW flat, managed, and tmpfs roots, plus a standard-memo The later Linux flat/managed/tmpfs/standard runs and macOS tmpfs run additionally assert unchanged Linux boot ID across ordinary pause/resume, resumed progress of the original counter workload, and guest wall clock within three seconds of the host after a ten-second pause. These checks do not constitute host-suspend, every clock-failure, or VM Generation ID notification testing. -The Python, Node, and Go live SDK checks passed on macOS: create with explicit CoW, pause, capture while paused, resume through a handle, restore a child, verify tmpfs contents and source/child write isolation, and stop a paused child. Windows ARM64 firmware build/export/load checks passed; native runtime compilation and live lifecycle coverage are still in progress. Windows explicit CoW requests remain unsupported and must fail without an eager fallback. +The Python, Node, and Go live SDK checks passed on macOS: create with explicit CoW, pause, capture while paused, resume through a handle, restore a child, verify tmpfs contents and source/child write isolation, and stop a paused child. A further Linux flat-root CoW run passed the live-resize sequence described above and the subsequent full lifecycle matrix. This verifies basic capacity convergence and retained marker contents, not physical host-memory reclamation or every unplug/replug invariant. + +Windows ARM64 firmware build/export/load checks and native `aarch64-pc-windows-msvc` runtime compilation passed. Explicit CoW rejection passed without starting a VM. Standard-memory lifecycle qualification failed as detailed below; Windows CoW remains unsupported. + +### Windows post-restore failure + +The first standard-memory flat-root run passed fresh boot, running full capture, pause/idempotence/status/admission checks, two paused captures, resume/idempotence, boot identity, ten-second pause clock correction, workload progress, and two installed-snapshot child restores with write isolation. Direct full archive creation took 10,247.52 ms and produced 20,630,604 bytes. Direct restore returned success after 4,912.00 ms, but the first guest command timed out after 120 seconds. That is a failed usable-restore result, not a 4.9-second successful restore benchmark. + +A repeat intended to retain the archive failed earlier: its first installed-snapshot child reported restored in 2,934.15 ms but its first guest command timed out after 120 seconds. No direct archive or archive unlink had been reached, ruling out archive deletion as the sole explanation. Three bounded restores of that retained installed snapshot subsequently all reported activation in 3,066.11–3,182.01 ms and all timed out at the 15-second guest-command bound. Do not infer successful activation implies a usable guest or use these values as successful restore latency. + +Trace evidence shows successful generation/clock acknowledgement and workload-thaw exchange, subsequent command bytes delivered to the virtual console, and continuing filesystem device activity during the command stall. The exact cause remains unresolved; the evidence does not establish a whole-VM freeze, archive corruption, or a specific interrupt/agent defect. All four VMs started by the first matrix and both started by the repeat received successful stop responses; cleanup also covered the rejected CoW sandbox record. A process inventory confirmed no remaining `cow8-windows-standard` or `cow8-windows-keep` runtime. Each bounded diagnostic also runs stop in `finally`. A separate fresh diagnostic snapshot reproduced the command timeout on all three children, but its attempted guest task-stack output did not reach `kernel.log` and supplies no task-stack diagnosis. Earlier unrelated development VMs were not terminated. ## Individual debug-build timings @@ -39,6 +51,6 @@ The macOS flat run logged APFS reflink reuse for repeated cache construction, in Rust SDK library tests: 668 passed, three ignored. Runtime tests: 179 passed. CLI library tests: 315 passed, plus three enabled CLI integration checks; platform-dependent ignored tests remain ignored. Node: 137 unit tests and typecheck passed. Go unit and native-FFI smoke tests passed. Python's focused API/stub tests passed (33 tests). The new backend-binding regression test verifies that pause observation and lifecycle requests use the handle's local backend rather than an ambient backend with the same sandbox name. CoW cache location is also passed from the owning backend at launch. -CoW virtio-mem unplug currently writes private zeros to prevent old backing bytes from reappearing, but does not reclaim those pages' host RAM. NUMA plus CoW is explicitly rejected. These are outstanding integration/performance limitations, not completed acceptance items. Further work includes backing-aware physical reclamation, resize/balloon live invariants, real shared/private resident-memory measurements, cache eviction and publication failure races, cancellation and lifecycle/maintenance concurrency, unsupported guest preparation, recovery/resume failures, Windows standard lifecycle qualification, and repeated release-build performance distributions. Public cache inspection/eviction workflow and comprehensive archive compatibility variants also remain to be completed. Do not mark #8 complete from these smoke results. +CoW virtio-mem unplug currently writes private zeros to prevent old backing bytes from reappearing, but does not reclaim those pages' host RAM. NUMA plus CoW is explicitly rejected. These are outstanding integration/performance limitations, not completed acceptance items. Further work includes backing-aware physical reclamation, deeper resize/balloon live invariants beyond the basic Linux sequence, real shared/private resident-memory measurements, cache eviction and publication failure races, cancellation and lifecycle/maintenance concurrency, unsupported guest preparation, recovery/resume failures, fixing and qualifying the Windows post-restore failure, and repeated release-build performance distributions. Public cache inspection/eviction workflow and comprehensive archive compatibility variants also remain to be completed. Do not mark #8 complete from these smoke results. Evidence locations: OVH `/home/ubuntu/msb-stack8.ElfKzf/`; macOS `/private/tmp/msb-stack8-mac-{results,managed-results,tmpfs-results,standard-results}/`; Windows isolated worktree `C:\Users\Stephen\AppData\Local\Temp\msb-stack8-20260907`. These are development outputs, not shipped artifacts. From 15ab8838da58061df5dd9560cbd85d648bb72351 Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Mon, 7 Sep 2026 22:24:30 +0100 Subject: [PATCH 06/29] fix(checkpoint): integrate arm and windows execution-state fixes Capture device workers before interrupt-controller state and RAM so late device completions cannot leave the checkpoint's queues and IRQs at different boundaries. Pin libkrun's processor, interrupt-controller and clock-state fixes. Add portable delayed-restore clock and offline-CPU live fixtures, and wait for application initialization in the lifecycle smoke runner. Record passing Windows ARM64 and nested Linux ARM64 live matrices, plus macOS and Linux x86 regressions. Keep native Windows x86 and Windows CoW cache qualification limits explicit. --- Cargo.lock | 22 +++--- Cargo.toml | 22 +++--- crates/runtime/lib/checkpoint/coordinator.rs | 43 ++++++----- scripts/smoke/cli/checkpoint-clock.py | 73 +++++++++++++++++++ scripts/smoke/cli/checkpoint-cpu-probe.rs | 34 +++++++++ scripts/smoke/cli/checkpoint-cpu-state.py | 54 ++++++++++++++ scripts/smoke/cli/cow-memory-lifecycle.py | 8 ++ .../cow-memory-lifecycle-2026-09-07.md | 2 + .../reports/execution-state-2026-09-07.md | 68 +++++++++++++++++ 9 files changed, 285 insertions(+), 41 deletions(-) create mode 100644 scripts/smoke/cli/checkpoint-clock.py create mode 100644 scripts/smoke/cli/checkpoint-cpu-probe.rs create mode 100644 scripts/smoke/cli/checkpoint-cpu-state.py create mode 100644 scripts/smoke/reports/execution-state-2026-09-07.md diff --git a/Cargo.lock b/Cargo.lock index 88caeb69c..f4bf7ed7d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4353,7 +4353,7 @@ dependencies = [ [[package]] name = "msb_krun" version = "0.1.33" -source = "git+https://github.com/superradcompany/libkrun?rev=94d680b21bf7ea7c2bed5262ed211833bd4379bd#94d680b21bf7ea7c2bed5262ed211833bd4379bd" +source = "git+https://github.com/superradcompany/libkrun?rev=51c1ed3b83dc826c02800fb297995538e4eac55d#51c1ed3b83dc826c02800fb297995538e4eac55d" dependencies = [ "crossbeam-channel", "kvm-bindings", @@ -4372,7 +4372,7 @@ dependencies = [ [[package]] name = "msb_krun_arch" version = "0.1.33" -source = "git+https://github.com/superradcompany/libkrun?rev=94d680b21bf7ea7c2bed5262ed211833bd4379bd#94d680b21bf7ea7c2bed5262ed211833bd4379bd" +source = "git+https://github.com/superradcompany/libkrun?rev=51c1ed3b83dc826c02800fb297995538e4eac55d#51c1ed3b83dc826c02800fb297995538e4eac55d" dependencies = [ "kvm-bindings", "kvm-ioctls", @@ -4386,12 +4386,12 @@ dependencies = [ [[package]] name = "msb_krun_arch_gen" version = "0.1.33" -source = "git+https://github.com/superradcompany/libkrun?rev=94d680b21bf7ea7c2bed5262ed211833bd4379bd#94d680b21bf7ea7c2bed5262ed211833bd4379bd" +source = "git+https://github.com/superradcompany/libkrun?rev=51c1ed3b83dc826c02800fb297995538e4eac55d#51c1ed3b83dc826c02800fb297995538e4eac55d" [[package]] name = "msb_krun_cpuid" version = "0.1.33" -source = "git+https://github.com/superradcompany/libkrun?rev=94d680b21bf7ea7c2bed5262ed211833bd4379bd#94d680b21bf7ea7c2bed5262ed211833bd4379bd" +source = "git+https://github.com/superradcompany/libkrun?rev=51c1ed3b83dc826c02800fb297995538e4eac55d#51c1ed3b83dc826c02800fb297995538e4eac55d" dependencies = [ "kvm-bindings", "kvm-ioctls", @@ -4401,7 +4401,7 @@ dependencies = [ [[package]] name = "msb_krun_devices" version = "0.1.33" -source = "git+https://github.com/superradcompany/libkrun?rev=94d680b21bf7ea7c2bed5262ed211833bd4379bd#94d680b21bf7ea7c2bed5262ed211833bd4379bd" +source = "git+https://github.com/superradcompany/libkrun?rev=51c1ed3b83dc826c02800fb297995538e4eac55d#51c1ed3b83dc826c02800fb297995538e4eac55d" dependencies = [ "bincode", "bitflags 1.3.2", @@ -4432,7 +4432,7 @@ dependencies = [ [[package]] name = "msb_krun_hvf" version = "0.1.33" -source = "git+https://github.com/superradcompany/libkrun?rev=94d680b21bf7ea7c2bed5262ed211833bd4379bd#94d680b21bf7ea7c2bed5262ed211833bd4379bd" +source = "git+https://github.com/superradcompany/libkrun?rev=51c1ed3b83dc826c02800fb297995538e4eac55d#51c1ed3b83dc826c02800fb297995538e4eac55d" dependencies = [ "crossbeam-channel", "libloading 0.8.9", @@ -4444,7 +4444,7 @@ dependencies = [ [[package]] name = "msb_krun_kernel" version = "0.1.33" -source = "git+https://github.com/superradcompany/libkrun?rev=94d680b21bf7ea7c2bed5262ed211833bd4379bd#94d680b21bf7ea7c2bed5262ed211833bd4379bd" +source = "git+https://github.com/superradcompany/libkrun?rev=51c1ed3b83dc826c02800fb297995538e4eac55d#51c1ed3b83dc826c02800fb297995538e4eac55d" dependencies = [ "msb-vm-memory", "msb_krun_utils", @@ -4453,7 +4453,7 @@ dependencies = [ [[package]] name = "msb_krun_polly" version = "0.1.33" -source = "git+https://github.com/superradcompany/libkrun?rev=94d680b21bf7ea7c2bed5262ed211833bd4379bd#94d680b21bf7ea7c2bed5262ed211833bd4379bd" +source = "git+https://github.com/superradcompany/libkrun?rev=51c1ed3b83dc826c02800fb297995538e4eac55d#51c1ed3b83dc826c02800fb297995538e4eac55d" dependencies = [ "libc", "msb_krun_utils", @@ -4462,7 +4462,7 @@ dependencies = [ [[package]] name = "msb_krun_smbios" version = "0.1.33" -source = "git+https://github.com/superradcompany/libkrun?rev=94d680b21bf7ea7c2bed5262ed211833bd4379bd#94d680b21bf7ea7c2bed5262ed211833bd4379bd" +source = "git+https://github.com/superradcompany/libkrun?rev=51c1ed3b83dc826c02800fb297995538e4eac55d#51c1ed3b83dc826c02800fb297995538e4eac55d" dependencies = [ "msb-vm-memory", ] @@ -4470,7 +4470,7 @@ dependencies = [ [[package]] name = "msb_krun_utils" version = "0.1.33" -source = "git+https://github.com/superradcompany/libkrun?rev=94d680b21bf7ea7c2bed5262ed211833bd4379bd#94d680b21bf7ea7c2bed5262ed211833bd4379bd" +source = "git+https://github.com/superradcompany/libkrun?rev=51c1ed3b83dc826c02800fb297995538e4eac55d#51c1ed3b83dc826c02800fb297995538e4eac55d" dependencies = [ "bitflags 1.3.2", "crossbeam-channel", @@ -4485,7 +4485,7 @@ dependencies = [ [[package]] name = "msb_krun_vmm" version = "0.1.33" -source = "git+https://github.com/superradcompany/libkrun?rev=94d680b21bf7ea7c2bed5262ed211833bd4379bd#94d680b21bf7ea7c2bed5262ed211833bd4379bd" +source = "git+https://github.com/superradcompany/libkrun?rev=51c1ed3b83dc826c02800fb297995538e4eac55d#51c1ed3b83dc826c02800fb297995538e4eac55d" dependencies = [ "bincode", "bzip2", diff --git a/Cargo.toml b/Cargo.toml index 0308b7983..36f34681e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -208,14 +208,14 @@ russh-sftp = "2.3.0" # #8 construction-time private memory and identity-preserving resume support. [patch.crates-io] -msb_krun = { git = "https://github.com/superradcompany/libkrun", rev = "94d680b21bf7ea7c2bed5262ed211833bd4379bd" } -msb_krun_utils = { git = "https://github.com/superradcompany/libkrun", rev = "94d680b21bf7ea7c2bed5262ed211833bd4379bd" } -msb_krun_vmm = { git = "https://github.com/superradcompany/libkrun", rev = "94d680b21bf7ea7c2bed5262ed211833bd4379bd" } -msb_krun_devices = { git = "https://github.com/superradcompany/libkrun", rev = "94d680b21bf7ea7c2bed5262ed211833bd4379bd" } -msb_krun_arch = { git = "https://github.com/superradcompany/libkrun", rev = "94d680b21bf7ea7c2bed5262ed211833bd4379bd" } -msb_krun_arch_gen = { git = "https://github.com/superradcompany/libkrun", rev = "94d680b21bf7ea7c2bed5262ed211833bd4379bd" } -msb_krun_cpuid = { git = "https://github.com/superradcompany/libkrun", rev = "94d680b21bf7ea7c2bed5262ed211833bd4379bd" } -msb_krun_hvf = { git = "https://github.com/superradcompany/libkrun", rev = "94d680b21bf7ea7c2bed5262ed211833bd4379bd" } -msb_krun_kernel = { git = "https://github.com/superradcompany/libkrun", rev = "94d680b21bf7ea7c2bed5262ed211833bd4379bd" } -msb_krun_polly = { git = "https://github.com/superradcompany/libkrun", rev = "94d680b21bf7ea7c2bed5262ed211833bd4379bd" } -msb_krun_smbios = { git = "https://github.com/superradcompany/libkrun", rev = "94d680b21bf7ea7c2bed5262ed211833bd4379bd" } +msb_krun = { git = "https://github.com/superradcompany/libkrun", rev = "51c1ed3b83dc826c02800fb297995538e4eac55d" } +msb_krun_utils = { git = "https://github.com/superradcompany/libkrun", rev = "51c1ed3b83dc826c02800fb297995538e4eac55d" } +msb_krun_vmm = { git = "https://github.com/superradcompany/libkrun", rev = "51c1ed3b83dc826c02800fb297995538e4eac55d" } +msb_krun_devices = { git = "https://github.com/superradcompany/libkrun", rev = "51c1ed3b83dc826c02800fb297995538e4eac55d" } +msb_krun_arch = { git = "https://github.com/superradcompany/libkrun", rev = "51c1ed3b83dc826c02800fb297995538e4eac55d" } +msb_krun_arch_gen = { git = "https://github.com/superradcompany/libkrun", rev = "51c1ed3b83dc826c02800fb297995538e4eac55d" } +msb_krun_cpuid = { git = "https://github.com/superradcompany/libkrun", rev = "51c1ed3b83dc826c02800fb297995538e4eac55d" } +msb_krun_hvf = { git = "https://github.com/superradcompany/libkrun", rev = "51c1ed3b83dc826c02800fb297995538e4eac55d" } +msb_krun_kernel = { git = "https://github.com/superradcompany/libkrun", rev = "51c1ed3b83dc826c02800fb297995538e4eac55d" } +msb_krun_polly = { git = "https://github.com/superradcompany/libkrun", rev = "51c1ed3b83dc826c02800fb297995538e4eac55d" } +msb_krun_smbios = { git = "https://github.com/superradcompany/libkrun", rev = "51c1ed3b83dc826c02800fb297995538e4eac55d" } diff --git a/crates/runtime/lib/checkpoint/coordinator.rs b/crates/runtime/lib/checkpoint/coordinator.rs index 703c9dafb..3ef84146f 100644 --- a/crates/runtime/lib/checkpoint/coordinator.rs +++ b/crates/runtime/lib/checkpoint/coordinator.rs @@ -708,25 +708,6 @@ impl CheckpointCoordinator { final_path: &Path, ) -> Result { let mut timings = PausedCaptureTimings::default(); - let execution_started = Instant::now(); - let execution = vm - .capture_execution_state() - .map_err(CheckpointFailure::resumable)?; - if execution.pause_generation() != pause_generation { - return Err(CheckpointFailure::resumable( - "execution state belongs to another pause generation", - )); - } - let execution_bytes = execution.encode().map_err(CheckpointFailure::resumable)?; - let execution_id = self - .store - .put_bytes(&execution_bytes) - .map_err(CheckpointFailure::resumable)?; - self.store - .link_into(&execution_id, staging) - .map_err(CheckpointFailure::resumable)?; - timings.execution_us = execution_started.elapsed().as_micros(); - let devices_started = Instant::now(); let mut pending_devices = Vec::with_capacity(inventory.len()); let mut disk_roots = Vec::new(); @@ -808,6 +789,30 @@ impl CheckpointCoordinator { .map_err(CheckpointFailure::resumable)?; timings.devices_us = devices_started.elapsed().as_micros(); + // Device capture parks each worker. Capture interrupt-controller state + // only after their final completions have been published; otherwise a + // used queue could survive in RAM without its corresponding interrupt. + // Execution capture must also precede RAM capture: KVM flushes its LPI + // pending tables into guest RAM as part of this operation. + let execution_started = Instant::now(); + let execution = vm + .capture_execution_state() + .map_err(CheckpointFailure::resumable)?; + if execution.pause_generation() != pause_generation { + return Err(CheckpointFailure::resumable( + "execution state belongs to another pause generation", + )); + } + let execution_bytes = execution.encode().map_err(CheckpointFailure::resumable)?; + let execution_id = self + .store + .put_bytes(&execution_bytes) + .map_err(CheckpointFailure::resumable)?; + self.store + .link_into(&execution_id, staging) + .map_err(CheckpointFailure::resumable)?; + timings.execution_us = execution_started.elapsed().as_micros(); + let memory_plan_started = Instant::now(); let (memory_plan, memory_mode, base_extents) = self.plan_memory(vm).map_err(CheckpointFailure::resumable)?; diff --git a/scripts/smoke/cli/checkpoint-clock.py b/scripts/smoke/cli/checkpoint-clock.py new file mode 100644 index 000000000..55b65f667 --- /dev/null +++ b/scripts/smoke/cli/checkpoint-clock.py @@ -0,0 +1,73 @@ +"""Cross-platform host runner for the static Linux guest clock fixture. + +Requires MSB_PATH, MSB_HOME, MSB_LIBKRUNFW_PATH, CLOCK_PROBE, CLOCK_OUT, +and a unique CLOCK_PREFIX. Never reuses an existing sandbox or snapshot. +""" +import json +import os +from pathlib import Path +import subprocess +import sys +import time + +binary = os.environ["MSB_PATH"] +out = Path(os.environ["CLOCK_OUT"]) +out.mkdir(parents=True, exist_ok=True) +prefix = os.environ["CLOCK_PREFIX"] +source, child, snapshot = prefix + "-source", prefix + "-child", prefix + "-full" +rows = [] + + +def run(label, *args, check=True, timeout=120): + started = time.perf_counter() + result = subprocess.run([binary, *args], capture_output=True, timeout=timeout) + (out / (label + ".stdout")).write_bytes(result.stdout) + (out / (label + ".stderr")).write_bytes(result.stderr) + row = {"case": label, "ms": round((time.perf_counter() - started) * 1000, 2), "exit": result.returncode} + rows.append(row) + print(json.dumps(row), flush=True) + if check and result.returncode: + raise RuntimeError(f"{label}: {result.stderr.decode(errors='replace')}") + return result + + +try: + run("create", "run", "-d", "-n", source, + "--root-disk", os.environ.get("CLOCK_LAYOUT", "flat:512M"), + "--cpus", os.environ.get("CLOCK_CPUS", "2"), "--memory", "256M", + "--memory-snapshot", os.environ.get("CLOCK_MEMORY", "standard"), + "alpine", "--", "sh", "-c", + "while [ ! -x /clock-probe ]; do sleep 0.05; done; exec /clock-probe") + run("copy-probe", "copy", os.environ["CLOCK_PROBE"], source + ":/clock-probe") + run("chmod-probe", "exec", source, "--", "chmod", "+x", "/clock-probe") + for attempt in range(30): + if run("ready-" + str(attempt), "exec", source, "--", "test", "-s", "/tmp/clock-records.csv", check=False).returncode == 0: + break + time.sleep(0.05) + else: + raise RuntimeError("guest clock fixture did not start") + run("capture", "snapshot", "create", snapshot, "--from", source, "--full", "--info") + if os.environ.get("CLOCK_INCREMENTAL") == "1": + snapshot = prefix + "-next" + run("capture-next", "snapshot", "create", snapshot, "--from", source, "--full", "--info") + if os.environ.get("CLOCK_ARCHIVE") == "1": + archive = str(out / "clock.msnap") + run("archive", "snapshot", "save", snapshot, archive) + snapshot = archive + run("stop-source", "stop", source) + time.sleep(float(os.environ.get("CLOCK_DELAY", "8"))) + (out / "restore-start.ns").write_text(str(time.time_ns())) + run("restore", "create", "-n", child, "--from-snapshot", snapshot, + "--memory-snapshot", os.environ.get("CLOCK_MEMORY", "standard"), "--info") + (out / "restore-end.ns").write_text(str(time.time_ns())) + time.sleep(6) + records = run("records", "exec", child, "--", "cat", "/tmp/clock-records.csv") + (out / "records.csv").write_bytes(records.stdout) + subprocess.run([sys.executable, str(Path(__file__).with_name("checkpoint-clock-analyze.py")), str(out)], check=True) +finally: + for name in (child, source): + try: + run("cleanup-" + name, "stop", name, check=False, timeout=20) + except Exception as error: + rows.append({"case": "cleanup-" + name, "error": str(error)}) + (out / "results.json").write_text(json.dumps(rows, indent=2)) diff --git a/scripts/smoke/cli/checkpoint-cpu-probe.rs b/scripts/smoke/cli/checkpoint-cpu-probe.rs new file mode 100644 index 000000000..b8aaf1f46 --- /dev/null +++ b/scripts/smoke/cli/checkpoint-cpu-probe.rs @@ -0,0 +1,34 @@ +//! Static Linux guest fixture: prove execution and timer wakeups on a chosen CPU. + +use std::fs; +use std::thread; +use std::time::{Duration, Instant}; + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +unsafe extern "C" { + fn sched_setaffinity(pid: i32, size: usize, mask: *const u64) -> i32; + fn sched_getcpu() -> i32; +} + +fn main() -> std::io::Result<()> { + let cpu: usize = std::env::args().nth(1).expect("CPU index").parse().unwrap(); + assert!(cpu < 64); + let mask = 1_u64 << cpu; + // A successful pin must be followed by observable execution on that CPU. + if unsafe { sched_setaffinity(0, 8, &mask) } != 0 { + return Err(std::io::Error::last_os_error()); + } + let start = Instant::now(); + for _ in 0..10 { + assert_eq!(unsafe { sched_getcpu() }, cpu as i32); + thread::sleep(Duration::from_millis(20)); + } + assert_eq!(unsafe { sched_getcpu() }, cpu as i32); + let marker = fs::read_to_string("/dev/shm/cow-marker")?; + assert_eq!(marker.trim(), "captured"); + println!("cpu={cpu} wakeups=10 elapsed_ms={}", start.elapsed().as_millis()); + Ok(()) +} diff --git a/scripts/smoke/cli/checkpoint-cpu-state.py b/scripts/smoke/cli/checkpoint-cpu-state.py new file mode 100644 index 000000000..78d86a024 --- /dev/null +++ b/scripts/smoke/cli/checkpoint-cpu-state.py @@ -0,0 +1,54 @@ +"""Live two-vCPU restore with CPU1 intentionally offline, then onlined again. + +Set MSB_PATH, MSB_HOME, MSB_LIBKRUNFW_PATH, CPU_PROBE (guest-architecture +checkpoint CPU fixture), CPU_PREFIX (unique), and CPU_OUT. This requires a +guest with CPU hotplug enabled; unsupported hotplug is a failure, not a pass. +""" +import json +import os +from pathlib import Path +import subprocess +import time + +binary = os.environ["MSB_PATH"] +out = Path(os.environ["CPU_OUT"]) +out.mkdir(parents=True, exist_ok=True) +prefix = os.environ["CPU_PREFIX"] +source, child = prefix + "-source", prefix + "-child" +rows = [] + + +def run(label, *args, check=True): + started = time.perf_counter() + result = subprocess.run([binary, *args], capture_output=True, timeout=90) + (out / (label + ".stdout")).write_bytes(result.stdout) + (out / (label + ".stderr")).write_bytes(result.stderr) + row = {"case": label, "ms": round((time.perf_counter() - started) * 1000, 2), "exit": result.returncode} + rows.append(row) + print(json.dumps(row), flush=True) + if check and result.returncode: + raise RuntimeError(f"{label}: {result.stderr.decode(errors='replace')}") + return result.stdout.strip() + + +try: + run("create", "create", "alpine", "-n", source, "--root-disk", "flat:512M", "--memory", "256M", "--cpus", "2") + run("marker", "exec", source, "--", "sh", "-c", "echo captured > /dev/shm/cow-marker") + run("copy-probe", "copy", os.environ["CPU_PROBE"], source + ":/cpu-probe") + run("chmod", "exec", source, "--", "chmod", "+x", "/cpu-probe") + run("cpu1-before", "exec", source, "--", "/cpu-probe", "1") + run("offline", "exec", source, "--", "sh", "-c", "echo 0 > /sys/devices/system/cpu/cpu1/online") + assert run("offline-before", "exec", source, "--", "cat", "/sys/devices/system/cpu/cpu1/online") == b"0" + run("capture", "snapshot", "create", prefix + "-full", "--from", source, "--full", "--info") + run("restore", "create", "-n", child, "--from-snapshot", prefix + "-full", "--info") + assert run("offline-after", "exec", child, "--", "cat", "/sys/devices/system/cpu/cpu1/online") == b"0" + run("cpu0-restored", "exec", child, "--", "/cpu-probe", "0") + run("online", "exec", child, "--", "sh", "-c", "echo 1 > /sys/devices/system/cpu/cpu1/online") + run("cpu1-restored", "exec", child, "--", "/cpu-probe", "1") +finally: + for name in (child, source): + try: + run("cleanup-" + name, "stop", name, check=False) + except Exception as error: + rows.append({"case": "cleanup-" + name, "error": str(error)}) + (out / "results.json").write_text(json.dumps(rows, indent=2)) diff --git a/scripts/smoke/cli/cow-memory-lifecycle.py b/scripts/smoke/cli/cow-memory-lifecycle.py index ba935cc97..aa16b1f9b 100644 --- a/scripts/smoke/cli/cow-memory-lifecycle.py +++ b/scripts/smoke/cli/cow-memory-lifecycle.py @@ -60,6 +60,14 @@ def run(label, *args, expected=0, timeout=120): "--root-disk", layout, "--memory", "256M", "--cpus", "2", *(["--max-memory", "512M"] if resize else []), "alpine", "--", "sh", "-c", "mkdir -p /dev/shm; echo captured > /dev/shm/cow-marker; i=0; while :; do echo $i > /tmp/cow-counter; i=$((i+1)); sleep 0.05; done") + # Detached launch acknowledges the runtime, not the application's first write. + for attempt in range(30): + ready = run("application-ready-" + str(attempt), "exec", source, "--", "test", "-s", "/dev/shm/cow-marker", expected=None) + if ready.returncode == 0: + break + time.sleep(0.1) + else: + raise RuntimeError("application did not initialize its marker") run("marker-source", "exec", source, "--", "cat", "/dev/shm/cow-marker") boot_id = run("boot-id-before", "exec", source, "--", "cat", "/proc/sys/kernel/random/boot_id").stdout.strip() process = run("process-before", "exec", source, "--", "sh", "-c", "for p in /proc/[0-9]*/cmdline; do tr '\\0' ' ' < $p; echo; done").stdout diff --git a/scripts/smoke/reports/cow-memory-lifecycle-2026-09-07.md b/scripts/smoke/reports/cow-memory-lifecycle-2026-09-07.md index bdcd8fc0e..ebd209bb2 100644 --- a/scripts/smoke/reports/cow-memory-lifecycle-2026-09-07.md +++ b/scripts/smoke/reports/cow-memory-lifecycle-2026-09-07.md @@ -1,5 +1,7 @@ # CoW memory and resident lifecycle — 2026-09-07 +Follow-up: [execution-state fixes and qualification](execution-state-2026-09-07.md) supersedes the Windows post-restore failure status below and adds Linux ARM64 coverage. The observations below describe the earlier backend revision. + Status: development integration and live smoke coverage, not full platform or performance qualification. Microsandbox #8 remains stacked directly on #7 `ce04099b`, with libkrun `94d680b21bf7ea7c2bed5262ed211833bd4379bd` and firmware `6cca413ac248f63e65d4ea4748b3bc36cd1b22f3`. The kernel and agentd used below were built from matching development sources on the authorized OVH host, including the ARM64 guest artifacts used on macOS and Windows. ## Reproduce diff --git a/scripts/smoke/reports/execution-state-2026-09-07.md b/scripts/smoke/reports/execution-state-2026-09-07.md new file mode 100644 index 000000000..9d3f0fe93 --- /dev/null +++ b/scripts/smoke/reports/execution-state-2026-09-07.md @@ -0,0 +1,68 @@ +# Execution-state fixes — 2026-09-07 + +The Windows ARM64 post-restore command hang is fixed in the tested cases. Linux ARM64 now has working full execution-state capture/restore with VGICv3. Windows x86-64 has a new implementation with successful cross-compilation and executable userspace tests, but no native x86 WHP live qualification. This report does not mark all of #8 complete. + +Backend revision: libkrun `51c1ed3b83dc826c02800fb297995538e4eac55d`. Firmware remains `6cca413ac248f63e65d4ea4748b3bc36cd1b22f3`, using matching ARM64 kernel and agentd builds. Microsandbox additionally captures device state before interrupt-controller state, and captures RAM afterward. Public CLI/SDK signatures and disk-only snapshot formats are unchanged. Old Windows ARM64 development full snapshots require recapture because the internal execution-state ABI now includes CPU activity and clock-frequency state. + +## What changed + +- Windows ARM64: capture and restore the architecture-specific internal activity register, including StartupSuspend. Previously, an online secondary CPU could remain startup-suspended after restore, leaving guest work waiting indefinitely. Preserve genuinely offline CPUs too; do not force all CPUs online. +- Windows ARM64 timers: freeze partition time at the all-vCPU pause barrier. After WHP activates partition time, reinstall saved timer compare/control values before releasing any vCPU. Installing them earlier allows WHP's time activation to rebase the deadline incorrectly. Preserve enable/mask state and validate counter frequency. +- Linux ARM64: enumerate complete variable-width KVM registers, retain MP and exception state, save VGICv3 private/global interrupt state, and retain virtual/physical counter origins and timer controls across pause. Restore VM-wide counter offsets once, not separately for each vCPU. Use the architectural host frequency rather than a nonexistent CNTFRQ GET_ONE_REG interface. Accommodate VGIC initialization ordering on Linux 6.12 without dropping the IIDR handshake. +- Windows x86-64: add register/MSR, XSAVE, local APIC, SynIC, userspace IOAPIC, partition capability/frequency and AP startup state. An already-running restored AP skips SIPI initialization so its saved instruction pointer is not overwritten. +- Shared controller: discard stale capture/restore replies when waiting for a later pause/resume acknowledgement. A failed capture on one CPU must not poison another CPU's source-resume response queue. +- Microsandbox: quiesce/capture device workers before capturing interrupt-controller state. Otherwise the saved queues and saved interrupt state can disagree about a late device completion. KVM's LPI pending-table flush must still precede RAM capture. + +## Live coverage + +All live VMs below used Alpine, two vCPUs, 256 MiB memory and a 512 MiB root. These are debug builds. Linux ARM64 ran inside Debian 13/Linux 6.12.107 on QEMU/HVF with EL2 exposed on the M5 Max Mac: `/dev/kvm` and nVHE initialization were verified, and Microsandbox used KVM, not software CPU emulation. This qualifies the exercised nested-KVM configuration, not every ARM host or kernel. + +| Check | Windows ARM64/WHP | Linux ARM64/KVM | +| --- | --- | --- | +| Running full capture and usable restore | Pass, standard memory | Pass, standard and CoW memory | +| Flat / layered root coverage | Flat clock/offline-CPU tests; layered full lifecycle; earlier flat full lifecycle | Flat standard full lifecycle; layered CoW full lifecycle | +| Idempotent pause/resume, status and paused exec rejection | Pass | Pass | +| Two captures while remaining user-paused | Pass | Pass | +| 15-second pause, boot identity and original workload progress | Pass | Pass | +| Two children, private RAM writes, source/child isolation | Pass | Pass | +| Direct full archive capture/restore and archive unlink survival | Pass | Pass | +| Stop paused source, child still usable | Pass | Pass | +| Six restores from the later paused snapshot; CPU0 and CPU1 pinned timer work on every child | Pass, 6/6 | Pass, 6/6 | +| CPU1 offline at capture, still offline after restore, then successfully onlined | Pass | Pass | +| Delayed restore: first-thaw wall time, elapsed clocks, timer expiration/cancellation | Pass | Pass, CoW | + +The clock fixture has four concurrently scheduled readers. Windows observed a 20,638.76 ms wall-time gap with only 46.29 ms monotonic/boottime advancement across the checkpoint boundary; its five-second elapsed-time timer fired once at 5,004.89 ms. Linux observed a 10,519.05 ms wall gap, 96.74 ms elapsed-clock advancement, and one timer expiration at 5,012.80 ms. Both reported no backward readings, expired the absolute wall timer once, canceled the cancel-on-clock-set timer, and passed the first-thaw wall-time bound. These are guest application observations, not just successful host API calls. + +macOS/HVF ARM64 and OVH Linux/KVM x86-64 also passed a fresh flat-root CoW lifecycle regression, including 15-second pause, full and repeated paused captures, independent children, direct archive restore and cleanup. This is a regression check of the shared ordering change, not a repeat of every earlier SDK/platform test. + +## Observed elapsed times + +Milliseconds, individual CLI wall-time observations. The hosts and memory modes differ; these are not controlled speedup comparisons, percentiles, or stop-the-world durations. The first diagnostic nested-KVM restore took 13,947 ms; subsequent lifecycle restores below were much shorter. Retain that cold/diagnostic outlier rather than claiming a stable distribution. + +| Operation | Windows ARM64 standard, layered | Linux ARM64 standard, flat | Linux ARM64 CoW, layered | +| --- | ---: | ---: | ---: | +| Running full capture | 7,542.96 | 1,084.90 | 1,238.35 | +| Resident pause | 73.82 | 26.57 | 34.11 | +| First / second paused capture | 2,988.71 / 2,754.49 | 360.17 / 340.20 | 506.82 / 315.73 | +| Resident resume | 81.48 | 71.15 | 87.20 | +| Restore child A / B | 2,647.74 / 3,083.40 | 781.67 / 798.37 | 337.50 / 359.42 | +| Direct full archive capture | 13,993.57 | 2,484.29 | 2,351.41 | +| Direct full archive restore | 4,019.68 | 1,727.62 | 1,090.74 | + +Six later-paused-snapshot restores took 2,906–3,296 ms on Windows ARM64 and 786.70–972.59 ms on Linux ARM64. Every child executed the marker read and both CPU-pinned probes successfully. The macOS regression's two CoW children restored in 975.91–1,007.92 ms; the OVH x86 Linux regression's children restored in 165.04–166.40 ms. + +## Reproduction and evidence + +Use `scripts/smoke/cli/cow-memory-lifecycle.py` as described in the preceding lifecycle report. The runner now waits for the detached application's marker before checking it, rather than assuming runtime readiness means the application's first write has happened. + +Build `checkpoint-clock-probe.rs` and `checkpoint-cpu-probe.rs` as static Linux musl binaries for the guest architecture. The portable host runners are `checkpoint-clock.py` and `checkpoint-cpu-state.py`. Their module docstrings specify the required environment variables. The CPU test deliberately offlines CPU1 before capture, checks the offline value after restore, and brings CPU1 online again before running the affinity/timer probe. The clock test runs `checkpoint-clock-analyze.py` against the captured application's observations. Every runner attempts source/child cleanup in `finally`. + +Local raw evidence: `/private/tmp/msb-execution-state-20260907/` (ARM64 Linux and Windows), `/private/tmp/mac-arch-fix-cow/` (macOS). OVH evidence: `/home/ubuntu/msb-stack8.ElfKzf/arch-fix-results/`. The nested Linux host retained `/root/stack8/` until shutdown; its test disk remains in `/private/tmp/msb-linux-arm64.lXCCOl/`. Snapshots/cache files remain isolated development artifacts; no test workload is intentionally left running. + +## Validation limits + +The native ARM64 VMM suite reported 67 passing tests, two ignored tests and two failures in existing MMIO test setup: `test_register_virtio_device` and `test_register_too_many_devices` call `create_irq_chip()` on ARM and fail with EINVAL. The new ARM execution-state tests passed (3), and focused VGIC tests passed (2). The ARM-only vCPU creation test's declaration-order error was fixed. Microsandbox's focused checkpoint suite passed 27 tests. macOS controller barrier tests passed, including the new stale-capture recovery case. + +Windows x86 compiled for `x86_64-pc-windows-msvc`. Executable tests on the Surface's x86 emulation passed for restored-running AP startup, pending SIPI, IOAPIC programming/serialization, and stale-capture response recovery. That does not exercise an x86 virtual processor, XSAVE/APIC restoration, or x86 timer delivery under WHP; a native x86 Windows machine is still required. + +Windows CoW memory remains explicitly unavailable at the protected shared-cache integration boundary; these fixes qualify standard-memory execution restore, not a Windows CoW cache implementation. Linux ARM64 execution snapshots require VGICv3: VGICv2 does not expose the same complete input-line/latch state interface, and its ordinary boot path remains unchanged. Host suspend, cross-host CPU compatibility, every interrupt injection race, all failure-injection cases and repeated release-build performance distributions remain outside this report. From 10ba613ca62f46341f2324a19d8a386bddd3d70b Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Tue, 8 Sep 2026 03:49:59 +0100 Subject: [PATCH 07/29] perf(snapshot): reduce restore staging and archive overhead Avoid repeated closure validation and disposable staging syncs while retaining payload admission, persistent disk durability and guest activation ordering. Serialize CoW cache misses without serializing warm hits. Preserve sparse archive members with long names, buffer archive extraction, and merge incremental memory extents with a consuming sweep. Resolve flat snapshot image metadata without materializing unused OCI disks and allow those sandboxes to restart without a VMDK. Add correctness regression coverage and record release-build macOS benchmarks, old/new archive reader checks, empty-cache restoration, concurrent CoW restores and child isolation tests. --- crates/image/lib/checkpoint/resolver.rs | 60 +++++-- crates/image/lib/registry/client.rs | 158 ++++++++++++++++-- crates/runtime/lib/checkpoint/coordinator.rs | 109 +++++++++--- crates/runtime/lib/checkpoint/memory_cache.rs | 52 +++++- crates/runtime/lib/control/executor.rs | 10 +- crates/runtime/lib/relay.rs | 5 +- docs/sandboxes/snapshots.mdx | 2 +- .../reports/restore-performance-2026-09-08.md | 82 +++++++++ sdk/rust/lib/backend/local/sandbox/create.rs | 46 ++++- sdk/rust/lib/backend/local/sandbox/mod.rs | 44 ++++- sdk/rust/lib/sandbox/builder.rs | 30 ++-- sdk/rust/lib/snapshot/archive.rs | 73 +++++++- sdk/rust/lib/snapshot/create.rs | 34 +++- sdk/rust/lib/snapshot/restore.rs | 17 +- 14 files changed, 610 insertions(+), 112 deletions(-) create mode 100644 scripts/smoke/reports/restore-performance-2026-09-08.md diff --git a/crates/image/lib/checkpoint/resolver.rs b/crates/image/lib/checkpoint/resolver.rs index 3958b91f5..d79b57fdf 100644 --- a/crates/image/lib/checkpoint/resolver.rs +++ b/crates/image/lib/checkpoint/resolver.rs @@ -45,6 +45,15 @@ pub struct CheckpointClosure { //-------------------------------------------------------------------------------------------------- impl CheckpointClosure { + /// Inspect the bounded, identity-verified root for construction planning, not payload admission. + /// The child closure must still be fully opened before its contents are consumed. + pub fn inspect_manifest( + root: &Path, + expected_root: Option<&ObjectId>, + ) -> ImageResult { + read_checkpoint_root(root, expected_root).map(|(_, manifest)| manifest) + } + /// Open and validate a checkpoint closure for restore on this host architecture. pub fn open(root: impl Into, expected_root: Option<&ObjectId>) -> ImageResult { Self::open_inner(root.into(), expected_root, true) @@ -66,22 +75,7 @@ impl CheckpointClosure { expected_root: Option<&ObjectId>, require_host_architecture: bool, ) -> ImageResult { - let metadata = std::fs::symlink_metadata(&root)?; - if !metadata.file_type().is_dir() { - return checkpoint_error("checkpoint root is not a directory"); - } - - let root_bytes = - read_regular_bounded(&root.join(CHECKPOINT_ROOT_FILE), MAX_MANIFEST_BYTES)?; - let root_id = ObjectId::from_bytes(&root_bytes)?; - if expected_root.is_some_and(|expected| expected != &root_id) { - return Err(ImageError::DigestMismatch { - digest: root_id.to_string(), - expected: expected_root.expect("checked Some").to_string(), - actual: root_id.to_string(), - }); - } - let checkpoint = CheckpointManifest::from_bytes(&root_bytes)?; + let (root_id, checkpoint) = read_checkpoint_root(&root, expected_root)?; if require_host_architecture && checkpoint.architecture != std::env::consts::ARCH { return checkpoint_error(format!( "checkpoint architecture {} cannot restore on {}", @@ -188,6 +182,25 @@ impl CheckpointClosure { // Functions //-------------------------------------------------------------------------------------------------- +fn read_checkpoint_root( + root: &Path, + expected_root: Option<&ObjectId>, +) -> ImageResult<(ObjectId, CheckpointManifest)> { + if !std::fs::symlink_metadata(root)?.file_type().is_dir() { + return checkpoint_error("checkpoint root is not a directory"); + } + let bytes = read_regular_bounded(&root.join(CHECKPOINT_ROOT_FILE), MAX_MANIFEST_BYTES)?; + let id = ObjectId::from_bytes(&bytes)?; + if let Some(expected) = expected_root.filter(|expected| *expected != &id) { + return Err(ImageError::DigestMismatch { + digest: id.to_string(), + expected: expected.to_string(), + actual: id.to_string(), + }); + } + Ok((id, CheckpointManifest::from_bytes(&bytes)?)) +} + fn validate_memory_objects(root: &Path, memory: &MemoryManifest) -> ImageResult<()> { let mut verified = BTreeSet::new(); for extent in &memory.extents { @@ -387,6 +400,21 @@ mod tests { (directory, root_id) } + #[test] + fn manifest_inspection_does_not_substitute_for_payload_admission() { + let (directory, root) = fixture(); + let manifest = CheckpointClosure::inspect_manifest(directory.path(), Some(&root)).unwrap(); + std::fs::remove_file(super::object_path( + directory.path(), + &manifest.execution_state, + )) + .unwrap(); + assert!(CheckpointClosure::inspect_manifest(directory.path(), Some(&root)).is_ok()); + assert!(CheckpointClosure::open(directory.path(), Some(&root)).is_err()); + let wrong = ObjectId::from_bytes(b"wrong root").unwrap(); + assert!(CheckpointClosure::inspect_manifest(directory.path(), Some(&wrong)).is_err()); + } + #[test] fn opens_complete_valid_closure() { let (directory, expected) = fixture(); diff --git a/crates/image/lib/registry/client.rs b/crates/image/lib/registry/client.rs index 133591530..41cc6e01f 100644 --- a/crates/image/lib/registry/client.rs +++ b/crates/image/lib/registry/client.rs @@ -167,12 +167,93 @@ impl Registry { manifest_digest: &Digest, ) -> ImageResult> { Ok( - resolve_cached_pull_result_by_manifest_digest_async(cache, manifest_digest) + resolve_cached_pull_result_by_manifest_digest_async(cache, manifest_digest, false) .await? .map(|cached| (cached.result, cached.metadata)), ) } + /// Resolve snapshot image defaults by immutable digest. Flat snapshots own + /// their complete disk, so they need metadata but no materialized OCI layers. + pub async fn pull_snapshot_cached( + cache: &GlobalCache, + references: &[oci_client::Reference], + manifest_digest: &Digest, + materialization: RootfsMaterialization, + ) -> ImageResult> { + let metadata_only = materialization == RootfsMaterialization::Flat; + let expected = manifest_digest.to_string(); + // Most snapshots retain either their original tag key or a pinned key. + // Avoid scanning every unrelated image on this common path. + for reference in references { + if let Some(metadata) = cache.read_image_metadata_async(reference).await? + && metadata.manifest_digest == expected + && let Some(cached) = + resolve_snapshot_metadata(cache, metadata, metadata_only).await? + { + return Ok(Some((cached.result, cached.metadata))); + } + } + Ok(resolve_cached_pull_result_by_manifest_digest_async( + cache, + manifest_digest, + metadata_only, + ) + .await? + .map(|cached| (cached.result, cached.metadata))) + } + + /// Fetch only the immutable manifest and config needed by a flat snapshot. + /// Normal pulls still independently require their filesystem artifacts. + pub async fn pull_snapshot_metadata( + &self, + reference: &oci_client::Reference, + ) -> ImageResult { + let expected = reference.digest().ok_or_else(|| { + ImageError::ManifestParse("snapshot metadata requires a digest-pinned reference".into()) + })?; + let (manifest_bytes, digest, config_bytes) = + self.fetch_manifest_and_config(reference).await?; + if digest != expected { + return Err(ImageError::ManifestParse( + "snapshot manifest digest differs from pinned reference".into(), + )); + } + let (manifest, config_bytes, resolved) = self + .parse_and_resolve_manifest(&manifest_bytes, config_bytes, reference) + .await?; + let (config, diff_ids) = ImageConfig::parse(&config_bytes)?; + let layers = self.extract_layer_digests(&manifest)?; + if layers.len() != diff_ids.len() { + return Err(ImageError::ManifestParse( + "snapshot manifest/config layer count mismatch".into(), + )); + } + let metadata = CachedImageMetadata { + manifest_digest: digest, + config_digest: manifest.config_digest().unwrap_or_default(), + raw_manifest_json: json_bytes_to_string(&resolved, "resolved manifest")?, + raw_config_json: json_bytes_to_string(&config_bytes, "image config")?, + config, + layers: layers + .iter() + .zip(diff_ids) + .map(|(layer, diff_id)| CachedLayerMetadata { + digest: layer.digest.to_string(), + media_type: layer.media_type.clone(), + size_bytes: layer.size, + diff_id, + }) + .collect(), + }; + let mut result = cached_pull_result(&metadata)?; + self.cache + .write_image_metadata_async(reference, &metadata) + .await?; + result.cached = false; + Ok(result) + } + /// Pull an image. Downloads blobs and materializes EROFS layers concurrently. pub async fn pull( &self, @@ -1652,9 +1733,9 @@ async fn resolve_cached_pull_result_async( async fn resolve_cached_pull_result_by_manifest_digest_async( cache: &GlobalCache, manifest_digest: &Digest, + metadata_only: bool, ) -> ImageResult> { let expected = manifest_digest.to_string(); - let platform = Platform::host_linux(); let mut entries = tokio::fs::read_dir(cache.manifests_dir()) .await .map_err(|e| ImageError::Cache { @@ -1683,14 +1764,7 @@ async fn resolve_cached_pull_result_by_manifest_digest_async( continue; } - if let Some(cached) = resolve_cached_metadata_pull_result_async( - cache, - metadata, - RootfsMaterialization::Layered, - &platform, - ) - .await? - { + if let Some(cached) = resolve_snapshot_metadata(cache, metadata, metadata_only).await? { return Ok(Some(cached)); } } @@ -1698,6 +1772,25 @@ async fn resolve_cached_pull_result_by_manifest_digest_async( Ok(None) } +async fn resolve_snapshot_metadata( + cache: &GlobalCache, + metadata: CachedImageMetadata, + metadata_only: bool, +) -> ImageResult> { + if metadata_only { + return Ok(cached_pull_result(&metadata) + .ok() + .map(|result| CachedPullInfo { result, metadata })); + } + resolve_cached_metadata_pull_result_async( + cache, + metadata, + RootfsMaterialization::Layered, + &Platform::host_linux(), + ) + .await +} + async fn resolve_cached_metadata_pull_result_async( cache: &GlobalCache, metadata: CachedImageMetadata, @@ -1961,6 +2054,51 @@ mod tests { assert_eq!(cached.1.manifest_digest, metadata.manifest_digest); } + #[tokio::test] + async fn snapshot_flat_metadata_does_not_require_disks_or_accept_moved_tags() { + let temp = tempdir().unwrap(); + let cache = GlobalCache::new(temp.path()).unwrap(); + let reference: oci_client::Reference = "docker.io/library/alpine:latest".parse().unwrap(); + let metadata = write_cached_image_fixture(&cache, &reference, &[false, false]); + let digest = parse_digest(&metadata.manifest_digest); + for refs in [vec![reference.clone()], vec![]] { + let found = super::Registry::pull_snapshot_cached( + &cache, + &refs, + &digest, + RootfsMaterialization::Flat, + ) + .await + .unwrap() + .unwrap(); + assert_eq!(found.0.manifest_digest, digest); + assert_eq!(found.0.config.env, metadata.config.env); + assert!( + super::Registry::pull_snapshot_cached( + &cache, + &refs, + &digest, + RootfsMaterialization::Layered + ) + .await + .unwrap() + .is_none() + ); + } + let other = parse_digest(&format!("sha256:{}", "f".repeat(64))); + assert!( + super::Registry::pull_snapshot_cached( + &cache, + &[reference], + &other, + RootfsMaterialization::Flat + ) + .await + .unwrap() + .is_none() + ); + } + #[tokio::test] async fn test_pull_cached_by_manifest_digest_requires_complete_artifacts() { let temp = tempdir().unwrap(); diff --git a/crates/runtime/lib/checkpoint/coordinator.rs b/crates/runtime/lib/checkpoint/coordinator.rs index 3ef84146f..3b13f0dad 100644 --- a/crates/runtime/lib/checkpoint/coordinator.rs +++ b/crates/runtime/lib/checkpoint/coordinator.rs @@ -1354,40 +1354,43 @@ fn overlay_extents( mut base: Vec, mut updates: Vec, ) -> Result, String> { + base.sort_by_key(|extent| extent.start); updates.sort_by_key(|extent| extent.start); + validate_non_overlapping(&base)?; validate_non_overlapping(&updates)?; + // Consume each old range once. A suffix split by an update remains at the + // front for the next update; object offsets are retained by slice_extent. + let mut pending = std::collections::VecDeque::from(base); + let mut output = Vec::with_capacity(pending.len() + updates.len()); for update in updates { - let update_end = update - .start - .checked_add(update.length) - .ok_or_else(|| "memory update overflows".to_string())?; - let mut next = Vec::with_capacity(base.len() + 1); - for extent in base { - let extent_end = extent - .start - .checked_add(extent.length) - .ok_or_else(|| "memory base extent overflows".to_string())?; - if extent_end <= update.start || extent.start >= update_end { - next.push(extent); + let update_end = update.start + update.length; + while let Some(extent) = pending.front() { + if extent.start >= update_end { + break; + } + let extent = pending.pop_front().expect("front was present"); + let extent_end = extent.start + extent.length; + if extent_end <= update.start { + output.push(extent); continue; } if extent.start < update.start { - next.push(slice_extent( + output.push(slice_extent( &extent, extent.start, update.start - extent.start, )); } if extent_end > update_end { - next.push(slice_extent(&extent, update_end, extent_end - update_end)); + pending.push_front(slice_extent(&extent, update_end, extent_end - update_end)); + break; } } - next.push(update); - next.sort_by_key(|extent| extent.start); - base = next; + output.push(update); } - validate_non_overlapping(&base)?; - Ok(coalesce_extents(base)) + output.extend(pending); + validate_non_overlapping(&output)?; + Ok(coalesce_extents(output)) } //-------------------------------------------------------------------------------------------------- @@ -1717,6 +1720,74 @@ mod tests { )); } + #[test] + fn incremental_merge_matches_byte_oracle_for_fragmented_ranges() { + let original = ObjectId::from_bytes(b"base").unwrap(); + let changed = ObjectId::from_bytes(b"update").unwrap(); + // Independent per-byte oracle includes holes, zero ranges, nonzero + // object offsets, unsorted input, and updates spanning multiple ranges. + let expand = |extents: &[MemoryExtent]| { + let mut bytes = vec![None; 256]; + for extent in extents { + for delta in 0..extent.length { + bytes[(extent.start + delta) as usize] = Some(match &extent.content { + MemoryExtentContent::Zero => (None, 0), + MemoryExtentContent::Object(content) => { + (Some(content.object.clone()), content.object_offset + delta) + } + }); + } + } + bytes + }; + let mut seed = 7u64; + for _ in 0..1000 { + let mut make = |object: &ObjectId| { + let mut ranges = Vec::new(); + let mut start = 0; + while start < 256 { + seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1); + let length = (1 + (seed >> 32) % 17).min(256 - start); + if seed % 5 != 0 { + ranges.push(MemoryExtent { + start, + length, + content: if seed % 3 == 0 { + MemoryExtentContent::Zero + } else { + MemoryExtentContent::Object(ContentRef { + object: object.clone(), + object_offset: 1024 + start, + }) + }, + }); + } + start += length; + } + ranges.reverse(); + ranges + }; + let base = make(&original); + let updates = make(&changed); + let mut expected = expand(&base); + for (slot, update) in expected.iter_mut().zip(expand(&updates)) { + if update.is_some() { + *slot = update; + } + } + assert_eq!(expand(&overlay_extents(base, updates).unwrap()), expected); + } + let zero = |start, length| MemoryExtent { + start, + length, + content: MemoryExtentContent::Zero, + }; + assert!(overlay_extents(vec![zero(0, 8), zero(4, 8)], vec![]).is_err()); + assert!(overlay_extents(vec![], vec![zero(u64::MAX, 2)]).is_err()); + assert!(overlay_extents(vec![], vec![zero(0, 0)]).is_err()); + assert!(overlay_extents(vec![], vec![zero(0, 8), zero(4, 8)]).is_err()); + } + #[test] fn managed_root_admits_only_runtime_owned_filesystems() { let bindings = runtime_owned_fs_bindings(true); diff --git a/crates/runtime/lib/checkpoint/memory_cache.rs b/crates/runtime/lib/checkpoint/memory_cache.rs index fefad72dd..6f0a910e4 100644 --- a/crates/runtime/lib/checkpoint/memory_cache.rs +++ b/crates/runtime/lib/checkpoint/memory_cache.rs @@ -130,6 +130,24 @@ impl MemoryCache { }); } + // Stable per-identity lock inodes serialize cache misses across processes, without + // placing warm hits or unrelated snapshots behind a global cache lock. Never unlink a + // build lock: waiters must not acquire different inodes for the same identity. + let build_lock = + microsandbox_utils::process_lock::open_lock_file(&path.with_extension("build-lock"))?; + microsandbox_utils::process_lock::lock_exclusive(&build_lock)?; + if let Some(file) = open_pinned(&path, length)? { + return Ok(CachedMemory { + path, + identity: identity.clone(), + file, + regions, + cache_hit: true, + reflink: false, + prepare_us: started.elapsed().as_micros(), + }); + } + let staging_dir = tempfile::Builder::new() .prefix(".memory-") .tempdir_in(&self.root)?; @@ -221,8 +239,8 @@ impl MemoryCache { staging.set_permissions(std::fs::Permissions::from_mode(0o400))?; } staging.sync_all()?; - // Publish the inode without replacement. Concurrent builders may do duplicate work, but - // no winner can overwrite backing another VM has already pinned or mapped. + // Keep no-replacement publication even under the build lock: older builders may not + // participate in single-flight, and eviction must never replace a live mapped inode. match std::fs::hard_link(&staging_path, &path) { Ok(()) => {} Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} @@ -536,7 +554,7 @@ mod tests { )) .is_err() ); - assert_eq!(std::fs::read_dir(directory.path()).unwrap().count(), 1); + assert_eq!(payload_count(directory.path()), 1); } #[test] @@ -548,9 +566,9 @@ mod tests { Err(io::Error::other("injected object read failure")) }); assert!(failure.is_err()); - assert_eq!(std::fs::read_dir(directory.path()).unwrap().count(), 0); + assert_eq!(payload_count(directory.path()), 0); assert!(cache.materialize(&manifest, &id, |_| Ok(vec![])).is_err()); - assert_eq!(std::fs::read_dir(directory.path()).unwrap().count(), 0); + assert_eq!(payload_count(directory.path()), 0); } #[test] @@ -571,6 +589,23 @@ mod tests { assert!(memory_regions(&manifest, 16384).is_err()); } + fn payload_count(root: &Path) -> usize { + // Build-lock inodes intentionally survive failed builders. Only RAM or staging entries + // count as payloads; removing lock files would permit two independent flock owners. + std::fs::read_dir(root) + .unwrap() + .filter(|entry| { + entry + .as_ref() + .unwrap() + .path() + .extension() + .and_then(|ext| ext.to_str()) + != Some("build-lock") + }) + .count() + } + #[test] fn concurrent_builders_publish_one_immutable_inode() { use std::os::unix::fs::MetadataExt; @@ -578,11 +613,13 @@ mod tests { let cache = MemoryCache::open(directory.path()).unwrap(); let (manifest, id, bytes) = fixture(cache.page_size); let barrier = std::sync::Barrier::new(2); + let reads = std::sync::atomic::AtomicUsize::new(0); std::thread::scope(|scope| { let run = || { + barrier.wait(); cache .materialize(&manifest, &id, |_| { - barrier.wait(); + reads.fetch_add(1, std::sync::atomic::Ordering::Relaxed); Ok(bytes.clone()) }) .unwrap() @@ -596,7 +633,8 @@ mod tests { second.file.metadata().unwrap().ino() ); }); - assert_eq!(std::fs::read_dir(directory.path()).unwrap().count(), 1); + assert_eq!(reads.load(std::sync::atomic::Ordering::Relaxed), 1); + assert_eq!(std::fs::read_dir(directory.path()).unwrap().count(), 2); } #[test] diff --git a/crates/runtime/lib/control/executor.rs b/crates/runtime/lib/control/executor.rs index d89329b99..52c876bab 100644 --- a/crates/runtime/lib/control/executor.rs +++ b/crates/runtime/lib/control/executor.rs @@ -6,9 +6,6 @@ use std::io::Write; use std::path::{Path, PathBuf}; use std::sync::Mutex; -#[cfg(unix)] -use std::fs::File; - use rand::Rng as _; use serde::{Deserialize, Serialize}; use sha2::{Digest as _, Sha256}; @@ -122,7 +119,7 @@ struct DedupEntry { //-------------------------------------------------------------------------------------------------- impl RuntimeControlExecutor { - /// Construct an executor and durably publish a fresh runtime boot identity. + /// Construct an executor and atomically publish a fresh runtime boot identity. pub fn new( vm: msb_krun::VmControl, #[cfg(feature = "net")] secrets: Option< @@ -625,11 +622,10 @@ fn persist_runtime_boot_id(runtime_dir: &Path, boot_id: &str) -> std::io::Result .open(&temporary)?; file.write_all(boot_id.as_bytes())?; file.write_all(b"\n")?; - file.sync_all()?; + // This file is diagnostic/live discovery, not a restart journal. Fencing uses the new + // in-memory identity on every boot; atomic visibility is sufficient here. drop(file); crate::checkpoint::replace_file(&temporary, &target)?; - #[cfg(unix)] - File::open(runtime_dir)?.sync_all()?; Ok(()) } diff --git a/crates/runtime/lib/relay.rs b/crates/runtime/lib/relay.rs index 199a2f216..7625a49c3 100644 --- a/crates/runtime/lib/relay.rs +++ b/crates/runtime/lib/relay.rs @@ -868,11 +868,10 @@ fn persist_restore_activation( ) .map_err(|error| RuntimeError::Custom(format!("encode restore activation: {error}")))?; file.write_all(b"\n")?; - file.sync_all()?; + // Activation ordering is enforced by the live VM barrier. This diagnostic record has no + // recovery reader; avoid making guest readiness wait for storage durability. drop(file); crate::checkpoint::replace_file(&temporary, &target)?; - #[cfg(unix)] - std::fs::File::open(runtime_dir)?.sync_all()?; Ok(()) } diff --git a/docs/sandboxes/snapshots.mdx b/docs/sandboxes/snapshots.mdx index 2427109a7..91f79565a 100644 --- a/docs/sandboxes/snapshots.mdx +++ b/docs/sandboxes/snapshots.mdx @@ -249,7 +249,7 @@ msb run --name worker --from-snapshot ./after-pip-install.msnap -- python -V The same archive path works with the SDK restore methods shown below. The archive is unpacked into child-owned staging, so no intermediate installed snapshot is loaded into `~/.microsandbox/snapshots`. -Direct capture records the pinned image identity, but does not bundle the cached OCI image. The target must already have that image or be able to pull it. For an offline target, create an installed snapshot and use `msb snapshot save --with-image` instead. +Direct capture records the pinned image identity, but does not bundle the cached OCI image. A flat snapshot already contains the complete disk, so restore only needs the pinned image's configuration from cache or the registry; it does not download the original disk layers again. Layered snapshots still need their base image artifacts. For an offline target, create an installed snapshot and use `msb snapshot save --with-image` instead. ## Boot from a snapshot diff --git a/scripts/smoke/reports/restore-performance-2026-09-08.md b/scripts/smoke/reports/restore-performance-2026-09-08.md new file mode 100644 index 000000000..882069279 --- /dev/null +++ b/scripts/smoke/reports/restore-performance-2026-09-08.md @@ -0,0 +1,82 @@ +# Restore optimization qualification — 2026-09-08 + +The optimization batch is implemented and live-tested on macOS/HVF ARM64. This report does not declare the entire #8 branch or every proposed performance optimization complete. Linux and Windows were not rerun in this pass. + +## Changes qualified + +- Preserve GNU sparse encoding for long checkpoint layer names, with the exact GNU long-name marker already accepted by the previous reader. Increase archive input and extraction buffers to 1 MiB while retaining bounded reads, transport hashing, sparse-map validation, and truncation rejection. +- Merge ordered incremental memory ranges with a consuming sweep rather than rescanning and sorting the entire evolving map for each update. Preserve zero ranges, object identity, slice offsets, and overlap/overflow rejection. +- Resolve flat snapshot image configuration without downloading or materializing unused OCI disks. Probe pinned/original cache keys before the existing digest scan. Keep digest pinning, registry authentication/TLS settings, `--pull never` refusal when metadata is absent, and full artifact requirements for layered roots. +- Make the stopped-sandbox startup check respect flat root disks. Live testing caught its unconditional VMDK requirement when the new metadata-only cache contained no VMDK; the fix was rebuilt and requalified. +- Avoid directory syncs for disposable child checkpoint staging, but keep durable installed-snapshot publication, persistent disk publication, and immutable RAM-cache synchronization. Avoid syncing diagnostic boot/activation records; guest activation/clock acknowledgement/thaw ordering is unchanged. +- Inspect only the bounded, identity-verified checkpoint root during builder planning, and remove the redundant pre-copy source validation. Child/runtime payload admission still runs before consumption. +- Serialize same-identity CoW cache misses with a process lock and recheck after acquiring it. Warm hits remain outside the build lock. Keep no-replacement inode publication for interoperability with earlier builders and pinned VMs. + +No snapshot schema, agent protocol, dependency pin, CLI flag, or public language-SDK option changed. + +## Environment and reproducibility + +Apple M5 Max, 36 GiB RAM, macOS 26.3, APFS, native ARM64/HVF. Guest: Alpine pinned at `sha256:e7a1a92a5bfeee40966aea60f0796b0e7917cc35591542701834f03a68fa3d18`, 256 MiB RAM, two vCPUs, flat 512 MiB root disk, 32 MiB random tmpfs payload, disk/RAM markers, and a shell workload. This is a development workstation, not an otherwise idle dedicated performance host. + +Source base: Microsandbox `15ab8838da58061df5dd9560cbd85d648bb72351` plus this optimization commit. Libkrun remains pinned to `51c1ed3b83dc826c02800fb297995538e4eac55d`. Matching guest agent: `/private/tmp/msb-stack8-agentd-arm`; firmware: `/private/tmp/msb-stack8-libkrunfw.5.dylib`. The final signed executable SHA-256 is `32530a3f4385a090fb385d8a5015429638a43b16d990353818c0ecba219c3d9f`. + +Build: + +```sh +MSB_AGENTD_PATH=/private/tmp/msb-stack8-agentd-arm cargo build --release --locked --no-default-features --features net,ssh,prebuilt -p microsandbox-cli +codesign --entitlements msb-entitlements.plist --force -s - /private/tmp/msb-resume-final.P83RaS/msb +``` + +The build executable was copied to the final test directory before signing. Final evidence and executable: `/private/tmp/msb-resume-final.P83RaS/`. `bench.py`, `qualify.py`, and `fanout.py` retain the exact test procedures; `results.json`, `summary.json`, `qualification.json`, `qualification-summary.json`, and `concurrent.json` contain individual times and exit codes. Tests use isolated homes, bounded subprocess timeouts, and cleanup in `finally`. Use fresh directories/names when repeating them. The earlier optimization run and corruption test are retained in `/private/tmp/msb-resume-opt.locaTL/`; the pre-optimization benchmark and actual older executable are in `/private/tmp/msb-resume-profile.osiuRj/`. + +## Performance + +These times measure process launch through successful CLI completion, not just RAM mapping, stop-the-world time, or application readiness. “First command” measures from that same initial launch until a subsequent guest command completes, including a second CLI process. RAM hash checks occur afterward. Warm CoW means the prepared backing exists; cache miss means that file was absent, not that the OS page cache was purged. No p95 claim is made from these sample counts. + +The strongest A/B comparison alternates the actual pre-optimization and final release binaries against the same installed snapshot and home, six samples per binary/mode: + +| Installed full restore | Before median | After median | Reduction | After range | +| --- | ---: | ---: | ---: | ---: | +| Standard RAM | 278.15 ms | 153.56 ms | 44.8% | 145.83–159.68 ms | +| Warm CoW | 229.64 ms | 115.18 ms | 49.8% | 112.22–116.87 ms | + +The complete final-binary matrix below compares with the previous measurement pass. Unlike the alternating A/B above, those passes used separate captures and ran at different times. Improvements describe the combined batch, not an isolated contribution from each optimization. + +| Path | n | Previous CLI median | Final CLI median | Final first-command median | +| --- | ---: | ---: | ---: | ---: | +| Resident resume, standard | 8 | 12.97 ms | 11.40 ms | 25.81 ms | +| Resident resume, CoW | 8 | 12.91 ms | 10.57 ms | 24.04 ms | +| Fresh boot, standard | 5 | 180.48 ms | 158.89 ms | 194.53 ms | +| Fresh boot, CoW configured | 5 | 179.43 ms | 166.89 ms | 219.02 ms | +| Stopped disk snapshot | 5 | 206.80 ms | 190.45 ms | 226.67 ms | +| Full snapshot, disk-only restore | 5 | 209.38 ms | 181.63 ms | 218.00 ms | +| Installed full, standard | 8 | 278.76 ms | 152.31 ms | 219.32 ms | +| Installed full, warm CoW | 8 | 228.47 ms | 111.66 ms | 176.29 ms | +| Installed full, CoW cache miss | 5 | 301.18 ms | 162.85 ms | 232.38 ms | +| Full archive, standard | 5 | 405.12 ms | 236.38 ms | 287.45 ms | +| Full archive, warm CoW | 5 | 347.50 ms | 200.94 ms | 256.26 ms | + +The final full archive was 54,767,005 bytes. A genuine incremental checkpoint produced by repeated capture while paused restored in median 165.20 ms with standard RAM and 120.20 ms with CoW (five samples each). That paused incremental case has no intentionally dirtied RAM between captures; it verifies the incremental path but does not measure heavily fragmented dirty-memory merge throughput. A separate post-workload capture fell back to full and is correctly recorded as `delta-*` with `capture_mode=full`, not presented as incremental performance. + +Single observations, not distributions: full capture 493.35 ms; subsequent full-fallback capture 466.94 ms; direct full archive capture 442.86 ms; stopped disk capture 37.38 ms. In the qualification run, a full baseline capture took 458.52 ms and the following genuinely incremental paused capture took 247.57 ms. These are caller-observed operation durations, not vCPU pause durations. + +Three simultaneous cold-CoW archive restores completed in 278.70–286.37 ms each and published one prepared RAM file. This is one fanout experiment, not a concurrency percentile. An empty image-cache archive restore took 3,138.93 ms including registry network access; only one metadata JSON file was cached, with no OCI disk artifacts. A subsequent offline `--pull never` CoW restore succeeded. Avoid interpreting network-bound cold metadata fetch as warm restore latency. + +## Correctness and compatibility evidence + +- 92 focused tests passed: 27 runtime checkpoint tests, 28 registry tests, 31 SDK snapshot tests, five checkpoint resolver tests, and the new flat-restart regression test. +- The memory merge was compared against an independent per-byte oracle over 1,000 generated fragmented cases, including holes, zero updates, nonzero object offsets, unsorted input, and updates spanning multiple old extents. Malformed overlap, empty, and overflowing ranges are rejected. +- Sparse long-name output is checked with a separate tar parser and encoded-size assertion, then loaded and restored. The actual pre-optimization #8 release binary successfully restored the new plain-tar and zstd archives without reader changes. The final binary restored a real pre-optimization archive and verified its tmpfs payload hash. macOS system tar listed both new archive encodings successfully. This is not qualification against every historical release. +- The live matrix checked full-state RAM SHA-256, retained workload markers, cold disk-only semantics, standard/CoW modes, installed/direct-archive inputs, prepared-cache hits/misses, resident resume, unchanged boot ID across resident resume, and cleanup. +- Missing metadata with `--pull never` refused cleanly; metadata-only network fetch succeeded; reuse with `--pull never` succeeded. Unit tests retain layered artifact requirements and reject moved-tag digest mismatches. +- Two children retained independent RAM/disk contents after deleting their input archive. Stopping/restarting the modified child preserved its private disk marker without OCI layers or VMDK. An isolated snapshot copy with corrupted execution-state object bytes was refused before successful VM creation. +- Three concurrent restores shared a cold cache entry, retained their markers, and isolated private RAM writes. A truncated zstd archive was refused with `zstd stream did not finish`. +- A process inventory after the runs found no runtimes belonging to either optimization test directory. Two unrelated pre-existing development VMs were left untouched. Test artifacts were retained for inspection. + +The first live cold-cache attempt used an overlong Unix socket path and was rerun with a shorter home. The later restart check exposed and fixed the real unconditional-VMDK bug described above. Both are retained in the earlier run's evidence; neither is counted as a passing initial attempt. + +`cargo fmt --all -- --check` and `git diff --check` passed. Strict Clippy was blocked by the existing `derivable_impls` warning in `crates/image/lib/snapshot/manifest.rs`; allowing that lint exposed the existing `too_many_arguments` warning on `RuntimeControlExecutor::new`. No unrelated lint cleanup was included. This pass did not rerun every workspace or language-SDK test. + +## Remaining performance work + +Warm full CoW branching is roughly twice as fast, but it is not consistently below 90 ms. The broader proposals for startup readiness notification, pipelined eager object reads, restore-only zero-preserving RAM construction, and avoiding immutable qcow2 ancestor relocation copies are not implemented by this batch. The remaining first-command latency also deserves separate measurement; CLI completion is not a substitute for measuring a ready application or a persistent SDK connection. Large-memory, deep-chain, heavily fragmented dirty-memory throughput, cache-builder process-death, and cross-platform performance qualification remain outside this pass. diff --git a/sdk/rust/lib/backend/local/sandbox/create.rs b/sdk/rust/lib/backend/local/sandbox/create.rs index 9d076b629..b77541243 100644 --- a/sdk/rust/lib/backend/local/sandbox/create.rs +++ b/sdk/rust/lib/backend/local/sandbox/create.rs @@ -208,7 +208,7 @@ impl LocalBackend { crate::sandbox::apply_checkpoint_restore_constraints( &mut config, state, - &closure, + closure.checkpoint(), overrides, )?; config.checkpoint_restore = Some(restore); @@ -914,6 +914,7 @@ impl LocalBackend { pinned_digest, pull_policy, registry_overrides, + materialization, progress, ) .await @@ -926,6 +927,7 @@ impl LocalBackend { pinned_digest: &str, pull_policy: PullPolicy, registry_overrides: RegistryOverrides, + materialization: microsandbox_image::RootfsMaterialization, progress: Option, ) -> MicrosandboxResult { let manifest_digest: Digest = pinned_digest.parse().map_err(|e| { @@ -936,8 +938,19 @@ impl LocalBackend { let pinned_reference = Self::digest_pinned_reference(reference, pinned_digest)?; let cache = GlobalCache::new_async(&self.cache_dir()).await?; - if let Some((pull_result, metadata)) = - Registry::pull_cached_by_manifest_digest(&cache, &manifest_digest).await? + let pinned_ref: Reference = pinned_reference.parse().map_err(|e| { + crate::MicrosandboxError::InvalidConfig(format!("invalid pinned reference: {e}")) + })?; + let original_ref: Reference = reference.parse().map_err(|e| { + crate::MicrosandboxError::InvalidConfig(format!("invalid image reference: {e}")) + })?; + if let Some((pull_result, metadata)) = Registry::pull_snapshot_cached( + &cache, + &[pinned_ref.clone(), original_ref], + &manifest_digest, + materialization, + ) + .await? { Self::emit_cached_pull_progress(progress.as_ref(), reference, &metadata); return Ok(ResolvedOciImage { @@ -954,6 +967,33 @@ impl LocalBackend { ))); } + if materialization == microsandbox_image::RootfsMaterialization::Flat { + // The snapshot supplies the complete disk. Fetch its pinned image + // defaults, not a second root filesystem that will never be used. + let global = self.config(); + let auth = match registry_overrides.auth { + Some(auth) => auth, + None => global.resolve_registry_auth(pinned_ref.registry())?, + }; + let mut ca_certs = global.resolve_ca_certs().await?; + ca_certs.extend(registry_overrides.ca_certs); + let mut insecure = global.insecure_registries(); + if registry_overrides.insecure { + insecure.push(pinned_ref.registry().to_string()); + } + let registry = Registry::builder(microsandbox_image::Platform::host_linux(), cache) + .auth(auth) + .extra_ca_certs(ca_certs) + .add_insecure_registries(insecure) + .build()?; + let pull_result = registry.pull_snapshot_metadata(&pinned_ref).await?; + return Ok(ResolvedOciImage { + pull_result, + metadata_reference: pinned_reference, + cached_metadata: None, + }); + } + // Pull by digest, never by the mutable source tag, when the exact // snapshot base is absent from the local cache. let pull_result = match self diff --git a/sdk/rust/lib/backend/local/sandbox/mod.rs b/sdk/rust/lib/backend/local/sandbox/mod.rs index eb65289df..da7994358 100644 --- a/sdk/rust/lib/backend/local/sandbox/mod.rs +++ b/sdk/rust/lib/backend/local/sandbox/mod.rs @@ -405,7 +405,13 @@ impl LocalBackend { ))); } - if let RootfsSource::Oci(_) = &config.spec.image + // Flat roots own their disk and never boot through the OCI VMDK. + // Metadata-only snapshot restores deliberately do not populate it. + if let RootfsSource::Oci(oci) = &config.spec.image + && !matches!( + oci.root_disk.as_ref(), + Some(crate::sandbox::RootDisk::Flat { .. }) + ) && let Some(ref digest_str) = config.manifest_digest { let cache_dir = self.cache_dir(); @@ -1498,6 +1504,42 @@ mod tests { let _ = backend.validate_start_state(&config, &sandbox_dir); } + #[tokio::test] + async fn flat_restart_does_not_require_layered_image_artifacts() { + let temp = tempdir().unwrap(); + let backend = LocalBackend::builder() + .home(temp.path()) + .build() + .await + .unwrap(); + let sandbox_dir = temp.path().join("persisted"); + fs::create_dir(&sandbox_dir).unwrap(); + let mut config = test_config_with_rootfs( + "persisted", + RootfsSource::Oci(OciRootfsSource { + reference: "alpine".into(), + root_disk: Some(crate::sandbox::RootDisk::Flat { + size_mib: Some(512), + fstype: None, + clone: microsandbox_types::FlatClone::Auto, + }), + }), + ); + config.manifest_digest = Some(format!("sha256:{}", "a".repeat(64))); + backend.validate_start_state(&config, &sandbox_dir).unwrap(); + let RootfsSource::Oci(oci) = &mut config.spec.image else { + unreachable!() + }; + oci.root_disk = None; + assert!( + backend + .validate_start_state(&config, &sandbox_dir) + .unwrap_err() + .to_string() + .contains("VMDK missing") + ); + } + /// Simulates the reaper sweep: queries all Running/Draining sandboxes and /// reconciles each. Verifies that only stale entries are reaped while /// live, stopped, crashed, and starting (no run record) sandboxes are diff --git a/sdk/rust/lib/sandbox/builder.rs b/sdk/rust/lib/sandbox/builder.rs index 880360886..6fb8f4768 100644 --- a/sdk/rust/lib/sandbox/builder.rs +++ b/sdk/rust/lib/sandbox/builder.rs @@ -1279,27 +1279,22 @@ impl SandboxBuilder { ) .map_err(|error| crate::MicrosandboxError::SnapshotIntegrity(error.to_string()))?; let closure = snap.path().join(crate::snapshot::CHECKPOINT_DIRECTORY); - let opened = match self.config.snapshot_restore_mode { - SnapshotRestoreMode::Full => { - microsandbox_image::checkpoint::CheckpointClosure::open( - &closure, - Some(&expected), - ) - } - SnapshotRestoreMode::DiskOnly => { - microsandbox_image::checkpoint::CheckpointClosure::open_portable( - &closure, - Some(&expected), - ) - } - } + let opened = microsandbox_image::checkpoint::CheckpointClosure::inspect_manifest( + &closure, + Some(&expected), + ) .map_err(|error| crate::MicrosandboxError::SnapshotIntegrity(error.to_string()))?; - if opened.checkpoint().checkpoint_id != state.checkpoint_id { + if opened.checkpoint_id != state.checkpoint_id { return Err(crate::MicrosandboxError::SnapshotIntegrity( "snapshot and checkpoint closure identities differ".into(), )); } if self.config.snapshot_restore_mode == SnapshotRestoreMode::Full { + if opened.architecture != std::env::consts::ARCH { + return Err(crate::MicrosandboxError::SnapshotIntegrity( + "checkpoint architecture cannot restore on this host".into(), + )); + } let restore_overrides = self.restore_override_intent(); apply_checkpoint_restore_constraints( &mut self.config, @@ -1848,13 +1843,12 @@ fn validate_config_script_name(name: &str) -> Result<(), String> { pub(crate) fn apply_checkpoint_restore_constraints( config: &mut SandboxConfig, state: &crate::snapshot::CheckpointSnapshotState, - closure: µsandbox_image::checkpoint::CheckpointClosure, + checkpoint: µsandbox_image::checkpoint::CheckpointManifest, overrides: RestoreOverrideIntent, ) -> MicrosandboxResult<()> { apply_checkpoint_resources(config, state, overrides)?; - let mut resources = closure - .checkpoint() + let mut resources = checkpoint .resources .iter() .filter(|resource| resource.kind == "network"); diff --git a/sdk/rust/lib/snapshot/archive.rs b/sdk/rust/lib/snapshot/archive.rs index be3d84522..a3278cf8c 100644 --- a/sdk/rust/lib/snapshot/archive.rs +++ b/sdk/rust/lib/snapshot/archive.rs @@ -942,7 +942,7 @@ pub(super) async fn load_snapshot_with_base( // Stream rather than slurp — archives carry the full upper layer and are // routinely multi-GB. let file = tokio::fs::File::open(archive).await?; - let mut buf = BufReader::new(file); + let mut buf = BufReader::with_capacity(1024 * 1024, file); let is_zstd = { let bytes = buf.fill_buf().await?; bytes.starts_with(&[0x28, 0xb5, 0x2f, 0xfd]) @@ -1115,7 +1115,7 @@ pub(crate) async fn materialize_archive_for_child_with_base( .tempdir_in(&cache_tmp_dir)?; let file = tokio::fs::File::open(archive).await?; - let mut buffered = BufReader::new(file); + let mut buffered = BufReader::with_capacity(1024 * 1024, file); let is_zstd = buffered .fill_buf() .await? @@ -1933,8 +1933,23 @@ where let mut header = Header::new_gnu(); header.set_metadata_in_mode(&meta, HeaderMode::Complete); if header.set_path(name).is_err() { - // Needs a GNU long-name entry; the dense path emits one. - return Ok(None); + // GNU long-name records apply to sparse members too. Canonical qcow2 + // checkpoint paths exceed the fixed name field by one byte. + let mut long = Header::new_gnu(); + // set_path normalizes away the leading dots; use the exact GNU + // marker emitted by the existing dense writer and accepted by readers. + long.as_gnu_mut().expect("GNU header").name[..13].copy_from_slice(b"././@LongLink"); + long.set_entry_type(EntryType::GNULongName); + long.set_mode(0o644); + long.set_size(name.len() as u64 + 1); + long.set_cksum(); + let dst = builder.get_mut(); + dst.write_all(long.as_bytes()).await?; + dst.write_all(name.as_bytes()).await?; + dst.write_all(&[0]).await?; + let padding = tar_pad(name.len() as u64 + 1) as usize; + dst.write_all(&[0u8; TAR_BLOCK as usize][..padding]).await?; + header.set_path("sparse-member")?; } header.set_entry_type(EntryType::GNUSparse); header.set_size(map.archived); @@ -2448,6 +2463,29 @@ fn tar_pad(size: u64) -> u64 { (TAR_BLOCK - size % TAR_BLOCK) % TAR_BLOCK } +/// Amortize async filesystem dispatch while preserving the caller's bounded +/// reader and transport hashing. Reuse the same buffer across sparse extents. +async fn copy_archive_payload( + reader: &mut R, + writer: &mut W, + buffer: &mut [u8], +) -> std::io::Result +where + R: tokio::io::AsyncRead + Unpin, + W: tokio::io::AsyncWrite + Unpin, +{ + use tokio::io::AsyncWriteExt; + let mut copied = 0; + loop { + let read = reader.read(buffer).await?; + if read == 0 { + return Ok(copied); + } + writer.write_all(&buffer[..read]).await?; + copied += read as u64; + } +} + /// Stream a dense entry's bytes into `target`. async fn unpack_dense_entry( reader: &mut R, @@ -2467,7 +2505,8 @@ where hasher: archive_transport_hasher(kind, archive_path, size, size, &[]), bytes_read: 0, }; - let copied = tokio::io::copy(&mut source, &mut file).await?; + let mut buffer = vec![0u8; 1024 * 1024]; + let copied = copy_archive_payload(&mut source, &mut file, &mut buffer).await?; if copied != size { return Err(MicrosandboxError::Custom( "archive truncated mid-entry".into(), @@ -2587,6 +2626,7 @@ where std_file.set_len(realsize)?; let mut file = tokio::fs::File::from_std(std_file); let mut transport = archive_transport_hasher(kind, archive_path, archived, realsize, &map); + let mut buffer = vec![0u8; 1024 * 1024]; for (offset, numbytes) in &map { if *numbytes == 0 { @@ -2598,7 +2638,7 @@ where hasher: transport, bytes_read: 0, }; - let copied = tokio::io::copy(&mut source, &mut file).await?; + let copied = copy_archive_payload(&mut source, &mut file, &mut buffer).await?; transport = source.hasher; if copied != *numbytes { return Err(MicrosandboxError::Custom( @@ -4341,14 +4381,14 @@ mod tests { drop(layer_file); // The canonical checkpoint qcow member is one byte too long for the - // fixed GNU header path field. It therefore exercises dense long-name - // fallback even though the source itself has a sparse extent map. + // fixed GNU header path field. It must retain sparse encoding even + // when a GNU long-name record precedes the sparse header. let archive_layer_path = format!("checkpoints/snap_00000000000000000000000000000002/layers/{layer_id}.qcow2"); assert_eq!(archive_layer_path.len(), 101); assert!( archive_encoded_size(&source_layer).await.unwrap() < 4 * 1024 * 1024, - "test source must remain sparse so dense fallback changes the encoded size" + "test source must remain sparse" ); let layer_integrity = sparse_file_integrity(&source_layer).unwrap(); let disk = DiskGenerationManifest { @@ -4430,6 +4470,21 @@ mod tests { ) .await .unwrap(); + // Inspect the actual transport, not just same-reader roundtrip results. + let compressed = tokio::fs::File::open(&archive).await.unwrap(); + let decoder = ZstdDecoder::new(tokio::io::BufReader::new(compressed)); + let mut tar = tokio_tar::Archive::new(decoder); + let mut entries = tar.entries().unwrap(); + let mut found_sparse = false; + while let Some(entry) = futures::StreamExt::next(&mut entries).await { + let entry = entry.unwrap(); + if entry.path().unwrap() == Path::new(&archive_layer_path) { + assert!(entry.header().entry_type().is_gnu_sparse()); + assert!(entry.header().entry_size().unwrap() < 4 * 1024 * 1024); + found_sparse = true; + } + } + assert!(found_sparse); std::fs::remove_dir_all(&source).unwrap(); let restored = materialize_archive_for_child(&local, &archive, &child_stage, false) .await diff --git a/sdk/rust/lib/snapshot/create.rs b/sdk/rust/lib/snapshot/create.rs index d1bc8cacf..0aaf9fd5f 100644 --- a/sdk/rust/lib/snapshot/create.rs +++ b/sdk/rust/lib/snapshot/create.rs @@ -1011,6 +1011,20 @@ fn validate_snapshot_name(name: &str) -> MicrosandboxResult<()> { pub(crate) fn materialize_checkpoint_closure( source: &Path, destination: &Path, +) -> std::io::Result<()> { + materialize_checkpoint_tree(source, destination, true) +} + +/// Construction-only closure: retain independent links, but do not make disposable staging +/// durable. Persistent disk successors are published separately before guest activation. +pub(crate) fn stage_checkpoint_closure(source: &Path, destination: &Path) -> std::io::Result<()> { + materialize_checkpoint_tree(source, destination, false) +} + +fn materialize_checkpoint_tree( + source: &Path, + destination: &Path, + durable: bool, ) -> std::io::Result<()> { let source_metadata = std::fs::symlink_metadata(source)?; if !source_metadata.file_type().is_dir() { @@ -1025,7 +1039,7 @@ pub(crate) fn materialize_checkpoint_closure( let source_member = source.join(member); match std::fs::symlink_metadata(&source_member) { Ok(metadata) if metadata.file_type().is_dir() => { - copy_checkpoint_directory(&source_member, &destination.join(member))?; + copy_checkpoint_directory(&source_member, &destination.join(member), durable)?; } Ok(_) => { return Err(std::io::Error::new( @@ -1042,10 +1056,17 @@ pub(crate) fn materialize_checkpoint_closure( &source.join("checkpoint.json"), &destination.join("checkpoint.json"), )?; - sync_directory(destination) + if durable { + sync_directory(destination)?; + } + Ok(()) } -fn copy_checkpoint_directory(source: &Path, destination: &Path) -> std::io::Result<()> { +fn copy_checkpoint_directory( + source: &Path, + destination: &Path, + durable: bool, +) -> std::io::Result<()> { std::fs::create_dir(destination)?; for entry in std::fs::read_dir(source)? { let entry = entry?; @@ -1053,7 +1074,7 @@ fn copy_checkpoint_directory(source: &Path, destination: &Path) -> std::io::Resu let destination_path = destination.join(entry.file_name()); let metadata = std::fs::symlink_metadata(&source_path)?; if metadata.file_type().is_dir() { - copy_checkpoint_directory(&source_path, &destination_path)?; + copy_checkpoint_directory(&source_path, &destination_path, durable)?; } else if metadata.file_type().is_file() { copy_checkpoint_file(&source_path, &destination_path)?; } else { @@ -1066,7 +1087,10 @@ fn copy_checkpoint_directory(source: &Path, destination: &Path) -> std::io::Resu )); } } - sync_directory(destination) + if durable { + sync_directory(destination)?; + } + Ok(()) } pub(crate) fn copy_checkpoint_file(source: &Path, destination: &Path) -> std::io::Result<()> { diff --git a/sdk/rust/lib/snapshot/restore.rs b/sdk/rust/lib/snapshot/restore.rs index dda0a75af..25202758f 100644 --- a/sdk/rust/lib/snapshot/restore.rs +++ b/sdk/rust/lib/snapshot/restore.rs @@ -8,7 +8,7 @@ use microsandbox_runtime::launch::{CheckpointRestoreConfig, RootfsUpperLayerConf use crate::{MicrosandboxError, MicrosandboxResult, Operation, UnsupportedReason}; -use super::create::{copy_checkpoint_file, materialize_checkpoint_closure}; +use super::create::{copy_checkpoint_file, stage_checkpoint_closure}; //-------------------------------------------------------------------------------------------------- // Constants @@ -44,23 +44,14 @@ pub(crate) async fn materialize_checkpoint_for_child( child_stage: &Path, root_disk: &SnapshotRootDisk, ) -> MicrosandboxResult { - let expected = ObjectId::new(&source.checkpoint_root) - .map_err(|error| MicrosandboxError::SnapshotIntegrity(error.to_string()))?; - let source_closure = CheckpointClosure::open(&source.closure, Some(&expected)) - .map_err(|error| MicrosandboxError::SnapshotIntegrity(error.to_string()))?; - if source_closure.checkpoint().checkpoint_id != source.checkpoint_id { - return Err(MicrosandboxError::SnapshotIntegrity( - "checkpoint restore source has another identity".into(), - )); - } - validate_root_disk_closure(&source_closure, root_disk, false)?; - + // Validate once after obtaining child-owned files. Validating the source first neither + // protects against a later source mutation nor substitutes for validation of the child. tokio::fs::create_dir_all(child_stage).await?; let closure_destination = child_stage.join(CHILD_CHECKPOINT_DIRECTORY); let source_path = source.closure.clone(); let destination_for_copy = closure_destination.clone(); tokio::task::spawn_blocking(move || { - materialize_checkpoint_closure(&source_path, &destination_for_copy) + stage_checkpoint_closure(&source_path, &destination_for_copy) }) .await .map_err(|error| MicrosandboxError::Custom(format!("checkpoint child copy task: {error}")))??; From 49b6670d19d0bca5c71cc53eb4ea17d5fbe514f1 Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Tue, 8 Sep 2026 21:31:08 +0100 Subject: [PATCH 08/29] feat(snapshot): add direct local branching and forked restores Create independent local children through the existing capture and restore paths without publishing a durable full snapshot. Reuse immutable RAM generations with reflinks and dirty ranges, and protect handoff and VM backing lifetimes with independent pins. Expose branching across the CLI and Rust, Python, TypeScript and Go SDKs. Replace the unreleased memory-snapshot creation policy with restore-only forked selection, and reject unsupported or incomplete restore requests instead of falling back to a fresh boot. Update documentation and add macOS live lifecycle, isolation, ownership and release benchmark harnesses. macOS release and SDK qualification passed; Linux and Windows qualification remains outstanding. --- COMPATIBILITY.md | 2 +- crates/cli/bin/main.rs | 3 + crates/cli/lib/commands/branch.rs | 37 ++ crates/cli/lib/commands/common.rs | 13 +- crates/cli/lib/commands/mod.rs | 1 + crates/cli/lib/sandbox_cmd.rs | 28 +- crates/runtime/lib/checkpoint/coordinator.rs | 309 +++++++++++++-- crates/runtime/lib/checkpoint/local.rs | 85 ++++ crates/runtime/lib/checkpoint/local_memory.rs | 373 ++++++++++++++++++ crates/runtime/lib/checkpoint/memory_cache.rs | 83 ++-- crates/runtime/lib/checkpoint/mod.rs | 4 + crates/runtime/lib/checkpoint/restore.rs | 163 +++++--- crates/runtime/lib/control.rs | 16 + crates/runtime/lib/control/executor.rs | 42 ++ crates/runtime/lib/launch.rs | 152 ++++++- crates/runtime/lib/vm.rs | 24 +- docs/sandboxes/snapshots.mdx | 26 +- packages/microsandbox-types/rust/lib/cloud.rs | 1 - .../microsandbox-types/rust/lib/domain.rs | 41 -- packages/microsandbox-types/rust/lib/lib.rs | 20 +- scripts/smoke/cli/branch-ownership.py | 58 +++ scripts/smoke/cli/checkpoint-clock.py | 3 +- scripts/smoke/cli/cow-memory-lifecycle.py | 45 ++- scripts/smoke/cli/direct-branch.py | 131 ++++++ sdk/go/cow_lifecycle_test.go | 38 +- sdk/go/internal/ffi/ffi.go | 46 ++- sdk/go/native/microsandbox_go_ffi.h | 10 + sdk/go/native/src/lib.rs | 42 +- sdk/go/options.go | 40 +- sdk/go/options_test.go | 27 +- sdk/go/sandbox.go | 20 +- sdk/node-ts/native/index.d.ts | 8 +- sdk/node-ts/native/sandbox.rs | 10 + sdk/node-ts/native/sandbox_builder.rs | 10 +- sdk/node-ts/native/sandbox_handle.rs | 8 + sdk/node-ts/src/internal/napi.ts | 4 +- sdk/node-ts/src/sandbox-handle.ts | 8 +- sdk/node-ts/src/sandbox.ts | 8 +- sdk/node-ts/tests/cow-lifecycle.test.ts | 12 +- sdk/node-ts/tests/unit/builders.test.ts | 11 +- sdk/python/microsandbox/__init__.py | 2 - sdk/python/microsandbox/_microsandbox.pyi | 7 +- sdk/python/microsandbox/types.py | 7 - sdk/python/src/helpers.rs | 8 +- sdk/python/src/sandbox.rs | 11 + sdk/python/src/sandbox_handle.rs | 11 + sdk/python/tests/test_cow_lifecycle.py | 16 +- sdk/python/tests/test_create_stub.py | 4 +- sdk/rust/lib/backend/cloud/sandbox.rs | 4 +- sdk/rust/lib/backend/local/sandbox/create.rs | 31 +- sdk/rust/lib/runtime/spawn.rs | 28 +- sdk/rust/lib/sandbox/branch.rs | 186 +++++++++ sdk/rust/lib/sandbox/builder.rs | 77 +++- sdk/rust/lib/sandbox/config.rs | 16 +- sdk/rust/lib/sandbox/config_patch.rs | 11 - sdk/rust/lib/sandbox/mod.rs | 3 +- sdk/rust/lib/snapshot/restore.rs | 4 + 57 files changed, 2049 insertions(+), 339 deletions(-) create mode 100644 crates/cli/lib/commands/branch.rs create mode 100644 crates/runtime/lib/checkpoint/local.rs create mode 100644 crates/runtime/lib/checkpoint/local_memory.rs create mode 100644 scripts/smoke/cli/branch-ownership.py create mode 100644 scripts/smoke/cli/direct-branch.py create mode 100644 sdk/rust/lib/sandbox/branch.rs diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index 8f321dd7f..55fc53aba 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -140,7 +140,7 @@ Compatibility-sensitive elements include descriptor numbers, ownership and close Sources: [`crates/runtime/lib/launch.rs`](crates/runtime/lib/launch.rs), [`crates/runtime/lib/vm.rs`](crates/runtime/lib/vm.rs), [`sdk/rust/lib/runtime/spawn.rs`](sdk/rust/lib/runtime/spawn.rs), and [`crates/cli/lib/sandbox_cmd.rs`](crates/cli/lib/sandbox_cmd.rs). -This protocol has no explicit version envelope. Treat additions as optional and consider adding explicit version or capability negotiation before allowing independently versioned launchers and runtimes. +Launch JSON requires an explicit `execution` intent (`boot` or `restore`) and rejects unknown fields. Restores also pass the internal `msb sandbox --restore` argument: a runtime predating this contract rejects the unknown argument rather than ignoring a JSON restore source and cold-booting. The argument, intent, and complete strictly validated `checkpoint_restore` source must agree before VM construction. Unsupported restore behavior is an error, never a fresh-boot fallback. These #8 development contracts replace superseded unreleased forms without shims; they do not change portable snapshot bytes. ## 6. Database, Configuration, and Migration History diff --git a/crates/cli/bin/main.rs b/crates/cli/bin/main.rs index 0e17ea395..4954ea709 100644 --- a/crates/cli/bin/main.rs +++ b/crates/cli/bin/main.rs @@ -110,6 +110,8 @@ enum Commands { Stop(stop::StopArgs), /// Suspend a resident sandbox without creating a snapshot. Pause(microsandbox_cli::commands::pause::PauseArgs), + /// Branch running execution into a new local CoW child without a durable full snapshot. + Branch(microsandbox_cli::commands::branch::BranchArgs), /// Resume a user-paused resident sandbox. Resume(microsandbox_cli::commands::pause::PauseArgs), @@ -672,6 +674,7 @@ fn run_async_command_anyhow( Commands::Start(args) => start::run(args).await, Commands::Stop(args) => stop::run(args).await, Commands::Pause(args) => microsandbox_cli::commands::pause::run(args, false).await, + Commands::Branch(args) => microsandbox_cli::commands::branch::run(args).await, Commands::Resume(args) => microsandbox_cli::commands::pause::run(args, true).await, Commands::Restart(args) => restart::run(args).await, Commands::Ping(args) => ping::run(args).await, diff --git a/crates/cli/lib/commands/branch.rs b/crates/cli/lib/commands/branch.rs new file mode 100644 index 000000000..6c2b096ed --- /dev/null +++ b/crates/cli/lib/commands/branch.rs @@ -0,0 +1,37 @@ +//! Direct local execution branching without a durable full snapshot. + +use clap::Args; +use microsandbox::Sandbox; + +use crate::ui; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// Create an independent child from a running or user-paused local sandbox. +#[derive(Args)] +pub struct BranchArgs { + /// Source sandbox name. + pub source: String, + /// Name of the new child sandbox. + #[arg(long)] + pub name: String, + /// Suppress progress output. + #[arg(short, long)] + pub quiet: bool, +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +/// Branch source execution. The child's CoW memory is inherent to this operation. +pub async fn run(args: BranchArgs) -> anyhow::Result<()> { + let source = Sandbox::get(&args.source).await?; + let child = source.branch(&args.name).await?; + if !args.quiet { + ui::success("Branched", child.name()); + } + Ok(()) +} diff --git a/crates/cli/lib/commands/common.rs b/crates/cli/lib/commands/common.rs index 227d0bddc..9bb50968d 100644 --- a/crates/cli/lib/commands/common.rs +++ b/crates/cli/lib/commands/common.rs @@ -123,9 +123,9 @@ pub struct SandboxOpts { #[arg(long, value_name = "POLICY", value_parser = ["always", "madvise", "never"])] pub thp: Option, - /// Memory snapshot representation; CoW is explicit and snapshots remain manual. - #[arg(long, value_name = "MODE", value_parser = ["standard", "cow"])] - pub memory_snapshot: Option, + /// Restore a full snapshot with private copy-on-write memory. + #[arg(long, requires = "from_snapshot", conflicts_with = "disk_only")] + pub forked: bool, /// Mount a host path or named volume into the sandbox (`SOURCE:DEST[:OPTIONS]`). /// OPTIONS may include paired `uid=,gid=` for directory-backed mounts. @@ -617,7 +617,7 @@ impl SandboxOpts { || self.memory.is_some() || self.max_memory.is_some() || self.thp.is_some() - || self.memory_snapshot.is_some() + || self.forked || !self.volume.is_empty() || !self.mount_dir.is_empty() || !self.mount_file.is_empty() @@ -888,9 +888,8 @@ fn apply_sandbox_opts_inner( .map_err(anyhow::Error::msg)?; builder = builder.thp(policy); } - if let Some(ref mode) = opts.memory_snapshot { - let mode = serde_json::from_value(serde_json::Value::String(mode.clone()))?; - builder = builder.memory_snapshot(mode); + if opts.forked { + builder = builder.forked(); } if let Some(ref workdir) = opts.workdir { builder = builder.workdir(workdir); diff --git a/crates/cli/lib/commands/mod.rs b/crates/cli/lib/commands/mod.rs index 96db57957..034bc9335 100644 --- a/crates/cli/lib/commands/mod.rs +++ b/crates/cli/lib/commands/mod.rs @@ -8,6 +8,7 @@ use crate::ui; // Exports //-------------------------------------------------------------------------------------------------- +pub mod branch; pub mod common; pub mod completion; pub mod context; diff --git a/crates/cli/lib/sandbox_cmd.rs b/crates/cli/lib/sandbox_cmd.rs index f177fcc5d..1e855c88e 100644 --- a/crates/cli/lib/sandbox_cmd.rs +++ b/crates/cli/lib/sandbox_cmd.rs @@ -33,6 +33,10 @@ use microsandbox_runtime::{ /// `--config-file` for manual invocation). See issue #997. #[derive(Debug, Args)] pub struct SandboxArgs { + /// Require captured execution; runtimes without this protocol reject the invocation. + #[arg(long, hide = true)] + pub restore: bool, + /// Name of the sandbox. #[arg(long = "name")] pub sandbox_name: String, @@ -268,7 +272,6 @@ pub fn run(args: SandboxArgs) -> ! { let vm_config = VmConfig { libkrunfw_path: launch.libkrunfw_path, thp: launch.thp, - memory_snapshot: launch.memory_snapshot, memory_cache_dir: launch.memory_cache_dir, vcpus: args.vcpus, memory_mib: args.memory_mib, @@ -402,7 +405,12 @@ fn load_launch_config(args: &SandboxArgs) -> Result { .map_err(|e| format!("failed to read --config-file {}: {e}", path.display()))?, None => return Err("missing --config-file for `msb sandbox`".to_string()), }; - serde_json::from_slice(&bytes).map_err(|e| format!("invalid launch config: {e}")) + let config = LaunchConfig::decode(&bytes)?; + if args.restore != (config.execution == microsandbox_runtime::launch::ExecutionIntent::Restore) + { + return Err("--restore and launch execution intent disagree".into()); + } + Ok(config) } /// Read the full contents of the inherited config fd, taking ownership so it @@ -700,6 +708,7 @@ mod tests { let _ = config_fd; SandboxArgs { + restore: false, sandbox_name: "test".to_string(), sandbox_id: 1, log_level: None, @@ -767,6 +776,21 @@ mod tests { ); } + #[test] + fn restore_argument_cannot_select_a_fresh_boot() { + use std::io::Write; + let mut file = tempfile::NamedTempFile::new().unwrap(); + file.write_all(&serde_json::to_vec(&LaunchConfig::default()).unwrap()) + .unwrap(); + let mut args = args_with(None, Some(file.path().to_path_buf())); + args.restore = true; + assert!( + load_launch_config(&args) + .unwrap_err() + .contains("intent disagree") + ); + } + #[test] fn test_old_launch_config_without_run_dir_remains_readable() { use std::io::Write; diff --git a/crates/runtime/lib/checkpoint/coordinator.rs b/crates/runtime/lib/checkpoint/coordinator.rs index 3b13f0dad..d1823c693 100644 --- a/crates/runtime/lib/checkpoint/coordinator.rs +++ b/crates/runtime/lib/checkpoint/coordinator.rs @@ -24,6 +24,7 @@ use msb_krun::{ }; use super::disk::RuntimeOwnedRootDisk; +use super::local_memory::{LocalMemoryCapture, LocalMemoryPin}; use crate::vm::VmConfig; //-------------------------------------------------------------------------------------------------- @@ -58,6 +59,9 @@ pub(crate) struct CheckpointCoordinator { previous_memory: Option, memory_cache: Option, cached_baseline: Option<(MemoryManifest, super::CachedMemory)>, + local_cache_root: Option, + local_baseline: Option, + boot_geometry: (u8, u8, u32, u32), } /// Published checkpoint identity returned to the control executor. @@ -88,7 +92,8 @@ struct AdmittedResources { struct PausedCapture { result: CheckpointResult, memory_plan: MemoryCapturePlan, - memory_manifest: MemoryManifest, + memory_manifest: Option, + local_memory: Option, timings: PausedCaptureTimings, } @@ -330,7 +335,11 @@ impl CheckpointCoordinator { fs_resource_bindings, network_resource_binding, previous_memory: None, - memory_cache: if vm.memory_snapshot == microsandbox_types::MemorySnapshotMode::Cow { + memory_cache: if vm + .checkpoint_restore + .as_ref() + .is_some_and(|restore| restore.forked) + { Some( super::MemoryCache::open(vm.memory_cache_dir.as_ref().ok_or_else(|| { "CoW memory requires its backend-resolved cache directory".to_string() @@ -341,6 +350,9 @@ impl CheckpointCoordinator { None }, cached_baseline: None, + local_cache_root: vm.memory_cache_dir.clone(), + local_baseline: None, + boot_geometry: (vm.vcpus, vm.max_cpus, vm.memory_mib, vm.max_memory_mib), }) } @@ -351,6 +363,82 @@ impl CheckpointCoordinator { checkpoint_id: &str, intent: CaptureIntent, user_pause: Option<&UserPause>, + ) -> Result { + self.capture_to(vm, checkpoint_id, intent, user_pause, None) + } + + /// Capture a local handoff directly, without publishing a portable RAM closure. + pub(crate) fn branch( + &mut self, + vm: &msb_krun::VmControl, + id: &str, + child_name: &str, + reserved_cache: &Path, + user_pause: Option<&UserPause>, + ) -> Result { + let cache = self.local_cache_root.as_ref().ok_or_else(|| { + CheckpointFailure::before_pause("runtime has no backend-resolved memory cache") + })?; + // Reject unsupported hosts before freezing or rolling over the source disk. + super::MemoryCache::open_namespace(cache.clone(), "branches") + .map_err(CheckpointFailure::before_pause)?; + if std::fs::canonicalize(cache).map_err(CheckpointFailure::before_pause)? + != std::fs::canonicalize(reserved_cache).map_err(CheckpointFailure::before_pause)? + { + return Err(CheckpointFailure::before_pause( + "branch handoff cache differs from the source runtime; use the source's original backend cache configuration", + )); + } + validate_checkpoint_id(id).map_err(CheckpointFailure::before_pause)?; + microsandbox_types::validate_sandbox_name(child_name) + .map_err(CheckpointFailure::before_pause)?; + // The SDK reserves a fresh child directory under this same backend. Never accept + // caller-selected host paths, symlinked children, or an existing handoff destination. + let source = self.root.parent().and_then(Path::parent).ok_or_else(|| { + CheckpointFailure::before_pause("source storage has no sandbox parent") + })?; + let parent = source + .parent() + .ok_or_else(|| CheckpointFailure::before_pause("missing sandbox storage root"))?; + let child = parent.join(child_name); + if child == source + || !std::fs::symlink_metadata(&child).is_ok_and(|m| m.file_type().is_dir()) + { + return Err(CheckpointFailure::before_pause( + "branch requires a reserved child directory", + )); + } + let reservation = child.join(".branch-reservation"); + if !std::fs::symlink_metadata(&reservation) + .is_ok_and(|m| m.file_type().is_file() && m.len() <= 128) + || std::fs::read_to_string(&reservation).map_err(CheckpointFailure::before_pause)? != id + { + return Err(CheckpointFailure::before_pause( + "child reservation does not match branch attempt", + )); + } + let destination = child.join(".branch-restore"); + if std::fs::symlink_metadata(&destination).is_ok() { + return Err(CheckpointFailure::before_pause( + "child already has a branch handoff", + )); + } + self.capture_to( + vm, + id, + CaptureIntent::FullSnapshot, + user_pause, + Some(&destination), + ) + } + + fn capture_to( + &mut self, + vm: &msb_krun::VmControl, + checkpoint_id: &str, + intent: CaptureIntent, + user_pause: Option<&UserPause>, + local_destination: Option<&Path>, ) -> Result { if let Some(paused) = user_pause { paused @@ -382,16 +470,21 @@ impl CheckpointCoordinator { .map_err(CheckpointFailure::before_pause)?; let admission_us = admission_started.elapsed().as_micros(); let staging_started = Instant::now(); - let final_path = self.root.join(checkpoint_id); + let final_path = local_destination + .map(Path::to_path_buf) + .unwrap_or_else(|| self.root.join(checkpoint_id)); if final_path.exists() { return Err(CheckpointFailure::before_pause( "checkpoint identity is already published", )); } - let staging = self.root.join(format!( - ".{checkpoint_id}.{}.staging", - rand::random::() - )); + let staging = final_path + .parent() + .ok_or_else(|| CheckpointFailure::before_pause("capture destination has no parent"))? + .join(format!( + ".{checkpoint_id}.{}.staging", + rand::random::() + )); std::fs::create_dir(&staging).map_err(CheckpointFailure::before_pause)?; let staging_us = staging_started.elapsed().as_micros(); @@ -447,6 +540,7 @@ impl CheckpointCoordinator { pause.get(), &staging, &final_path, + local_destination.is_some(), ); let paused_capture_us = paused_capture_started.elapsed().as_micros(); let captured = match paused { @@ -521,18 +615,19 @@ impl CheckpointCoordinator { } let thaw_us = thaw_started.elapsed().as_micros(); let workload_unavailable_us = workload_unavailable_started.elapsed().as_micros(); - if let Some(cache) = &self.memory_cache { + if let (Some(cache), Some(memory_manifest)) = + (&self.memory_cache, &captured.memory_manifest) + { // Source execution has resumed (unless explicitly user-paused). Read only the // completed immutable capture, never live RAM, while preparing child acceleration. let prepared = (|| -> Result { - let bytes = captured - .memory_manifest + let bytes = memory_manifest .to_canonical_bytes() .map_err(|e| e.to_string())?; let identity = ObjectId::from_bytes(&bytes).map_err(|e| e.to_string())?; cache .materialize_with_baseline( - &captured.memory_manifest, + memory_manifest, &identity, self.cached_baseline .as_ref() @@ -557,7 +652,7 @@ impl CheckpointCoordinator { match prepared { Ok(cached) => { tracing::info!(target: "microsandbox_checkpoint_timing", operation = "memory_cache", prepare_us = cached.prepare_us, cache_hit = cached.cache_hit, reflink = cached.reflink, "prepared immutable capture cache"); - self.cached_baseline = Some((captured.memory_manifest.clone(), cached)); + self.cached_baseline = Some((memory_manifest.clone(), cached)); } Err(error) => { // Publication already succeeded. Losing optional acceleration does not @@ -567,9 +662,11 @@ impl CheckpointCoordinator { } } if baseline_published { - self.previous_memory = Some(captured.memory_manifest); + self.previous_memory = captured.memory_manifest; + self.local_baseline = captured.local_memory; } else { self.previous_memory = None; + self.local_baseline = None; } tracing::info!( target: "microsandbox_checkpoint_timing", @@ -706,11 +803,13 @@ impl CheckpointCoordinator { pause_generation: u64, staging: &Path, final_path: &Path, + local: bool, ) -> Result { let mut timings = PausedCaptureTimings::default(); let devices_started = Instant::now(); let mut pending_devices = Vec::with_capacity(inventory.len()); let mut disk_roots = Vec::new(); + let mut local_disks = Vec::new(); for (device_type, device_id) in inventory { let runtime_owned_root = self .root_disk @@ -732,18 +831,21 @@ impl CheckpointCoordinator { published: None, })?; timings.managed_disk_us += disk_started.elapsed().as_micros(); - let manifest_bytes = rollover - .manifest - .to_canonical_bytes() - .map_err(CheckpointFailure::resumable)?; - let manifest_id = self - .store - .put_bytes(&manifest_bytes) - .map_err(CheckpointFailure::resumable)?; - self.store - .link_into(&manifest_id, staging) - .map_err(CheckpointFailure::resumable)?; - disk_roots.push(manifest_id); + if !local { + let manifest_bytes = rollover + .manifest + .to_canonical_bytes() + .map_err(CheckpointFailure::resumable)?; + let manifest_id = self + .store + .put_bytes(&manifest_bytes) + .map_err(CheckpointFailure::resumable)?; + self.store + .link_into(&manifest_id, staging) + .map_err(CheckpointFailure::resumable)?; + disk_roots.push(manifest_id); + } + local_disks.push(rollover.manifest); rollover.device_state } else if *device_type == TYPE_BLOCK { vm.capture_block_device_state(device_id) @@ -785,8 +887,21 @@ impl CheckpointCoordinator { bytes, }); } - let device_refs = persist_device_states(&self.store, staging, &pending_devices) - .map_err(CheckpointFailure::resumable)?; + let device_refs = if local { + pending_devices + .iter() + .map(|device| { + Ok(DeviceStateRef { + device_type: device.device_type, + device_id: device.device_id.clone(), + state: put_local_object(staging, &device.bytes)?, + }) + }) + .collect::, String>>() + } else { + persist_device_states(&self.store, staging, &pending_devices) + } + .map_err(CheckpointFailure::resumable)?; timings.devices_us = devices_started.elapsed().as_micros(); // Device capture parks each worker. Capture interrupt-controller state @@ -804,15 +919,101 @@ impl CheckpointCoordinator { )); } let execution_bytes = execution.encode().map_err(CheckpointFailure::resumable)?; - let execution_id = self - .store - .put_bytes(&execution_bytes) - .map_err(CheckpointFailure::resumable)?; - self.store - .link_into(&execution_id, staging) - .map_err(CheckpointFailure::resumable)?; + let execution_id = if local { + put_local_object(staging, &execution_bytes).map_err(CheckpointFailure::resumable)? + } else { + let id = self + .store + .put_bytes(&execution_bytes) + .map_err(CheckpointFailure::resumable)?; + self.store + .link_into(&id, staging) + .map_err(CheckpointFailure::resumable)?; + id + }; timings.execution_us = execution_started.elapsed().as_micros(); + if local { + let (memory_plan, incremental) = self + .plan_local_memory(vm) + .map_err(CheckpointFailure::resumable)?; + let captured = (|| { + let started = Instant::now(); + let mut sink = LocalMemoryCapture::new( + self.local_cache_root + .as_ref() + .expect("validated local cache"), + checkpoint_id, + if incremental { + self.local_baseline.as_ref() + } else { + None + }, + ) + .map_err(CheckpointFailure::resumable)?; + let reflink = sink.reflink; + let stats = vm + .capture_memory( + &memory_plan, + MemoryCaptureOptions::new(MEMORY_SCAN_CHUNK_SIZE, true) + .map_err(CheckpointFailure::resumable)?, + &mut sink, + ) + .map_err(CheckpointFailure::resumable)?; + let memory = sink + .finish(memory_plan.generation().get(), memory_plan.topology().get()) + .map_err(CheckpointFailure::resumable)?; + timings.memory_capture_us = started.elapsed().as_micros(); + let state = super::LocalBranchState { + id: checkpoint_id.into(), + architecture: std::env::consts::ARCH.into(), + pause_generation, + execution_state: execution_id, + devices: device_refs, + resources, + disks: local_disks, + memory: memory.memory.clone(), + vcpus: self.boot_geometry.0, + max_cpus: self.boot_geometry.1, + memory_mib: self.boot_geometry.2, + max_memory_mib: self.boot_geometry.3, + }; + let bytes = serde_json::to_vec(&state).map_err(CheckpointFailure::resumable)?; + // This handoff has no snapshot root or RAM object manifest. Child-owned disk + // links and bounded metadata are installed before acknowledging the capture. + std::fs::write(staging.join("branch.json"), bytes) + .map_err(CheckpointFailure::resumable)?; + std::fs::rename(staging, final_path).map_err(CheckpointFailure::resumable)?; + tracing::info!(target: "microsandbox_checkpoint_timing", operation = "local_memory_capture", incremental, reflink, capture_us = timings.memory_capture_us); + Ok((memory, stats)) + })(); + let (memory, stats) = match captured { + Ok(captured) => captured, + Err(error) => { + let _ = vm.abandon_memory_capture(&memory_plan); + return Err(error); + } + }; + return Ok(PausedCapture { + result: CheckpointResult { + checkpoint_id: checkpoint_id.into(), + checkpoint_root: String::new(), + path: final_path.into(), + memory_mode: if incremental { + MemoryCaptureMode::Incremental + } else { + MemoryCaptureMode::Full + }, + memory_logical_bytes: stats.logical_bytes, + memory_emitted_bytes: stats.emitted_bytes, + }, + memory_plan, + memory_manifest: None, + local_memory: Some(memory), + timings, + }); + } + let memory_plan_started = Instant::now(); let (memory_plan, memory_mode, base_extents) = self.plan_memory(vm).map_err(CheckpointFailure::resumable)?; @@ -948,11 +1149,37 @@ impl CheckpointCoordinator { memory_emitted_bytes: stats.emitted_bytes, }, memory_plan, - memory_manifest, + memory_manifest: Some(memory_manifest), + local_memory: None, timings, }) } + fn plan_local_memory( + &self, + vm: &msb_krun::VmControl, + ) -> Result<(MemoryCapturePlan, bool), String> { + if let (Some(baseline), Some(previous)) = + (vm.retained_memory_baseline(), self.local_baseline.as_ref()) + && previous.memory.generation == baseline.generation().get() + && previous.memory.topology == baseline.topology().get() + { + match vm + .plan_incremental_memory_capture(baseline) + .map_err(|e| e.to_string())? + { + IncrementalCaptureDecision::Incremental(plan) => return Ok((plan, true)), + IncrementalCaptureDecision::Complete { capture, .. } => { + return Ok((capture, false)); + } + IncrementalCaptureDecision::FullRequired(_) => {} + } + } + vm.plan_full_memory_capture() + .map(|plan| (plan, false)) + .map_err(|e| e.to_string()) + } + fn plan_memory( &self, vm: &msb_krun::VmControl, @@ -1501,6 +1728,18 @@ fn resource_kind(device_type: u32) -> &'static str { /// Persist independent device envelopes concurrently after every device has reached the same /// paused epoch. Immutable-object publication is thread-safe, and the returned vector retains the /// inventory order required by the checkpoint manifest. +/// Local handoffs reuse the state codecs and object paths, but make no crash-recovery promise. +/// Only bounded CPU/device state reaches this helper; RAM goes straight to its mmap backing. +fn put_local_object(staging: &Path, bytes: &[u8]) -> Result { + let id = ObjectId::from_bytes(bytes).map_err(|e| e.to_string())?; + let store = LocalObjectStore::open(staging).map_err(|e| e.to_string())?; + let path = store.object_path(&id); + std::fs::create_dir_all(path.parent().expect("confined object parent")) + .map_err(|e| e.to_string())?; + std::fs::write(path, bytes).map_err(|e| e.to_string())?; + Ok(id) +} + fn persist_device_states( store: &LocalObjectStore, staging: &Path, diff --git a/crates/runtime/lib/checkpoint/local.rs b/crates/runtime/lib/checkpoint/local.rs new file mode 100644 index 000000000..84cd6c8c8 --- /dev/null +++ b/crates/runtime/lib/checkpoint/local.rs @@ -0,0 +1,85 @@ +//! Bounded process-local branch handoff, deliberately distinct from a full snapshot. + +use std::fs::File; +use std::io::{self, Read}; +use std::path::Path; + +use microsandbox_image::checkpoint::{ + DeviceStateRef, DiskGenerationManifest, LocalObjectStore, ObjectId, ResourceDescriptor, +}; +use serde::{Deserialize, Serialize}; + +use super::local_memory::LocalMemory; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// Same-epoch local execution handoff. RAM has no portable object representation here. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LocalBranchState { + /// Unique capture attempt, shared with the workload freeze latch. + pub id: String, + /// Host architecture required by the captured execution. + pub architecture: String, + /// Shared CPU/device/RAM pause boundary. + pub pause_generation: u64, + /// Encoded CPU and interrupt-controller state. + pub execution_state: ObjectId, + /// Existing device state encodings. + pub devices: Vec, + /// Existing resource bindings, including the captured agent identity. + pub resources: Vec, + /// Complete sealed disk generations. + pub disks: Vec, + /// Complete, immutable, mmap-ready RAM; never a partial memory manifest. + pub memory: LocalMemory, + /// Boot CPU count and configured capacity, not a mutable guest online count. + pub vcpus: u8, + /// Maximum CPU count used for device construction. + pub max_cpus: u8, + /// Boot RAM geometry in MiB. + pub memory_mib: u32, + /// Configured hotplug capacity in MiB. + pub max_memory_mib: u32, +} + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl LocalBranchState { + /// Read bounded handoff metadata; ordinary snapshot readers never recognize this file. + pub fn open(root: &Path) -> io::Result { + let bytes = read_bounded(&root.join("branch.json"), 16 * 1024 * 1024)?; + let state: Self = serde_json::from_slice(&bytes).map_err(io::Error::other)?; + if state.architecture != std::env::consts::ARCH { + return Err(io::Error::other("branch architecture differs")); + } + Ok(state) + } + + /// Read an existing bounded state object, checking its recorded identity. + pub fn read_object(root: &Path, id: &ObjectId, limit: u64) -> io::Result> { + let store = LocalObjectStore::open(root).map_err(io::Error::other)?; + let bytes = read_bounded(&store.object_path(id), limit)?; + if ObjectId::from_bytes(&bytes).map_err(io::Error::other)? != *id { + return Err(io::Error::other("branch state object differs")); + } + Ok(bytes) + } +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +fn read_bounded(path: &Path, limit: u64) -> io::Result> { + let mut bytes = Vec::new(); + File::open(path)?.take(limit + 1).read_to_end(&mut bytes)?; + if bytes.len() as u64 > limit { + return Err(io::Error::other("local state exceeds size bound")); + } + Ok(bytes) +} diff --git a/crates/runtime/lib/checkpoint/local_memory.rs b/crates/runtime/lib/checkpoint/local_memory.rs new file mode 100644 index 000000000..ed6aa8a9a --- /dev/null +++ b/crates/runtime/lib/checkpoint/local_memory.rs @@ -0,0 +1,373 @@ +//! Direct local RAM generations. The source's live mappings are never replaced. + +use std::fs::{File, OpenOptions}; +use std::io::{self, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; + +use msb_krun::{GuestMemoryRange, MemoryCaptureSink}; +use serde::{Deserialize, Serialize}; + +use super::memory_cache::open_pinned; +use super::{CachedMemoryRegion, MemoryCache}; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// Complete local memory geometry; this is not a portable memory manifest. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LocalMemory { + /// Immutable backend-owned file, independent of the source sandbox directory. + pub path: PathBuf, + /// Native mapping geometry in guest-physical order. + pub regions: Vec, + /// Retained memory generation used only for incremental capture continuity. + pub generation: u64, + /// Memory topology to which the generation belongs. + pub topology: u64, +} + +pub(crate) struct LocalMemoryPin { + pub(crate) memory: LocalMemory, + pub(crate) _file: File, +} + +pub(super) struct LocalMemoryCapture { + staging: tempfile::TempDir, + file: File, + path: PathBuf, + regions: Vec, + length: u64, + incremental: bool, + page_size: u64, + pub(super) reflink: bool, +} + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl LocalMemory { + /// Reserve publication-to-pin ownership before asking the source to capture RAM. + /// Stable lock inodes are never unlinked, so eviction cannot race a replacement lock. + pub fn reserve(root: &Path, id: &str) -> io::Result { + if id.is_empty() + || id.len() > 128 + || !id + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'-')) + { + return Err(io::Error::other("invalid local generation identity")); + } + let cache = MemoryCache::open_namespace(root.into(), "branches")?; + let path = cache + .root + .join(format!("{id}-{}.handoff-lock", cache.page_size)); + let file = microsandbox_utils::process_lock::open_lock_file(&path)?; + microsandbox_utils::process_lock::lock_exclusive(&file)?; + Ok(file) + } + + /// Reclaim only after pending handoff, retained-baseline and VM pins have been released. + pub fn evict(&self) -> io::Result { + let handoff = microsandbox_utils::process_lock::open_lock_file( + &self.path.with_extension("handoff-lock"), + )?; + if !microsandbox_utils::process_lock::try_lock_exclusive(&handoff)? { + return Ok(false); + } + super::memory_cache::evict_unpinned(&self.path) + } + + /// Acquire independent backing ownership before launching or mapping a child. + pub fn pin(&self) -> io::Result { + let mut file_end = 0; + let mut guest_end = 0; + for region in &self.regions { + if region.length == 0 + || region.file_offset != file_end + || region.guest_address < guest_end + { + return Err(io::Error::other("invalid local memory geometry")); + } + file_end = region + .file_offset + .checked_add(region.length) + .ok_or_else(|| io::Error::other("memory size overflow"))?; + guest_end = region + .guest_address + .checked_add(region.length) + .ok_or_else(|| io::Error::other("guest range overflow"))?; + } + if file_end == 0 { + return Err(io::Error::other("empty local memory")); + } + open_pinned(&self.path, file_end)? + .ok_or_else(|| io::Error::other("local memory backing is missing")) + } +} + +impl LocalMemoryCapture { + pub(super) fn new( + root: &Path, + id: &str, + baseline: Option<&LocalMemoryPin>, + ) -> io::Result { + let cache = MemoryCache::open_namespace(root.into(), "branches")?; + let staging = tempfile::Builder::new() + .prefix(".capture-") + .tempdir_in(&cache.root)?; + let temporary = staging.path().join("memory"); + let mut reflink = false; + if let Some(base) = baseline { + let (_, strategy) = + microsandbox_utils::copy::fast_copy_with_strategy(&base.memory.path, &temporary)?; + reflink = strategy == microsandbox_utils::copy::FastCopyStrategy::Reflink; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&temporary, std::fs::Permissions::from_mode(0o600))?; + } + } + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(&temporary)?; + let length = file.metadata()?.len(); + Ok(Self { + staging, + file, + length, + reflink, + path: cache.root.join(format!("{id}-{}.ram", cache.page_size)), + regions: baseline + .map(|base| base.memory.regions.clone()) + .unwrap_or_default(), + incremental: baseline.is_some(), + page_size: cache.page_size, + }) + } + + fn offset(&mut self, range: GuestMemoryRange) -> io::Result { + let end = range + .start() + .checked_add(range.length()) + .ok_or_else(|| io::Error::other("range overflow"))?; + if self.incremental { + let region = self + .regions + .iter() + .find(|region| { + range.start() >= region.guest_address + && end <= region.guest_address + region.length + }) + .ok_or_else(|| io::Error::other("delta falls outside retained memory topology"))?; + return Ok(region.file_offset + range.start() - region.guest_address); + } + let offset = self.length; + if let Some(last) = self.regions.last_mut() { + let previous_end = last.guest_address + last.length; + if range.start() < previous_end { + return Err(io::Error::other("unordered full memory capture")); + } + if range.start() == previous_end { + last.length += range.length(); + } else { + self.regions.push(CachedMemoryRegion { + guest_address: range.start(), + length: range.length(), + file_offset: offset, + }); + } + } else { + self.regions.push(CachedMemoryRegion { + guest_address: range.start(), + length: range.length(), + file_offset: offset, + }); + } + self.length = self + .length + .checked_add(range.length()) + .ok_or_else(|| io::Error::other("memory file overflow"))?; + Ok(offset) + } + + pub(super) fn finish(self, generation: u64, topology: u64) -> io::Result { + for region in &self.regions { + if !region.guest_address.is_multiple_of(self.page_size) + || !region.length.is_multiple_of(self.page_size) + || !region.file_offset.is_multiple_of(self.page_size) + { + return Err(io::Error::other( + "local memory geometry is not native-page aligned", + )); + } + } + self.file.set_len(self.length)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + self.file + .set_permissions(std::fs::Permissions::from_mode(0o400))?; + } + // Local branching promises process-independent ownership, not crash recovery. Closing + // the writer and publishing the completed inode suffices; no RAM-sized fsync/hash pass. + drop(self.file); + let memory = LocalMemory { + path: self.path, + regions: self.regions, + generation, + topology, + }; + let file = open_pinned(&self.staging.path().join("memory"), self.length)? + .ok_or_else(|| io::Error::other("capture staging disappeared"))?; + std::fs::hard_link(self.staging.path().join("memory"), &memory.path)?; + Ok(LocalMemoryPin { + memory, + _file: file, + }) + } +} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +impl MemoryCaptureSink for LocalMemoryCapture { + fn write_bytes(&mut self, range: GuestMemoryRange, bytes: &[u8]) -> io::Result<()> { + if range.length() != bytes.len() as u64 { + return Err(io::Error::other("capture length mismatch")); + } + let offset = self.offset(range)?; + self.file.seek(SeekFrom::Start(offset))?; + self.file.write_all(bytes) + } + + fn write_zero(&mut self, range: GuestMemoryRange) -> io::Result<()> { + let offset = self.offset(range)?; + if self.incremental { + // A zero/discard delta must replace old bytes, not leave stale private content. + self.file.seek(SeekFrom::Start(offset))?; + let zeroes = [0u8; 65536]; + let mut remaining = range.length(); + while remaining != 0 { + let count = remaining.min(zeroes.len() as u64) as usize; + self.file.write_all(&zeroes[..count])?; + remaining -= count as u64; + } + } + Ok(()) + } +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(all(test, unix))] +mod tests { + use std::io::Read; + use std::os::unix::fs::PermissionsExt; + + use super::*; + + fn range(start: u64, length: u64) -> GuestMemoryRange { + GuestMemoryRange::new(start, length).unwrap() + } + + #[test] + fn direct_generation_is_sparse_complete_and_independently_pinned() { + let dir = tempfile::tempdir().unwrap(); + let page = MemoryCache::open(dir.path()).unwrap().page_size; + let mut sink = LocalMemoryCapture::new(dir.path(), "first", None).unwrap(); + sink.write_bytes(range(0, page), &vec![7; page as usize]) + .unwrap(); + sink.write_zero(range(page, page)).unwrap(); + sink.write_bytes(range(4 * page, page), &vec![9; page as usize]) + .unwrap(); + let captured = sink.finish(1, 1).unwrap(); + assert_eq!(captured.memory.regions.len(), 2); + assert_eq!(captured.memory.regions[1].file_offset, 2 * page); + assert_eq!( + captured._file.metadata().unwrap().permissions().mode() & 0o777, + 0o400 + ); + let mut child = captured.memory.pin().unwrap(); + std::fs::remove_file(&captured.memory.path).unwrap(); + drop(captured); + let mut bytes = Vec::new(); + child.read_to_end(&mut bytes).unwrap(); + assert_eq!(bytes.len(), (3 * page) as usize); + assert!( + bytes[page as usize..(2 * page) as usize] + .iter() + .all(|byte| *byte == 0) + ); + } + + #[test] + fn incremental_zero_and_write_do_not_mutate_the_baseline() { + let dir = tempfile::tempdir().unwrap(); + let page = MemoryCache::open(dir.path()).unwrap().page_size; + let mut full = LocalMemoryCapture::new(dir.path(), "base", None).unwrap(); + full.write_bytes(range(0, 2 * page), &vec![7; (2 * page) as usize]) + .unwrap(); + let base = full.finish(1, 1).unwrap(); + let mut delta = LocalMemoryCapture::new(dir.path(), "delta", Some(&base)).unwrap(); + delta.write_zero(range(0, page)).unwrap(); + delta + .write_bytes(range(page, page), &vec![9; page as usize]) + .unwrap(); + let child = delta.finish(2, 1).unwrap(); + assert!( + std::fs::read(&base.memory.path) + .unwrap() + .iter() + .all(|b| *b == 7) + ); + let bytes = std::fs::read(&child.memory.path).unwrap(); + assert!(bytes[..page as usize].iter().all(|b| *b == 0)); + assert!(bytes[page as usize..].iter().all(|b| *b == 9)); + } + + #[test] + fn capture_rejects_overlap_unaligned_geometry_and_identity_collision() { + let dir = tempfile::tempdir().unwrap(); + let page = MemoryCache::open(dir.path()).unwrap().page_size; + let mut sink = LocalMemoryCapture::new(dir.path(), "same", None).unwrap(); + sink.write_zero(range(0, page)).unwrap(); + assert!(sink.write_zero(range(0, page)).is_err()); + let _pin = sink.finish(1, 1).unwrap(); + let mut collision = LocalMemoryCapture::new(dir.path(), "same", None).unwrap(); + collision.write_zero(range(0, page)).unwrap(); + assert!(collision.finish(2, 1).is_err()); + let mut unaligned = LocalMemoryCapture::new(dir.path(), "unaligned", None).unwrap(); + unaligned.write_zero(range(0, 4096)).unwrap(); + if page > 4096 { + assert!(unaligned.finish(3, 1).is_err()); + } + } + + #[test] + fn pending_handoff_survives_source_pin_loss_and_eviction() { + let dir = tempfile::tempdir().unwrap(); + let page = MemoryCache::open(dir.path()).unwrap().page_size; + let reservation = LocalMemory::reserve(dir.path(), "handoff").unwrap(); + let mut sink = LocalMemoryCapture::new(dir.path(), "handoff", None).unwrap(); + sink.write_zero(range(0, page)).unwrap(); + let source = sink.finish(1, 1).unwrap(); + let memory = source.memory.clone(); + drop(source); // source exits before the SDK receives the response + assert!(!memory.evict().unwrap()); + let child = memory.pin().unwrap(); + drop(reservation); + assert!(!memory.evict().unwrap()); + drop(child); + assert!(memory.evict().unwrap()); + assert!(memory.pin().is_err()); + } +} diff --git a/crates/runtime/lib/checkpoint/memory_cache.rs b/crates/runtime/lib/checkpoint/memory_cache.rs index 6f0a910e4..fe5cdff61 100644 --- a/crates/runtime/lib/checkpoint/memory_cache.rs +++ b/crates/runtime/lib/checkpoint/memory_cache.rs @@ -16,7 +16,8 @@ use microsandbox_image::checkpoint::{MemoryExtentContent, MemoryManifest, Object //-------------------------------------------------------------------------------------------------- /// One native-aligned, contiguous guest address span in a flat cache file. -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] pub struct CachedMemoryRegion { /// Start of the guest physical span. pub guest_address: u64, @@ -44,8 +45,8 @@ pub struct CachedMemory { /// Host-local, immutable memory cache. No entry is ever modified in place. pub struct MemoryCache { - root: PathBuf, - page_size: u64, + pub(super) root: PathBuf, + pub(super) page_size: u64, } //-------------------------------------------------------------------------------------------------- @@ -55,18 +56,24 @@ pub struct MemoryCache { impl MemoryCache { /// Open a dedicated cache directory using this host's native mapping alignment. pub fn open(root: impl Into) -> io::Result { + Self::open_namespace(root.into(), "snapshots") + } + + pub(super) fn open_namespace(root: PathBuf, namespace: &str) -> io::Result { #[cfg(unix)] { let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; if page_size <= 0 { return Err(io::Error::last_os_error()); } - let root = root.into(); std::fs::create_dir_all(&root)?; // Cache contents are guest RAM, not public image data. Restrict traversal even // when the caller's umask permits other local users to read ordinary cache files. use std::os::unix::fs::PermissionsExt; std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700))?; + let root = root.join(namespace); + std::fs::create_dir_all(&root)?; + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700))?; Ok(Self { root, page_size: page_size as u64, @@ -74,7 +81,7 @@ impl MemoryCache { } #[cfg(not(unix))] { - let _ = root; + let _ = (root, namespace); Err(io::Error::new( io::ErrorKind::Unsupported, "private memory cache is not qualified on this backend", @@ -271,31 +278,7 @@ impl MemoryCache { /// reader that opened the inode immediately before an eviction acquired its exclusive lock. pub fn evict(&self, identity: &ObjectId) -> io::Result { let path = self.entry_path(identity); - let file = match open_readonly(&path) { - Ok(file) => file, - Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), - Err(error) => return Err(error), - }; - if !microsandbox_utils::process_lock::try_lock_exclusive(&file)? { - return Ok(false); - } - // A competing evictor can have removed this same inode while we waited to acquire it. - // Do not unlink a new realization published at the old name in the meantime. - #[cfg(unix)] - { - use std::os::unix::fs::MetadataExt; - let opened = file.metadata()?; - let current = match std::fs::symlink_metadata(&path) { - Ok(metadata) => metadata, - Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), - Err(error) => return Err(error), - }; - if (opened.dev(), opened.ino()) != (current.dev(), current.ino()) { - return Ok(false); - } - } - std::fs::remove_file(path)?; - Ok(true) + evict_unpinned(&path) } fn entry_path(&self, identity: &ObjectId) -> PathBuf { @@ -351,6 +334,35 @@ fn memory_regions( Ok(regions) } +/// Both cache namespaces use the same inode/lock checks and never mutate mapped RAM. +pub(super) fn evict_unpinned(path: &Path) -> io::Result { + let file = match open_readonly(path) { + Ok(file) => file, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error), + }; + if !microsandbox_utils::process_lock::try_lock_exclusive(&file)? { + return Ok(false); + } + // A competing evictor can have removed this same inode while we waited to acquire it. + // Do not unlink a new realization published at the old name in the meantime. + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let opened = file.metadata()?; + let current = match std::fs::symlink_metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error), + }; + if (opened.dev(), opened.ino()) != (current.dev(), current.ino()) { + return Ok(false); + } + } + std::fs::remove_file(path)?; + Ok(true) +} + fn open_readonly(path: &Path) -> io::Result { let mut options = OpenOptions::new(); options.read(true); @@ -362,7 +374,7 @@ fn open_readonly(path: &Path) -> io::Result { options.open(path) } -fn open_pinned(path: &Path, length: u64) -> io::Result> { +pub(super) fn open_pinned(path: &Path, length: u64) -> io::Result> { let file = match open_readonly(path) { Ok(file) => file, Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), @@ -592,7 +604,7 @@ mod tests { fn payload_count(root: &Path) -> usize { // Build-lock inodes intentionally survive failed builders. Only RAM or staging entries // count as payloads; removing lock files would permit two independent flock owners. - std::fs::read_dir(root) + std::fs::read_dir(root.join("snapshots")) .unwrap() .filter(|entry| { entry @@ -634,7 +646,12 @@ mod tests { ); }); assert_eq!(reads.load(std::sync::atomic::Ordering::Relaxed), 1); - assert_eq!(std::fs::read_dir(directory.path()).unwrap().count(), 2); + assert_eq!( + std::fs::read_dir(directory.path().join("snapshots")) + .unwrap() + .count(), + 2 + ); } #[test] diff --git a/crates/runtime/lib/checkpoint/mod.rs b/crates/runtime/lib/checkpoint/mod.rs index 5a42018cd..62cb6714e 100644 --- a/crates/runtime/lib/checkpoint/mod.rs +++ b/crates/runtime/lib/checkpoint/mod.rs @@ -2,6 +2,8 @@ mod coordinator; mod disk; +mod local; +mod local_memory; mod memory_cache; mod restore; @@ -15,6 +17,8 @@ pub use disk::{ DiskCompactionResult, RuntimeOwnedRootChain, RuntimeOwnedRootLayer, compact_stopped_root, grow_stopped_root, load_runtime_owned_root_chain, recover_stopped_root_growth, }; +pub use local::LocalBranchState; +pub use local_memory::LocalMemory; pub use memory_cache::{CachedMemory, CachedMemoryRegion, MemoryCache}; pub(crate) use restore::{PreparedCheckpointRestore, RestoredAgentState}; diff --git a/crates/runtime/lib/checkpoint/restore.rs b/crates/runtime/lib/checkpoint/restore.rs index 6e9131ff9..c6ca99a35 100644 --- a/crates/runtime/lib/checkpoint/restore.rs +++ b/crates/runtime/lib/checkpoint/restore.rs @@ -30,7 +30,8 @@ const MAX_MEMORY_OBJECT_BYTES: u64 = 32 * 1024 * 1024; pub(crate) struct PreparedCheckpointRestore { execution: msb_krun::ExecutionState, devices: Vec, - memory: CheckpointMemoryRestore, + memory: Option, + local_memory: Option, agent: RestoredAgentState, } @@ -61,6 +62,52 @@ struct CheckpointMemoryRestore { //-------------------------------------------------------------------------------------------------- impl PreparedCheckpointRestore { + /// Decode a local handoff and pin its RAM before constructing any guest mappings. + pub(crate) fn open_local(root: PathBuf, expected_id: &str) -> Result { + let state = super::LocalBranchState::open(&root).map_err(|e| e.to_string())?; + if state.id != expected_id { + return Err("local branch identity differs".into()); + } + let read = |id: &ObjectId, limit| { + super::LocalBranchState::read_object(&root, id, limit).map_err(|e| e.to_string()) + }; + let execution = msb_krun::ExecutionState::decode(&read( + &state.execution_state, + MAX_EXECUTION_STATE_BYTES, + )?) + .map_err(|e| e.to_string())?; + if execution.pause_generation() != state.pause_generation { + return Err("branch execution epoch differs".into()); + } + let devices = decode_devices(&state.devices, state.pause_generation, read)?; + let resource = state + .resources + .iter() + .find(|r| r.id == "guest:agentd") + .ok_or("branch has no captured agent identity")?; + let agent = parse_restored_agent_resource(resource, &state.id)?; + let file = state.memory.pin().map_err(|e| e.to_string())?; + let regions = state + .memory + .regions + .into_iter() + .map(|region| msb_krun::PrivateMemoryRegion { + guest_address: region.guest_address, + length: region.length, + file_offset: region.file_offset, + }) + .collect(); + let backing = + msb_krun::PrivateMemoryBacking::new(file, regions).map_err(|e| e.to_string())?; + Ok(Self { + execution, + devices, + memory: None, + local_memory: Some(backing), + agent, + }) + } + /// Resolve and decode every construction-time state envelope before building the VM. pub(crate) fn open(root: PathBuf, expected_root: &str) -> Result { let total_started = Instant::now(); @@ -89,50 +136,11 @@ impl PreparedCheckpointRestore { let execution_us = execution_started.elapsed().as_micros(); let devices_started = Instant::now(); - let mut devices = Vec::with_capacity(closure.checkpoint().devices.len()); - for device in &closure.checkpoint().devices { - let max_state_bytes = if device.device_type == TYPE_FS { - MAX_FS_DEVICE_STATE_BYTES - } else { - MAX_DEVICE_STATE_BYTES - }; - let bytes = closure - .read_object(&device.state, max_state_bytes) - .map_err(|error| format!("read checkpoint device {}: {error}", device.device_id))?; - if device.device_type == 2 { - let state = msb_krun::BlockDeviceState::decode(&bytes).map_err(|error| { - format!( - "decode checkpoint block device {}: {error}", - device.device_id - ) - })?; - if state.pause_generation != pause_generation { - return Err(format!( - "block device {} does not belong to the checkpoint epoch", - device.device_id - )); - } - devices.push(PreparedDeviceRestore::Block { - device_id: device.device_id.clone(), - state, - }); - } else { - let state = msb_krun::VirtioDeviceState::decode(&bytes).map_err(|error| { - format!( - "decode checkpoint virtio device {}: {error}", - device.device_id - ) - })?; - if state.pause_generation != pause_generation || state.device_id != device.device_id - { - return Err(format!( - "virtio device {} does not belong to the checkpoint binding/epoch", - device.device_id - )); - } - devices.push(PreparedDeviceRestore::Virtio(state)); - } - } + let devices = decode_devices( + &closure.checkpoint().devices, + pause_generation, + |id, limit| closure.read_object(id, limit).map_err(|e| e.to_string()), + )?; let devices_us = devices_started.elapsed().as_micros(); tracing::info!( target: "microsandbox_checkpoint_timing", @@ -151,7 +159,8 @@ impl PreparedCheckpointRestore { Ok(Self { execution, devices, - memory: CheckpointMemoryRestore { closure }, + memory: Some(CheckpointMemoryRestore { closure }), + local_memory: None, agent, }) } @@ -163,8 +172,14 @@ impl PreparedCheckpointRestore { cache_root: Option, ) -> Result { vm.set_execution_restore(self.execution); - if let Some(root) = cache_root { - let closure = &self.memory.closure; + if let Some(backing) = self.local_memory { + vm.set_private_memory_backing(backing); + } else if let Some(root) = cache_root { + let closure = &self + .memory + .as_ref() + .expect("durable restore memory") + .closure; let cache = super::MemoryCache::open(root).map_err(|e| e.to_string())?; let cached = cache .materialize(closure.memory(), &closure.checkpoint().memory, |id| { @@ -191,7 +206,7 @@ impl PreparedCheckpointRestore { .map_err(|e| e.to_string())?; vm.set_private_memory_backing(backing); } else { - vm.set_memory_restore(self.memory); + vm.set_memory_restore(self.memory.expect("durable restore memory")); } for device in self.devices { match device { @@ -301,6 +316,56 @@ impl msb_krun::VmMemoryRestoreSource for CheckpointMemoryRestore { // Functions //-------------------------------------------------------------------------------------------------- +fn decode_devices( + references: &[microsandbox_image::checkpoint::DeviceStateRef], + pause_generation: u64, + mut read: impl FnMut(&ObjectId, u64) -> Result, String>, +) -> Result, String> { + let mut devices = Vec::with_capacity(references.len()); + for device in references { + let max_state_bytes = if device.device_type == TYPE_FS { + MAX_FS_DEVICE_STATE_BYTES + } else { + MAX_DEVICE_STATE_BYTES + }; + let bytes = read(&device.state, max_state_bytes) + .map_err(|error| format!("read checkpoint device {}: {error}", device.device_id))?; + if device.device_type == 2 { + let state = msb_krun::BlockDeviceState::decode(&bytes).map_err(|error| { + format!( + "decode checkpoint block device {}: {error}", + device.device_id + ) + })?; + if state.pause_generation != pause_generation { + return Err(format!( + "block device {} does not belong to the checkpoint epoch", + device.device_id + )); + } + devices.push(PreparedDeviceRestore::Block { + device_id: device.device_id.clone(), + state, + }); + } else { + let state = msb_krun::VirtioDeviceState::decode(&bytes).map_err(|error| { + format!( + "decode checkpoint virtio device {}: {error}", + device.device_id + ) + })?; + if state.pause_generation != pause_generation || state.device_id != device.device_id { + return Err(format!( + "virtio device {} does not belong to the checkpoint binding/epoch", + device.device_id + )); + } + devices.push(PreparedDeviceRestore::Virtio(state)); + } + } + Ok(devices) +} + fn parse_restored_agent(closure: &CheckpointClosure) -> Result { let resource = closure .checkpoint() diff --git a/crates/runtime/lib/control.rs b/crates/runtime/lib/control.rs index 7400c7982..9469a04ca 100644 --- a/crates/runtime/lib/control.rs +++ b/crates/runtime/lib/control.rs @@ -42,6 +42,15 @@ pub const CONTROL_SOCKET_EXTENSION: &str = "control.sock"; #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(tag = "op", rename_all = "snake_case")] pub enum ControlRequest { + /// Capture directly into a reserved child-owned local handoff directory. + BranchCreate { + /// Unique capture identity matching the child's reservation. + branch_id: String, + /// Reserved sandbox name in this runtime's backend, never a host path. + child_name: String, + /// Cache in which the caller holds its handoff lock; must match the source runtime. + memory_cache_dir: PathBuf, + }, /// Retain a resident pause until an explicit resume or stop. Pause, /// Resume a user-owned resident pause. @@ -151,6 +160,9 @@ pub struct SecretValue(pub String); /// The reply to any control request. #[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct ControlResponse { + /// Completed local handoff, deliberately not a portable checkpoint identity. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub branch: Option, /// Resident pause status for lifecycle operations. #[serde(default, skip_serializing_if = "Option::is_none")] pub pause: Option, @@ -227,6 +239,9 @@ pub struct RootDiskGrowthResult { /// resize-capable and secrets-incapable. #[derive(Debug, Default, Clone, Copy, Serialize, Deserialize)] pub struct ControlCapabilities { + /// Direct local branch capture is supported on this host. + #[serde(default)] + pub branch_create: bool, /// Resident pause/resume with identity-preserving clock correction. #[serde(default)] pub pause_resume: bool, @@ -605,6 +620,7 @@ mod tests { let response = ControlResponse { ok: true, capabilities: Some(ControlCapabilities { + branch_create: true, pause_resume: true, root_disk_grow: true, disk_compact: true, diff --git a/crates/runtime/lib/control/executor.rs b/crates/runtime/lib/control/executor.rs index 52c876bab..71811854d 100644 --- a/crates/runtime/lib/control/executor.rs +++ b/crates/runtime/lib/control/executor.rs @@ -269,6 +269,7 @@ impl RuntimeControlExecutor { | ControlRequest::CpuTarget { .. } | ControlRequest::SecretsUpdate { .. } | ControlRequest::CheckpointCreate { .. } + | ControlRequest::BranchCreate { .. } | ControlRequest::DiskCompact { dry_run: false, .. } | ControlRequest::Pause | ControlRequest::Resume @@ -279,6 +280,7 @@ impl RuntimeControlExecutor { ControlRequest::Pause | ControlRequest::Resume | ControlRequest::CheckpointCreate { .. } + | ControlRequest::BranchCreate { .. } ); if mutation && state.lifecycle != RuntimeLifecycle::Running && !resident_operation { return control_error( @@ -364,6 +366,44 @@ impl RuntimeControlExecutor { } } } + ControlRequest::BranchCreate { + branch_id, + child_name, + memory_cache_dir, + } => { + state.lifecycle = RuntimeLifecycle::Quiescing; + match state.checkpoint.branch( + &self.vm, + &branch_id, + &child_name, + &memory_cache_dir, + state.user_pause.as_ref(), + ) { + Ok(result) => { + state.lifecycle = if state.user_pause.is_some() { + RuntimeLifecycle::Quiesced + } else { + RuntimeLifecycle::Running + }; + ControlResponse { + ok: true, + branch: Some(result.path), + ..Default::default() + } + } + Err(error) => { + if error.keep_paused { + state.user_pause = None; + } + state.lifecycle = if error.keep_paused || state.user_pause.is_some() { + RuntimeLifecycle::Quiesced + } else { + RuntimeLifecycle::Running + }; + control_error("branch_failed", error.to_string()) + } + } + } ControlRequest::CheckpointCreate { checkpoint_id, intent, @@ -476,6 +516,7 @@ impl RuntimeControlExecutor { memory_resize: self.vm.memory_resize_supported(), secrets_update: self.secrets_update_supported(), checkpoint_create: true, + branch_create: cfg!(unix), disk_compact: true, root_disk_grow: true, pause_resume: self.vm.clock_sync_supported(), @@ -498,6 +539,7 @@ impl RuntimeControlExecutor { ControlRequest::CpuState => cpu(self.vm.cpu_state()), ControlRequest::SecretsUpdate { changes } => self.handle_secrets_update(changes), ControlRequest::CheckpointCreate { .. } + | ControlRequest::BranchCreate { .. } | ControlRequest::Pause | ControlRequest::Resume | ControlRequest::PauseState diff --git a/crates/runtime/lib/launch.rs b/crates/runtime/lib/launch.rs index deb72eb13..a34f777ee 100644 --- a/crates/runtime/lib/launch.rs +++ b/crates/runtime/lib/launch.rs @@ -27,7 +27,11 @@ use crate::vm::{MetricsSlotHandoff, StartupCommand}; /// The bulk `msb sandbox` configuration delivered over the config fd. #[derive(Debug, Default, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct LaunchConfig { + /// Required execution intent. Restore intent must never be inferred from optional hints. + pub execution: ExecutionIntent, + /// Path to the sandbox database file. pub db_path: PathBuf, @@ -74,11 +78,7 @@ pub struct LaunchConfig { #[serde(default)] pub thp: TransparentHugePagePolicy, - /// Explicit memory representation selected at construction. - #[serde(default)] - pub memory_snapshot: microsandbox_types::MemorySnapshotMode, - - /// Backend-resolved protected cache; required only for explicit CoW construction. + /// Backend-resolved protected cache for explicit memory captures and restores. #[serde(default, skip_serializing_if = "Option::is_none")] pub memory_cache_dir: Option, @@ -146,6 +146,11 @@ pub struct LaunchConfig { #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct CheckpointRestoreConfig { + /// Restore a local branch handoff instead of a durable checkpoint closure. + pub local_branch: bool, + /// Require private CoW memory rather than eager restoration. + /// Kept inside the strict restore contract: an unsupported mode must not become a boot. + pub forked: bool, /// Path to the complete eager checkpoint closure. pub closure: PathBuf, /// Expected algorithm-qualified composite checkpoint root. @@ -154,6 +159,17 @@ pub struct CheckpointRestoreConfig { pub checkpoint_id: String, } +/// Required process-construction intent, independent of any guest startup command. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ExecutionIntent { + /// Construct a fresh guest from its disk/image state. + #[default] + Boot, + /// Continue captured execution; a complete recognized restore source is mandatory. + Restore, +} + /// Lifetime bounds for the sandbox. #[derive(Debug, Default, Clone, Serialize, Deserialize)] pub struct Lifecycle { @@ -238,3 +254,129 @@ pub struct RootfsUpperLayerConfig { /// On-disk format (`raw` or `qcow2`). pub format: String, } + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl LaunchConfig { + /// Decode and validate execution intent before allocating or starting a VM. + pub fn decode(bytes: &[u8]) -> Result { + let config: Self = serde_json::from_slice(bytes) + .map_err(|error| format!("invalid launch config: {error}"))?; + match (config.execution, config.checkpoint_restore.as_ref()) { + (ExecutionIntent::Boot, None) => {} + (ExecutionIntent::Restore, Some(restore)) => { + if restore.closure.as_os_str().is_empty() + || (!restore.local_branch && restore.checkpoint_root.is_empty()) + || restore.checkpoint_id.is_empty() + { + return Err("restore requires a complete checkpoint source".into()); + } + if restore.local_branch && (!restore.forked || !restore.checkpoint_root.is_empty()) + { + return Err( + "local branch requires private memory and no durable checkpoint root" + .into(), + ); + } + if config.startup.is_some() { + return Err("restore cannot execute a fresh startup command".into()); + } + } + _ => return Err("execution intent and checkpoint restore source disagree".into()), + } + Ok(config) + } +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + fn restore_request() -> serde_json::Value { + serde_json::to_value(LaunchConfig { + execution: ExecutionIntent::Restore, + checkpoint_restore: Some(CheckpointRestoreConfig { + local_branch: false, + forked: true, + closure: "/owned/child/restore".into(), + checkpoint_root: "blake3:captured-root".into(), + checkpoint_id: "captured".into(), + }), + ..Default::default() + }) + .unwrap() + } + + fn decode(value: serde_json::Value) -> Result { + LaunchConfig::decode(&serde_json::to_vec(&value).unwrap()) + } + + #[test] + fn matching_boot_and_restore_intents_are_accepted() { + assert!(decode(serde_json::to_value(LaunchConfig::default()).unwrap()).is_ok()); + let restored = decode(restore_request()).unwrap(); + assert!(restored.checkpoint_restore.unwrap().forked); + } + + #[test] + fn unsupported_or_missing_restore_never_becomes_boot() { + for mutation in [ + "missing", + "null", + "unknown_outer", + "unknown_nested", + "unknown_intent", + "boot", + ] { + let mut request = restore_request(); + match mutation { + "missing" => { + request + .as_object_mut() + .unwrap() + .remove("checkpoint_restore"); + } + "null" => request["checkpoint_restore"] = serde_json::Value::Null, + "unknown_outer" => request["branch_restore"] = serde_json::json!({}), + "unknown_nested" => request["checkpoint_restore"]["unsupported"] = true.into(), + "unknown_intent" => request["execution"] = "future_restore".into(), + "boot" => request["execution"] = "boot".into(), + _ => unreachable!(), + } + assert!(decode(request).is_err(), "{mutation}"); + } + } + + #[test] + fn launch_requires_explicit_intent_and_memory_policy() { + let mut request = restore_request(); + request.as_object_mut().unwrap().remove("execution"); + assert!(decode(request).is_err()); + let mut request = restore_request(); + request["checkpoint_restore"] + .as_object_mut() + .unwrap() + .remove("forked"); + assert!(decode(request).is_err()); + } + + #[test] + fn local_branch_requires_restore_and_private_memory_without_a_fake_root() { + let mut request = restore_request(); + request["checkpoint_restore"]["local_branch"] = true.into(); + assert!(decode(request.clone()).is_err()); + request["checkpoint_restore"]["checkpoint_root"] = "".into(); + assert!(decode(request.clone()).is_ok()); + request["checkpoint_restore"]["forked"] = false.into(); + assert!(decode(request.clone()).is_err()); + request["checkpoint_restore"]["forked"] = true.into(); + request["execution"] = "boot".into(); + assert!(decode(request).is_err()); + } +} diff --git a/crates/runtime/lib/vm.rs b/crates/runtime/lib/vm.rs index eaa623024..e42262983 100644 --- a/crates/runtime/lib/vm.rs +++ b/crates/runtime/lib/vm.rs @@ -280,9 +280,6 @@ pub struct VmConfig { /// Guest transparent huge-page policy selected at boot. pub thp: microsandbox_types::TransparentHugePagePolicy, - /// Explicit construction-time memory representation. - pub memory_snapshot: microsandbox_types::MemorySnapshotMode, - /// Protected memory cache resolved by the sandbox's owning local backend. pub memory_cache_dir: Option, @@ -2088,12 +2085,20 @@ fn build_vm( .build() .map_err(|e| RuntimeError::Custom(format!("build VM: {e}")))?; let restored_agent = if let Some(restore) = &config.vm.checkpoint_restore { - let prepared = crate::checkpoint::PreparedCheckpointRestore::open( - restore.closure.clone(), - &restore.checkpoint_root, - ) + let prepared = if restore.local_branch { + crate::checkpoint::PreparedCheckpointRestore::open_local( + restore.closure.clone(), + &restore.checkpoint_id, + ) + } else { + crate::checkpoint::PreparedCheckpointRestore::open( + restore.closure.clone(), + &restore.checkpoint_root, + ) + } .map_err(|error| RuntimeError::Custom(format!("prepare checkpoint restore: {error}")))?; - let cache_root = (config.vm.memory_snapshot == microsandbox_types::MemorySnapshotMode::Cow) + let cache_root = restore + .forked .then(|| { config.vm.memory_cache_dir.clone().ok_or_else(|| { RuntimeError::Custom( @@ -2112,9 +2117,6 @@ fn build_vm( }; let bootstrap_frame = if restored_agent.is_none() { - if config.vm.memory_snapshot == microsandbox_types::MemorySnapshotMode::Cow { - vm.set_private_memory_boot(true); - } Some(encode_bootstrap_frame(&bootstrap)?) } else { None diff --git a/docs/sandboxes/snapshots.mdx b/docs/sandboxes/snapshots.mdx index 91f79565a..fcfe3889b 100644 --- a/docs/sandboxes/snapshots.mdx +++ b/docs/sandboxes/snapshots.mdx @@ -25,18 +25,32 @@ Both modes produce the same schema-1 `snapshot.json` descriptor. Its closed `sta ## Share restored memory with CoW -On Linux and macOS, request `--memory-snapshot cow` when creating a sandbox to use private file-backed memory. Children restored from the same full snapshot can share clean memory pages; writes stay private to each child. Each child must opt in explicitly. The default, `standard`, keeps eager anonymous memory. +On Linux and macOS, add `--forked` when restoring a full snapshot to share clean memory pages between children. Writes stay private to each child. Without the flag, restore eagerly copies memory. The source needs no special creation option. ```bash -msb create alpine --name baseline --root-disk flat:1G --memory-snapshot cow +msb create alpine --name baseline --root-disk flat:1G msb snapshot create ready --from baseline --full -msb create --name worker-a --from-snapshot ready --memory-snapshot cow -msb create --name worker-b --from-snapshot ready --memory-snapshot cow +msb create --name worker-a --from-snapshot ready --forked +msb create --name worker-b --from-snapshot ready --forked ``` -The SDK creation options are Rust `.memory_snapshot(MemorySnapshotMode::Cow)`, Python `memory_snapshot=MemorySnapshotMode.COW`, TypeScript `.memorySnapshot("cow")`, and Go `WithMemorySnapshot(MemorySnapshotCow)`. +The SDK creation options are Rust `.forked()`, Python `forked=True`, TypeScript `.forked()`, and Go `WithForked()`. The option requires a full snapshot and cannot be combined with `--disk-only` or an image cold boot. -Snapshots are still manual. CoW uses a protected local memory cache; the first uncached restore must build it, while later children reuse it. Repeated captures use filesystem reflinks when available. Removing the input archive does not invalidate a running child's backing. Windows CoW and explicit NUMA placement with CoW are not supported yet; requests fail instead of silently switching modes. +Snapshots are still manual. CoW uses a protected local memory cache; the first uncached restore must build it, while later children reuse it. Captures from forked children can prepare later restore backing with filesystem reflinks when available. Removing the input archive does not invalidate a running child's backing. Windows CoW and explicit NUMA placement with CoW are not supported yet; requests fail instead of silently switching modes. + +## Branch a running sandbox + +Use `branch` for an independent local child without first saving a full snapshot: + +```bash +msb branch baseline --name worker +``` + +The child resumes the captured processes, memory, and disk state. Its writes do not affect the source. CoW memory is built in—no `--forked` flag is needed. A running source resumes after capture; a user-paused source stays paused. Branches work on supported Linux and macOS hosts, require a new child name, and cannot inherit published host ports. + +The SDK methods are Rust `source.branch("worker").await?`, Python `await source.branch("worker")`, TypeScript `await source.branch("worker")`, and Go `source.Branch(ctx, "worker")`. They also work on sandbox handles returned by `get`. + +Local branching still writes a consistent memory backing file, but skips portable RAM packaging and snapshot registration. It is not a saved recovery point. Use `snapshot create --full` when you need a durable, exportable snapshot. ## Quick start diff --git a/packages/microsandbox-types/rust/lib/cloud.rs b/packages/microsandbox-types/rust/lib/cloud.rs index 941ca70aa..d69516a8b 100644 --- a/packages/microsandbox-types/rust/lib/cloud.rs +++ b/packages/microsandbox-types/rust/lib/cloud.rs @@ -972,7 +972,6 @@ impl TryFrom for SandboxSpec { }; let resources = SandboxResources { - memory_snapshot: crate::MemorySnapshotMode::Standard, cpus: spec.resources.vcpus, memory_mib: spec.resources.memory_mib, // The cloud wire type has no boot-capacity fields yet; treat the diff --git a/packages/microsandbox-types/rust/lib/domain.rs b/packages/microsandbox-types/rust/lib/domain.rs index 39ca7f978..686c370e9 100644 --- a/packages/microsandbox-types/rust/lib/domain.rs +++ b/packages/microsandbox-types/rust/lib/domain.rs @@ -832,9 +832,6 @@ pub struct SandboxSpec { #[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] #[cfg_attr(feature = "ts", derive(ts_rs::TS))] pub struct SandboxResources { - /// Explicit construction-time memory representation; does not create automatic snapshots. - #[serde(default, skip_serializing_if = "MemorySnapshotMode::is_standard")] - pub memory_snapshot: MemorySnapshotMode, /// Number of virtual CPUs currently presented to the guest at boot. pub cpus: u8, @@ -860,19 +857,6 @@ pub struct SandboxResources { pub thp: TransparentHugePagePolicy, } -/// Memory representation selected when constructing a sandbox. -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -#[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] -#[cfg_attr(feature = "ts", derive(ts_rs::TS))] -pub enum MemorySnapshotMode { - /// Anonymous memory with eager full restore. - #[default] - Standard, - /// Private file-backed memory, sharing immutable pages between restored children. - Cow, -} - /// Controls how Microsandbox places vCPU threads on host processors. #[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] @@ -1541,17 +1525,9 @@ impl Default for RootfsSource { } } -impl MemorySnapshotMode { - /// Whether the default anonymous representation was selected. - pub fn is_standard(&self) -> bool { - *self == Self::Standard - } -} - impl Default for SandboxResources { fn default() -> Self { Self { - memory_snapshot: MemorySnapshotMode::Standard, cpus: DEFAULT_SANDBOX_CPUS, memory_mib: DEFAULT_SANDBOX_MEMORY_MIB, max_cpus: DEFAULT_SANDBOX_CPUS, @@ -1582,8 +1558,6 @@ impl<'de> Deserialize<'de> for SandboxResources { placement_profile: Option, #[serde(default)] thp: TransparentHugePagePolicy, - #[serde(default)] - memory_snapshot: MemorySnapshotMode, } let raw = RawResources::deserialize(deserializer)?; @@ -1598,7 +1572,6 @@ impl<'de> Deserialize<'de> for SandboxResources { cpu_placement: raw.cpu_placement, placement_profile: raw.placement_profile, thp: raw.thp, - memory_snapshot: raw.memory_snapshot, }) } } @@ -2886,20 +2859,6 @@ impl fmt::Display for NetworkRateLimitDirection { mod tests { use super::*; - #[test] - fn memory_snapshot_policy_is_explicit_and_old_resources_default_to_standard() { - let standard = serde_json::to_value(SandboxResources::default()).unwrap(); - assert!(standard.get("memory_snapshot").is_none()); - let decoded: SandboxResources = serde_json::from_value(standard.clone()).unwrap(); - assert_eq!(decoded.memory_snapshot, MemorySnapshotMode::Standard); - let mut cow = standard; - cow["memory_snapshot"] = serde_json::json!("cow"); - let decoded: SandboxResources = serde_json::from_value(cow.clone()).unwrap(); - assert_eq!(decoded.memory_snapshot, MemorySnapshotMode::Cow); - cow["memory_snapshot"] = serde_json::json!("automatic"); - assert!(serde_json::from_value::(cow).is_err()); - } - fn tmpfs_mount(guest: &str) -> VolumeMount { VolumeMount::Tmpfs { guest: guest.to_owned(), diff --git a/packages/microsandbox-types/rust/lib/lib.rs b/packages/microsandbox-types/rust/lib/lib.rs index 9af47f799..99f691511 100644 --- a/packages/microsandbox-types/rust/lib/lib.rs +++ b/packages/microsandbox-types/rust/lib/lib.rs @@ -31,16 +31,16 @@ pub use domain::{ DEFAULT_SANDBOX_CPUS, DEFAULT_SANDBOX_MEMORY_MIB, DeploymentProfile, Destination, DestinationGroup, Direction, DiskImageFormat, DnsConfig, EnvVar, FlatClone, HandoffInit, HostPattern, HostPermissions, InterceptCaConfig, InterfaceOverrides, LogSource, - MAX_SECRET_PLACEHOLDER_BYTES, MemoryPlacement, MemorySnapshotMode, MountOptions, - NamedVolumeCreate, NamedVolumeMode, NetworkPolicy, NetworkRateLimitDirection, - NetworkRateLimiterConfig, NetworkSpec, NumaPlacement, OciRootfsSource, Patch, PlacementProfile, - PortProtocol, PortRange, Protocol, PublishedPortSpec, PullPolicy, RateLimitConfigError, - RateLimiterConfig, Rlimit, RlimitResource, RootDisk, RootfsSource, Rule, SandboxLogLevel, - SandboxPolicy, SandboxResources, SandboxRuntimeOptions, SandboxSpec, ScopedUpstreamCaCert, - ScopedVerifyUpstream, SecretConfigError, SecretEntry, SecretInjection, SecretsConfig, - SecurityProfile, SnapshotSpec, StatVirtualization, TlsConfig, TokenBucketConfig, - TransparentHugePagePolicy, ViolationAction, VolumeKind, VolumeMount, VolumeSpec, - VsockRouteSpec, VsockSocketType, VsockSpec, canonicalize_volume_mounts, + MAX_SECRET_PLACEHOLDER_BYTES, MemoryPlacement, MountOptions, NamedVolumeCreate, + NamedVolumeMode, NetworkPolicy, NetworkRateLimitDirection, NetworkRateLimiterConfig, + NetworkSpec, NumaPlacement, OciRootfsSource, Patch, PlacementProfile, PortProtocol, PortRange, + Protocol, PublishedPortSpec, PullPolicy, RateLimitConfigError, RateLimiterConfig, Rlimit, + RlimitResource, RootDisk, RootfsSource, Rule, SandboxLogLevel, SandboxPolicy, SandboxResources, + SandboxRuntimeOptions, SandboxSpec, ScopedUpstreamCaCert, ScopedVerifyUpstream, + SecretConfigError, SecretEntry, SecretInjection, SecretsConfig, SecurityProfile, SnapshotSpec, + StatVirtualization, TlsConfig, TokenBucketConfig, TransparentHugePagePolicy, ViolationAction, + VolumeKind, VolumeMount, VolumeSpec, VsockRouteSpec, VsockSocketType, VsockSpec, + canonicalize_volume_mounts, }; pub use error::{TypesError, TypesResult}; pub use modify::{ diff --git a/scripts/smoke/cli/branch-ownership.py b/scripts/smoke/cli/branch-ownership.py new file mode 100644 index 000000000..8040feda5 --- /dev/null +++ b/scripts/smoke/cli/branch-ownership.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Live Unix pin lifetime and same-name reservation checks for direct branching.""" + +import fcntl +import json +import os +from pathlib import Path +import subprocess + +binary = os.environ["MSB_PATH"] +home = Path(os.environ["MSB_HOME"]) +prefix = f"branch-own-{os.getpid()}" +names = [prefix, prefix + ".child", prefix + ".race"] + + +def call(*args, expected=0): + result = subprocess.run([binary, *args], capture_output=True, text=True, timeout=120) + if expected is not None: + assert result.returncode == expected, result.stderr + return result + + +def evictable(path): + # Same OS primitive used by production eviction. Never unlink or modify live backing. + with path.open("rb") as file: + try: + fcntl.flock(file, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + return False + return True + + +try: + call("create", "alpine", "--name", prefix, "--root-disk", "tmpfs:128M", "--memory", "256M") + cache = home / "cache" / "memory" / "branches" + before = set(cache.glob("*.ram")) + call("branch", prefix, "--name", names[1]) + paths = set(cache.glob("*.ram")) - before + assert len(paths) == 1 + backing = paths.pop() + assert backing.stat().st_mode & 0o777 == 0o400 + assert not evictable(backing), "source/child pins disappeared" + assert not (home / "sandboxes" / names[1] / ".branch-restore").exists() + attempts = [subprocess.Popen([binary, "branch", prefix, "--name", names[2]], stdout=subprocess.PIPE, stderr=subprocess.PIPE) for _ in range(2)] + statuses = [] + for attempt in attempts: + attempt.communicate(timeout=120) + statuses.append(attempt.returncode) + assert sorted(statuses) == [0, 1], statuses + call("stop", prefix) + assert not evictable(backing), "child depended on the source's pin" + call("exec", names[1], "--", "true") + call("stop", names[1]) + assert evictable(backing), "pin leaked after final VM teardown" + print(json.dumps({"independent_child_pin": "pass", "release_after_teardown": "pass", "same_name_race": "pass", "dot_name": "pass"})) +finally: + for name in reversed(names): + call("stop", name, expected=None) diff --git a/scripts/smoke/cli/checkpoint-clock.py b/scripts/smoke/cli/checkpoint-clock.py index 55b65f667..2979ac7a3 100644 --- a/scripts/smoke/cli/checkpoint-clock.py +++ b/scripts/smoke/cli/checkpoint-clock.py @@ -35,7 +35,6 @@ def run(label, *args, check=True, timeout=120): run("create", "run", "-d", "-n", source, "--root-disk", os.environ.get("CLOCK_LAYOUT", "flat:512M"), "--cpus", os.environ.get("CLOCK_CPUS", "2"), "--memory", "256M", - "--memory-snapshot", os.environ.get("CLOCK_MEMORY", "standard"), "alpine", "--", "sh", "-c", "while [ ! -x /clock-probe ]; do sleep 0.05; done; exec /clock-probe") run("copy-probe", "copy", os.environ["CLOCK_PROBE"], source + ":/clock-probe") @@ -58,7 +57,7 @@ def run(label, *args, check=True, timeout=120): time.sleep(float(os.environ.get("CLOCK_DELAY", "8"))) (out / "restore-start.ns").write_text(str(time.time_ns())) run("restore", "create", "-n", child, "--from-snapshot", snapshot, - "--memory-snapshot", os.environ.get("CLOCK_MEMORY", "standard"), "--info") + *(["--forked"] if os.environ.get("CLOCK_FORKED") == "1" else []), "--info") (out / "restore-end.ns").write_text(str(time.time_ns())) time.sleep(6) records = run("records", "exec", child, "--", "cat", "/tmp/clock-records.csv") diff --git a/scripts/smoke/cli/cow-memory-lifecycle.py b/scripts/smoke/cli/cow-memory-lifecycle.py index aa16b1f9b..094075887 100644 --- a/scripts/smoke/cli/cow-memory-lifecycle.py +++ b/scripts/smoke/cli/cow-memory-lifecycle.py @@ -10,7 +10,9 @@ root = Path(os.environ["STACK8_OUT"]) root.mkdir(parents=True, exist_ok=True) prefix = os.environ.get("STACK8_PREFIX", "cow8") -mode = os.environ.get("STACK8_MODE", "cow") +mode = os.environ.get("STACK8_MODE", "forked") +assert mode in ("forked", "eager") +restore_flags = ["--forked"] if mode == "forked" else [] layout = os.environ.get("STACK8_LAYOUT", "flat:512M") resize = os.environ.get("STACK8_LIVE_RESIZE") == "1" rows = [] @@ -44,19 +46,13 @@ def run(label, *args, expected=0, timeout=120): return result try: - if os.environ.get("STACK8_CHECK_COW_REJECTION") == "1": - refused = prefix + "-cow-refused" - names.append(refused) - result = run("cow-unsupported", "create", "alpine", "-n", refused, - "--memory", "256M", "--memory-snapshot", "cow", expected=None) - assert result.returncode != 0, "unsupported CoW must not silently start eagerly" - assert "not qualified" in result.stderr or "not qualified" in result.stdout - inspected = run("cow-refused-inspect", "inspect", refused, "--format", "json", expected=None) - if inspected.returncode == 0: - assert json.loads(inspected.stdout)["status"] not in ("Running", "Paused") + refused = prefix + "-forked-boot" + result = run("forked-boot-rejected", "create", "alpine", "-n", refused, + "--forked", expected=None) + assert result.returncode != 0, "forked must require captured RAM" source = prefix + "-source" names.append(source) - run("fresh-" + mode, "run", "-d", "-n", source, "--memory-snapshot", mode, + run("fresh-" + mode, "run", "-d", "-n", source, "--root-disk", layout, "--memory", "256M", "--cpus", "2", *(["--max-memory", "512M"] if resize else []), "alpine", "--", "sh", "-c", "mkdir -p /dev/shm; echo captured > /dev/shm/cow-marker; i=0; while :; do echo $i > /tmp/cow-counter; i=$((i+1)); sleep 0.05; done") @@ -115,24 +111,45 @@ def run(label, *args, expected=0, timeout=120): child = prefix + "-" + suffix names.append(child) run("restore-" + suffix, "create", "-n", child, "--from-snapshot", snap, - "--memory-snapshot", mode, "--info") + *restore_flags, "--info") result = run("marker-" + suffix, "exec", child, "--", "cat", "/dev/shm/cow-marker") assert result.stdout.strip() == "captured" run("mutate-a", "exec", prefix + "-a", "--", "sh", "-c", "echo private-a > /dev/shm/cow-marker") assert run("isolation-b", "exec", prefix + "-b", "--", "cat", "/dev/shm/cow-marker").stdout.strip() == "captured" assert run("isolation-source", "exec", source, "--", "cat", "/dev/shm/cow-marker").stdout.strip() == "captured" + # A restored child remains a normal capture source; no creation-time memory opt-in exists. + child_snapshot = prefix + "-child-full" + run("capture-restored-child", "snapshot", "create", child_snapshot, + "--from", prefix + "-a", "--full", "--info") + grandchild = prefix + "-grandchild" + names.append(grandchild) + run("restore-grandchild", "create", "-n", grandchild, "--from-snapshot", child_snapshot, + *restore_flags, "--info") + assert run("grandchild-marker", "exec", grandchild, "--", "cat", "/dev/shm/cow-marker").stdout.strip() == "private-a" archive = str(root / "direct.msnap") run("direct-full", "snapshot", "create", prefix + "-direct", "--from", source, "--full", "--archive", archive, "--info") child = prefix + "-archive" names.append(child) run("direct-restore", "create", "-n", child, "--from-snapshot", archive, - "--memory-snapshot", mode, "--info") + *restore_flags, "--info") if os.environ.get("STACK8_KEEP_ARCHIVE") != "1": Path(archive).unlink() assert run("archive-child-exec", "exec", child, "--", "cat", "/dev/shm/cow-marker").stdout.strip() == "captured" run("pause-for-stop", "pause", source) run("stop-paused", "stop", source, timeout=20) + disk_snapshot = prefix + "-disk" + run("stopped-disk-capture", "snapshot", "create", disk_snapshot, "--from", source) + disk_archive = str(root / "disk.msnap") + run("disk-archive", "snapshot", "save", disk_snapshot, disk_archive) + for label, snapshot in (("installed", disk_snapshot), ("archive", disk_archive)): + refused_name = prefix + "-refused-" + label + names.append(refused_name) + result = run("forked-disk-" + label, "create", "-n", refused_name, + "--from-snapshot", snapshot, "--forked", expected=None) + assert result.returncode != 0 and "forked requires a full snapshot" in result.stderr + inspected = run("refused-inspect-" + label, "inspect", refused_name, "--format", "json", expected=None) + assert inspected.returncode != 0, "invalid restore must not publish a sandbox row" assert run("child-after-source-stop", "exec", prefix + "-a", "--", "cat", "/dev/shm/cow-marker").stdout.strip() == "private-a" finally: for name in reversed(names): diff --git a/scripts/smoke/cli/direct-branch.py b/scripts/smoke/cli/direct-branch.py new file mode 100644 index 000000000..0d98630d7 --- /dev/null +++ b/scripts/smoke/cli/direct-branch.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +"""Opt-in direct branch invariants and release timing; stops every test VM.""" + +import json +import os +from pathlib import Path +import subprocess +import time + +binary = os.environ["MSB_PATH"] +root = Path(os.environ["STACK8_OUT"]) +root.mkdir(parents=True, exist_ok=True) +prefix = os.environ.get("STACK8_PREFIX", f"branch8-{os.getpid()}") +layout = os.environ.get("STACK8_LAYOUT", "flat:512M") +rows = [] +names = [] + + +def run(label, *args, expected=0): + started = time.perf_counter() + result = subprocess.run([binary, *args], text=True, capture_output=True, timeout=120) + elapsed = (time.perf_counter() - started) * 1000 + (root / f"{label}.stdout").write_text(result.stdout) + (root / f"{label}.stderr").write_text(result.stderr) + row = {"case": label, "ms": round(elapsed, 2), "exit": result.returncode} + rows.append(row) + print(json.dumps(row), flush=True) + if expected is not None and result.returncode != expected: + raise RuntimeError(f"{label}: {result.stderr[-4000:]}") + return result + + +def exec_guest(name, script, label): + return run(label, "exec", name, "--", "sh", "-c", script).stdout.strip() + + +def branch(source, child, label): + names.append(child) + run(label, "branch", source, "--name", child) + assert exec_guest(child, "cat /dev/shm/branch-marker", label + "-ready") == "source" + + +def benchmark(source): + # One source and at most one measured child: do not let accumulating VMs distort later + # samples. Each CLI return includes activation; first guest command is recorded separately. + for i in range(10): + if os.environ.get("STACK8_BENCH_COMPACT") == "1" and i > 1: + run(f"setup-compact-branch-{i}", "modify", source, "--compact", "--format", "json") + child = prefix + f"-bench-{i}" + branch(source, child, f"branch-{i}") + run(f"stop-branch-{i}", "stop", child) + saved = prefix + "-warm" + if os.environ.get("STACK8_BENCH_COMPACT") == "1": + run("setup-compact-snapshot", "modify", source, "--compact", "--format", "json") + run("capture-warm-source", "snapshot", "create", saved, "--from", source, "--full") + for mode in ("forked", "eager"): + for i in range(8): + child = prefix + f"-{mode}-{i}" + names.append(child) + run(f"{mode}-restore-{i}", "create", "--name", child, "--from-snapshot", saved, + *(["--forked"] if mode == "forked" else [])) + assert exec_guest(child, "cat /dev/shm/branch-marker", f"{mode}-ready-{i}") == "source" + run(f"stop-{mode}-{i}", "stop", child) + for i in range(5): + if os.environ.get("STACK8_BENCH_COMPACT") == "1": + run(f"setup-compact-pipeline-{i}", "modify", source, "--compact", "--format", "json") + saved = prefix + f"-full-{i}" + child = prefix + f"-durable-{i}" + names.append(child) + run(f"pipeline-capture-{i}", "snapshot", "create", saved, "--from", source, "--full") + run(f"pipeline-restore-{i}", "create", "--name", child, "--from-snapshot", saved, "--forked") + assert exec_guest(child, "cat /dev/shm/branch-marker", f"pipeline-ready-{i}") == "source" + run(f"stop-pipeline-{i}", "stop", child) + + +try: + source = prefix + "-source" + names.append(source) + run("boot", "create", "alpine", "--name", source, "--root-disk", layout, + "--memory", "256M", "--cpus", "2") + exec_guest(source, "echo source > /dev/shm/branch-marker; echo source > /disk-marker; sh -c 'i=0; while :; do i=$((i+1)); echo $i > /dev/shm/branch-counter; sleep 0.02; done' >/tmp/branch-counter.log 2>&1 /dev/shm/branch-marker; echo child > /disk-marker", "mutate-child") + assert exec_guest(source, "cat /dev/shm/branch-marker; cat /disk-marker", "source-isolation") == "source\nsource" + assert exec_guest(prefix + "-repeat-0", "cat /dev/shm/branch-marker; cat /disk-marker", "sibling-isolation") == "source\nsource" + # Branch a branch after private writes; capturing its original file would lose these writes. + grandchild = prefix + "-grandchild" + names.append(grandchild) + run("branch-child", "branch", child, "--name", grandchild) + assert exec_guest(grandchild, "cat /dev/shm/branch-marker; cat /disk-marker", "grandchild-private-writes") == "child\nchild" + counter = int(exec_guest(grandchild, "cat /dev/shm/branch-counter", "counter-before")) + time.sleep(0.1) + assert int(exec_guest(grandchild, "cat /dev/shm/branch-counter", "counter-after")) > counter + run("pause-source", "pause", source) + branch(source, prefix + "-paused", "paused-branch") + rejected = run("source-still-paused", "exec", source, "--", "true", expected=None) + assert rejected.returncode != 0 + run("resume-source", "resume", source) + # Compare durable capture+forked-child against the same source and readiness endpoint. + for i in range(3): + snap = prefix + f"-saved-{i}" + run(f"full-capture-{i}", "snapshot", "create", snap, "--from", source, "--full") + name = prefix + f"-restored-{i}" + names.append(name) + run(f"forked-restore-{i}", "create", "--name", name, "--from-snapshot", snap, "--forked") + assert exec_guest(name, "cat /dev/shm/branch-marker", f"restore-ready-{i}") == "source" + if os.environ.get("STACK8_MAINTENANCE") == "1" and not layout.startswith("tmpfs"): + run("grow-source", "modify", source, "--root-disk", "768M", "--format", "json") + run("compact-source", "modify", source, "--compact", "--format", "json") + branch(source, prefix + "-after-grow", "branch-after-grow") + run("stop-source", "stop", source) + run("stop-child", "stop", child) + assert exec_guest(grandchild, "cat /dev/shm/branch-marker; cat /disk-marker", "survives-source-stop") == "child\nchild" +finally: + for name in reversed(names): + run("cleanup-" + name, "stop", name, expected=None) + (root / "results.json").write_text(json.dumps(rows, indent=2)) diff --git a/sdk/go/cow_lifecycle_test.go b/sdk/go/cow_lifecycle_test.go index c79b24ca7..f54778330 100644 --- a/sdk/go/cow_lifecycle_test.go +++ b/sdk/go/cow_lifecycle_test.go @@ -19,7 +19,7 @@ func TestCowResidentCapture(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() name := fmt.Sprintf("cow8-go-%d", os.Getpid()) - source, err := CreateSandbox(ctx, name, WithImage("alpine"), WithRootDisk(RootDisk.Managed(512)), WithMemory(256), WithMemorySnapshot(MemorySnapshotCow)) + source, err := CreateSandbox(ctx, name, WithImage("alpine"), WithRootDisk(RootDisk.Managed(512)), WithMemory(256)) if err != nil { t.Fatal(err) } @@ -41,13 +41,30 @@ func TestCowResidentCapture(t *testing.T) { if paused.Status() != SandboxStatusPaused { t.Fatalf("got status %s", paused.Status()) } + branched, err := paused.Branch(ctx, name+"-paused-branch") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := branched.Stop(context.Background()); err != nil { + t.Error(err) + } + branched.Close() + }) + branchResult, err := branched.Exec(ctx, "cat", []string{"/dev/shm/sdk-marker"}) + if err != nil { + t.Fatal(err) + } + if strings.TrimSpace(branchResult.Stdout()) != "source" { + t.Fatal("branch lost captured RAM") + } if _, err := Snapshot.Create(ctx, SnapshotCreateOptions{Name: name + "-full", FromSandbox: name, Full: true}); err != nil { t.Fatal(err) } if err := paused.Resume(ctx); err != nil { t.Fatal(err) } - child, err := CreateSandbox(ctx, name+"-child", WithFromSnapshot(name+"-full"), WithMemorySnapshot(MemorySnapshotCow)) + child, err := CreateSandbox(ctx, name+"-child", WithFromSnapshot(name+"-full"), WithForked()) if err != nil { t.Fatal(err) } @@ -76,4 +93,21 @@ func TestCowResidentCapture(t *testing.T) { if err := child.Pause(ctx); err != nil { t.Fatal(err) } + descendant, err := child.Branch(ctx, name+"-branch") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := descendant.Stop(context.Background()); err != nil { + t.Error(err) + } + descendant.Close() + }) + branchResult, err = descendant.Exec(ctx, "cat", []string{"/dev/shm/sdk-marker"}) + if err != nil { + t.Fatal(err) + } + if strings.TrimSpace(branchResult.Stdout()) != "child" { + t.Fatal("branch lost private writes") + } } diff --git a/sdk/go/internal/ffi/ffi.go b/sdk/go/internal/ffi/ffi.go index 26a81f973..dee9de114 100644 --- a/sdk/go/internal/ffi/ffi.go +++ b/sdk/go/internal/ffi/ffi.go @@ -122,6 +122,7 @@ typedef char *(*msb_sandbox_detach_fn)(uint64_t cancel_id, uint64_t handle, uint typedef char *(*msb_sandbox_stop_fn)(uint64_t cancel_id, uint64_t handle, uint64_t timeout_ms, uint8_t *buf, size_t buf_len); typedef char *(*msb_sandbox_request_stop_fn)(uint64_t cancel_id, uint64_t handle, uint8_t *buf, size_t buf_len); typedef char *(*msb_sandbox_pause_fn)(uint64_t cancel_id, uint64_t handle, uint8_t *buf, size_t buf_len); +typedef char *(*msb_sandbox_branch_fn)(uint64_t cancel_id, uint64_t handle, const char *source, const char *child, uint8_t *buf, size_t buf_len); typedef char *(*msb_sandbox_resume_fn)(uint64_t cancel_id, uint64_t handle, uint8_t *buf, size_t buf_len); typedef char *(*msb_sandbox_handle_pause_fn)(uint64_t cancel_id, const char *name, uint8_t *buf, size_t buf_len); typedef char *(*msb_sandbox_handle_resume_fn)(uint64_t cancel_id, const char *name, uint8_t *buf, size_t buf_len); @@ -278,6 +279,7 @@ static msb_sandbox_detach_fn ptr_msb_sandbox_detach = NULL; static msb_sandbox_stop_fn ptr_msb_sandbox_stop = NULL; static msb_sandbox_request_stop_fn ptr_msb_sandbox_request_stop = NULL; static msb_sandbox_pause_fn ptr_msb_sandbox_pause = NULL; +static msb_sandbox_branch_fn ptr_msb_sandbox_branch = NULL; static msb_sandbox_resume_fn ptr_msb_sandbox_resume = NULL; static msb_sandbox_handle_pause_fn ptr_msb_sandbox_handle_pause = NULL; static msb_sandbox_handle_resume_fn ptr_msb_sandbox_handle_resume = NULL; @@ -460,6 +462,7 @@ const char *load_microsandbox(const char *path) { RESOLVE(msb_sandbox_stop); RESOLVE(msb_sandbox_request_stop); RESOLVE(msb_sandbox_pause); + RESOLVE(msb_sandbox_branch); RESOLVE(msb_sandbox_resume); RESOLVE(msb_sandbox_handle_pause); RESOLVE(msb_sandbox_handle_resume); @@ -663,6 +666,9 @@ char *call_msb_sandbox_request_stop(uint64_t cancel_id, uint64_t handle, uint8_t char *call_msb_sandbox_pause(uint64_t cancel_id, uint64_t handle, uint8_t *buf, size_t buf_len) { return ptr_msb_sandbox_pause ? ptr_msb_sandbox_pause(cancel_id, handle, buf, buf_len) : NULL; } +char *call_msb_sandbox_branch(uint64_t cancel_id, uint64_t handle, const char *source, const char *child, uint8_t *buf, size_t buf_len) { + return ptr_msb_sandbox_branch ? ptr_msb_sandbox_branch(cancel_id, handle, source, child, buf, buf_len) : NULL; +} char *call_msb_sandbox_resume(uint64_t cancel_id, uint64_t handle, uint8_t *buf, size_t buf_len) { return ptr_msb_sandbox_resume ? ptr_msb_sandbox_resume(cancel_id, handle, buf, buf_len) : NULL; } @@ -1596,7 +1602,7 @@ type CreateOptions struct { CPUPlacement string `json:"cpu_placement,omitempty"` PlacementProfile string `json:"placement_profile,omitempty"` THP string `json:"thp,omitempty"` - MemorySnapshot string `json:"memory_snapshot,omitempty"` + Forked bool `json:"forked,omitempty"` Workdir string `json:"workdir,omitempty"` Shell string `json:"shell,omitempty"` SecurityProfile string `json:"security_profile,omitempty"` @@ -2318,6 +2324,44 @@ func (s *Sandbox) RequestStop(ctx context.Context) error { return err } +// Branch creates an independent local child through the host runtime. +func (s *Sandbox) Branch(ctx context.Context, name string) (*Sandbox, error) { + return branchSandbox(ctx, uint64(s.h()), s.name, name) +} + +// BranchSandboxByName branches execution without an agent connection to the source. +func BranchSandboxByName(ctx context.Context, source, name string) (*Sandbox, error) { + return branchSandbox(ctx, 0, source, name) +} + +func branchSandbox(ctx context.Context, handle uint64, source, name string) (*Sandbox, error) { + if err := ensureLoaded(); err != nil { + return nil, err + } + cSource, cName := C.CString(source), C.CString(name) + defer C.free(unsafe.Pointer(cSource)) + defer C.free(unsafe.Pointer(cName)) + out, err := call(ctx, func(cancelID C.uint64_t, buf *C.uint8_t, bufLen C.size_t) *C.char { + return C.call_msb_sandbox_branch(cancelID, C.uint64_t(handle), cSource, cName, buf, bufLen) + }) + if err != nil { + return nil, err + } + var resp struct { + Handle uint64 `json:"handle"` + BackendKind string `json:"backend_kind"` + } + if err := json.Unmarshal([]byte(out), &resp); err != nil { + if h := salvageHandle(out); h != 0 { + releaseHandle(h) + } + return nil, fmt.Errorf("parse branch response: %w", err) + } + s := &Sandbox{name: name, backendKind: resp.BackendKind} + s.handle.Store(resp.Handle) + return s, nil +} + // Pause controls resident execution through the host runtime. func (s *Sandbox) Pause(ctx context.Context) error { if err := ensureLoaded(); err != nil { diff --git a/sdk/go/native/microsandbox_go_ffi.h b/sdk/go/native/microsandbox_go_ffi.h index ff28b9a95..45882e689 100644 --- a/sdk/go/native/microsandbox_go_ffi.h +++ b/sdk/go/native/microsandbox_go_ffi.h @@ -160,6 +160,16 @@ char *msb_sandbox_stop(uint64_t cancel_id, char *msb_sandbox_pause(uint64_t cancel_id, Handle handle, unsigned char *buf, uintptr_t buf_len); +/** + * Branch by live handle, or by persisted name when handle is zero. + */ +char *msb_sandbox_branch(uint64_t cancel_id, + Handle handle, + const char *source, + const char *child, + unsigned char *buf, + uintptr_t buf_len); + char *msb_sandbox_resume(uint64_t cancel_id, Handle handle, unsigned char *buf, uintptr_t buf_len); char *msb_sandbox_request_stop(uint64_t cancel_id, diff --git a/sdk/go/native/src/lib.rs b/sdk/go/native/src/lib.rs index bb85824f1..9290d6806 100644 --- a/sdk/go/native/src/lib.rs +++ b/sdk/go/native/src/lib.rs @@ -1016,7 +1016,7 @@ struct SandboxCreateOpts { cpu_placement: Option, placement_profile: Option, thp: Option, - memory_snapshot: Option, + forked: Option, workdir: Option, shell: Option, env: Option>, @@ -2227,8 +2227,8 @@ pub unsafe extern "C" fn msb_sandbox_create( .map_err(FfiError::invalid_argument)?; builder = builder.thp(policy); } - if let Some(mode) = opts.memory_snapshot { - builder = builder.memory_snapshot(mode); + if opts.forked.unwrap_or(false) { + builder = builder.forked(); } if let Some(w) = opts.workdir { builder = builder.workdir(w); @@ -2933,6 +2933,42 @@ pub unsafe extern "C" fn msb_sandbox_pause( }) } +/// Branch by live handle, or by persisted name when handle is zero. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn msb_sandbox_branch( + cancel_id: u64, + handle: Handle, + source: *const c_char, + child: *const c_char, + buf: *mut c_uchar, + buf_len: usize, +) -> *mut c_char { + run_c(cancel_id, buf, buf_len, || { + let child = unsafe { cstr(child) }?; + let source = unsafe { cstr(source) }?; + let live = if handle == 0 { + None + } else { + Some(get(handle)?) + }; + Ok(Box::pin(async move { + let sb = if let Some(live) = live { + live.branch(child).await.map_err(FfiError::from)? + } else { + Sandbox::get(&source) + .await + .map_err(FfiError::from)? + .branch(child) + .await + .map_err(FfiError::from)? + }; + let backend_kind = sb.backend_kind().as_str(); + let handle = register(sb)?; + Ok(serde_json::json!({ "handle": handle, "backend_kind": backend_kind }).to_string()) + })) + }) +} + #[unsafe(no_mangle)] pub unsafe extern "C" fn msb_sandbox_resume( cancel_id: u64, diff --git a/sdk/go/options.go b/sdk/go/options.go index bb1f53e9a..6166b1552 100644 --- a/sdk/go/options.go +++ b/sdk/go/options.go @@ -36,7 +36,7 @@ type SandboxConfig struct { CPUPlacement CPUPlacement PlacementProfile string THP THPPolicy - MemorySnapshot MemorySnapshotMode + Forked bool Workdir string Shell string SecurityProfile SecurityProfile @@ -134,14 +134,13 @@ type persistedInitConfig struct { } type persistedResources struct { - MemorySnapshot MemorySnapshotMode `json:"memory_snapshot"` - CPUs uint8 `json:"cpus"` - MemoryMiB uint32 `json:"memory_mib"` - MaxCPUs uint8 `json:"max_cpus"` - MaxMemoryMiB uint32 `json:"max_memory_mib"` - CPUPlacement CPUPlacement `json:"cpu_placement"` - PlacementProfile string `json:"placement_profile"` - THP THPPolicy `json:"thp"` + CPUs uint8 `json:"cpus"` + MemoryMiB uint32 `json:"memory_mib"` + MaxCPUs uint8 `json:"max_cpus"` + MaxMemoryMiB uint32 `json:"max_memory_mib"` + CPUPlacement CPUPlacement `json:"cpu_placement"` + PlacementProfile string `json:"placement_profile"` + THP THPPolicy `json:"thp"` } type persistedRuntime struct { @@ -213,7 +212,6 @@ func (c *SandboxConfig) UnmarshalJSON(data []byte) error { OCIUpperSizeMiB: upperSizeMiB, ociUpperSizeSet: upperSizeSet, MemoryMiB: raw.memoryMiB(), - MemorySnapshot: raw.memorySnapshot(), CPUs: raw.cpus(), MaxMemoryMiB: raw.maxMemoryMiB(), MaxCPUs: raw.maxCPUs(), @@ -270,13 +268,6 @@ func (c persistedSandboxConfig) maxCPUs() uint8 { return c.CPUs } -func (c persistedSandboxConfig) memorySnapshot() MemorySnapshotMode { - if c.Resources != nil && c.Resources.MemorySnapshot != "" { - return c.Resources.MemorySnapshot - } - return MemorySnapshotStandard -} - func (c persistedSandboxConfig) maxMemoryMiB() uint32 { if c.Resources != nil { if c.Resources.MaxMemoryMiB != 0 { @@ -488,17 +479,10 @@ const ( // THPPolicy selects the guest transparent huge-page policy at boot. type THPPolicy string -// MemorySnapshotMode selects anonymous or explicit private file-backed memory. -type MemorySnapshotMode string - -const ( - MemorySnapshotStandard MemorySnapshotMode = "standard" - MemorySnapshotCow MemorySnapshotMode = "cow" -) - -// WithMemorySnapshot selects memory representation; snapshots remain manual. -func WithMemorySnapshot(mode MemorySnapshotMode) SandboxOption { - return func(o *SandboxConfig) { o.MemorySnapshot = mode } +// WithForked restores a full snapshot with private copy-on-write memory. +// It cannot be combined with a fresh boot or disk-only restore. +func WithForked() SandboxOption { + return func(o *SandboxConfig) { o.Forked = true } } const ( diff --git a/sdk/go/options_test.go b/sdk/go/options_test.go index 8b42690fd..f4220f427 100644 --- a/sdk/go/options_test.go +++ b/sdk/go/options_test.go @@ -15,25 +15,18 @@ func TestWithImage(t *testing.T) { } } -func TestMemorySnapshotPolicy(t *testing.T) { +func TestForkedRestoreOption(t *testing.T) { var config SandboxConfig - WithMemorySnapshot(MemorySnapshotCow)(&config) - if config.MemorySnapshot != MemorySnapshotCow { - t.Fatal("CoW option was lost") + WithForked()(&config) + if !config.Forked { + t.Fatal("forked option was lost") } - for _, tc := range []struct { - json string - want MemorySnapshotMode - }{ - {`{"resources":{"cpus":1,"memory_mib":128}}`, MemorySnapshotStandard}, - {`{"resources":{"cpus":1,"memory_mib":128,"memory_snapshot":"cow"}}`, MemorySnapshotCow}, - } { - if err := json.Unmarshal([]byte(tc.json), &config); err != nil { - t.Fatal(err) - } - if config.MemorySnapshot != tc.want { - t.Fatalf("got %q, want %q", config.MemorySnapshot, tc.want) - } + // Restore policy is construction-only, not a property of stopped sandbox disks. + if err := json.Unmarshal([]byte(`{"resources":{"cpus":1,"memory_mib":128}}`), &config); err != nil { + t.Fatal(err) + } + if config.Forked { + t.Fatal("forked option leaked into persisted configuration") } } diff --git a/sdk/go/sandbox.go b/sdk/go/sandbox.go index fde33b788..abb92cd29 100644 --- a/sdk/go/sandbox.go +++ b/sdk/go/sandbox.go @@ -83,7 +83,7 @@ func buildFFICreateOptions(o SandboxConfig) ffi.CreateOptions { CPUPlacement: string(o.CPUPlacement), PlacementProfile: o.PlacementProfile, THP: string(o.THP), - MemorySnapshot: string(o.MemorySnapshot), + Forked: o.Forked, Workdir: o.Workdir, Shell: o.Shell, SecurityProfile: string(o.SecurityProfile), @@ -772,6 +772,15 @@ func (h *SandboxHandle) RequestStop(ctx context.Context) error { return wrapFFI(ffi.RequestStopSandboxByName(ctx, h.name)) } +// Branch creates an independent local CoW child without publishing a durable full snapshot. +func (h *SandboxHandle) Branch(ctx context.Context, name string) (*Sandbox, error) { + inner, err := ffi.BranchSandboxByName(ctx, h.name, name) + if err != nil { + return nil, wrapFFI(err) + } + return &Sandbox{inner: inner}, nil +} + // Pause controls resident execution without creating a snapshot. func (h *SandboxHandle) Pause(ctx context.Context) error { return wrapFFI(ffi.PauseSandboxByName(ctx, h.name)) @@ -840,6 +849,15 @@ func (s *Sandbox) Pause(ctx context.Context) error { return wrapFFI(s.inner.Pause(ctx)) } +// Branch creates an independent local CoW child without publishing a durable full snapshot. +func (s *Sandbox) Branch(ctx context.Context, name string) (*Sandbox, error) { + inner, err := s.inner.Branch(ctx, name) + if err != nil { + return nil, wrapFFI(err) + } + return &Sandbox{inner: inner}, nil +} + // Resume controls resident execution without creating a snapshot. func (s *Sandbox) Resume(ctx context.Context) error { return wrapFFI(s.inner.Resume(ctx)) diff --git a/sdk/node-ts/native/index.d.ts b/sdk/node-ts/native/index.d.ts index 7510cf12d..e9ba9429c 100644 --- a/sdk/node-ts/native/index.d.ts +++ b/sdk/node-ts/native/index.d.ts @@ -984,6 +984,8 @@ export declare class Sandbox { attachShell(): Promise /** Stop the sandbox gracefully and wait for it to exit. */ stop(): Promise + /** Create an independent local CoW child without a durable full snapshot. */ + branch(name: string): Promise /** Explicit resident pause through host control. */ pause(): Promise /** Explicit resident resume through host control. */ @@ -1090,8 +1092,8 @@ export declare class SandboxBuilder { maxMemory(mib: number): this /** Guest transparent huge-page policy selected at boot. */ thp(policy: 'always' | 'madvise' | 'never'): this - /** Select explicit private file-backed memory or standard anonymous memory. */ - memorySnapshot(mode: 'standard' | 'cow'): this + /** Restore a full snapshot with private copy-on-write memory. */ + forked(): this /** Override log verbosity: `"trace" | "debug" | "info" | "warn" | "error"`. */ logLevel(level: string): this /** Suppress sandbox logs. */ @@ -1362,6 +1364,8 @@ export declare class SandboxHandle { * override with `stopWithTimeout(timeoutMs)`. */ stop(): Promise + /** Create an independent local CoW child without a durable full snapshot. */ + branch(name: string): Promise /** Explicit resident pause through host control. */ pause(): Promise /** Explicit resident resume through host control. */ diff --git a/sdk/node-ts/native/sandbox.rs b/sdk/node-ts/native/sandbox.rs index eb9b4f907..4f043effe 100644 --- a/sdk/node-ts/native/sandbox.rs +++ b/sdk/node-ts/native/sandbox.rs @@ -529,6 +529,16 @@ impl Sandbox { sb.stop().await.map_err(to_napi_error) } + /// Create an independent local CoW child without a durable full snapshot. + #[napi] + pub async fn branch(&self, name: String) -> Result { + let guard = self.inner.lock().await; + let sb = guard.as_ref().ok_or_else(consumed_error)?; + Ok(Sandbox::from_rust( + sb.branch(name).await.map_err(to_napi_error)?, + )) + } + /// Explicit resident pause through host control. #[napi] pub async fn pause(&self) -> Result<()> { diff --git a/sdk/node-ts/native/sandbox_builder.rs b/sdk/node-ts/native/sandbox_builder.rs index facad0632..50ab270ed 100644 --- a/sdk/node-ts/native/sandbox_builder.rs +++ b/sdk/node-ts/native/sandbox_builder.rs @@ -231,16 +231,14 @@ impl JsSandboxBuilder { Ok(self) } - /// Select explicit private file-backed memory or standard anonymous memory. - #[napi(ts_args_type = "mode: 'standard' | 'cow'")] - pub fn memory_snapshot(&mut self, mode: String) -> Result<&Self> { - let mode = serde_json::from_value(serde_json::Value::String(mode)) - .map_err(|e| napi::Error::from_reason(e.to_string()))?; + /// Restore a full snapshot with private copy-on-write memory. + #[napi] + pub fn forked(&mut self) -> Result<&Self> { let prev = self .inner .take() .ok_or_else(|| napi::Error::from_reason("builder already consumed"))?; - self.inner = Some(prev.memory_snapshot(mode)); + self.inner = Some(prev.forked()); Ok(self) } diff --git a/sdk/node-ts/native/sandbox_handle.rs b/sdk/node-ts/native/sandbox_handle.rs index 57f0ca089..ad723ba03 100644 --- a/sdk/node-ts/native/sandbox_handle.rs +++ b/sdk/node-ts/native/sandbox_handle.rs @@ -162,6 +162,14 @@ impl JsSandboxHandle { self.inner.stop().await.map_err(to_napi_error) } + /// Create an independent local CoW child without a durable full snapshot. + #[napi] + pub async fn branch(&self, name: String) -> Result { + Ok(crate::sandbox::Sandbox::from_rust( + self.inner.branch(name).await.map_err(to_napi_error)?, + )) + } + /// Explicit resident pause through host control. #[napi] pub async fn pause(&self) -> Result<()> { diff --git a/sdk/node-ts/src/internal/napi.ts b/sdk/node-ts/src/internal/napi.ts index 1dcaebe82..d301c1941 100644 --- a/sdk/node-ts/src/internal/napi.ts +++ b/sdk/node-ts/src/internal/napi.ts @@ -187,7 +187,7 @@ export interface NapiSandboxBuilderSetters { memory(mib: number): this; maxMemory(mib: number): this; thp(policy: "always" | "madvise" | "never"): this; - memorySnapshot(mode: "standard" | "cow"): this; + forked(): this; logLevel(level: string): this; quietLogs(): this; detached(enabled: boolean): this; @@ -274,6 +274,7 @@ export interface NapiSandbox { attachWithBuilder(cmd: string, builder: NapiAttachOptionsBuilder): Promise; attachShell(): Promise; stop(): Promise; + branch(name: string): Promise; pause(): Promise; resume(): Promise; requestStop(): Promise; @@ -306,6 +307,7 @@ export interface NapiSandboxHandle { connect(): Promise; connectWithTimeout(timeoutMs: number): Promise; stop(): Promise; + branch(name: string): Promise; pause(): Promise; resume(): Promise; requestStop(): Promise; diff --git a/sdk/node-ts/src/sandbox-handle.ts b/sdk/node-ts/src/sandbox-handle.ts index db26b2f8f..3e6fd193e 100644 --- a/sdk/node-ts/src/sandbox-handle.ts +++ b/sdk/node-ts/src/sandbox-handle.ts @@ -160,7 +160,13 @@ export class SandboxHandle { await withMappedErrors(() => this.inner.stop()); } - /** Explicit resident pause; no snapshot is created. */ + /** Create an independent local CoW child without a durable full snapshot. */ + async branch(name: string): Promise { + const child = await withMappedErrors(() => this.inner.branch(name)); + return new Sandbox(child, name, false); + } + + /** Suspend this resident VM without creating a snapshot. */ async pause(): Promise { await withMappedErrors(() => this.inner.pause()); } diff --git a/sdk/node-ts/src/sandbox.ts b/sdk/node-ts/src/sandbox.ts index 2a50cd536..1de23b1d6 100644 --- a/sdk/node-ts/src/sandbox.ts +++ b/sdk/node-ts/src/sandbox.ts @@ -491,7 +491,13 @@ export class Sandbox implements AsyncDisposable { await withMappedErrors(() => this.inner.stop()); } - /** Explicit resident pause; no snapshot is created. */ + /** Create an independent local CoW child without a durable full snapshot. */ + async branch(name: string): Promise { + const child = await withMappedErrors(() => this.inner.branch(name)); + return new Sandbox(child, name, false); + } + + /** Suspend this resident VM without creating a snapshot. */ async pause(): Promise { await withMappedErrors(() => this.inner.pause()); } diff --git a/sdk/node-ts/tests/cow-lifecycle.test.ts b/sdk/node-ts/tests/cow-lifecycle.test.ts index b171b4fff..8cc3dc08a 100644 --- a/sdk/node-ts/tests/cow-lifecycle.test.ts +++ b/sdk/node-ts/tests/cow-lifecycle.test.ts @@ -4,21 +4,29 @@ import { Sandbox, Snapshot } from "../dist/index.js"; // Opt-in because this starts real VMs with a matching development runtime/kernel bundle. it.skipIf(process.env.MSB_COW_LIVE !== "1")("captures a resident pause and restores private memory", async () => { const name = `cow8-node-${process.pid}`; - const source = await Sandbox.builder(name).image("alpine").rootDisk(512).memory(256).memorySnapshot("cow").create(); + const source = await Sandbox.builder(name).image("alpine").rootDisk(512).memory(256).create(); let child: Sandbox | undefined; + const branches: Sandbox[] = []; try { await source.exec("sh", ["-c", "echo source > /dev/shm/sdk-marker"]); await source.pause(); const paused = await Sandbox.get(name); expect(paused.status).toBe("paused"); + const branched = await paused.branch(`${name}-paused-branch`); + branches.push(branched); + expect((await branched.exec("cat", ["/dev/shm/sdk-marker"])).stdout().trim()).toBe("source"); await Snapshot.builder(`${name}-full`).fromSandbox(name).full().create(); await paused.resume(); - child = await Sandbox.builder(`${name}-child`).fromSnapshot(`${name}-full`).memorySnapshot("cow").create(); + child = await Sandbox.builder(`${name}-child`).fromSnapshot(`${name}-full`).forked().create(); expect((await child.exec("cat", ["/dev/shm/sdk-marker"])).stdout().trim()).toBe("source"); await child.exec("sh", ["-c", "echo child > /dev/shm/sdk-marker"]); + const descendant = await child.branch(`${name}-branch`); + branches.push(descendant); + expect((await descendant.exec("cat", ["/dev/shm/sdk-marker"])).stdout().trim()).toBe("child"); expect((await source.exec("cat", ["/dev/shm/sdk-marker"])).stdout().trim()).toBe("source"); await child.pause(); } finally { + for (const branch of branches.reverse()) await branch.stop(); await child?.stop(); await source.stop(); } diff --git a/sdk/node-ts/tests/unit/builders.test.ts b/sdk/node-ts/tests/unit/builders.test.ts index 3cabe7ef5..59003a05f 100644 --- a/sdk/node-ts/tests/unit/builders.test.ts +++ b/sdk/node-ts/tests/unit/builders.test.ts @@ -351,14 +351,9 @@ describe("PatchBuilder", () => { }); describe("SandboxBuilder.build", () => { - it("makes CoW memory an explicit creation policy", async () => { - const config = await Sandbox.builder("cow-policy") - .image("alpine") - .memorySnapshot("cow") - .build(); - expect((config.resources as { memorySnapshot: string }).memorySnapshot).toBe("cow"); - const standard = await Sandbox.builder("standard-policy").image("alpine").build(); - expect((standard.resources as { memorySnapshot?: string }).memorySnapshot).toBeUndefined(); + it("rejects forked for a fresh boot", async () => { + await expect(Sandbox.builder("forked-policy").image("alpine").forked().build()) + .rejects.toThrow("forked requires a full snapshot"); }); it("requires .image()", async () => { diff --git a/sdk/python/microsandbox/__init__.py b/sdk/python/microsandbox/__init__.py index c1d593f3d..eb6a6dc6c 100644 --- a/sdk/python/microsandbox/__init__.py +++ b/sdk/python/microsandbox/__init__.py @@ -117,7 +117,6 @@ LogLevel, LogReadSource, LogSource, - MemorySnapshotMode, MiB, ModificationConflict, ModificationDisposition, @@ -313,7 +312,6 @@ "FlatClone", "PullPolicy", "CpuPlacement", - "MemorySnapshotMode", "RegistryAuth", "LogLevel", "DeploymentProfile", diff --git a/sdk/python/microsandbox/_microsandbox.pyi b/sdk/python/microsandbox/_microsandbox.pyi index 3f8d6b336..1fcd0a1c2 100644 --- a/sdk/python/microsandbox/_microsandbox.pyi +++ b/sdk/python/microsandbox/_microsandbox.pyi @@ -20,7 +20,6 @@ from microsandbox.types import ( LogLevel, LogReadSource, LogSource, - MemorySnapshotMode, ModificationPolicy, MountConfig, NamedVolumeMode, @@ -96,7 +95,7 @@ class Sandbox: from_snapshot: str | os.PathLike[str] | None = None, disk_only: bool = False, snapshot_base: str | None = None, - memory_snapshot: MemorySnapshotMode | None = None, + forked: bool = False, memory: int | None = None, cpus: int | None = None, max_memory: int | None = None, @@ -154,7 +153,7 @@ class Sandbox: from_snapshot: str | os.PathLike[str] | None = None, disk_only: bool = False, snapshot_base: str | None = None, - memory_snapshot: MemorySnapshotMode | None = None, + forked: bool = False, memory: int | None = None, cpus: int | None = None, max_memory: int | None = None, @@ -325,6 +324,7 @@ class Sandbox: follow: bool = False, ) -> LogStream: ... async def stop(self, timeout: float | None = None) -> None: ... + async def branch(self, name: str) -> Sandbox: ... async def pause(self) -> None: ... async def resume(self) -> None: ... async def request_stop(self) -> None: ... @@ -422,6 +422,7 @@ class SandboxHandle: async def refresh(self) -> SandboxHandle: ... async def connect(self, timeout: float | None = None) -> Sandbox: ... async def stop(self, timeout: float | None = None) -> None: ... + async def branch(self, name: str) -> Sandbox: ... async def pause(self) -> None: ... async def resume(self) -> None: ... async def request_stop(self) -> None: ... diff --git a/sdk/python/microsandbox/types.py b/sdk/python/microsandbox/types.py index bab28d76a..40c0923a0 100644 --- a/sdk/python/microsandbox/types.py +++ b/sdk/python/microsandbox/types.py @@ -41,13 +41,6 @@ class PullPolicy(StrEnum): NEVER = "never" -class MemorySnapshotMode(StrEnum): - """Explicit memory representation; snapshots are still created manually.""" - - STANDARD = "standard" - COW = "cow" - - class CpuPlacement(StrEnum): """Host placement policy for sandbox vCPU threads.""" diff --git a/sdk/python/src/helpers.rs b/sdk/python/src/helpers.rs index d24bd87b6..f4fb24ef1 100644 --- a/sdk/python/src/helpers.rs +++ b/sdk/python/src/helpers.rs @@ -25,7 +25,7 @@ const KNOWN_CREATE_KWARGS: &[&str] = &[ "cpu_placement", "placement_profile", "thp", - "memory_snapshot", + "forked", "workdir", "shell", "security", @@ -337,10 +337,8 @@ pub fn sandbox_builder_from_args( .map_err(pyo3::exceptions::PyValueError::new_err)?; builder = builder.thp(policy); } - if let Some(mode) = extract_opt::(kwargs, "memory_snapshot")? { - let mode = serde_json::from_value(serde_json::Value::String(mode)) - .map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))?; - builder = builder.memory_snapshot(mode); + if extract_opt::(kwargs, "forked")?.unwrap_or(false) { + builder = builder.forked(); } if let Some(workdir) = extract_opt::(kwargs, "workdir")? { builder = builder.workdir(workdir); diff --git a/sdk/python/src/sandbox.rs b/sdk/python/src/sandbox.rs index 6b47dea7a..f37158732 100644 --- a/sdk/python/src/sandbox.rs +++ b/sdk/python/src/sandbox.rs @@ -963,6 +963,17 @@ impl PySandbox { }) } + /// Create an independent local CoW child without a durable full snapshot. + fn branch<'py>(&self, py: Python<'py>, name: String) -> PyResult> { + let inner = self.inner.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let sandbox = Self::clone_sandbox(&inner).await?; + Ok(PySandbox::from_rust( + sandbox.branch(name).await.map_err(to_py_err)?, + )) + }) + } + /// Suspend this resident VM without releasing RAM. fn pause<'py>(&self, py: Python<'py>) -> PyResult> { let inner = self.inner.clone(); diff --git a/sdk/python/src/sandbox_handle.rs b/sdk/python/src/sandbox_handle.rs index 2446150cb..4714e79d9 100644 --- a/sdk/python/src/sandbox_handle.rs +++ b/sdk/python/src/sandbox_handle.rs @@ -360,6 +360,17 @@ impl PySandboxHandle { }) } + /// Create an independent local CoW child without a durable full snapshot. + fn branch<'py>(&self, py: Python<'py>, name: String) -> PyResult> { + let inner = self.inner.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let guard = inner.lock().await; + Ok(PySandbox::from_rust( + guard.branch(name).await.map_err(to_py_err)?, + )) + }) + } + /// Suspend this resident VM without releasing RAM. fn pause<'py>(&self, py: Python<'py>) -> PyResult> { let inner = self.inner.clone(); diff --git a/sdk/python/tests/test_cow_lifecycle.py b/sdk/python/tests/test_cow_lifecycle.py index 8ce67c960..82b66e6ec 100644 --- a/sdk/python/tests/test_cow_lifecycle.py +++ b/sdk/python/tests/test_cow_lifecycle.py @@ -4,7 +4,7 @@ import pytest -from microsandbox import MemorySnapshotMode, Sandbox, Snapshot +from microsandbox import Sandbox, Snapshot @pytest.mark.skipif(os.environ.get("MSB_COW_LIVE") != "1", reason="requires matching live bundle") @@ -12,26 +12,36 @@ async def test_cow_resident_capture_and_child_isolation(): name = f"cow8-python-{os.getpid()}" source = await Sandbox.create( - name, image="alpine", memory=256, memory_snapshot=MemorySnapshotMode.COW + name, image="alpine", memory=256 ) child = None + branches = [] try: await source.exec("sh", ["-c", "echo source > /dev/shm/sdk-marker"]) await source.pause() paused = await Sandbox.get(name) assert str(paused.status) == "paused" + branched = await paused.branch(f"{name}-paused-branch") + branches.append(branched) + assert (await branched.exec("cat", ["/dev/shm/sdk-marker"])).stdout_text.strip() == "source" await Snapshot.create(f"{name}-full", from_sandbox=name, full=True) await paused.resume() child = await Sandbox.create( - f"{name}-child", from_snapshot=f"{name}-full", memory_snapshot=MemorySnapshotMode.COW + f"{name}-child", from_snapshot=f"{name}-full", forked=True ) result = await child.exec("cat", ["/dev/shm/sdk-marker"]) assert result.stdout_text.strip() == "source" await child.exec("sh", ["-c", "echo child > /dev/shm/sdk-marker"]) + descendant = await child.branch(f"{name}-branch") + branches.append(descendant) + result = await descendant.exec("cat", ["/dev/shm/sdk-marker"]) + assert result.stdout_text.strip() == "child" result = await source.exec("cat", ["/dev/shm/sdk-marker"]) assert result.stdout_text.strip() == "source" await child.pause() finally: + for branched in reversed(branches): + await branched.stop() if child is not None: await child.stop() await source.stop() diff --git a/sdk/python/tests/test_create_stub.py b/sdk/python/tests/test_create_stub.py index a9c678f98..b86a82890 100644 --- a/sdk/python/tests/test_create_stub.py +++ b/sdk/python/tests/test_create_stub.py @@ -12,7 +12,7 @@ "from_snapshot", "disk_only", "snapshot_base", - "memory_snapshot", + "forked", "memory", "cpus", "max_memory", @@ -85,7 +85,7 @@ def test_create_closed_values_are_precisely_typed() -> None: } assert annotations["security"] == "SecurityProfile | None" - assert annotations["memory_snapshot"] == "MemorySnapshotMode | None" + assert annotations["forked"] == "bool" assert annotations["init"] == "str | InitConfig | InitOptions | None" assert annotations["pull_policy"] == "PullPolicy | None" assert annotations["log_level"] == "LogLevel | None" diff --git a/sdk/rust/lib/backend/cloud/sandbox.rs b/sdk/rust/lib/backend/cloud/sandbox.rs index 2cc1ceaa5..34888dac5 100644 --- a/sdk/rust/lib/backend/cloud/sandbox.rs +++ b/sdk/rust/lib/backend/cloud/sandbox.rs @@ -292,10 +292,10 @@ impl TryFrom for CloudCreateBody { /// Build the cloud create body from an SDK config, rejecting the /// create-time options the cloud does not accept. fn try_from(mut config: SandboxConfig) -> MicrosandboxResult { - if !config.spec.resources.memory_snapshot.is_standard() { + if config.forked { return Err(MicrosandboxError::unsupported( Operation::SandboxCreate, - UnsupportedReason::ConfigField("memory_snapshot"), + UnsupportedReason::ConfigField("forked"), )); } if config.replace_existing { diff --git a/sdk/rust/lib/backend/local/sandbox/create.rs b/sdk/rust/lib/backend/local/sandbox/create.rs index b77541243..23c6cdf17 100644 --- a/sdk/rust/lib/backend/local/sandbox/create.rs +++ b/sdk/rust/lib/backend/local/sandbox/create.rs @@ -153,6 +153,18 @@ impl LocalBackend { let db = self.db().await?; let sandbox_dir = self.sandboxes_dir().join(&config.spec.name); Self::prepare_create_target(db, &config, &sandbox_dir, &self.config().run_dir()).await?; + // Hold the existing lifecycle lock across reservation, capture and spawn. Recheck + // under the lock so two creates cannot both own the same child staging directory. + let lifecycle_guard = crate::runtime::acquire_sandbox_lifecycle_guard( + &self.config().run_dir(), + &config.spec.name, + std::time::Duration::from_secs(5), + ) + .await?; + let mut reserved_config = config.clone(); + reserved_config.replace_existing = false; + Self::prepare_create_target(db, &reserved_config, &sandbox_dir, &self.config().run_dir()) + .await?; let mut child_stage_guard = None; // Preserve only the installed-snapshot source that existed on entry. Direct archive // materialization below installs its checkpoint closure directly into child staging, so @@ -161,6 +173,16 @@ impl LocalBackend { let installed_checkpoint_restore = config.checkpoint_restore.take(); let installed_file_sources = std::mem::take(&mut config.snapshot_root_layer_sources); let installed_file_virtual_size = config.snapshot_root_virtual_size.take(); + let _branch_pin = if let Some(source) = config.branch_source.take() { + tokio::fs::create_dir(&sandbox_dir).await?; + child_stage_guard = Some(ChildStageGuard::new(sandbox_dir.clone())); + Some( + crate::sandbox::branch::capture_child(self, &mut config, &source, &sandbox_dir) + .await?, + ) + } else { + None + }; // A direct archive restore streams its layer into the ordinary child // staging location before image resolution. The archive supplies the @@ -260,6 +282,13 @@ impl LocalBackend { } } } + // Archive descriptors are resolved here, after the builder's initial validation. + // Do not let a disk archive turn an explicit CoW restore into a fresh boot. + if config.forked && config.checkpoint_restore.is_none() { + return Err(crate::MicrosandboxError::InvalidConfig( + "forked requires a full snapshot restore".into(), + )); + } if !installed_file_sources.is_empty() { child_stage_guard = Some(ChildStageGuard::new(sandbox_dir.clone())); let virtual_size = installed_file_virtual_size.ok_or_else(|| { @@ -598,7 +627,7 @@ impl LocalBackend { .as_ref() .map(|restore| restore.closure.clone()); let created = self - .create_sandbox_inner(config, sandbox_id, mode, None) + .create_sandbox_inner(config, sandbox_id, mode, Some(lifecycle_guard)) .await; if let Some(closure) = restore_closure && let Err(error) = remove_dir_if_exists(&closure) diff --git a/sdk/rust/lib/runtime/spawn.rs b/sdk/rust/lib/runtime/spawn.rs index 9057a578a..761ce9998 100644 --- a/sdk/rust/lib/runtime/spawn.rs +++ b/sdk/rust/lib/runtime/spawn.rs @@ -2492,6 +2492,12 @@ fn sandbox_cli_args( // typed `LaunchConfig`, delivered over the config fd. See issue #997. let mut visible = vec![OsString::from("sandbox")]; + // An old binary might ignore unknown JSON fields, including the whole restore source. + // An explicit argv requirement instead fails in its command parser, before any VM exists. + if config.checkpoint_restore.is_some() { + visible.push(OsString::from("--restore")); + } + if let Some(log_level) = config.spec.runtime.log_level { visible.push(OsString::from(sandbox_log_level_cli_flag(log_level))); } @@ -2548,17 +2554,22 @@ fn sandbox_cli_args( agent_sock: agent_sock_path.to_path_buf(), libkrunfw_path: libkrunfw_path.to_path_buf(), thp: config.spec.resources.thp, - memory_snapshot: config.spec.resources.memory_snapshot, - memory_cache_dir: (config.spec.resources.memory_snapshot - == microsandbox_types::MemorySnapshotMode::Cow) - .then(|| local.cache_dir().join("memory")), + memory_cache_dir: Some(local.cache_dir().join("memory")), startup: startup_command(config), lifecycle: Lifecycle { max_duration_secs: config.spec.lifecycle.max_duration_secs, idle_timeout_secs: config.spec.lifecycle.idle_timeout_secs, }, vsock: config.spec.vsock.routes.clone(), - checkpoint_restore: config.checkpoint_restore.clone(), + execution: if config.checkpoint_restore.is_some() { + microsandbox_runtime::launch::ExecutionIntent::Restore + } else { + microsandbox_runtime::launch::ExecutionIntent::Boot + }, + checkpoint_restore: config.checkpoint_restore.clone().map(|mut restore| { + restore.forked = config.forked; + restore + }), #[cfg(feature = "net")] deployment_profile: config.spec.deployment_profile, bootstrap: GuestBootstrap { @@ -3951,6 +3962,8 @@ mod tests { }, ]; config.checkpoint_restore = Some(CheckpointRestoreConfig { + local_branch: false, + forked: false, closure: PathBuf::from("/tmp/checkpoint"), checkpoint_root: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb".into(), @@ -3960,6 +3973,11 @@ mod tests { let launch = render_launch(&config); assert!(launch.rootfs.upper.is_none()); + assert_eq!( + launch.execution, + microsandbox_runtime::launch::ExecutionIntent::Restore + ); + assert!(render_args(&config).contains(&"--restore".to_string())); assert_eq!(launch.rootfs.upper_layers.len(), 2); assert_eq!(launch.rootfs.upper_layers[0].format, "raw"); assert_eq!(launch.rootfs.upper_layers[1].format, "qcow2"); diff --git a/sdk/rust/lib/sandbox/branch.rs b/sdk/rust/lib/sandbox/branch.rs new file mode 100644 index 000000000..5f8e629e0 --- /dev/null +++ b/sdk/rust/lib/sandbox/branch.rs @@ -0,0 +1,186 @@ +//! Direct local execution branching through the existing control and restore paths. + +use std::fs::File; +use std::path::Path; +use std::sync::Arc; + +use microsandbox_runtime::checkpoint::LocalBranchState; +use microsandbox_runtime::control::ControlRequest; +use microsandbox_runtime::launch::{CheckpointRestoreConfig, RootfsUpperLayerConfig}; + +use crate::backend::{Backend, LocalBackend}; +use crate::{MicrosandboxError, MicrosandboxResult}; + +use super::{Sandbox, SandboxConfig, SandboxHandle, SandboxStatus, modify}; + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl Sandbox { + /// Branch current execution into an independent local child using private CoW RAM. + /// The source keeps its running/paused state; no durable full snapshot is created. + pub async fn branch(&self, name: impl Into) -> MicrosandboxResult { + branch(self.backend().clone(), self.name(), name.into()).await + } +} + +impl SandboxHandle { + /// Branch a running or user-paused local sandbox without connecting to its guest. + pub async fn branch(&self, name: impl Into) -> MicrosandboxResult { + branch(self.backend.clone(), self.name(), name.into()).await + } +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +async fn branch( + backend: Arc, + source: &str, + name: String, +) -> MicrosandboxResult { + super::validate_sandbox_name(&name)?; + let local = backend.as_local().ok_or_else(|| { + MicrosandboxError::InvalidConfig("direct branching requires a local backend".into()) + })?; + let handle = backend.sandboxes().get(backend.clone(), source).await?; + if !matches!( + handle.status_snapshot(), + SandboxStatus::Running | SandboxStatus::Paused + ) { + return Err(MicrosandboxError::InvalidConfig( + "branch requires a running or user-paused source".into(), + )); + } + let mut config = handle + .active_config()? + .unwrap_or(handle.config()?) + .clone_for_persistence(); + if !config.spec.network.ports.is_empty() { + return Err(MicrosandboxError::InvalidConfig( + "branch cannot inherit published host ports; remove port publications before branching" + .into(), + )); + } + let capabilities = + modify::control_request_for(local, source, "{\"op\":\"capabilities\"}\n".into()).await?; + if !capabilities.capabilities.is_some_and(|c| c.branch_create) { + return Err(MicrosandboxError::Runtime( + "source runtime does not support direct local branching".into(), + )); + } + config.spec.name = name; + config.replace_existing = false; + config.spec.patches.clear(); + config.branch_source = Some(source.into()); + config.suppress_launch_for_full_restore(); + backend + .sandboxes() + .create_detached(backend.clone(), config) + .await +} + +/// Called only after the ordinary create path reserves the child name and directory. +/// Retain this pin through spawn, until the runtime owns its independent mapping handle. +pub(crate) async fn capture_child( + local: &LocalBackend, + config: &mut SandboxConfig, + source: &str, + child: &Path, +) -> MicrosandboxResult { + let id = format!("branch_{:032x}", rand::random::()); + // Acquired before publication: source exit or another capture cannot create an unpinned + // eviction window before this caller opens the completed memory file. + let _handoff = microsandbox_runtime::checkpoint::LocalMemory::reserve( + &local.cache_dir().join("memory"), + &id, + )?; + tokio::fs::write(child.join(".branch-reservation"), &id).await?; + let request = ControlRequest::BranchCreate { + branch_id: id.clone(), + child_name: config.spec.name.clone(), + memory_cache_dir: local.cache_dir().join("memory"), + }; + let response = modify::control_request_for( + local, + source, + format!("{}\n", serde_json::to_string(&request)?), + ) + .await?; + let closure = child.join(".branch-restore"); + if response.branch.as_ref() != Some(&closure) { + return Err(MicrosandboxError::Runtime( + "branch returned an unexpected handoff path".into(), + )); + } + let state = LocalBranchState::open(&closure)?; + if state.id != id { + return Err(MicrosandboxError::Runtime("branch identity differs".into())); + } + let pin = state.memory.pin()?; + config.spec.resources.cpus = state.vcpus; + config.spec.resources.max_cpus = state.max_cpus; + config.spec.resources.memory_mib = state.memory_mib; + config.spec.resources.max_memory_mib = state.max_memory_mib; + // The captured effective address wins over launch-time pools/defaults. Each user-mode + // network stack is isolated; host listeners were rejected before source mutation. + config.spec.network.interface = None; + super::builder::apply_capture_network(config, &state.resources)?; + let layout = match config.spec.image.oci_root_disk() { + Some(super::RootDisk::Flat { .. }) => crate::snapshot::SnapshotRootDisk::Flat, + Some(super::RootDisk::Tmpfs { size_mib }) => crate::snapshot::SnapshotRootDisk::Tmpfs { + size_mib: *size_mib, + }, + _ => crate::snapshot::SnapshotRootDisk::Managed, + }; + match state.disks.as_slice() { + [] if matches!(layout, crate::snapshot::SnapshotRootDisk::Tmpfs { .. }) => {} + [disk] => { + disk.to_canonical_bytes() + .map_err(|e| MicrosandboxError::SnapshotIntegrity(e.to_string()))?; + if disk.pause_generation != state.pause_generation { + return Err(MicrosandboxError::SnapshotIntegrity( + "branch disk epoch differs".into(), + )); + } + let sources = disk + .layers + .iter() + .map(|layer| RootfsUpperLayerConfig { + path: closure + .join("layers") + .join(format!("{}.{}", layer.layer_id, layer.format)), + format: layer.format.clone(), + }) + .collect::>(); + let size = disk + .layers + .last() + .ok_or_else(|| MicrosandboxError::SnapshotIntegrity("branch disk is empty".into()))? + .virtual_size; + let materialized = crate::snapshot::materialize_file_snapshot_for_child( + &sources, size, child, &layout, + ) + .await?; + config.snapshot_upper_layers = materialized.upper_layers; + } + _ => { + return Err(MicrosandboxError::SnapshotIntegrity( + "branch disk closure differs from root layout".into(), + )); + } + } + config.checkpoint_restore = Some(CheckpointRestoreConfig { + local_branch: true, + forked: true, + closure, + checkpoint_root: String::new(), + checkpoint_id: id, + }); + config.forked = true; + config.suppress_launch_for_full_restore(); + tokio::fs::remove_file(child.join(".branch-reservation")).await?; + Ok(pin) +} diff --git a/sdk/rust/lib/sandbox/builder.rs b/sdk/rust/lib/sandbox/builder.rs index 6fb8f4768..5a2a9b6d8 100644 --- a/sdk/rust/lib/sandbox/builder.rs +++ b/sdk/rust/lib/sandbox/builder.rs @@ -378,9 +378,12 @@ impl SandboxBuilder { self } - /// Select explicit private file-backed memory or the default anonymous representation. - pub fn memory_snapshot(mut self, mode: microsandbox_types::MemorySnapshotMode) -> Self { - self.config.spec.resources.memory_snapshot = mode; + /// Restore a full snapshot using private copy-on-write memory. + /// + /// Clean pages can be shared by children; writes remain private. This requires + /// a full snapshot and cannot be combined with a fresh boot or disk-only restore. + pub fn forked(mut self) -> Self { + self.config.forked = true; self } @@ -1306,6 +1309,8 @@ impl SandboxBuilder { } self.config.checkpoint_restore = Some(microsandbox_runtime::launch::CheckpointRestoreConfig { + local_branch: false, + forked: false, closure, checkpoint_root: state.checkpoint_root.clone(), checkpoint_id: state.checkpoint_id.clone(), @@ -1564,6 +1569,16 @@ impl SandboxBuilder { "disk_only must be combined with from_snapshot".into(), )); } + if self.config.forked + && (self.config.snapshot_restore_mode == SnapshotRestoreMode::DiskOnly + || (self.config.checkpoint_restore.is_none() + && self.config.snapshot_archive_source.is_none())) + { + return Err(crate::MicrosandboxError::InvalidConfig( + "forked requires a full snapshot restore and cannot be combined with disk_only" + .into(), + )); + } if self.config.checkpoint_restore.is_some() && !self.config.spec.patches.is_empty() { return Err(crate::MicrosandboxError::InvalidConfig( "patches cannot be combined with full snapshot restore".into(), @@ -1847,9 +1862,14 @@ pub(crate) fn apply_checkpoint_restore_constraints( overrides: RestoreOverrideIntent, ) -> MicrosandboxResult<()> { apply_checkpoint_resources(config, state, overrides)?; + apply_capture_network(config, &checkpoint.resources) +} - let mut resources = checkpoint - .resources +pub(crate) fn apply_capture_network( + config: &mut SandboxConfig, + captured_resources: &[microsandbox_image::checkpoint::ResourceDescriptor], +) -> MicrosandboxResult<()> { + let mut resources = captured_resources .iter() .filter(|resource| resource.kind == "network"); let Some(resource) = resources.next() else { @@ -3192,4 +3212,51 @@ mod tests { vec!["/workspace", "/workspace/persist"] ); } + #[tokio::test] + async fn forked_rejects_fresh_boot() { + let error = SandboxBuilder::new("forked-boot") + .image("alpine") + .forked() + .build() + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("forked requires a full snapshot") + ); + } + + #[tokio::test] + async fn forked_restore_is_transient_and_requires_execution() { + let mut builder = SandboxBuilder::new("forked-child").image("alpine").forked(); + builder.config.checkpoint_restore = + Some(microsandbox_runtime::launch::CheckpointRestoreConfig { + local_branch: false, + forked: false, + closure: "/owned/checkpoint".into(), + checkpoint_root: "blake3:captured-root".into(), + checkpoint_id: "captured".into(), + }); + builder.validate().unwrap(); + let config = builder.config.clone(); + assert!(config.forked); + assert!(!config.clone_for_persistence().forked); + assert!( + serde_json::to_value(&config) + .unwrap() + .get("forked") + .is_none() + ); + builder.config.snapshot_restore_mode = + crate::sandbox::config::SnapshotRestoreMode::DiskOnly; + assert!( + builder + .build() + .await + .unwrap_err() + .to_string() + .contains("forked") + ); + } } diff --git a/sdk/rust/lib/sandbox/config.rs b/sdk/rust/lib/sandbox/config.rs index 5373a9212..49a9e598c 100644 --- a/sdk/rust/lib/sandbox/config.rs +++ b/sdk/rust/lib/sandbox/config.rs @@ -220,6 +220,14 @@ pub struct SandboxConfig { #[serde(skip)] pub(crate) checkpoint_restore: Option, + /// Source name for a one-shot direct local branch, consumed under child reservation. + #[serde(skip)] + pub(crate) branch_source: Option, + + /// Restore captured RAM through private CoW mappings; never a cold-boot policy. + #[serde(skip)] + pub(crate) forked: bool, + /// Transient checkpoint materialization policy selected by the caller. #[serde(skip)] pub(crate) snapshot_restore_mode: SnapshotRestoreMode, @@ -277,6 +285,8 @@ impl SandboxConfig { pub(crate) fn clone_for_persistence(&self) -> Self { let mut config = self.clone(); config.checkpoint_restore = None; + config.branch_source = None; + config.forked = false; config.snapshot_restore_mode = SnapshotRestoreMode::Full; config.resumed_from_full_snapshot = false; config.snapshot_root_layer_sources.clear(); @@ -779,7 +789,6 @@ impl Default for SandboxConfig { Self { spec: SandboxSpec { resources: SandboxResources { - memory_snapshot: Default::default(), cpus: default_cpus(), memory_mib: default_memory_mib(), max_cpus: default_cpus(), @@ -810,6 +819,8 @@ impl Default for SandboxConfig { snapshot_archive_source: None, snapshot_base: None, checkpoint_restore: None, + branch_source: None, + forked: false, snapshot_restore_mode: SnapshotRestoreMode::Full, resumed_from_full_snapshot: false, snapshot_upper_layers: Vec::new(), @@ -1536,7 +1547,6 @@ mod tests { resources: SandboxResources { cpus: 2, memory_mib: 1024, - memory_snapshot: Default::default(), max_cpus: 2, max_memory_mib: 1024, cpu_placement: Default::default(), @@ -1788,6 +1798,8 @@ mod tests { }, snapshot_restore_mode: restore_mode, checkpoint_restore: Some(CheckpointRestoreConfig { + local_branch: false, + forked: false, closure: PathBuf::from("/tmp/checkpoint"), checkpoint_root: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" diff --git a/sdk/rust/lib/sandbox/config_patch.rs b/sdk/rust/lib/sandbox/config_patch.rs index aca44f834..07a6e1e4e 100644 --- a/sdk/rust/lib/sandbox/config_patch.rs +++ b/sdk/rust/lib/sandbox/config_patch.rs @@ -67,7 +67,6 @@ pub enum SandboxImagePatch { /// Sparse resource and lifecycle limits. #[derive(Debug, Clone, Default)] pub struct ResourceConfigPatch { - memory_snapshot: Option, cpus: Option, memory_mib: Option, max_duration_secs: Option, @@ -361,12 +360,6 @@ impl ResourceConfigPatch { Self::default() } - /// Select memory representation for the next VM construction. - pub fn memory_snapshot(mut self, mode: super::MemorySnapshotMode) -> Self { - self.memory_snapshot = Some(mode); - self - } - /// Set the initial vCPU count. pub fn cpus(mut self, cpus: u8) -> Self { self.cpus = Some(cpus); @@ -400,7 +393,6 @@ impl ResourceConfigPatch { /// Overlay another resource patch. pub fn overlay(mut self, higher: Self) -> Self { replace(&mut self.cpus, higher.cpus); - replace(&mut self.memory_snapshot, higher.memory_snapshot); replace(&mut self.memory_mib, higher.memory_mib); replace(&mut self.max_duration_secs, higher.max_duration_secs); replace(&mut self.idle_timeout_secs, higher.idle_timeout_secs); @@ -409,9 +401,6 @@ impl ResourceConfigPatch { } fn apply_to(self, mut builder: SandboxBuilder) -> SandboxBuilder { - if let Some(mode) = self.memory_snapshot { - builder = builder.memory_snapshot(mode); - } if let Some(cpus) = self.cpus { builder = builder.cpus(cpus); } diff --git a/sdk/rust/lib/sandbox/mod.rs b/sdk/rust/lib/sandbox/mod.rs index 357ad3b73..204835fb8 100644 --- a/sdk/rust/lib/sandbox/mod.rs +++ b/sdk/rust/lib/sandbox/mod.rs @@ -6,6 +6,7 @@ //! for guest communication. pub(crate) mod attach; +pub(crate) mod branch; mod builder; mod compact; pub(crate) mod config; @@ -140,7 +141,7 @@ pub use microsandbox_network::policy::{ }; pub use microsandbox_runtime::control::PauseControlState as SandboxPauseState; pub use microsandbox_runtime::logging::LogLevel; -pub use microsandbox_types::{CpuPlacement, MemorySnapshotMode, PullPolicy}; +pub use microsandbox_types::{CpuPlacement, PullPolicy}; pub use microsandbox_types::{ EnvVar, MAX_HOSTNAME_BYTES, MAX_SANDBOX_NAME_BYTES, NetworkSpec, PortProtocol, PublishedPortSpec, SandboxLogLevel, SandboxResources, SandboxRuntimeOptions, SandboxSpec, diff --git a/sdk/rust/lib/snapshot/restore.rs b/sdk/rust/lib/snapshot/restore.rs index 25202758f..5b2cfc2d8 100644 --- a/sdk/rust/lib/snapshot/restore.rs +++ b/sdk/rust/lib/snapshot/restore.rs @@ -110,6 +110,8 @@ pub(crate) async fn materialize_checkpoint_child_state( Ok(CheckpointChildMaterialization { restore: CheckpointRestoreConfig { + local_branch: false, + forked: false, closure: closure_destination.to_path_buf(), checkpoint_root: checkpoint_root.to_string(), checkpoint_id: checkpoint_id.to_string(), @@ -447,6 +449,8 @@ mod tests { let checkpoint_root = ObjectId::from_bytes(&checkpoint_bytes).unwrap(); std::fs::write(source.join("checkpoint.json"), checkpoint_bytes).unwrap(); let restore = CheckpointRestoreConfig { + local_branch: false, + forked: false, closure: source.clone(), checkpoint_root: checkpoint_root.to_string(), checkpoint_id: checkpoint.checkpoint_id, From f68c13295b6d338d05dcf86cf29372b758be0754 Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Wed, 9 Sep 2026 01:35:38 +0100 Subject: [PATCH 09/29] fix(snapshot): qualify forked restores across supported hosts Enable Windows private RAM backing with protected cache entries and cross-platform lifetime pins. Persist pending restore intent until creation succeeds so failed children cannot cold-boot or mutate sealed snapshot layers through later lifecycle operations. Pin the published native memory and ARM64 interrupt/barrier fixes. Add failed-restore and retained-sibling timer regressions, port ownership checks to Windows, and document the four-host live qualification results. --- COMPATIBILITY.md | 2 + Cargo.lock | 25 ++-- Cargo.toml | 23 +-- crates/runtime/Cargo.toml | 2 +- crates/runtime/lib/checkpoint/local_memory.rs | 4 +- crates/runtime/lib/checkpoint/memory_cache.rs | 136 ++++++++++++++++-- crates/runtime/lib/control/executor.rs | 2 +- crates/utils/lib/process_lock.rs | 32 +++++ docs/sandboxes/snapshots.mdx | 8 +- scripts/smoke/cli/branch-ownership.py | 31 +++- scripts/smoke/cli/branch-timer-progress.py | 75 ++++++++++ scripts/smoke/cli/checkpoint-cpu-state.py | 3 +- scripts/smoke/cli/failed-restore.py | 88 ++++++++++++ .../reports/cow-platform-fixes-2026-09-09.md | 56 ++++++++ .../reports/execution-state-2026-09-07.md | 2 + sdk/rust/lib/backend/local/sandbox/create.rs | 99 +++++++++++-- sdk/rust/lib/backend/local/sandbox/mod.rs | 3 + sdk/rust/lib/sandbox/compact.rs | 3 + sdk/rust/lib/sandbox/config.rs | 8 +- sdk/rust/lib/sandbox/modify.rs | 3 + sdk/rust/lib/snapshot/create.rs | 3 + 21 files changed, 547 insertions(+), 61 deletions(-) create mode 100644 scripts/smoke/cli/branch-timer-progress.py create mode 100644 scripts/smoke/cli/failed-restore.py create mode 100644 scripts/smoke/reports/cow-platform-fixes-2026-09-09.md diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index 55fc53aba..d3c878cfb 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -142,6 +142,8 @@ Sources: [`crates/runtime/lib/launch.rs`](crates/runtime/lib/launch.rs), [`crate Launch JSON requires an explicit `execution` intent (`boot` or `restore`) and rejects unknown fields. Restores also pass the internal `msb sandbox --restore` argument: a runtime predating this contract rejects the unknown argument rather than ignoring a JSON restore source and cold-booting. The argument, intent, and complete strictly validated `checkpoint_restore` source must agree before VM construction. Unsupported restore behavior is an error, never a fresh-boot fallback. These #8 development contracts replace superseded unreleased forms without shims; they do not change portable snapshot bytes. +The child database config retains `checkpoint_restore` while construction is incomplete. Only successful restore activation and creation finalization remove it. A failed or interrupted attempt retains its child-owned staging and rejects start, auto-start through exec, modification, compaction, and snapshot creation; remove and recreate it from the intact input snapshot. This replaces the earlier unreleased #8 behavior that discarded restore intent before success. Do not reopen these development rows with older #8 binaries that skip that field. Successful restores retain the ordinary later stop/start lifecycle; no portable snapshot format or schema version changes. + ## 6. Database, Configuration, and Migration History The SQLite database under `MSB_HOME` is a durable protocol between releases. Host and runtime processes must also agree on WAL, busy timeout, foreign-key, synchronous, and writer settings. diff --git a/Cargo.lock b/Cargo.lock index f4bf7ed7d..66b6bdddb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4342,8 +4342,7 @@ dependencies = [ [[package]] name = "msb-vm-memory" version = "0.18.0-msb.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c31dfbf17e640f6d661ae17e098b87c019821fe3eb4aba0e9eeada4ba678096" +source = "git+https://github.com/superradcompany/rust-vmm?rev=f798d4f274db22a3c458ba756900db2cd03e6fe9#f798d4f274db22a3c458ba756900db2cd03e6fe9" dependencies = [ "libc", "thiserror", @@ -4353,7 +4352,7 @@ dependencies = [ [[package]] name = "msb_krun" version = "0.1.33" -source = "git+https://github.com/superradcompany/libkrun?rev=51c1ed3b83dc826c02800fb297995538e4eac55d#51c1ed3b83dc826c02800fb297995538e4eac55d" +source = "git+https://github.com/superradcompany/libkrun?rev=862d68422b40ac7c26c731b3577025a5b1f64b89#862d68422b40ac7c26c731b3577025a5b1f64b89" dependencies = [ "crossbeam-channel", "kvm-bindings", @@ -4372,7 +4371,7 @@ dependencies = [ [[package]] name = "msb_krun_arch" version = "0.1.33" -source = "git+https://github.com/superradcompany/libkrun?rev=51c1ed3b83dc826c02800fb297995538e4eac55d#51c1ed3b83dc826c02800fb297995538e4eac55d" +source = "git+https://github.com/superradcompany/libkrun?rev=862d68422b40ac7c26c731b3577025a5b1f64b89#862d68422b40ac7c26c731b3577025a5b1f64b89" dependencies = [ "kvm-bindings", "kvm-ioctls", @@ -4386,12 +4385,12 @@ dependencies = [ [[package]] name = "msb_krun_arch_gen" version = "0.1.33" -source = "git+https://github.com/superradcompany/libkrun?rev=51c1ed3b83dc826c02800fb297995538e4eac55d#51c1ed3b83dc826c02800fb297995538e4eac55d" +source = "git+https://github.com/superradcompany/libkrun?rev=862d68422b40ac7c26c731b3577025a5b1f64b89#862d68422b40ac7c26c731b3577025a5b1f64b89" [[package]] name = "msb_krun_cpuid" version = "0.1.33" -source = "git+https://github.com/superradcompany/libkrun?rev=51c1ed3b83dc826c02800fb297995538e4eac55d#51c1ed3b83dc826c02800fb297995538e4eac55d" +source = "git+https://github.com/superradcompany/libkrun?rev=862d68422b40ac7c26c731b3577025a5b1f64b89#862d68422b40ac7c26c731b3577025a5b1f64b89" dependencies = [ "kvm-bindings", "kvm-ioctls", @@ -4401,7 +4400,7 @@ dependencies = [ [[package]] name = "msb_krun_devices" version = "0.1.33" -source = "git+https://github.com/superradcompany/libkrun?rev=51c1ed3b83dc826c02800fb297995538e4eac55d#51c1ed3b83dc826c02800fb297995538e4eac55d" +source = "git+https://github.com/superradcompany/libkrun?rev=862d68422b40ac7c26c731b3577025a5b1f64b89#862d68422b40ac7c26c731b3577025a5b1f64b89" dependencies = [ "bincode", "bitflags 1.3.2", @@ -4432,7 +4431,7 @@ dependencies = [ [[package]] name = "msb_krun_hvf" version = "0.1.33" -source = "git+https://github.com/superradcompany/libkrun?rev=51c1ed3b83dc826c02800fb297995538e4eac55d#51c1ed3b83dc826c02800fb297995538e4eac55d" +source = "git+https://github.com/superradcompany/libkrun?rev=862d68422b40ac7c26c731b3577025a5b1f64b89#862d68422b40ac7c26c731b3577025a5b1f64b89" dependencies = [ "crossbeam-channel", "libloading 0.8.9", @@ -4444,7 +4443,7 @@ dependencies = [ [[package]] name = "msb_krun_kernel" version = "0.1.33" -source = "git+https://github.com/superradcompany/libkrun?rev=51c1ed3b83dc826c02800fb297995538e4eac55d#51c1ed3b83dc826c02800fb297995538e4eac55d" +source = "git+https://github.com/superradcompany/libkrun?rev=862d68422b40ac7c26c731b3577025a5b1f64b89#862d68422b40ac7c26c731b3577025a5b1f64b89" dependencies = [ "msb-vm-memory", "msb_krun_utils", @@ -4453,7 +4452,7 @@ dependencies = [ [[package]] name = "msb_krun_polly" version = "0.1.33" -source = "git+https://github.com/superradcompany/libkrun?rev=51c1ed3b83dc826c02800fb297995538e4eac55d#51c1ed3b83dc826c02800fb297995538e4eac55d" +source = "git+https://github.com/superradcompany/libkrun?rev=862d68422b40ac7c26c731b3577025a5b1f64b89#862d68422b40ac7c26c731b3577025a5b1f64b89" dependencies = [ "libc", "msb_krun_utils", @@ -4462,7 +4461,7 @@ dependencies = [ [[package]] name = "msb_krun_smbios" version = "0.1.33" -source = "git+https://github.com/superradcompany/libkrun?rev=51c1ed3b83dc826c02800fb297995538e4eac55d#51c1ed3b83dc826c02800fb297995538e4eac55d" +source = "git+https://github.com/superradcompany/libkrun?rev=862d68422b40ac7c26c731b3577025a5b1f64b89#862d68422b40ac7c26c731b3577025a5b1f64b89" dependencies = [ "msb-vm-memory", ] @@ -4470,7 +4469,7 @@ dependencies = [ [[package]] name = "msb_krun_utils" version = "0.1.33" -source = "git+https://github.com/superradcompany/libkrun?rev=51c1ed3b83dc826c02800fb297995538e4eac55d#51c1ed3b83dc826c02800fb297995538e4eac55d" +source = "git+https://github.com/superradcompany/libkrun?rev=862d68422b40ac7c26c731b3577025a5b1f64b89#862d68422b40ac7c26c731b3577025a5b1f64b89" dependencies = [ "bitflags 1.3.2", "crossbeam-channel", @@ -4485,7 +4484,7 @@ dependencies = [ [[package]] name = "msb_krun_vmm" version = "0.1.33" -source = "git+https://github.com/superradcompany/libkrun?rev=51c1ed3b83dc826c02800fb297995538e4eac55d#51c1ed3b83dc826c02800fb297995538e4eac55d" +source = "git+https://github.com/superradcompany/libkrun?rev=862d68422b40ac7c26c731b3577025a5b1f64b89#862d68422b40ac7c26c731b3577025a5b1f64b89" dependencies = [ "bincode", "bzip2", diff --git a/Cargo.toml b/Cargo.toml index 36f34681e..1afc657af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -208,14 +208,15 @@ russh-sftp = "2.3.0" # #8 construction-time private memory and identity-preserving resume support. [patch.crates-io] -msb_krun = { git = "https://github.com/superradcompany/libkrun", rev = "51c1ed3b83dc826c02800fb297995538e4eac55d" } -msb_krun_utils = { git = "https://github.com/superradcompany/libkrun", rev = "51c1ed3b83dc826c02800fb297995538e4eac55d" } -msb_krun_vmm = { git = "https://github.com/superradcompany/libkrun", rev = "51c1ed3b83dc826c02800fb297995538e4eac55d" } -msb_krun_devices = { git = "https://github.com/superradcompany/libkrun", rev = "51c1ed3b83dc826c02800fb297995538e4eac55d" } -msb_krun_arch = { git = "https://github.com/superradcompany/libkrun", rev = "51c1ed3b83dc826c02800fb297995538e4eac55d" } -msb_krun_arch_gen = { git = "https://github.com/superradcompany/libkrun", rev = "51c1ed3b83dc826c02800fb297995538e4eac55d" } -msb_krun_cpuid = { git = "https://github.com/superradcompany/libkrun", rev = "51c1ed3b83dc826c02800fb297995538e4eac55d" } -msb_krun_hvf = { git = "https://github.com/superradcompany/libkrun", rev = "51c1ed3b83dc826c02800fb297995538e4eac55d" } -msb_krun_kernel = { git = "https://github.com/superradcompany/libkrun", rev = "51c1ed3b83dc826c02800fb297995538e4eac55d" } -msb_krun_polly = { git = "https://github.com/superradcompany/libkrun", rev = "51c1ed3b83dc826c02800fb297995538e4eac55d" } -msb_krun_smbios = { git = "https://github.com/superradcompany/libkrun", rev = "51c1ed3b83dc826c02800fb297995538e4eac55d" } +msb_krun = { git = "https://github.com/superradcompany/libkrun", rev = "862d68422b40ac7c26c731b3577025a5b1f64b89" } +msb_krun_utils = { git = "https://github.com/superradcompany/libkrun", rev = "862d68422b40ac7c26c731b3577025a5b1f64b89" } +msb_krun_vmm = { git = "https://github.com/superradcompany/libkrun", rev = "862d68422b40ac7c26c731b3577025a5b1f64b89" } +msb_krun_devices = { git = "https://github.com/superradcompany/libkrun", rev = "862d68422b40ac7c26c731b3577025a5b1f64b89" } +msb_krun_arch = { git = "https://github.com/superradcompany/libkrun", rev = "862d68422b40ac7c26c731b3577025a5b1f64b89" } +msb_krun_arch_gen = { git = "https://github.com/superradcompany/libkrun", rev = "862d68422b40ac7c26c731b3577025a5b1f64b89" } +msb_krun_cpuid = { git = "https://github.com/superradcompany/libkrun", rev = "862d68422b40ac7c26c731b3577025a5b1f64b89" } +msb_krun_hvf = { git = "https://github.com/superradcompany/libkrun", rev = "862d68422b40ac7c26c731b3577025a5b1f64b89" } +msb_krun_kernel = { git = "https://github.com/superradcompany/libkrun", rev = "862d68422b40ac7c26c731b3577025a5b1f64b89" } +msb_krun_polly = { git = "https://github.com/superradcompany/libkrun", rev = "862d68422b40ac7c26c731b3577025a5b1f64b89" } +msb_krun_smbios = { git = "https://github.com/superradcompany/libkrun", rev = "862d68422b40ac7c26c731b3577025a5b1f64b89" } +msb-vm-memory = { git = "https://github.com/superradcompany/rust-vmm", rev = "f798d4f274db22a3c458ba756900db2cd03e6fe9" } diff --git a/crates/runtime/Cargo.toml b/crates/runtime/Cargo.toml index e95dd42c8..287088b28 100644 --- a/crates/runtime/Cargo.toml +++ b/crates/runtime/Cargo.toml @@ -48,7 +48,7 @@ tracing.workspace = true zeroize.workspace = true [target.'cfg(windows)'.dependencies] -windows-sys = { version = "0.61.2", features = ["Win32_Foundation", "Win32_Storage_FileSystem", "Win32_System_Kernel", "Win32_System_SystemInformation", "Win32_System_Threading"] } +windows-sys = { version = "0.61.2", features = ["Win32_Foundation", "Win32_Security", "Win32_Security_Authorization", "Win32_Storage_FileSystem", "Win32_System_Kernel", "Win32_System_SystemInformation", "Win32_System_Threading"] } [target.'cfg(unix)'.dependencies] microsandbox-agent-client = { workspace = true, features = ["uds"] } diff --git a/crates/runtime/lib/checkpoint/local_memory.rs b/crates/runtime/lib/checkpoint/local_memory.rs index ed6aa8a9a..178691b1c 100644 --- a/crates/runtime/lib/checkpoint/local_memory.rs +++ b/crates/runtime/lib/checkpoint/local_memory.rs @@ -268,9 +268,10 @@ impl MemoryCaptureSink for LocalMemoryCapture { // Tests //-------------------------------------------------------------------------------------------------- -#[cfg(all(test, unix))] +#[cfg(test)] mod tests { use std::io::Read; + #[cfg(unix)] use std::os::unix::fs::PermissionsExt; use super::*; @@ -292,6 +293,7 @@ mod tests { let captured = sink.finish(1, 1).unwrap(); assert_eq!(captured.memory.regions.len(), 2); assert_eq!(captured.memory.regions[1].file_offset, 2 * page); + #[cfg(unix)] assert_eq!( captured._file.metadata().unwrap().permissions().mode() & 0o777, 0o400 diff --git a/crates/runtime/lib/checkpoint/memory_cache.rs b/crates/runtime/lib/checkpoint/memory_cache.rs index fe5cdff61..15fda774c 100644 --- a/crates/runtime/lib/checkpoint/memory_cache.rs +++ b/crates/runtime/lib/checkpoint/memory_cache.rs @@ -79,7 +79,24 @@ impl MemoryCache { page_size: page_size as u64, }) } - #[cfg(not(unix))] + #[cfg(windows)] + { + use windows_sys::Win32::System::SystemInformation::{GetSystemInfo, SYSTEM_INFO}; + let mut info: SYSTEM_INFO = unsafe { std::mem::zeroed() }; + unsafe { + GetSystemInfo(&mut info); + } + std::fs::create_dir_all(&root)?; + restrict_cache_directory(&root)?; + let root = root.join(namespace); + std::fs::create_dir_all(&root)?; + restrict_cache_directory(&root)?; + Ok(Self { + root, + page_size: u64::from(info.dwPageSize), + }) + } + #[cfg(not(any(unix, windows)))] { let _ = (root, namespace); Err(io::Error::new( @@ -246,6 +263,9 @@ impl MemoryCache { staging.set_permissions(std::fs::Permissions::from_mode(0o400))?; } staging.sync_all()?; + // Windows readers deliberately deny write sharing. Close the completed writer before + // publishing/opening its immutable view; keeping it open would cause a sharing violation. + drop(staging); // Keep no-replacement publication even under the build lock: older builders may not // participate in single-flight, and eviction must never replace a live mapped inode. match std::fs::hard_link(&staging_path, &path) { @@ -359,6 +379,17 @@ pub(super) fn evict_unpinned(path: &Path) -> io::Result { return Ok(false); } } + #[cfg(windows)] + { + let current = match open_readonly(path) { + Ok(file) => file, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error), + }; + if windows_file_identity(&file)? != windows_file_identity(¤t)? { + return Ok(false); + } + } std::fs::remove_file(path)?; Ok(true) } @@ -371,6 +402,12 @@ fn open_readonly(path: &Path) -> io::Result { use std::os::unix::fs::OpenOptionsExt; options.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK); } + #[cfg(windows)] + { + use std::os::windows::fs::OpenOptionsExt; + use windows_sys::Win32::Storage::FileSystem::{FILE_SHARE_DELETE, FILE_SHARE_READ}; + options.share_mode(FILE_SHARE_READ | FILE_SHARE_DELETE); + } options.open(path) } @@ -386,20 +423,70 @@ pub(super) fn open_pinned(path: &Path, length: u64) -> io::Result> "memory cache entry has invalid type or length; evict and rebuild it", )); } - #[cfg(unix)] + microsandbox_utils::process_lock::lock_shared(&file)?; + Ok(Some(file)) +} + +#[cfg(windows)] +fn windows_file_identity(file: &File) -> io::Result<(u32, u32, u32)> { + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::Storage::FileSystem::{ + BY_HANDLE_FILE_INFORMATION, GetFileInformationByHandle, + }; + let mut info: BY_HANDLE_FILE_INFORMATION = unsafe { std::mem::zeroed() }; + if unsafe { GetFileInformationByHandle(file.as_raw_handle(), &mut info) } == 0 { + return Err(io::Error::last_os_error()); + } + Ok(( + info.dwVolumeSerialNumber, + info.nFileIndexHigh, + info.nFileIndexLow, + )) +} + +/// Guest RAM must not inherit broad read permissions from a custom cache parent. +#[cfg(windows)] +fn restrict_cache_directory(path: &Path) -> io::Result<()> { + use std::os::windows::ffi::OsStrExt; + use windows_sys::Win32::Foundation::LocalFree; + use windows_sys::Win32::Security::Authorization::ConvertStringSecurityDescriptorToSecurityDescriptorW; + use windows_sys::Win32::Security::{ + DACL_SECURITY_INFORMATION, PROTECTED_DACL_SECURITY_INFORMATION, SetFileSecurityW, + }; + // OWNER RIGHTS follows the actual owner; SYSTEM is retained for OS maintenance. Children + // inherit these ACEs. This is local-user confidentiality, not an adversarial-host boundary. + let sddl: Vec = "D:P(A;OICI;FA;;;OW)(A;OICI;FA;;;SY)\0" + .encode_utf16() + .collect(); + let mut descriptor = std::ptr::null_mut(); + if unsafe { + ConvertStringSecurityDescriptorToSecurityDescriptorW( + sddl.as_ptr(), + 1, + &mut descriptor, + std::ptr::null_mut(), + ) + } == 0 { - use std::os::fd::AsRawFd; - loop { - if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_SH) } == 0 { - break; - } - let error = io::Error::last_os_error(); - if error.kind() != io::ErrorKind::Interrupted { - return Err(error); - } - } + return Err(io::Error::last_os_error()); } - Ok(Some(file)) + let path: Vec = path.as_os_str().encode_wide().chain(Some(0)).collect(); + let success = unsafe { + SetFileSecurityW( + path.as_ptr(), + DACL_SECURITY_INFORMATION | PROTECTED_DACL_SECURITY_INFORMATION, + descriptor, + ) + }; + let result = if success == 0 { + Err(io::Error::last_os_error()) + } else { + Ok(()) + }; + unsafe { + LocalFree(descriptor); + } + result } fn invalid(message: &str) -> io::Error { @@ -410,11 +497,25 @@ fn invalid(message: &str) -> io::Error { // Tests //-------------------------------------------------------------------------------------------------- -#[cfg(all(test, unix))] +#[cfg(test)] mod tests { use super::*; use microsandbox_image::checkpoint::{ContentRef, MemoryCaptureMode, MemoryExtent}; + #[cfg(unix)] use std::os::unix::fs::FileExt; + #[cfg(windows)] + trait ReadAt { + fn read_exact_at(&self, bytes: &mut [u8], offset: u64) -> io::Result<()>; + } + #[cfg(windows)] + impl ReadAt for File { + fn read_exact_at(&self, bytes: &mut [u8], offset: u64) -> io::Result<()> { + use std::io::Read; + let mut file = self.try_clone()?; + file.seek(SeekFrom::Start(offset))?; + file.read_exact(bytes) + } + } fn fixture(page: u64) -> (MemoryManifest, ObjectId, Vec) { let bytes = vec![0x5a; page as usize]; @@ -620,6 +721,7 @@ mod tests { #[test] fn concurrent_builders_publish_one_immutable_inode() { + #[cfg(unix)] use std::os::unix::fs::MetadataExt; let directory = tempfile::tempdir().unwrap(); let cache = MemoryCache::open(directory.path()).unwrap(); @@ -640,10 +742,16 @@ mod tests { let second = scope.spawn(run); let first = first.join().unwrap(); let second = second.join().unwrap(); + #[cfg(unix)] assert_eq!( first.file.metadata().unwrap().ino(), second.file.metadata().unwrap().ino() ); + #[cfg(windows)] + assert_eq!( + windows_file_identity(&first.file).unwrap(), + windows_file_identity(&second.file).unwrap() + ); }); assert_eq!(reads.load(std::sync::atomic::Ordering::Relaxed), 1); assert_eq!( diff --git a/crates/runtime/lib/control/executor.rs b/crates/runtime/lib/control/executor.rs index 71811854d..ecd3f5554 100644 --- a/crates/runtime/lib/control/executor.rs +++ b/crates/runtime/lib/control/executor.rs @@ -516,7 +516,7 @@ impl RuntimeControlExecutor { memory_resize: self.vm.memory_resize_supported(), secrets_update: self.secrets_update_supported(), checkpoint_create: true, - branch_create: cfg!(unix), + branch_create: cfg!(any(unix, windows)), disk_compact: true, root_disk_grow: true, pause_resume: self.vm.clock_sync_supported(), diff --git a/crates/utils/lib/process_lock.rs b/crates/utils/lib/process_lock.rs index 04a43d07f..fd6a6cc31 100644 --- a/crates/utils/lib/process_lock.rs +++ b/crates/utils/lib/process_lock.rs @@ -43,6 +43,38 @@ pub fn lock_exclusive(file: &File) -> io::Result<()> { lock_exclusive_inner(file, false).map(|_| ()) } +/// Pins immutable data against cooperative exclusive eviction until the file closes. +pub fn lock_shared(file: &File) -> io::Result<()> { + #[cfg(unix)] + loop { + if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_SH) } == 0 { + return Ok(()); + } + let error = io::Error::last_os_error(); + if error.kind() != io::ErrorKind::Interrupted { + return Err(error); + } + } + #[cfg(windows)] + { + let mut overlapped: OVERLAPPED = unsafe { std::mem::zeroed() }; + let result = unsafe { + LockFileEx( + file.as_raw_handle() as HANDLE, + 0, + 0, + u32::MAX, + u32::MAX, + &mut overlapped, + ) + }; + if result == 0 { + return Err(io::Error::last_os_error()); + } + Ok(()) + } +} + /// Attempts to acquire an exclusive process-held lock without blocking. /// /// Returns `Ok(false)` only when another process currently owns the lock. diff --git a/docs/sandboxes/snapshots.mdx b/docs/sandboxes/snapshots.mdx index fcfe3889b..9bd0ac61e 100644 --- a/docs/sandboxes/snapshots.mdx +++ b/docs/sandboxes/snapshots.mdx @@ -25,7 +25,7 @@ Both modes produce the same schema-1 `snapshot.json` descriptor. Its closed `sta ## Share restored memory with CoW -On Linux and macOS, add `--forked` when restoring a full snapshot to share clean memory pages between children. Writes stay private to each child. Without the flag, restore eagerly copies memory. The source needs no special creation option. +On supported Linux, macOS, and Windows hosts, add `--forked` when restoring a full snapshot to share clean memory pages between children. Writes stay private to each child. Without the flag, restore eagerly copies memory. The source needs no special creation option. ```bash msb create alpine --name baseline --root-disk flat:1G @@ -36,7 +36,9 @@ msb create --name worker-b --from-snapshot ready --forked The SDK creation options are Rust `.forked()`, Python `forked=True`, TypeScript `.forked()`, and Go `WithForked()`. The option requires a full snapshot and cannot be combined with `--disk-only` or an image cold boot. -Snapshots are still manual. CoW uses a protected local memory cache; the first uncached restore must build it, while later children reuse it. Captures from forked children can prepare later restore backing with filesystem reflinks when available. Removing the input archive does not invalidate a running child's backing. Windows CoW and explicit NUMA placement with CoW are not supported yet; requests fail instead of silently switching modes. +Snapshots are still manual. CoW uses a protected local memory cache; the first uncached restore must build it, while later children reuse it. Captures from forked children can prepare later restore backing with filesystem reflinks when available. Removing the input archive does not invalidate a running child's backing. Explicit NUMA placement with CoW is not supported yet; requests fail instead of silently switching modes. + +If a restore fails after creating a child record, remove that child and create a new one from the snapshot. An incomplete restore cannot be started as a fresh VM or modified; the original snapshot remains reusable. ## Branch a running sandbox @@ -46,7 +48,7 @@ Use `branch` for an independent local child without first saving a full snapshot msb branch baseline --name worker ``` -The child resumes the captured processes, memory, and disk state. Its writes do not affect the source. CoW memory is built in—no `--forked` flag is needed. A running source resumes after capture; a user-paused source stays paused. Branches work on supported Linux and macOS hosts, require a new child name, and cannot inherit published host ports. +The child resumes the captured processes, memory, and disk state. Its writes do not affect the source. CoW memory is built in—no `--forked` flag is needed. A running source resumes after capture; a user-paused source stays paused. Branches work on supported Linux, macOS, and Windows hosts, require a new child name, and cannot inherit published host ports. The SDK methods are Rust `source.branch("worker").await?`, Python `await source.branch("worker")`, TypeScript `await source.branch("worker")`, and Go `source.Branch(ctx, "worker")`. They also work on sandbox handles returned by `get`. diff --git a/scripts/smoke/cli/branch-ownership.py b/scripts/smoke/cli/branch-ownership.py index 8040feda5..639757b46 100644 --- a/scripts/smoke/cli/branch-ownership.py +++ b/scripts/smoke/cli/branch-ownership.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 -"""Live Unix pin lifetime and same-name reservation checks for direct branching.""" +"""Live pin lifetime and same-name reservation checks for direct branching.""" -import fcntl import json import os from pathlib import Path @@ -23,6 +22,31 @@ def call(*args, expected=0): def evictable(path): # Same OS primitive used by production eviction. Never unlink or modify live backing. with path.open("rb") as file: + if os.name == "nt": + import ctypes + from ctypes import wintypes + import msvcrt + + class Overlapped(ctypes.Structure): + _fields_ = [("internal", ctypes.c_size_t), ("internal_high", ctypes.c_size_t), + ("offset", wintypes.DWORD), ("offset_high", wintypes.DWORD), + ("event", wintypes.HANDLE)] + + kernel = ctypes.WinDLL("kernel32", use_last_error=True) + kernel.LockFileEx.argtypes = [wintypes.HANDLE, wintypes.DWORD, wintypes.DWORD, + wintypes.DWORD, wintypes.DWORD, ctypes.POINTER(Overlapped)] + kernel.UnlockFileEx.argtypes = [wintypes.HANDLE, wintypes.DWORD, wintypes.DWORD, + wintypes.DWORD, ctypes.POINTER(Overlapped)] + handle = msvcrt.get_osfhandle(file.fileno()) + overlap = Overlapped() + # Fail-immediately + exclusive, over the same whole-file range as production. + if not kernel.LockFileEx(handle, 3, 0, 0xffffffff, 0xffffffff, ctypes.byref(overlap)): + error = ctypes.get_last_error() + assert error == 33, ctypes.WinError(error) + return False + assert kernel.UnlockFileEx(handle, 0, 0xffffffff, 0xffffffff, ctypes.byref(overlap)) + return True + import fcntl try: fcntl.flock(file, fcntl.LOCK_EX | fcntl.LOCK_NB) except BlockingIOError: @@ -38,7 +62,8 @@ def evictable(path): paths = set(cache.glob("*.ram")) - before assert len(paths) == 1 backing = paths.pop() - assert backing.stat().st_mode & 0o777 == 0o400 + if os.name != "nt": + assert backing.stat().st_mode & 0o777 == 0o400 assert not evictable(backing), "source/child pins disappeared" assert not (home / "sandboxes" / names[1] / ".branch-restore").exists() attempts = [subprocess.Popen([binary, "branch", prefix, "--name", names[2]], stdout=subprocess.PIPE, stderr=subprocess.PIPE) for _ in range(2)] diff --git a/scripts/smoke/cli/branch-timer-progress.py b/scripts/smoke/cli/branch-timer-progress.py new file mode 100644 index 000000000..8cc81cac9 --- /dev/null +++ b/scripts/smoke/cli/branch-timer-progress.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Repeated branching with retained siblings and a timer-driven guest workload. + +Use an isolated MSB_HOME, matching MSB_PATH/MSB_LIBKRUNFW_PATH, and unique +STACK8_PREFIX/STACK8_OUT. STACK8_REPEATS defaults to 100 and STACK8_RETAIN to 8. +Retained immutable RAM cache entries need disk space even after VMs stop. +""" +import json +import os +from pathlib import Path +import subprocess +import time + +binary = os.environ["MSB_PATH"] +out = Path(os.environ["STACK8_OUT"]) +out.mkdir(parents=True, exist_ok=True) +prefix = os.environ["STACK8_PREFIX"] +repeats = int(os.environ.get("STACK8_REPEATS", "100")) +retain = int(os.environ.get("STACK8_RETAIN", "8")) +assert repeats > 0 and retain > 0 +source = prefix + "-source" +names = [source] +live_children = [] +rows = [] + + +def run(label, *args, check=True): + start = time.monotonic() + try: + result = subprocess.run([binary, *args], capture_output=True, text=True, timeout=30) + except subprocess.TimeoutExpired: + rows.append({"case": label, "exit": "timeout"}) + raise + row = {"case": label, "exit": result.returncode, + "ms": round((time.monotonic() - start) * 1000, 2)} + rows.append(row) + (out / (label + ".stdout")).write_text(result.stdout) + (out / (label + ".stderr")).write_text(result.stderr) + if check: + assert result.returncode == 0, (row, result.stderr) + return result.stdout.strip() + + +try: + run("create", "create", "alpine", "--name", source, "--root-disk", "tmpfs:128M", + "--memory", "256M", "--cpus", "2") + # Atomic replacement prevents a concurrent reader mistaking a truncated + # counter file for a stalled timer. A background process survives exec exit. + run("prepare", "exec", source, "--", "sh", "-c", + "sh -c 'i=0; while :; do i=$((i+1)); echo $i > /dev/shm/count.next; " + "mv /dev/shm/count.next /dev/shm/count; sleep 0.02; done' " + ">/tmp/counter.log 2>&1 retain: + run("retire-" + str(index), "stop", live_children.pop(0)) + print(json.dumps({"branch": index, "timer_progress": "pass"}), flush=True) +finally: + # Failed creation must not be followed by exec/start: that would test an + # unintended cold boot instead of the failed restore. Stop is always safe. + for name in reversed(names): + try: + run("cleanup-" + name, "stop", name, check=False) + except Exception as error: + rows.append({"case": "cleanup-" + name, "error": str(error)}) + (out / "results.json").write_text(json.dumps(rows, indent=2)) diff --git a/scripts/smoke/cli/checkpoint-cpu-state.py b/scripts/smoke/cli/checkpoint-cpu-state.py index 78d86a024..68aef4103 100644 --- a/scripts/smoke/cli/checkpoint-cpu-state.py +++ b/scripts/smoke/cli/checkpoint-cpu-state.py @@ -40,7 +40,8 @@ def run(label, *args, check=True): run("offline", "exec", source, "--", "sh", "-c", "echo 0 > /sys/devices/system/cpu/cpu1/online") assert run("offline-before", "exec", source, "--", "cat", "/sys/devices/system/cpu/cpu1/online") == b"0" run("capture", "snapshot", "create", prefix + "-full", "--from", source, "--full", "--info") - run("restore", "create", "-n", child, "--from-snapshot", prefix + "-full", "--info") + run("restore", "create", "-n", child, "--from-snapshot", prefix + "-full", + *(["--forked"] if os.environ.get("CPU_FORKED") == "1" else []), "--info") assert run("offline-after", "exec", child, "--", "cat", "/sys/devices/system/cpu/cpu1/online") == b"0" run("cpu0-restored", "exec", child, "--", "/cpu-probe", "0") run("online", "exec", child, "--", "sh", "-c", "echo 1 > /sys/devices/system/cpu/cpu1/online") diff --git a/scripts/smoke/cli/failed-restore.py b/scripts/smoke/cli/failed-restore.py new file mode 100644 index 000000000..165647a06 --- /dev/null +++ b/scripts/smoke/cli/failed-restore.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Late restore failure must preserve sealed bytes and refuse cold-boot paths. + +Run only with an isolated MSB_HOME: this temporarily blocks its memory cache. +""" +import hashlib +import json +import os +from pathlib import Path +import subprocess +import time + +assert os.environ.get("MSB_TEST_DISPOSABLE_HOME") == "1", "requires an isolated test home" +binary = os.environ["MSB_PATH"] +home = Path(os.environ["MSB_HOME"]) +out = Path(os.environ["STACK8_OUT"]) +out.mkdir(parents=True, exist_ok=True) +prefix = os.environ.get("STACK8_PREFIX", "failed-restore-" + str(os.getpid())) +source, child, healthy = (prefix + suffix for suffix in ("-source", "-failed", "-healthy")) +snapshot = prefix + "-saved" +cache = home / "cache" / "memory" +held = cache.with_name("memory-held-" + prefix) +rows = [] + +def call(label, *args, expected=0): + start = time.monotonic() + result = subprocess.run([binary, *args], capture_output=True, text=True, timeout=120) + rows.append(dict(case=label, exit=result.returncode, ms=(time.monotonic()-start)*1000, + stdout=result.stdout, stderr=result.stderr)) + if expected is not None: + assert result.returncode == expected, rows[-1] + return result + +def layers(): + root = home / "snapshots" / snapshot + paths = sorted(p for p in root.rglob('*') if p.is_file() and p.suffix in ('.raw', '.qcow2', '.ext4')) + assert paths, "fixture must include sealed disk bytes" + result = {} + for path in paths: + with path.open('rb') as file: + digest = hashlib.sha256() + for chunk in iter(lambda: file.read(1024 * 1024), b''): + digest.update(chunk) + result[str(path.relative_to(root))] = digest.hexdigest() + return result + +blocked = False +try: + call("source", "create", "alpine", "--name", source, "--root-disk", os.environ.get("STACK8_LAYOUT", "flat:512M"), "--memory", "256M") + call("marker", "exec", source, "--", "sh", "-c", "echo preserved > /dev/shm/restore-marker; echo disk-preserved > /restore-disk-marker") + call("capture", "snapshot", "create", snapshot, "--from", source, "--full") + before = layers() + # Trigger a real host I/O error during memory installation, after child staging + # and DB insertion. No production test-only failure hook is necessary. + assert not held.exists() + if cache.exists(): + cache.rename(held) + cache.write_bytes(b"intentional isolated test obstruction") + blocked = True + failure = call("restore-fails", "create", "--name", child, "--from-snapshot", snapshot, "--forked", expected=None) + assert failure.returncode != 0 + call("failed-row-exists", "inspect", child) + cache.unlink() + blocked = False + if held.exists(): + held.rename(cache) + for label, args in [("start", ["start", child]), ("exec", ["exec", child, "--", "true"]), + ("modify", ["modify", child, "--root-disk", "8G"]), + ("compact", ["modify", child, "--compact"]), + ("snapshot", ["snapshot", "create", prefix+'-invalid', "--from", child])]: + refused = call(label + "-refused", *args, expected=None) + assert refused.returncode != 0 and "incomplete restore" in refused.stderr, rows[-1] + assert layers() == before, "failed restore or later lifecycle mutated sealed disk bytes" + call("healthy-restore", "create", "--name", healthy, "--from-snapshot", snapshot, "--forked") + assert call("healthy-marker", "exec", healthy, "--", "cat", "/dev/shm/restore-marker").stdout.strip() == "preserved" + call("healthy-stop", "stop", healthy) + call("healthy-later-start", "start", healthy) + assert call("healthy-disk-marker", "exec", healthy, "--", "cat", "/restore-disk-marker").stdout.strip() == "disk-preserved" + assert layers() == before, "ordinary later startup mutated the original sealed snapshot" + print(json.dumps({"late_failure": "pass", "start_exec_modify_compact_snapshot_refused": "pass", "sealed_bytes": "unchanged", "fresh_restore_and_later_start": "pass"})) +finally: + if blocked: + cache.unlink() + if held.exists(): + held.rename(cache) + for name in (healthy, child, source): + call("cleanup-" + name, "stop", name, expected=None) + (out / "results.json").write_text(json.dumps(rows, indent=2)) diff --git a/scripts/smoke/reports/cow-platform-fixes-2026-09-09.md b/scripts/smoke/reports/cow-platform-fixes-2026-09-09.md new file mode 100644 index 000000000..cb6c59105 --- /dev/null +++ b/scripts/smoke/reports/cow-platform-fixes-2026-09-09.md @@ -0,0 +1,56 @@ +# CoW platform fixes — 2026-09-09 + +Development qualification of the #8 changes on top of Microsandbox `49b6670d`. The tested companion sources are published and pinned to libkrun `862d68422b40ac7c26c731b3577025a5b1f64b89` and rust-vmm `f798d4f274db22a3c458ba756900db2cd03e6fe9`. Firmware is unchanged from the matching execution-state test build. The live test builds used explicit local source overrides for those sources before publication. These are debug-build correctness runs, not release performance benchmarks. + +## Changes + +- Windows uses native private file views for restored RAM. All GPA slots are slices of one owned view, so 4 KiB slot offsets do not require separate 64 KiB-aligned mappings. Siblings cannot change each other's RAM or the backing file. The final view owner unmaps it exactly once. +- The Windows RAM cache has owner/SYSTEM-only directory ACLs, immutable reader sharing, shared lifetime pins, and identity-checked exclusive eviction. The completed writer closes before readers open the published file. +- Pending restore intent survives database insertion and process failure. It is cleared only after restore activation and create finalization. A failed child cannot later start as an ordinary VM, auto-start through exec, grow or compact its staged disk, or create a snapshot. Remove/recreate it from the original input. Successful children retain normal later stop/start behavior. +- ARM64 KVM restores distributor control before pending levels and interrupt enables, with every vCPU still paused. KVM's userspace `GICD_CTLR` setter changes the enable flag without requeueing pending interrupts; enabling it last can strand an already-high timer line. See the [Linux VGIC implementation](https://github.com/torvalds/linux/blob/v6.12/arch/arm64/kvm/vgic/vgic-mmio-v3.c). +- Resume acknowledgements now get the configured one-second barrier deadline. Only pause uses ten-millisecond sub-waits for periodic kicks. This corrects an accidental ten-millisecond resume deadline, without adding delay to successful acknowledgements. + +## Live coverage + +| Check | macOS ARM64/HVF | Linux x86-64/KVM | Windows ARM64/WHP | +| --- | --- | --- | --- | +| Full eager/forked restore, flat and layered roots | Pass | Pass | Pass | +| Direct full archive capture/restore and input-archive unlink | Pass | Pass | Pass | +| Pause/resume, idempotence, capture while paused | Pass | Pass | Pass | +| Source/sibling RAM isolation and modified-child capture/restore | Pass | Pass | Pass | +| Direct branch, paused-source branch, branch of branch; flat/layered/tmpfs | Pass | Pass | Pass | +| Independent backing pins, final release, same-name race | Pass | Pass | Pass after porting the test's Unix-only lock probe | +| Failed restore refuses start/exec/modify/compact/snapshot; sealed bytes unchanged | Pass | Pass | Pass | +| Fresh restore after failure, then ordinary stop/start | Pass | Pass | Pass | +| Retained-sibling timer-progress regression | 20/20 | 20/20 | 20/20 | +| Branch after live root growth and compaction | Previously passed; not added to final Mac rerun | Previously passed; not added to final x86 rerun | Pass, flat and layered | +| Delayed incremental archive forked restore: wall clock, monotonic/boottime, timers | Prior coverage retained | Prior coverage retained | Pass | +| Offline CPU retained and subsequently onlined after forked restore | Prior eager coverage retained | Prior eager coverage retained | Pass | + +Linux ARM64 nested-KVM also passed all eight final matrix suites, including branch-after-growth/compaction for flat and layered disks. Its failed-restore test passed every refusal, snapshot byte preservation, fresh restore, and later stop/start. Delayed incremental archive forked restore passed wall-clock, monotonic/boottime, and relative-timer checks. The offline-CPU forked restore and later CPU online check passed. The committed timer-progress fixture passed another 8/8 iterations after the 100-iteration diagnostic run. Windows x86-64 was deliberately not tested. This report does not claim every possible failure injection, memory-pressure scenario, NUMA configuration, or SDK-language/platform combination has been qualified. + +## ARM64 diagnosis and regression + +The original retained-sibling loop repeatedly hung after successful restore activation. The same failure occurred with forced full RAM capture, eager anonymous memory, and a single vCPU; those experiments did not fix it. A cold-start control and an ordinary pause/resume control each passed 100 iterations. + +A stuck guest had both CPUs idle, virtual timers enabled/unmasked with expired deadlines, and timer interrupt line levels high, but no active interrupt. Reasserting the pending interrupt recovered workload progress in a disposable diagnostic guest. That experiment was not shipped as a workaround. Inspection of KVM's userspace distributor-enable semantics identified the ordering bug above. No forced eager fallback, synthetic timer injection, periodic wakeup, or disabled incremental capture remains in the production changes. + +The corrected-order run first exposed the separate resume-deadline bug. With both fixes, a subsequent run passed 20 retained-child iterations before the disposable 32 GiB VM exhausted disk space. Unpinned local RAM cache entries were evicted under exclusive file locks; snapshots and pinned backings were not removed. A fresh run then passed all 100 retained-child iterations, including first exec and continuing timer-driven workload progress, with no failed cleanup. The final broader ARM64 matrix and focused regressions also passed. + +## Reproduce and evidence + +- `scripts/smoke/cli/failed-restore.py`: requires `MSB_TEST_DISPOSABLE_HOME=1`, deliberately obstructs only the disposable memory cache, retains the failed database row, verifies every refusal and sealed-file digest, and tests a fresh child's later stop/start. +- `scripts/smoke/cli/branch-timer-progress.py`: repeated direct branches, retained siblings, atomic counter publication, first-command success and continued guest timer progress. Set unique `STACK8_PREFIX`, `STACK8_OUT`, optional `STACK8_REPEATS` and `STACK8_RETAIN`, plus the usual runtime/home/firmware environment. +- Existing `cow-memory-lifecycle.py`, `direct-branch.py`, `branch-ownership.py`, `checkpoint-clock.py`, and `checkpoint-cpu-state.py` supply the broader matrix. `CPU_FORKED=1` selects forked restore for the offline-CPU fixture. +- Mac final matrix: `/private/tmp/msb-cow-fix-final-mac-results`; failure/restart: `/private/tmp/msb-cow-fix-restart-mac`; timer stress: `/private/tmp/msb-cow-fix-mac-stress`. +- OVH final matrix: `/home/ubuntu/msb-cow-fix-final-results`; failure/restart: `/home/ubuntu/msb-cow-fix-restart-results`. A separate benchmark task overlapped some qualification work; do not use these timings as an uncontended performance comparison. +- Surface evidence under `C:\Users\Stephen\AppData\Local\Temp`: `msb-cow-fix-49b6670d\results`, `msb-cow-fix-restart-results`, `msb-cow-fix-maint-flat`, `msb-cow-fix-maint-layered`, `msb-cow-fix-clock-results`, `msb-cow-fix-cpu-results`, and `msb-cow-fix-win-stress`. +- Nested ARM64: `/root/msb-cow-fix-qualified-results` and `/root/msb-cow-fix-arm-final-results` in the disposable QEMU VM; text-only evidence copied to `/private/tmp/msb-cow-fix-evidence/arm64` on the Mac. + +All runners stop their own test VMs in `finally`; final process checks found no remaining runtimes from this pass on the Mac, nested ARM64 VM, or Surface. Three pre-existing Surface branch-5 runtimes were left untouched. The disposable nested ARM64 VM was shut down after evidence collection. Unrelated development VMs are not targets for cleanup. Raw artifacts can contain guest state and are not committed. + +## Other validation + +After publication, the Mac CLI build passed with the pinned Git dependencies and no local source overrides: `cargo build -p microsandbox-cli --no-default-features --features net,ssh`. The generated lockfile changes only the twelve intended dependency sources. Formatting, diff whitespace checks, and Python smoke-script compilation passed. + +The Mac libkrun VMM library tests passed 52/52, including the resume-deadline regression. ARM64 VGIC tests passed 3/3, including the distributor/pending-state ordering regression. The failed-restore database regression passed. Runtime checkpoint tests passed 31/31 on Mac and Windows. The Windows public Rust mapping harness passed alias/sibling/backing isolation, bounds, parent-drop, and mapped-file-unlink checks. The standalone vm-memory unit harness cannot currently compile on Windows because its existing vmm-sys-util development dependency references Unix clock APIs; this is not reported as a passing unit run. The production mapping code was exercised by both the standalone Rust binary and the live WHP matrix. diff --git a/scripts/smoke/reports/execution-state-2026-09-07.md b/scripts/smoke/reports/execution-state-2026-09-07.md index 9d3f0fe93..7f780b9a1 100644 --- a/scripts/smoke/reports/execution-state-2026-09-07.md +++ b/scripts/smoke/reports/execution-state-2026-09-07.md @@ -1,5 +1,7 @@ # Execution-state fixes — 2026-09-07 +Follow-up: [CoW platform fixes and qualification](cow-platform-fixes-2026-09-09.md) covers Windows private memory, failed-restore lifecycle protection, and the later ARM64 pending-interrupt regression. The backend revision and observations below remain historical. + The Windows ARM64 post-restore command hang is fixed in the tested cases. Linux ARM64 now has working full execution-state capture/restore with VGICv3. Windows x86-64 has a new implementation with successful cross-compilation and executable userspace tests, but no native x86 WHP live qualification. This report does not mark all of #8 complete. Backend revision: libkrun `51c1ed3b83dc826c02800fb297995538e4eac55d`. Firmware remains `6cca413ac248f63e65d4ea4748b3bc36cd1b22f3`, using matching ARM64 kernel and agentd builds. Microsandbox additionally captures device state before interrupt-controller state, and captures RAM afterward. Public CLI/SDK signatures and disk-only snapshot formats are unchanged. Old Windows ARM64 development full snapshots require recapture because the internal execution-state ABI now includes CPU activity and clock-frequency state. diff --git a/sdk/rust/lib/backend/local/sandbox/create.rs b/sdk/rust/lib/backend/local/sandbox/create.rs index 23c6cdf17..479d1c595 100644 --- a/sdk/rust/lib/backend/local/sandbox/create.rs +++ b/sdk/rust/lib/backend/local/sandbox/create.rs @@ -605,7 +605,10 @@ impl LocalBackend { // Insert the sandbox record and keep its stable database ID. let write_db = db.write(); - let persisted_config = config.clone_for_persistence(); + let mut persisted_config = config.clone_for_persistence(); + // Persist pending restore intent before a row can be discovered by start/exec. Only + // successful creation clears it; process errors and client death leave a safe refusal. + persisted_config.checkpoint_restore = config.checkpoint_restore.clone(); let sandbox_id = match Self::insert_sandbox_record(write_db, &persisted_config).await { Ok(sandbox_id) => sandbox_id, Err(err) => { @@ -629,15 +632,6 @@ impl LocalBackend { let created = self .create_sandbox_inner(config, sandbox_id, mode, Some(lifecycle_guard)) .await; - if let Some(closure) = restore_closure - && let Err(error) = remove_dir_if_exists(&closure) - { - tracing::warn!( - error = %error, - path = %closure.display(), - "failed to remove consumed eager checkpoint closure" - ); - } let (local_state, mut returned_config) = match created { Ok(pair) => pair, Err(e) => { @@ -726,9 +720,46 @@ impl LocalBackend { } } + if let Some(closure) = restore_closure { + // Do not lose the recovery discriminator if any preceding creation check failed. + // RAM/device state has been consumed and the runtime owns its disk chain and pins. + if let Err(error) = Self::complete_sandbox_restore(write_db, sandbox_id).await { + let _ = sandbox.stop().await; + return Err(error); + } + if let Err(error) = remove_dir_if_exists(&closure) { + tracing::warn!(error = %error, path = %closure.display(), "failed to remove consumed checkpoint closure"); + } + } Ok(sandbox) } + /// Clear only the pending construction intent, preserving any concurrent desired edits. + async fn complete_sandbox_restore( + db: &DbWriteConnection, + sandbox_id: i32, + ) -> MicrosandboxResult<()> { + sandbox_entity::Entity::update_many() + .col_expr( + sandbox_entity::Column::Config, + Expr::cust("json_remove(config, '$.checkpoint_restore')"), + ) + .filter(sandbox_entity::Column::Id.eq(sandbox_id)) + .exec(db) + .await?; + Ok(()) + } + + pub(crate) fn validate_completed_restore(config: &SandboxConfig) -> MicrosandboxResult<()> { + if config.checkpoint_restore.is_some() { + return Err(crate::MicrosandboxError::InvalidConfig(format!( + "sandbox {:?} has an incomplete restore; remove and recreate it from the snapshot; refusing a cold boot", + config.spec.name + ))); + } + Ok(()) + } + /// Inner local create logic separated for error-cleanup wrapper. Returns /// the local-variant state plus the (possibly mutated) config. pub(super) async fn create_sandbox_inner( @@ -1911,6 +1942,54 @@ mod tests { fs::remove_dir(path).unwrap(); } + #[tokio::test] + async fn incomplete_restore_survives_failure_until_explicit_completion() { + let temp = tempdir().unwrap(); + let pools = open_test_pools(&temp.path().join("test.db")).await; + let mut config = test_config_with_rootfs("pending", bind_rootfs(temp.path().to_path_buf())); + config.checkpoint_restore = Some(microsandbox_runtime::launch::CheckpointRestoreConfig { + local_branch: false, + forked: true, + closure: temp.path().join("checkpoint"), + checkpoint_root: "blake3:pending".into(), + checkpoint_id: "pending".into(), + }); + let id = LocalBackend::insert_sandbox_record(pools.write(), &config) + .await + .unwrap(); + LocalBackend::update_sandbox_status(pools.write(), id, SandboxStatus::Stopped) + .await + .unwrap(); + let model = sandbox_entity::Entity::find_by_id(id) + .one(pools.read()) + .await + .unwrap() + .unwrap(); + let pending: SandboxConfig = serde_json::from_str(&model.config).unwrap(); + assert!( + LocalBackend::validate_completed_restore(&pending) + .unwrap_err() + .to_string() + .contains("refusing a cold boot") + ); + assert!(pending.checkpoint_restore.as_ref().unwrap().forked); + + // Ordinary post-success/snapshot projections must not perpetuate one-shot restore input. + assert!(pending.clone_for_persistence().checkpoint_restore.is_none()); + LocalBackend::complete_sandbox_restore(pools.write(), id) + .await + .unwrap(); + let model = sandbox_entity::Entity::find_by_id(id) + .one(pools.read()) + .await + .unwrap() + .unwrap(); + let completed: SandboxConfig = serde_json::from_str(&model.config).unwrap(); + assert!(completed.checkpoint_restore.is_none()); + assert_eq!(completed.spec.name, "pending"); + LocalBackend::validate_completed_restore(&completed).unwrap(); + } + #[tokio::test] async fn test_persist_oci_manifest_pin_upserts_rootfs_record() { let temp = tempdir().unwrap(); diff --git a/sdk/rust/lib/backend/local/sandbox/mod.rs b/sdk/rust/lib/backend/local/sandbox/mod.rs index da7994358..158df6e74 100644 --- a/sdk/rust/lib/backend/local/sandbox/mod.rs +++ b/sdk/rust/lib/backend/local/sandbox/mod.rs @@ -137,6 +137,9 @@ impl LocalBackend { } let mut config: SandboxConfig = serde_json::from_str(&model.config)?; + // A failed or interrupted first restore is not a stopped ordinary VM. In particular, + // its sealed base may be hard-linked to a snapshot and must never become a boot disk. + Self::validate_completed_restore(&config)?; self.apply_deployment_profile(&mut config); config.apply_runtime_defaults(); validate_hostname(config.spec.runtime.hostname.as_deref())?; diff --git a/sdk/rust/lib/sandbox/compact.rs b/sdk/rust/lib/sandbox/compact.rs index 3a628f330..621667b8d 100644 --- a/sdk/rust/lib/sandbox/compact.rs +++ b/sdk/rust/lib/sandbox/compact.rs @@ -65,6 +65,7 @@ impl DiskCompactionBuilder { .await? .ok_or_else(|| MicrosandboxError::SandboxNotFound(self.name.clone()))?; let config: SandboxConfig = serde_json::from_str(&model.config)?; + crate::LocalBackend::validate_completed_restore(&config)?; use microsandbox_types::RootDisk; if config.manifest_digest.is_none() || matches!( @@ -111,6 +112,8 @@ impl DiskCompactionBuilder { )); } let runtime_dir = local.sandboxes_dir().join(&self.name).join("runtime"); + let current_config: SandboxConfig = serde_json::from_str(¤t.config)?; + crate::LocalBackend::validate_completed_restore(¤t_config)?; tokio::task::spawn_blocking(move || { // Dropping an SDK future does not cancel spawn_blocking. Keep disk ownership in the // worker until it finishes, even when its caller disconnects or cancels the await. diff --git a/sdk/rust/lib/sandbox/config.rs b/sdk/rust/lib/sandbox/config.rs index 49a9e598c..ef032f0a6 100644 --- a/sdk/rust/lib/sandbox/config.rs +++ b/sdk/rust/lib/sandbox/config.rs @@ -213,11 +213,13 @@ pub struct SandboxConfig { #[serde(skip)] pub(crate) snapshot_base: Option, - /// Child-owned checkpoint closure used only for this process construction. + /// Child-owned checkpoint closure for an unfinished restore construction. /// /// The builder initially points this at an installed snapshot. The local create path copies - /// the closure into child staging and rewrites the path before spawning the runtime. - #[serde(skip)] + /// the closure into child staging and rewrites the path before spawning the runtime. Local + /// creation persists this intent until activation succeeds; an interrupted restore must not + /// subsequently be interpreted as an ordinary cold boot. + #[serde(default, skip_serializing_if = "Option::is_none")] pub(crate) checkpoint_restore: Option, /// Source name for a one-shot direct local branch, consumed under child reservation. diff --git a/sdk/rust/lib/sandbox/modify.rs b/sdk/rust/lib/sandbox/modify.rs index d0a28cf5b..61850a666 100644 --- a/sdk/rust/lib/sandbox/modify.rs +++ b/sdk/rust/lib/sandbox/modify.rs @@ -274,6 +274,9 @@ impl SandboxModificationBuilder { .await?; let status = handle.status_snapshot(); let mut config = handle.config()?; + // A failed restore can still own staged immutable lower layers. Do not let + // offline disk growth or a restart-backed modification bypass its launch gate. + crate::LocalBackend::validate_completed_restore(&config)?; let mut active = handle.active_config().ok().flatten(); let live = live_control(&self.name, status).await; let mut plan = build_plan( diff --git a/sdk/rust/lib/snapshot/create.rs b/sdk/rust/lib/snapshot/create.rs index 0aaf9fd5f..9506acbbe 100644 --- a/sdk/rust/lib/snapshot/create.rs +++ b/sdk/rust/lib/snapshot/create.rs @@ -133,6 +133,7 @@ pub(super) async fn create_snapshot( } let sandbox_config: SandboxConfig = serde_json::from_str(¤t.config)?; + LocalBackend::validate_completed_restore(&sandbox_config)?; // Only OCI-rooted sandboxes can be snapshotted today; non-OCI // rootfs (passthrough, disk-image-rootfs) are out of scope. @@ -405,6 +406,7 @@ pub(super) async fn create_snapshot_archive( return Err(MicrosandboxError::SnapshotSandboxRunning(source_sandbox)); } let sandbox_config: SandboxConfig = serde_json::from_str(¤t.config)?; + LocalBackend::validate_completed_restore(&sandbox_config)?; let manifest_digest = sandbox_config.manifest_digest.clone().ok_or_else(|| { MicrosandboxError::InvalidConfig( "only OCI-rooted sandboxes with a pinned image can be snapshotted".into(), @@ -503,6 +505,7 @@ async fn capture_full_snapshot( )); } let sandbox_config: SandboxConfig = serde_json::from_str(&model.config)?; + LocalBackend::validate_completed_restore(&sandbox_config)?; let manifest_digest = sandbox_config.manifest_digest.clone().ok_or_else(|| { MicrosandboxError::InvalidConfig(format!( "sandbox '{source_sandbox}' has no OCI image pinned; full snapshots require an OCI root" From 8e68722c47d7f06f76e51b78f44563939d47b4da Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Wed, 9 Sep 2026 03:16:10 +0100 Subject: [PATCH 10/29] feat(snapshot): support live disk-only capture Expose disk-only capture for running and user-paused managed and flat sandboxes through the existing snapshot CLI and shared SDK paths. Serialize root rollover in runtime control without capturing RAM or advancing its baseline, preserve user pauses, and capability-gate older runtimes. Keep stopped lifecycle locking and direct archive publication. Update API documentation and record live flat/layered coverage on macOS ARM64, Linux x86-64, and Windows ARM64, including active writes, source isolation, direct archives, and mixed disk/full capture sequences. --- COMPATIBILITY.md | 2 + crates/cli/lib/commands/snapshot.rs | 4 +- crates/runtime/lib/checkpoint/coordinator.rs | 65 +++++- crates/runtime/lib/control.rs | 41 ++++ crates/runtime/lib/control/executor.rs | 36 +++ docs/sandboxes/snapshots.mdx | 4 +- docs/sdk/rust/snapshots.mdx | 10 +- scripts/smoke/cli/live-disk-snapshot.py | 174 +++++++++++++++ .../reports/live-disk-snapshot-2026-09-09.md | 67 ++++++ sdk/go/sandbox.go | 2 +- sdk/go/snapshot.go | 2 +- sdk/node-ts/native/sandbox_handle.rs | 2 +- sdk/node-ts/src/sandbox-handle.ts | 6 +- sdk/python/src/sandbox_handle.rs | 2 +- sdk/python/src/snapshot.rs | 2 +- sdk/rust/lib/sandbox/handle.rs | 6 +- sdk/rust/lib/sandbox/mod.rs | 1 + sdk/rust/lib/sandbox/modify.rs | 27 ++- sdk/rust/lib/snapshot/create.rs | 209 ++++++++++++++---- sdk/rust/lib/snapshot/mod.rs | 5 +- 20 files changed, 604 insertions(+), 63 deletions(-) create mode 100644 scripts/smoke/cli/live-disk-snapshot.py create mode 100644 scripts/smoke/reports/live-disk-snapshot-2026-09-09.md diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index d3c878cfb..5abc2e70e 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -132,6 +132,8 @@ Sources: [`crates/runtime/lib/control.rs`](crates/runtime/lib/control.rs) and [` Add operations and optional fields rather than redefining existing ones. Capability-gate behavior whose absence cannot be interpreted safely by older clients. +Live disk-only snapshots use the distinct `disk_checkpoint_create` operation and capability. An absent capability is false: callers refuse before capture rather than silently capturing RAM or copying a writable disk. The runtime serializes the disk rollover with other control mutations and preserves a user's pause. This does not change the agent protocol or snapshot format; the result uses the existing file-state layer descriptor. Stopped disk capture retains its lifecycle lock and existing behavior. + ## 5. Launcher-to-Runtime Process Protocol Starting a sandbox crosses a private process boundary. On Unix, launch JSON is passed through inherited descriptor 96, the parent watchdog uses descriptor 97, startup JSON uses descriptor 98, and the lifecycle lock uses descriptor 99. Windows uses a short-lived launch-config file and platform-specific startup plumbing. Detach acknowledgement bytes and graceful-shutdown signals are also part of this contract. diff --git a/crates/cli/lib/commands/snapshot.rs b/crates/cli/lib/commands/snapshot.rs index 689ee06d9..d99592f34 100644 --- a/crates/cli/lib/commands/snapshot.rs +++ b/crates/cli/lib/commands/snapshot.rs @@ -20,7 +20,7 @@ pub struct SnapshotArgs { /// Snapshot subcommands. #[derive(Debug, Subcommand)] pub enum SnapshotCommands { - /// Create a disk snapshot from a stopped sandbox or a full snapshot from a running or paused one. + /// Create a disk snapshot, or include memory and execution state with --full. Create(SnapshotCreateArgs), /// List indexed snapshots. @@ -54,7 +54,7 @@ pub struct SnapshotCreateArgs { /// (or under `--dest-dir` when given). pub name: String, - /// Source sandbox name. Must be stopped (or crashed). + /// Source sandbox name. Disk capture also supports running and user-paused sources. #[arg(long, value_name = "SANDBOX")] pub from: String, diff --git a/crates/runtime/lib/checkpoint/coordinator.rs b/crates/runtime/lib/checkpoint/coordinator.rs index d1823c693..0311f0c64 100644 --- a/crates/runtime/lib/checkpoint/coordinator.rs +++ b/crates/runtime/lib/checkpoint/coordinator.rs @@ -356,7 +356,70 @@ impl CheckpointCoordinator { }) } - /// Capture a same-epoch checkpoint while preserving the caller's prior execution state. + /// Seal the owned disk at a crash-consistent cut without capturing RAM or guest execution. + pub(crate) fn capture_disk( + &mut self, + vm: &msb_krun::VmControl, + checkpoint_id: &str, + user_pause: Option<&UserPause>, + ) -> Result + { + use super::disk::RootDiskRolloverError as Failure; + let started = Instant::now(); + validate_checkpoint_id(checkpoint_id).map_err(Failure::pre_rebind)?; + if let Some(paused) = user_pause { + paused.validate(vm).map_err(Failure::pre_rebind)?; + } + let disk = self.root_disk.as_mut().ok_or_else(|| { + Failure::pre_rebind("disk-only capture requires an owned managed or flat root disk") + })?; + if disk.growth_pending() { + return Err(Failure::pre_rebind( + "complete pending root-disk growth before snapshotting", + )); + } + let path = self.root.join(checkpoint_id); + std::fs::create_dir(&path).map_err(Failure::pre_rebind)?; + let paused_at = Instant::now(); + let pause = match user_pause + .map(|p| Ok(p.generation)) + .unwrap_or_else(|| vm.pause()) + { + Ok(pause) => pause, + Err(error) => { + let _ = std::fs::remove_dir_all(&path); + return Err(Failure::pre_rebind(error)); + } + }; + // Only the root block worker is drained and switched. Rollover inspects its state, + // but no full CPU/device payload, RAM scan, guest handshake, or dirty-baseline update + // is needed. The result is a crash-consistent disk cut, not an execution checkpoint. + let result = disk.rollover(vm, &self.runtime, &path, pause.get()); + if user_pause.is_none() && !result.as_ref().is_err_and(|e| e.keep_paused) { + vm.resume(pause).map_err(Failure::post_journal)?; + } + let pause_us = paused_at.elapsed().as_micros(); + match result { + Ok(captured) => { + tracing::info!(target: "microsandbox_checkpoint_timing", operation = "capture_disk", + checkpoint_id, source_already_paused = user_pause.is_some(), pause_us, + total_us = started.elapsed().as_micros(), "disk-only checkpoint timing"); + Ok(crate::control::DiskCheckpointControlState { + checkpoint_id: checkpoint_id.into(), + path, + disk: captured.manifest, + }) + } + Err(error) => { + // The runtime's forward journal owns any committed new head. Only discard the + // unreturned immutable closure, never source layers or its recovery journal. + let _ = std::fs::remove_dir_all(&path); + Err(error) + } + } + } + + /// Capture a same-epoch full checkpoint while preserving prior execution state. pub(crate) fn capture( &mut self, vm: &msb_krun::VmControl, diff --git a/crates/runtime/lib/control.rs b/crates/runtime/lib/control.rs index 9469a04ca..7f323788d 100644 --- a/crates/runtime/lib/control.rs +++ b/crates/runtime/lib/control.rs @@ -42,6 +42,11 @@ pub const CONTROL_SOCKET_EXTENSION: &str = "control.sock"; #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(tag = "op", rename_all = "snake_case")] pub enum ControlRequest { + /// Seal only the owned root disk; never capture guest RAM or execution state. + DiskCheckpointCreate { + /// Caller-selected safe capture identity. + checkpoint_id: String, + }, /// Capture directly into a reserved child-owned local handoff directory. BranchCreate { /// Unique capture identity matching the child's reservation. @@ -199,6 +204,9 @@ pub struct ControlResponse { /// failure such as an unsuccessful source resume. #[serde(default, skip_serializing_if = "Option::is_none")] pub checkpoint: Option, + /// Sealed disk-only capture, with no RAM or execution-state closure. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub disk_checkpoint: Option, } /// Published checkpoint information returned by the runtime control executor. @@ -218,6 +226,17 @@ pub struct CheckpointControlState { pub memory_emitted_bytes: u64, } +/// Immutable disk closure returned after a live root-head rollover. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct DiskCheckpointControlState { + /// Capture identity echoed from the request. + pub checkpoint_id: String, + /// Runtime-owned closure, independent of the source's new writable head. + pub path: PathBuf, + /// Complete base-to-head disk generation; contains no memory or device payloads. + pub disk: microsandbox_image::checkpoint::DiskGenerationManifest, +} + /// Verified capacity and measured phases of a completed online root growth. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RootDiskGrowthResult { @@ -263,6 +282,9 @@ pub struct ControlCapabilities { /// Same-epoch composite checkpoint capture is available. #[serde(default)] pub checkpoint_create: bool, + /// Disk-only live capture is available without full-state admission. + #[serde(default)] + pub disk_checkpoint_create: bool, } /// Host-confirmed resident suspension state. @@ -615,6 +637,24 @@ mod tests { )); } + #[test] + fn disk_only_capture_has_a_distinct_wire_operation() { + let request = ControlRequest::DiskCheckpointCreate { + checkpoint_id: "disk_test".into(), + }; + let json = serde_json::to_string(&request).unwrap(); + assert_eq!( + serde_json::from_str::(&json).unwrap()["op"], + "disk_checkpoint_create" + ); + assert!( + matches!(serde_json::from_str::(&json).unwrap(), ControlRequest::DiskCheckpointCreate { checkpoint_id } if checkpoint_id == "disk_test") + ); + // An older runtime's capability response cannot accidentally opt into this operation. + let old: ControlCapabilities = serde_json::from_str(r#"{"cpu_resize":false,"memory_resize":false,"secrets_update":false,"checkpoint_create":true}"#).unwrap(); + assert!(!old.disk_checkpoint_create); + } + #[test] fn capabilities_response_serializes_flags() { let response = ControlResponse { @@ -628,6 +668,7 @@ mod tests { memory_resize: false, secrets_update: true, checkpoint_create: true, + disk_checkpoint_create: true, }), ..Default::default() }; diff --git a/crates/runtime/lib/control/executor.rs b/crates/runtime/lib/control/executor.rs index ecd3f5554..ef6525311 100644 --- a/crates/runtime/lib/control/executor.rs +++ b/crates/runtime/lib/control/executor.rs @@ -269,6 +269,7 @@ impl RuntimeControlExecutor { | ControlRequest::CpuTarget { .. } | ControlRequest::SecretsUpdate { .. } | ControlRequest::CheckpointCreate { .. } + | ControlRequest::DiskCheckpointCreate { .. } | ControlRequest::BranchCreate { .. } | ControlRequest::DiskCompact { dry_run: false, .. } | ControlRequest::Pause @@ -280,6 +281,7 @@ impl RuntimeControlExecutor { ControlRequest::Pause | ControlRequest::Resume | ControlRequest::CheckpointCreate { .. } + | ControlRequest::DiskCheckpointCreate { .. } | ControlRequest::BranchCreate { .. } ); if mutation && state.lifecycle != RuntimeLifecycle::Running && !resident_operation { @@ -290,6 +292,38 @@ impl RuntimeControlExecutor { } let response = match request { + ControlRequest::DiskCheckpointCreate { checkpoint_id } => { + state.lifecycle = RuntimeLifecycle::Quiescing; + match state.checkpoint.capture_disk( + &self.vm, + &checkpoint_id, + state.user_pause.as_ref(), + ) { + Ok(result) => { + state.lifecycle = if state.user_pause.is_some() { + RuntimeLifecycle::Quiesced + } else { + RuntimeLifecycle::Running + }; + ControlResponse { + ok: true, + disk_checkpoint: Some(result), + ..Default::default() + } + } + Err(error) => { + if error.keep_paused { + state.user_pause = None; + } + state.lifecycle = if error.keep_paused || state.user_pause.is_some() { + RuntimeLifecycle::Quiesced + } else { + RuntimeLifecycle::Running + }; + control_error("disk_checkpoint_failed", error.to_string()) + } + } + } ControlRequest::Pause => { if state.user_pause.is_none() { self.resident_paused @@ -516,6 +550,7 @@ impl RuntimeControlExecutor { memory_resize: self.vm.memory_resize_supported(), secrets_update: self.secrets_update_supported(), checkpoint_create: true, + disk_checkpoint_create: true, branch_create: cfg!(any(unix, windows)), disk_compact: true, root_disk_grow: true, @@ -539,6 +574,7 @@ impl RuntimeControlExecutor { ControlRequest::CpuState => cpu(self.vm.cpu_state()), ControlRequest::SecretsUpdate { changes } => self.handle_secrets_update(changes), ControlRequest::CheckpointCreate { .. } + | ControlRequest::DiskCheckpointCreate { .. } | ControlRequest::BranchCreate { .. } | ControlRequest::Pause | ControlRequest::Resume diff --git a/docs/sandboxes/snapshots.mdx b/docs/sandboxes/snapshots.mdx index 9bd0ac61e..fbb7526de 100644 --- a/docs/sandboxes/snapshots.mdx +++ b/docs/sandboxes/snapshots.mdx @@ -9,7 +9,7 @@ icon: "code-branch" A snapshot is a portable artifact that can hold either a sandbox's writable disk state or a full checkpoint of a running sandbox. Managed and flat OCI roots are supported; the descriptor preserves the root layout so a flat `rootfs.raw` is never mistaken for a managed OverlayFS upper. Move it with `scp`, archive it as `.msnap`, or create a child sandbox from it. -Disk snapshots capture stopped or crashed sandboxes. Full snapshots use `--full` and capture a running or user-paused sandbox. You do not need to pause it first; if you do, capture leaves it paused. +Disk snapshots work with running, paused, stopped, or crashed sandboxes. A running disk capture briefly pauses the VM, seals its disk, and resumes it without copying RAM. Full snapshots use `--full` to include memory and execution state. A user-paused source stays paused in either mode. ## What gets captured @@ -136,7 +136,7 @@ msb snapshot create after-pip-install --from baseline --dest-dir /mnt/big -The default disk mode requires a stopped or crashed sandbox; running sandboxes are rejected. If the sandbox has previously produced a full snapshot, its writable state may already be a raw/qcow2 chain. Disk snapshotting preserves that complete chain, including writes made after the earlier full snapshot, and every restored child receives a fresh private writable head. +The default disk mode captures only the owned managed or flat root disk. Live captures are crash-consistent: unsaved application buffers and tmpfs contents are not included. Use `--full` when you need memory too. Disk snapshotting preserves the complete raw/qcow2 chain, and every restored child receives a fresh private writable head. Direct `--archive ./saved.msnap` capture works in either mode without installing a snapshot directory. ## Capture a running sandbox diff --git a/docs/sdk/rust/snapshots.mdx b/docs/sdk/rust/snapshots.mdx index f53af1560..ca6f57412 100644 --- a/docs/sdk/rust/snapshots.mdx +++ b/docs/sdk/rust/snapshots.mdx @@ -5,7 +5,7 @@ description: Rust SDK - Snapshot API reference Local-only -Create disk snapshots of stopped sandboxes and full checkpoints of running sandboxes. See [Snapshots](/sandboxes/snapshots) for usage and lifecycle concepts. +Create disk snapshots of running, paused, stopped, or crashed sandboxes, or full checkpoints of running and paused sandboxes. See [Snapshots](/sandboxes/snapshots) for usage and lifecycle concepts. ## Disk maintenance and incremental export @@ -69,7 +69,7 @@ let snap = Snapshot::builder("baseline") async fn create(config: SnapshotConfig) -> MicrosandboxResult ``` -Create an installed snapshot artifact atomically, then best-effort update the rebuildable local index. Disk mode captures a stopped or crashed sandbox; full mode captures a running sandbox's checkpoint closure. Most callers use the [builder](#snapshotbuilder)'s [`create()`](#create) instead of constructing a [`SnapshotConfig`](#snapshotconfig) by hand. +Create an installed snapshot artifact atomically, then best-effort update the rebuildable local index. Disk mode supports running, paused, stopped, and crashed sources; full mode includes memory and execution state from a running or paused source. Most callers use the [builder](#snapshotbuilder)'s [`create()`](#create) instead of constructing a [`SnapshotConfig`](#snapshotconfig) by hand.

Parameters

@@ -736,7 +736,7 @@ Cold-boot only the disk state carried by a full snapshot. Chain after [`from_sna async fn snapshot(&self, name: &str) -> MicrosandboxResult ``` -`SandboxHandle` method. Snapshot this sandbox under a bare name in the default snapshots directory (`~/.microsandbox/snapshots//`). The sandbox must be stopped or crashed; running sandboxes are rejected with `SnapshotSandboxRunning`. Local handles only. To place the artifact elsewhere, use [`Snapshot::save()`](#snapshotsave) / [`Snapshot::load()`](#snapshotload) or move the self-contained artifact directory. +`SandboxHandle` method. Snapshot this sandbox's disk under a bare name in the default snapshots directory (`~/.microsandbox/snapshots//`). Live captures are crash-consistent and preserve the source's running/paused state. Local handles only. To place the artifact elsewhere, use [`Snapshot::save()`](#snapshotsave) / [`Snapshot::load()`](#snapshotload) or move the self-contained artifact directory.

Parameters

@@ -794,7 +794,7 @@ Set the sandbox to capture. Required; [`build()`](#build) and [`create()`](#crea
source_sandboximpl Into<String>
-
Name of the source sandbox. Must be stopped or crashed, and rooted on an OCI image.
+
Name of the OCI-rooted source sandbox. Disk capture also supports running and paused sources.
@@ -912,7 +912,7 @@ Inputs to create a snapshot. A type alias for `SnapshotSpec`. Usually built via |-------|------|-------------| | name | `String` | Bare snapshot name; always the artifact directory's basename | | dest_dir | `Option` | Parent directory for the artifact; `None` = the default snapshots directory | -| source_sandbox | `String` | Name of the source sandbox; must be stopped | +| source_sandbox | `String` | Name of the source sandbox; disk capture preserves running/paused state | | labels | `Vec<(String, String)>` | User-supplied labels | | force | `bool` | Overwrite an existing artifact with the same name | | record_integrity | `bool` | Compute and record upper-layer integrity at creation | diff --git a/scripts/smoke/cli/live-disk-snapshot.py b/scripts/smoke/cli/live-disk-snapshot.py new file mode 100644 index 000000000..ea886ed95 --- /dev/null +++ b/scripts/smoke/cli/live-disk-snapshot.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Live disk-only capture on an isolated MSB_HOME with a matching runtime/firmware. + +MSB_PATH, MSB_HOME, STACK8_PREFIX and STACK8_OUT are required. The fixture stops +its own VMs and records whole CLI times, not just the VM pause interval. +""" +import hashlib +import json +import os +from pathlib import Path +import subprocess +import time + +binary = os.environ["MSB_PATH"] +home = Path(os.environ["MSB_HOME"]) +out = Path(os.environ["STACK8_OUT"]) +out.mkdir(parents=True, exist_ok=True) +prefix = os.environ["STACK8_PREFIX"] +rows = [] +names = [] + + +def run(label, *args, ok=True): + start = time.monotonic() + result = subprocess.run([binary, *args], capture_output=True, text=True, timeout=180) + row = dict(case=label, ms=round((time.monotonic() - start) * 1000, 2), + exit=result.returncode, stdout=result.stdout, stderr=result.stderr) + rows.append(row) + (out / "results.json").write_text(json.dumps(rows, indent=2)) + if ok: + assert result.returncode == 0, row + return result + + +def files(root): + return {str(p.relative_to(root)): (p.stat().st_size, p.stat().st_mtime_ns) + for p in root.rglob("*") if p.is_file()} if root.exists() else {} + + +def sealed_hashes(root): + result = {} + for path in root.rglob("*"): + if path.is_file() and path.suffix in (".raw", ".qcow2", ".ext4"): + digest = hashlib.sha256() + with path.open("rb") as file: + for chunk in iter(lambda: file.read(1024 * 1024), b""): + digest.update(chunk) + result[str(path)] = digest.hexdigest() + assert result + return result + + +try: + for layout in ("flat", "managed"): + source = prefix + "-" + layout + names.append(source) + run("create-" + layout, "create", "alpine", "--name", source, + "--root-disk", "flat:512M" if layout == "flat" else "512M", "--memory", "256M") + run("prepare-" + layout, "exec", source, "--", "sh", "-c", + "echo before > /disk-marker; echo ram-only > /dev/shm/ram-marker; sync") + boot = run("boot-" + layout, "exec", source, "--", "cat", "/proc/sys/kernel/random/boot_id").stdout + runtime = home / "sandboxes" / source / "runtime" + memory_before = files(runtime / "checkpoint-store") + cache_before = files(home / "cache" / "memory") + for mode in ("installed", "integrity", "archive", "plain", "paused"): + snap = source + "-" + mode + child = snap + "-child" + names.append(child) + run("reset-" + snap, "exec", source, "--", "sh", "-c", "echo before > /disk-marker; sync") + if mode == "paused": + run("pause-" + layout, "pause", source) + args = ["snapshot", "create", snap, "--from", source] + archive = out / (snap + (".tar" if mode == "plain" else ".msnap")) + installed_before = set((home / "snapshots").glob("*")) + if mode in ("archive", "plain"): + args += ["--archive", str(archive)] + if mode == "plain": + args += ["--plain-tar"] + if mode == "integrity": + args += ["--integrity"] + run("capture-" + snap, *args) + assert files(runtime / "checkpoint-store") == memory_before, "disk capture touched the RAM/object store" + assert files(home / "cache" / "memory") == cache_before, "disk capture touched the RAM cache" + assert not list((runtime / "checkpoints").iterdir()), "consumed disk staging leaked" + if mode == "paused": + # Public repeated pause must be idempotent, and an exec must not release it. + paused_exec = run("paused-exec-" + layout, "exec", source, "--", "true", ok=False) + assert paused_exec.returncode != 0 + run("resume-" + layout, "resume", source) + assert run("source-boot-" + snap, "exec", source, "--", "cat", "/proc/sys/kernel/random/boot_id").stdout == boot + assert run("source-ram-" + snap, "exec", source, "--", "cat", "/dev/shm/ram-marker").stdout.strip() == "ram-only" + run("diverge-" + snap, "exec", source, "--", "sh", "-c", "echo after > /disk-marker; sync") + if mode in ("archive", "plain"): + assert set((home / "snapshots").glob("*")) == installed_before, "direct archive installed a snapshot" + from_snapshot = str(archive) + else: + artifact = home / "snapshots" / snap + manifest = json.loads((artifact / "snapshot.json").read_text()) + assert manifest["scope"] == "file" and manifest["state"]["kind"] == "file", manifest + assert not (artifact / "checkpoint").exists() + before = sealed_hashes(artifact) + from_snapshot = snap + run("restore-" + snap, "create", "--name", child, "--from-snapshot", from_snapshot) + assert run("child-disk-" + snap, "exec", child, "--", "cat", "/disk-marker").stdout.strip() == "before" + run("child-no-ram-" + snap, "exec", child, "--", "sh", "-c", "test ! -e /dev/shm/ram-marker") + assert run("child-boot-" + snap, "exec", child, "--", "cat", "/proc/sys/kernel/random/boot_id").stdout != boot + run("child-write-" + snap, "exec", child, "--", "sh", "-c", "echo child > /disk-marker; sync") + assert run("source-disk-" + snap, "exec", source, "--", "cat", "/disk-marker").stdout.strip() == "after" + if mode in ("archive", "plain"): + archive.unlink() + else: + assert sealed_hashes(artifact) == before, "source or child mutated sealed layers" + run("remove-" + snap, "snapshot", "remove", snap) + assert run("after-delete-" + snap, "exec", child, "--", "cat", "/disk-marker").stdout.strip() == "child" + run("stop-child-" + snap, "stop", child) + # Exercise the cut while the guest actually submits writes, not only after sync. + # Keep ownership in the guest, as in the branch timer fixture. This does not depend + # on an additional host client's stdin/console lifetime while capture runs. + run("start-writer-" + layout, "exec", source, "--", "sh", "-c", + "sh -c 'i=1; while [ ! -e /stop-counter ]; do echo $i > /counter.tmp; " + "mv /counter.tmp /counter; sync; i=$((i+1)); sleep 0.02; done; " + "touch /counter-done' >/tmp/counter.log 2>&1 = start_counter + run("stop-busy-child-" + layout, "stop", busy_child) + # A later full checkpoint must still work after disk-only generations. + full = source + "-full" + run("full-after-disk-" + layout, "snapshot", "create", full, "--from", source, "--full") + full_child = source + "-full-child" + names.append(full_child) + run("full-restore-" + layout, "create", "--name", full_child, "--from-snapshot", full) + assert run("full-ram-" + layout, "exec", full_child, "--", "cat", "/dev/shm/ram-marker").stdout.strip() == "ram-only" + # A disk-only cut between full captures must not consume/advance the RAM baseline. + memory_before = files(runtime / "checkpoint-store") + run("disk-between-full-" + layout, "snapshot", "create", source + "-between", "--from", source) + assert files(runtime / "checkpoint-store") == memory_before + run("change-ram-" + layout, "exec", source, "--", "sh", "-c", "echo updated > /dev/shm/ram-marker") + run("second-full-" + layout, "snapshot", "create", full + "-next", "--from", source, "--full") + next_child = source + "-next-child" + names.append(next_child) + run("second-full-restore-" + layout, "create", "--name", next_child, "--from-snapshot", full + "-next") + assert run("second-full-ram-" + layout, "exec", next_child, "--", "cat", "/dev/shm/ram-marker").stdout.strip() == "updated" + # A name collision must fail before publication and leave both the source and snapshot usable. + refused = run("duplicate-refused-" + layout, "snapshot", "create", source + "-between", "--from", source, ok=False) + assert refused.returncode != 0 + run("source-after-refusal-" + layout, "exec", source, "--", "true") + run("stop-source-" + layout, "stop", source) + run("stopped-after-live-" + layout, "snapshot", "create", source + "-stopped", "--from", source) + tmpfs = prefix + "-tmpfs" + names.append(tmpfs) + run("tmpfs-create", "create", "alpine", "--name", tmpfs, "--root-disk", "tmpfs:128M", "--memory", "256M") + refused = run("tmpfs-refused", "snapshot", "create", tmpfs + "-bad", "--from", tmpfs, ok=False) + assert refused.returncode != 0 and "tmpfs" in refused.stderr + run("tmpfs-still-running", "exec", tmpfs, "--", "true") + print(json.dumps({"result": "pass", "layouts": ["flat", "managed"], "modes": ["installed", "integrity", "archive", "plain", "paused"]})) +finally: + for name in reversed(names): + try: + run("cleanup-" + name, "stop", name, ok=False) + except Exception as error: + rows.append({"case": "cleanup-" + name, "error": str(error)}) + (out / "results.json").write_text(json.dumps(rows, indent=2)) diff --git a/scripts/smoke/reports/live-disk-snapshot-2026-09-09.md b/scripts/smoke/reports/live-disk-snapshot-2026-09-09.md new file mode 100644 index 000000000..1108d4e1d --- /dev/null +++ b/scripts/smoke/reports/live-disk-snapshot-2026-09-09.md @@ -0,0 +1,67 @@ +# Live disk-only snapshots — 2026-09-09 + +Implemented on the #8 branch above Microsandbox `f68c1329`, using existing pinned libkrun `862d6842`, rust-vmm `f798d4f2`, and matching firmware. No companion changes, dependency overrides, schema change, or new public flags were required. + +## Behavior + +```sh +# Source stays running. No RAM checkpoint or --full workaround. +msb snapshot create saved --from source +msb create --name child --from-snapshot saved + +# Direct archive: no installed snapshot directory or index row. +msb snapshot create exported --from source --archive ./exported.msnap +msb create --name archive-child --from-snapshot ./exported.msnap +``` + +Running and user-paused managed/flat OCI roots use the serialized runtime control executor and a distinct capability-gated `disk_checkpoint_create` operation. Rollover seals the disk and selects a private successor. Running sources resume before SDK packaging; user-paused sources stay paused. Stopped/crashed copies retain their lifecycle lock. The SDK packages only the immutable disk closure, rechecks source identity, and removes consumed staging. + +Live disk cuts are **crash-consistent**, not application/filesystem-quiesced. Unsaved application buffers, guest page-cache writes not submitted to disk, and tmpfs are not promised. Children cold-boot with private writable heads. The block worker is inspected by existing rollover machinery, but no RAM, full CPU/device checkpoint, freezer handshake, or RAM-baseline update is required. Full capture retains its existing behavior. Cloud, tmpfs disk-only, and user-owned disk-image restrictions remain unchanged. Unsupported old runtimes are refused without a full-capture or writable-file-copy fallback. + +Existing Rust, Python, TypeScript and Go snapshot APIs already route through the shared Rust implementation. Documentation was corrected; no duplicate API was added. Native language packages were not rebuilt/live-tested in this pass. + +## Live coverage + +The committed `scripts/smoke/cli/live-disk-snapshot.py` passed on macOS ARM64/HVF, OVH Linux x86-64/KVM, and Surface Windows ARM64/WHP. Each tested flat and layered roots with 512 MiB disk capacity, 256 MiB RAM, and a cached Alpine image. + +| Checks, on both root layouts | Mac | Linux | Windows | +| --- | --- | --- | --- | +| Running installed capture and cold restore; optional integrity | Pass | Pass | Pass | +| Direct compressed `.msnap` and plain `.tar`; no installed intermediate | Pass | Pass | Pass | +| User-paused capture remains paused; exec refuses until explicit resume | Pass | Pass | Pass | +| Source RAM/boot ID retained; disk child has a new boot ID and no tmpfs marker | Pass | Pass | Pass | +| Source/child writes isolated; sealed payload hashes unchanged | Pass | Pass | Pass | +| Child usable after snapshot/archive deletion | Pass | Pass | Pass | +| Disk capture leaves RAM store/cache untouched; transient staging released | Pass | Pass | Pass | +| Full after disk; disk between full captures; updated RAM restores | Pass | Pass | Pass | +| Stopped capture after live rollover; duplicate-name and tmpfs refusal | Pass | Pass | Pass | +| Capture during active writes; writer progresses; captured counter cold-boots | Pass | Pass | Pass | + +An initial live test caught incorrect acquisition of the runtime-owned lifecycle lock; live SDK capture now uses runtime serialization, while stopped capture retains the lock. The first Windows active-write fixture failed to start its separate long-running host client before capture. The final portable fixture uses a guest-owned background worker, as existing branch tests do, and verifies startup, progress, and termination. + +## Timings + +Debug-build whole CLI wall times in milliseconds, one observation per case. These are correctness-run observations, not release benchmarks, percentiles, or controlled host-to-host comparisons. Cache state, hardware, sequential chain depth, and workload matter. Guest contents and command success were checked separately after create returned. + +| Host / root | Installed capture | Installed cold restore | Compressed archive capture | Archive cold restore | +| --- | ---: | ---: | ---: | ---: | +| Mac / flat | 203.20 | 415.35 | 366.84 | 629.40 | +| Mac / layered | 94.74 | 311.51 | 139.37 | 402.38 | +| Linux / flat | 24.57 | 319.45 | 144.48 | 346.42 | +| Linux / layered | 14.18 | 320.52 | 22.19 | 335.16 | +| Windows / flat | 234.00 | 2359.00 | 906.00 | 1328.00 | +| Windows / layered | 157.00 | 2016.00 | 250.00 | 1141.00 | + +These samples come from Mac `live-disk-mac-h`, Linux `live-disk-linux-d`, and Windows `live-disk-win-c`. Runtime `capture_disk.pause_us` separately measures pause request through resume acknowledgement for running sources. First installed captures measured 174.32/70.56 ms for flat/layered on Mac and 145.21/49.04 ms on Windows. Linux's preceding `live-disk-linux-c` run measured 5.91/3.32 ms. User-paused measurements describe rollover work, not the user's entire suspension interval. + +Disk-only skips RAM capture but is not constant-time or zero-I/O. Existing rollover still performs layer integrity work and prepares its immutable closure while paused. Allocated bytes, chain depth, copying fallback, outstanding I/O and durability latency can increase pause time; this change does not optimize that existing hot path away. SDK publication and archive compression follow source resume. + +## Reproduction and limits + +Set `MSB_PATH`, isolated `MSB_HOME`, matching `MSB_LIBKRUNFW_PATH`, unique `STACK8_PREFIX`, and `STACK8_OUT`, then run `python3 scripts/smoke/cli/live-disk-snapshot.py`. The fixture stops only its own VMs in `finally`. + +Final portable-fixture evidence: `/private/tmp/msb-live-disk-mac-i/results.json`; `/home/ubuntu/msb-live-disk-linux-e/results.json` on OVH; `C:\Users\Stephen\AppData\Local\Temp\msb-live-disk-win-e\results.json` on Surface. Earlier timing runs retain the corresponding result directories. Guest disks/RAM are not committed. + +Runtime control tests passed 6/6; checkpoint-filtered runtime tests 32/32; SDK snapshot-filtered tests 31/31. Formatting and diff whitespace checks passed. CLI builds passed on all three hosts with pinned Git sources; the Mac runnable binary was codesigned. + +Linux ARM64/nested KVM and Windows x86-64 were not rerun for this addition. Native SDK packages, exhaustive crash/publication fault injection, disk-full, near-limit chains, concurrent maintenance, large cold-cache workloads and release p50/p95 performance are not qualified by this pass. diff --git a/sdk/go/sandbox.go b/sdk/go/sandbox.go index abb92cd29..8ea2ae6e0 100644 --- a/sdk/go/sandbox.go +++ b/sdk/go/sandbox.go @@ -817,7 +817,7 @@ func (h *SandboxHandle) Remove(ctx context.Context) error { return RemoveSandbox(ctx, h.name) } -// Snapshot captures this stopped sandbox under a bare name in the default +// Snapshot captures this sandbox's disk under a bare name in the default // snapshots directory. func (h *SandboxHandle) Snapshot(ctx context.Context, name string) (*SnapshotArtifact, error) { info, err := ffi.SandboxHandleSnapshot(ctx, h.name, name) diff --git a/sdk/go/snapshot.go b/sdk/go/snapshot.go index 89255dfe4..0ce93f52f 100644 --- a/sdk/go/snapshot.go +++ b/sdk/go/snapshot.go @@ -17,7 +17,7 @@ type SnapshotCreateOptions struct { // Snapshot name, resolved under the default snapshots directory // (or under DestDir when set). Name string - // Source sandbox to snapshot. Must be stopped. Required. + // Source sandbox to snapshot. Disk capture preserves running/paused state. Required. FromSandbox string // Parent directory to create the artifact in; empty = the default // snapshots directory. The artifact lands at DestDir/. diff --git a/sdk/node-ts/native/sandbox_handle.rs b/sdk/node-ts/native/sandbox_handle.rs index ad723ba03..507ad290e 100644 --- a/sdk/node-ts/native/sandbox_handle.rs +++ b/sdk/node-ts/native/sandbox_handle.rs @@ -278,7 +278,7 @@ impl JsSandboxHandle { crate::sandbox::spawn_log_stream_from_stream(stream).await } - /// Snapshot this (stopped) sandbox under a bare name. + /// Snapshot this sandbox's disk under a bare name, preserving its running/paused state. /// /// Resolves under `~/.microsandbox/snapshots//`. Move /// artifacts with `Snapshot.save`/`Snapshot.load`. diff --git a/sdk/node-ts/src/sandbox-handle.ts b/sdk/node-ts/src/sandbox-handle.ts index 3e6fd193e..ac431b4aa 100644 --- a/sdk/node-ts/src/sandbox-handle.ts +++ b/sdk/node-ts/src/sandbox-handle.ts @@ -244,12 +244,12 @@ export class SandboxHandle { } /** - * Snapshot this (stopped) sandbox under a bare name. Resolves under + * Snapshot this sandbox's disk under a bare name. Resolves under * `~/.microsandbox/snapshots//`. For an explicit filesystem * destination, move the artifact with `Snapshot.save`/`Snapshot.load`. * - * The sandbox must be stopped (or crashed); running sandboxes are - * rejected with a `SnapshotSandboxRunning` error. + * Running and paused sources are supported. A live cut is crash-consistent + * and preserves the source's running/paused state. */ async snapshot(name: string): Promise { const raw = await withMappedErrors(() => this.inner.snapshot(name)); diff --git a/sdk/python/src/sandbox_handle.rs b/sdk/python/src/sandbox_handle.rs index 4714e79d9..1b766bf12 100644 --- a/sdk/python/src/sandbox_handle.rs +++ b/sdk/python/src/sandbox_handle.rs @@ -456,7 +456,7 @@ impl PySandboxHandle { }) } - /// Snapshot this (stopped) sandbox under a bare name. Resolves + /// Snapshot this sandbox's disk under a bare name, preserving its running/paused state. Resolves /// under `~/.microsandbox/snapshots//`. Move artifacts with /// `Snapshot.save`/`Snapshot.load`. fn snapshot<'py>(&self, py: Python<'py>, name: String) -> PyResult> { diff --git a/sdk/python/src/snapshot.rs b/sdk/python/src/snapshot.rs index 9d57e99a1..9775d754b 100644 --- a/sdk/python/src/snapshot.rs +++ b/sdk/python/src/snapshot.rs @@ -42,7 +42,7 @@ pub struct PySnapshotHandle { #[pymethods] impl PySnapshot { - /// Create a disk snapshot from a stopped sandbox or a full snapshot from a running one. + /// Create a disk snapshot, or include memory and execution state with full=True. /// /// The artifact is created under `~/.microsandbox/snapshots//`, /// or under `dest_dir=` when given; move artifacts with `save`/`load`. diff --git a/sdk/rust/lib/sandbox/handle.rs b/sdk/rust/lib/sandbox/handle.rs index c130f292e..03d2ca89a 100644 --- a/sdk/rust/lib/sandbox/handle.rs +++ b/sdk/rust/lib/sandbox/handle.rs @@ -469,9 +469,9 @@ impl SandboxHandle { /// Snapshot this sandbox to a bare name under the default snapshots /// directory (`~/.microsandbox/snapshots//`). /// - /// The sandbox must be stopped (or crashed); running sandboxes are - /// rejected with `MicrosandboxError::SnapshotSandboxRunning`. **Local - /// handles only** — cloud snapshot semantics are deferred. + /// Captures disk only, including running and paused sources. A live cut is + /// crash-consistent and preserves the source's running/paused state. + /// **Local handles only** — cloud snapshot semantics are deferred. pub async fn snapshot( &self, name: &str, diff --git a/sdk/rust/lib/sandbox/mod.rs b/sdk/rust/lib/sandbox/mod.rs index 204835fb8..d01499c13 100644 --- a/sdk/rust/lib/sandbox/mod.rs +++ b/sdk/rust/lib/sandbox/mod.rs @@ -92,6 +92,7 @@ pub(crate) fn reserved_label_prefix(key: &str) -> Option<&'static str> { // local backend's lifecycle and create methods under `backend/local/` call. pub(crate) use builder::{apply_checkpoint_restore_constraints, apply_snapshot_root_layout}; pub(crate) use modify::control_checkpoint_create; +pub(crate) use modify::control_disk_checkpoint_create; pub(crate) use patch::{apply_patches, build_flat_tree, build_upper_tree}; #[cfg(windows)] pub(crate) use reap::reap_leaked_runtime_process; diff --git a/sdk/rust/lib/sandbox/modify.rs b/sdk/rust/lib/sandbox/modify.rs index 61850a666..95c002d51 100644 --- a/sdk/rust/lib/sandbox/modify.rs +++ b/sdk/rust/lib/sandbox/modify.rs @@ -5,11 +5,11 @@ use std::sync::Arc; use microsandbox_types::{EnvVar, RootDisk, RootfsSource}; use sea_orm::{ActiveModelTrait, ColumnTrait, EntityTrait, QueryFilter, Set}; -use crate::MicrosandboxResult; use crate::backend::Backend; use crate::db::entity::{sandbox as sandbox_entity, sandbox_label as sandbox_label_entity}; use crate::error::{Operation, UnsupportedReason}; use crate::size::Mebibytes; +use crate::{MicrosandboxError, MicrosandboxResult}; use super::{SandboxConfig, SandboxStatus}; @@ -904,6 +904,31 @@ pub(crate) async fn control_checkpoint_create( ))) } +/// Request disk-only capture without falling back to full-state capture or a stopped copy. +pub(crate) async fn control_disk_checkpoint_create( + local: &crate::backend::LocalBackend, + name: &str, + checkpoint_id: String, +) -> MicrosandboxResult { + let capabilities = + control_request_for(local, name, "{\"op\":\"capabilities\"}\n".into()).await?; + if !capabilities + .capabilities + .is_some_and(|c| c.disk_checkpoint_create) + { + return Err(MicrosandboxError::unsupported(Operation::SnapshotOps, + UnsupportedReason::NotAvailable("this runtime does not support live disk-only snapshots; recreate the sandbox with the updated runtime".into()))); + } + let request = + microsandbox_runtime::control::ControlRequest::DiskCheckpointCreate { checkpoint_id }; + let mut line = serde_json::to_string(&request)?; + line.push('\n'); + let response = control_request_for(local, name, line).await?; + response.disk_checkpoint.ok_or_else(|| { + MicrosandboxError::Runtime("runtime omitted the disk-only capture result".into()) + }) +} + /// Send the value-bearing live secret batch to the sandbox process. The /// request travels only over the private per-sandbox control endpoint and is /// never logged; failures surface the runtime's error, which carries secret diff --git a/sdk/rust/lib/snapshot/create.rs b/sdk/rust/lib/snapshot/create.rs index 9506acbbe..e865f71dd 100644 --- a/sdk/rust/lib/snapshot/create.rs +++ b/sdk/rust/lib/snapshot/create.rs @@ -1,4 +1,4 @@ -//! Snapshot creation from a stopped sandbox. +//! Disk-only and full snapshot creation with source lifecycle preservation. use std::collections::BTreeMap; use std::path::{Path, PathBuf}; @@ -47,6 +47,22 @@ struct SnapshotDiskSource { struct SnapshotDiskClosure { sources: Vec, virtual_size: u64, + /// A live capture owns immutable runtime staging until artifact publication completes. + capture_root: Option, +} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +impl Drop for SnapshotDiskClosure { + fn drop(&mut self) { + if let Some(path) = &self.capture_root + && let Err(error) = std::fs::remove_dir_all(path) + { + tracing::warn!(%error, "failed to remove consumed disk-only capture staging"); + } + } } //-------------------------------------------------------------------------------------------------- @@ -99,33 +115,39 @@ pub(super) async fn create_snapshot( .await; } - if matches!( - model.status, - SandboxStatus::Running | SandboxStatus::Draining | SandboxStatus::Paused - ) { + if model.status == SandboxStatus::Draining { return Err(MicrosandboxError::SnapshotSandboxRunning( source_sandbox.clone(), )); } - // Reuse the runtime's existing lifecycle ownership lock so start, - // replacement, and removal cannot race the upper copy. - let _lifecycle_guard = crate::runtime::acquire_sandbox_lifecycle_guard( - &local.config().run_dir(), - &source_sandbox, - std::time::Duration::from_secs(5), - ) - .await?; + // Resident runtimes own the lifecycle lock and serialize the disk cut through control. + // Stopped copies acquire it here; the SDK never reads a live writable head. + let live = matches!(model.status, SandboxStatus::Running | SandboxStatus::Paused); + let _lifecycle_guard = if live { + None + } else { + Some( + crate::runtime::acquire_sandbox_lifecycle_guard( + &local.config().run_dir(), + &source_sandbox, + std::time::Duration::from_secs(5), + ) + .await?, + ) + }; let current = sandbox_entity::Entity::find() .filter(sandbox_entity::Column::Name.eq(&source_sandbox)) .one(local.db().await?.read()) .await? .ok_or_else(|| MicrosandboxError::SandboxNotFound(source_sandbox.clone()))?; if current.id != model.id - || matches!( - current.status, - SandboxStatus::Running | SandboxStatus::Draining | SandboxStatus::Paused - ) + || current.status == SandboxStatus::Draining + || live + != matches!( + current.status, + SandboxStatus::Running | SandboxStatus::Paused + ) { return Err(MicrosandboxError::SnapshotSandboxRunning( source_sandbox.clone(), @@ -152,7 +174,15 @@ pub(super) async fn create_snapshot( } let sandbox_dir = local.sandboxes_dir().join(&source_sandbox); - let disk = snapshot_disk_closure(&sandbox_dir, &root_disk)?; + let disk = capture_disk_source( + local, + &sandbox_dir, + &source_sandbox, + current.id, + current.status, + &root_disk, + ) + .await?; // Stage the artifact in a sibling directory, so a failed create never // leaves a partial artifact at the destination (which would poison @@ -206,13 +236,13 @@ pub(super) async fn create_snapshot( let index_us = index_started.elapsed().as_micros(); tracing::info!( target: "microsandbox_checkpoint_timing", - operation = "snapshot_create_installed_stopped", + operation = "snapshot_create_installed_disk", source_sandbox, total_us = total_started.elapsed().as_micros(), artifact_build_us, promote_us, index_us, - "stopped snapshot creation timing" + "disk snapshot creation timing" ); Ok(Snapshot::from_parts(dest_dir, digest, manifest, labels)) @@ -313,7 +343,7 @@ async fn create_full_snapshot( )) } -/// Capture directly from a stopped sandbox into an archive without creating +/// Capture a disk or full snapshot directly into an archive without creating /// an installed artifact directory or index row. pub(super) async fn create_snapshot_archive( local: &LocalBackend, @@ -380,28 +410,34 @@ pub(super) async fn create_snapshot_archive( captured.labels, )); } - if matches!( - model.status, - SandboxStatus::Running | SandboxStatus::Draining | SandboxStatus::Paused - ) { + if model.status == SandboxStatus::Draining { return Err(MicrosandboxError::SnapshotSandboxRunning(source_sandbox)); } - let _lifecycle_guard = crate::runtime::acquire_sandbox_lifecycle_guard( - &local.config().run_dir(), - &source_sandbox, - std::time::Duration::from_secs(5), - ) - .await?; + let live = matches!(model.status, SandboxStatus::Running | SandboxStatus::Paused); + let _lifecycle_guard = if live { + None + } else { + Some( + crate::runtime::acquire_sandbox_lifecycle_guard( + &local.config().run_dir(), + &source_sandbox, + std::time::Duration::from_secs(5), + ) + .await?, + ) + }; let current = sandbox_entity::Entity::find() .filter(sandbox_entity::Column::Name.eq(&source_sandbox)) .one(local.db().await?.read()) .await? .ok_or_else(|| MicrosandboxError::SandboxNotFound(source_sandbox.clone()))?; if current.id != model.id - || matches!( - current.status, - SandboxStatus::Running | SandboxStatus::Draining | SandboxStatus::Paused - ) + || current.status == SandboxStatus::Draining + || live + != matches!( + current.status, + SandboxStatus::Running | SandboxStatus::Paused + ) { return Err(MicrosandboxError::SnapshotSandboxRunning(source_sandbox)); } @@ -420,7 +456,15 @@ pub(super) async fn create_snapshot_archive( ))); } let sandbox_dir = local.sandboxes_dir().join(&source_sandbox); - let disk = snapshot_disk_closure(&sandbox_dir, &root_disk)?; + let disk = capture_disk_source( + local, + &sandbox_dir, + &source_sandbox, + current.id, + current.status, + &root_disk, + ) + .await?; let integrity_started = Instant::now(); let integrities = vec![None; disk.sources.len()]; let labels: BTreeMap<_, _> = labels.into_iter().collect(); @@ -471,7 +515,7 @@ pub(super) async fn create_snapshot_archive( let archive_us = archive_started.elapsed().as_micros(); tracing::info!( target: "microsandbox_checkpoint_timing", - operation = "snapshot_create_archive_stopped", + operation = "snapshot_create_archive_disk", source_sandbox, plain_tar, record_integrity, @@ -479,7 +523,7 @@ pub(super) async fn create_snapshot_archive( total_us = total_started.elapsed().as_micros(), integrity_us, archive_us, - "direct stopped snapshot archive timing" + "direct disk snapshot archive timing" ); Ok(SnapshotArchive::from_parts( out.to_path_buf(), @@ -718,7 +762,7 @@ async fn build_artifact( payload_sync_us, integrity_us, descriptor_us, - "stopped snapshot artifact build timing" + "disk snapshot artifact build timing" ); Ok((digest, manifest)) @@ -863,6 +907,90 @@ fn snapshot_root_disk( } } +/// Resident runtimes return a sealed closure while retaining their lifecycle lock. +/// Stopped callers own that lock themselves. Packaging never reads a live writable head. +async fn capture_disk_source( + local: &LocalBackend, + sandbox_dir: &Path, + source: &str, + source_id: i32, + status: SandboxStatus, + root_disk: &SnapshotRootDisk, +) -> MicrosandboxResult { + if !matches!(status, SandboxStatus::Running | SandboxStatus::Paused) { + return snapshot_disk_closure(sandbox_dir, root_disk); + } + let id = format!("disk_{:032x}", rand::random::()); + let captured = + crate::sandbox::control_disk_checkpoint_create(local, source, id.clone()).await?; + let expected_path = sandbox_dir.join("runtime").join("checkpoints").join(&id); + let expected_device = match root_disk { + SnapshotRootDisk::Flat => "vda", + SnapshotRootDisk::Managed => "vdb", + SnapshotRootDisk::Tmpfs { .. } => { + return Err(MicrosandboxError::InvalidConfig( + "tmpfs requires a full snapshot".into(), + )); + } + }; + if captured.checkpoint_id != id + || captured.path != expected_path + || captured.disk.device_id != expected_device + { + return Err(MicrosandboxError::SnapshotIntegrity( + "disk capture identity, path, or root device mismatch".into(), + )); + } + captured + .disk + .validate() + .map_err(|e| MicrosandboxError::SnapshotIntegrity(e.to_string()))?; + let sources = captured + .disk + .layers + .iter() + .map(|layer| { + let format = match layer.format.as_str() { + "raw" => SnapshotFormat::Raw, + "qcow2" => SnapshotFormat::Qcow2, + other => { + return Err(MicrosandboxError::SnapshotIntegrity(format!( + "unsupported live disk format {other}" + ))); + } + }; + Ok(SnapshotDiskSource { + path: captured + .path + .join("layers") + .join(format!("{}.{}", layer.layer_id, layer.format)), + format, + }) + }) + .collect::>>()?; + let size = captured + .disk + .layers + .last() + .ok_or_else(|| MicrosandboxError::SnapshotIntegrity("empty disk capture".into()))? + .virtual_size; + let mut disk = validate_snapshot_disk_sources(sources, size)?; + disk.capture_root = Some(expected_path); + // A live runtime owns the lifecycle lock, not this SDK call. If the source was replaced + // between lookup and capture, never publish its disk under the original image/config. + let current = sandbox_entity::Entity::find() + .filter(sandbox_entity::Column::Name.eq(source)) + .one(local.db().await?.read()) + .await?; + if !current.is_some_and(|model| model.id == source_id) { + return Err(MicrosandboxError::Runtime( + "snapshot source was replaced during disk capture; retry with the current sandbox" + .into(), + )); + } + Ok(disk) +} + fn snapshot_disk_closure( sandbox_dir: &Path, root_disk: &SnapshotRootDisk, @@ -957,6 +1085,7 @@ fn validate_snapshot_disk_sources( Ok(SnapshotDiskClosure { sources, virtual_size, + capture_root: None, }) } @@ -1313,6 +1442,7 @@ mod tests { let source = temp.path().join("source.ext4"); std::fs::write(&source, b"snapshot payload").unwrap(); let disk = SnapshotDiskClosure { + capture_root: None, sources: vec![SnapshotDiskSource { path: source, format: SnapshotFormat::Raw, @@ -1370,6 +1500,7 @@ mod tests { .await .unwrap(); let disk = SnapshotDiskClosure { + capture_root: None, sources: vec![ SnapshotDiskSource { path: raw, diff --git a/sdk/rust/lib/snapshot/mod.rs b/sdk/rust/lib/snapshot/mod.rs index d6e60e0a2..6f5757424 100644 --- a/sdk/rust/lib/snapshot/mod.rs +++ b/sdk/rust/lib/snapshot/mod.rs @@ -95,8 +95,9 @@ impl Snapshot { /// Create an installed disk or full snapshot artifact. /// - /// Disk capture requires a stopped or crashed sandbox. A builder configured with - /// [`full`](SnapshotBuilder::full) captures a running sandbox's checkpoint closure. + /// Disk capture supports resident and stopped sources, briefly quiescing a running root + /// without capturing RAM. A user-paused source remains paused. A builder configured with + /// [`full`](SnapshotBuilder::full) also captures memory and execution state. /// Publication is atomic and the local index remains a rebuildable cache. pub async fn create(config: SnapshotConfig) -> MicrosandboxResult { let backend = crate::backend::default_backend(); From 75429feacb9dad9eeafe5a831774eeedf199cf7a Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Wed, 9 Sep 2026 16:24:12 +0100 Subject: [PATCH 11/29] build(deps): use published snapshot runtime dependencies Replace the development libkrun and vm-memory Git patches with the published msb_krun 0.1.34 family and msb-vm-memory 0.18.0-msb.2. Update imago to 0.1.7 so disk and runtime code share one memory crate. Retain the existing firmware pin and the #7 stack base. Record release validation, 32 checkpoint and 39 snapshot regressions, and the successful Mac live branch/restore/growth/compaction run. --- Cargo.lock | 64 +++++++++++-------- Cargo.toml | 21 +----- .../registry-dependencies-2026-09-09.md | 49 ++++++++++++++ 3 files changed, 90 insertions(+), 44 deletions(-) create mode 100644 scripts/smoke/reports/registry-dependencies-2026-09-09.md diff --git a/Cargo.lock b/Cargo.lock index 66b6bdddb..eb7b583e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4322,9 +4322,9 @@ dependencies = [ [[package]] name = "msb-imago" -version = "0.1.5" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8563624d245da0b51b13959708758f06d1412cacdd17000b001f5ccae80cf6af" +checksum = "9b6feac477a4387d62fde89aad52900fa0ea7ccb4fec686f50b93a575cbf0ea5" dependencies = [ "async-trait", "cfg-if", @@ -4341,8 +4341,9 @@ dependencies = [ [[package]] name = "msb-vm-memory" -version = "0.18.0-msb.1" -source = "git+https://github.com/superradcompany/rust-vmm?rev=f798d4f274db22a3c458ba756900db2cd03e6fe9#f798d4f274db22a3c458ba756900db2cd03e6fe9" +version = "0.18.0-msb.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6646492f6bc51c4126c73854d377ba8ce4cebb1443cc9efe2517ab2215d87b49" dependencies = [ "libc", "thiserror", @@ -4351,8 +4352,9 @@ dependencies = [ [[package]] name = "msb_krun" -version = "0.1.33" -source = "git+https://github.com/superradcompany/libkrun?rev=862d68422b40ac7c26c731b3577025a5b1f64b89#862d68422b40ac7c26c731b3577025a5b1f64b89" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6a401c1ec192a3dfbc382901d8c0275f42c36097991a6426f5502a30f017ff0" dependencies = [ "crossbeam-channel", "kvm-bindings", @@ -4370,8 +4372,9 @@ dependencies = [ [[package]] name = "msb_krun_arch" -version = "0.1.33" -source = "git+https://github.com/superradcompany/libkrun?rev=862d68422b40ac7c26c731b3577025a5b1f64b89#862d68422b40ac7c26c731b3577025a5b1f64b89" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7362283e6bc924947c2a2415d7cd9025616fa45ea61bc4ec3ea00bddf8df36df" dependencies = [ "kvm-bindings", "kvm-ioctls", @@ -4384,13 +4387,15 @@ dependencies = [ [[package]] name = "msb_krun_arch_gen" -version = "0.1.33" -source = "git+https://github.com/superradcompany/libkrun?rev=862d68422b40ac7c26c731b3577025a5b1f64b89#862d68422b40ac7c26c731b3577025a5b1f64b89" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4101f8a221a95e8e747684c65d001b6d49899f88e81b41122f9de0fba3eefc3" [[package]] name = "msb_krun_cpuid" -version = "0.1.33" -source = "git+https://github.com/superradcompany/libkrun?rev=862d68422b40ac7c26c731b3577025a5b1f64b89#862d68422b40ac7c26c731b3577025a5b1f64b89" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dacf8d69bc3a568970bfc98918f2f60786a5438f8b5701b16dc0ac9f40af6f58" dependencies = [ "kvm-bindings", "kvm-ioctls", @@ -4399,8 +4404,9 @@ dependencies = [ [[package]] name = "msb_krun_devices" -version = "0.1.33" -source = "git+https://github.com/superradcompany/libkrun?rev=862d68422b40ac7c26c731b3577025a5b1f64b89#862d68422b40ac7c26c731b3577025a5b1f64b89" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe443bd0f03c68eaf33d63da12e8b3084ec080a6c105d8768d711cab062f2c60" dependencies = [ "bincode", "bitflags 1.3.2", @@ -4430,8 +4436,9 @@ dependencies = [ [[package]] name = "msb_krun_hvf" -version = "0.1.33" -source = "git+https://github.com/superradcompany/libkrun?rev=862d68422b40ac7c26c731b3577025a5b1f64b89#862d68422b40ac7c26c731b3577025a5b1f64b89" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f5833d1d7e6404ae1e3cc3be50a5ad3eceb1c61c2bdc3bb296f223ac0615a2a" dependencies = [ "crossbeam-channel", "libloading 0.8.9", @@ -4442,8 +4449,9 @@ dependencies = [ [[package]] name = "msb_krun_kernel" -version = "0.1.33" -source = "git+https://github.com/superradcompany/libkrun?rev=862d68422b40ac7c26c731b3577025a5b1f64b89#862d68422b40ac7c26c731b3577025a5b1f64b89" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "189508d48404ed0768d882f7e93e8ca1a093a77621462c4aa96f0637a02792be" dependencies = [ "msb-vm-memory", "msb_krun_utils", @@ -4451,8 +4459,9 @@ dependencies = [ [[package]] name = "msb_krun_polly" -version = "0.1.33" -source = "git+https://github.com/superradcompany/libkrun?rev=862d68422b40ac7c26c731b3577025a5b1f64b89#862d68422b40ac7c26c731b3577025a5b1f64b89" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e849910d2531272df93806a1e2f5830145d76e64c2d0161aa75a69dcc24ead8" dependencies = [ "libc", "msb_krun_utils", @@ -4460,16 +4469,18 @@ dependencies = [ [[package]] name = "msb_krun_smbios" -version = "0.1.33" -source = "git+https://github.com/superradcompany/libkrun?rev=862d68422b40ac7c26c731b3577025a5b1f64b89#862d68422b40ac7c26c731b3577025a5b1f64b89" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cf963ea3fcf2e564dd8982ae663339629eaf19a0ec1a4bd161d42c43e23a4ad" dependencies = [ "msb-vm-memory", ] [[package]] name = "msb_krun_utils" -version = "0.1.33" -source = "git+https://github.com/superradcompany/libkrun?rev=862d68422b40ac7c26c731b3577025a5b1f64b89#862d68422b40ac7c26c731b3577025a5b1f64b89" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "078445461df117b827681679390565e6f1ecf06a8d1f309d641dc6a1e390531c" dependencies = [ "bitflags 1.3.2", "crossbeam-channel", @@ -4483,8 +4494,9 @@ dependencies = [ [[package]] name = "msb_krun_vmm" -version = "0.1.33" -source = "git+https://github.com/superradcompany/libkrun?rev=862d68422b40ac7c26c731b3577025a5b1f64b89#862d68422b40ac7c26c731b3577025a5b1f64b89" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "223013c35221736169599da5a71fb5a801346c7a5e4439f21461bf5e503f16bb" dependencies = [ "bincode", "bzip2", diff --git a/Cargo.toml b/Cargo.toml index 1afc657af..235df60ae 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -88,8 +88,8 @@ microsandbox-protocol = { version = "=0.6.15", path = "crates/protocol" } microsandbox-runtime = { version = "=0.6.15", path = "crates/runtime", default-features = false } microsandbox-utils = { version = "=0.6.15", path = "crates/utils" } microsandbox-vsock = { version = "=0.6.15", path = "crates/vsock" } -msb_krun = "=0.1.33" -msb_krun_utils = "=0.1.33" +msb_krun = "=0.1.34" +msb_krun_utils = "=0.1.34" test-macros = { path = "crates/testing/macros" } test-utils = { path = "crates/testing/utils" } @@ -118,7 +118,7 @@ hex = "0.4" hickory-net = "0.26.1" hickory-proto = "0.26.1" httlib-hpack = "0.1.3" -imago = { package = "msb-imago", version = "0.1.5" } +imago = { package = "msb-imago", version = "0.1.7" } indicatif = "0.18" ipnetwork = { version = "0.21.0", features = ["serde"] } libc = "0.2" @@ -205,18 +205,3 @@ parking_lot = "0.12" rpassword = "7" russh = "0.62.4" russh-sftp = "2.3.0" - -# #8 construction-time private memory and identity-preserving resume support. -[patch.crates-io] -msb_krun = { git = "https://github.com/superradcompany/libkrun", rev = "862d68422b40ac7c26c731b3577025a5b1f64b89" } -msb_krun_utils = { git = "https://github.com/superradcompany/libkrun", rev = "862d68422b40ac7c26c731b3577025a5b1f64b89" } -msb_krun_vmm = { git = "https://github.com/superradcompany/libkrun", rev = "862d68422b40ac7c26c731b3577025a5b1f64b89" } -msb_krun_devices = { git = "https://github.com/superradcompany/libkrun", rev = "862d68422b40ac7c26c731b3577025a5b1f64b89" } -msb_krun_arch = { git = "https://github.com/superradcompany/libkrun", rev = "862d68422b40ac7c26c731b3577025a5b1f64b89" } -msb_krun_arch_gen = { git = "https://github.com/superradcompany/libkrun", rev = "862d68422b40ac7c26c731b3577025a5b1f64b89" } -msb_krun_cpuid = { git = "https://github.com/superradcompany/libkrun", rev = "862d68422b40ac7c26c731b3577025a5b1f64b89" } -msb_krun_hvf = { git = "https://github.com/superradcompany/libkrun", rev = "862d68422b40ac7c26c731b3577025a5b1f64b89" } -msb_krun_kernel = { git = "https://github.com/superradcompany/libkrun", rev = "862d68422b40ac7c26c731b3577025a5b1f64b89" } -msb_krun_polly = { git = "https://github.com/superradcompany/libkrun", rev = "862d68422b40ac7c26c731b3577025a5b1f64b89" } -msb_krun_smbios = { git = "https://github.com/superradcompany/libkrun", rev = "862d68422b40ac7c26c731b3577025a5b1f64b89" } -msb-vm-memory = { git = "https://github.com/superradcompany/rust-vmm", rev = "f798d4f274db22a3c458ba756900db2cd03e6fe9" } diff --git a/scripts/smoke/reports/registry-dependencies-2026-09-09.md b/scripts/smoke/reports/registry-dependencies-2026-09-09.md new file mode 100644 index 000000000..3a2bb5dac --- /dev/null +++ b/scripts/smoke/reports/registry-dependencies-2026-09-09.md @@ -0,0 +1,49 @@ +# Registry dependency qualification — 2026-09-09 + +This follow-up to #8 replaces development Cargo Git patches with published crates. It starts from Microsandbox `8e68722c47d7f06f76e51b78f44563939d47b4da`, retains the #7 stack base, and does not change the firmware pin, agent protocol, snapshot format, or public API. These are debug-build correctness checks, not release performance measurements. + +## Published dependencies + +| Package | Registry version | Release-source commit | +| --- | --- | --- | +| `msb-vm-memory` | `0.18.0-msb.2` | rust-vmm `c8aad4c`, on `appcypher/windows-private-memory` | +| `msb-imago` | `0.1.7` | imago `cba9c0c`, on `appcypher/release-memory-dependency` | +| All 15 `msb_krun` family crates | `0.1.34` | libkrun `b20d31a`, on `appcypher/registry-memory-dependencies` | + +All uploads completed and Cargo confirmed registry availability. Release-source branches were pushed; this does not claim they have merged into their default branches. No Microsandbox package was published. + +The memory release contains the Windows private-view implementation already used by #8's pinned `f798d4f` source. Imago's exact memory dependency had to move with it to avoid incompatible copies of the shared memory types; `0.1.7` also retains the previously released `0.1.6` tail-discard fix. Libkrun now uses those registry dependencies without a Git override. Microsandbox's lockfile changes only the intended 13 packages and resolves one `msb-vm-memory` version across imago and libkrun. + +## Release checks + +- Memory: Linux x86-64 passed 127 unit tests and 34 doctests, plus Clippy. macOS Clippy passed; 121 unit tests passed and five existing 4 KiB page-assumption tests failed on the 16 KiB-page host. Comparing against `0.18.0-msb.1` confirmed no Unix memory source changes. Windows ARM64 and x86-64 production-feature compilation passed; this is not a native Windows test run. Packaged dry run passed. +- Imago: all-feature build, Clippy, formatting, packaged dry run, 37 unit tests and four doctests passed; two tests were ignored. Windows ARM64 and x86-64 compilation passed. +- Libkrun: `cargo build --all --locked`, `cargo test --all --locked` (281 passed, six ignored), `cargo clippy --all --locked -- -D warnings`, formatting, and the coordinated 15-crate packaged dry run passed. `cargo check --locked -p msb_krun --features blk --target aarch64-pc-windows-msvc` passed with registry dependencies. + +## Microsandbox checks + +```bash +cargo build --locked -p microsandbox-cli --no-default-features --features net,ssh +cargo test --locked -p microsandbox-runtime --no-default-features --features net --lib checkpoint +cargo test --locked -p microsandbox --no-default-features --features net,ssh --lib snapshot +cargo fmt --all -- --check +git diff --check +``` + +Build and formatting passed; checkpoint tests passed 32/32 and snapshot tests passed 39/39. Cargo reported the existing future-incompatibility warning for `proc-macro-error2 2.0.1`. The CLI was codesigned with `msb-entitlements.plist` before live testing. + +## Mac live checks + +The existing `scripts/smoke/cli/direct-branch.py` harness ran with `STACK8_MAINTENANCE=1`, a new disposable `MSB_HOME`, flat 512 MiB disk, 256 MiB RAM, and two vCPUs. All 56 recorded steps had their expected result, including the two deliberate refusals: + +- First branch plus five repeated branches with retained siblings, without installed snapshot publication. +- Same-name refusal, private RAM/disk writes, source/sibling isolation, and a grandchild retaining its parent's private writes. +- Continuing guest timer-driven counter progress. +- Branch from a paused source, refusal to execute on the still-paused source, and ordinary resume. +- Three durable full captures and forked restores with subsequent guest reads. +- Live root growth to 768 MiB, explicit compaction, and another usable branch. +- Grandchild survival after source and parent stop; successful cleanup of all 13 test VMs. A final process check found no remaining runtimes with the test prefix. + +Evidence: `/private/tmp/msb8-registry-live.b1LAEv/results/results.json` and adjacent per-command logs. The signed debug CLI SHA-256 was `cdbfc66c9fae75d1a8d3e4015020889fa0f48b5ee01932c08de87a01282c8ab7`. Firmware SHA-256 was `ea0d458cdc12a0fa6dac8d192542ddc39717f816da41176582905e31a8bf868c`; the embedded agent SHA-256 was `4c467d1e93d9ba78168d0eb3ec6c0a5d03793b972c496e250fe0288a42e51e43`, matching the existing stack test artifacts. Raw guest state is not committed. + +This registry-only update was not live-rerun on Linux or Windows, and does not claim a new full platform matrix or language-SDK qualification. The Surface SSH connection timed out during this pass. Earlier successful platform coverage remains recorded separately in [CoW platform fixes](cow-platform-fixes-2026-09-09.md) and [live disk snapshot](live-disk-snapshot-2026-09-09.md); their historical source versions and measurements are unchanged. From 18c8fb8693957ac6e8ded964e42cfbe63f8506ad Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Wed, 9 Sep 2026 20:17:33 +0100 Subject: [PATCH 12/29] perf(snapshot): reduce repeated I/O and lifecycle overhead Pipeline capture persistence with two writers and three reusable 32 MiB packs. Reuse runtime-owned immutable RAM object receipts across captures and batch directory durability before the existing root-last publication. Pipeline eager restoration and cold memory backing construction with bounded verified buffers. Seed root journals from unchanged admitted disk files, retaining at most 32 file handles and hashing copied or rewritten layers. Keep SDK payload verification unchanged. Use one authoritative pause/resume mutation and validate its resulting state. Combine cgroup notifications with bounded state rechecks so kernel notification throttling cannot impose an avoidable 10 ms wait. Preserve snapshot formats, dirty-baseline ordering, clock-before-thaw, durability and failure cleanup. Add work counters and regression tests for corruption, cancellation, concurrent writers and low descriptor budgets. --- COMPATIBILITY.md | 6 + crates/agentd/lib/workload.rs | 288 ++++++- crates/cli/lib/commands/pause.rs | 2 +- crates/image/lib/checkpoint/admitted_disk.rs | 284 +++++++ crates/image/lib/checkpoint/mod.rs | 8 +- crates/image/lib/checkpoint/resolver.rs | 229 ++++- crates/image/lib/checkpoint/store.rs | 781 +++++++++++++++++- .../lib/checkpoint/capture_pipeline.rs | 428 ++++++++++ crates/runtime/lib/checkpoint/coordinator.rs | 228 ++--- crates/runtime/lib/checkpoint/disk.rs | 330 +++++++- crates/runtime/lib/checkpoint/memory_cache.rs | 174 +++- crates/runtime/lib/checkpoint/mod.rs | 4 +- .../runtime/lib/checkpoint/object_pipeline.rs | 248 ++++++ crates/runtime/lib/checkpoint/restore.rs | 102 ++- crates/runtime/lib/control/executor.rs | 37 + crates/runtime/lib/vm.rs | 6 + sdk/rust/lib/backend/local/sandbox/mod.rs | 67 +- sdk/rust/lib/sandbox/pause.rs | 134 ++- 18 files changed, 3068 insertions(+), 288 deletions(-) create mode 100644 crates/image/lib/checkpoint/admitted_disk.rs create mode 100644 crates/runtime/lib/checkpoint/capture_pipeline.rs create mode 100644 crates/runtime/lib/checkpoint/object_pipeline.rs diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index 5abc2e70e..f12833be0 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -134,6 +134,8 @@ Add operations and optional fields rather than redefining existing ones. Capabil Live disk-only snapshots use the distinct `disk_checkpoint_create` operation and capability. An absent capability is false: callers refuse before capture rather than silently capturing RAM or copying a writable disk. The runtime serializes the disk rollover with other control mutations and preserves a user's pause. This does not change the agent protocol or snapshot format; the result uses the existing file-state layer descriptor. Stopped disk capture retains its lifecycle lock and existing behavior. +Resident pause/resume sends one authoritative mutation rather than first observing pause state and querying capabilities. The runtime checks support before mutation, including idempotent requests, and the client requires the expected state in the response; unknown operations and incomplete replies fail. Ordinary get/list pause projection remains unchanged. Guest freezing retains one `cgroup.events` descriptor and waits for notifications with a fixed deadline; each poll timeout is capped at 1 ms so rate-limited kernel notifications cannot delay the next authoritative state check. Clock correction still precedes workload thaw, and no control or agent wire format changes. + ## 5. Launcher-to-Runtime Process Protocol Starting a sandbox crosses a private process boundary. On Unix, launch JSON is passed through inherited descriptor 96, the parent watchdog uses descriptor 97, startup JSON uses descriptor 98, and the lifecycle lock uses descriptor 99. Windows uses a short-lived launch-config file and platform-specific startup plumbing. Detach acknowledgement bytes and graceful-shutdown signals are also part of this contract. @@ -198,6 +200,10 @@ Evolution rules: Sources: [`crates/image/lib/snapshot/manifest.rs`](crates/image/lib/snapshot/manifest.rs), [`crates/image/lib/snapshot/migration.rs`](crates/image/lib/snapshot/migration.rs), and [`sdk/rust/lib/snapshot/archive.rs`](sdk/rust/lib/snapshot/archive.rs). +Runtime restore admits disk payloads before activation. Journal creation may reuse the verified root only for that same unchanged immutable file. Its in-process cache retains at most 32 file handles, preferring larger physical files; every layer still receives full admission, and uncached, copied or rewritten layers receive a fresh hash. Detected mutation of a retained admitted file fails. Candidates are opened once per lookup, with comparisons bounded by the cache size. This reuse is not a persistent "verified" flag or a path-only cache, and the cache bound does not reduce supported chain depth. + +Incremental capture retains immutable object receipts only within the owning runtime's store lifetime. Reopened stores and unadmitted objects still verify bytes. Receipts retain no file descriptors; active operations open, check and temporarily pin the exact file. Capture uses two writers and three recycled 32 MiB packs, then synchronizes new directory entries before the existing root-last publication. Eager restore and cold memory-cache construction use at most four reusable 32 MiB read/hash buffers. Errors join workers before cleanup. These changes preserve the snapshot format and restored bytes, dirty-baseline rollover, pause/publication ordering and durability barriers; summed worker timings must not be interpreted as additive wall time. + ## 10. OCI Cache and Materializer ABI The cache is rebuildable, but cache entries and closures can cross releases through `MSB_HOME` and snapshot archives. OCI semantics are externally defined: compressed descriptor digests, uncompressed diff IDs, ordered layers, whiteouts, opaque directories, hardlinks, extended attributes, non-UTF-8 paths, special files, and permissions must retain their meaning. diff --git a/crates/agentd/lib/workload.rs b/crates/agentd/lib/workload.rs index b1daadbde..41cff90a6 100644 --- a/crates/agentd/lib/workload.rs +++ b/crates/agentd/lib/workload.rs @@ -1,8 +1,8 @@ //! Checkpoint-time execution latch for agentd-managed workloads. use std::fs::{File, OpenOptions}; -use std::io; -use std::os::fd::{AsRawFd, OwnedFd}; +use std::io::{self, Read, Seek, SeekFrom}; +use std::os::fd::{AsRawFd, OwnedFd, RawFd}; use std::path::{Path, PathBuf}; use std::time::{Duration, Instant}; @@ -12,7 +12,7 @@ use std::time::{Duration, Instant}; const CGROUP_ROOT: &str = "/sys/fs/cgroup/microsandbox-workload"; const FREEZE_TIMEOUT: Duration = Duration::from_secs(5); -const FREEZE_POLL_INTERVAL: Duration = Duration::from_millis(1); +const FREEZE_STATE_RECHECK_INTERVAL: Duration = Duration::from_millis(1); const MAX_ATTEMPT_ID_BYTES: usize = 128; //-------------------------------------------------------------------------------------------------- @@ -44,6 +44,7 @@ trait FreezerControl: Send { struct CgroupFreezer { root: PathBuf, cgroup_procs: File, + cgroup_events: File, } /// A child-owned cgroup placement handle prepared before `fork`. @@ -231,27 +232,36 @@ impl CgroupFreezer { Ok(Self { root: root.to_path_buf(), cgroup_procs, + cgroup_events: File::open(events)?, }) } fn wait_for_state(&self, expected: bool) -> io::Result<()> { - let deadline = Instant::now() + FREEZE_TIMEOUT; - loop { - let events = std::fs::read_to_string(self.root.join("cgroup.events"))?; - if parse_frozen_event(&events) == Some(expected) { - return Ok(()); - } - if Instant::now() >= deadline { - return Err(io::Error::new( - io::ErrorKind::TimedOut, - format!( - "cgroup did not report frozen={} within {FREEZE_TIMEOUT:?}", - expected as u8 - ), - )); - } - std::thread::sleep(FREEZE_POLL_INTERVAL); - } + let mut events = &self.cgroup_events; + let fd = events.as_raw_fd(); + let mut contents = String::with_capacity(128); + // cgroup.events sends POLLPRI/POLLERR when frozen changes. Read on the same open + // descriptor before each wait: an early completion is observed immediately, and a + // change between read and poll remains pending on this descriptor's kernfs counter. + // cgroup_file_notify rate-limits notifications, however, so bounded poll timeouts + // also recheck authoritative state instead of waiting for a delayed notification. + wait_for_frozen_event( + expected, + Instant::now() + FREEZE_TIMEOUT, + || { + events.seek(SeekFrom::Start(0))?; + contents.clear(); + events.read_to_string(&mut contents)?; + parse_frozen_event(&contents).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "cgroup.events omitted frozen state", + ) + }) + }, + |remaining| wait_for_cgroup_event(fd, remaining), + Instant::now, + ) } } @@ -309,6 +319,68 @@ impl FreezerControl for CgroupFreezer { // Functions //-------------------------------------------------------------------------------------------------- +fn wait_for_frozen_event( + expected: bool, + deadline: Instant, + mut read_state: impl FnMut() -> io::Result, + mut wait: impl FnMut(Duration) -> io::Result<()>, + mut now: impl FnMut() -> Instant, +) -> io::Result<()> { + loop { + let interrupted = match read_state() { + Ok(state) if state == expected => return Ok(()), + Ok(_) => false, + Err(error) if error.kind() == io::ErrorKind::Interrupted => true, + Err(error) => return Err(error), + }; + let remaining = deadline.saturating_duration_since(now()); + if remaining.is_zero() { + return Err(io::Error::new( + io::ErrorKind::TimedOut, + format!( + "cgroup did not report frozen={} within {FREEZE_TIMEOUT:?}", + expected as u8 + ), + )); + } + if interrupted { + continue; + } + match wait(remaining.min(FREEZE_STATE_RECHECK_INTERVAL)) { + Ok(()) => {} + // Recheck state and the original deadline after interruptions or spurious events. + Err(error) if error.kind() == io::ErrorKind::Interrupted => {} + Err(error) => return Err(error), + } + } +} + +fn wait_for_cgroup_event(fd: RawFd, remaining: Duration) -> io::Result<()> { + let mut event = libc::pollfd { + fd, + events: libc::POLLPRI | libc::POLLERR, + revents: 0, + }; + // A notification can wake poll immediately. The one-millisecond timeout ceiling also + // covers transitions whose cgroup notification is deferred by the kernel's rate limit. + let timeout = remaining + .as_nanos() + .div_ceil(1_000_000) + .min(i32::MAX as u128) as i32; + let result = unsafe { libc::poll(&mut event, 1, timeout) }; + if result < 0 { + return Err(io::Error::last_os_error()); + } + if event.revents & (libc::POLLNVAL | libc::POLLHUP) != 0 { + return Err(io::Error::other( + "cgroup.events descriptor became unavailable", + )); + } + // POLLERR accompanies normal kernfs notifications; it is not by itself an I/O failure. + // Timeout and other wakeups both lead to a fresh state read and deadline check. + Ok(()) +} + fn validate_attempt_id(attempt_id: &str) -> Result<(), WorkloadLatchError> { if attempt_id.is_empty() || attempt_id.len() > MAX_ATTEMPT_ID_BYTES @@ -341,11 +413,187 @@ fn parse_frozen_event(events: &str) -> Option { #[cfg(test)] mod tests { + use std::cell::Cell; use std::collections::VecDeque; use std::sync::{Arc, Mutex}; use super::*; + #[test] + fn completed_freezer_transition_does_not_wait() { + let start = Instant::now(); + wait_for_frozen_event( + true, + start + FREEZE_TIMEOUT, + || Ok(true), + |_| panic!("already frozen"), + || start, + ) + .unwrap(); + } + + #[test] + fn freezer_transition_between_read_and_wait_is_not_lost() { + let start = Instant::now(); + let frozen = Cell::new(false); + let waits = Cell::new(0); + wait_for_frozen_event( + true, + start + FREEZE_TIMEOUT, + || Ok(frozen.get()), + |_| { + // Model a pending kernfs notification from a transition after the state read. + frozen.set(true); + waits.set(waits.get() + 1); + Ok(()) + }, + || start, + ) + .unwrap(); + assert_eq!(waits.get(), 1); + } + + #[test] + fn freezer_spurious_notifications_and_eintr_recheck_state() { + let start = Instant::now(); + let waits = Cell::new(0); + wait_for_frozen_event( + false, + start + FREEZE_TIMEOUT, + || Ok(waits.get() < 3), + |_| { + waits.set(waits.get() + 1); + if waits.get() == 2 { + Err(io::ErrorKind::Interrupted.into()) + } else { + Ok(()) + } + }, + || start, + ) + .unwrap(); + assert_eq!(waits.get(), 3); + } + + #[test] + fn freezer_delayed_notification_does_not_delay_the_authoritative_state_read() { + let start = Instant::now(); + let elapsed = Cell::new(Duration::ZERO); + let frozen = Cell::new(false); + let waits = Cell::new(0); + wait_for_frozen_event( + true, + start + FREEZE_TIMEOUT, + || Ok(frozen.get()), + |remaining| { + assert_eq!(remaining, Duration::from_millis(1)); + // The state is ready but cgroup_file_notify defers its notification by + // roughly 10 ms. A bounded timeout observes readiness without that event. + frozen.set(true); + elapsed.set(elapsed.get() + remaining); + waits.set(waits.get() + 1); + Ok(()) + }, + || start + elapsed.get(), + ) + .unwrap(); + assert_eq!(waits.get(), 1); + assert_eq!(elapsed.get(), Duration::from_millis(1)); + } + + #[test] + fn freezer_interruptions_do_not_extend_original_deadline() { + let start = Instant::now(); + let elapsed = Cell::new(Duration::ZERO); + let error = wait_for_frozen_event( + true, + start + Duration::from_millis(3), + || Ok(false), + |remaining| { + assert_eq!( + remaining, + (Duration::from_millis(3) - elapsed.get()).min(FREEZE_STATE_RECHECK_INTERVAL) + ); + elapsed.set(elapsed.get() + Duration::from_millis(1)); + Err(io::ErrorKind::Interrupted.into()) + }, + || start + elapsed.get(), + ) + .unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::TimedOut); + assert_eq!(elapsed.get(), Duration::from_millis(3)); + } + + #[test] + fn freezer_read_interruptions_retry_with_a_bounded_deadline() { + let start = Instant::now(); + let reads = Cell::new(0); + let error = wait_for_frozen_event( + true, + start + Duration::from_millis(3), + || { + reads.set(reads.get() + 1); + Err(io::ErrorKind::Interrupted.into()) + }, + |_| panic!("an interrupted read must be retried before waiting"), + || start + Duration::from_millis(reads.get()), + ) + .unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::TimedOut); + assert_eq!(reads.get(), 3); + } + + #[test] + fn freezer_read_and_notification_errors_are_not_success() { + let start = Instant::now(); + let error = wait_for_frozen_event( + true, + start + FREEZE_TIMEOUT, + || Err(io::ErrorKind::InvalidData.into()), + |_| unreachable!(), + || start, + ) + .unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + let error = wait_for_frozen_event( + true, + start + FREEZE_TIMEOUT, + || Ok(false), + |_| Err(io::ErrorKind::BrokenPipe.into()), + || start, + ) + .unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::BrokenPipe); + } + + #[test] + fn freezer_reads_final_state_before_reporting_timeout() { + let start = Instant::now(); + let completed = Cell::new(false); + wait_for_frozen_event( + true, + start + FREEZE_TIMEOUT, + || Ok(completed.get()), + |_| { + completed.set(true); + Ok(()) + }, + || { + if completed.get() { + start + FREEZE_TIMEOUT + } else { + start + } + }, + ) + .unwrap(); + } + + #[test] + fn invalid_cgroup_poll_descriptor_is_an_error() { + assert!(wait_for_cgroup_event(i32::MAX, Duration::from_millis(1)).is_err()); + } + struct FakeFreezer { states: Arc>>, } diff --git a/crates/cli/lib/commands/pause.rs b/crates/cli/lib/commands/pause.rs index f68c33ddc..28fca5343 100644 --- a/crates/cli/lib/commands/pause.rs +++ b/crates/cli/lib/commands/pause.rs @@ -25,7 +25,7 @@ pub struct PauseArgs { /// Change resident execution state through host control, without opening the guest agent. pub async fn run(args: PauseArgs, resume: bool) -> anyhow::Result<()> { - let sandbox = Sandbox::get(&args.name).await?; + let sandbox = Sandbox::get_for_control(&args.name).await?; if resume { sandbox.resume().await?; } else { diff --git a/crates/image/lib/checkpoint/admitted_disk.rs b/crates/image/lib/checkpoint/admitted_disk.rs new file mode 100644 index 000000000..6e561aa63 --- /dev/null +++ b/crates/image/lib/checkpoint/admitted_disk.rs @@ -0,0 +1,284 @@ +//! In-process reuse of a verified immutable disk file, never a path-only verification cache. + +use std::fs::{File, Metadata, OpenOptions}; +use std::io; +use std::path::Path; +use std::sync::Arc; +use std::time::SystemTime; + +use super::sparse_file_integrity; +use crate::error::{ImageError, ImageResult}; + +//-------------------------------------------------------------------------------------------------- +// Constants +//-------------------------------------------------------------------------------------------------- + +/// Admission still checks every layer; only this many file handles are kept for hash reuse. +const MAX_ADMITTED_DISK_FILES: usize = 32; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// A bounded optimization cache, not a limit on the number of admitted disk layers. +#[derive(Clone, Debug, Default)] +pub(super) struct AdmittedDiskLayers { + layers: Vec, +} + +/// A file handle keeps the admitted inode alive even if its original name is removed. +#[derive(Clone, Debug)] +struct AdmittedDiskLayer { + file: Arc, + stamp: FileStamp, + root: String, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct FileStamp { + identity: (u64, u64), + length: u64, + modified: SystemTime, +} + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl AdmittedDiskLayers { + pub(super) fn admit(&mut self, path: &Path, expected: &str) -> ImageResult<()> { + // Always verify, even when the resulting receipt will not fit in the cache. Retaining + // the largest physical files usually keeps the expensive base disk rather than tiny + // overlays; uncached layers simply take the caller's normal fresh-hash path. + let layer = AdmittedDiskLayer::admit(path, expected)?; + if self.layers.len() < MAX_ADMITTED_DISK_FILES { + self.layers.push(layer); + } else if let Some((index, smallest)) = self + .layers + .iter() + .enumerate() + .min_by_key(|(_, candidate)| candidate.stamp.length) + && layer.stamp.length > smallest.stamp.length + { + self.layers[index] = layer; + } + Ok(()) + } + + pub(super) fn reuse_for(&self, path: &Path) -> ImageResult> { + if self.layers.is_empty() { + return Ok(None); + } + // Open once per candidate, not once per admitted ancestor. The remaining checks are + // bounded by the cache size, including detection of a changed retained inode. + let candidate = FileStamp::read(&open_regular(path)?)?; + for layer in &self.layers { + if let Some(root) = layer.reuse_for(&candidate)? { + return Ok(Some(root)); + } + } + Ok(None) + } +} + +impl AdmittedDiskLayer { + fn admit(path: &Path, expected: &str) -> ImageResult { + let file = open_regular(path)?; + let stamp = FileStamp::read(&file)?; + let integrity = sparse_file_integrity(path)?; + if integrity.root != expected { + return Err(ImageError::DigestMismatch { + digest: path.display().to_string(), + expected: expected.into(), + actual: integrity.root, + }); + } + // Cooperative writers never mutate sealed files. Detect accidental replacement or a + // concurrent writer before binding the computed root to this exact owned file. + if FileStamp::read(&file)? != stamp || FileStamp::read(&open_regular(path)?)? != stamp { + return Err(io::Error::other("disk layer changed during admission").into()); + } + Ok(Self { + file: Arc::new(file), + stamp, + root: expected.into(), + }) + } + + fn reuse_for(&self, candidate: &FileStamp) -> ImageResult> { + // A copied or header-relocated layer needs a fresh identity, but an admitted sealed + // inode changing is corruption. Never bless its replacement contents as a new root. + if FileStamp::read(&self.file)? != self.stamp + || (candidate.identity == self.stamp.identity && *candidate != self.stamp) + { + return Err(io::Error::other("admitted disk layer was modified").into()); + } + if *candidate == self.stamp { + Ok(Some(&self.root)) + } else { + Ok(None) + } + } +} + +impl FileStamp { + fn read(file: &File) -> io::Result { + let metadata = file.metadata()?; + Ok(Self { + identity: file_identity(file, &metadata)?, + length: metadata.len(), + modified: metadata.modified()?, + }) + } +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +fn open_regular(path: &Path) -> io::Result { + if !std::fs::symlink_metadata(path)?.is_file() { + return Err(io::Error::other("disk layer is not a regular file")); + } + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK); + } + #[cfg(windows)] + { + use std::os::windows::fs::OpenOptionsExt; + use windows_sys::Win32::Storage::FileSystem::{FILE_SHARE_DELETE, FILE_SHARE_READ}; + options.share_mode(FILE_SHARE_READ | FILE_SHARE_DELETE); + } + let file = options.open(path)?; + if !file.metadata()?.is_file() { + return Err(io::Error::other("disk layer is not a regular file")); + } + Ok(file) +} + +#[cfg(unix)] +fn file_identity(_file: &File, metadata: &Metadata) -> io::Result<(u64, u64)> { + use std::os::unix::fs::MetadataExt; + Ok((metadata.dev(), metadata.ino())) +} + +#[cfg(windows)] +fn file_identity(file: &File, _metadata: &Metadata) -> io::Result<(u64, u64)> { + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::Storage::FileSystem::{ + BY_HANDLE_FILE_INFORMATION, GetFileInformationByHandle, + }; + let mut info: BY_HANDLE_FILE_INFORMATION = unsafe { std::mem::zeroed() }; + // The live File owns this handle; the API fills the complete fixed-size output structure. + if unsafe { GetFileInformationByHandle(file.as_raw_handle(), &mut info) } == 0 { + return Err(io::Error::last_os_error()); + } + Ok(( + u64::from(info.dwVolumeSerialNumber), + u64::from(info.nFileIndexHigh) << 32 | u64::from(info.nFileIndexLow), + )) +} + +#[cfg(not(any(unix, windows)))] +fn file_identity(_file: &File, _metadata: &Metadata) -> io::Result<(u64, u64)> { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "disk file identity is unavailable", + )) +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn admission_reuses_a_hardlink_but_not_a_copy_or_replacement() { + let dir = tempfile::tempdir().unwrap(); + let source = dir.path().join("source"); + let linked = dir.path().join("linked"); + let copied = dir.path().join("copied"); + std::fs::write(&source, b"sealed disk").unwrap(); + let root = sparse_file_integrity(&source).unwrap().root; + let mut admitted = AdmittedDiskLayers::default(); + admitted.admit(&source, &root).unwrap(); + std::fs::hard_link(&source, &linked).unwrap(); + std::fs::copy(&source, &copied).unwrap(); + assert_eq!(admitted.reuse_for(&linked).unwrap(), Some(root.as_str())); + assert_eq!(admitted.reuse_for(&copied).unwrap(), None); + std::fs::remove_file(&source).unwrap(); + std::fs::write(&source, b"sealed disk").unwrap(); + assert_eq!(admitted.reuse_for(&source).unwrap(), None); + assert_eq!(admitted.reuse_for(&linked).unwrap(), Some(root.as_str())); + } + + #[test] + fn admission_rejects_corruption() { + let dir = tempfile::tempdir().unwrap(); + let source = dir.path().join("source"); + std::fs::write(&source, b"original").unwrap(); + let root = sparse_file_integrity(&source).unwrap().root; + std::fs::write(&source, b"modified").unwrap(); + assert!(AdmittedDiskLayers::default().admit(&source, &root).is_err()); + } + + #[cfg(unix)] + #[test] + fn detected_mutation_is_not_treated_as_an_unrelated_copy() { + use std::fs::FileTimes; + use std::time::Duration; + + let dir = tempfile::tempdir().unwrap(); + let source = dir.path().join("source"); + let copy = dir.path().join("copy"); + std::fs::write(&source, b"original").unwrap(); + std::fs::copy(&source, ©).unwrap(); + let root = sparse_file_integrity(&source).unwrap().root; + let mut admitted = AdmittedDiskLayers::default(); + admitted.admit(&source, &root).unwrap(); + std::fs::write(&source, b"modified").unwrap(); + let writer = OpenOptions::new().write(true).open(&source).unwrap(); + writer + .set_times( + FileTimes::new() + .set_modified(admitted.layers[0].stamp.modified + Duration::from_secs(1)), + ) + .unwrap(); + assert!(admitted.reuse_for(&source).is_err()); + assert!(admitted.reuse_for(©).is_err()); + } + + #[test] + fn receipt_cache_retains_largest_files_and_still_admits_uncached_layers() { + let dir = tempfile::tempdir().unwrap(); + let mut admitted = AdmittedDiskLayers::default(); + let mut paths = Vec::new(); + for length in 1..=MAX_ADMITTED_DISK_FILES + 3 { + let path = dir.path().join(format!("layer-{length}")); + std::fs::write(&path, vec![0x55; length]).unwrap(); + let root = sparse_file_integrity(&path).unwrap().root; + admitted.admit(&path, &root).unwrap(); + assert!(admitted.layers.len() <= MAX_ADMITTED_DISK_FILES); + paths.push((path, root)); + } + assert_eq!(admitted.layers.len(), MAX_ADMITTED_DISK_FILES); + for (index, (path, root)) in paths.iter().enumerate() { + let expected = (index >= 3).then_some(root.as_str()); + assert_eq!(admitted.reuse_for(path).unwrap(), expected); + } + + // The smallest layer will not receive a retained receipt, but its bytes must still + // pass full admission. A full cache is never permission to skip verification. + let (smallest, root) = &paths[0]; + std::fs::write(smallest, b"X").unwrap(); + assert!(admitted.admit(smallest, root).is_err()); + assert_eq!(admitted.layers.len(), MAX_ADMITTED_DISK_FILES); + } +} diff --git a/crates/image/lib/checkpoint/mod.rs b/crates/image/lib/checkpoint/mod.rs index 69ded76d9..21e8b859c 100644 --- a/crates/image/lib/checkpoint/mod.rs +++ b/crates/image/lib/checkpoint/mod.rs @@ -3,6 +3,7 @@ //! Checkpoint artifacts keep guest-visible state in canonical, content-addressed objects. Mutable //! operation progress, runtime ownership, and provider locations deliberately live elsewhere. +mod admitted_disk; mod compact; mod layer_selection; mod manifest; @@ -26,5 +27,8 @@ pub use manifest::{ ResourceDescriptor, ResourceTreatment, }; pub use qcow::{create_qcow2_overlay, relocate_qcow2_backing, relocated_qcow2_header}; -pub use resolver::CheckpointClosure; -pub use store::{LocalObjectStore, ObjectId, SparseFileIntegrity, sparse_file_integrity}; +pub use resolver::{CheckpointClosure, CheckpointObjectReadTiming}; +pub use store::{ + AdmittedObject, CaptureObjectBatch, CaptureObjectBatchStats, LocalObjectStore, ObjectId, + SparseFileIntegrity, sparse_file_integrity, +}; diff --git a/crates/image/lib/checkpoint/resolver.rs b/crates/image/lib/checkpoint/resolver.rs index d79b57fdf..6fc00578e 100644 --- a/crates/image/lib/checkpoint/resolver.rs +++ b/crates/image/lib/checkpoint/resolver.rs @@ -4,12 +4,14 @@ use std::collections::BTreeSet; use std::fs::File; use std::io::Read; use std::path::{Path, PathBuf}; +use std::time::Instant; use sha2::{Digest as _, Sha256}; +use super::admitted_disk::AdmittedDiskLayers; use super::{ CheckpointManifest, DiskGenerationManifest, DiskLayerRef, MemoryExtentContent, MemoryManifest, - ObjectId, sparse_file_integrity, + ObjectId, }; use crate::error::{ImageError, ImageResult}; @@ -38,6 +40,16 @@ pub struct CheckpointClosure { checkpoint: CheckpointManifest, memory: MemoryManifest, disks: Vec, + admitted_disks: AdmittedDiskLayers, +} + +/// Separate wall times for loading and verifying one checkpoint object. +#[derive(Clone, Copy, Debug, Default)] +pub struct CheckpointObjectReadTiming { + /// File open, allocation or buffer growth, and read time in microseconds. + pub read_us: u128, + /// Identity verification time in microseconds. + pub hash_us: u128, } //-------------------------------------------------------------------------------------------------- @@ -105,6 +117,7 @@ impl CheckpointClosure { } let mut disks = Vec::with_capacity(checkpoint.disks.len()); + let mut admitted_disks = AdmittedDiskLayers::default(); let mut volumes = BTreeSet::new(); for disk_id in &checkpoint.disks { let bytes = read_object_verified(&root, disk_id, MAX_MANIFEST_BYTES)?; @@ -115,7 +128,11 @@ impl CheckpointClosure { if !volumes.insert(disk.volume_id.clone()) { return checkpoint_error("checkpoint repeats a logical disk volume"); } - validate_disk_layers(&root, &disk)?; + for layer in &disk.layers { + let path = disk_layer_path(&root, layer); + open_regular(&path)?; + admitted_disks.admit(&path, &layer.integrity_root)?; + } disks.push(disk); } @@ -125,6 +142,7 @@ impl CheckpointClosure { checkpoint, memory, disks, + admitted_disks, }) } @@ -153,11 +171,29 @@ impl CheckpointClosure { read_object_verified(&self.root, id, max_len) } + /// Load and verify one object into reusable storage, without changing its identity contract. + /// The buffer must not be consumed when this method fails. + pub fn read_object_into( + &self, + id: &ObjectId, + max_len: u64, + bytes: &mut Vec, + ) -> ImageResult { + read_object_verified_into(&self.root, id, max_len, bytes) + } + /// Return the confined path of a validated disk layer. pub fn disk_layer_path(&self, layer: &DiskLayerRef) -> PathBuf { - self.root - .join("layers") - .join(format!("{}.{}", layer.layer_id, layer.format)) + disk_layer_path(&self.root, layer) + } + + /// Reuse a disk root only while the candidate is the exact unchanged admitted file. + /// Copies, rewritten qcow headers, replaced names, and uncached layers require a new integrity + /// computation. Retained file handles are bounded independently of admitted chain depth. + pub fn reused_disk_integrity(&self, path: &Path) -> ImageResult> { + self.admitted_disks + .reuse_for(path) + .map(|root| root.map(str::to_owned)) } /// Stream and verify every immutable memory payload referenced by the logical generation. @@ -225,28 +261,24 @@ fn validate_memory_objects(root: &Path, memory: &MemoryManifest) -> ImageResult< Ok(()) } -fn validate_disk_layers(root: &Path, disk: &DiskGenerationManifest) -> ImageResult<()> { - for layer in &disk.layers { - let path = root - .join("layers") - .join(format!("{}.{}", layer.layer_id, layer.format)); - let metadata = std::fs::symlink_metadata(&path)?; - if !metadata.file_type().is_file() { - return checkpoint_error("checkpoint disk layer is not a regular file"); - } - let integrity = sparse_file_integrity(&path)?; - if integrity.root != layer.integrity_root { - return Err(ImageError::DigestMismatch { - digest: layer.layer_id.clone(), - expected: layer.integrity_root.clone(), - actual: integrity.root, - }); - } - } - Ok(()) +fn disk_layer_path(root: &Path, layer: &DiskLayerRef) -> PathBuf { + root.join("layers") + .join(format!("{}.{}", layer.layer_id, layer.format)) } fn read_object_verified(root: &Path, id: &ObjectId, max_len: u64) -> ImageResult> { + let mut bytes = Vec::new(); + read_object_verified_into(root, id, max_len, &mut bytes)?; + Ok(bytes) +} + +fn read_object_verified_into( + root: &Path, + id: &ObjectId, + max_len: u64, + bytes: &mut Vec, +) -> ImageResult { + let started = Instant::now(); let path = object_path(root, id); let mut file = open_regular(&path)?; let length = file.metadata()?.len(); @@ -255,9 +287,17 @@ fn read_object_verified(root: &Path, id: &ObjectId, max_len: u64) -> ImageResult } let length = usize::try_from(length) .map_err(|_| checkpoint_error_value("checkpoint object exceeds host limits"))?; - let mut bytes = Vec::with_capacity(length); - file.read_to_end(&mut bytes)?; - let actual = ObjectId::from_bytes(&bytes)?; + // Keep the initialized buffer between packs. Unlike clear + resize this does not zero + // an entire reused pack before the file read overwrites it. A growing file cannot make + // read_to_end allocate beyond the admitted object bound. + bytes.resize(length, 0); + file.read_exact(bytes)?; + if file.read(&mut [0u8; 1])? != 0 { + return checkpoint_error("checkpoint object changed length during read"); + } + let read_us = started.elapsed().as_micros(); + let hash_started = Instant::now(); + let actual = ObjectId::from_bytes(bytes)?; if &actual != id { return Err(ImageError::DigestMismatch { digest: id.to_string(), @@ -265,7 +305,10 @@ fn read_object_verified(root: &Path, id: &ObjectId, max_len: u64) -> ImageResult actual: actual.to_string(), }); } - Ok(bytes) + Ok(CheckpointObjectReadTiming { + read_us, + hash_us: hash_started.elapsed().as_micros(), + }) } fn verify_object_streaming(root: &Path, id: &ObjectId) -> ImageResult<()> { @@ -345,6 +388,27 @@ mod tests { ResourceDescriptor, ResourceTreatment, }; + #[test] + fn reusable_object_reader_checks_each_identity_and_reuses_allocation() { + let directory = tempfile::tempdir().unwrap(); + let store = super::super::LocalObjectStore::open(directory.path()).unwrap(); + let first = store.put_bytes(b"first payload").unwrap(); + let second = store.put_bytes(b"next payload!").unwrap(); + let mut buffer = Vec::with_capacity(64); + let allocation = buffer.as_ptr(); + read_object_verified_into(directory.path(), &first, 64, &mut buffer).unwrap(); + assert_eq!(buffer, b"first payload"); + read_object_verified_into(directory.path(), &second, 64, &mut buffer).unwrap(); + assert_eq!(buffer, b"next payload!"); + assert_eq!(buffer.as_ptr(), allocation); + assert!(read_object_verified_into(directory.path(), &first, 4, &mut buffer).is_err()); + std::fs::write(store.object_path(&second), b"bad payload!!").unwrap(); + assert!(matches!( + read_object_verified_into(directory.path(), &second, 64, &mut buffer), + Err(ImageError::DigestMismatch { .. }) + )); + } + fn fixture() -> (tempfile::TempDir, ObjectId) { let directory = tempfile::tempdir().unwrap(); let store = super::super::LocalObjectStore::open(directory.path()).unwrap(); @@ -425,6 +489,115 @@ mod tests { assert_eq!(closure.memory().pause_generation, 7); } + #[test] + fn deep_closure_admission_keeps_file_handles_bounded() { + #[cfg(unix)] + if std::env::var_os("MSB_TEST_DEEP_ADMISSION_LOW_FD").is_none() { + use std::os::unix::process::CommandExt; + + // Isolate the process-wide limit from concurrently running tests. The old one-FD- + // per-layer implementation cannot admit these 512 files with only 64 descriptors. + let mut child = std::process::Command::new(std::env::current_exe().unwrap()); + child + .args([ + "--exact", + "checkpoint::resolver::tests::deep_closure_admission_keeps_file_handles_bounded", + "--nocapture", + ]) + .env("MSB_TEST_DEEP_ADMISSION_LOW_FD", "1"); + // SAFETY: the pre-exec callback only invokes async-signal-safe libc resource-limit + // operations; it does not allocate or acquire locks in the forked child. + unsafe { + child.pre_exec(|| { + let mut limit = std::mem::zeroed::(); + if libc::getrlimit(libc::RLIMIT_NOFILE, &mut limit) != 0 { + return Err(std::io::Error::last_os_error()); + } + limit.rlim_cur = limit.rlim_max.min(64); + if libc::setrlimit(libc::RLIMIT_NOFILE, &limit) != 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + let output = child.output().unwrap(); + assert!( + output.status.success(), + "low-FD admission failed: {}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!(String::from_utf8_lossy(&output.stdout).contains("1 passed")); + return; + } + + let (directory, _) = fixture(); + let store = super::super::LocalObjectStore::open(directory.path()).unwrap(); + let root_path = directory.path().join(CHECKPOINT_ROOT_FILE); + let mut checkpoint = + CheckpointManifest::from_bytes(&std::fs::read(&root_path).unwrap()).unwrap(); + std::fs::create_dir_all(directory.path().join("layers")).unwrap(); + let mut paths = Vec::new(); + // Each volume stays inside the existing 256-layer manifest limit. Disk header codecs + // are runtime concerns: this resolver fixture exercises byte admission and membership. + for volume in 0..2 { + let mut layers = Vec::new(); + for index in 0..256 { + let layer_id = format!("volume_{volume}_layer_{index}"); + let format = if index == 0 { "raw" } else { "qcow2" }; + let path = directory + .path() + .join("layers") + .join(format!("{layer_id}.{format}")); + std::fs::write(&path, [0x55]).unwrap(); + let integrity_root = super::super::sparse_file_integrity(&path).unwrap().root; + layers.push(DiskLayerRef { + layer_id, + format: format.into(), + virtual_size: 4096, + predecessor: (index > 0) + .then(|| format!("volume_{volume}_layer_{}", index - 1)), + integrity_root, + }); + paths.push(path); + } + let disk = DiskGenerationManifest { + schema: "microsandbox.disk-generation/1".into(), + volume_id: format!("volume_{volume}"), + device_id: format!("device_{volume}"), + generation: 1, + head: layers.last().unwrap().layer_id.clone(), + layers, + pause_generation: checkpoint.pause_generation, + }; + checkpoint.disks.push( + store + .put_bytes(&disk.to_canonical_bytes().unwrap()) + .unwrap(), + ); + } + let bytes = checkpoint.to_canonical_bytes().unwrap(); + let root = ObjectId::from_bytes(&bytes).unwrap(); + std::fs::write(&root_path, bytes).unwrap(); + + let closure = CheckpointClosure::open(directory.path(), Some(&root)).unwrap(); + assert_eq!(closure.disks().len(), 2); + assert!(closure.disks().iter().all(|disk| disk.layers.len() == 256)); + let reusable = paths + .iter() + .filter(|path| closure.reused_disk_integrity(path).unwrap().is_some()) + .count(); + assert_eq!(reusable, 32); + drop(closure); + + // A layer outside the retained receipt set must still be verified during admission. + std::fs::write(paths.last().unwrap(), [0xAA]).unwrap(); + assert!(matches!( + CheckpointClosure::open(directory.path(), Some(&root)), + Err(ImageError::DigestMismatch { .. }) + )); + } + #[test] fn portable_open_separates_integrity_from_restore_architecture() { let (directory, _expected) = fixture(); diff --git a/crates/image/lib/checkpoint/store.rs b/crates/image/lib/checkpoint/store.rs index f56e74544..d9e033c45 100644 --- a/crates/image/lib/checkpoint/store.rs +++ b/crates/image/lib/checkpoint/store.rs @@ -1,9 +1,13 @@ //! Crash-safe local immutable-object storage. +use std::collections::{BTreeMap, BTreeSet}; use std::fmt; use std::fs::{File, OpenOptions}; use std::io::{Read, Seek, SeekFrom, Write}; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::SystemTime; use serde::{Deserialize, Serialize}; use sha2::{Digest as _, Sha256}; @@ -32,6 +36,61 @@ pub struct ObjectId(String); #[derive(Clone, Debug)] pub struct LocalObjectStore { root: PathBuf, + ownership: Arc, +} + +#[derive(Debug, Default)] +struct StoreOwnership { + publication: Mutex<()>, +} + +/// A verified immutable inode identity in a live runtime's owned object store. +/// +/// This receipt is neither serializable nor constructible from a path, and is scoped to its store +/// instance. The owning runtime must retain its published object names and never mutate their bytes. +/// Reuse opens and pins the exact inode only for the active operation, checking identity and stamp; +/// missing, replaced or modified members fail closed. Retaining generations therefore costs no FD +/// per object. Unadmitted stores/imports still verify payloads instead of trusting these receipts. +#[derive(Clone, Debug)] +pub struct AdmittedObject { + id: ObjectId, + path: PathBuf, + stamp: ObjectStamp, + ownership: Arc, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct ObjectStamp { + identity: (u64, u64), + length: u64, + modified: SystemTime, +} + +/// Capture-local object publication with deferred directory durability, but durable file data. +/// +/// Finish this batch before publishing any manifest root. Existing `LocalObjectStore::put_bytes` +/// keeps its immediate durability contract; only this explicitly scoped API batches directory sync. +pub struct CaptureObjectBatch { + store: LocalObjectStore, + admitted: Mutex>, + directories: Mutex>, + hashed_bytes: AtomicU64, + linked_bytes: AtomicU64, + copied_bytes: AtomicU64, + directory_syncs: AtomicU64, +} + +/// Actual work performed by a capture object batch, independent of its logical RAM size. +#[derive(Clone, Copy, Debug, Default)] +pub struct CaptureObjectBatchStats { + /// Bytes hashed to create or admit immutable objects. + pub hashed_bytes: u64, + /// Bytes referenced by newly installed closure links, including copy fallbacks. + pub linked_bytes: u64, + /// Bytes physically copied when hardlinks were unavailable. + pub copied_bytes: u64, + /// Directory durability barriers issued by this batch. + pub directory_syncs: u64, } /// Sparse-aware immutable identity of one physical layer file. @@ -94,7 +153,10 @@ impl LocalObjectStore { pub fn open(root: impl Into) -> ImageResult { let root = root.into(); std::fs::create_dir_all(root.join("objects").join("sha256"))?; - Ok(Self { root }) + Ok(Self { + root, + ownership: Arc::new(StoreOwnership::default()), + }) } /// Store exact bytes durably and return their immutable identity. @@ -103,6 +165,7 @@ impl LocalObjectStore { let path = self.object_path(&id); if path.exists() { self.verify_existing(&id, &path)?; + sync_directories_through(path.parent().expect("object parent"), &self.root)?; return Ok(id); } let parent = path.parent().expect("object path has a parent"); @@ -115,17 +178,10 @@ impl LocalObjectStore { file.write_all(bytes)?; file.sync_all()?; drop(file); - match std::fs::rename(&temporary, &path) { - Ok(()) => {} - Err(_error) if path.exists() => { - let _ = std::fs::remove_file(&temporary); - self.verify_existing(&id, &path)?; - return Ok(id); - } - Err(error) => { - let _ = std::fs::remove_file(&temporary); - return Err(error.into()); - } + let published = publish_object_file(&temporary, &path, &self.ownership); + let _ = std::fs::remove_file(&temporary); + if !published? { + self.verify_existing(&id, &path)?; } sync_directories_through(parent, &self.root)?; Ok(id) @@ -201,6 +257,303 @@ impl LocalObjectStore { } } +impl CaptureObjectBatch { + /// Start a new batch, retaining only explicitly supplied previous-generation capabilities. + pub fn new(store: LocalObjectStore, previous: &[AdmittedObject]) -> Self { + let admitted = previous + .iter() + .filter(|object| Arc::ptr_eq(&object.ownership, &store.ownership)) + .map(|object| (object.id.clone(), object.clone())) + .collect(); + Self { + store, + admitted: Mutex::new(admitted), + directories: Mutex::new(BTreeSet::new()), + hashed_bytes: AtomicU64::new(0), + linked_bytes: AtomicU64::new(0), + copied_bytes: AtomicU64::new(0), + directory_syncs: AtomicU64::new(0), + } + } + + /// Hash new bytes once and store durable file data. Directory entries commit at `finish`. + pub fn put_bytes(&self, bytes: &[u8]) -> ImageResult { + let id = ObjectId::from_bytes(bytes)?; + self.hashed_bytes + .fetch_add(bytes.len() as u64, Ordering::Relaxed); + if let Some(object) = self.admitted.lock().unwrap().get(&id).cloned() { + object.validate()?; + return Ok(id); + } + let path = self.store.object_path(&id); + if path.exists() { + self.admit(&id)?; + // Also sync the path of an object left by a previously interrupted batch. + self.record_directories(path.parent().unwrap(), &self.store.root); + return Ok(id); + } + let parent = path.parent().expect("object parent"); + std::fs::create_dir_all(parent)?; + let temporary = parent.join(format!(".{}.{}.tmp", id.hex(), rand::random::())); + let result = (|| -> ImageResult { + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(&temporary)?; + file.write_all(bytes)?; + file.sync_all()?; + drop(file); + if publish_object_file(&temporary, &path, &self.store.ownership)? { + self.open_admitted(id.clone(), path.clone()) + } else { + self.admit(&id) + } + })(); + let _ = std::fs::remove_file(&temporary); + let object = result?; + self.admitted.lock().unwrap().insert(id.clone(), object); + self.record_directories(parent, &self.store.root); + Ok(id) + } + + /// Link admitted bytes without rehashing the generation's complete inherited RAM payload. + pub fn link_into(&self, id: &ObjectId, closure_root: &Path) -> ImageResult { + // A receipt itself is not sufficient authority to read bytes. Pin/check it once below; + // avoid the redundant open that admitting an already-owned receipt would otherwise do. + let known = self.admitted.lock().unwrap().get(id).cloned(); + let object = match known { + Some(object) => object, + None => self.admit(id)?, + }; + let pinned = object.pin()?; + let encoded = id.hex(); + let target = closure_root + .join("objects") + .join("sha256") + .join(&encoded[..2]) + .join(encoded); + let parent = target.parent().expect("closure object parent"); + std::fs::create_dir_all(parent)?; + if target.exists() { + // Existing targets are not automatically part of this batch's ownership. Retain + // the checked public behavior, except for the exact already-admitted inode. + let file = File::open(&target)?; + if ObjectStamp::read(&file)? != object.stamp { + self.verify_and_sync_target(id, &target)?; + } + } else { + match std::fs::hard_link(&object.path, &target) { + Ok(()) => { + if ObjectStamp::read(&File::open(&target)?)? != object.stamp { + let _ = std::fs::remove_file(&target); + return Err( + std::io::Error::other("admitted object path was replaced").into() + ); + } + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + self.verify_and_sync_target(id, &target)?; + } + Err(_) => { + // Copy the retained inode, not a potentially replaced path. Positioned + // reads avoid shared cursor races when several closures reuse one object. + copy_admitted_object(&object, &pinned, &target)?; + self.copied_bytes + .fetch_add(object.stamp.length, Ordering::Relaxed); + } + } + object.validate_pin(&pinned)?; + self.linked_bytes + .fetch_add(object.stamp.length, Ordering::Relaxed); + } + self.record_directories(parent, closure_root); + Ok(target) + } + + fn verify_and_sync_target(&self, id: &ObjectId, path: &Path) -> ImageResult<()> { + // A pre-existing independent copy may have been written without a durability barrier. + // Verify and sync the very same open file, rather than hashing one path then flushing a + // replacement. Windows requires write access for FlushFileBuffers. + #[cfg(unix)] + let file = File::open(path)?; + #[cfg(windows)] + let file = OpenOptions::new().read(true).write(true).open(path)?; + let stamp = ObjectStamp::read(&file)?; + let mut hasher = Sha256::new(); + let mut buffer = vec![0; 1024 * 1024]; + let mut offset = 0; + loop { + let count = read_object_at(&file, &mut buffer, offset)?; + if count == 0 { + break; + } + hasher.update(&buffer[..count]); + offset += count as u64; + } + self.hashed_bytes.fetch_add(offset, Ordering::Relaxed); + let actual = format!("sha256:{}", hex::encode(hasher.finalize())); + if actual != id.as_str() { + return Err(ImageError::DigestMismatch { + digest: id.to_string(), + expected: id.to_string(), + actual, + }); + } + file.sync_all()?; + if ObjectStamp::read(&file)? != stamp || ObjectStamp::read(&File::open(path)?)? != stamp { + return Err(std::io::Error::other("closure object changed during verification").into()); + } + Ok(()) + } + + /// Make every new directory entry durable before its caller publishes a root descriptor. + /// Call only after all batch writers have joined. A failed sync leaves the set available for retry. + pub fn finish(&self) -> ImageResult { + let mut directories = self.directories.lock().unwrap(); + let mut ordered = directories.iter().collect::>(); + ordered.sort_by_key(|path| std::cmp::Reverse(path.components().count())); + for path in ordered { + #[cfg(unix)] + { + File::open(path)?.sync_all()?; + self.directory_syncs.fetch_add(1, Ordering::Relaxed); + } + #[cfg(not(unix))] + let _ = path; + } + directories.clear(); + Ok(self.stats()) + } + + /// Retain exactly the next generation's referenced receipts without keeping object FDs open. + pub fn retained_objects(&self, ids: &[ObjectId]) -> ImageResult> { + ids.iter().map(|id| self.admit(id)).collect() + } + + /// Read counters without adding timing or content-verification work. + pub fn stats(&self) -> CaptureObjectBatchStats { + CaptureObjectBatchStats { + hashed_bytes: self.hashed_bytes.load(Ordering::Relaxed), + linked_bytes: self.linked_bytes.load(Ordering::Relaxed), + copied_bytes: self.copied_bytes.load(Ordering::Relaxed), + directory_syncs: self.directory_syncs.load(Ordering::Relaxed), + } + } + + fn admit(&self, id: &ObjectId) -> ImageResult { + if let Some(object) = self.admitted.lock().unwrap().get(id).cloned() { + object.validate()?; + return Ok(object); + } + let object = self.open_admitted(id.clone(), self.store.object_path(id))?; + let pinned = object.pin()?; + let mut hasher = Sha256::new(); + let mut buffer = vec![0; 1024 * 1024]; + let mut offset = 0; + loop { + let count = read_object_at(&pinned, &mut buffer, offset)?; + if count == 0 { + break; + } + hasher.update(&buffer[..count]); + offset += count as u64; + } + self.hashed_bytes.fetch_add(offset, Ordering::Relaxed); + let actual = format!("sha256:{}", hex::encode(hasher.finalize())); + if actual != id.as_str() { + return Err(ImageError::DigestMismatch { + digest: id.to_string(), + expected: id.to_string(), + actual, + }); + } + object.validate_pin(&pinned)?; + self.admitted + .lock() + .unwrap() + .insert(id.clone(), object.clone()); + Ok(object) + } + + fn open_admitted(&self, id: ObjectId, path: PathBuf) -> ImageResult { + let file = File::open(&path)?; + let stamp = ObjectStamp::read(&file)?; + Ok(AdmittedObject { + id, + path, + stamp, + ownership: Arc::clone(&self.store.ownership), + }) + } + + fn record_directories(&self, path: &Path, stop: &Path) { + let mut directories = self.directories.lock().unwrap(); + for directory in path.ancestors() { + directories.insert(directory.to_path_buf()); + if directory == stop { + break; + } + } + } +} + +impl AdmittedObject { + fn validate(&self) -> ImageResult<()> { + self.pin().map(|_| ()) + } + + fn pin(&self) -> ImageResult { + let file = File::open(&self.path)?; + self.validate_pin(&file)?; + Ok(file) + } + + fn validate_pin(&self, file: &File) -> ImageResult<()> { + if ObjectStamp::read(file)? != self.stamp { + return Err(std::io::Error::other("admitted immutable object was modified").into()); + } + Ok(()) + } +} + +impl ObjectStamp { + fn read(file: &File) -> std::io::Result { + let metadata = file.metadata()?; + if !metadata.is_file() { + return Err(std::io::Error::other( + "immutable object is not a regular file", + )); + } + #[cfg(unix)] + let identity = { + use std::os::unix::fs::MetadataExt; + (metadata.dev(), metadata.ino()) + }; + #[cfg(windows)] + let identity = { + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::Storage::FileSystem::{ + BY_HANDLE_FILE_INFORMATION, GetFileInformationByHandle, + }; + let mut info = std::mem::MaybeUninit::::uninit(); + // SAFETY: the file owns a valid handle and the API initializes the output on success. + if unsafe { GetFileInformationByHandle(file.as_raw_handle(), info.as_mut_ptr()) } == 0 { + return Err(std::io::Error::last_os_error()); + } + let info = unsafe { info.assume_init() }; + ( + u64::from(info.dwVolumeSerialNumber), + (u64::from(info.nFileIndexHigh) << 32) | u64::from(info.nFileIndexLow), + ) + }; + Ok(Self { + identity, + length: metadata.len(), + modified: metadata.modified()?, + }) + } +} + impl MerkleAccumulator { fn new(height: u32) -> Self { Self { @@ -259,6 +612,73 @@ impl From for String { // Functions: Helpers //-------------------------------------------------------------------------------------------------- +fn publish_object_file( + temporary: &Path, + path: &Path, + ownership: &StoreOwnership, +) -> std::io::Result { + // Atomic no-replace publication keeps admitted inode bindings stable for concurrent writers. + match std::fs::hard_link(temporary, path) { + Ok(()) => Ok(true), + Err(_) if path.exists() => Ok(false), + Err(_) => { + // Some filesystems do not support hardlinks. All writers in this runtime's store + // share the fallback namespace lock; it covers only the final check and rename. + let _publication = ownership.publication.lock().unwrap(); + if path.exists() { + return Ok(false); + } + std::fs::rename(temporary, path)?; + Ok(true) + } + } +} + +fn read_object_at(file: &File, bytes: &mut [u8], offset: u64) -> std::io::Result { + #[cfg(unix)] + { + use std::os::unix::fs::FileExt; + file.read_at(bytes, offset) + } + #[cfg(windows)] + { + use std::os::windows::fs::FileExt; + file.seek_read(bytes, offset) + } +} + +fn copy_admitted_object(object: &AdmittedObject, source: &File, target: &Path) -> ImageResult<()> { + let mut destination = OpenOptions::new() + .write(true) + .create_new(true) + .open(target)?; + let result = (|| -> ImageResult<()> { + let mut buffer = vec![0; 1024 * 1024]; + let mut offset = 0; + while offset < object.stamp.length { + let length = buffer.len().min((object.stamp.length - offset) as usize); + let count = read_object_at(source, &mut buffer[..length], offset)?; + if count == 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "immutable object was truncated", + ) + .into()); + } + destination.write_all(&buffer[..count])?; + offset += count as u64; + } + object.validate_pin(source)?; + destination.sync_all()?; + Ok(()) + })(); + drop(destination); + if result.is_err() { + let _ = std::fs::remove_file(target); + } + result +} + fn sync_directories_through(path: &Path, stop: &Path) -> ImageResult<()> { #[cfg(unix)] { @@ -278,6 +698,7 @@ fn sync_directories_through(path: &Path, stop: &Path) -> ImageResult<()> { /// Compute a sparse-aware fixed-leaf Merkle root without reading unallocated holes. pub fn sparse_file_integrity(path: &Path) -> ImageResult { + let started = std::time::Instant::now(); let mut file = File::open(path)?; let logical_size = file.metadata()?.len(); let logical_leaves = logical_size.div_ceil(FILE_MERKLE_LEAF_SIZE as u64).max(1); @@ -289,6 +710,8 @@ pub fn sparse_file_integrity(path: &Path) -> ImageResult { let mut accumulator = MerkleAccumulator::new(tree_height); let mut cursor = 0u64; let mut buffer = vec![0u8; FILE_MERKLE_LEAF_SIZE]; + let mut read_bytes = 0u64; + let mut read_leaves = 0u64; for (start, end) in ranges { push_zero_range(&mut accumulator, &zero_roots, cursor, start); @@ -300,6 +723,8 @@ pub fn sparse_file_integrity(path: &Path) -> ImageResult { buffer.fill(0); file.seek(SeekFrom::Start(offset))?; file.read_exact(&mut buffer[..readable])?; + read_bytes += readable as u64; + read_leaves += 1; accumulator.push_subtree(0, hash_leaf(&buffer)); } cursor = end; @@ -312,6 +737,7 @@ pub fn sparse_file_integrity(path: &Path) -> ImageResult { root.update(&(FILE_MERKLE_LEAF_SIZE as u32).to_le_bytes()); root.update(&tree_height.to_le_bytes()); root.update(&accumulator.finish(tree_height)); + tracing::info!(target: "microsandbox_checkpoint_timing", operation = "disk_hash", logical_bytes = logical_size, read_bytes, read_leaves, hash_us = started.elapsed().as_micros(), "sealed disk integrity timing"); Ok(SparseFileIntegrity { root: format!("blake3:{}", root.finalize().to_hex()), logical_size, @@ -399,6 +825,337 @@ fn hash_parent(left: &[u8; 32], right: &[u8; 32]) -> [u8; 32] { mod tests { use super::*; + #[test] + fn capture_batch_reuses_owned_objects_and_syncs_each_directory_once() { + let directory = tempfile::tempdir().unwrap(); + let store = LocalObjectStore::open(directory.path().join("store")).unwrap(); + let first = CaptureObjectBatch::new(store.clone(), &[]); + let id = first.put_bytes(b"captured immutable RAM").unwrap(); + first + .link_into(&id, &directory.path().join("first")) + .unwrap(); + let stats = first.finish().unwrap(); + assert_eq!( + stats.hashed_bytes, 22, + "new objects must not be rehashed when linked" + ); + let retained = first.retained_objects(std::slice::from_ref(&id)).unwrap(); + let second = CaptureObjectBatch::new(store, &retained); + second + .link_into(&id, &directory.path().join("second")) + .unwrap(); + second + .link_into(&id, &directory.path().join("second")) + .unwrap(); + let stats = second.finish().unwrap(); + assert_eq!(stats.hashed_bytes, 0); + assert_eq!(stats.linked_bytes, 22); + #[cfg(unix)] + assert_eq!( + stats.directory_syncs, 4, + "prefix, algorithm, objects and closure directories" + ); + assert_eq!( + second.finish().unwrap().directory_syncs, + stats.directory_syncs + ); + } + + #[test] + fn capture_batch_checks_unadmitted_data_and_pinned_copy_keeps_exact_inode() { + let directory = tempfile::tempdir().unwrap(); + let store = LocalObjectStore::open(directory.path().join("store")).unwrap(); + let id = store.put_bytes(b"original").unwrap(); + let batch = CaptureObjectBatch::new(store.clone(), &[]); + batch + .link_into(&id, &directory.path().join("first")) + .unwrap(); + assert_eq!(batch.stats().hashed_bytes, 8); + let admitted = batch + .retained_objects(std::slice::from_ref(&id)) + .unwrap() + .remove(0); + let pinned = admitted.pin().unwrap(); + // An active operation's pin is sufficient for a copy even if its original entry is + // unlinked. A later operation must refuse: the runtime no longer owns that name. + std::fs::remove_file(store.object_path(&id)).unwrap(); + let target = directory.path().join("pinned-copy"); + copy_admitted_object(&admitted, &pinned, &target).unwrap(); + assert_eq!(std::fs::read(target).unwrap(), b"original"); + assert!( + batch + .link_into(&id, &directory.path().join("second")) + .is_err() + ); + + let corrupt_id = store.put_bytes(b"must be checked").unwrap(); + std::fs::write(store.object_path(&corrupt_id), b"corrupted").unwrap(); + let fresh = CaptureObjectBatch::new(store, &[]); + assert!( + fresh + .link_into(&corrupt_id, &directory.path().join("bad")) + .is_err() + ); + } + + #[test] + fn capture_batch_rejects_replaced_or_modified_admitted_inodes() { + let directory = tempfile::tempdir().unwrap(); + let store = LocalObjectStore::open(directory.path().join("store")).unwrap(); + let batch = CaptureObjectBatch::new(store.clone(), &[]); + let id = batch.put_bytes(b"original").unwrap(); + let path = store.object_path(&id); + std::fs::remove_file(&path).unwrap(); + std::fs::write(&path, b"replaced").unwrap(); + assert!( + batch + .link_into(&id, &directory.path().join("replaced")) + .is_err() + ); + + let other = batch.put_bytes(b"another object").unwrap(); + // Length change is portable and reliably visible even on coarse timestamp filesystems. + std::fs::write(store.object_path(&other), b"short").unwrap(); + assert!( + batch + .link_into(&other, &directory.path().join("mutated")) + .is_err() + ); + assert!(batch.put_bytes(b"another object").is_err()); + } + + #[test] + fn capture_batch_directory_failure_does_not_report_durable_completion() { + let directory = tempfile::tempdir().unwrap(); + let store = LocalObjectStore::open(directory.path().join("store")).unwrap(); + let batch = CaptureObjectBatch::new(store.clone(), &[]); + let id = batch + .put_bytes(b"durable data pending publication") + .unwrap(); + let closure = directory.path().join("closure"); + batch.link_into(&id, &closure).unwrap(); + #[cfg(unix)] + { + std::fs::remove_dir_all(&closure).unwrap(); + assert!(batch.finish().is_err()); + } + assert!(!closure.join("checkpoint.json").exists()); + assert!(store.object_path(&id).is_file()); + } + + #[test] + fn existing_independent_closure_copies_are_verified_before_reuse() { + let directory = tempfile::tempdir().unwrap(); + let store = LocalObjectStore::open(directory.path().join("store")).unwrap(); + let batch = CaptureObjectBatch::new(store.clone(), &[]); + let id = batch.put_bytes(b"original").unwrap(); + let closure = directory.path().join("closure"); + let target = LocalObjectStore::open(&closure).unwrap().object_path(&id); + std::fs::create_dir_all(target.parent().unwrap()).unwrap(); + // A separate inode, initially written without sync_all, must not inherit source admission. + std::fs::write(&target, b"original").unwrap(); + batch.link_into(&id, &closure).unwrap(); + assert_eq!(batch.stats().hashed_bytes, 16); + batch.finish().unwrap(); + std::fs::write(&target, b"modified").unwrap(); + assert!(batch.link_into(&id, &closure).is_err()); + assert_eq!( + std::fs::read(&target).unwrap(), + b"modified", + "never delete a pre-existing target on verification failure" + ); + assert_eq!(std::fs::read(store.object_path(&id)).unwrap(), b"original"); + } + + #[test] + fn concurrent_checked_and_batched_writers_keep_the_winning_inode() { + let directory = tempfile::tempdir().unwrap(); + let store = LocalObjectStore::open(directory.path().join("store")).unwrap(); + let batch = Arc::new(CaptureObjectBatch::new(store.clone(), &[])); + let gate = std::sync::Barrier::new(8); + let payload = vec![7; 65536]; + std::thread::scope(|scope| { + let handles = (0..8) + .map(|index| { + let gate = &gate; + let payload = &payload; + let batch = &batch; + let store = &store; + scope.spawn(move || { + gate.wait(); + if index % 2 == 0 { + store.put_bytes(payload) + } else { + batch.put_bytes(payload) + } + .unwrap() + }) + }) + .collect::>(); + let expected = ObjectId::from_bytes(&payload).unwrap(); + for handle in handles { + assert_eq!(handle.join().unwrap(), expected); + } + }); + let id = ObjectId::from_bytes(&payload).unwrap(); + batch + .link_into(&id, &directory.path().join("closure")) + .unwrap(); + batch.finish().unwrap(); + let stamp = ObjectStamp::read(&File::open(store.object_path(&id)).unwrap()).unwrap(); + assert_eq!(batch.retained_objects(&[id]).unwrap()[0].stamp, stamp); + } + + #[test] + #[ignore = "opt-in old/new object-store experiment; prints measured times, not a CI latency threshold"] + fn capture_store_full_incremental_experiment() { + const COUNT: usize = 16; + const SIZE: usize = 1024 * 1024; + const DELTA: usize = 4096; + let directory = tempfile::tempdir().unwrap(); + let payloads = (0..COUNT) + .map(|index| vec![index as u8 + 1; SIZE]) + .collect::>(); + let changed = vec![255; DELTA]; + for round in 0..3 { + let old = + LocalObjectStore::open(directory.path().join(format!("old-{round}"))).unwrap(); + let started = std::time::Instant::now(); + let mut old_ids = Vec::new(); + for bytes in &payloads { + let id = old.put_bytes(bytes).unwrap(); + old.link_into(&id, &directory.path().join(format!("old-full-{round}"))) + .unwrap(); + old_ids.push(id); + } + let old_full_us = started.elapsed().as_micros(); + let started = std::time::Instant::now(); + old_ids.push(old.put_bytes(&changed).unwrap()); + for id in &old_ids { + old.link_into(id, &directory.path().join(format!("old-delta-{round}"))) + .unwrap(); + } + let old_delta_us = started.elapsed().as_micros(); + + let new = + LocalObjectStore::open(directory.path().join(format!("new-{round}"))).unwrap(); + let full = CaptureObjectBatch::new(new.clone(), &[]); + let started = std::time::Instant::now(); + let mut new_ids = Vec::new(); + for bytes in &payloads { + let id = full.put_bytes(bytes).unwrap(); + full.link_into(&id, &directory.path().join(format!("new-full-{round}"))) + .unwrap(); + new_ids.push(id); + } + let full_stats = full.finish().unwrap(); + let receipts = full.retained_objects(&new_ids).unwrap(); + let new_full_us = started.elapsed().as_micros(); + let delta = CaptureObjectBatch::new(new, &receipts); + let started = std::time::Instant::now(); + new_ids.push(delta.put_bytes(&changed).unwrap()); + for id in &new_ids { + delta + .link_into(id, &directory.path().join(format!("new-delta-{round}"))) + .unwrap(); + } + let delta_stats = delta.finish().unwrap(); + let new_delta_us = started.elapsed().as_micros(); + assert_eq!( + old_ids, new_ids, + "the optimized publication preserves content identities" + ); + assert_eq!(full_stats.hashed_bytes, (COUNT * SIZE) as u64); + assert_eq!( + delta_stats.hashed_bytes, DELTA as u64, + "inherited payload must not be read again" + ); + #[cfg(unix)] + { + assert!(full_stats.directory_syncs <= (2 * (COUNT + 3)) as u64); + assert!(delta_stats.directory_syncs <= (COUNT + 8) as u64); + } + // Old byte/sync counts follow the unchanged checked public path's exact loop; new + // counts come from runtime counters. Timings are measured; no speed ratio is asserted. + println!( + "{}", + serde_json::json!({ + "experiment": "capture_object_store", "round": round, + "baseline_bytes": COUNT * SIZE, "changed_bytes": DELTA, + "old_full_us": old_full_us, "new_full_us": new_full_us, + "old_incremental_us": old_delta_us, "new_incremental_us": new_delta_us, + "old_full_expected_hashed_bytes": 2 * COUNT * SIZE, + "old_incremental_expected_hashed_bytes": COUNT * SIZE + 2 * DELTA, + "new_full_hashed_bytes": full_stats.hashed_bytes, + "new_incremental_hashed_bytes": delta_stats.hashed_bytes, + "new_full_directory_syncs": full_stats.directory_syncs, + "new_incremental_directory_syncs": delta_stats.directory_syncs + }) + ); + } + } + + #[test] + fn admission_receipts_do_not_escape_their_store_lifetime() { + let directory = tempfile::tempdir().unwrap(); + let store = LocalObjectStore::open(directory.path()).unwrap(); + let first = CaptureObjectBatch::new(store.clone(), &[]); + let id = first.put_bytes(b"payload").unwrap(); + let receipts = first.retained_objects(std::slice::from_ref(&id)).unwrap(); + first.finish().unwrap(); + // Opening the same path does not confer the old runtime's ownership. Re-admit bytes. + let reopened = + CaptureObjectBatch::new(LocalObjectStore::open(directory.path()).unwrap(), &receipts); + reopened + .retained_objects(std::slice::from_ref(&id)) + .unwrap(); + assert_eq!(reopened.stats().hashed_bytes, 7); + } + + #[cfg(unix)] + #[test] + fn retained_receipts_fit_low_fd_budget() { + const CHILD: &str = "MSB_STORE_LOW_FD_TEST_CHILD"; + if std::env::var_os(CHILD).is_none() { + let result = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "checkpoint::store::tests::retained_receipts_fit_low_fd_budget", + "--nocapture", + ]) + .env(CHILD, "1") + .status() + .unwrap(); + assert!(result.success()); + return; + } + // Set a low limit only in this isolated test process, never in the parallel test runner. + let mut limit = std::mem::MaybeUninit::::uninit(); + assert_eq!( + unsafe { libc::getrlimit(libc::RLIMIT_NOFILE, limit.as_mut_ptr()) }, + 0 + ); + let mut limit = unsafe { limit.assume_init() }; + limit.rlim_cur = limit.rlim_cur.min(64); + assert_eq!(unsafe { libc::setrlimit(libc::RLIMIT_NOFILE, &limit) }, 0); + let directory = tempfile::tempdir().unwrap(); + let store = LocalObjectStore::open(directory.path().join("store")).unwrap(); + let first = CaptureObjectBatch::new(store.clone(), &[]); + let ids = (0_u32..512) + .map(|index| first.put_bytes(&index.to_le_bytes()).unwrap()) + .collect::>(); + first.finish().unwrap(); + let receipts = first.retained_objects(&ids).unwrap(); + drop(first); + let second = CaptureObjectBatch::new(store, &receipts); + for id in &ids { + second + .link_into(id, &directory.path().join("closure")) + .unwrap(); + } + assert_eq!(second.finish().unwrap().hashed_bytes, 0); + } + #[test] fn identical_objects_are_reused_and_linked_into_a_closure() { let directory = tempfile::tempdir().unwrap(); diff --git a/crates/runtime/lib/checkpoint/capture_pipeline.rs b/crates/runtime/lib/checkpoint/capture_pipeline.rs new file mode 100644 index 000000000..5df9700a5 --- /dev/null +++ b/crates/runtime/lib/checkpoint/capture_pipeline.rs @@ -0,0 +1,428 @@ +//! Bounded ownership transfer from the paused RAM reader to immutable-object writers. + +use std::io; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, mpsc}; +use std::thread::JoinHandle; +use std::time::Instant; + +use microsandbox_image::checkpoint::{ + CaptureObjectBatch, ContentRef, MemoryExtent, MemoryExtentContent, ObjectId, +}; +use msb_krun::{GuestMemoryRange, MemoryCaptureSink}; + +//-------------------------------------------------------------------------------------------------- +// Constants +//-------------------------------------------------------------------------------------------------- + +pub(super) const MEMORY_OBJECT_PACK_SIZE: usize = 32 * 1024 * 1024; +const WRITERS: usize = 2; +const BUFFER_COUNT: usize = WRITERS + 1; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +pub(super) struct MemoryObjectSink { + sender: Option>, + completed: mpsc::Receiver, + workers: Vec>, + cancelled: Arc, + pending: Pack, + free: Vec>, + updates: Vec, + in_flight: usize, + stats: MemoryPipelineStats, +} + +type PackWriter = dyn Fn(&[u8]) -> Result + Send + Sync; + +#[derive(Default)] +struct Pack { + bytes: Vec, + extents: Vec<(u64, u64, u64)>, +} + +struct CompletedPack { + pack: Pack, + object: Result, + persist_us: u128, +} + +#[derive(Clone, Copy, Debug, Default)] +pub(super) struct MemoryPipelineStats { + pub(super) wait_us: u128, + pub(super) persist_us: u128, + pub(super) packs: u64, + pub(super) peak_in_flight_bytes: usize, +} + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl MemoryObjectSink { + pub(super) fn new(batch: Arc) -> io::Result { + Self::with_writer(Arc::new(move |bytes| { + batch.put_bytes(bytes).map_err(|error| error.to_string()) + })) + } + + fn with_writer(write: Arc) -> io::Result { + let (sender, receiver) = mpsc::sync_channel::(WRITERS); + let receiver = Arc::new(Mutex::new(receiver)); + let (completed_sender, completed) = mpsc::channel(); + let cancelled = Arc::new(AtomicBool::new(false)); + let mut sink = Self { + sender: Some(sender), + completed, + workers: Vec::with_capacity(WRITERS), + cancelled, + pending: Pack { + bytes: Vec::with_capacity(MEMORY_OBJECT_PACK_SIZE), + extents: Vec::new(), + }, + free: (1..BUFFER_COUNT) + .map(|_| Vec::with_capacity(MEMORY_OBJECT_PACK_SIZE)) + .collect(), + updates: Vec::new(), + in_flight: 0, + stats: MemoryPipelineStats::default(), + }; + for index in 0..WRITERS { + let receiver = Arc::clone(&receiver); + let completed_sender = completed_sender.clone(); + let cancelled = Arc::clone(&sink.cancelled); + let write = Arc::clone(&write); + let worker = std::thread::Builder::new() + .name(format!("capture-pack-{index}")) + .spawn(move || { + loop { + // The queue mutex protects receive only; never hold it during hashing or I/O. + let Ok(pack) = receiver.lock().unwrap_or_else(|e| e.into_inner()).recv() + else { + break; + }; + let started = Instant::now(); + let object = if cancelled.load(Ordering::Acquire) { + Err("memory capture cancelled".to_string()) + } else { + // Always return a buffer/completion even on a panicking storage worker, + // so the producer cannot wait forever for an in-flight pack. + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + write(&pack.bytes) + })) + .map_err(|_| "memory object writer panicked".to_string()) + .and_then(|result| result) + }; + if object.is_err() { + cancelled.store(true, Ordering::Release); + } + if completed_sender + .send(CompletedPack { + pack, + object, + persist_us: started.elapsed().as_micros(), + }) + .is_err() + { + break; + } + } + })?; + sink.workers.push(worker); + } + Ok(sink) + } + + pub(super) fn finish(mut self) -> io::Result<(Vec, MemoryPipelineStats)> { + self.flush_pending()?; + self.sender.take(); + while self.in_flight != 0 { + self.receive()?; + } + self.join()?; + Ok((std::mem::take(&mut self.updates), self.stats)) + } + + fn flush_pending(&mut self) -> io::Result<()> { + if self.pending.bytes.is_empty() { + return Ok(()); + } + if self.cancelled.load(Ordering::Acquire) { + return Err(io::Error::other("memory object writer failed")); + } + // No borrowed guest-memory slice leaves write_bytes. At most three owned packs exist, + // including this producer's pack; a slow disk applies backpressure instead of allocating. + let pack = std::mem::take(&mut self.pending); + let started = Instant::now(); + self.sender + .as_ref() + .expect("capture is open") + .send(pack) + .map_err(|_| io::Error::other("memory object writers disconnected"))?; + self.stats.wait_us += started.elapsed().as_micros(); + self.in_flight += 1; + self.stats.packs += 1; + self.stats.peak_in_flight_bytes = self + .stats + .peak_in_flight_bytes + .max(self.in_flight * MEMORY_OBJECT_PACK_SIZE); + while self.free.is_empty() { + self.receive()?; + } + self.pending.bytes = self.free.pop().expect("received reusable buffer"); + Ok(()) + } + + fn receive(&mut self) -> io::Result<()> { + let started = Instant::now(); + let completed = self + .completed + .recv() + .map_err(|_| io::Error::other("memory object writers disconnected"))?; + self.stats.wait_us += started.elapsed().as_micros(); + self.in_flight -= 1; + self.stats.persist_us += completed.persist_us; + let object = completed.object.map_err(io::Error::other)?; + let mut pack = completed.pack; + self.updates.extend( + pack.extents + .drain(..) + .map(|(start, length, object_offset)| MemoryExtent { + start, + length, + content: MemoryExtentContent::Object(ContentRef { + object: object.clone(), + object_offset, + }), + }), + ); + pack.bytes.clear(); + self.free.push(pack.bytes); + Ok(()) + } + + fn join(&mut self) -> io::Result<()> { + let mut failed = false; + for worker in self.workers.drain(..) { + failed |= worker.join().is_err(); + } + if failed { + return Err(io::Error::other("memory object writer panicked")); + } + Ok(()) + } +} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +impl Drop for MemoryObjectSink { + fn drop(&mut self) { + self.cancelled.store(true, Ordering::Release); + self.sender.take(); + // Finish/drop cannot let a writer recreate staging files after failure cleanup starts. + let _ = self.join(); + } +} + +impl MemoryCaptureSink for MemoryObjectSink { + fn write_bytes(&mut self, range: GuestMemoryRange, bytes: &[u8]) -> io::Result<()> { + if bytes.len() as u64 != range.length() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "memory sink range length does not match bytes", + )); + } + // libkrun currently supplies <=2MiB ranges. Splitting also keeps the bound valid if a + // future caller supplies a larger range, without changing that range's guest projection. + let mut consumed = 0; + while consumed < bytes.len() { + if self.pending.bytes.len() == MEMORY_OBJECT_PACK_SIZE { + self.flush_pending()?; + } + let count = + (MEMORY_OBJECT_PACK_SIZE - self.pending.bytes.len()).min(bytes.len() - consumed); + let offset = self.pending.bytes.len() as u64; + self.pending + .bytes + .extend_from_slice(&bytes[consumed..consumed + count]); + self.pending + .extents + .push((range.start() + consumed as u64, count as u64, offset)); + consumed += count; + } + Ok(()) + } + + fn write_zero(&mut self, range: GuestMemoryRange) -> io::Result<()> { + self.updates.push(MemoryExtent { + start: range.start(), + length: range.length(), + content: MemoryExtentContent::Zero, + }); + Ok(()) + } +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use microsandbox_image::checkpoint::LocalObjectStore; + + #[test] + fn sparse_ranges_keep_exact_object_offsets() { + let dir = tempfile::tempdir().unwrap(); + let store = LocalObjectStore::open(dir.path()).unwrap(); + let batch = Arc::new(CaptureObjectBatch::new(store.clone(), &[])); + let mut sink = MemoryObjectSink::new(Arc::clone(&batch)).unwrap(); + sink.write_bytes(GuestMemoryRange::new(4096, 3).unwrap(), b"abc") + .unwrap(); + sink.write_zero(GuestMemoryRange::new(8192, 4).unwrap()) + .unwrap(); + sink.write_bytes(GuestMemoryRange::new(12288, 2).unwrap(), b"de") + .unwrap(); + let (mut extents, stats) = sink.finish().unwrap(); + batch.finish().unwrap(); + extents.sort_by_key(|extent| extent.start); + assert_eq!(stats.packs, 1); + assert!(matches!(extents[1].content, MemoryExtentContent::Zero)); + let MemoryExtentContent::Object(first) = &extents[0].content else { + panic!() + }; + let MemoryExtentContent::Object(last) = &extents[2].content else { + panic!() + }; + assert_eq!(first.object, last.object); + assert_eq!(last.object_offset, 3); + assert_eq!( + std::fs::read(store.object_path(&first.object)).unwrap(), + b"abcde" + ); + } + + #[test] + fn oversized_input_remains_bounded_and_storage_failure_joins_workers() { + let dir = tempfile::tempdir().unwrap(); + let store = LocalObjectStore::open(dir.path()).unwrap(); + let batch = Arc::new(CaptureObjectBatch::new(store, &[])); + let mut sink = MemoryObjectSink::new(Arc::clone(&batch)).unwrap(); + let bytes = vec![7; MEMORY_OBJECT_PACK_SIZE * 4 + 1]; + sink.write_bytes( + GuestMemoryRange::new(0, bytes.len() as u64).unwrap(), + &bytes, + ) + .unwrap(); + let (extents, stats) = sink.finish().unwrap(); + assert_eq!(stats.packs, 5); + assert_eq!( + extents.iter().map(|extent| extent.length).sum::(), + bytes.len() as u64 + ); + assert!(stats.peak_in_flight_bytes <= BUFFER_COUNT * MEMORY_OBJECT_PACK_SIZE); + + let bad = LocalObjectStore::open(dir.path().join("bad")).unwrap(); + std::fs::remove_dir_all(dir.path().join("bad/objects")).unwrap(); + std::fs::write(dir.path().join("bad/objects"), b"not a directory").unwrap(); + let batch = Arc::new(CaptureObjectBatch::new(bad, &[])); + let mut sink = MemoryObjectSink::new(Arc::clone(&batch)).unwrap(); + sink.write_bytes(GuestMemoryRange::new(0, 1).unwrap(), b"x") + .unwrap(); + assert!(sink.finish().is_err()); + assert_eq!( + Arc::strong_count(&batch), + 1, + "all writer references must be joined" + ); + } + + #[test] + fn panicking_writer_returns_an_error_instead_of_stranding_a_pack() { + let mut sink = + MemoryObjectSink::with_writer(Arc::new(|_| panic!("injected pack writer panic"))) + .unwrap(); + sink.write_bytes(GuestMemoryRange::new(0, 1).unwrap(), b"x") + .unwrap(); + assert!(sink.finish().unwrap_err().to_string().contains("panicked")); + } + + #[test] + fn dropping_capture_waits_until_active_writes_have_finished() { + use std::sync::atomic::AtomicUsize; + let active = Arc::new(AtomicUsize::new(0)); + let barrier = Arc::new(std::sync::Barrier::new(2)); + let (started_sender, started) = mpsc::channel(); + let writer_active = Arc::clone(&active); + let writer_barrier = Arc::clone(&barrier); + let mut sink = MemoryObjectSink::with_writer(Arc::new(move |bytes| { + writer_active.fetch_add(1, Ordering::SeqCst); + started_sender.send(()).unwrap(); + writer_barrier.wait(); + let id = ObjectId::from_bytes(bytes).map_err(|error| error.to_string()); + writer_active.fetch_sub(1, Ordering::SeqCst); + id + })) + .unwrap(); + sink.write_bytes(GuestMemoryRange::new(0, 1).unwrap(), b"x") + .unwrap(); + sink.flush_pending().unwrap(); + started.recv().unwrap(); + let cancelled = Arc::clone(&sink.cancelled); + let dropping = std::thread::spawn(move || drop(sink)); + while !cancelled.load(Ordering::Acquire) { + std::thread::yield_now(); + } + assert_eq!(active.load(Ordering::SeqCst), 1); + barrier.wait(); + dropping.join().unwrap(); + assert_eq!(active.load(Ordering::SeqCst), 0); + } + + #[test] + fn out_of_order_writers_preserve_each_packs_guest_projection() { + let (release_first, wait_first) = mpsc::channel(); + let wait_first = Mutex::new(wait_first); + let mut sink = MemoryObjectSink::with_writer(Arc::new(move |bytes| { + if bytes == b"a" { + wait_first.lock().unwrap().recv().unwrap(); + } + ObjectId::from_bytes(bytes).map_err(|error| error.to_string()) + })) + .unwrap(); + sink.write_bytes(GuestMemoryRange::new(4096, 1).unwrap(), b"a") + .unwrap(); + sink.flush_pending().unwrap(); + sink.write_bytes(GuestMemoryRange::new(8192, 1).unwrap(), b"b") + .unwrap(); + sink.flush_pending().unwrap(); + sink.receive().unwrap(); + assert_eq!( + sink.updates[0].start, 8192, + "the later pack must finish first in this test" + ); + release_first.send(()).unwrap(); + let (extents, _) = sink.finish().unwrap(); + assert_eq!( + extents + .iter() + .map(|extent| extent.start) + .collect::>(), + vec![8192, 4096] + ); + for (extent, bytes) in extents.iter().zip([b"b", b"a"]) { + let MemoryExtentContent::Object(content) = &extent.content else { + panic!() + }; + assert_eq!(content.object, ObjectId::from_bytes(bytes).unwrap()); + assert_eq!(content.object_offset, 0); + } + // The coordinator's overlay_extents sorts and validates these projections before any + // canonical manifest is encoded. Worker completion ordering is never artifact ordering. + } +} diff --git a/crates/runtime/lib/checkpoint/coordinator.rs b/crates/runtime/lib/checkpoint/coordinator.rs index 0311f0c64..6ec556b5f 100644 --- a/crates/runtime/lib/checkpoint/coordinator.rs +++ b/crates/runtime/lib/checkpoint/coordinator.rs @@ -4,13 +4,14 @@ use std::collections::{BTreeMap, BTreeSet}; use std::fmt; use std::io::{self, Read}; use std::path::{Path, PathBuf}; +use std::sync::Arc; use std::time::{Duration, Instant}; use microsandbox_agent_client::AgentClient; use microsandbox_image::checkpoint::{ - CaptureIntent, CheckpointManifest, ContentRef, DeviceStateRef, LocalObjectStore, - MemoryCaptureMode, MemoryExtent, MemoryExtentContent, MemoryManifest, ObjectId, - ResourceDescriptor, ResourceTreatment, + AdmittedObject, CaptureIntent, CaptureObjectBatch, CheckpointManifest, ContentRef, + DeviceStateRef, LocalObjectStore, MemoryCaptureMode, MemoryExtent, MemoryExtentContent, + MemoryManifest, ObjectId, ResourceDescriptor, ResourceTreatment, }; use microsandbox_protocol::bootstrap::GuestBootstrap; use microsandbox_protocol::core::{ @@ -18,11 +19,9 @@ use microsandbox_protocol::core::{ WorkloadThaw, WorkloadThawed, }; use microsandbox_protocol::message::{Message, MessageType}; -use msb_krun::{ - GuestMemoryRange, IncrementalCaptureDecision, MemoryCaptureOptions, MemoryCapturePlan, - MemoryCaptureSink, -}; +use msb_krun::{IncrementalCaptureDecision, MemoryCaptureOptions, MemoryCapturePlan}; +use super::capture_pipeline::{MEMORY_OBJECT_PACK_SIZE, MemoryObjectSink}; use super::disk::RuntimeOwnedRootDisk; use super::local_memory::{LocalMemoryCapture, LocalMemoryPin}; use crate::vm::VmConfig; @@ -40,7 +39,6 @@ pub(super) const TYPE_FS: u32 = 26; // object store. Independently pack non-zero ranges into larger immutable objects to amortize // hashing, fsync, directory publication, and restore-time object opens. const MEMORY_SCAN_CHUNK_SIZE: usize = 2 * 1024 * 1024; -const MEMORY_OBJECT_PACK_SIZE: usize = 32 * 1024 * 1024; const WORKLOAD_CONTROL_TIMEOUT: Duration = Duration::from_secs(10); //-------------------------------------------------------------------------------------------------- @@ -57,6 +55,7 @@ pub(crate) struct CheckpointCoordinator { fs_resource_bindings: BTreeMap>, network_resource_binding: Option, previous_memory: Option, + previous_memory_objects: Vec, memory_cache: Option, cached_baseline: Option<(MemoryManifest, super::CachedMemory)>, local_cache_root: Option, @@ -93,6 +92,7 @@ struct PausedCapture { result: CheckpointResult, memory_plan: MemoryCapturePlan, memory_manifest: Option, + memory_objects: Vec, local_memory: Option, timings: PausedCaptureTimings, } @@ -107,6 +107,14 @@ struct PausedCaptureTimings { extent_overlay_us: u128, memory_manifest_us: u128, checkpoint_publish_us: u128, + pipeline_wait_us: u128, + object_persist_worker_us: u128, + object_packs: u64, + peak_in_flight_bytes: usize, + object_hashed_bytes: u64, + object_linked_bytes: u64, + object_copied_bytes: u64, + object_directory_syncs: u64, } struct FrozenWorkload { @@ -123,19 +131,6 @@ pub(crate) struct UserPause { pub(crate) capture_unavailable: Option, } -struct MemoryObjectSink<'a> { - store: &'a LocalObjectStore, - updates: Vec, - pending_bytes: Vec, - pending_extents: Vec, -} - -struct PendingMemoryExtent { - start: u64, - length: u64, - object_offset: u64, -} - struct PendingDeviceState { device_type: u32, device_id: String, @@ -335,6 +330,7 @@ impl CheckpointCoordinator { fs_resource_bindings, network_resource_binding, previous_memory: None, + previous_memory_objects: Vec::new(), memory_cache: if vm .checkpoint_restore .as_ref() @@ -726,9 +722,11 @@ impl CheckpointCoordinator { } if baseline_published { self.previous_memory = captured.memory_manifest; + self.previous_memory_objects = captured.memory_objects; self.local_baseline = captured.local_memory; } else { self.previous_memory = None; + self.previous_memory_objects.clear(); self.local_baseline = None; } tracing::info!( @@ -753,6 +751,14 @@ impl CheckpointCoordinator { extent_overlay_us = captured.timings.extent_overlay_us, memory_manifest_us = captured.timings.memory_manifest_us, checkpoint_publish_us = captured.timings.checkpoint_publish_us, + pipeline_wait_us = captured.timings.pipeline_wait_us, + object_persist_worker_us = captured.timings.object_persist_worker_us, + object_packs = captured.timings.object_packs, + peak_in_flight_bytes = captured.timings.peak_in_flight_bytes, + object_hashed_bytes = captured.timings.object_hashed_bytes, + object_linked_bytes = captured.timings.object_linked_bytes, + object_copied_bytes = captured.timings.object_copied_bytes, + object_directory_syncs = captured.timings.object_directory_syncs, baseline_publish_us, resume_us, thaw_us, @@ -869,6 +875,14 @@ impl CheckpointCoordinator { local: bool, ) -> Result { let mut timings = PausedCaptureTimings::default(); + let batch = Arc::new(CaptureObjectBatch::new( + self.store.clone(), + if local { + &[] + } else { + &self.previous_memory_objects + }, + )); let devices_started = Instant::now(); let mut pending_devices = Vec::with_capacity(inventory.len()); let mut disk_roots = Vec::new(); @@ -899,11 +913,10 @@ impl CheckpointCoordinator { .manifest .to_canonical_bytes() .map_err(CheckpointFailure::resumable)?; - let manifest_id = self - .store + let manifest_id = batch .put_bytes(&manifest_bytes) .map_err(CheckpointFailure::resumable)?; - self.store + batch .link_into(&manifest_id, staging) .map_err(CheckpointFailure::resumable)?; disk_roots.push(manifest_id); @@ -962,7 +975,7 @@ impl CheckpointCoordinator { }) .collect::, String>>() } else { - persist_device_states(&self.store, staging, &pending_devices) + persist_device_states(&batch, staging, &pending_devices) } .map_err(CheckpointFailure::resumable)?; timings.devices_us = devices_started.elapsed().as_micros(); @@ -985,11 +998,10 @@ impl CheckpointCoordinator { let execution_id = if local { put_local_object(staging, &execution_bytes).map_err(CheckpointFailure::resumable)? } else { - let id = self - .store + let id = batch .put_bytes(&execution_bytes) .map_err(CheckpointFailure::resumable)?; - self.store + batch .link_into(&id, staging) .map_err(CheckpointFailure::resumable)?; id @@ -1072,6 +1084,7 @@ impl CheckpointCoordinator { }, memory_plan, memory_manifest: None, + memory_objects: Vec::new(), local_memory: Some(memory), timings, }); @@ -1081,11 +1094,12 @@ impl CheckpointCoordinator { let (memory_plan, memory_mode, base_extents) = self.plan_memory(vm).map_err(CheckpointFailure::resumable)?; timings.memory_plan_us = memory_plan_started.elapsed().as_micros(); - let mut sink = MemoryObjectSink { - store: &self.store, - updates: Vec::new(), - pending_bytes: Vec::with_capacity(MEMORY_OBJECT_PACK_SIZE), - pending_extents: Vec::new(), + let mut sink = match MemoryObjectSink::new(Arc::clone(&batch)) { + Ok(sink) => sink, + Err(error) => { + let _ = vm.abandon_memory_capture(&memory_plan); + return Err(CheckpointFailure::resumable(error)); + } }; let memory_capture_started = Instant::now(); let stats = match vm.capture_memory( @@ -1096,18 +1110,24 @@ impl CheckpointCoordinator { ) { Ok(stats) => stats, Err(error) => { + // Stop/join queued writers before the caller can remove this capture's staging. + drop(sink); let _ = vm.abandon_memory_capture(&memory_plan); return Err(CheckpointFailure::resumable(error)); } }; - let updates = match sink.finish() { - Ok(updates) => updates, + let (updates, pipeline_stats) = match sink.finish() { + Ok(result) => result, Err(error) => { let _ = vm.abandon_memory_capture(&memory_plan); return Err(CheckpointFailure::resumable(error)); } }; timings.memory_capture_us = memory_capture_started.elapsed().as_micros(); + timings.pipeline_wait_us = pipeline_stats.wait_us; + timings.object_persist_worker_us = pipeline_stats.persist_us; + timings.object_packs = pipeline_stats.packs; + timings.peak_in_flight_bytes = pipeline_stats.peak_in_flight_bytes; let extent_overlay_started = Instant::now(); let extents = match overlay_extents(base_extents, updates) { Ok(extents) => extents, @@ -1143,22 +1163,19 @@ impl CheckpointCoordinator { linked_memory_objects.insert(content.object.clone()); } } - if let Err(error) = parallel_link_objects( - &self.store, - staging, - &linked_memory_objects.into_iter().collect::>(), - ) { + let linked_memory_objects = linked_memory_objects.into_iter().collect::>(); + if let Err(error) = parallel_link_objects(&batch, staging, &linked_memory_objects) { let _ = vm.abandon_memory_capture(&memory_plan); return Err(CheckpointFailure::resumable(error)); } - let memory_id = match self.store.put_bytes(&memory_bytes) { + let memory_id = match batch.put_bytes(&memory_bytes) { Ok(id) => id, Err(error) => { let _ = vm.abandon_memory_capture(&memory_plan); return Err(CheckpointFailure::resumable(error)); } }; - if let Err(error) = self.store.link_into(&memory_id, staging) { + if let Err(error) = batch.link_into(&memory_id, staging) { let _ = vm.abandon_memory_capture(&memory_plan); return Err(CheckpointFailure::resumable(error)); } @@ -1185,17 +1202,32 @@ impl CheckpointCoordinator { return Err(CheckpointFailure::resumable(error)); } }; - let checkpoint_root = match self.store.put_bytes(&checkpoint_bytes) { + let checkpoint_root = match batch.put_bytes(&checkpoint_bytes) { Ok(id) => id, Err(error) => { let _ = vm.abandon_memory_capture(&memory_plan); return Err(CheckpointFailure::resumable(error)); } }; - if let Err(error) = self.store.link_into(&checkpoint_root, staging) { + if let Err(error) = batch.link_into(&checkpoint_root, staging) { let _ = vm.abandon_memory_capture(&memory_plan); return Err(CheckpointFailure::resumable(error)); } + let memory_objects = match batch + .retained_objects(&linked_memory_objects) + .and_then(|objects| batch.finish().map(|_| objects)) + { + Ok(objects) => objects, + Err(error) => { + let _ = vm.abandon_memory_capture(&memory_plan); + return Err(CheckpointFailure::resumable(error)); + } + }; + let object_stats = batch.stats(); + timings.object_hashed_bytes = object_stats.hashed_bytes; + timings.object_linked_bytes = object_stats.linked_bytes; + timings.object_copied_bytes = object_stats.copied_bytes; + timings.object_directory_syncs = object_stats.directory_syncs; if let Err(error) = publish_root_last(staging, final_path, &checkpoint_bytes) { let _ = vm.abandon_memory_capture(&memory_plan); return Err(CheckpointFailure::resumable(error)); @@ -1213,6 +1245,7 @@ impl CheckpointCoordinator { }, memory_plan, memory_manifest: Some(memory_manifest), + memory_objects, local_memory: None, timings, }) @@ -1339,37 +1372,6 @@ impl FrozenWorkload { } } -impl MemoryObjectSink<'_> { - /// Publish the final partial content pack and return its exact guest-address projection. - fn finish(mut self) -> io::Result> { - self.flush_pending()?; - Ok(self.updates) - } - - /// Store up to one bounded chunk containing bytes from multiple sparse guest ranges. - fn flush_pending(&mut self) -> io::Result<()> { - if self.pending_bytes.is_empty() { - return Ok(()); - } - let bytes = std::mem::take(&mut self.pending_bytes); - let object = self - .store - .put_bytes(&bytes) - .map_err(|error| io::Error::other(error.to_string()))?; - self.updates - .extend(self.pending_extents.drain(..).map(|extent| MemoryExtent { - start: extent.start, - length: extent.length, - content: MemoryExtentContent::Object(ContentRef { - object: object.clone(), - object_offset: extent.object_offset, - }), - })); - self.pending_bytes = Vec::with_capacity(MEMORY_OBJECT_PACK_SIZE); - Ok(()) - } -} - //-------------------------------------------------------------------------------------------------- // Trait Implementations //-------------------------------------------------------------------------------------------------- @@ -1382,56 +1384,6 @@ impl fmt::Display for CheckpointFailure { impl std::error::Error for CheckpointFailure {} -impl MemoryCaptureSink for MemoryObjectSink<'_> { - fn write_bytes(&mut self, range: GuestMemoryRange, bytes: &[u8]) -> io::Result<()> { - if bytes.len() as u64 != range.length() { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "memory sink range length does not match bytes", - )); - } - - if !self.pending_bytes.is_empty() - && self.pending_bytes.len().saturating_add(bytes.len()) > MEMORY_OBJECT_PACK_SIZE - { - self.flush_pending()?; - } - if bytes.len() > MEMORY_OBJECT_PACK_SIZE { - let object = self - .store - .put_bytes(bytes) - .map_err(|error| io::Error::other(error.to_string()))?; - self.updates.push(MemoryExtent { - start: range.start(), - length: range.length(), - content: MemoryExtentContent::Object(ContentRef { - object, - object_offset: 0, - }), - }); - return Ok(()); - } - - let object_offset = self.pending_bytes.len() as u64; - self.pending_bytes.extend_from_slice(bytes); - self.pending_extents.push(PendingMemoryExtent { - start: range.start(), - length: range.length(), - object_offset, - }); - Ok(()) - } - - fn write_zero(&mut self, range: GuestMemoryRange) -> io::Result<()> { - self.updates.push(MemoryExtent { - start: range.start(), - length: range.length(), - content: MemoryExtentContent::Zero, - }); - Ok(()) - } -} - //-------------------------------------------------------------------------------------------------- // Functions //-------------------------------------------------------------------------------------------------- @@ -1804,7 +1756,7 @@ fn put_local_object(staging: &Path, bytes: &[u8]) -> Result { } fn persist_device_states( - store: &LocalObjectStore, + store: &CaptureObjectBatch, staging: &Path, pending: &[PendingDeviceState], ) -> Result, String> { @@ -1849,10 +1801,9 @@ fn persist_device_states( }) } -/// Link independent immutable memory objects concurrently. Each object remains fully verified by -/// `LocalObjectStore::link_into`; this only overlaps hashing and filesystem durability waits. +/// Link independent immutable memory objects concurrently, reusing this batch's inode ownership. fn parallel_link_objects( - store: &LocalObjectStore, + store: &CaptureObjectBatch, staging: &Path, objects: &[ObjectId], ) -> Result<(), String> { @@ -1913,11 +1864,13 @@ mod tests { }; use microsandbox_image::checkpoint::{ - ContentRef, LocalObjectStore, MemoryExtent, MemoryExtentContent, ObjectId, + CaptureObjectBatch, ContentRef, LocalObjectStore, MemoryExtent, MemoryExtentContent, + ObjectId, }; use microsandbox_protocol::core::{CoreError, CoreErrorKind, WorkloadFrozen}; use microsandbox_protocol::message::{Message, MessageType}; use msb_krun::{GuestMemoryRange, MemoryCaptureSink}; + use std::sync::Arc; #[test] fn unavailable_freezer_requires_explicit_matching_evidence() { @@ -2176,12 +2129,8 @@ mod tests { fn sparse_memory_ranges_share_one_bounded_content_object() { let temp = tempfile::tempdir().unwrap(); let store = LocalObjectStore::open(temp.path()).unwrap(); - let mut sink = MemoryObjectSink { - store: &store, - updates: Vec::new(), - pending_bytes: Vec::new(), - pending_extents: Vec::new(), - }; + let batch = Arc::new(CaptureObjectBatch::new(store.clone(), &[])); + let mut sink = MemoryObjectSink::new(Arc::clone(&batch)).unwrap(); sink.write_bytes(GuestMemoryRange::new(0x1000, 3).unwrap(), b"abc") .unwrap(); @@ -2189,7 +2138,8 @@ mod tests { .unwrap(); sink.write_bytes(GuestMemoryRange::new(0x3000, 2).unwrap(), b"de") .unwrap(); - let mut extents = sink.finish().unwrap(); + let (mut extents, _) = sink.finish().unwrap(); + batch.finish().unwrap(); extents.sort_by_key(|extent| extent.start); assert_eq!(extents.len(), 3); @@ -2223,7 +2173,9 @@ mod tests { }) .collect::>(); - let persisted = persist_device_states(&store, &staging, &pending).unwrap(); + let batch = CaptureObjectBatch::new(store.clone(), &[]); + let persisted = persist_device_states(&batch, &staging, &pending).unwrap(); + batch.finish().unwrap(); assert_eq!(persisted.len(), pending.len()); for (index, state) in persisted.iter().enumerate() { diff --git a/crates/runtime/lib/checkpoint/disk.rs b/crates/runtime/lib/checkpoint/disk.rs index b951763d6..f7d166eb3 100644 --- a/crates/runtime/lib/checkpoint/disk.rs +++ b/crates/runtime/lib/checkpoint/disk.rs @@ -10,10 +10,12 @@ use std::time::Instant; #[cfg(unix)] use std::fs::File; +use microsandbox_image::checkpoint::{ + CheckpointClosure, DiskGenerationManifest, DiskLayerRef, sparse_file_integrity, +}; use microsandbox_image::checkpoint::{ CompactLayer, DiskCompactionPlan, compact_layer_capacity, materialize_compact_prefix, }; -use microsandbox_image::checkpoint::{DiskGenerationManifest, DiskLayerRef, sparse_file_integrity}; use serde::{Deserialize, Serialize}; use crate::vm::{UpperLayerSpec, UpperSpec, VmConfig}; @@ -194,6 +196,14 @@ impl RuntimeOwnedRootDisk { /// Open the authoritative chain journal or initialize it from a sandbox-owned root disk. pub(crate) fn open(runtime_dir: &Path, vm: &VmConfig) -> Result, String> { + Self::open_with_admitted(runtime_dir, vm, None) + } + + fn open_with_admitted( + runtime_dir: &Path, + vm: &VmConfig, + admitted: Option<&CheckpointClosure>, + ) -> Result, String> { let Some(layout) = configured_layout(vm) else { return Ok(None); }; @@ -225,13 +235,28 @@ impl RuntimeOwnedRootDisk { .collect(), }; let last = state.layers.len() - 1; + let mut reused_layers = 0_u64; + let mut hashed_layers = 0_u64; + let started = Instant::now(); for layer in state.layers.iter_mut().take(last) { - layer.integrity_root = Some( + let reused = admitted + .map(|closure| closure.reused_disk_integrity(&layer.path)) + .transpose() + .map_err(|error| format!("reuse admitted root ancestor: {error}"))? + .flatten(); + layer.integrity_root = Some(if let Some(root) = reused { + reused_layers += 1; + root + } else { + // Copies and relocated qcow headers are different physical artifacts. + // Never reuse their predecessor's root merely because sizes match. + hashed_layers += 1; sparse_file_integrity(&layer.path) .map_err(|error| format!("hash sealed root ancestor: {error}"))? - .root, - ); + .root + }); } + tracing::info!(target: "microsandbox_checkpoint_timing", operation = "root_journal_admission", reused_layers, hashed_layers, total_us = started.elapsed().as_micros(), "root journal admission timing"); write_state(&state_path, &state)?; state }; @@ -623,6 +648,18 @@ impl std::error::Error for RootDiskRolloverError {} // Functions //-------------------------------------------------------------------------------------------------- +/// Seed a new child's journal from disk admission already completed in this runtime process. +/// The existing journal, when present, remains authoritative; transformed or copied files are +/// hashed instead of inheriting an identity belonging to their source representation. +pub(crate) fn seed_restored_root_disk( + runtime_dir: &Path, + vm: &VmConfig, + admitted: &CheckpointClosure, +) -> Result<(), String> { + RuntimeOwnedRootDisk::open_with_admitted(runtime_dir, vm, Some(admitted))?; + Ok(()) +} + /// Apply the durable forward chain before VM construction after a runtime restart. pub(crate) fn recover_runtime_owned_root( runtime_dir: &Path, @@ -1015,6 +1052,291 @@ fn sync_directory(path: &Path) -> std::io::Result<()> { #[cfg(test)] mod tests { + fn admitted_fixture( + root: &std::path::Path, + sources: &[super::UpperLayerSpec], + ) -> microsandbox_image::checkpoint::CheckpointClosure { + use microsandbox_image::checkpoint::{ + CaptureIntent, CheckpointClosure, CheckpointManifest, DiskGenerationManifest, + DiskLayerRef, LocalObjectStore, MemoryCaptureMode, MemoryExtent, MemoryExtentContent, + MemoryManifest, ObjectId, sparse_file_integrity, + }; + + let store = LocalObjectStore::open(root).unwrap(); + std::fs::create_dir(root.join("layers")).unwrap(); + let memory = MemoryManifest { + schema: "microsandbox.memory/1".into(), + architecture: std::env::consts::ARCH.into(), + guest_page_size: 4096, + topology_generation: 1, + generation: 1, + capture_mode: MemoryCaptureMode::Full, + pause_generation: 7, + extents: vec![MemoryExtent { + start: 0, + length: 4096, + content: MemoryExtentContent::Zero, + }], + }; + let layers: Vec<_> = sources + .iter() + .enumerate() + .map(|(index, source)| { + let layer_id = format!("sealed_{index}"); + let format = match source.format { + msb_krun::DiskImageFormat::Raw => "raw", + msb_krun::DiskImageFormat::Qcow2 => "qcow2", + _ => panic!("unsupported fixture format"), + }; + let target = root.join("layers").join(format!("{layer_id}.{format}")); + std::fs::hard_link(&source.path, &target).unwrap(); + DiskLayerRef { + layer_id, + format: format.into(), + virtual_size: 131072, + predecessor: index + .checked_sub(1) + .map(|previous| format!("sealed_{previous}")), + integrity_root: sparse_file_integrity(&target).unwrap().root, + } + }) + .collect(); + let disk = DiskGenerationManifest { + schema: "microsandbox.disk-generation/1".into(), + volume_id: "root".into(), + device_id: "vda".into(), + generation: 1, + head: layers.last().unwrap().layer_id.clone(), + layers, + pause_generation: 7, + }; + // The closure checks opaque execution bytes; only live restore decodes their codec. + let checkpoint = CheckpointManifest { + schema: "microsandbox.checkpoint/1".into(), + checkpoint_id: "journal-fixture".into(), + capture_intent: CaptureIntent::FullSnapshot, + architecture: std::env::consts::ARCH.into(), + pause_generation: 7, + execution_state: store.put_bytes(b"execution fixture").unwrap(), + memory: store + .put_bytes(&memory.to_canonical_bytes().unwrap()) + .unwrap(), + disks: vec![ + store + .put_bytes(&disk.to_canonical_bytes().unwrap()) + .unwrap(), + ], + devices: Vec::new(), + resources: Vec::new(), + requires: Vec::new(), + }; + let bytes = checkpoint.to_canonical_bytes().unwrap(); + let id = ObjectId::from_bytes(&bytes).unwrap(); + std::fs::write(root.join("checkpoint.json"), bytes).unwrap(); + CheckpointClosure::open(root, Some(&id)).unwrap() + } + + fn root_vm( + layout: super::RootDiskLayout, + layers: Vec, + ) -> super::VmConfig { + let spec = super::UpperSpec { + layers, + read_only: false, + }; + let mut vm = super::VmConfig { + libkrunfw_path: Default::default(), + thp: Default::default(), + memory_cache_dir: None, + vcpus: 1, + memory_mib: 256, + max_cpus: 1, + max_memory_mib: 256, + cpu_placement: Default::default(), + placement_profile_name: None, + placement_profile: None, + block_writeback_limit_bytes: None, + rootfs_path: None, + rootfs_follow_root_symlinks: false, + rootfs_disk: None, + rootfs_disk_format: None, + rootfs_disk_readonly: false, + rootfs_disk_spec: None, + rootfs_disk_runtime_owned: false, + rootfs_vmdk: None, + rootfs_upper: None, + rootfs_upper_spec: None, + mounts: Vec::new(), + disks: Vec::new(), + vsock: Vec::new(), + #[cfg(unix)] + backends: Vec::new(), + init_path: None, + bootstrap: Default::default(), + exec_path: None, + exec_args: Vec::new(), + #[cfg(feature = "net")] + network: Default::default(), + #[cfg(feature = "net")] + deployment_profile: Default::default(), + #[cfg(feature = "net")] + sandbox_slot: 1, + checkpoint_restore: None, + }; + match layout { + super::RootDiskLayout::ManagedUpper => { + vm.rootfs_vmdk = Some("fixture.vmdk".into()); + vm.rootfs_upper_spec = Some(spec); + } + super::RootDiskLayout::FlatRoot => { + vm.rootfs_disk_runtime_owned = true; + vm.rootfs_disk_spec = Some(spec); + } + } + vm + } + + #[tokio::test] + async fn admitted_raw_hardlink_seeds_once_and_reopens_without_the_snapshot() { + use super::*; + for layout in [RootDiskLayout::ManagedUpper, RootDiskLayout::FlatRoot] { + let directory = tempfile::tempdir().unwrap(); + let source = directory.path().join("source.raw"); + std::fs::write(&source, vec![17; 131072]).unwrap(); + let admitted = admitted_fixture( + &directory.path().join("snapshot"), + &[UpperLayerSpec { + path: source.clone(), + format: msb_krun::DiskImageFormat::Raw, + }], + ); + let child_base = directory.path().join("child.raw"); + std::fs::hard_link(&source, &child_base).unwrap(); + let expected = admitted.disks()[0].layers[0].integrity_root.clone(); + assert_eq!( + admitted.reused_disk_integrity(&child_base).unwrap(), + Some(expected.clone()) + ); + let runtime = directory.path().join("runtime"); + std::fs::create_dir(&runtime).unwrap(); + let head = runtime.join("head.qcow2"); + microsandbox_image::checkpoint::create_qcow2_overlay(&head, 131072, &child_base, "raw") + .await + .unwrap(); + let vm = root_vm( + layout, + vec![ + UpperLayerSpec { + path: child_base.clone(), + format: msb_krun::DiskImageFormat::Raw, + }, + UpperLayerSpec { + path: head, + format: msb_krun::DiskImageFormat::Qcow2, + }, + ], + ); + seed_restored_root_disk(&runtime, &vm, &admitted).unwrap(); + let journal = runtime.join(ROOT_DISK_STATE_FILE); + let first = std::fs::read(&journal).unwrap(); + let state = read_state(&journal).unwrap(); + assert_eq!(state.layers[0].integrity_root.as_ref(), Some(&expected)); + assert!( + state.layers[1].integrity_root.is_none(), + "writable head must not be sealed" + ); + assert_eq!(state.layout, layout); + seed_restored_root_disk(&runtime, &vm, &admitted).unwrap(); + assert_eq!(std::fs::read(&journal).unwrap(), first); + let snapshot_layer = admitted.disk_layer_path(&admitted.disks()[0].layers[0]); + drop(admitted); + std::fs::remove_file(source).unwrap(); + std::fs::remove_file(snapshot_layer).unwrap(); + RuntimeOwnedRootDisk::open(&runtime, &vm).unwrap().unwrap(); + assert_eq!(std::fs::read(&journal).unwrap(), first); + assert_eq!(std::fs::read(child_base).unwrap(), vec![17; 131072]); + } + } + + #[tokio::test] + async fn copied_raw_and_relocated_qcow_seed_their_own_physical_integrities() { + use super::*; + use microsandbox_image::checkpoint::{create_qcow2_overlay, relocate_qcow2_backing}; + let directory = tempfile::tempdir().unwrap(); + let source = directory.path().join("source.raw"); + let overlay = directory.path().join("source.qcow2"); + std::fs::write(&source, vec![31; 131072]).unwrap(); + create_qcow2_overlay(&overlay, 131072, &source, "raw") + .await + .unwrap(); + let original_overlay = std::fs::read(&overlay).unwrap(); + let admitted = admitted_fixture( + &directory.path().join("snapshot"), + &[ + UpperLayerSpec { + path: source.clone(), + format: msb_krun::DiskImageFormat::Raw, + }, + UpperLayerSpec { + path: overlay.clone(), + format: msb_krun::DiskImageFormat::Qcow2, + }, + ], + ); + let base_copy = directory.path().join("copied-base.raw"); + let overlay_copy = directory.path().join("copied-overlay.qcow2"); + std::fs::copy(&source, &base_copy).unwrap(); + std::fs::copy(&overlay, &overlay_copy).unwrap(); + relocate_qcow2_backing(&overlay_copy, &base_copy).unwrap(); + assert!( + admitted + .reused_disk_integrity(&base_copy) + .unwrap() + .is_none() + ); + assert!( + admitted + .reused_disk_integrity(&overlay_copy) + .unwrap() + .is_none() + ); + let expected_raw = sparse_file_integrity(&base_copy).unwrap().root; + let expected_qcow = sparse_file_integrity(&overlay_copy).unwrap().root; + assert_ne!(expected_qcow, admitted.disks()[0].layers[1].integrity_root); + let runtime = directory.path().join("runtime"); + std::fs::create_dir(&runtime).unwrap(); + let head = runtime.join("head.qcow2"); + create_qcow2_overlay(&head, 131072, &overlay_copy, "qcow2") + .await + .unwrap(); + let vm = root_vm( + RootDiskLayout::FlatRoot, + vec![ + UpperLayerSpec { + path: base_copy, + format: msb_krun::DiskImageFormat::Raw, + }, + UpperLayerSpec { + path: overlay_copy, + format: msb_krun::DiskImageFormat::Qcow2, + }, + UpperLayerSpec { + path: head, + format: msb_krun::DiskImageFormat::Qcow2, + }, + ], + ); + seed_restored_root_disk(&runtime, &vm, &admitted).unwrap(); + let state = read_state(&runtime.join(ROOT_DISK_STATE_FILE)).unwrap(); + assert_eq!(state.layers[0].integrity_root.as_ref(), Some(&expected_raw)); + assert_eq!( + state.layers[1].integrity_root.as_ref(), + Some(&expected_qcow) + ); + assert!(state.layers[2].integrity_root.is_none()); + assert_eq!(std::fs::read(overlay).unwrap(), original_overlay); + } + #[test] fn stopped_growth_preserves_ancestors_and_recovers_pending_target() { use super::*; diff --git a/crates/runtime/lib/checkpoint/memory_cache.rs b/crates/runtime/lib/checkpoint/memory_cache.rs index 15fda774c..7cab99be7 100644 --- a/crates/runtime/lib/checkpoint/memory_cache.rs +++ b/crates/runtime/lib/checkpoint/memory_cache.rs @@ -9,7 +9,11 @@ use std::io::{self, Seek, SeekFrom, Write}; use std::path::{Path, PathBuf}; use std::time::Instant; -use microsandbox_image::checkpoint::{MemoryExtentContent, MemoryManifest, ObjectId}; +use microsandbox_image::checkpoint::{ + CheckpointObjectReadTiming, MemoryExtentContent, MemoryManifest, ObjectId, +}; + +use super::object_pipeline::{ObjectPipelineTiming, consume_verified_objects}; //-------------------------------------------------------------------------------------------------- // Types @@ -49,6 +53,8 @@ pub struct MemoryCache { pub(super) page_size: u64, } +type ObjectSlices = BTreeMap>; + //-------------------------------------------------------------------------------------------------- // Methods //-------------------------------------------------------------------------------------------------- @@ -120,6 +126,32 @@ impl MemoryCache { self.materialize_with_baseline(manifest, identity, None, read_object) } + /// Prepare a durable restore backing with bounded parallel verification and reusable buffers. + /// Readers must enforce the 32 MiB portable memory-object bound and return verified bytes. + pub fn materialize_parallel( + &self, + manifest: &MemoryManifest, + identity: &ObjectId, + read_object: impl Fn(&ObjectId, &mut Vec) -> io::Result + Sync, + ) -> io::Result { + self.materialize_with_baseline_inner(manifest, identity, None, |objects, staging| { + let timings = consume_verified_objects(objects, read_object, |slices, bytes| { + write_object_slices(staging, slices, bytes) + })?; + tracing::info!( + target: "microsandbox_checkpoint_timing", + operation = "memory_cache_objects", + object_io_worker_us = timings.read_us, + object_hash_worker_us = timings.hash_us, + object_write_us = timings.consume_us, + object_pipeline_us = timings.elapsed_us, + object_bytes = timings.object_bytes, + "parallel memory cache object timing" + ); + Ok(timings) + }) + } + /// Reuse a pinned complete baseline before overlaying immutable changed object slices. /// The source VM is never read or remapped here; both inputs are completed captures. pub fn materialize_with_baseline( @@ -128,6 +160,30 @@ impl MemoryCache { identity: &ObjectId, baseline: Option<(&MemoryManifest, &CachedMemory)>, mut read_object: impl FnMut(&ObjectId) -> io::Result>, + ) -> io::Result { + self.materialize_with_baseline_inner(manifest, identity, baseline, |objects, staging| { + let started = Instant::now(); + let mut timings = ObjectPipelineTiming::default(); + for (id, slices) in objects { + let reading = Instant::now(); + let bytes = read_object(&id)?; + timings.read_us += reading.elapsed().as_micros(); + timings.object_bytes += bytes.len() as u64; + let writing = Instant::now(); + write_object_slices(staging, slices, &bytes)?; + timings.consume_us += writing.elapsed().as_micros(); + } + timings.elapsed_us = started.elapsed().as_micros(); + Ok(timings) + }) + } + + fn materialize_with_baseline_inner( + &self, + manifest: &MemoryManifest, + identity: &ObjectId, + baseline: Option<(&MemoryManifest, &CachedMemory)>, + consume_objects: impl FnOnce(ObjectSlices, &mut File) -> io::Result, ) -> io::Result { let started = Instant::now(); let canonical = manifest.to_canonical_bytes().map_err(io::Error::other)?; @@ -240,29 +296,15 @@ impl MemoryCache { } } } - for (id, slices) in objects { - let bytes = read_object(&id)?; - for (target, offset, count) in slices { - let start = usize::try_from(offset) - .map_err(|_| invalid("memory object offset overflows"))?; - let count = usize::try_from(count) - .map_err(|_| invalid("memory object length overflows"))?; - let end = start - .checked_add(count) - .ok_or_else(|| invalid("memory object slice overflows"))?; - let bytes = bytes - .get(start..end) - .ok_or_else(|| invalid("memory object slice exceeds verified bytes"))?; - staging.seek(SeekFrom::Start(target))?; - staging.write_all(bytes)?; - } - } + let objects = consume_objects(objects, &mut staging)?; #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; staging.set_permissions(std::fs::Permissions::from_mode(0o400))?; } + let syncing = Instant::now(); staging.sync_all()?; + let file_sync_us = syncing.elapsed().as_micros(); // Windows readers deliberately deny write sharing. Close the completed writer before // publishing/opening its immutable view; keeping it open would cause a sharing violation. drop(staging); @@ -273,14 +315,27 @@ impl MemoryCache { Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} Err(error) => return Err(error), } + let syncing = Instant::now(); #[cfg(unix)] File::open(&self.root)?.sync_all()?; + let directory_sync_us = syncing.elapsed().as_micros(); let file = open_pinned(&path, length)?.ok_or_else(|| { io::Error::new( io::ErrorKind::NotFound, "memory cache was evicted before it could be pinned; retry restore", ) })?; + tracing::info!( + target: "microsandbox_checkpoint_timing", + operation = "memory_cache_materialize", + total_us = started.elapsed().as_micros(), + object_pipeline_us = objects.elapsed_us, + object_write_us = objects.consume_us, + object_bytes = objects.object_bytes, + file_sync_us, + directory_sync_us, + "memory cache construction timing" + ); Ok(CachedMemory { path, identity: identity.clone(), @@ -316,6 +371,28 @@ impl MemoryCache { // Functions //-------------------------------------------------------------------------------------------------- +fn write_object_slices( + staging: &mut File, + slices: Vec<(u64, u64, u64)>, + bytes: &[u8], +) -> io::Result<()> { + for (target, offset, count) in slices { + let start = + usize::try_from(offset).map_err(|_| invalid("memory object offset overflows"))?; + let count = + usize::try_from(count).map_err(|_| invalid("memory object length overflows"))?; + let end = start + .checked_add(count) + .ok_or_else(|| invalid("memory object slice overflows"))?; + let bytes = bytes + .get(start..end) + .ok_or_else(|| invalid("memory object slice exceeds verified bytes"))?; + staging.seek(SeekFrom::Start(target))?; + staging.write_all(bytes)?; + } + Ok(()) +} + fn memory_regions( manifest: &MemoryManifest, page_size: u64, @@ -592,6 +669,67 @@ mod tests { assert!(!cache.evict(&id).unwrap()); } + #[test] + fn parallel_materialization_preserves_holes_zeroes_and_warm_pins() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + let directory = tempfile::tempdir().unwrap(); + let cache = MemoryCache::open(directory.path()).unwrap(); + let (manifest, id, bytes) = fixture(cache.page_size); + let reads = AtomicUsize::new(0); + let first = cache + .materialize_parallel(&manifest, &id, |_, buffer| { + reads.fetch_add(1, Ordering::Relaxed); + buffer.resize(bytes.len(), 0); + buffer.copy_from_slice(&bytes); + Ok(CheckpointObjectReadTiming::default()) + }) + .unwrap(); + assert_eq!(reads.load(Ordering::Relaxed), 1); + let mut actual = vec![0xff; cache.page_size as usize * 3]; + first.file.read_exact_at(&mut actual, 0).unwrap(); + assert_eq!(&actual[..bytes.len()], bytes); + assert!( + actual[bytes.len()..bytes.len() * 2] + .iter() + .all(|byte| *byte == 0) + ); + assert_eq!(&actual[bytes.len() * 2..], bytes); + let second = cache + .materialize_parallel(&manifest, &id, |_, _| panic!("warm restore reread objects")) + .unwrap(); + assert!(second.cache_hit); + assert!(!cache.evict(&id).unwrap()); + drop(first); + assert!(!cache.evict(&id).unwrap()); + drop(second); + assert!(cache.evict(&id).unwrap()); + } + + #[test] + fn failed_parallel_read_or_slice_never_publishes_a_cache_entry() { + let directory = tempfile::tempdir().unwrap(); + let cache = MemoryCache::open(directory.path()).unwrap(); + let (manifest, id, _) = fixture(cache.page_size); + assert!( + cache + .materialize_parallel(&manifest, &id, |_, _| { + Err(io::Error::other("injected verification failure")) + }) + .is_err() + ); + assert_eq!(payload_count(directory.path()), 0); + assert!( + cache + .materialize_parallel(&manifest, &id, |_, buffer| { + buffer.resize(1, 0); + Ok(CheckpointObjectReadTiming::default()) + }) + .is_err() + ); + assert_eq!(payload_count(directory.path()), 0); + } + #[test] fn descendant_reuses_unchanged_objects_and_clears_new_zero_ranges() { let directory = tempfile::tempdir().unwrap(); diff --git a/crates/runtime/lib/checkpoint/mod.rs b/crates/runtime/lib/checkpoint/mod.rs index 62cb6714e..5c864418a 100644 --- a/crates/runtime/lib/checkpoint/mod.rs +++ b/crates/runtime/lib/checkpoint/mod.rs @@ -1,10 +1,12 @@ //! Runtime-owned composite checkpoint production. +mod capture_pipeline; mod coordinator; mod disk; mod local; mod local_memory; mod memory_cache; +mod object_pipeline; mod restore; //-------------------------------------------------------------------------------------------------- @@ -12,11 +14,11 @@ mod restore; //-------------------------------------------------------------------------------------------------- pub(crate) use coordinator::{CheckpointCoordinator, CheckpointResult, UserPause}; -pub(crate) use disk::recover_runtime_owned_root; pub use disk::{ DiskCompactionResult, RuntimeOwnedRootChain, RuntimeOwnedRootLayer, compact_stopped_root, grow_stopped_root, load_runtime_owned_root_chain, recover_stopped_root_growth, }; +pub(crate) use disk::{recover_runtime_owned_root, seed_restored_root_disk}; pub use local::LocalBranchState; pub use local_memory::LocalMemory; pub use memory_cache::{CachedMemory, CachedMemoryRegion, MemoryCache}; diff --git a/crates/runtime/lib/checkpoint/object_pipeline.rs b/crates/runtime/lib/checkpoint/object_pipeline.rs new file mode 100644 index 000000000..f8e97d496 --- /dev/null +++ b/crates/runtime/lib/checkpoint/object_pipeline.rs @@ -0,0 +1,248 @@ +//! Bounded object verification with construction-thread-only consumption. + +use std::io; +use std::sync::mpsc; +use std::time::Instant; + +use microsandbox_image::checkpoint::{CheckpointObjectReadTiming, ObjectId}; + +//-------------------------------------------------------------------------------------------------- +// Constants +//-------------------------------------------------------------------------------------------------- + +const MAX_READERS: usize = 4; +const MAX_OBJECT_BYTES: usize = 32 * 1024 * 1024; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +#[derive(Debug, Default)] +pub(super) struct ObjectPipelineTiming { + /// Sum of worker read times; parallel worker times are not pipeline wall time. + pub read_us: u128, + pub hash_us: u128, + pub consume_us: u128, + pub elapsed_us: u128, + pub object_bytes: u64, +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +/// Verify at most four objects ahead, returning each buffer to its reader after consumption. +/// +/// Only the invoking thread consumes bytes. Reader completion order may differ from object-ID +/// order, so callers must supply disjoint destination slices. On any error, dropping the work +/// channels cancels queued work and all active readers are joined before their inputs disappear. +pub(super) fn consume_verified_objects( + objects: impl IntoIterator, + read: impl Fn(&ObjectId, &mut Vec) -> io::Result + Sync, + mut consume: impl FnMut(T, &[u8]) -> io::Result<()>, +) -> io::Result { + let objects: Vec<_> = objects.into_iter().collect(); + let readers = std::thread::available_parallelism() + .map(usize::from) + .unwrap_or(1) + .min(MAX_READERS) + .min(objects.len()); + run_pipeline(objects, readers, &read, &mut consume) +} + +fn run_pipeline( + objects: Vec<(ObjectId, T)>, + readers: usize, + read: &(impl Fn(&ObjectId, &mut Vec) -> io::Result + Sync), + consume: &mut impl FnMut(T, &[u8]) -> io::Result<()>, +) -> io::Result { + let started = Instant::now(); + if objects.is_empty() { + return Ok(ObjectPipelineTiming::default()); + } + let readers = readers.clamp(1, MAX_READERS).min(objects.len()); + std::thread::scope(|scope| { + let (ready_tx, ready_rx) = mpsc::sync_channel(readers); + let mut senders = Vec::with_capacity(readers); + let mut handles = Vec::with_capacity(readers); + for worker in 0..readers { + let (work_tx, work_rx) = mpsc::sync_channel::<(ObjectId, T, Vec)>(1); + let ready_tx = ready_tx.clone(); + let handle = std::thread::Builder::new() + .name("checkpoint-reader".into()) + .spawn_scoped(scope, move || { + while let Ok((id, item, mut bytes)) = work_rx.recv() { + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + read(&id, &mut bytes) + })) + .unwrap_or_else(|_| Err(io::Error::other("checkpoint reader panicked"))) + .and_then(|timing| { + if bytes.len() > MAX_OBJECT_BYTES { + Err(io::Error::new( + io::ErrorKind::InvalidData, + "checkpoint reader exceeded the memory object limit", + )) + } else { + Ok(timing) + } + }); + if ready_tx.send((worker, item, bytes, result)).is_err() { + break; + } + } + }); + match handle { + Ok(handle) => { + senders.push(work_tx); + handles.push(handle); + } + Err(error) => { + drop(senders); + drop(ready_rx); + for handle in handles { + let _ = handle.join(); + } + return Err(error); + } + } + } + drop(ready_tx); + let result = (|| { + let total = objects.len(); + let mut pending = objects.into_iter(); + for sender in &senders { + let (id, item) = pending.next().expect("one initial item per reader"); + sender + .send((id, item, Vec::with_capacity(MAX_OBJECT_BYTES))) + .map_err(|_| io::Error::other("checkpoint reader stopped before reading"))?; + } + let mut timings = ObjectPipelineTiming::default(); + for _ in 0..total { + let (worker, item, bytes, timing) = ready_rx + .recv() + .map_err(|_| io::Error::other("checkpoint reader stopped before completion"))?; + let timing = timing?; + timings.read_us += timing.read_us; + timings.hash_us += timing.hash_us; + timings.object_bytes += bytes.len() as u64; + let consuming = Instant::now(); + consume(item, &bytes)?; + timings.consume_us += consuming.elapsed().as_micros(); + if let Some((id, item)) = pending.next() { + senders[worker].send((id, item, bytes)).map_err(|_| { + io::Error::other("checkpoint reader stopped before its next object") + })?; + } + } + timings.elapsed_us = started.elapsed().as_micros(); + Ok(timings) + })(); + // Break both directions before joining: an errored consumer must not leave workers + // blocked on a full completion queue or waiting for their next returned buffer. + drop(senders); + drop(ready_rx); + let mut panicked = false; + for handle in handles { + panicked |= handle.join().is_err(); + } + if panicked { + return Err(io::Error::other("checkpoint object reader panicked")); + } + result + }) +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + use std::sync::Mutex; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use super::*; + + fn jobs(count: usize) -> Vec<(ObjectId, usize)> { + (0..count) + .map(|index| (ObjectId::from_bytes(&index.to_le_bytes()).unwrap(), index)) + .collect() + } + + #[test] + fn buffers_are_bounded_reused_and_consumed_on_the_caller_thread() { + let pointers = Mutex::new(BTreeSet::new()); + let owner = std::thread::current().id(); + let mut observed = BTreeSet::new(); + let timings = run_pipeline( + jobs(20), + 2, + &|_, bytes| { + pointers.lock().unwrap().insert(bytes.as_ptr() as usize); + assert_eq!(bytes.capacity(), MAX_OBJECT_BYTES); + bytes.resize(100, 7); + Ok(CheckpointObjectReadTiming { + read_us: 2, + hash_us: 3, + }) + }, + &mut |index, bytes| { + assert_eq!(std::thread::current().id(), owner); + assert_eq!(bytes, &[7; 100]); + observed.insert(index); + Ok(()) + }, + ) + .unwrap(); + assert_eq!(pointers.lock().unwrap().len(), 2); + assert_eq!(observed.len(), 20); + assert_eq!(timings.object_bytes, 2_000); + assert_eq!((timings.read_us, timings.hash_us), (40, 60)); + } + + #[test] + fn consumption_failure_joins_readers_and_does_not_start_remaining_jobs() { + let reads = AtomicUsize::new(0); + let active = AtomicUsize::new(0); + let result = run_pipeline( + jobs(20), + 2, + &|_, _| { + active.fetch_add(1, Ordering::SeqCst); + reads.fetch_add(1, Ordering::SeqCst); + active.fetch_sub(1, Ordering::SeqCst); + Ok(CheckpointObjectReadTiming::default()) + }, + &mut |_, _| Err(io::Error::other("injected guest write failure")), + ); + assert!(result.is_err()); + assert_eq!(active.load(Ordering::SeqCst), 0); + assert!(reads.load(Ordering::SeqCst) <= 2); + } + + #[test] + fn unverified_bytes_never_reach_the_consumer() { + let result = run_pipeline( + jobs(1), + 1, + &|_, bytes| { + bytes.extend_from_slice(b"corrupt"); + Err(io::Error::new(io::ErrorKind::InvalidData, "bad digest")) + }, + &mut |_, _| panic!("unverified object was consumed"), + ); + assert_eq!(result.unwrap_err().kind(), io::ErrorKind::InvalidData); + } + + #[test] + fn reader_panic_does_not_leave_other_readers_waiting_forever() { + let result = run_pipeline( + jobs(4), + 2, + &|_, _| panic!("injected reader panic"), + &mut |_, _| panic!("panicked reader was consumed"), + ); + assert!(result.unwrap_err().to_string().contains("panicked")); + } +} diff --git a/crates/runtime/lib/checkpoint/restore.rs b/crates/runtime/lib/checkpoint/restore.rs index c6ca99a35..a408a1933 100644 --- a/crates/runtime/lib/checkpoint/restore.rs +++ b/crates/runtime/lib/checkpoint/restore.rs @@ -62,6 +62,11 @@ struct CheckpointMemoryRestore { //-------------------------------------------------------------------------------------------------- impl PreparedCheckpointRestore { + /// Borrow disk admission while the prepared durable restore owns its validated closure. + pub(crate) fn disk_closure(&self) -> Option<&CheckpointClosure> { + self.memory.as_ref().map(|memory| &memory.closure) + } + /// Decode a local handoff and pin its RAM before constructing any guest mappings. pub(crate) fn open_local(root: PathBuf, expected_id: &str) -> Result { let state = super::LocalBranchState::open(&root).map_err(|e| e.to_string())?; @@ -182,11 +187,15 @@ impl PreparedCheckpointRestore { .closure; let cache = super::MemoryCache::open(root).map_err(|e| e.to_string())?; let cached = cache - .materialize(closure.memory(), &closure.checkpoint().memory, |id| { - closure - .read_object(id, MAX_MEMORY_OBJECT_BYTES) - .map_err(io::Error::other) - }) + .materialize_parallel( + closure.memory(), + &closure.checkpoint().memory, + |id, bytes| { + closure + .read_object_into(id, MAX_MEMORY_OBJECT_BYTES, bytes) + .map_err(io::Error::other) + }, + ) .map_err(|e| e.to_string())?; tracing::info!( cache_hit = cached.cache_hit, @@ -230,9 +239,7 @@ impl msb_krun::VmMemoryRestoreSource for CheckpointMemoryRestore { let total_started = Instant::now(); let mut zero_write_us = 0u128; let mut zero_bytes = 0u64; - let mut object_read_us = 0u128; let mut guest_write_us = 0u128; - let mut object_bytes = 0u64; let mut guest_object_bytes = 0u64; let mut object_extent_count = 0usize; let mut objects: BTreeMap> = @@ -257,53 +264,58 @@ impl msb_krun::VmMemoryRestoreSource for CheckpointMemoryRestore { } } - // Read and identity-check each packed object exactly once, write all of its referenced - // guest ranges, then release the small object buffer. This fuses integrity with the - // unavoidable restore pass without retaining a RAM-sized cache. + // Read and identity-check each packed object exactly once with bounded read-ahead. + // Guest ranges are disjoint and only this construction thread writes them; workers + // never obtain guest-memory access or permit activation before verification completes. let object_count = objects.len(); - for (id, extents) in objects { - let read_started = Instant::now(); - let bytes = self - .closure - .read_object(&id, MAX_MEMORY_OBJECT_BYTES) - .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error.to_string()))?; - object_read_us += read_started.elapsed().as_micros(); - object_bytes = object_bytes.saturating_add(bytes.len() as u64); - for (range, offset) in extents { - let start = usize::try_from(offset).map_err(|_| { - io::Error::new( - io::ErrorKind::InvalidData, - "memory object offset is too large", - ) - })?; - let length = usize::try_from(range.length()).map_err(|_| { - io::Error::new(io::ErrorKind::InvalidData, "memory extent is too large") - })?; - let end = start.checked_add(length).ok_or_else(|| { - io::Error::new(io::ErrorKind::InvalidData, "memory object slice overflows") - })?; - let slice = bytes.get(start..end).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "memory object slice exceeds verified bytes", - ) - })?; - let write_started = Instant::now(); - target.write_bytes(range, slice)?; - guest_write_us += write_started.elapsed().as_micros(); - guest_object_bytes = guest_object_bytes.saturating_add(range.length()); - } - } + let pipeline = super::object_pipeline::consume_verified_objects( + objects, + |id, bytes| { + self.closure + .read_object_into(id, MAX_MEMORY_OBJECT_BYTES, bytes) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error.to_string())) + }, + |extents, bytes| { + for (range, offset) in extents { + let start = usize::try_from(offset).map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidData, + "memory object offset is too large", + ) + })?; + let length = usize::try_from(range.length()).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidData, "memory extent is too large") + })?; + let end = start.checked_add(length).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "memory object slice overflows") + })?; + let slice = bytes.get(start..end).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "memory object slice exceeds verified bytes", + ) + })?; + let write_started = Instant::now(); + target.write_bytes(range, slice)?; + guest_write_us += write_started.elapsed().as_micros(); + guest_object_bytes = guest_object_bytes.saturating_add(range.length()); + } + Ok(()) + }, + )?; tracing::info!( target: "microsandbox_checkpoint_timing", operation = "restore_memory", total_us = total_started.elapsed().as_micros(), - object_read_us, + object_read_us = pipeline.read_us + pipeline.hash_us, + object_io_worker_us = pipeline.read_us, + object_hash_worker_us = pipeline.hash_us, + object_pipeline_us = pipeline.elapsed_us, guest_write_us, zero_write_us, object_count, object_extent_count, - object_bytes, + object_bytes = pipeline.object_bytes, guest_object_bytes, zero_bytes, "checkpoint memory restore timing" diff --git a/crates/runtime/lib/control/executor.rs b/crates/runtime/lib/control/executor.rs index ef6525311..2067fec86 100644 --- a/crates/runtime/lib/control/executor.rs +++ b/crates/runtime/lib/control/executor.rs @@ -262,6 +262,14 @@ impl RuntimeControlExecutor { state: &mut ExecutorState, request: ControlRequest, ) -> ControlResponse { + // Gate the authoritative operation, including idempotent Resume on a running VM. + // Clients need no separate capability exchange, and refusal never changes ownership. + if matches!(request, ControlRequest::Pause | ControlRequest::Resume) + && let Some(response) = + unsupported_lifecycle_request(&request, self.vm.clock_sync_supported()) + { + return response; + } let mutation = matches!( request, ControlRequest::MemoryTarget { .. } @@ -641,6 +649,18 @@ impl RuntimeControlExecutor { // Functions //-------------------------------------------------------------------------------------------------- +fn unsupported_lifecycle_request( + request: &ControlRequest, + clock_sync: bool, +) -> Option { + (!clock_sync && matches!(request, ControlRequest::Pause | ControlRequest::Resume)).then(|| { + control_error( + "pause_resume_unavailable", + "resident pause/resume requires a runtime and guest kernel with clock-only resume support", + ) + }) +} + fn new_runtime_boot_id() -> String { let mut bytes = [0u8; 16]; rand::rng().fill_bytes(&mut bytes); @@ -785,6 +805,23 @@ fn control_error(code: &str, message: impl Into) -> ControlResponse { mod tests { use super::*; + #[test] + fn unsupported_pause_and_resume_refuse_before_idempotent_mutation() { + for request in [ControlRequest::Pause, ControlRequest::Resume] { + let refused = unsupported_lifecycle_request(&request, false).unwrap(); + assert!(!refused.ok); + assert_eq!( + refused.error_code.as_deref(), + Some("pause_resume_unavailable") + ); + assert!(refused.pause.is_none()); + assert!(unsupported_lifecycle_request(&request, true).is_none()); + } + // Observation remains safe without a kernel clock callback. + assert!(unsupported_lifecycle_request(&ControlRequest::PauseState, false).is_none()); + assert!(unsupported_lifecycle_request(&ControlRequest::Capabilities, false).is_none()); + } + #[test] fn control_ids_are_bounded_and_printable() { assert!(valid_control_id("request_42")); diff --git a/crates/runtime/lib/vm.rs b/crates/runtime/lib/vm.rs index e42262983..3a507e3b8 100644 --- a/crates/runtime/lib/vm.rs +++ b/crates/runtime/lib/vm.rs @@ -2097,6 +2097,12 @@ fn build_vm( ) } .map_err(|error| RuntimeError::Custom(format!("prepare checkpoint restore: {error}")))?; + if let Some(admitted) = prepared.disk_closure() { + // Reuse this process's exact admitted file bindings before the closure is moved + // into RAM restoration. The later coordinator opens the completed journal. + crate::checkpoint::seed_restored_root_disk(&config.runtime_dir, &config.vm, admitted) + .map_err(RuntimeError::Custom)?; + } let cache_root = restore .forked .then(|| { diff --git a/sdk/rust/lib/backend/local/sandbox/mod.rs b/sdk/rust/lib/backend/local/sandbox/mod.rs index 158df6e74..a33c694ed 100644 --- a/sdk/rust/lib/backend/local/sandbox/mod.rs +++ b/sdk/rust/lib/backend/local/sandbox/mod.rs @@ -312,7 +312,7 @@ impl LocalBackend { } /// Load the local DB row + active PID for a sandbox handle. - async fn sandbox_handle_state( + pub(crate) async fn sandbox_handle_state( &self, name: &str, ) -> MicrosandboxResult<(sandbox_entity::Model, Option)> { @@ -1190,6 +1190,71 @@ mod tests { pid } + #[cfg(unix)] + #[tokio::test] + async fn control_lookup_skips_observation_but_get_and_list_still_project_pause() { + use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; + + let home = tempfile::tempdir_in("/tmp").unwrap(); + let backend = Arc::new( + LocalBackend::builder() + .home(home.path()) + .build() + .await + .unwrap(), + ); + let pools = backend.db().await.unwrap(); + let name = "resident"; + let id = LocalBackend::insert_sandbox_record(pools.write(), &test_config(name)) + .await + .unwrap(); + run_entity::Entity::insert(run_entity::ActiveModel { + sandbox_id: Set(id), + pid: Set(Some(std::process::id() as i32)), + status: Set(run_entity::RunStatus::Running), + ..Default::default() + }) + .exec(pools.write()) + .await + .unwrap(); + let agent = + crate::runtime::sandbox_agent_socket_path_candidates_for(&backend, name).remove(0); + let path = microsandbox_runtime::control::control_socket_path_for(&agent); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + let listener = tokio::net::UnixListener::bind(path).unwrap(); + let server = tokio::spawn(async move { + // The mutation arrives first. Ordinary observational APIs retain their projection. + for operation in ["pause", "pause_state", "pause_state"] { + let (stream, _) = listener.accept().await.unwrap(); + let mut stream = BufReader::new(stream); + let mut line = String::new(); + stream.read_line(&mut line).await.unwrap(); + assert_eq!(line, format!("{{\"op\":\"{operation}\"}}\n")); + stream + .get_mut() + .write_all( + b"{\"ok\":true,\"pause\":{\"paused\":true,\"recovery_required\":false}}\n", + ) + .await + .unwrap(); + } + }); + let backend_dyn: Arc = backend; + crate::backend::with_backend(backend_dyn, async { + let handle = crate::Sandbox::get_for_control(name).await.unwrap(); + handle.pause().await.unwrap(); + assert_eq!( + crate::Sandbox::get(name).await.unwrap().status_snapshot(), + SandboxStatus::Paused + ); + let page = crate::Sandbox::list().await.unwrap(); + assert_eq!(page.sandboxes.len(), 1); + assert_eq!(page.sandboxes[0].status_snapshot(), SandboxStatus::Paused); + }) + .await; + server.await.unwrap(); + } + #[tokio::test] async fn follow_logs_replays_filtered_history_then_streams_from_snapshot_cursor() { let temp = tempdir().unwrap(); diff --git a/sdk/rust/lib/sandbox/pause.rs b/sdk/rust/lib/sandbox/pause.rs index b2736088e..9a0d3938c 100644 --- a/sdk/rust/lib/sandbox/pause.rs +++ b/sdk/rust/lib/sandbox/pause.rs @@ -13,6 +13,20 @@ use super::{Sandbox, SandboxHandle, SandboxPauseState, modify}; //-------------------------------------------------------------------------------------------------- impl Sandbox { + /// Internal CLI lookup for an immediately following authoritative control mutation. + /// + /// Keep database/runtime reconciliation, but skip the pause observation used by ordinary + /// `get`/`list`: that observation is already stale by the time the mutation executes. + #[doc(hidden)] + pub async fn get_for_control(name: &str) -> MicrosandboxResult { + let backend = crate::backend::default_backend(); + if let Some(local) = backend.as_local() { + let (model, pid) = local.sandbox_handle_state(name).await?; + return Ok(SandboxHandle::from_local_model(backend, model, pid)); + } + backend.sandboxes().get(backend.clone(), name).await + } + /// Suspend this resident VM without creating a snapshot or releasing RAM. pub async fn pause(&self) -> MicrosandboxResult<()> { lifecycle(self.name(), self.backend().as_ref(), ControlRequest::Pause) @@ -106,20 +120,26 @@ async fn lifecycle( let local = backend .as_local() .ok_or_else(|| MicrosandboxError::local_only(operation))?; - // Do not send a new operation to an old runtime that cannot implement its semantics. - let capabilities = - modify::control_request_for(local, name, "{\"op\":\"capabilities\"}\n".into()).await?; - if !capabilities - .capabilities - .is_some_and(|caps| caps.pause_resume) - { - return Err(MicrosandboxError::Runtime("resident pause/resume requires a runtime and guest kernel with clock-only resume support".into())); - } + // The mutation itself is authoritative. Unknown operations fail on older runtimes, and + // successful replies must carry pause state; neither case can silently become a no-op. let line = format!("{}\n", serde_json::to_string(&request)?); let response = modify::control_request_for(local, name, line).await?; - response + let state = response .pause - .ok_or_else(|| MicrosandboxError::Runtime("control response omitted pause state".into())) + .ok_or_else(|| MicrosandboxError::Runtime("control response omitted pause state".into()))?; + // An acknowledgement must confirm the requested transition, not just contain some + // observation. State inspection itself must still be able to report recovery required. + let expected = match request { + ControlRequest::Pause => Some(true), + ControlRequest::Resume => Some(false), + _ => None, + }; + if expected.is_some_and(|paused| state.paused != paused || state.recovery_required) { + return Err(MicrosandboxError::Runtime( + "control response did not confirm the requested pause transition".into(), + )); + } + Ok(state) } //-------------------------------------------------------------------------------------------------- @@ -158,17 +178,14 @@ mod tests { std::fs::create_dir_all(path.parent().unwrap()).unwrap(); let listener = tokio::net::UnixListener::bind(path).unwrap(); let server = tokio::spawn(async move { - // One observation plus the capability-gated public lifecycle exchange. - for _ in 0..3 { + // Each observation is one exchange; it needs no capabilities preflight. + for _ in 0..2 { let (stream, _) = listener.accept().await.unwrap(); let mut stream = BufReader::new(stream); let mut line = String::new(); stream.read_line(&mut line).await.unwrap(); - let response = if line.contains("capabilities") { - "{\"ok\":true,\"capabilities\":{\"pause_resume\":true,\"cpu_resize\":false,\"memory_resize\":false,\"secrets_update\":false}}\n" - } else { - "{\"ok\":true,\"pause\":{\"paused\":true,\"recovery_required\":false,\"capture_unavailable\":null}}\n" - }; + assert_eq!(line, "{\"op\":\"pause_state\"}\n"); + let response = "{\"ok\":true,\"pause\":{\"paused\":true,\"recovery_required\":false,\"capture_unavailable\":null}}\n"; stream .get_mut() .write_all(response.as_bytes()) @@ -191,4 +208,85 @@ mod tests { .await; server.await.unwrap(); } + + #[tokio::test] + async fn lifecycle_sends_one_mutation_and_requires_an_authoritative_reply() { + for (operation, response, accepted) in [ + ( + "pause", + "{\"ok\":true,\"pause\":{\"paused\":true,\"recovery_required\":false}}\n", + true, + ), + ( + "resume", + "{\"ok\":true,\"pause\":{\"paused\":false,\"recovery_required\":false}}\n", + true, + ), + // Old runtime unknown-operation errors and unsupported current kernels must fail. + ( + "pause", + "{\"ok\":false,\"error\":\"unknown variant pause\"}\n", + false, + ), + ( + "resume", + "{\"ok\":false,\"error\":\"pause/resume unavailable\"}\n", + false, + ), + ("resume", "{\"ok\":true}\n", false), + ( + "pause", + "{\"ok\":true,\"pause\":{\"paused\":false,\"recovery_required\":false}}\n", + false, + ), + ( + "resume", + "{\"ok\":true,\"pause\":{\"paused\":true,\"recovery_required\":false}}\n", + false, + ), + ( + "pause", + "{\"ok\":true,\"pause\":{\"paused\":true,\"recovery_required\":true}}\n", + false, + ), + ] { + let home = tempfile::tempdir_in("/tmp").unwrap(); + let backend = LocalBackend::builder() + .home(home.path()) + .build() + .await + .unwrap(); + let agent = + crate::runtime::sandbox_agent_socket_path_candidates_for(&backend, "source") + .remove(0); + let path = microsandbox_runtime::control::control_socket_path_for(&agent); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + let listener = tokio::net::UnixListener::bind(path).unwrap(); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut stream = BufReader::new(stream); + let mut line = String::new(); + stream.read_line(&mut line).await.unwrap(); + assert_eq!(line, format!("{{\"op\":\"{operation}\"}}\n")); + stream + .get_mut() + .write_all(response.as_bytes()) + .await + .unwrap(); + }); + let request = if operation == "pause" { + ControlRequest::Pause + } else { + ControlRequest::Resume + }; + let result = tokio::time::timeout( + std::time::Duration::from_secs(2), + lifecycle("source", &backend, request), + ) + .await + .unwrap(); + assert_eq!(result.is_ok(), accepted, "{operation}: {response}"); + server.await.unwrap(); + } + } } From e20269e1b260149e4cf629f43fac142a35e1458b Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Thu, 10 Sep 2026 01:26:37 +0100 Subject: [PATCH 13/29] feat(snapshot): export reusable RAM as archive dependencies Extend --since exports to omit RAM objects supplied by the explicit base alongside the physical disk prefix. Preserve complete memory maps and CPU/device metadata, and resolve borrowed objects into owned staging for both snapshot load and direct archive restore. Keep --last-layers limited to disk selection. Update CLI/SDK documentation and add malformed-input, corruption, retention and twelve-generation archive regressions, plus a reproducible live qualification harness. Validate 44 snapshot tests on both macOS and Linux and live twelve-step chains for flat, layered and tmpfs roots with eager and forked restores. Record timings and qualification limits in the smoke report. BREAKING CHANGE: replace the unreleased disk-only archive dependency encoding with msb-snapshot-dependencies-v1, without a compatibility shim. Standalone archives and snapshot descriptors remain unchanged. --- COMPATIBILITY.md | 2 + crates/cli/lib/commands/snapshot.rs | 2 +- docs/sandboxes/snapshots.mdx | 4 +- docs/sdk/go/snapshots.mdx | 2 + docs/sdk/python/snapshots.mdx | 2 + docs/sdk/rust/snapshots.mdx | 2 + docs/sdk/typescript/snapshots.mdx | 2 + scripts/smoke/cli/incremental-full-archive.py | 148 +++++ .../incremental-full-archives-2026-09-10.md | 130 ++++ sdk/go/snapshot.go | 2 +- sdk/node-ts/native/index.d.ts | 2 +- sdk/node-ts/native/sandbox_builder.rs | 2 +- sdk/node-ts/native/snapshot.rs | 2 +- sdk/node-ts/src/snapshot.ts | 2 +- sdk/python/src/snapshot.rs | 3 + sdk/rust/lib/sandbox/builder.rs | 2 +- sdk/rust/lib/snapshot/archive.rs | 7 +- sdk/rust/lib/snapshot/archive/delta.rs | 382 ++++++++--- sdk/rust/lib/snapshot/archive/delta_tests.rs | 597 ++++++++++++++++++ sdk/rust/lib/snapshot/mod.rs | 2 +- 20 files changed, 1189 insertions(+), 108 deletions(-) create mode 100644 scripts/smoke/cli/incremental-full-archive.py create mode 100644 scripts/smoke/reports/incremental-full-archives-2026-09-10.md create mode 100644 sdk/rust/lib/snapshot/archive/delta_tests.rs diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index f12833be0..82c2c7298 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -190,6 +190,8 @@ Snapshot descriptor bytes are identity-bearing: their canonical bytes determine Archive compatibility includes compression detection, `archive.json`, canonical inventory order, transport digests, accepted path grammar, legacy paths, cache-closure entries, and rejection of duplicate, missing, or escaping paths. +Unreleased #8 incremental exports use `completeness: "dependent"` and the must-understand `msb-snapshot-dependencies-v1` extension. `--since` records omitted physical disk-prefix layers and reusable RAM-object identities; `--last-layers` only omits disk layers. The complete target memory manifest and CPU/device state remain included. Loading and direct archive restore resolve the explicitly supplied base into owned staging before opening the complete target. This replaces the unreleased disk-only dependency encoding without a compatibility shim or snapshot descriptor change. Readers that do not understand this requirement refuse it; ordinary standalone archives are unchanged. + Evolution rules: - Do not make semantically harmless serialization changes to identity-bearing bytes without treating them as an identity format change. diff --git a/crates/cli/lib/commands/snapshot.rs b/crates/cli/lib/commands/snapshot.rs index d99592f34..5647b7a2a 100644 --- a/crates/cli/lib/commands/snapshot.rs +++ b/crates/cli/lib/commands/snapshot.rs @@ -167,7 +167,7 @@ pub struct SnapshotSaveArgs { /// CPU but much larger file for sparse uppers. #[arg(long)] pub plain_tar: bool, - /// Export disk layers after an exact base snapshot or standalone base archive. + /// Omit disk layers and RAM objects supplied by an exact base snapshot or standalone archive. #[arg(long, conflicts_with_all = ["last_layers", "with_parents"])] pub since: Option, /// Export only the newest N sealed disk layers (load requires the omitted base). diff --git a/docs/sandboxes/snapshots.mdx b/docs/sandboxes/snapshots.mdx index fbb7526de..0bbbe8950 100644 --- a/docs/sandboxes/snapshots.mdx +++ b/docs/sandboxes/snapshots.mdx @@ -196,14 +196,14 @@ msb modify worker --compact --layers 3 `--layers 3` merges the oldest three physical layers, **including the base**. It never includes the writable head, even when stopped. Omit `--layers` to merge all sealed layers; a chain with fewer than two sealed layers is unchanged. Managed and flat roots support compaction, running or stopped. Existing snapshots remain valid and retain their storage until you remove them separately. -`--since` requires an exact physical-prefix base. Alternatively, `--last-layers 2` includes the newest two sealed checkpoint layers. A full checkpoint export still includes all required memory and device state. Unlike ordinary standalone exports, these smaller archives require the omitted base when loading or restoring: +`--since` omits disk layers and, for full checkpoints, RAM objects already supplied by the base. The disk base must be an exact physical prefix. Each archive still includes its complete memory map and CPU/device state; it does not replay earlier memory images. Alternatively, `--last-layers 2` selects only disk layers and keeps all required RAM objects. These smaller archives require an explicit base when loading or restoring: ```bash msb snapshot load changes.msnap --base checkpoint-a msb create --name child --from-snapshot changes.msnap --snapshot-base checkpoint-a ``` -The base can also be a standalone snapshot archive. Restore copies the required closure into child-owned storage, without installing an intermediate snapshot. Add `--disk-only` to cold-boot only disk state. After compaction, export a new standalone baseline before resuming incremental exports: the old physical prefix no longer matches. Do not combine compaction with unrelated `modify` options, or incremental export with `--with-parents`. +The base can also be a standalone snapshot archive. Load a dependent base first, then pass the installed path printed by `snapshot load` to the next load or restore. Loads resolve disk and RAM dependencies without starting a VM; only the final sandbox creation resumes execution. Missing or incorrect dependencies fail before execution. Loaded snapshots and restored children own their required files, so removing the base later does not break them. Direct restore skips installing an intermediate snapshot. Add `--disk-only` to cold-boot only disk state. After compaction, export a new standalone baseline before resuming incremental exports: the old physical prefix no longer matches. Do not combine compaction with unrelated `modify` options, or incremental export with `--with-parents`. ## Capture directly to an archive diff --git a/docs/sdk/go/snapshots.mdx b/docs/sdk/go/snapshots.mdx index f723978cb..c6069a1e0 100644 --- a/docs/sdk/go/snapshots.mdx +++ b/docs/sdk/go/snapshots.mdx @@ -9,6 +9,8 @@ Create disk snapshots of stopped sandboxes and full checkpoints of running sandb ## Disk maintenance and incremental export +Exporting since a base omits its reusable disk layers and RAM objects. Full checkpoints keep the complete memory map and CPU/device state. The last-layers option only selects disk layers and leaves RAM payloads complete. Supply the base when loading or restoring a dependent archive; a dependent base archive must be loaded first. + ```go worker, err := m.GetSandbox(ctx, "worker") if err != nil { return err } diff --git a/docs/sdk/python/snapshots.mdx b/docs/sdk/python/snapshots.mdx index 462a0f1d6..56d924d16 100644 --- a/docs/sdk/python/snapshots.mdx +++ b/docs/sdk/python/snapshots.mdx @@ -101,6 +101,8 @@ sb = await Sandbox.create("worker", image="python:3.12") ## Disk maintenance and incremental export +Exporting since a base omits its reusable disk layers and RAM objects. Full checkpoints keep the complete memory map and CPU/device state. The last-layers option only selects disk layers and leaves RAM payloads complete. Supply the base when loading or restoring a dependent archive; a dependent base archive must be loaded first. + ```python worker = await Sandbox.get("worker") plan = await worker.compact(layers=3, dry_run=True) diff --git a/docs/sdk/rust/snapshots.mdx b/docs/sdk/rust/snapshots.mdx index ca6f57412..f41d84af8 100644 --- a/docs/sdk/rust/snapshots.mdx +++ b/docs/sdk/rust/snapshots.mdx @@ -9,6 +9,8 @@ Create disk snapshots of running, paused, stopped, or crashed sandboxes, or full ## Disk maintenance and incremental export +Exporting since a base omits its reusable disk layers and RAM objects. Full checkpoints keep the complete memory map and CPU/device state. The last-layers option only selects disk layers and leaves RAM payloads complete. Supply the base when loading or restoring a dependent archive; a dependent base archive must be loaded first. + ```rust let worker = Sandbox::get("worker").await?; let plan = worker.compact().layers(3).dry_run().await?; diff --git a/docs/sdk/typescript/snapshots.mdx b/docs/sdk/typescript/snapshots.mdx index 8ba0755ed..2b3acf2fd 100644 --- a/docs/sdk/typescript/snapshots.mdx +++ b/docs/sdk/typescript/snapshots.mdx @@ -91,6 +91,8 @@ const snap = await h.snapshot("after-pip-install"); ## Disk maintenance and incremental export +Exporting since a base omits its reusable disk layers and RAM objects. Full checkpoints keep the complete memory map and CPU/device state. The last-layers option only selects disk layers and leaves RAM payloads complete. Supply the base when loading or restoring a dependent archive; a dependent base archive must be loaded first. + ```typescript const worker = await Sandbox.get("worker"); const plan = await worker.compact({ layers: 3, dryRun: true }); diff --git a/scripts/smoke/cli/incremental-full-archive.py b/scripts/smoke/cli/incremental-full-archive.py new file mode 100644 index 000000000..370380503 --- /dev/null +++ b/scripts/smoke/cli/incremental-full-archive.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +"""Opt-in full archive dependency chain: real guest RAM/disk, eager/forked restore, cleanup. + +Set MSB_PATH, STACK8_OUT, and optionally STACK8_LAYOUT (512M, flat:512M, tmpfs:128M). +Each run creates isolated source/destination MSB_HOME directories below STACK8_OUT. +""" + +import json +import os +from pathlib import Path +import shutil +import subprocess +import time + +binary = os.environ["MSB_PATH"] +root = Path(os.environ["STACK8_OUT"]) +root.mkdir(parents=True, exist_ok=False) +layout = os.environ.get("STACK8_LAYOUT", "flat:512M") +source_home = root / "source-home" +dest_home = root / "destination-home" +prefix = f"ramdelta-{os.getpid()}" +rows = [] +names = [] + + +def run(label, home, *args, fail=False): + env = dict(os.environ, MSB_HOME=str(home)) + started = time.perf_counter() + result = subprocess.run([binary, *map(str, args)], env=env, capture_output=True, text=True, timeout=180) + elapsed = round((time.perf_counter() - started) * 1000, 2) + (root / f"{label}.stdout").write_text(result.stdout) + (root / f"{label}.stderr").write_text(result.stderr) + rows.append({"case": label, "ms": elapsed, "exit": result.returncode}) + print(json.dumps(rows[-1]), flush=True) + if (result.returncode != 0) != fail: + raise RuntimeError(f"{label}: exit={result.returncode}: {result.stderr[-3000:]}") + return result.stdout.strip() + + +def guest(label, home, name, script): + return run(label, home, "exec", name, "--", "sh", "-ec", script) + + +def inventory(archive): + # System tar decodes plain and zstd input by magic, independently of the SDK loader. + return json.loads(subprocess.check_output(["tar", "-xOf", str(archive), "archive.json"])) + + +def restore(label, snapshot, expected, base=None, forked=False): + name = prefix + "-" + label + names.append((dest_home, name)) + args = ["create", "--name", name, "--from-snapshot", str(snapshot)] + if base: + args += ["--snapshot-base", str(base)] + if forked: + args += ["--forked"] + run(label, dest_home, *args) + actual = guest(label + "-read", dest_home, name, "cat /dev/shm/marker; cat /disk-marker; sha256sum /dev/shm/blob | cut -d' ' -f1") + assert actual == f"{expected}\n{expected}\n{blob_hash}", actual + guest(label + "-write", dest_home, name, "echo child > /dev/shm/marker; echo child > /disk-marker") + return name + + +try: + source = prefix + "-source" + names.append((source_home, source)) + # Reuse only immutable OCI artifacts, never a prior VM or memory backing cache. + seed = os.environ.get("STACK8_SEED_CACHE") + if seed: + for kind in ("layers", "manifests", "fsmeta", "vmdk"): + origin = Path(seed) / kind + if origin.exists(): + shutil.copytree(origin, source_home / "cache" / kind) + run("boot", source_home, "create", "alpine", "--name", source, "--root-disk", layout, "--memory", "256M", "--cpus", "2") + # Image transport is separate from checkpoint dependencies. Independently populate the + # destination's OCI cache so this matrix does not depend on --with-image or source paths. + seed_vm = prefix + "-image-seed" + names.append((dest_home, seed_vm)) + run("prepare-destination-image", dest_home, "create", "alpine", "--name", seed_vm, "--root-disk", layout, "--memory", "256M", "--cpus", "2") + run("stop-image-seed", dest_home, "stop", seed_vm) + blob_hash = guest("prepare", source_home, source, "dd if=/dev/urandom of=/dev/shm/blob bs=1M count=24 2>/dev/null; sha256sum /dev/shm/blob | cut -d' ' -f1") + previous_source = None + installed_base = None + archive_rows = [] + for n in range(1, 13): + guest(f"mutate-{n}", source_home, source, f"echo {n} > /dev/shm/marker; echo {n} > /disk-marker; dd if=/dev/zero of=/dev/shm/zero bs=4096 count=1 2>/dev/null") + if n == 6: + run("pause-source", source_home, "pause", source) + name = f"cp{n:02}" + run(f"capture-{n}", source_home, "snapshot", "create", name, "--from", source, "--full") + if n == 6: + run("resume-source", source_home, "resume", source) + artifact = source_home / "snapshots" / name + archive = root / f"cp{n:02}.msnap" + args = ["snapshot", "save", artifact, archive] + if previous_source: + args += ["--since", previous_source] + run(f"export-{n}", source_home, *args) + inv = inventory(archive) + omitted_ram = [e for e in inv["entries"] if not e["included"] and e["kind"] == "checkpoint-object"] + if n > 1: + assert omitted_ram, "incremental export did not omit any reusable RAM objects" + assert all(e["kind"] in ("checkpoint-object", "checkpoint-disk-layer") for e in inv["entries"] if not e["included"]) + archive_rows.append({"checkpoint": n, "archive_bytes": archive.stat().st_size, "omitted_ram_objects": len(omitted_ram), "omitted_ram_bytes": sum(e["apparent_size"] for e in omitted_ram)}) + if n == 2: + run("missing-base", dest_home, "snapshot", "load", archive, fail=True) + run("wrong-base", dest_home, "snapshot", "load", archive, "--base", root / "absent", fail=True) + if n == 12: + final_archive, final_artifact = archive, artifact + break + load_args = ["snapshot", "load", archive] + if installed_base: + load_args += ["--base", installed_base] + old_base = installed_base + installed_base = Path(run(f"load-{n}", dest_home, *load_args).splitlines()[-1]) + assert installed_base.parent == dest_home / "snapshots" + if old_base: + run(f"remove-base-{n-1}", dest_home, "snapshot", "remove", old_base) + previous_source = artifact + run("stop-source", source_home, "stop", source) + before = set((dest_home / "snapshots").iterdir()) + eager = restore("direct-eager", final_archive, 12, installed_base) + forked = restore("direct-forked", final_archive, 12, installed_base, True) + assert set((dest_home / "snapshots").iterdir()) == before, "direct restore installed the target snapshot" + final_loaded = Path(run("load-final", dest_home, "snapshot", "load", final_archive, "--base", installed_base).splitlines()[-1]) + run("remove-final-base", dest_home, "snapshot", "remove", installed_base) + # Live children must retain both their captured RAM backing and private disk writes + # after the explicit base disappears. The installed target must remain independent too. + for mode, child in (("eager", eager), ("forked", forked)): + actual = guest("deleted-base-" + mode, dest_home, child, "cat /dev/shm/marker; cat /disk-marker; sha256sum /dev/shm/blob | cut -d' ' -f1") + assert actual == f"child\nchild\n{blob_hash}", actual + run("stop-" + mode, dest_home, "stop", child) + run("verify-final", dest_home, "snapshot", "verify", final_loaded) + for mode in ("eager", "forked"): + child = restore("installed-" + mode, final_loaded, 12, forked=mode == "forked") + run("stop-installed-" + mode, dest_home, "stop", child) + complete = root / "standalone.msnap" + run("export-standalone", source_home, "snapshot", "save", final_artifact, complete) + assert all(e["included"] for e in inventory(complete)["entries"]) + archive_rows[-1]["standalone_bytes"] = complete.stat().st_size + (root / "archive-sizes.json").write_text(json.dumps(archive_rows, indent=2)) + print(json.dumps({"pass": True, "layout": layout, "archives": archive_rows}), flush=True) +finally: + # Stop only test-owned names, including children whose creation failed part-way through. + for home, name in reversed(names): + result = subprocess.run([binary, "stop", name], env=dict(os.environ, MSB_HOME=str(home)), capture_output=True, text=True, timeout=30) + (root / ("cleanup-" + name + ".log")).write_text(result.stdout + result.stderr) + (root / "results.json").write_text(json.dumps(rows, indent=2)) diff --git a/scripts/smoke/reports/incremental-full-archives-2026-09-10.md b/scripts/smoke/reports/incremental-full-archives-2026-09-10.md new file mode 100644 index 000000000..29e0dcb2a --- /dev/null +++ b/scripts/smoke/reports/incremental-full-archives-2026-09-10.md @@ -0,0 +1,130 @@ +# Incremental full-snapshot archive qualification — 2026-09-10 + +## Result and scope + +RAM-aware `snapshot save --since` passed 12-checkpoint live chains on macOS ARM64/HVF and OVH Linux x86-64/KVM with flat, managed/layered, and tmpfs roots. The final checkpoint resumed with its expected RAM and filesystem contents through direct-archive and installed-snapshot restore, both eager and forked. This qualifies the archive dependency change, not unrelated concurrent lifecycle work. Linux ARM64 and Windows were not rerun for this change. + +The implementation replaces the unreleased disk-only dependency encoding with `msb-snapshot-dependencies-v1`. It omits reusable RAM objects as well as the exact physical disk prefix, keeps the target memory manifest and CPU/device state complete, and resolves the explicit base into destination-owned staging. `--last-layers` remains disk-only selection. Standalone archives and snapshot descriptors are unchanged; there is no compatibility shim for earlier unreleased #8 dependency archives. + +## macOS build and fixture + +- Base commit: `18c8fb8693957ac6e8ded964e42cfbe63f8506ad`, plus this change's archive implementation, tests, and CLI help. An isolated worktree excluded concurrent agentd, database, and control-path edits in the shared #8 checkout. +- Host: macOS 26.3, build `25D2125`, ARM64/HVF. +- Binary: debug build, `net,ssh` features, codesigned with `msb-entitlements.plist`. SHA-256: `dc21397227f028bb32ffeed7f8322f33caf2767f16bac264fae2ede80a8f6cdb`. +- Guest agent SHA-256: `4c467d1e93d9ba78168d0eb3ec6c0a5d03793b972c496e250fe0288a42e51e43`. +- Firmware SHA-256: `ea0d458cdc12a0fa6dac8d192542ddc39717f816da41176582905e31a8bf868c`. +- Each source: Alpine, 256 MiB RAM, two vCPUs. Flat/managed disks: 512 MiB; tmpfs root: 128 MiB. +- Workload: a retained 24 MiB random RAM file plus a RAM marker and root-filesystem marker changed before each capture. Checkpoint 6 was captured while explicitly paused; other captures were from a running source. +- Source and destination used separate `MSB_HOME` directories. Each destination independently populated its OCI image cache. Image bundling was not part of this test. + +## macOS size and timing + +These are individual end-to-end CLI wall times from sequential debug-build qualification runs, not release-build latency claims. Filesystem caches were warm, and forked restores ran after eager restores. MB means decimal megabytes. Capture times include publication, not just the pause interval. Export comparisons below use the same checkpoint 12 with and without `--since`. + +| Checkpoint 12 result | Flat | Managed/layered | Tmpfs | +| --- | ---: | ---: | ---: | +| Standalone archive bytes | 51,778,517 | 48,269,890 | 47,274,984 | +| Incremental archive bytes | 535,798 | 598,498 | 501,731 | +| Archive reduction | 98.97% | 98.76% | 98.94% | +| Omitted RAM objects | 13 | 13 | 13 | +| Omitted RAM object bytes | 131,334,144 | 135,737,344 | 126,586,880 | +| Full capture | 927.64 ms | 733.54 ms | 312.09 ms | +| Standalone export | 3,228.32 ms | 3,020.35 ms | 3,233.28 ms | +| Incremental export | 1,272.95 ms | 771.61 ms | 189.83 ms | +| Load final dependent archive | 2,222.48 ms | 1,717.76 ms | 1,042.21 ms | +| Direct archive → eager child | 3,361.47 ms | 2,687.48 ms | 1,836.25 ms | +| Direct archive → forked child | 3,367.98 ms | 2,936.96 ms | 1,680.98 ms | +| Installed snapshot → eager child | 1,256.04 ms | 1,076.20 ms | 752.70 ms | +| Installed snapshot → forked child | 792.42 ms | 602.98 ms | 237.71 ms | +| Sum of timed test commands | 75.84 s | 52.63 s | 38.96 s | + +The size reduction is workload-dependent. Reuse is at the existing RAM-object granularity, not a new byte-diff encoding. An object can contain ranges not selected by the current memory map, so omitted object bytes are not the number of live unchanged guest bytes. Loading still reconstructs a complete local snapshot and verifies the borrowed RAM objects; a small archive does not imply equally small local storage or load work. Capture, live RAM allocation, and CoW materialization were not redesigned here. + +## macOS live invariants + +| Check | Flat | Managed/layered | Tmpfs | +| --- | --- | --- | --- | +| Standalone baseline plus 11 dependent exports | Pass | Pass | Pass | +| Every dependent export actually omits RAM payloads | Pass | Pass | Pass | +| Load checkpoints 1–11 without starting checkpoint VMs | Pass | Pass | Pass | +| Delete each prior installed base after resolving the next | Pass | Pass | Pass | +| Missing/nonexistent explicit base fails | Pass | Pass | Pass | +| Capture while paused, then resume source | Pass | Pass | Pass | +| Direct archive eager/forked restore of checkpoint 12 | Pass | Pass | Pass | +| Direct restore does not install the target snapshot | Pass | Pass | Pass | +| Restored RAM blob hash and both markers match checkpoint 12 | Pass | Pass | Pass | +| Child changes do not alter other restores or installed snapshot | Pass | Pass | Pass | +| Installed snapshot eager/forked restore after deleting its base | Pass | Pass | Pass | +| Live children retain RAM and disk writes after base deletion | Not separately injected | Pass | Pass | +| Final snapshot verification and standalone re-export | Pass | Pass | Pass | +| All test-owned VM processes stopped | Pass | Pass | Pass | + +The tmpfs root marker is RAM-backed: this exercises a dependent archive with no disk-layer omissions. The flat run stopped its direct children before deleting the base; the later managed/tmpfs runs additionally deleted the base while those children remained running. + +## Automated checks + +- `cargo test --locked -p microsandbox --no-default-features --features net,ssh --lib snapshot --offline --target-dir /private/tmp/rd-target-10`: **44 passed** in 7.43 s. +- New archive fixtures cover disk+RAM and RAM-only 12-generation chains, packed-object offsets, zero extents, changed object identities, complete CPU/device metadata, installed and standalone-archive bases, direct restore staging, base deletion, corrupted/missing RAM, malformed omissions, undeclared or unreferenced dependencies, truncated input, and standalone re-export. Existing disk-only archive and released-format regression tests remain passing. +- `cargo build --locked -p microsandbox-cli --no-default-features --features net,ssh --offline --target-dir /private/tmp/rd-target-10`: passed. +- `cargo fmt --all -- --check`, `git diff --check`, and Python smoke-script syntax compilation: passed in the isolated worktree. +- Strict Clippy is blocked by pre-existing warnings: `derivable_impls` in `crates/image/lib/snapshot/manifest.rs` and `too_many_arguments` in `sdk/rust/lib/snapshot/create.rs`. The scoped SDK check passed with `--no-deps -- -D warnings -A clippy::too_many_arguments`; no source lint allowances were added. +- A default-stack regression test exposed a large nested async future introduced by RAM verification. Boxing the buffered reader fixed it; the passing suite uses the default test stack, not an increased stack limit. + +## Reproduction and local evidence + +The repository smoke harness is `scripts/smoke/cli/incremental-full-archive.py`. Use a short, fresh output directory on macOS because runtime socket paths have a length limit: + +```bash +MSB_PATH=/path/to/codesigned/msb \ +MSB_LIBKRUNFW_PATH=/path/to/matching/libkrunfw.5.dylib \ +STACK8_OUT=/private/tmp/ram-delta-flat \ +STACK8_LAYOUT=flat:512M \ +python3 scripts/smoke/cli/incremental-full-archive.py +``` + +For managed roots use `STACK8_LAYOUT=512M`; for tmpfs use `STACK8_LAYOUT=tmpfs:128M`. An optional `STACK8_SEED_CACHE` reuses immutable OCI cache artifacts, never VM RAM caches. The harness records command logs, `results.json`, and `archive-sizes.json`, and stops only its own named VMs in cleanup. + +Successful run directories: `/private/tmp/rd-f11`, `/private/tmp/rd-m12`, and `/private/tmp/rd-t11`. These include test RAM and disk artifacts and are local evidence, not files to publish. An explicit process check after all runs found no remaining test VMs. + +Discarded fixture attempts are not counted as passes: an older installed firmware lacked the matching VMGenID readiness support; an overly long temporary path exceeded the socket-path limit; optional `--with-image` export from a flat-only cache lacked fsmeta; and `managed:512M` was not valid CLI syntax. Final runs used matching firmware, short directories, independent OCI caches, and the verified root-disk syntax. The `--with-image` cache limitation was not fixed by this archive dependency change. + +## Linux x86-64/KVM follow-up + +All three root layouts passed the same 12-checkpoint live harness on OVH, including deleting the supplied base while direct eager and forked children remained running. Each run exported the baseline standalone, exported checkpoints 2–12 with `--since`, loaded 1–11 sequentially with `--base`, and restored checkpoint 12 both directly from its archive and from its installed snapshot. Guest reads verified the checkpoint-12 RAM/root markers and the original 24 MiB RAM-file hash. Child writes remained private. Missing bases failed, capture while explicitly paused passed, and final verification and standalone re-export passed. No additional implementation fix was needed for Linux. + +### Linux build and storage + +- Same isolated `18c8fb8693957ac6e8ded964e42cfbe63f8506ad` baseline plus archive edits; no concurrent #8 lifecycle changes. The transferred `delta.rs` SHA-256 matched the Mac source: `25894fc39499d9d664c02d6bd126dfd73a3c6c325c55d4f39fd879c23af6460e`. +- Host kernel: `7.0.0-28-generic`, x86-64/KVM. Rust: `1.97.1`. Guest fixture sizes and workload match the Mac tests. +- Rebuilt the matching static x86-64 guest agent from source; its release build took 20.83 s. SHA-256: `31c648803053be07bc4dc8491b2e16035b44dbf79d21da097a69e609c2658814`. +- Debug CLI build with `--locked --no-default-features --features net,ssh --offline` took 38.62 s. SHA-256: `9bdd6c7ed090520be31e95da7776655339fd62d819640f4a8c87af5553acd10b`. +- Matching existing firmware SHA-256: `6acfb3c81238e64f60ab1dcac95e7b2c2c57161a5e6ab10fbab18e457a205d80`. +- Final archive, cache, sandbox, and temporary staging directories were on the host's ext4 `/dev/md3` filesystem. `TMPDIR=/home/ubuntu/rdl.pa6szm` kept SDK temporary staging and launch-configuration files there too. These results are not from host tmpfs storage; the guest tmpfs-root case still correctly stores its filesystem contents in guest RAM. +- The Linux snapshot regression suite passed **44/44 tests** in 0.45 s with the same test command and features as the Mac. The new host build and all live tests used the rebuilt matching agent. + +### Linux measurements + +One sequential debug-build run per layout, warm filesystem caches; these are end-to-end CLI times, not pause durations or release latency claims. Forked restores ran after eager restores. The final standalone and incremental exports use the same checkpoint 12. + +| Checkpoint 12 result | Flat | Managed/layered | Tmpfs | +| --- | ---: | ---: | ---: | +| Standalone archive bytes | 43,430,398 | 39,755,415 | 39,378,027 | +| Incremental archive bytes | 258,923 | 257,546 | 232,087 | +| Archive reduction | 99.40% | 99.35% | 99.41% | +| Omitted RAM objects | 13 | 13 | 13 | +| Omitted RAM object bytes | 112,947,200 | 121,892,864 | 106,696,704 | +| Full capture | 128.87 ms | 125.26 ms | 84.44 ms | +| Standalone export | 992.76 ms | 900.48 ms | 838.71 ms | +| Incremental export | 139.09 ms | 128.20 ms | 104.27 ms | +| Load final dependent archive | 1,592.66 ms | 1,690.74 ms | 1,459.00 ms | +| Direct archive → eager child | 2,382.62 ms | 2,501.35 ms | 2,179.46 ms | +| Direct archive → forked child | 2,451.04 ms | 2,565.34 ms | 2,244.45 ms | +| Installed snapshot → eager child | 806.07 ms | 809.40 ms | 732.56 ms | +| Installed snapshot → forked child | 235.81 ms | 239.48 ms | 198.96 ms | +| Sum of timed test commands | 50.36 s | 44.48 s | 40.72 s | + +### Linux evidence and fixture corrections + +The isolated source/build directory is `/home/ubuntu/ram-delta-linux.mdyJh7`. Successful disk-backed run directories are `/home/ubuntu/rdl.pa6szm/{flat-2,layered-2,tmpfs-2}`. Timing and size JSON files were copied to `/private/tmp/rd-linux-evidence-10/{flat,layered,tmpfs}` on the Mac; raw RAM and disk artifacts were not copied or published. The final process check found no remaining runtime using the isolated test binary, including the diagnostic probe. + +An initial long runtime path exceeded the Unix socket-path limit. A subsequent flat-root run under `/tmp/rdl.W5axYo/flat` passed, but `/tmp` on this host is RAM-backed, so its timings are excluded from the table. The first disk-backed rerun then encountered `EDQUOT` while writing an anonymous launch-configuration file in `/tmp`, not while writing a snapshot. A scoped syscall trace identified that location. Setting only the test process's `TMPDIR` to the private disk-backed directory resolved it; no host quota, system configuration, or unrelated files were changed. The three final disk-backed runs above completed successfully. diff --git a/sdk/go/snapshot.go b/sdk/go/snapshot.go index 0ce93f52f..ebbe5a44f 100644 --- a/sdk/go/snapshot.go +++ b/sdk/go/snapshot.go @@ -30,7 +30,7 @@ type SnapshotCreateOptions struct { // SnapshotSaveOptions configures Snapshot.Save. type SnapshotSaveOptions struct { - // Since identifies an exact base snapshot or standalone base archive. + // Since omits disk layers and RAM objects supplied by a base snapshot or standalone archive. Since string // LastLayers includes the newest N sealed disk layers. Mutually exclusive with Since. LastLayers *uint32 diff --git a/sdk/node-ts/native/index.d.ts b/sdk/node-ts/native/index.d.ts index e9ba9429c..43bcf450f 100644 --- a/sdk/node-ts/native/index.d.ts +++ b/sdk/node-ts/native/index.d.ts @@ -1074,7 +1074,7 @@ export declare class SandboxBuilder { * snapshot already pins the image reference and digest. */ fromSnapshot(pathOrName: string): this - /** Supply the exact base for a disk-dependent snapshot archive. */ + /** Supply the base for omitted disk layers and RAM objects in a snapshot archive. */ snapshotBase(base: string): this /** Cold-boot only the disk state carried by a full snapshot. */ diskOnly(): this diff --git a/sdk/node-ts/native/sandbox_builder.rs b/sdk/node-ts/native/sandbox_builder.rs index 50ab270ed..08a51da0e 100644 --- a/sdk/node-ts/native/sandbox_builder.rs +++ b/sdk/node-ts/native/sandbox_builder.rs @@ -142,7 +142,7 @@ impl JsSandboxBuilder { self } - /// Supply the exact base for a disk-dependent snapshot archive. + /// Supply the base for omitted disk layers and RAM objects in a snapshot archive. #[napi] pub fn snapshot_base(&mut self, base: String) -> &Self { let prev = self.take_inner(); diff --git a/sdk/node-ts/native/snapshot.rs b/sdk/node-ts/native/snapshot.rs index beec3ecd5..383658413 100644 --- a/sdk/node-ts/native/snapshot.rs +++ b/sdk/node-ts/native/snapshot.rs @@ -45,7 +45,7 @@ pub struct JsSaveOpts { pub with_image: Option, /// Skip zstd compression and write a plain `.tar`. pub plain_tar: Option, - /// Exact base snapshot or standalone archive for incremental disk export. + /// Base snapshot or standalone archive supplying reusable disk layers and RAM objects. pub since: Option, /// Newest N immutable disk layers to include. pub last_layers: Option, diff --git a/sdk/node-ts/src/snapshot.ts b/sdk/node-ts/src/snapshot.ts index ee32daa4d..c5486f9ba 100644 --- a/sdk/node-ts/src/snapshot.ts +++ b/sdk/node-ts/src/snapshot.ts @@ -52,7 +52,7 @@ export type SnapshotState = * Bundle options for `Snapshot.save`. */ export interface SaveOpts { - /** Exact base snapshot or standalone archive; mutually exclusive with lastLayers/withParents. */ + /** Omit disk layers and RAM objects supplied by this base; mutually exclusive with lastLayers/withParents. */ since?: string; /** Newest N sealed disk layers. Full snapshots still include all memory/device state. */ lastLayers?: number; diff --git a/sdk/python/src/snapshot.rs b/sdk/python/src/snapshot.rs index 9775d754b..560dc4e8d 100644 --- a/sdk/python/src/snapshot.rs +++ b/sdk/python/src/snapshot.rs @@ -228,6 +228,9 @@ impl PySnapshot { /// Bundle a snapshot into a `.tar.zst` archive. /// + /// `since` omits disk layers and RAM objects supplied by the base; `last_layers` only + /// selects disk layers. Dependent archives require a base when loaded or restored. + /// /// The recorded manifest is archived as-is, so create the snapshot /// with `record_integrity=True` if receivers must verify content. #[staticmethod] diff --git a/sdk/rust/lib/sandbox/builder.rs b/sdk/rust/lib/sandbox/builder.rs index 5a2a9b6d8..cca630ac1 100644 --- a/sdk/rust/lib/sandbox/builder.rs +++ b/sdk/rust/lib/sandbox/builder.rs @@ -1158,7 +1158,7 @@ impl SandboxBuilder { self } - /// Supply the exact base snapshot or standalone base archive for a disk-dependent archive. + /// Supply the base snapshot or standalone archive for omitted disk layers and RAM objects. pub fn snapshot_base(mut self, base: impl Into) -> Self { self.config.snapshot_base = Some(base.into()); self diff --git a/sdk/rust/lib/snapshot/archive.rs b/sdk/rust/lib/snapshot/archive.rs index a3278cf8c..ff88cb39e 100644 --- a/sdk/rust/lib/snapshot/archive.rs +++ b/sdk/rust/lib/snapshot/archive.rs @@ -70,7 +70,8 @@ pub struct SaveOpts { pub with_image: bool, /// Skip zstd compression and write a plain `.tar`. Default: zstd. pub plain_tar: bool, - /// Export only disk layers after this exact base snapshot (name, directory, or archive). + /// Omit disk layers and RAM objects supplied by this base (name, directory, or archive). + /// The base must be an exact physical disk prefix; full-snapshot metadata stays complete. /// Mutually exclusive with `last_layers` and `with_parents`. pub since: Option, /// Export the newest N sealed disk layers, requiring an explicit base when loading omissions. @@ -1327,7 +1328,7 @@ async fn write_archive_entries( cache_files: &[(PathBuf, String)], head: &Snapshot, opts: &SaveOpts, - dependencies: Option<&delta::DiskDependencies>, + dependencies: Option<&delta::Dependencies>, ) -> MicrosandboxResult<()> where W: tokio::io::AsyncWrite + Unpin + Send, @@ -2864,7 +2865,7 @@ async fn validate_archive_inventory( } if !matches!( inventory.completeness.as_str(), - "boot-complete" | "disk-dependent" + "boot-complete" | "dependent" ) { return Err(MicrosandboxError::unsupported( Operation::SnapshotOps, diff --git a/sdk/rust/lib/snapshot/archive/delta.rs b/sdk/rust/lib/snapshot/archive/delta.rs index acb537562..41665886b 100644 --- a/sdk/rust/lib/snapshot/archive/delta.rs +++ b/sdk/rust/lib/snapshot/archive/delta.rs @@ -1,4 +1,4 @@ -//! Exact physical-prefix dependencies for explicitly incremental disk exports. +//! Explicit disk-prefix and immutable RAM-object dependencies for incremental exports. use microsandbox_image::checkpoint::{DiskLayerExportPlan, DiskLayerRef}; use microsandbox_image::snapshot::{DiskLayer, Manifest}; @@ -9,7 +9,7 @@ use super::*; // Constants //-------------------------------------------------------------------------------------------------- -pub(super) const REQUIREMENT: &str = "msb-disk-layer-dependencies-v1"; +pub(super) const REQUIREMENT: &str = "msb-snapshot-dependencies-v1"; //-------------------------------------------------------------------------------------------------- // Types @@ -36,8 +36,9 @@ struct RequiredLayer { #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] -pub(super) struct DiskDependencies { - required: Vec, +pub(super) struct Dependencies { + disks: Vec, + memory: Vec, } struct PhysicalLayer { @@ -47,7 +48,7 @@ struct PhysicalLayer { struct BaseSnapshot { snapshot: Snapshot, - // Keep archive staging alive until all required layers have been copied to the destination. + // Keep archive staging alive until all required payloads belong to the destination. _stage: Option, } @@ -59,64 +60,84 @@ pub(super) async fn selection( local: &LocalBackend, head: &Snapshot, opts: &SaveOpts, -) -> MicrosandboxResult> { +) -> MicrosandboxResult> { if opts.since.is_none() && opts.last_layers.is_none() { return Ok(None); } if opts.with_parents || (opts.since.is_some() && opts.last_layers.is_some()) { return Err(MicrosandboxError::InvalidConfig( - "disk-layer export takes either since or last_layers, without with_parents".into(), + "incremental export takes either since or last_layers, without with_parents".into(), )); } let layers = physical_layers(head.manifest(), head.path())?; - let plan = if let Some(base) = &opts.since { - let base = open_base(local, base).await?; + let mut memory = Vec::new(); + let required = if let Some(base) = &opts.since { + // Base archives carry buffered decoder/verification futures; keep them off the caller's + // stack, including when this planner is nested inside a direct restore or SDK call. + let base = Box::pin(open_base(local, base)).await?; let baseline = physical_layers(base.snapshot.manifest(), base.snapshot.path())?; - DiskLayerExportPlan::since( - &layers - .iter() - .map(|layer| &layer.required.identity) - .collect::>(), - &baseline - .iter() - .map(|layer| &layer.required.identity) - .collect::>(), - ) + let available = memory_objects(&base.snapshot)?; + memory = memory_objects(head)? + .intersection(&available) + .cloned() + .collect(); + // Tmpfs-root full snapshots have no disks, but may still depend on RAM objects. + // Do not let disk completeness suppress an independent memory dependency. + if layers.is_empty() && baseline.is_empty() { + 0..0 + } else { + DiskLayerExportPlan::since( + &layers + .iter() + .map(|layer| &layer.required.identity) + .collect::>(), + &baseline + .iter() + .map(|layer| &layer.required.identity) + .collect::>(), + ) + .map_err(|error| MicrosandboxError::InvalidConfig(error.to_string()))? + .required() + } } else { DiskLayerExportPlan::last(layers.len(), opts.last_layers.expect("selector checked")) - } - .map_err(|error| MicrosandboxError::InvalidConfig(error.to_string()))?; - if plan.is_disk_complete() { + .map_err(|error| MicrosandboxError::InvalidConfig(error.to_string()))? + .required() + }; + if required.is_empty() && memory.is_empty() { return Ok(None); } - Ok(Some(DiskDependencies { - required: layers[plan.required()] + Ok(Some(Dependencies { + disks: layers[required] .iter() .map(|layer| layer.required.clone()) .collect(), + memory, })) } pub(super) fn apply( inventory: &mut ArchiveInventory, - dependencies: &DiskDependencies, + dependencies: &Dependencies, ) -> MicrosandboxResult<()> { - for required in &dependencies.required { - let entry = inventory - .entries - .iter_mut() - .find(|entry| entry.path == required.path) - .ok_or_else(|| { - MicrosandboxError::SnapshotIntegrity( - "required disk layer is absent from archive inventory".into(), - ) - })?; + let paths = dependency_paths(&inventory.head, dependencies); + let mut found = 0; + for entry in &mut inventory.entries { + if !paths.contains(entry.path.as_str()) { + continue; + } + found += 1; entry.included = false; entry.encoded_size = 0; entry.sparse_ranges.clear(); entry.transport_integrity = None; } - inventory.completeness = "disk-dependent".into(); + if found != paths.len() { + return Err(MicrosandboxError::SnapshotIntegrity( + "required payload is absent from archive inventory".into(), + )); + } + inventory.completeness = "dependent".into(); inventory.requires.push(REQUIREMENT.into()); inventory.requires.sort(); inventory @@ -142,14 +163,9 @@ pub(super) fn apply( Ok(()) } -pub(super) fn validate( - inventory: &ArchiveInventory, -) -> MicrosandboxResult> { +pub(super) fn validate(inventory: &ArchiveInventory) -> MicrosandboxResult> { let extension = inventory.extensions.get(REQUIREMENT); - let required = inventory - .requires - .iter() - .any(|requirement| requirement == REQUIREMENT); + let required = inventory.requires.iter().any(|value| value == REQUIREMENT); if inventory.completeness == "boot-complete" && !required && extension.is_none() { if inventory.entries.iter().any(|entry| !entry.included) { return Err(MicrosandboxError::SnapshotIntegrity( @@ -158,41 +174,60 @@ pub(super) fn validate( } return Ok(None); } - if inventory.completeness != "disk-dependent" || !required || extension.is_none() { + if inventory.completeness != "dependent" || !required || extension.is_none() { return Err(MicrosandboxError::SnapshotIntegrity( - "invalid disk dependency capability/completeness binding".into(), + "invalid snapshot dependency capability/completeness binding".into(), )); } - let dependencies: DiskDependencies = serde_json::from_value(extension.unwrap().clone())?; - if dependencies.required.is_empty() || dependencies.required.len() > 256 { + let dependencies: Dependencies = serde_json::from_value(extension.unwrap().clone())?; + if (dependencies.disks.is_empty() && dependencies.memory.is_empty()) + || dependencies.disks.len() > 256 + || dependencies.memory.len() > inventory.entries.len() + || dependencies + .memory + .windows(2) + .any(|pair| pair[0] >= pair[1]) + { return Err(MicrosandboxError::SnapshotIntegrity( - "invalid disk dependency count".into(), + "invalid snapshot dependency count or object ordering".into(), )); } - let mut paths = HashSet::new(); - for layer in &dependencies.required { - let entry = inventory - .entries - .iter() - .find(|entry| entry.path == layer.path) - .ok_or_else(|| { - MicrosandboxError::SnapshotIntegrity( - "disk dependency lacks an inventory entry".into(), - ) - })?; - if entry.included - || !matches!( + let entries: HashMap<_, _> = inventory + .entries + .iter() + .map(|entry| (entry.path.as_str(), entry)) + .collect(); + let paths = dependency_paths(&inventory.head, &dependencies); + if paths.len() != dependencies.disks.len() + dependencies.memory.len() { + return Err(MicrosandboxError::SnapshotIntegrity( + "duplicate snapshot dependency".into(), + )); + } + for path in &paths { + let entry = entries.get(path.as_str()).ok_or_else(|| { + MicrosandboxError::SnapshotIntegrity("dependency lacks an inventory entry".into()) + })?; + let is_memory = dependencies + .memory + .binary_search_by(|id| memory_archive_path(&inventory.head, id).cmp(path)) + .is_ok(); + let valid_kind = if is_memory { + entry.kind == "checkpoint-object" + } else { + matches!( entry.kind.as_str(), "file-payload" | "checkpoint-disk-layer" ) + }; + if entry.included + || !valid_kind || entry.owner_snapshot.as_deref() != Some(inventory.head.as_str()) || entry.encoded_size != 0 || !entry.sparse_ranges.is_empty() || entry.transport_integrity.is_some() - || !paths.insert(&layer.path) { return Err(MicrosandboxError::SnapshotIntegrity( - "invalid omitted disk-layer binding".into(), + "invalid omitted payload binding".into(), )); } } @@ -204,13 +239,13 @@ pub(super) fn validate( != paths.len() { return Err(MicrosandboxError::SnapshotIntegrity( - "archive omits a non-disk dependency".into(), + "archive omits an undeclared dependency".into(), )); } Ok(Some(dependencies)) } -/// Resolve only a caller-supplied base; never search ambient directories or qcow backing paths. +/// Resolve only a caller-supplied base; never search ambient directories or backing paths. pub(super) async fn resolve( local: &LocalBackend, inventory: &ArchiveInventory, @@ -221,56 +256,203 @@ pub(super) async fn resolve( let Some(dependencies) = validate(inventory)? else { return Ok(()); }; - let base = base.ok_or_else(|| MicrosandboxError::InvalidConfig( - "this disk-dependent archive requires an explicit base snapshot or standalone base archive".into(), - ))?; - let base = open_base(local, base).await?; + let base = base.ok_or_else(|| { + MicrosandboxError::InvalidConfig( + "this dependent archive requires an explicit base snapshot or standalone base archive" + .into(), + ) + })?; + let base = Box::pin(open_base(local, base)).await?; let available = physical_layers(base.snapshot.manifest(), base.snapshot.path())?; - if available.len() != dependencies.required.len() - || available - .iter() - .zip(&dependencies.required) - .any(|(layer, required)| layer.required.identity != required.identity) + if !dependencies.disks.is_empty() + && (available.len() != dependencies.disks.len() + || available + .iter() + .zip(&dependencies.disks) + .any(|(layer, required)| layer.required.identity != required.identity)) { return Err(MicrosandboxError::SnapshotIntegrity( "supplied base is not the exact required physical disk prefix".into(), )); } - // Copy into operation-owned staging. Imported artifacts must survive deleting the supplied - // base and must never inherit a writable hardlink into another sandbox. - for (source, required) in available.iter().zip(&dependencies.required) { + let available_memory = memory_objects(&base.snapshot)?; + if dependencies + .memory + .iter() + .any(|id| !available_memory.contains(id)) + { + return Err(MicrosandboxError::SnapshotIntegrity( + "supplied base does not contain the required RAM objects".into(), + )); + } + + // Every dependency is copied into operation-owned staging. In particular, a restored child + // must not inherit a writable hardlink into the base; deleting the base must be harmless. + for (source, required) in available.iter().zip(&dependencies.disks) { let target = inventory_entry_target(&required.path, snapshots_dir, cache_dir)?; - if target.exists() { - return Err(MicrosandboxError::SnapshotIntegrity( - "dependency collides with an extracted member".into(), - )); - } - if let Some(parent) = target.parent() { - tokio::fs::create_dir_all(parent).await?; + copy_dependency(&source.source, &target).await?; + } + for id in &dependencies.memory { + let source = checkpoint_object_path(&base.snapshot.path().join(CHECKPOINT_DIRECTORY), id); + let target = inventory_entry_target( + &memory_archive_path(&inventory.head, id), + snapshots_dir, + cache_dir, + )?; + copy_dependency(&source, &target).await?; + // Verify only the objects actually borrowed, in the destination-owned copy. Export + // selection is metadata-only for RAM; it must not scan the base's entire guest memory. + // This reader owns a 64 KiB buffer; boxing prevents every enclosing archive/SDK + // future from embedding another copy of that buffer in its own stack frame. + let actual = format!( + "sha256:{}", + hex::encode(Box::pin(file_sha256(&target)).await?) + ); + if actual != id.as_str() { + return Err(MicrosandboxError::SnapshotIntegrity(format!( + "base RAM object content does not match {id}" + ))); } - let source = source.source.clone(); - tokio::task::spawn_blocking(move || microsandbox_utils::copy::fast_copy(&source, &target)) - .await - .map_err(|error| MicrosandboxError::Runtime(format!("base layer copy: {error}")))??; } + + // Open the complete target only after filling omissions. This retains its normal metadata, + // range, epoch and disk-integrity validation instead of introducing a partial-closure mode. let artifact = snapshots_dir.join(&inventory.head); let manifest = Manifest::from_bytes(&tokio::fs::read(artifact.join(DESCRIPTOR_FILENAME)).await?) .map_err(|error| MicrosandboxError::SnapshotIntegrity(error.to_string()))?; let target = physical_layers(&manifest, &artifact)?; - if target.len() < dependencies.required.len() + if target.len() < dependencies.disks.len() || target .iter() - .zip(&dependencies.required) + .zip(&dependencies.disks) .any(|(layer, required)| &layer.required != required) { return Err(MicrosandboxError::SnapshotIntegrity( "dependency list is not the target descriptor's exact disk prefix".into(), )); } + let target_memory = if dependencies.memory.is_empty() { + BTreeSet::new() + } else { + let snapshot = store::open_snapshot(local, artifact.to_string_lossy().as_ref()).await?; + memory_objects(&snapshot)? + }; + if dependencies + .memory + .iter() + .any(|id| !target_memory.contains(id)) + { + return Err(MicrosandboxError::SnapshotIntegrity( + "omitted object is not a target RAM payload; metadata must remain included".into(), + )); + } + let entries: HashMap<_, _> = inventory + .entries + .iter() + .map(|entry| (entry.path.as_str(), entry)) + .collect(); + for id in &dependencies.memory { + let path = memory_archive_path(&inventory.head, id); + let entry = entries + .get(path.as_str()) + .expect("dependency inventory was validated"); + let target = inventory_entry_target(&path, snapshots_dir, cache_dir)?; + if tokio::fs::metadata(target).await?.len() != entry.apparent_size { + return Err(MicrosandboxError::SnapshotIntegrity( + "resolved RAM object size differs from inventory".into(), + )); + } + } Ok(()) } +async fn copy_dependency(source: &Path, target: &Path) -> MicrosandboxResult<()> { + if tokio::fs::symlink_metadata(target).await.is_ok() { + return Err(MicrosandboxError::SnapshotIntegrity( + "dependency collides with an extracted member".into(), + )); + } + if let Some(parent) = target.parent() { + tokio::fs::create_dir_all(parent).await?; + } + let source = source.to_path_buf(); + let target = target.to_path_buf(); + tokio::task::spawn_blocking(move || microsandbox_utils::copy::fast_copy(&source, &target)) + .await + .map_err(|error| MicrosandboxError::Runtime(format!("base payload copy: {error}")))??; + Ok(()) +} + +fn memory_archive_path(snapshot_id: &str, id: &ObjectId) -> String { + let hash = id + .as_str() + .strip_prefix("sha256:") + .expect("validated ObjectId"); + format!( + "checkpoints/{snapshot_id}/objects/sha256/{}/{hash}", + &hash[..2] + ) +} + +fn checkpoint_object_path(root: &Path, id: &ObjectId) -> PathBuf { + let hash = id + .as_str() + .strip_prefix("sha256:") + .expect("validated ObjectId"); + root.join("objects") + .join("sha256") + .join(&hash[..2]) + .join(hash) +} + +fn dependency_paths(head: &str, dependencies: &Dependencies) -> BTreeSet { + dependencies + .disks + .iter() + .map(|layer| layer.path.clone()) + .chain( + dependencies + .memory + .iter() + .map(|id| memory_archive_path(head, id)), + ) + .collect() +} + +/// Return reusable RAM payload IDs, never metadata objects, even if bytes happen to coincide. +fn memory_objects(snapshot: &Snapshot) -> MicrosandboxResult> { + let SnapshotState::Checkpoint(state) = &snapshot.manifest().state else { + return Ok(BTreeSet::new()); + }; + let expected = ObjectId::new(&state.checkpoint_root) + .map_err(|error| MicrosandboxError::SnapshotIntegrity(error.to_string()))?; + let closure = CheckpointClosure::open_portable( + snapshot.path().join(CHECKPOINT_DIRECTORY), + Some(&expected), + ) + .map_err(|error| MicrosandboxError::SnapshotIntegrity(error.to_string()))?; + let checkpoint = closure.checkpoint(); + let mut objects: BTreeSet<_> = closure + .memory() + .extents + .iter() + .filter_map(|extent| match &extent.content { + MemoryExtentContent::Object(content) => Some(content.object.clone()), + MemoryExtentContent::Zero => None, + }) + .collect(); + objects.remove(&checkpoint.memory); + objects.remove(&checkpoint.execution_state); + for id in &checkpoint.disks { + objects.remove(id); + } + for device in &checkpoint.devices { + objects.remove(&device.state); + } + Ok(objects) +} + fn physical_layers( manifest: &Manifest, directory: &Path, @@ -295,14 +477,15 @@ fn physical_layers( .map_err(|error| MicrosandboxError::SnapshotIntegrity(error.to_string()))?; let closure = CheckpointClosure::open_portable(&root, Some(&expected)) .map_err(|error| MicrosandboxError::SnapshotIntegrity(error.to_string()))?; - if closure.disks().len() != 1 { + if closure.disks().len() > 1 { return Err(MicrosandboxError::InvalidConfig( - "disk-layer selection requires exactly one checkpoint disk".into(), + "disk-layer selection supports at most one checkpoint disk".into(), )); } - Ok(closure.disks()[0] - .layers + Ok(closure + .disks() .iter() + .flat_map(|disk| &disk.layers) .map(|layer| PhysicalLayer { required: RequiredLayer { path: format!( @@ -322,7 +505,9 @@ async fn open_base(local: &LocalBackend, input: &str) -> MicrosandboxResult MicrosandboxResult MicrosandboxResult, + disk: bool, +) -> Snapshot { + let root = path.join(CHECKPOINT_DIRECTORY); + let store = LocalObjectStore::open(&root).unwrap(); + // Packed objects deliberately contain bytes not selected by the map. Export is object-based. + let original = store.put_bytes(&vec![0xa5; 8192]).unwrap(); + let changed = store + .put_bytes(&vec![(generation / 3) as u8; 8192]) + .unwrap(); + let memory = MemoryManifest { + schema: "microsandbox.memory/1".into(), + architecture: std::env::consts::ARCH.into(), + guest_page_size: 4096, + topology_generation: 1, + generation, + capture_mode: if generation == 1 { + MemoryCaptureMode::Full + } else { + MemoryCaptureMode::Incremental + }, + pause_generation: generation, + extents: vec![ + MemoryExtent { + start: 0, + length: 4096, + content: MemoryExtentContent::Object(ContentRef { + object: original, + object_offset: 4096, + }), + }, + MemoryExtent { + start: 4096, + length: 4096, + content: MemoryExtentContent::Object(ContentRef { + object: changed, + object_offset: 4096, + }), + }, + MemoryExtent { + start: 8192, + length: 4096, + content: MemoryExtentContent::Zero, + }, + ], + }; + let mut disks = Vec::new(); + if disk { + std::fs::create_dir_all(root.join("layers")).unwrap(); + let mut layers = Vec::new(); + if let Some(previous) = previous { + for old in physical_layers(previous.manifest(), previous.path()).unwrap() { + let LayerIdentity::Checkpoint(layer) = old.required.identity else { + unreachable!() + }; + let dest = root + .join("layers") + .join(format!("{}.{}", layer.layer_id, layer.format)); + microsandbox_utils::copy::fast_copy(&old.source, &dest).unwrap(); + layers.push(layer); + } + } + let layer_id = format!("layer_{generation:032x}"); + let format = if layers.is_empty() { "raw" } else { "qcow2" }; + let layer_path = root.join("layers").join(format!("{layer_id}.{format}")); + if let Some(base) = layers.last() { + microsandbox_image::checkpoint::create_qcow2_overlay( + &layer_path, + 65536, + &root + .join("layers") + .join(format!("{}.{}", base.layer_id, base.format)), + &base.format, + ) + .await + .unwrap(); + } else { + std::fs::write(&layer_path, vec![17u8; 65536]).unwrap(); + } + let layer = DiskLayerRef { + layer_id: layer_id.clone(), + format: format.into(), + virtual_size: 65536, + predecessor: layers.last().map(|layer| layer.layer_id.clone()), + integrity_root: sparse_file_integrity(&layer_path).unwrap().root, + }; + layers.push(layer); + let disk = DiskGenerationManifest { + schema: "microsandbox.disk-generation/1".into(), + volume_id: "vol_test".into(), + device_id: "vdb".into(), + generation, + layers, + head: layer_id, + pause_generation: generation, + }; + disks.push( + store + .put_bytes(&disk.to_canonical_bytes().unwrap()) + .unwrap(), + ); + } + let checkpoint = CheckpointManifest { + schema: "microsandbox.checkpoint/1".into(), + checkpoint_id: format!("checkpoint_{generation}"), + capture_intent: CaptureIntent::FullSnapshot, + architecture: std::env::consts::ARCH.into(), + pause_generation: generation, + execution_state: store + .put_bytes(format!("execution-{generation}").as_bytes()) + .unwrap(), + memory: store + .put_bytes(&memory.to_canonical_bytes().unwrap()) + .unwrap(), + disks, + devices: vec![DeviceStateRef { + device_type: 4, + device_id: "rng".into(), + state: store.put_bytes(b"unchanged-device-state").unwrap(), + }], + resources: Vec::new(), + requires: Vec::new(), + }; + let bytes = checkpoint.to_canonical_bytes().unwrap(); + let root_id = ObjectId::from_bytes(&bytes).unwrap(); + std::fs::write(root.join("checkpoint.json"), bytes).unwrap(); + let manifest = Manifest { + schema: "microsandbox.snapshot/1".into(), + snapshot_id: SnapshotId::new(format!("snap_{generation:032x}")).unwrap(), + scope: SnapshotScope::Full, + root_disk: if disk { + SnapshotRootDisk::Managed + } else { + SnapshotRootDisk::Tmpfs { size_mib: None } + }, + state: SnapshotState::Checkpoint(CheckpointSnapshotState { + checkpoint_id: checkpoint.checkpoint_id, + checkpoint_root: root_id.to_string(), + restore_intents: vec!["clone".into(), "resume".into()], + requirements_summary: BTreeMap::from([ + ("vcpus".into(), 1.into()), + ("max_vcpus".into(), 1.into()), + ("memory_mib".into(), 128.into()), + ("max_memory_mib".into(), 128.into()), + ]), + }), + capture: SnapshotCapture { + created_at: "2026-09-10T00:00:00Z".into(), + source_lineage: None, + source_checkpoint: None, + consistency: SnapshotConsistency::ApplicationConsistent, + }, + image: ImageRef { + reference: "docker.io/library/alpine:3.20".into(), + manifest_digest: format!("sha256:{}", "0".repeat(64)), + }, + parent: None, + extensions: BTreeMap::new(), + requires: Vec::new(), + }; + std::fs::write( + path.join(DESCRIPTOR_FILENAME), + manifest.to_canonical_bytes().unwrap(), + ) + .unwrap(); + store::open_snapshot(local, path.to_str().unwrap()) + .await + .unwrap() +} + +fn assert_ram(path: &Path, generation: u64) { + let closure = CheckpointClosure::open_portable(path.join(CHECKPOINT_DIRECTORY), None).unwrap(); + closure.verify_memory_objects().unwrap(); + let mut ram = Vec::new(); + for extent in &closure.memory().extents { + match &extent.content { + MemoryExtentContent::Zero => ram.extend(vec![0; extent.length as usize]), + MemoryExtentContent::Object(content) => { + let object = closure.read_object(&content.object, 8192).unwrap(); + let start = content.object_offset as usize; + ram.extend_from_slice(&object[start..start + extent.length as usize]); + } + } + } + assert_eq!(&ram[..4096], &vec![0xa5; 4096]); + assert_eq!(&ram[4096..8192], &vec![(generation / 3) as u8; 4096]); + assert_eq!(&ram[8192..], &vec![0; 4096]); + assert_eq!( + closure + .read_object(&closure.checkpoint().execution_state, 128) + .unwrap(), + format!("execution-{generation}").as_bytes() + ); +} + +async fn unpack(path: &Path, stage: &Path) -> ArchiveInventory { + let file = BufReader::new(tokio::fs::File::open(path).await.unwrap()); + let cache = stage.join("cache"); + tokio::fs::create_dir_all(&cache).await.unwrap(); + // Tests use plain tar to inspect exactly which payloads were physically transported. + unpack_archive(file, stage, &cache) + .await + .unwrap() + .inventory + .unwrap() +} + +async fn chain(disk: bool) { + let temp = tempfile::tempdir().unwrap(); + let local = LocalBackend::builder() + .home(temp.path().join("home")) + .build() + .await + .unwrap(); + let mut previous = None; + let mut loaded: Option = None; + for generation in 1..=12 { + let source = fixture( + &local, + &temp.path().join(format!("source-{generation}")), + generation, + previous.as_ref(), + disk, + ) + .await; + let archive = temp.path().join(format!("cp{generation:02}.msnap")); + save_snapshot( + &local, + source.path().to_str().unwrap(), + &archive, + SaveOpts { + since: previous + .as_ref() + .map(|snapshot: &Snapshot| snapshot.path().to_string_lossy().into_owned()), + plain_tar: true, + ..Default::default() + }, + ) + .await + .unwrap(); + let stage = temp.path().join(format!("unpacked-{generation}")); + let inventory = unpack(&archive, &stage).await; + if generation > 1 { + let dependencies = validate(&inventory).unwrap().unwrap(); + assert_eq!( + dependencies.disks.len(), + if disk { generation as usize - 1 } else { 0 } + ); + assert!(!dependencies.memory.is_empty()); + for id in &dependencies.memory { + let path = memory_archive_path(source.id().as_str(), id); + assert!( + !inventory_entry_target(&path, &stage, &stage.join("cache")) + .unwrap() + .exists() + ); + } + let closure = + CheckpointClosure::open_portable(source.path().join(CHECKPOINT_DIRECTORY), None) + .unwrap(); + for id in [ + &closure.checkpoint().memory, + &closure.checkpoint().execution_state, + &closure.checkpoint().devices[0].state, + ] { + assert!(inventory.entries.iter().any(|entry| entry.path + == memory_archive_path(source.id().as_str(), id) + && entry.included)); + } + assert!(load_snapshot(&local, &archive, None).await.is_err()); + } else { + assert!(validate(&inventory).unwrap().is_none()); + } + let base = loaded + .as_ref() + .map(|snapshot| snapshot.path().to_str().unwrap()); + if generation == 12 { + // Direct archive restore must use the same dependency resolver, without installation. + let child = temp.path().join("child"); + let result = + materialize_archive_for_child_with_base(&local, &archive, &child, false, base) + .await + .unwrap(); + assert!(result.checkpoint_restore.is_some()); + let closure = + CheckpointClosure::open_portable(child.join(".checkpoint-restore"), None).unwrap(); + closure.verify_memory_objects().unwrap(); + assert!(!local.snapshots_dir().join(source.id().as_str()).exists()); + } + let current = load_snapshot_with_base(&local, &archive, None, base) + .await + .unwrap(); + assert_ram(current.path(), generation); + if let Some(old) = loaded.take() { + // These are exclusively test-owned artifacts; later loads cannot depend on their paths. + std::fs::remove_dir_all(old.path()).unwrap(); + assert_ram(current.path(), generation); + } + loaded = Some(current); + previous = Some(source); + } + let final_snapshot = loaded.unwrap(); + let standalone = temp.path().join("standalone.msnap"); + save_snapshot( + &local, + final_snapshot.path().to_str().unwrap(), + &standalone, + SaveOpts::default(), + ) + .await + .unwrap(); + let other = load_snapshot(&local, &standalone, Some(&temp.path().join("other-host"))) + .await + .unwrap(); + assert_ram(other.path(), 12); +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[tokio::test] +async fn twelve_ram_only_archives_resolve_without_intermediate_vms() { + chain(false).await; +} + +#[tokio::test] +async fn twelve_disk_and_ram_archives_resolve_without_intermediate_vms() { + chain(true).await; +} + +#[tokio::test] +async fn last_layers_keeps_ram_complete_and_wrong_ram_base_fails() { + let temp = tempfile::tempdir().unwrap(); + let local = LocalBackend::builder() + .home(temp.path().join("home")) + .build() + .await + .unwrap(); + let base = fixture(&local, &temp.path().join("base"), 1, None, true).await; + let target = fixture(&local, &temp.path().join("target"), 2, Some(&base), true).await; + let selection = selection( + &local, + &target, + &SaveOpts { + last_layers: Some(1), + ..Default::default() + }, + ) + .await + .unwrap() + .unwrap(); + assert!(selection.memory.is_empty()); + let archive = temp.path().join("delta.msnap"); + save_snapshot( + &local, + target.path().to_str().unwrap(), + &archive, + SaveOpts { + since: Some(base.path().to_string_lossy().into_owned()), + plain_tar: true, + ..Default::default() + }, + ) + .await + .unwrap(); + let ids = memory_objects(&base).unwrap(); + let missing = checkpoint_object_path( + &base.path().join(CHECKPOINT_DIRECTORY), + ids.first().unwrap(), + ); + let saved = std::fs::read(&missing).unwrap(); + std::fs::remove_file(&missing).unwrap(); + assert!( + load_snapshot_with_base(&local, &archive, None, Some(base.path().to_str().unwrap())) + .await + .is_err() + ); + assert!(!local.snapshots_dir().join(target.id().as_str()).exists()); + std::fs::write(&missing, vec![0x33; saved.len()]).unwrap(); + assert!( + load_snapshot_with_base(&local, &archive, None, Some(base.path().to_str().unwrap())) + .await + .is_err() + ); + std::fs::write(&missing, saved).unwrap(); + let loaded = + load_snapshot_with_base(&local, &archive, None, Some(base.path().to_str().unwrap())) + .await + .unwrap(); + assert_ram(loaded.path(), 2); +} + +#[tokio::test] +async fn memory_dependency_validation_rejects_incomplete_and_misbound_inventories() { + let temp = tempfile::tempdir().unwrap(); + let local = LocalBackend::builder() + .home(temp.path().join("home")) + .build() + .await + .unwrap(); + let base = fixture(&local, &temp.path().join("base"), 1, None, false).await; + let target = fixture(&local, &temp.path().join("target"), 4, None, false).await; + let archive = temp.path().join("delta.tar"); + save_snapshot( + &local, + target.path().to_str().unwrap(), + &archive, + SaveOpts { + since: Some(base.path().to_string_lossy().into_owned()), + plain_tar: true, + ..Default::default() + }, + ) + .await + .unwrap(); + let stage = temp.path().join("stage"); + let inventory = unpack(&archive, &stage).await; + assert!(validate(&inventory).unwrap().is_some()); + let clone = || { + serde_json::from_value::(serde_json::to_value(&inventory).unwrap()) + .unwrap() + }; + let mut missing_requirement = clone(); + missing_requirement + .requires + .retain(|name| name != REQUIREMENT); + assert!(validate(&missing_requirement).is_err()); + let mut no_dependencies = clone(); + no_dependencies.extensions.insert( + REQUIREMENT.into(), + serde_json::json!({"disks": [], "memory": []}), + ); + assert!(validate(&no_dependencies).is_err()); + let mut duplicate = clone(); + let mut deps = validate(&duplicate).unwrap().unwrap(); + deps.memory.push(deps.memory[0].clone()); + duplicate + .extensions + .insert(REQUIREMENT.into(), serde_json::to_value(&deps).unwrap()); + assert!(validate(&duplicate).is_err()); + for kind in ["snapshot-descriptor", "checkpoint-root", "image-object"] { + let mut wrong_kind = clone(); + wrong_kind + .entries + .iter_mut() + .find(|entry| !entry.included) + .unwrap() + .kind = kind.into(); + assert!(validate(&wrong_kind).is_err()); + } + let mut wrong_owner = clone(); + wrong_owner + .entries + .iter_mut() + .find(|entry| !entry.included) + .unwrap() + .owner_snapshot = Some(base.id().to_string()); + assert!(validate(&wrong_owner).is_err()); + let mut undeclared = clone(); + undeclared + .entries + .iter_mut() + .find(|entry| entry.kind == "checkpoint-root") + .unwrap() + .included = false; + assert!(validate(&undeclared).is_err()); + + // An inventory can describe an existing base object that the target does not reference. + // Structural inventory validation alone is insufficient: resolve must check the target map. + let mut unreferenced = clone(); + let extra = memory_objects(&base) + .unwrap() + .difference(&memory_objects(&target).unwrap()) + .next() + .unwrap() + .clone(); + let mut deps = validate(&unreferenced).unwrap().unwrap(); + deps.memory.push(extra.clone()); + deps.memory.sort(); + unreferenced + .extensions + .insert(REQUIREMENT.into(), serde_json::to_value(&deps).unwrap()); + let mut entry = serde_json::from_value::( + serde_json::to_value( + unreferenced + .entries + .iter() + .find(|entry| !entry.included) + .unwrap(), + ) + .unwrap(), + ) + .unwrap(); + entry.path = memory_archive_path(target.id().as_str(), &extra); + unreferenced.entries.push(entry); + assert!( + resolve( + &local, + &unreferenced, + &stage, + &stage.join("cache"), + Some(base.path().to_str().unwrap()) + ) + .await + .is_err() + ); + assert!(!local.snapshots_dir().join(target.id().as_str()).exists()); + + let truncated = temp.path().join("truncated.msnap"); + let bytes = std::fs::read(&archive).unwrap(); + std::fs::write(&truncated, &bytes[..bytes.len() / 2]).unwrap(); + assert!( + load_snapshot_with_base( + &local, + &truncated, + None, + Some(base.path().to_str().unwrap()) + ) + .await + .is_err() + ); + assert!(!local.snapshots_dir().join(target.id().as_str()).exists()); +} + +#[tokio::test] +async fn standalone_base_archive_resolves_ram_but_dependent_base_archive_is_refused() { + let temp = tempfile::tempdir().unwrap(); + let local = LocalBackend::builder() + .home(temp.path().join("home")) + .build() + .await + .unwrap(); + let base = fixture(&local, &temp.path().join("base"), 1, None, false).await; + let target = fixture(&local, &temp.path().join("target"), 4, None, false).await; + let base_archive = temp.path().join("base.msnap"); + save_snapshot( + &local, + base.path().to_str().unwrap(), + &base_archive, + SaveOpts::default(), + ) + .await + .unwrap(); + let delta = temp.path().join("delta.msnap"); + save_snapshot( + &local, + target.path().to_str().unwrap(), + &delta, + SaveOpts { + since: Some(base_archive.to_string_lossy().into_owned()), + ..Default::default() + }, + ) + .await + .unwrap(); + assert!(open_base(&local, delta.to_str().unwrap()).await.is_err()); + let loaded = + load_snapshot_with_base(&local, &delta, None, Some(base_archive.to_str().unwrap())) + .await + .unwrap(); + assert_ram(loaded.path(), 4); + let child = temp.path().join("child"); + assert!( + materialize_archive_for_child_with_base( + &local, + &delta, + &child, + false, + Some(base_archive.to_str().unwrap()) + ) + .await + .unwrap() + .checkpoint_restore + .is_some() + ); +} diff --git a/sdk/rust/lib/snapshot/mod.rs b/sdk/rust/lib/snapshot/mod.rs index 6f5757424..511662b9e 100644 --- a/sdk/rust/lib/snapshot/mod.rs +++ b/sdk/rust/lib/snapshot/mod.rs @@ -263,7 +263,7 @@ impl Snapshot { archive::load_snapshot(local, archive_path, dest).await } - /// Load a disk-dependent archive using its exact base snapshot or standalone base archive. + /// Load a dependent archive using its base snapshot or standalone base archive. /// The imported snapshot owns a complete local closure after this call. pub async fn load_with_base( archive_path: &Path, From 8713ded2ee13c48a9bcce02cf7f127185aef7e6d Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Thu, 10 Sep 2026 02:05:46 +0100 Subject: [PATCH 14/29] perf(lifecycle): reduce resident pause and resume overhead Remove automatic guest-wide sync from execution-state capture while retaining workload freezing, host disk durability and clock-before-thaw ordering. Document the crash-consistent disk-only extraction contract. Use bounded sub-millisecond freezer rechecks, a lightweight CLI executor, and WAL-aware read-only control lookups with existing installation and schema gates. Reuse ambient configuration and route Go control lookups through the same path. Add dirty-memory checkpoint coverage and the macOS/Linux release report, including 480 measured lifecycle cycles and flat/layered restore checks. --- crates/agentd/lib/workload.rs | 88 ++++-- crates/cli/bin/main.rs | 24 +- crates/db/lib/connection.rs | 73 +++++ crates/runtime/lib/maintenance.rs | 5 +- docs/sandboxes/snapshots.mdx | 2 +- scripts/smoke/cli/dirty-memory-checkpoint.py | 184 ++++++++++++ .../reports/resident-lifecycle-2026-09-10.md | 69 +++++ sdk/go/native/src/lib.rs | 8 +- sdk/rust/lib/backend/local/control_lookup.rs | 272 ++++++++++++++++++ sdk/rust/lib/backend/local/mod.rs | 10 + sdk/rust/lib/backend/local/sandbox/mod.rs | 4 +- sdk/rust/lib/backend/profile.rs | 31 +- sdk/rust/lib/sandbox/pause.rs | 5 +- 13 files changed, 738 insertions(+), 37 deletions(-) create mode 100644 scripts/smoke/cli/dirty-memory-checkpoint.py create mode 100644 scripts/smoke/reports/resident-lifecycle-2026-09-10.md create mode 100644 sdk/rust/lib/backend/local/control_lookup.rs diff --git a/crates/agentd/lib/workload.rs b/crates/agentd/lib/workload.rs index 41cff90a6..8e35c02ef 100644 --- a/crates/agentd/lib/workload.rs +++ b/crates/agentd/lib/workload.rs @@ -13,6 +13,8 @@ use std::time::{Duration, Instant}; const CGROUP_ROOT: &str = "/sys/fs/cgroup/microsandbox-workload"; const FREEZE_TIMEOUT: Duration = Duration::from_secs(5); const FREEZE_STATE_RECHECK_INTERVAL: Duration = Duration::from_millis(1); +const FREEZE_FAST_RECHECK_INTERVAL: Duration = Duration::from_micros(100); +const FREEZE_FAST_RECHECK_WINDOW: Duration = Duration::from_millis(1); const MAX_ATTEMPT_ID_BYTES: usize = 128; //-------------------------------------------------------------------------------------------------- @@ -150,10 +152,9 @@ impl WorkloadLatch { attempt_id: attempt_id.to_string(), }; self.freezer()?.set_frozen(true)?; - // Agentd itself remains outside the workload cgroup, so it can flush every mounted - // filesystem after user processes stop mutating them and before the host pauses the VM. - // `sync(2)` has no error return; completion is the durability boundary exposed by Linux. - unsafe { libc::sync() }; + // This latch stops execution, not guest writeback. Full captures preserve dirty guest + // cache pages in RAM alongside the matching device/disk cut; disk-only extraction is + // crash-consistent. Host block draining and durable publication remain separate gates. self.state = LatchState::Frozen { attempt_id: attempt_id.to_string(), }; @@ -326,6 +327,7 @@ fn wait_for_frozen_event( mut wait: impl FnMut(Duration) -> io::Result<()>, mut now: impl FnMut() -> Instant, ) -> io::Result<()> { + let fast_until = now() + FREEZE_FAST_RECHECK_WINDOW; loop { let interrupted = match read_state() { Ok(state) if state == expected => return Ok(()), @@ -333,7 +335,8 @@ fn wait_for_frozen_event( Err(error) if error.kind() == io::ErrorKind::Interrupted => true, Err(error) => return Err(error), }; - let remaining = deadline.saturating_duration_since(now()); + let observed_at = now(); + let remaining = deadline.saturating_duration_since(observed_at); if remaining.is_zero() { return Err(io::Error::new( io::ErrorKind::TimedOut, @@ -346,7 +349,14 @@ fn wait_for_frozen_event( if interrupted { continue; } - match wait(remaining.min(FREEZE_STATE_RECHECK_INTERVAL)) { + // Cgroup notifications can be delayed even after frozen=1. Brief sleeping rechecks + // avoid a whole millisecond of observation lag without spinning for the deadline. + let interval = if observed_at < fast_until { + FREEZE_FAST_RECHECK_INTERVAL + } else { + FREEZE_STATE_RECHECK_INTERVAL + }; + match wait(remaining.min(interval)) { Ok(()) => {} // Recheck state and the original deadline after interruptions or spurious events. Err(error) if error.kind() == io::ErrorKind::Interrupted => {} @@ -361,13 +371,24 @@ fn wait_for_cgroup_event(fd: RawFd, remaining: Duration) -> io::Result<()> { events: libc::POLLPRI | libc::POLLERR, revents: 0, }; - // A notification can wake poll immediately. The one-millisecond timeout ceiling also - // covers transitions whose cgroup notification is deferred by the kernel's rate limit. - let timeout = remaining - .as_nanos() - .div_ceil(1_000_000) - .min(i32::MAX as u128) as i32; - let result = unsafe { libc::poll(&mut event, 1, timeout) }; + // Linux guests use ppoll so sub-millisecond waits are not rounded back up to 1 ms. + // Non-Linux builds only exercise the portable unit-test fallback, never a guest freezer. + #[cfg(target_os = "linux")] + let result = { + let timeout = libc::timespec { + tv_sec: remaining.as_secs().min(libc::time_t::MAX as u64) as libc::time_t, + tv_nsec: remaining.subsec_nanos().into(), + }; + unsafe { libc::ppoll(&mut event, 1, &timeout, std::ptr::null()) } + }; + #[cfg(not(target_os = "linux"))] + let result = { + let timeout = remaining + .as_nanos() + .div_ceil(1_000_000) + .min(i32::MAX as u128) as i32; + unsafe { libc::poll(&mut event, 1, timeout) } + }; if result < 0 { return Err(io::Error::last_os_error()); } @@ -486,7 +507,7 @@ mod tests { start + FREEZE_TIMEOUT, || Ok(frozen.get()), |remaining| { - assert_eq!(remaining, Duration::from_millis(1)); + assert_eq!(remaining, FREEZE_FAST_RECHECK_INTERVAL); // The state is ready but cgroup_file_notify defers its notification by // roughly 10 ms. A bounded timeout observes readiness without that event. frozen.set(true); @@ -498,7 +519,7 @@ mod tests { ) .unwrap(); assert_eq!(waits.get(), 1); - assert_eq!(elapsed.get(), Duration::from_millis(1)); + assert_eq!(elapsed.get(), FREEZE_FAST_RECHECK_INTERVAL); } #[test] @@ -512,7 +533,13 @@ mod tests { |remaining| { assert_eq!( remaining, - (Duration::from_millis(3) - elapsed.get()).min(FREEZE_STATE_RECHECK_INTERVAL) + (Duration::from_millis(3) - elapsed.get()).min( + if elapsed.get() < FREEZE_FAST_RECHECK_WINDOW { + FREEZE_FAST_RECHECK_INTERVAL + } else { + FREEZE_STATE_RECHECK_INTERVAL + } + ) ); elapsed.set(elapsed.get() + Duration::from_millis(1)); Err(io::ErrorKind::Interrupted.into()) @@ -524,6 +551,35 @@ mod tests { assert_eq!(elapsed.get(), Duration::from_millis(3)); } + #[test] + fn freezer_fast_rechecks_back_off_and_respect_short_final_wait() { + let start = Instant::now(); + let elapsed = Cell::new(Duration::ZERO); + let waits = Cell::new(0); + let timeout = Duration::from_micros(2_050); + let error = wait_for_frozen_event( + true, + start + timeout, + || Ok(false), + |duration| { + let expected = if elapsed.get() < FREEZE_FAST_RECHECK_WINDOW { + FREEZE_FAST_RECHECK_INTERVAL + } else { + FREEZE_STATE_RECHECK_INTERVAL + }; + assert_eq!(duration, expected.min(timeout - elapsed.get())); + elapsed.set(elapsed.get() + duration); + waits.set(waits.get() + 1); + Ok(()) + }, + || start + elapsed.get(), + ) + .unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::TimedOut); + assert_eq!(waits.get(), 12); + assert_eq!(elapsed.get(), timeout); + } + #[test] fn freezer_read_interruptions_retry_with_a_bounded_deadline() { let start = Instant::now(); diff --git a/crates/cli/bin/main.rs b/crates/cli/bin/main.rs index 4954ea709..74a8ee4f3 100644 --- a/crates/cli/bin/main.rs +++ b/crates/cli/bin/main.rs @@ -297,7 +297,9 @@ fn main() { // Handle --tree before Cli::parse() so it works even when // required arguments (e.g. `msb run --tree`) are missing. - if let Some(tree) = microsandbox_cli::tree::try_show_tree(&Cli::command()) { + if std::env::args_os().any(|arg| arg == "--tree") + && let Some(tree) = microsandbox_cli::tree::try_show_tree(&Cli::command()) + { println!("{tree}"); return; } @@ -640,13 +642,19 @@ fn run_async_command_anyhow( // Pull and create can overlap network I/O, decompression, and progress UI. // Use a small-but-not-tiny worker pool so foreground UI tasks still get // scheduled while multiple layers are downloading and materializing. - let worker_threads = std::thread::available_parallelism() - .map(|count| count.get().clamp(4, 8)) - .unwrap_or(4); - let runtime = tokio::runtime::Builder::new_multi_thread() - .worker_threads(worker_threads) - .enable_all() - .build()?; + // Resident control performs one IPC exchange. It needs I/O and timers, not the image + // pipeline's worker pool. Blocking filesystem/SQLite work keeps its normal executor. + let mut builder = if matches!(command, Commands::Pause(_) | Commands::Resume(_)) { + tokio::runtime::Builder::new_current_thread() + } else { + let worker_threads = std::thread::available_parallelism() + .map(|count| count.get().clamp(4, 8)) + .unwrap_or(4); + let mut builder = tokio::runtime::Builder::new_multi_thread(); + builder.worker_threads(worker_threads); + builder + }; + let runtime = builder.enable_all().build()?; runtime.block_on(async move { // Stale-sandbox reaping and ephemeral cleanup are owned by host diff --git a/crates/db/lib/connection.rs b/crates/db/lib/connection.rs index cd540a1df..650c9daee 100644 --- a/crates/db/lib/connection.rs +++ b/crates/db/lib/connection.rs @@ -71,6 +71,30 @@ impl DbReadConnection { Ok(Self(conn)) } + /// Open an existing catalog without creating it or changing its journal mode. + /// + /// Intended for short control lookups after the caller coordinates with migrations. + /// This is a normal WAL-aware reader, never an immutable-file shortcut. + pub async fn open_read_only( + db_path: &Path, + connect_timeout: Duration, + busy_timeout: Duration, + ) -> Result { + let options = sqlx::sqlite::SqliteConnectOptions::new() + .filename(db_path) + .read_only(true) + .create_if_missing(false) + .busy_timeout(busy_timeout); + let pool = sqlx::sqlite::SqlitePoolOptions::new() + .max_connections(1) + .acquire_timeout(connect_timeout) + .connect_with(options) + .await?; + Ok(Self(sea_orm::SqlxSqliteConnector::from_sqlx_sqlite_pool( + pool, + ))) + } + /// Borrow the underlying sea-orm connection. pub fn inner(&self) -> &DatabaseConnection { &self.0 @@ -217,6 +241,55 @@ mod tests { const TIMEOUT: Duration = Duration::from_secs(5); + #[tokio::test] + async fn strict_reader_sees_wal_commits_but_cannot_write() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("catalog.db"); + let writer = DbWriteConnection::open(&path, TIMEOUT, TIMEOUT) + .await + .unwrap(); + writer + .execute_unprepared("CREATE TABLE control_test (value INTEGER)") + .await + .unwrap(); + let reader = DbReadConnection::open_read_only(&path, TIMEOUT, TIMEOUT) + .await + .unwrap(); + // Keep the writer alive: control reads must see WAL commits, not an immutable + // view of only the main database file. + writer + .execute_unprepared("INSERT INTO control_test VALUES (42)") + .await + .unwrap(); + let row = reader + .query_one_raw(Statement::from_string( + DbBackend::Sqlite, + "SELECT value FROM control_test", + )) + .await + .unwrap() + .unwrap(); + assert_eq!(row.try_get_by_index::(0).unwrap(), 42); + assert!( + reader + .execute_unprepared("INSERT INTO control_test VALUES (43)") + .await + .is_err() + ); + } + + #[tokio::test] + async fn strict_reader_never_creates_a_catalog() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("missing.db"); + assert!( + DbReadConnection::open_read_only(&path, TIMEOUT, TIMEOUT) + .await + .is_err() + ); + assert!(!path.exists()); + } + #[tokio::test] async fn read_open_does_not_create_db() { // Existing directory, missing DB file. diff --git a/crates/runtime/lib/maintenance.rs b/crates/runtime/lib/maintenance.rs index 8ec52ca77..4cbb88281 100644 --- a/crates/runtime/lib/maintenance.rs +++ b/crates/runtime/lib/maintenance.rs @@ -27,7 +27,8 @@ use microsandbox_db::entity::{ }; use sea_orm::sea_query::{Expr, OnConflict}; use sea_orm::{ - ColumnTrait, Condition, DbErr, EntityTrait, QueryFilter, QueryOrder, QuerySelect, Set, + ColumnTrait, Condition, ConnectionTrait, DbErr, EntityTrait, QueryFilter, QueryOrder, + QuerySelect, Set, }; use crate::{RuntimeError, RuntimeResult}; @@ -655,7 +656,7 @@ pub async fn clear_install_exclusive_lease_idempotent( /// migration yet. In that case startup continues so normal migrations can /// create it. Once the table exists, the install-exclusive row becomes a hard /// refusal while unexpired. -pub async fn refuse_if_install_exclusive_held(db: &DbWriteConnection) -> RuntimeResult<()> { +pub async fn refuse_if_install_exclusive_held(db: &C) -> RuntimeResult<()> { let now = chrono::Utc::now().naive_utc(); let lease = match lease_entity::Entity::find_by_id(lease_entity::INSTALL_EXCLUSIVE) .one(db) diff --git a/docs/sandboxes/snapshots.mdx b/docs/sandboxes/snapshots.mdx index 0bbbe8950..3b489be34 100644 --- a/docs/sandboxes/snapshots.mdx +++ b/docs/sandboxes/snapshots.mdx @@ -351,7 +351,7 @@ msb run --name worker --from-snapshot worker-checkpoint --disk-only ``` -Disk-only restore copies only the checkpoint's disk chain, creates a fresh writable qcow2 head, and performs an ordinary boot. It does not require the destination to support the checkpoint's memory or execution codec. The full capture freezes workloads and syncs guest filesystems before pausing the VM, so this cold-boot view is filesystem-clean. +Disk-only restore copies only the checkpoint's disk chain, creates a fresh writable qcow2 head, and performs an ordinary boot. It does not require the destination to support the checkpoint's memory or execution codec. Like a live disk snapshot, this disk-only view is crash-consistent: recent writes still buffered in guest RAM may be absent, and filesystem journal recovery may run during boot. Full restore preserves those buffers in captured RAM. Pause and full capture do not automatically sync guest filesystems; host disk flushes and durable snapshot publication are still enforced. ## List, inspect, and remove diff --git a/scripts/smoke/cli/dirty-memory-checkpoint.py b/scripts/smoke/cli/dirty-memory-checkpoint.py new file mode 100644 index 000000000..98176c8eb --- /dev/null +++ b/scripts/smoke/cli/dirty-memory-checkpoint.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +"""Live no-sync checkpoint qualification in an isolated MSB_HOME. + +Requires a matching release runtime/guest agent. Checks dirty block-backed page +cache, shared/private mmap, heap, tmpfs, inherited incremental memory, and the +separate crash-consistent disk-only contract. It does not benchmark throughput. +""" +import argparse +import json +import os +from pathlib import Path +import subprocess +import tempfile +import time + + +WORKER = r''' +import json, mmap, os, time, uuid +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path + +# Make the dirty-memory case deliberate: normal background writeback must not +# turn this into a test that passes only because everything reached disk first. +Path('/proc/sys/vm/dirty_writeback_centisecs').write_text('0') +Path('/proc/sys/vm/dirty_expire_centisecs').write_text('600000') +Path('/proc/sys/vm/dirty_ratio').write_text('90') +Path('/proc/sys/vm/dirty_background_ratio').write_text('80') +Path('/persisted').write_bytes(b'persisted-before-checkpoint') +fd = os.open('/persisted', os.O_RDONLY); os.fsync(fd); os.close(fd) +fd = os.open('/', os.O_RDONLY); os.fsync(fd); os.close(fd) +size = 64 * 1024 * 1024 +fd = os.open('/dirty-cache', os.O_RDWR | os.O_CREAT | os.O_TRUNC, 0o600) +os.ftruncate(fd, size) +shared = mmap.mmap(fd, size, access=mmap.ACCESS_WRITE) +shared[:] = b'A' * size +private = mmap.mmap(fd, size, access=mmap.ACCESS_COPY) +private[:8] = b'private0' +heap = bytearray(b'heap0000') +Path('/dev/shm/latch-marker').write_bytes(b'tmpfs000') +nonce = str(uuid.uuid4()) +class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): pass + def do_GET(self): + if self.path == '/mutate': + shared[:8] = b'shared01'; private[:8] = b'private1'; heap[:] = b'heap0001' + Path('/dev/shm/latch-marker').write_bytes(b'tmpfs001') + dirty = next(int(x.split()[1]) for x in Path('/proc/meminfo').read_text().splitlines() if x.startswith('Dirty:')) + body = json.dumps(dict(nonce=nonce, heap=heap.decode(), shared=shared[:8].decode(), + private=private[:8].decode(), tmpfs=Path('/dev/shm/latch-marker').read_text(), + disk_cache=Path('/dirty-cache').read_bytes()[:8].decode(), dirty_kib=dirty, + clock=time.time())).encode() + self.send_response(200); self.send_header('Content-Length', str(len(body))); self.end_headers(); self.wfile.write(body) +HTTPServer(('0.0.0.0', 8080), Handler).serve_forever() +''' + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--binary', required=True) + parser.add_argument('--output', required=True, type=Path) + parser.add_argument('--layout', choices=['flat', 'layered'], required=True) + parser.add_argument('--home-parent', type=Path, help='Filesystem for the isolated sandbox home') + args = parser.parse_args() + args.output.mkdir(parents=True, exist_ok=False) + parent = args.home_parent or Path('/private/tmp' if os.uname().sysname == 'Darwin' else '/tmp') + home = Path(tempfile.mkdtemp(prefix='dirty-', dir=parent)) + env = dict(os.environ, MSB_HOME=str(home), MSB_BACKEND='local') + names, rows = [], [] + active = 'source' + report = dict(home=str(home), binary=args.binary, layout=args.layout, rows=rows, status='running') + + def run(label, *command): + start = time.monotonic() + result = subprocess.run([args.binary, *command], env=env, capture_output=True, text=True, timeout=180) + row = dict(case=label, seconds=time.monotonic()-start, returncode=result.returncode, + stdout=result.stdout, stderr=result.stderr) + rows.append(row) + (args.output / 'report.json').write_text(json.dumps(report, indent=2)) + print(json.dumps({k: row[k] for k in ('case', 'seconds', 'returncode')}), flush=True) + assert result.returncode == 0, row + return result.stdout + + def state(path='/'): + # Use the guest loopback endpoint. A branch must not inherit host port + # publications, and this test is about memory, not ingress reconfiguration. + return json.loads(run('state-' + active, 'exec', active, '--', 'python3', '-c', + f"import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080{path}').read().decode())")) + + def matches(expected, actual=None): + actual = state() if actual is None else actual + for key in ('nonce', 'heap', 'shared', 'private', 'tmpfs', 'disk_cache'): + assert actual[key] == expected[key], (key, actual, expected) + assert abs(actual['clock'] - time.time()) < 3, actual + return actual + + def stop(name): + run('stop-' + name, 'stop', name) + + try: + source = 'source'; names.append(source) + run('create', 'run', '-d', '--name', source, '--memory', '512M', '--cpus', '2', + '--root-disk', 'flat:1G' if args.layout == 'flat' else '1G', + 'python:3.13-alpine3.22', '--', 'python3', '-u', '-c', WORKER) + deadline = time.monotonic() + 30 + while True: + try: initial = state(); break + except Exception: + if time.monotonic() >= deadline: raise + time.sleep(.1) + assert initial['dirty_kib'] >= 32 * 1024, initial + report['initial'] = initial + run('full-dirty', 'snapshot', 'create', 'dirty-full', '--from', source, '--full') + matches(initial) + names.append('running-branch') + run('branch-dirty', 'branch', source, '--name', 'running-branch') + matches(initial, json.loads(run('branch-state', 'exec', 'running-branch', '--', 'python3', '-c', + "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080').read().decode())"))) + branch_changed = json.loads(run('mutate-branch', 'exec', 'running-branch', '--', 'python3', '-c', + "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/mutate').read().decode())")) + matches(initial) + names.append('branch-grandchild') + run('branch-of-branch', 'branch', 'running-branch', '--name', 'branch-grandchild') + matches(branch_changed, json.loads(run('branch-grandchild-state', 'exec', 'branch-grandchild', '--', 'python3', '-c', + "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080').read().decode())"))) + stop('branch-grandchild') + stop('running-branch'); matches(initial) + run('pause', 'pause', source) + for suffix in ('one', 'two'): + run('capture-paused-' + suffix, 'snapshot', 'create', 'paused-' + suffix, '--from', source, '--full') + assert json.loads(run('inspect-paused-' + suffix, 'inspect', source, '--format', 'json'))['status'] == 'Paused' + names.append('paused-branch') + run('branch-paused-dirty', 'branch', source, '--name', 'paused-branch') + matches(initial, json.loads(run('paused-branch-state', 'exec', 'paused-branch', '--', 'python3', '-c', + "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080').read().decode())"))) + assert json.loads(run('source-still-paused', 'inspect', source, '--format', 'json'))['status'] == 'Paused' + stop('paused-branch') + run('resume', 'resume', source); matches(initial) + # Source shutdown also proves restored state does not rely on live source RAM. + stop(source) + for mode in ('eager', 'forked'): + child = mode; names.append(child) + run('restore-' + mode, 'create', '--name', child, '--from-snapshot', 'dirty-full', + *(['--forked'] if mode == 'forked' else [])) + active = child + matches(initial) + if mode == 'forked': + # A restored child starts a fresh capture lineage. Establish its + # baseline before mutating, then verify the next capture is incremental. + run('child-baseline', 'snapshot', 'create', 'child-baseline', '--from', child, '--full') + changed = state('/mutate') + assert changed['private'] == 'private1' + run('incremental-dirty', 'snapshot', 'create', 'dirty-incremental', '--from', child, '--full') + checkpoint = home / 'snapshots/dirty-incremental/checkpoint' + descriptor = json.loads((checkpoint / 'checkpoint.json').read_text()) + algorithm, digest = descriptor['memory'].split(':', 1) + memory = json.loads((checkpoint / 'objects' / algorithm / digest[:2] / digest).read_text()) + report['incremental_capture_mode'] = memory['capture_mode'] + assert memory['capture_mode'] == 'incremental', memory['capture_mode'] + matches(changed) + stop(child) + names.append('grandchild') + run('restore-incremental', 'create', '--name', 'grandchild', '--from-snapshot', 'dirty-incremental', '--forked') + active = 'grandchild' + matches(changed); stop('grandchild') + names.append('disk-only') + run('disk-only', 'create', '--name', 'disk-only', '--from-snapshot', 'dirty-full', '--disk-only') + assert run('persisted-disk-data', 'exec', 'disk-only', '--', 'cat', '/persisted').strip() == 'persisted-before-checkpoint' + run('no-tmpfs-in-disk-view', 'exec', 'disk-only', '--', 'test', '!', '-e', '/dev/shm/latch-marker') + # Unsynced disk bytes are deliberately not asserted either present or absent. + report['status'] = 'passed' + except Exception as error: + report.update(status='failed', error=repr(error)); raise + finally: + errors = [] + for name in reversed(names): + result = subprocess.run([args.binary, 'stop', name], env=env, capture_output=True, text=True, timeout=30) + if result.returncode and 'already stopped' not in result.stderr.lower(): + errors.append(dict(name=name, error=result.stderr)) + report['cleanup_errors'] = errors + (args.output / 'report.json').write_text(json.dumps(report, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/scripts/smoke/reports/resident-lifecycle-2026-09-10.md b/scripts/smoke/reports/resident-lifecycle-2026-09-10.md new file mode 100644 index 000000000..9cacd7e08 --- /dev/null +++ b/scripts/smoke/reports/resident-lifecycle-2026-09-10.md @@ -0,0 +1,69 @@ +# Resident pause/resume optimization qualification + +## Scope and implementation + +This pass implements the approved change from automatic guest-wide filesystem syncing to execution-state preservation: resident pause, full capture, and direct branch no longer call guest `sync()`. Full restore retains dirty guest memory; deliberately extracting only the disk is crash-consistent. Host block draining, sealed-layer durability, workload freezing, heartbeat gating, and clock acknowledgement before workload thaw remain in place. + +The CLI avoids a second command-tree construction unless `--tree` is present and uses a current-thread Tokio runtime for pause/resume. Ambient local backend resolution reuses its loaded configuration document. A healthy current-catalog control lookup uses a WAL-aware read-only pool without snapshot reconciliation or writer initialization. Schema/install/downgrade checks remain; older catalogs and stale targets use the existing slow path. Go's name-based pause/resume calls use that observation-free lookup. + +The independent libkrun clock-observation change suppresses notifications when the entire observed predicate is unchanged, under the same mutex used by waiters. It passes device unit tests but is **not included in the release binaries benchmarked below**; these use published `msb_krun` 0.1.34. No firmware or protocol version was bumped. + +The Node shared-ownership rewrite was approved and implemented in the subsequent [Node ownership follow-up](node-lifecycle-concurrency-2026-09-10.md); it is not part of the CLI/direct measurements below. Retained-handle boot fencing and the Python ownership micro-optimization are not implemented in this pass. Existing runtime-authoritative control remains; this report does not claim to have eliminated the existing name-reuse race for stale retained handles. + +## Release benchmark results + +Measurements are complete request-to-acknowledgement durations, not isolated hypervisor pause instructions. The direct route opens the same existing control socket and invokes the same runtime operation as the CLI. No snapshot occurs between resident pause and resume. Each entry is the median of 30 samples; CLI/direct order alternates within each fixture. Baseline and candidate each use fresh Small/Large fixtures, run sequentially per host, not randomized across builds. Treat small differences and tail estimates accordingly. + +| Host | Profile | Interface | Pause before → after, ms | Resume ACK before → after, ms | Resume through app response before → after, ms | +| --- | --- | --- | --- | --- | --- | +| macOS ARM64/HVF | Small | CLI | 11.980 → 5.305 | 6.582 → 4.836 | 7.236 → 5.417 | +| macOS ARM64/HVF | Small | Direct | 5.531 → 0.410 | 0.166 → 0.160 | 0.615 → 0.518 | +| macOS ARM64/HVF | Large | CLI | 11.957 → 5.348 | 6.449 → 4.970 | 7.081 → 5.585 | +| macOS ARM64/HVF | Large | Direct | 5.561 → 0.428 | 0.183 → 0.175 | 0.652 → 0.547 | +| Linux x86-64/KVM | Small | CLI | 2.717 → 1.916 | 2.795 → 2.078 | 3.223 → 2.532 | +| Linux x86-64/KVM | Small | Direct | 0.275 → 0.228 | 0.347 → 0.343 | 0.800 → 0.804 | +| Linux x86-64/KVM | Large | CLI | 2.773 → 1.944 | 2.748 → 2.018 | 3.182 → 2.486 | +| Linux x86-64/KVM | Large | Direct | 0.307 → 0.244 | 0.307 → 0.307 | 0.763 → 0.767 | + +All 480 measured baseline/candidate cycles completed successfully, with application identity, memory state, progress, and SQLite integrity checks. This is 240 candidate cycles and 240 baseline cycles across both hosts, including both interfaces. Small uses one vCPU/256 MiB RAM; Large uses two vCPUs/4096 MiB RAM. These are release builds with matching guest agents; macOS binaries were codesigned with the repository's VM entitlements. The existing bounded stage tracing is unchanged between builds; no new per-pause diagnostic logging was added. + +Mac Small direct pause improves about 13.5×; its CLI pause improves about 2.26×. Linux Small CLI pause improves about 1.42×. Direct resume is essentially unchanged, particularly on Linux. The remaining CLI cost must not be described as a slow VM resume primitive. This pass does not rerun CH/FC, isolate each optimization's individual contribution, or qualify p99 latency. Mac Large has one candidate CLI pause outlier of 64.8 ms; median improvements do not eliminate scheduling tails. + +## Correctness coverage + +The live fixture is `scripts/smoke/cli/dirty-memory-checkpoint.py`. It disables ordinary periodic guest dirty-page writeback for the fixture, sets generous dirty thresholds, dirties a 64 MiB shared mapping, and requires at least 32 MiB reported dirty before capture. It separately retains a private mapping, heap bytes, tmpfs data, and a file explicitly persisted with file/directory `fsync`. Guest settings affect only the disposable test VM. + +| Invariant | macOS flat | macOS layered | Linux flat | Linux layered | +| --- | --- | --- | --- | --- | +| Running full capture retains dirty page-cache/shared mmap, private mmap, heap and tmpfs | Pass | Pass | Pass | Pass | +| Eager and forked restore after source shutdown | Pass | Pass | Pass | Pass | +| Running/paused direct branch; source remains paused after paused capture/branch | Pass | Pass | Pass | Pass | +| Mutated direct child stays private; branch-of-branch retains mutations | Pass | Pass | Pass | Pass | +| Dirty incremental capture verified by `capture_mode = incremental`, then restored | Pass | Pass | Pass | Pass | +| Disk-only extraction retains explicitly persisted file and excludes tmpfs | Pass | Pass | Pass | Pass | + +The first fixture iteration captured a restored child's initial full baseline and therefore did not prove incremental dirty capture. The tightened fixture creates the child's baseline first, mutates it, checks the next memory manifest explicitly says `incremental`, and restores that generation. Only the tightened run is credited for the incremental invariant. The paused checks validate public paused state and resulting memory; they are not an instruction-entry-counter proof that no guest instruction executed during capture. + +The tightened Linux tmpfs run returned an empty control response during baseline capture from a forked child, followed by SQLite disk-I/O errors during cleanup. Its runtime log ends during capture preparation without an explicit panic; no matching kernel crash/OOM report was found. The shared `/tmp` is a 32 GiB tmpfs and was heavily occupied. This is a suspected host-storage/resource interaction, **not an established root cause** and not a fixed product defect. The failed fixture is retained at `/tmp/dirty-42ts79bx` on OVH; none of its runtime processes remained alive when checked. Disk-backed reruns are tracked separately below. + +Disk-backed tightened Linux reruns passed both layouts after freeing tmpfs quota and directing temporary files to disk-backed storage. The intermediate disk-backed attempt had also failed at creation with explicit `Disk quota exceeded (os error 122)` because helper temporary files still used `/tmp`; mount inspection confirmed `/tmp` has `usrquota`. Two completed, stopped earlier fixture homes were moved intact from `/tmp/dirty-aic065db` and `/tmp/dirty-1ob_e60_` to `retained-dirty-flat` and `retained-dirty-layered` under the remote stage, freeing about 5 GiB without deleting their artifacts. Final reports are `dirty-flat-disk-r2/report.json` and `dirty-layered-disk-r2/report.json` under that stage. The quota failure is established; attribution of the earlier empty-response incident specifically to it remains an inference, not a proved runtime fix. + +Other checks passed: 17 Linux guest-freezer tests (predicate/event races, EINTR, error paths, bounded fast polling/backoff, timeout and latch ownership); 4 read-pool tests (including WAL visibility and write refusal); 4 control-lookup tests; 25 backend/profile tests; 2 control-socket reply tests; 15 libkrun clock-device tests; Go native binding `cargo check`; CLI `--tree`, `pause --tree`, and `pause --help`; focused Rust formatting and `git diff --check`. The restricted socket test initially failed to bind sockets and passed when rerun with appropriate permissions. The Go check initially lacked network/prebuilt artifacts and passed with an isolated development bundle and dependency access. + +Windows ARM64 and Linux ARM64/KVM were not live-tested in this pass. The subsequent [Node ownership follow-up](node-lifecycle-concurrency-2026-09-10.md) qualifies same-object exec/filesystem/lifecycle concurrency on Mac and Linux x86-64. Other unqualified cases include runtime-share custom files/mappings, detailed injected block-queue/ENOSPC failures, and a full per-language SDK timing matrix. Do not present these reports as exhaustive cross-platform correctness qualification or completion of every research recommendation. + +## Artifacts and provenance + +Base Microsandbox commit: `18c8fb8693957ac6e8ded964e42cfbe63f8506ad`. Worktree: `/private/tmp/msb-cow-8.PIhgYp`, branch `appcypher/cow-memory-lifecycle`. Release source staging copied that commit plus the lifecycle implementation files only, excluding concurrent incremental-archive changes in the shared checkout. Test-only/formatting additions made after staging do not change the measured runtime behavior. Qualification preceded the separately authorized commit and push. + +- Local build, harnesses, raw benchmark JSON, and live reports: `/private/tmp/msb-resident-perf.qHpm0p`. +- Linux build, raw results, and logs: `/home/ubuntu/msb-resident-perf.zqr4vR` on OVH. +- macOS candidate binary SHA-256: `23d7d9db282f63cc9d1ee1b1b2e49bed0661814f3cce3e98cbde2f5e4bc2dd101`. +- macOS baseline binary SHA-256: `82eb66d6cfc6370f172c3af597913069d03cb197688ee129bdde125308a01126`. +- Linux candidate binary SHA-256: `f87b4063dd1adc64ce996d306e0d334c3816f7ff9ca3b7a9474dad4acce25372`. +- ARM64 guest agent SHA-256: `1425dc4b6974c10983db03e023c2b868c890fc197857ea45d6289a83df593aa8`. +- x86-64 guest agent SHA-256: `ae70a9d63c7df953340c8d6dc2c674ccad18e9f490c114977ba7972fe63bb6d0`. +- macOS firmware SHA-256: `ea0d458cdc12a0fa6dac8d192542ddc39717f816da41176582905e31a8bf868c`. +- Companion libkrun worktree: `/private/tmp/krun-registry-release.Ordgfp`, baseline `b20d31aba2fcf996512b0540bde0176dd7db7ad8`; only the clock-observation file is modified. + +Passed live fixtures stop their own VMs in `finally`; artifacts are retained for inspection rather than deleting unrelated test data. Initial harness failures (old system Python, prohibited inherited host ports, and incorrect layered-root CLI spelling) are retained as failed attempts and are not counted as product passes. diff --git a/sdk/go/native/src/lib.rs b/sdk/go/native/src/lib.rs index 9290d6806..cd2881e43 100644 --- a/sdk/go/native/src/lib.rs +++ b/sdk/go/native/src/lib.rs @@ -2615,7 +2615,9 @@ pub unsafe extern "C" fn msb_sandbox_handle_pause( run_c(cancel_id, buf, buf_len, || { let name = unsafe { cstr(name) }?; Ok(Box::pin(async move { - let sb = Sandbox::get(&name).await.map_err(FfiError::from)?; + let sb = Sandbox::get_for_control(&name) + .await + .map_err(FfiError::from)?; sb.pause().await.map_err(FfiError::from)?; Ok(r#"{"ok":true}"#.into()) })) @@ -2632,7 +2634,9 @@ pub unsafe extern "C" fn msb_sandbox_handle_resume( run_c(cancel_id, buf, buf_len, || { let name = unsafe { cstr(name) }?; Ok(Box::pin(async move { - let sb = Sandbox::get(&name).await.map_err(FfiError::from)?; + let sb = Sandbox::get_for_control(&name) + .await + .map_err(FfiError::from)?; sb.resume().await.map_err(FfiError::from)?; Ok(r#"{"ok":true}"#.into()) })) diff --git a/sdk/rust/lib/backend/local/control_lookup.rs b/sdk/rust/lib/backend/local/control_lookup.rs new file mode 100644 index 000000000..38348ed6a --- /dev/null +++ b/sdk/rust/lib/backend/local/control_lookup.rs @@ -0,0 +1,272 @@ +//! Read-only lookup for a live control target, without unrelated snapshot reconciliation. + +use std::time::Duration; + +use microsandbox_db::DbReadConnection; +use microsandbox_migration::schema_metadata; +use sea_orm::{ColumnTrait, ConnectionTrait, DatabaseBackend, EntityTrait, QueryFilter, Statement}; + +use super::{ + LocalBackend, acquire_migration_lock, refuse_incomplete_self_downgrade, refuse_schema_ahead, +}; +use crate::db::entity::sandbox as sandbox_entity; +use crate::sandbox::SandboxStatus; +use crate::{MicrosandboxError, MicrosandboxResult}; + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl LocalBackend { + /// Return a healthy live target from a current catalog. `None` requests the existing + /// migration/stale-runtime path; errors must never become an unvalidated fast path. + pub(crate) async fn try_control_handle_state( + &self, + name: &str, + ) -> MicrosandboxResult)>> { + let db_dir = self.config().home().join(microsandbox_utils::DB_SUBDIR); + let db_path = db_dir.join(microsandbox_utils::DB_FILENAME); + match std::fs::metadata(&db_path) { + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.into()), + } + // Use the same installation coordination as the slow path. In particular, a rolled + // back database can look current while an incomplete downgrade still owns its files. + let _migration_lock = acquire_migration_lock(&db_dir).await?; + refuse_incomplete_self_downgrade(&db_dir)?; + let database = &self.config().database; + let read = if let Some(pools) = self.db.get() { + pools.read().clone() + } else { + DbReadConnection::open_read_only( + &db_path, + Duration::from_secs(database.connect_timeout_secs), + Duration::from_secs(database.busy_timeout_secs), + ) + .await + .map_err(|error| { + MicrosandboxError::Custom(format!( + "read control catalog {}: {error}", + db_path.display() + )) + })? + }; + microsandbox_runtime::maintenance::refuse_if_install_exclusive_held(&read) + .await + .map_err(|error| MicrosandboxError::Runtime(error.to_string()))?; + refuse_schema_ahead(read.inner()).await?; + let row = match read + .query_one_raw(Statement::from_string( + DatabaseBackend::Sqlite, + "SELECT COUNT(*) FROM seaql_migrations", + )) + .await + { + Ok(Some(row)) => row, + Ok(None) => return Ok(None), + Err(error) if super::is_missing_migrations_table(&error) => return Ok(None), + Err(error) => return Err(error.into()), + }; + if row.try_get_by_index::(0)? != schema_metadata::migration_ids().count() as i64 { + return Ok(None); + } + let model = sandbox_entity::Entity::find() + .filter(sandbox_entity::Column::Name.eq(name)) + .one(&read) + .await? + .ok_or_else(|| MicrosandboxError::SandboxNotFound(name.into()))?; + if !matches!( + model.status, + SandboxStatus::Running | SandboxStatus::Draining + ) { + return Ok(None); + } + let run = Self::load_active_run(&read, model.id).await?; + let pid = Self::pid_from_run(run.as_ref()); + // Do not clean up sockets from this read-only observation. The slow path rechecks + // the exact row/run under lifecycle ownership before touching stale artifacts. + if pid.is_some_and(Self::pid_is_alive) { + Ok(Some((model, pid))) + } else { + Ok(None) + } + } +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + async fn fixture() -> (tempfile::TempDir, LocalBackend) { + let home = tempfile::tempdir().unwrap(); + let backend = LocalBackend::builder().home(home.path()).build_lazy(); + let pools = backend.db().await.unwrap(); + pools.write().execute_unprepared( + "INSERT INTO sandbox (id, name, config, status, ephemeral) VALUES (1, 'source', '{}', 'Running', 0)", + ).await.unwrap(); + pools + .write() + .execute_raw(Statement::from_sql_and_values( + DatabaseBackend::Sqlite, + "INSERT INTO run (sandbox_id, pid, status) VALUES (1, ?, 'Running')", + [i64::from(std::process::id()).into()], + )) + .await + .unwrap(); + (home, backend) + } + + #[tokio::test] + async fn healthy_lookup_does_not_initialize_writer_or_reconcile_snapshots() { + let (home, _) = fixture().await; + // A non-directory snapshot path would fail normal reconciliation. The live + // control path has no reason to inspect it or mutate the target's catalog. + let snapshots = home.path().join("snapshots"); + if snapshots.exists() { + std::fs::remove_dir(&snapshots).unwrap(); + } + std::fs::write(&snapshots, b"unrelated snapshot inventory").unwrap(); + let backend = LocalBackend::builder().home(home.path()).build_lazy(); + let (model, pid) = backend + .try_control_handle_state("source") + .await + .unwrap() + .unwrap(); + assert_eq!(model.id, 1); + assert_eq!(pid, Some(std::process::id() as i32)); + assert!(backend.db.get().is_none()); + assert_eq!( + std::fs::read(snapshots).unwrap(), + b"unrelated snapshot inventory" + ); + } + + #[tokio::test] + async fn absent_catalog_and_terminal_target_request_slow_path() { + let home = tempfile::tempdir().unwrap(); + let empty = LocalBackend::builder().home(home.path()).build_lazy(); + assert!( + empty + .try_control_handle_state("source") + .await + .unwrap() + .is_none() + ); + assert!(!home.path().join("db").exists()); + let (_home, backend) = fixture().await; + backend + .db() + .await + .unwrap() + .write() + .execute_unprepared("UPDATE sandbox SET status = 'Stopped' WHERE id = 1") + .await + .unwrap(); + assert!( + backend + .try_control_handle_state("source") + .await + .unwrap() + .is_none() + ); + } + + #[tokio::test] + async fn schema_and_install_gates_are_not_bypassed() { + let (home, backend) = fixture().await; + let writer = backend.db().await.unwrap().write(); + writer.execute_unprepared( + "INSERT INTO seaql_migrations (version, applied_at) VALUES ('future_unknown_migration', 0)", + ).await.unwrap(); + assert!( + backend + .try_control_handle_state("source") + .await + .unwrap_err() + .to_string() + .contains("schema is newer") + ); + writer + .execute_unprepared( + "DELETE FROM seaql_migrations WHERE version = 'future_unknown_migration'", + ) + .await + .unwrap(); + writer.execute_raw(Statement::from_sql_and_values( + DatabaseBackend::Sqlite, + "INSERT OR REPLACE INTO maintenance_lease (name, holder_pid, lease_expires_at) VALUES ('install_exclusive', ?, ?)", + [(std::process::id() as i32).into(), (chrono::Utc::now().naive_utc() + chrono::Duration::minutes(10)).into()], + )).await.unwrap(); + assert!( + backend + .try_control_handle_state("source") + .await + .unwrap_err() + .to_string() + .contains("install operation in progress") + ); + writer + .execute_unprepared("DELETE FROM maintenance_lease WHERE name = 'install_exclusive'") + .await + .unwrap(); + let journal_dir = home.path().join("db/self-downgrade/test-operation"); + std::fs::create_dir_all(&journal_dir).unwrap(); + std::fs::write( + journal_dir.join("journal.json"), + br#"{"phase":"preparing"}"#, + ) + .unwrap(); + assert!( + backend + .try_control_handle_state("source") + .await + .unwrap_err() + .to_string() + .contains("self_downgrade_recovery_required") + ); + } + + #[tokio::test] + async fn older_schema_and_missing_active_run_fall_back_without_cleanup() { + let (home, backend) = fixture().await; + let writer = backend.db().await.unwrap().write(); + writer + .execute_unprepared("DELETE FROM run WHERE sandbox_id = 1") + .await + .unwrap(); + let runtime = home.path().join("sandboxes/source/runtime"); + std::fs::create_dir_all(&runtime).unwrap(); + let hint = runtime.join("runtime-boot-id"); + std::fs::write(&hint, b"do-not-clean-during-observation").unwrap(); + assert!( + backend + .try_control_handle_state("source") + .await + .unwrap() + .is_none() + ); + assert!(hint.exists()); + let last = schema_metadata::migration_ids().last().unwrap(); + writer + .execute_raw(Statement::from_sql_and_values( + DatabaseBackend::Sqlite, + "DELETE FROM seaql_migrations WHERE version = ?", + [last.into()], + )) + .await + .unwrap(); + assert!( + backend + .try_control_handle_state("source") + .await + .unwrap() + .is_none() + ); + assert!(hint.exists()); + } +} diff --git a/sdk/rust/lib/backend/local/mod.rs b/sdk/rust/lib/backend/local/mod.rs index e68ee3639..38509a514 100644 --- a/sdk/rust/lib/backend/local/mod.rs +++ b/sdk/rust/lib/backend/local/mod.rs @@ -15,6 +15,7 @@ //! the bulk of the old global config singleton plus the SQLite pool, so multiple //! backends can hold different configurations for tests / migrations. +mod control_lookup; mod sandbox; use std::{ @@ -129,6 +130,15 @@ impl LocalBackend { profile: Option, ) -> Self { let config = load_persisted_config_or_default().unwrap_or_default(); + Self::lazy_with_config(config, selection_source, profile) + } + + /// Reuse the configuration document already read by ambient profile resolution. + pub(crate) fn lazy_with_config( + config: LocalConfig, + selection_source: BackendSelectionSource, + profile: Option, + ) -> Self { Self { config: Arc::new(config), db: OnceCell::new(), diff --git a/sdk/rust/lib/backend/local/sandbox/mod.rs b/sdk/rust/lib/backend/local/sandbox/mod.rs index a33c694ed..94c559fc6 100644 --- a/sdk/rust/lib/backend/local/sandbox/mod.rs +++ b/sdk/rust/lib/backend/local/sandbox/mod.rs @@ -640,7 +640,7 @@ impl LocalBackend { } /// Extract a live PID from a run record, if the process is still alive. - fn pid_from_run(run: Option<&run_entity::Model>) -> Option { + pub(super) fn pid_from_run(run: Option<&run_entity::Model>) -> Option { run.and_then(|model| model.pid) .filter(|pid| Self::pid_is_alive(*pid)) } @@ -792,7 +792,7 @@ impl LocalBackend { } /// Whether `pid` refers to a live process. - fn pid_is_alive(pid: i32) -> bool { + pub(super) fn pid_is_alive(pid: i32) -> bool { microsandbox_utils::process::pid_is_alive(pid) } diff --git a/sdk/rust/lib/backend/profile.rs b/sdk/rust/lib/backend/profile.rs index ed4734bef..9303be0b9 100644 --- a/sdk/rust/lib/backend/profile.rs +++ b/sdk/rust/lib/backend/profile.rs @@ -100,9 +100,15 @@ enum BackendSelection { /// Missing file → `Ok(SdkConfig::default())`. Malformed JSON → `Err`. /// Honours `MSB_CONFIG_PATH` env override for the file path. pub fn load_sdk_config() -> MicrosandboxResult { + load_sdk_config_document().map(|(config, _)| config) +} + +/// Keep the source document for the local half of ambient backend resolution. Local field +/// errors retain the lazy backend's existing default fallback; SDK profile errors remain fatal. +fn load_sdk_config_document() -> MicrosandboxResult<(SdkConfig, Option)> { let path = sdk_config_path(); if !path.exists() { - return Ok(SdkConfig::default()); + return Ok((SdkConfig::default(), None)); } let raw = fs::read_to_string(&path).map_err(|e| { MicrosandboxError::InvalidConfig(format!( @@ -119,7 +125,7 @@ pub fn load_sdk_config() -> MicrosandboxResult { path.display() )) })?; - Ok(cfg) + Ok((cfg, Some(raw))) } /// Resolve the default backend according to the Q1 precedence ladder. @@ -155,7 +161,13 @@ pub fn resolve_default_backend() -> MicrosandboxResult> { )); } - let cfg = load_sdk_config()?; + let (cfg, document) = load_sdk_config_document()?; + let local_config = || { + document + .as_deref() + .and_then(|raw| serde_json::from_str::(raw).ok()) + .unwrap_or_default() + }; let env_profile = std::env::var("MSB_PROFILE").ok(); let selection = select_backend( backend_kind.as_deref(), @@ -165,7 +177,8 @@ pub fn resolve_default_backend() -> MicrosandboxResult> { )?; match selection { - BackendSelection::Local => Ok(Arc::new(LocalBackend::lazy_with_selection( + BackendSelection::Local => Ok(Arc::new(LocalBackend::lazy_with_config( + local_config(), BackendSelectionSource::Default, None, ))), @@ -205,7 +218,15 @@ pub fn resolve_default_backend() -> MicrosandboxResult> { } else { BackendSelectionSource::ActiveProfile }; - backend_from_profile(&name, profile, source) + if profile.backend == ProfileBackend::Local { + Ok(Arc::new(LocalBackend::lazy_with_config( + local_config(), + source, + Some(name), + ))) + } else { + backend_from_profile(&name, profile, source) + } } } } diff --git a/sdk/rust/lib/sandbox/pause.rs b/sdk/rust/lib/sandbox/pause.rs index 9a0d3938c..2e508dc5b 100644 --- a/sdk/rust/lib/sandbox/pause.rs +++ b/sdk/rust/lib/sandbox/pause.rs @@ -21,7 +21,10 @@ impl Sandbox { pub async fn get_for_control(name: &str) -> MicrosandboxResult { let backend = crate::backend::default_backend(); if let Some(local) = backend.as_local() { - let (model, pid) = local.sandbox_handle_state(name).await?; + let (model, pid) = match local.try_control_handle_state(name).await? { + Some(target) => target, + None => local.sandbox_handle_state(name).await?, + }; return Ok(SandboxHandle::from_local_model(backend, model, pid)); } backend.sandboxes().get(backend.clone(), name).await From 86873fa68806914a8417cd0fa4e5f6eaa068105b Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Thu, 10 Sep 2026 02:06:02 +0100 Subject: [PATCH 15/29] fix(node): release sandbox ownership locks before guest work Admit operations through short-lived shared-handle locks so pending exec, filesystem I/O and shutdown waits cannot block pause or resume. Preserve consumed-handle rejection while admitted operations retain ownership. Decode filesystem error envelopes before their payloads and propagate paused-runtime diagnostics through unary and streaming operations. Add ownership, cancellation and response-decoding tests, six live concurrency cases on macOS and Linux, and the release timing report. Both hosts passed all live cases and 137 Node unit tests; focused Rust tests, TypeScript checks and native linting also passed. --- docs/sdk/typescript/sandbox.mdx | 4 + .../node-lifecycle-concurrency-2026-09-10.md | 50 ++++ sdk/node-ts/native/fs.rs | 50 ++-- sdk/node-ts/native/lib.rs | 1 + sdk/node-ts/native/sandbox.rs | 147 +++++------- sdk/node-ts/native/shared_handle.rs | 102 +++++++++ sdk/node-ts/src/sandbox.ts | 5 + .../tests/lifecycle-concurrency.test.ts | 215 ++++++++++++++++++ sdk/rust/lib/sandbox/fs.rs | 145 ++++++++++-- 9 files changed, 575 insertions(+), 144 deletions(-) create mode 100644 scripts/smoke/reports/node-lifecycle-concurrency-2026-09-10.md create mode 100644 sdk/node-ts/native/shared_handle.rs create mode 100644 sdk/node-ts/tests/lifecycle-concurrency.test.ts diff --git a/docs/sdk/typescript/sandbox.mdx b/docs/sdk/typescript/sandbox.mdx index 12525ac83..d05556a97 100644 --- a/docs/sdk/typescript/sandbox.mdx +++ b/docs/sdk/typescript/sandbox.mdx @@ -259,6 +259,8 @@ A running `Sandbox` also exposes two read-only properties: `name` (`string`, the Command execution (`exec`, `execWith`, `execStream`, `execStreamWith`, `shell`, `shellStream`) and foreground attachment (`attach`, `attachWith`, `attachShell`) live on the [Execution](/sdk/typescript/execution) page. +Lifecycle calls do not wait for unrelated guest operations on the same handle to finish. For example, you can pause and resume while an `exec()` promise is pending. Await operations explicitly when their order matters; stopping a sandbox may interrupt pending commands or filesystem requests. + #### sandbox.config() ```typescript @@ -301,6 +303,8 @@ await sandbox.detach(); // keeps running in the background Release the handle without stopping the sandbox. The sandbox continues running as a background process. Reconnect later with [`Sandbox.get()`](#sandbox-get). +After detaching, this handle and its existing `fs()` objects reject new guest operations. Already-started operations retain their connection; await them before detaching if you need their results first. + #### sandbox.fs() ```typescript diff --git a/scripts/smoke/reports/node-lifecycle-concurrency-2026-09-10.md b/scripts/smoke/reports/node-lifecycle-concurrency-2026-09-10.md new file mode 100644 index 000000000..c5b1408ed --- /dev/null +++ b/scripts/smoke/reports/node-lifecycle-concurrency-2026-09-10.md @@ -0,0 +1,50 @@ +# Node lifecycle concurrency qualification — 2026-09-10 + +The approved Node ownership change is implemented. The wrapper now acquires an `Arc` under a short admission lock instead of holding the lock across guest execution, filesystem I/O, or lifecycle waits. Normal operations do not clone the sandbox configuration. Detach/removal consume the shared slot, so subsequent calls through the wrapper or an existing filesystem facade fail; admitted operations retain their references. The runtime still decides whether concurrent operations are valid. This is not a guarantee that work completes successfully after stopping or removing its sandbox. + +Live tests also found an existing filesystem response-decoding bug: the host relay rejects new guest work while paused using `core.error`, but filesystem helpers attempted to deserialize it as `FsResponse` or ignored it on streams. They now preserve the existing diagnostic through the SDK's existing unexpected-response helper. The wire protocol, snapshot format, and public method signatures are unchanged. No additional state lookup, guest round trip, or retry was introduced. + +## Release timing + +Each sample measures the Node promise from invocation to completion using `performance.now()`, while a command on the same object is already executing and waiting for an explicit release marker. There are 20 pause/resume pairs per host in the final sample. Sandbox configuration is Alpine, 256 MiB RAM, and a 512 MiB managed root disk. These are SDK call timings, not CLI startup, snapshot restore, or application-response timings. + +| Host | Pause median | Resume median | Pause min–max | Resume min–max | +| --- | --- | --- | --- | --- | +| macOS ARM64/HVF | 0.216 ms | 0.182 ms | 0.187–0.407 ms | 0.170–0.367 ms | +| Linux x86-64/KVM | 0.195 ms | 0.313 ms | 0.178–0.457 ms | 0.299–0.549 ms | + +A freshly built old macOS Node binding, using the same runtime and firmware, fails the pending-exec regression at the 2,000 ms pause deadline. The command intentionally waits indefinitely for a marker, so this is a demonstrated lock blockage, not a baseline latency distribution or a meaningful speedup ratio. The new binding completes pause/resume without releasing that command, then verifies the original command completes correctly. The old binding's failure and final candidate reports are retained separately. CH/FC and the snapshot/restore benchmark matrix were not rerun in this follow-up. + +## Live coverage + +Fixture: `sdk/node-ts/tests/lifecycle-concurrency.test.ts`, enabled with `MSB_NODE_LIFECYCLE_LIVE=1`. Set `MSB_NODE_TIMINGS` to save raw call timings. Use a short isolated `MSB_HOME` and explicitly select the matching development `MSB_PATH` and `MSB_LIBKRUNFW_PATH`. + +| Case | macOS ARM64 | Linux x86-64 | +| --- | --- | --- | +| Pending exec: 20 pause/resume cycles, public paused state, original command completion | Pass | Pass | +| Filesystem request while paused: useful rejection and successful read after resume | Pass | Pass | +| Detach during exec: consumed-handle errors, idempotent detach, original command and VM survive | Pass | Pass | +| Blocked filesystem upload: pause/resume/detach progress, existing facade rejects new work, admitted upload finishes with correct bytes | Pass | Pass | +| Removal during exec: wrapper consumption is independent of runtime lifecycle locking; admitted command completes | Pass | Pass | +| Terminal-state wait concurrent with stop, followed by successful removal and consumed-handle errors | Pass | Pass | + +The upload fixture uses a host FIFO and observes the first bytes in the guest before pausing, proving the guest stream has started rather than racing its initial admission. The removal fixture permits the runtime's existing lifecycle lock to defer removal until shutdown or reject it. If removal wins before stop's final database observation, a missing-row stop error is accepted only after verifying that the concurrent removal succeeded. The tests do not weaken runtime lifecycle locking or silently ignore arbitrary cleanup errors. + +Final live test-body totals, including VM creation and cleanup: macOS 1.72 seconds; Linux approximately 13.53 seconds. The longer Linux fixture total includes shutdown and is not the pause/resume latency shown above. Both final runs passed all six cases. No sandbox records or matching runtime processes remained in either isolated test home after cleanup. + +Other validation: 137 Node unit tests passed on each host; three Rust shared-ownership tests passed (pending operation, competing consumers, cancellation/lifetime); five Rust filesystem response tests passed (normal success/failure, paused diagnostic, unexpected envelope, read-stream rejection, terminal response without waiting for channel close); TypeScript build/typecheck and separate live-fixture typecheck passed; Node native Clippy with `-D warnings`, focused Rust formatting, and `git diff --check` passed. + +Initial attempts are retained rather than counted as passes: the first Mac home exceeded Unix socket path limits; Linux's first runner installation omitted its optional native bundler dependency; macOS archive metadata sidecars were initially collected as tests on Linux and were excluded with `--exclude '**/._*'` while all 137 actual tests ran. Early upload/removal assertions raced guest admission or incorrectly imposed a 2-second shutdown deadline; the final fixture checks the actual contracts above. The paused-filesystem decoding failure was a product bug and was fixed, not suppressed in the test. + +## Artifacts and scope + +- Local stage: `/private/tmp/msb-resident-perf.qHpm0p`; candidate package `node-candidate`, old binding `node-baseline`, final reports `node-live-results-final.json`, `node-timings-final.json`, and `node-unit-results-final.json`. Linux live/timing reports are also copied here with a `linux-` prefix. +- Linux stage: `/home/ubuntu/msb-resident-perf.zqr4vR`; final reports use the same names. Runtime `/bin/msb` under that stage is the previously qualified lifecycle candidate; live tests explicitly override the SDK's prebuilt discovery paths. +- Test homes: `/private/tmp/nl.u0biju` and `/home/ubuntu/nl.eRfSLi`. Test sandboxes were removed; image caches and reports are retained. +- macOS candidate Node binding SHA-256: `3ee1a932985d102e6977cf23da37a63eac374429906dbf02a8e0df2f5814147f`. +- macOS old Node binding SHA-256: `dae1ef24eb8d9716521f2c803d3c258d690e8902d7af8a2e4e53c12f9fab4bca`. +- Linux candidate Node binding SHA-256: `5467985868389923447bf0495b0cf7b42d8f49619b9b9ae20b4d7182170fda33`. + +Runtime and guest-firmware provenance is recorded in [the resident lifecycle report](resident-lifecycle-2026-09-10.md). The local worktree advanced from `18c8fb86` to `e20269e1` through another contributor's archive commit during this work; those unrelated changes were preserved. Linux uses the existing isolated lifecycle source stage plus this Node/FS patch. Qualification preceded the separately authorized commit and push. + +This qualifies the Node ownership follow-up on macOS ARM64 and Linux x86-64, not every mobility invariant. Windows, Linux ARM64, cloud execution, every SSH/streaming variant, retained-handle boot fencing, Python ownership optimization, and companion clock-patch integration are outside this run. No generated binding declarations, dependency versions, lockfiles, or submodule pointers were changed by this work. diff --git a/sdk/node-ts/native/fs.rs b/sdk/node-ts/native/fs.rs index 4f645dc91..13e512a88 100644 --- a/sdk/node-ts/native/fs.rs +++ b/sdk/node-ts/native/fs.rs @@ -9,6 +9,7 @@ use napi_derive::napi; use tokio::sync::Mutex; use crate::error::to_napi_error; +use crate::shared_handle::SharedHandle; use crate::types::*; //-------------------------------------------------------------------------------------------------- @@ -18,7 +19,7 @@ use crate::types::*; /// Filesystem operations on a running sandbox (via agent protocol). #[napi(js_name = "SandboxFsOps")] pub struct JsSandboxFsOps { - sandbox: Arc>>, + sandbox: Arc>, } pub type JsSandboxFs = JsSandboxFsOps; @@ -48,7 +49,7 @@ pub struct JsFsWriteSink { //-------------------------------------------------------------------------------------------------- impl JsSandboxFsOps { - pub fn new(sandbox: Arc>>) -> Self { + pub(crate) fn new(sandbox: Arc>) -> Self { Self { sandbox } } } @@ -58,8 +59,7 @@ impl JsSandboxFsOps { /// Read a file as a Buffer. #[napi] pub async fn read(&self, path: String) -> Result { - let guard = self.sandbox.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.sandbox.get().await.ok_or_else(consumed_error)?; let data = sb.fs().read(&path).await.map_err(to_napi_error)?; Ok(data.to_vec().into()) } @@ -67,8 +67,7 @@ impl JsSandboxFsOps { /// Read a file as a UTF-8 string. #[napi] pub async fn read_string(&self, path: String) -> Result { - let guard = self.sandbox.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.sandbox.get().await.ok_or_else(consumed_error)?; sb.fs().read_to_string(&path).await.map_err(to_napi_error) } @@ -76,16 +75,14 @@ impl JsSandboxFsOps { #[napi] pub async fn write(&self, path: String, data: Buffer) -> Result<()> { let bytes: Vec = data.to_vec(); - let guard = self.sandbox.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.sandbox.get().await.ok_or_else(consumed_error)?; sb.fs().write(&path, &bytes).await.map_err(to_napi_error) } /// List directory contents. #[napi] pub async fn list(&self, path: String) -> Result> { - let guard = self.sandbox.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.sandbox.get().await.ok_or_else(consumed_error)?; let entries = sb.fs().list(&path).await.map_err(to_napi_error)?; Ok(entries.iter().map(fs_entry_to_js).collect()) } @@ -93,48 +90,42 @@ impl JsSandboxFsOps { /// Create a directory. #[napi] pub async fn mkdir(&self, path: String) -> Result<()> { - let guard = self.sandbox.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.sandbox.get().await.ok_or_else(consumed_error)?; sb.fs().mkdir(&path).await.map_err(to_napi_error) } /// Remove a directory. #[napi] pub async fn remove_dir(&self, path: String) -> Result<()> { - let guard = self.sandbox.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.sandbox.get().await.ok_or_else(consumed_error)?; sb.fs().remove_dir(&path).await.map_err(to_napi_error) } /// Remove a file. #[napi] pub async fn remove(&self, path: String) -> Result<()> { - let guard = self.sandbox.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.sandbox.get().await.ok_or_else(consumed_error)?; sb.fs().remove(&path).await.map_err(to_napi_error) } /// Copy a file within the sandbox. #[napi] pub async fn copy(&self, from: String, to: String) -> Result<()> { - let guard = self.sandbox.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.sandbox.get().await.ok_or_else(consumed_error)?; sb.fs().copy(&from, &to).await.map_err(to_napi_error) } /// Rename a file within the sandbox. #[napi] pub async fn rename(&self, from: String, to: String) -> Result<()> { - let guard = self.sandbox.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.sandbox.get().await.ok_or_else(consumed_error)?; sb.fs().rename(&from, &to).await.map_err(to_napi_error) } /// Get file or directory metadata. #[napi] pub async fn stat(&self, path: String) -> Result { - let guard = self.sandbox.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.sandbox.get().await.ok_or_else(consumed_error)?; let meta = sb.fs().stat(&path).await.map_err(to_napi_error)?; Ok(fs_metadata_to_js(&meta)) } @@ -142,16 +133,14 @@ impl JsSandboxFsOps { /// Check if a path exists. #[napi] pub async fn exists(&self, path: String) -> Result { - let guard = self.sandbox.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.sandbox.get().await.ok_or_else(consumed_error)?; sb.fs().exists(&path).await.map_err(to_napi_error) } /// Copy a file from the host into the sandbox. #[napi] pub async fn copy_from_host(&self, host_path: String, guest_path: String) -> Result<()> { - let guard = self.sandbox.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.sandbox.get().await.ok_or_else(consumed_error)?; sb.fs() .copy_from_host(&host_path, &guest_path) .await @@ -161,8 +150,7 @@ impl JsSandboxFsOps { /// Copy a file from the sandbox to the host. #[napi] pub async fn copy_to_host(&self, guest_path: String, host_path: String) -> Result<()> { - let guard = self.sandbox.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.sandbox.get().await.ok_or_else(consumed_error)?; sb.fs() .copy_to_host(&guest_path, &host_path) .await @@ -172,8 +160,7 @@ impl JsSandboxFsOps { /// Read a file with streaming (~3 MiB chunks). #[napi(js_name = "readStream")] pub async fn read_stream(&self, path: String) -> Result { - let guard = self.sandbox.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.sandbox.get().await.ok_or_else(consumed_error)?; let stream = sb.fs().read_stream(&path).await.map_err(to_napi_error)?; Ok(JsFsReadStream { inner: Arc::new(Mutex::new(stream)), @@ -183,8 +170,7 @@ impl JsSandboxFsOps { /// Write a file with streaming. Returns a sink the caller writes to. #[napi(js_name = "writeStream")] pub async fn write_stream(&self, path: String) -> Result { - let guard = self.sandbox.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.sandbox.get().await.ok_or_else(consumed_error)?; let sink = sb.fs().write_stream(&path).await.map_err(to_napi_error)?; Ok(JsFsWriteSink { inner: Arc::new(Mutex::new(Some(sink))), diff --git a/sdk/node-ts/native/lib.rs b/sdk/node-ts/native/lib.rs index cb85ad043..7171799f7 100644 --- a/sdk/node-ts/native/lib.rs +++ b/sdk/node-ts/native/lib.rs @@ -31,6 +31,7 @@ mod sandbox_builder; mod sandbox_handle; mod secret_builder; mod setup; +mod shared_handle; mod snapshot; mod snapshot_builder; mod ssh; diff --git a/sdk/node-ts/native/sandbox.rs b/sdk/node-ts/native/sandbox.rs index 4f043effe..580266e6f 100644 --- a/sdk/node-ts/native/sandbox.rs +++ b/sdk/node-ts/native/sandbox.rs @@ -13,6 +13,7 @@ use crate::exec::{ExecOutput, JsExecHandle}; use crate::exec_options_builder::JsExecOptionsBuilder; use crate::fs::JsSandboxFs; use crate::sandbox_handle::JsSandboxHandle; +use crate::shared_handle::SharedHandle; use crate::ssh::{JsSshClient, JsSshServer, apply_client_options, apply_server_options}; use crate::types::*; @@ -26,7 +27,7 @@ use crate::types::*; /// to the guest VM and can execute commands, access the filesystem, and query metrics. #[napi] pub struct Sandbox { - inner: Arc>>, + inner: Arc>, backend_kind: &'static str, } @@ -73,7 +74,7 @@ impl Sandbox { pub fn from_rust(inner: microsandbox::sandbox::Sandbox) -> Self { let backend_kind = inner.backend_kind().as_str(); Sandbox { - inner: Arc::new(Mutex::new(Some(inner))), + inner: Arc::new(SharedHandle::new(inner)), backend_kind, } } @@ -175,16 +176,14 @@ impl Sandbox { /// Sandbox name. Names are limited to 128 UTF-8 bytes. #[napi(getter)] pub async fn name(&self) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; Ok(sb.name().to_string()) } /// Whether this handle owns the sandbox lifecycle (attached mode). #[napi(getter)] pub async fn owns_lifecycle(&self) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; Ok(sb.owns_lifecycle()) } @@ -193,8 +192,7 @@ impl Sandbox { /// The TS layer parses + camelCase-remaps this into a plain object. #[napi(js_name = "configJson")] pub async fn config_json(&self) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; serde_json::to_string(sb.config()) .map_err(|e| napi::Error::from_reason(format!("failed to serialize config: {e}"))) } @@ -206,8 +204,7 @@ impl Sandbox { /// Execute the sandbox's effective OCI entrypoint and CMD. #[napi] pub async fn exec_default(&self) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let output = sb.exec_default().await.map_err(to_napi_error)?; Ok(ExecOutput::from_rust(output)) } @@ -219,8 +216,7 @@ impl Sandbox { builder: &mut JsExecOptionsBuilder, ) -> Result { let opts_builder = builder.take_inner_builder()?; - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let output = sb .exec_default_with(|_default| opts_builder) .await @@ -231,8 +227,7 @@ impl Sandbox { /// Execute the sandbox's effective OCI entrypoint and CMD with streaming I/O. #[napi] pub async fn exec_default_stream(&self) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let handle = sb.exec_default_stream().await.map_err(to_napi_error)?; Ok(JsExecHandle::from_rust(handle)) } @@ -244,8 +239,7 @@ impl Sandbox { builder: &mut JsExecOptionsBuilder, ) -> Result { let opts_builder = builder.take_inner_builder()?; - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let handle = sb .exec_default_stream_with(|_default| opts_builder) .await @@ -256,8 +250,7 @@ impl Sandbox { /// Execute a command and wait for completion. #[napi] pub async fn exec(&self, cmd: String, args: Option>) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let args_owned = args.unwrap_or_default(); let output = sb.exec(&cmd, args_owned).await.map_err(to_napi_error)?; Ok(ExecOutput::from_rust(output)) @@ -272,8 +265,7 @@ impl Sandbox { builder: &mut JsExecOptionsBuilder, ) -> Result { let opts_builder = builder.take_inner_builder()?; - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let output = sb .exec_with(&cmd, |_default| opts_builder) .await @@ -288,8 +280,7 @@ impl Sandbox { cmd: String, args: Option>, ) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let args_owned = args.unwrap_or_default(); let handle = sb .exec_stream(&cmd, args_owned) @@ -309,8 +300,7 @@ impl Sandbox { builder: &mut JsExecOptionsBuilder, ) -> Result { let opts_builder = builder.take_inner_builder()?; - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let handle = sb .exec_stream_with(&cmd, |_default| opts_builder) .await @@ -321,8 +311,7 @@ impl Sandbox { /// Execute a shell command using the sandbox's configured shell. #[napi] pub async fn shell(&self, script: String) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let output = sb.shell(&script).await.map_err(to_napi_error)?; Ok(ExecOutput::from_rust(output)) } @@ -330,8 +319,7 @@ impl Sandbox { /// Execute a shell command with streaming I/O. #[napi] pub async fn shell_stream(&self, script: String) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let handle = sb.shell_stream(&script).await.map_err(to_napi_error)?; Ok(JsExecHandle::from_rust(handle)) } @@ -353,8 +341,7 @@ impl Sandbox { /// Connect a native in-process SSH client to this sandbox. #[napi(js_name = "sshConnect")] pub async fn ssh_connect(&self, options: Option) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let client = sb .ssh() .connect_with(|builder| apply_client_options(options, builder)) @@ -366,8 +353,7 @@ impl Sandbox { /// Prepare a reusable SSH server endpoint for this sandbox. #[napi(js_name = "sshServer")] pub async fn ssh_server(&self, options: Option) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let server = sb .ssh() .server_with(|builder| apply_server_options(options, builder)) @@ -383,8 +369,7 @@ impl Sandbox { /// Get point-in-time resource metrics. #[napi] pub async fn metrics(&self) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let m = sb.metrics().await.map_err(to_napi_error)?; Ok(metrics_to_js(&m)) } @@ -396,8 +381,7 @@ impl Sandbox { /// Check whether agentd is reachable without refreshing idle activity. #[napi] pub async fn ping(&self) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let result = sb.ping().await.map_err(to_napi_error)?; Ok(sandbox_ping_result_to_js(result)) } @@ -405,8 +389,7 @@ impl Sandbox { /// Explicitly refresh this sandbox's idle activity timer. #[napi] pub async fn touch(&self) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let result = sb.touch().await.map_err(to_napi_error)?; Ok(sandbox_touch_result_to_js(result)) } @@ -416,8 +399,7 @@ impl Sandbox { #[napi] pub async fn modify(&self, options: Option) -> Result { let builder = { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; configure_modify(sb.modify(), options.as_ref())? }; run_modify(builder, modify_dry_run(options.as_ref())).await @@ -426,18 +408,15 @@ impl Sandbox { /// Compact the immutable disk prefix; the count includes the base, not the writable head. #[napi] pub async fn compact(&self, layers: Option, dry_run: Option) -> Result { - let builder = { - let guard = self.inner.lock().await; - guard.as_ref().ok_or_else(consumed_error)?.compact() - }; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; + let builder = sb.compact(); run_compact(builder, layers, dry_run.unwrap_or(false)).await } /// Stream metrics snapshots at the requested interval (in milliseconds). #[napi] pub async fn metrics_stream(&self, interval_ms: f64) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let interval = Duration::from_millis(interval_ms as u64); let mut stream = Box::pin(sb.metrics_stream(interval)); @@ -463,8 +442,7 @@ impl Sandbox { /// Attach to the sandbox's effective OCI entrypoint and CMD. #[napi] pub async fn attach_default(&self) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; sb.attach_default().await.map_err(to_napi_error) } @@ -475,8 +453,7 @@ impl Sandbox { builder: &mut JsAttachOptionsBuilder, ) -> Result { let opts_builder = builder.take_inner_builder()?; - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; sb.attach_default_with(|_default| opts_builder) .await .map_err(to_napi_error) @@ -487,8 +464,7 @@ impl Sandbox { /// Bridges the host terminal to the guest process. Returns the exit code. #[napi] pub async fn attach(&self, cmd: String, args: Option>) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let args_owned = args.unwrap_or_default(); sb.attach(&cmd, args_owned).await.map_err(to_napi_error) } @@ -502,8 +478,7 @@ impl Sandbox { builder: &mut JsAttachOptionsBuilder, ) -> Result { let opts_builder = builder.take_inner_builder()?; - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; sb.attach_with(&cmd, |_default| opts_builder) .await .map_err(to_napi_error) @@ -512,8 +487,7 @@ impl Sandbox { /// Attach to the sandbox's default shell. #[napi] pub async fn attach_shell(&self) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; sb.attach_shell().await.map_err(to_napi_error) } @@ -524,16 +498,14 @@ impl Sandbox { /// Stop the sandbox gracefully and wait for it to exit. #[napi] pub async fn stop(&self) -> Result<()> { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; sb.stop().await.map_err(to_napi_error) } /// Create an independent local CoW child without a durable full snapshot. #[napi] pub async fn branch(&self, name: String) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; Ok(Sandbox::from_rust( sb.branch(name).await.map_err(to_napi_error)?, )) @@ -542,24 +514,21 @@ impl Sandbox { /// Explicit resident pause through host control. #[napi] pub async fn pause(&self) -> Result<()> { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; sb.pause().await.map_err(to_napi_error) } /// Explicit resident resume through host control. #[napi] pub async fn resume(&self) -> Result<()> { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; sb.resume().await.map_err(to_napi_error) } /// Stop and wait for exit, returning the exit status. #[napi] pub async fn stop_and_wait(&self) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let status = sb.stop_and_wait().await.map_err(to_napi_error)?; Ok(exit_status_to_js(status)) } @@ -567,16 +536,14 @@ impl Sandbox { /// Request graceful shutdown without waiting for observed exit. #[napi] pub async fn request_stop(&self) -> Result<()> { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; sb.request_stop().await.map_err(to_napi_error) } /// Stop gracefully with an explicit timeout before escalating to SIGKILL. #[napi] pub async fn stop_with_timeout(&self, timeout_ms: u32) -> Result<()> { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let timeout = Duration::from_millis(timeout_ms.into()); sb.stop_with_timeout(timeout).await.map_err(to_napi_error) } @@ -584,24 +551,21 @@ impl Sandbox { /// Kill the sandbox immediately and wait for observed exit. #[napi] pub async fn kill(&self) -> Result<()> { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; sb.kill().await.map_err(to_napi_error) } /// Request force termination without waiting for observed exit. #[napi] pub async fn request_kill(&self) -> Result<()> { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; sb.request_kill().await.map_err(to_napi_error) } /// Force-kill the sandbox with an explicit observation timeout. #[napi] pub async fn kill_with_timeout(&self, timeout_ms: u32) -> Result<()> { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let timeout = Duration::from_millis(timeout_ms.into()); sb.kill_with_timeout(timeout).await.map_err(to_napi_error) } @@ -609,24 +573,21 @@ impl Sandbox { /// Graceful drain (SIGUSR1 — for load balancing). #[napi] pub async fn drain(&self) -> Result<()> { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; sb.drain().await.map_err(to_napi_error) } /// Request graceful drain without waiting for observed exit. #[napi] pub async fn request_drain(&self) -> Result<()> { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; sb.request_drain().await.map_err(to_napi_error) } /// Wait until the sandbox is observed in a terminal non-running state. #[napi] pub async fn wait_until_stopped(&self) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let result = sb.wait_until_stopped().await.map_err(to_napi_error)?; Ok(sandbox_stop_result_to_js(result)) } @@ -634,27 +595,29 @@ impl Sandbox { /// Wait for the sandbox process to exit. #[napi(js_name = "wait")] pub async fn wait_for_exit(&self) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let status = sb.wait().await.map_err(to_napi_error)?; Ok(exit_status_to_js(status)) } /// Detach from the sandbox — it will continue running after this handle is dropped. + /// New operations are rejected; already admitted operations retain their connection. #[napi] pub async fn detach(&self) -> Result<()> { - let mut guard = self.inner.lock().await; - if let Some(sb) = guard.take() { - sb.detach().await; + if let Some(sb) = self.inner.take().await { + // Only detach consumes the Rust value. Ordinary operations clone the Arc, + // not the sandbox configuration; admitted operations keep their reference. + Arc::unwrap_or_clone(sb).detach().await; } Ok(()) } /// Remove the persisted database record after stopping. + /// Consumes this wrapper even on failure. Already admitted operations may finish or + /// fail at the runtime boundary; removal does not wait for guest operations to drain. #[napi] pub async fn remove_persisted(&self) -> Result<()> { - let mut guard = self.inner.lock().await; - let sb = guard.take().ok_or_else(consumed_error)?; + let sb = self.inner.take().await.ok_or_else(consumed_error)?; sb.remove_persisted().await.map_err(to_napi_error) } @@ -665,8 +628,7 @@ impl Sandbox { /// protocol traffic. #[napi] pub async fn logs(&self, opts: Option) -> Result> { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let rust_opts = log_options_from_js(opts).map_err(napi::Error::from_reason)?; let entries = sb.logs(&rust_opts).await.map_err(to_napi_error)?; Ok(entries.into_iter().map(log_entry_to_js).collect()) @@ -680,8 +642,7 @@ impl Sandbox { /// entry. #[napi] pub async fn log_stream(&self, opts: Option) -> Result { - let guard = self.inner.lock().await; - let sb = guard.as_ref().ok_or_else(consumed_error)?; + let sb = self.inner.get().await.ok_or_else(consumed_error)?; let rust_opts = log_stream_options_from_js(opts).map_err(napi::Error::from_reason)?; let stream = sb.log_stream(&rust_opts).await.map_err(to_napi_error)?; spawn_log_stream_from_stream(stream).await diff --git a/sdk/node-ts/native/shared_handle.rs b/sdk/node-ts/native/shared_handle.rs new file mode 100644 index 000000000..12f4fe7bc --- /dev/null +++ b/sdk/node-ts/native/shared_handle.rs @@ -0,0 +1,102 @@ +use std::sync::Arc; + +use tokio::sync::Mutex; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// A consumable wrapper whose admitted operations own their reference independently. +/// +/// The slot lock orders admission against consumption, not guest work against lifecycle work. +/// Holding it across guest I/O would let a paused guest prevent its own resume. +pub(crate) struct SharedHandle { + inner: Mutex>>, +} + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl SharedHandle { + pub(crate) fn new(value: T) -> Self { + Self { + inner: Mutex::new(Some(Arc::new(value))), + } + } + + /// Admit an operation without cloning the underlying sandbox configuration. + pub(crate) async fn get(&self) -> Option> { + self.inner.lock().await.clone() + } + + /// Reject future admissions; already admitted operations retain their references. + pub(crate) async fn take(&self) -> Option> { + self.inner.lock().await.take() + } +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + + #[tokio::test] + async fn pending_operation_does_not_block_admission_or_consumption() { + let handle = Arc::new(SharedHandle::new(String::from("sandbox"))); + let (admitted_tx, admitted_rx) = tokio::sync::oneshot::channel(); + let (finish_tx, finish_rx) = tokio::sync::oneshot::channel(); + let task_handle = handle.clone(); + let operation = tokio::spawn(async move { + let reference = task_handle.get().await.unwrap(); + admitted_tx.send(()).unwrap(); + finish_rx.await.unwrap(); + assert_eq!(reference.as_str(), "sandbox"); + }); + admitted_rx.await.unwrap(); + + tokio::time::timeout(Duration::from_secs(1), async { + let second = handle.get().await.unwrap(); + let consumed = handle.take().await.unwrap(); + assert!(Arc::ptr_eq(&second, &consumed)); + assert!(handle.get().await.is_none()); + assert!(handle.take().await.is_none()); + }) + .await + .expect("a pending operation must not hold the admission lock"); + finish_tx.send(()).unwrap(); + operation.await.unwrap(); + } + + #[tokio::test] + async fn concurrent_consumers_have_exactly_one_winner() { + let handle = SharedHandle::new(()); + let (first, second) = tokio::join!(handle.take(), handle.take()); + assert_ne!(first.is_some(), second.is_some()); + assert!(handle.get().await.is_none()); + } + + #[tokio::test] + async fn cancellation_releases_last_admitted_reference() { + let handle = Arc::new(SharedHandle::new(())); + let reference = handle.get().await.unwrap(); + let weak = Arc::downgrade(&reference); + let (admitted_tx, admitted_rx) = tokio::sync::oneshot::channel(); + let operation = tokio::spawn(async move { + let _reference = reference; + admitted_tx.send(()).unwrap(); + std::future::pending::<()>().await; + }); + admitted_rx.await.unwrap(); + drop(handle.take().await); + assert!(weak.upgrade().is_some()); + operation.abort(); + assert!(operation.await.unwrap_err().is_cancelled()); + assert!(weak.upgrade().is_none()); + } +} diff --git a/sdk/node-ts/src/sandbox.ts b/sdk/node-ts/src/sandbox.ts index 1de23b1d6..414a2a32c 100644 --- a/sdk/node-ts/src/sandbox.ts +++ b/sdk/node-ts/src/sandbox.ts @@ -537,6 +537,11 @@ export class Sandbox implements AsyncDisposable { ); } + /** + * Consume this handle without stopping the sandbox. New guest and filesystem + * operations on this handle are rejected; already admitted operations retain + * their connection. Await operations first when their completion matters. + */ async detach(): Promise { await withMappedErrors(() => this.inner.detach()); } diff --git a/sdk/node-ts/tests/lifecycle-concurrency.test.ts b/sdk/node-ts/tests/lifecycle-concurrency.test.ts new file mode 100644 index 000000000..283ce4afb --- /dev/null +++ b/sdk/node-ts/tests/lifecycle-concurrency.test.ts @@ -0,0 +1,215 @@ +import { createRequire } from "node:module"; +import { execFileSync } from "node:child_process"; +import { mkdtemp, open, rmdir, unlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { performance } from "node:perf_hooks"; +import { setTimeout as delay } from "node:timers/promises"; +import { describe, expect, it } from "vitest"; +import { Sandbox, SandboxNotFoundError } from "../dist/index.js"; + +const native: typeof import("../native/index.js") = createRequire(import.meta.url)("../native/index.cjs"); +const consumed = /Sandbox handle has been consumed/; + +async function bounded(promise: Promise, label: string, timeoutMs = 2_000): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`${label} timed out after ${timeoutMs} ms`)), timeoutMs); + }), + ]); + } finally { + clearTimeout(timer); + } +} + +async function fixture(label: string, run: (source: Sandbox, observer: Sandbox) => Promise) { + const name = `node-concurrency-${label}-${process.pid}`; + const source = await Sandbox.builder(name).image("alpine").rootDisk(512).memory(256).create(); + let observer: Sandbox | undefined; + try { + observer = await (await Sandbox.get(name)).connect(); + await run(source, observer); + } finally { + // A separate handle can recover a paused VM even if the tested wrapper is + // consumed or regresses to holding its lock while waiting for the guest. + const handle = await Sandbox.get(name).catch(error => { + if (error instanceof SandboxNotFoundError) return undefined; + throw error; + }); + if (handle) { + if (handle.status === "paused") await handle.resume(); + await handle.stop(); + await Sandbox.remove(name); + } + } +} + +async function gatedExec(source: Sandbox, observer: Sandbox) { + const pending = source.exec("sh", ["-c", "touch /dev/shm/started; while [ ! -e /dev/shm/release ]; do sleep 0.02; done; echo completed"]); + // Attach a rejection handler immediately so fixture cleanup cannot produce + // an unhandled rejection if a later assertion fails and stops the VM. + void pending.catch(() => undefined); + await bounded((async () => { + while (!(await observer.fs().exists("/dev/shm/started"))) await delay(10); + })(), "guest admission"); + return { pending }; +} + +describe.skipIf(process.env.MSB_NODE_LIFECYCLE_LIVE !== "1")("same-object lifecycle concurrency", () => { + it("pauses and resumes while exec is pending, then completes the original exec", async () => { + await fixture("exec", async (source, observer) => { + const { pending } = await gatedExec(source, observer); + let settled = false; + void pending.finally(() => { settled = true; }).catch(() => undefined); + const pauses: number[] = []; + const resumes: number[] = []; + for (let cycle = 0; cycle < 20; cycle++) { + let started = performance.now(); + await bounded(source.pause(), "pause during exec"); + pauses.push(performance.now() - started); + expect((await Sandbox.get(source.name)).status).toBe("paused"); + started = performance.now(); + await bounded(source.resume(), "resume during exec"); + resumes.push(performance.now() - started); + expect(settled).toBe(false); + } + await observer.fs().write("/dev/shm/release", "go"); + expect((await bounded(pending, "exec completion")).stdout().trim()).toBe("completed"); + const timings = { platform: process.platform, arch: process.arch, pauses_ms: pauses, resumes_ms: resumes }; + console.log(JSON.stringify(timings)); + if (process.env.MSB_NODE_TIMINGS) await writeFile(process.env.MSB_NODE_TIMINGS, JSON.stringify(timings, null, 2)); + }); + }); + + it("does not let a filesystem request to a paused guest block resume", async () => { + await fixture("fs", async (source) => { + const fs = source.fs(); + await fs.write("/dev/shm/payload", "retained"); + await source.pause(); + const read = fs.readToString("/dev/shm/payload"); + // Both a prompt paused-state refusal and a read waiting for resume are + // valid; neither is allowed to monopolize the wrapper lock. + const outcome = read.then(value => ({ value }), error => ({ error })); + await delay(50); + await bounded(source.resume(), "resume during filesystem read"); + const result = await bounded(outcome, "filesystem completion"); + if ("error" in result) expect(String(result.error)).toMatch(/paused/i); + else expect(result.value).toBe("retained"); + expect(await fs.readToString("/dev/shm/payload")).toBe("retained"); + }); + }); + + it("detach consumes new calls and existing filesystem facades without draining exec", async () => { + await fixture("detach", async (source, observer) => { + const fs = source.fs(); + const { pending } = await gatedExec(source, observer); + await bounded(source.detach(), "detach during exec"); + await expect(source.pause()).rejects.toThrow(consumed); + await expect(source.exec("true")).rejects.toThrow(consumed); + await expect(fs.exists("/dev/shm/started")).rejects.toThrow(consumed); + await source.detach(); // Detach remains idempotent. + await observer.fs().write("/dev/shm/release", "go"); + expect((await bounded(pending, "detached exec completion")).stdout().trim()).toBe("completed"); + expect((await observer.exec("echo", ["alive"])).stdout().trim()).toBe("alive"); + }); + }); + + it.skipIf(process.platform === "win32")("pause, resume and detach progress during a blocked filesystem upload", async () => { + await fixture("upload", async (source, observer) => { + const directory = await mkdtemp(join(tmpdir(), "msb-node-upload-")); + const fifo = join(directory, "input"); + execFileSync("mkfifo", [fifo]); + const fs = source.fs(); + const upload = fs.copyFromHost(fifo, "/dev/shm/uploaded"); + void upload.catch(() => undefined); + // Opening the writer proves the native filesystem operation has opened + // its reader. With no bytes or EOF, the upload cannot complete yet. + const writer = await open(fifo, "w"); + let closed = false; + try { + await writer.writeFile("initial "); + await bounded((async () => { + while (true) { + try { + if ((await observer.fs().stat("/dev/shm/uploaded")).size >= 8) break; + } catch (error) { + if (!String(error).includes("No such file")) throw error; + } + await delay(10); + } + })(), "guest upload admission"); + await bounded(source.pause(), "pause during upload"); + await bounded(source.resume(), "resume during upload"); + await bounded(source.detach(), "detach during upload"); + await expect(fs.exists("/tmp")).rejects.toThrow(consumed); + await writer.writeFile("private upload payload"); + await writer.close(); + closed = true; + await bounded(upload, "admitted upload after detach"); + expect(await observer.fs().readToString("/dev/shm/uploaded")).toBe("initial private upload payload"); + } finally { + if (!closed) await writer.close(); + await unlink(fifo); + await rmdir(directory); + } + }); + }); + + it("removal consumes its wrapper while exec is pending, independently of lifecycle locking", async () => { + await fixture("remove", async (source, observer) => { + const raw = await (await native.Sandbox.get(source.name)).connect(); + const fs = raw.fs(); + const pending = raw.exec("sh", ["-c", "touch /dev/shm/started; while [ ! -e /dev/shm/release ]; do sleep 0.02; done; echo completed"]); + void pending.catch(() => undefined); + await bounded((async () => { + while (!(await observer.fs().exists("/dev/shm/started"))) await delay(10); + })(), "native exec admission"); + const removal = raw.removePersisted().then(() => ({ removed: true }), error => ({ error })); + await delay(50); + await expect(bounded(raw.pause(), "consumed pause")).rejects.toThrow(consumed); + await expect(bounded(fs.exists("/dev/shm/started"), "consumed filesystem")).rejects.toThrow(consumed); + await expect(raw.removePersisted()).rejects.toThrow(consumed); + await observer.fs().write("/dev/shm/release", "go"); + expect((await bounded(pending, "exec after failed removal")).success).toBe(true); + // Runtime lifecycle locking can defer removal until shutdown or reject + // the running VM. Neither should keep the Node admission slot locked. + const stopError = await observer.stop().then(() => undefined, error => error); + const result = await bounded(removal, "removal after shutdown", 10_000); + if (stopError !== undefined) { + // Removal can win after the VM exits but before stop's final database + // observation. Accept a missing row only when this removal succeeded. + expect(stopError).toBeInstanceOf(SandboxNotFoundError); + expect("removed" in result).toBe(true); + } + if ("error" in result) expect(String(result.error)).toMatch(/running|stopped|lock|timed out/i); + }); + }); + + it("a terminal-state waiter does not block stopping and successful removal", async () => { + const name = `node-concurrency-stopped-${process.pid}`; + const raw = await new native.SandboxBuilder(name).image("alpine").rootDisk(512).memory(256).create(); + try { + const fs = raw.fs(); + const waiter = raw.waitUntilStopped(); + void waiter.catch(() => undefined); + await delay(50); + await bounded(raw.stop(), "stop during wait", 10_000); + await bounded(waiter, "terminal-state observation"); + await raw.removePersisted(); + await expect(raw.resume()).rejects.toThrow(consumed); + await expect(fs.exists("/tmp")).rejects.toThrow(consumed); + await expect(native.Sandbox.get(name)).rejects.toThrow(); + } finally { + // Successful removal already deleted the record; only recover a fixture + // that still exists. Unexpected cleanup errors must remain visible. + const handle = await Sandbox.get(name).catch(() => undefined); + if (handle) { + await handle.stop(); + await Sandbox.remove(name); + } + } + }); +}); diff --git a/sdk/rust/lib/sandbox/fs.rs b/sdk/rust/lib/sandbox/fs.rs index 2170f57f6..cface406c 100644 --- a/sdk/rust/lib/sandbox/fs.rs +++ b/sdk/rust/lib/sandbox/fs.rs @@ -551,9 +551,10 @@ impl FsReadStream { return Ok(Some(Bytes::from(chunk.data))); } } - MessageType::FsResponse => { - let resp: FsResponse = msg.payload()?; + MessageType::FsResponse | MessageType::CoreError => { + let response = filesystem_response(msg); let close_result = self.close_owned_handle().await; + let resp = response?; if !resp.ok { return Err(MicrosandboxError::SandboxFsOps( resp.error.unwrap_or_else(|| "unknown error".into()), @@ -685,9 +686,22 @@ fn entry_info_to_metadata(info: &FsEntryInfo) -> FsMetadata { } } +/// Check the envelope before decoding: a paused runtime rejects new work with +/// `core.error`, which has no filesystem `ok` field. Do not hide that diagnostic +/// behind a CBOR decoding error (or silently discard it on a stream). +fn filesystem_response(msg: Message) -> MicrosandboxResult { + if msg.t != MessageType::FsResponse { + return Err(super::unexpected_agent_response( + "filesystem operation", + &msg, + )); + } + Ok(msg.payload()?) +} + /// Deserialize and check a simple ok/error `FsResponse`. fn check_response(msg: Message) -> MicrosandboxResult<()> { - let resp: FsResponse = msg.payload()?; + let resp = filesystem_response(msg)?; if resp.ok { Ok(()) } else { @@ -700,7 +714,7 @@ fn check_response(msg: Message) -> MicrosandboxResult<()> { /// Wait for and check a terminal `FsResponse` from a subscription channel. async fn wait_for_ok_response(rx: &mut mpsc::Receiver) -> MicrosandboxResult<()> { while let Some(msg) = rx.recv().await { - if msg.t == MessageType::FsResponse { + if matches!(msg.t, MessageType::FsResponse | MessageType::CoreError) { return check_response(msg); } } @@ -742,10 +756,7 @@ pub(crate) mod agent { use bytes::Bytes; use microsandbox_protocol::{ - fs::{ - FS_CHUNK_SIZE, FsData, FsOp, FsOpenOptions, FsRequest, FsResponse, FsResponseData, - FsSetAttrs, - }, + fs::{FS_CHUNK_SIZE, FsData, FsOp, FsOpenOptions, FsRequest, FsResponseData, FsSetAttrs}, message::MessageType, }; use tokio::io::AsyncReadExt; @@ -754,7 +765,7 @@ pub(crate) mod agent { use super::{ FsEntry, FsHandle, FsMetadata, FsReadStream, FsWriteSink, check_response, - entry_info_to_fs_entry, entry_info_to_metadata, wait_for_ok_response, + entry_info_to_fs_entry, entry_info_to_metadata, filesystem_response, wait_for_ok_response, }; /// Open a fresh agent connection for the named sandbox. @@ -785,7 +796,7 @@ pub(crate) mod agent { }, }; let resp_msg = client.request(MessageType::FsRequest, &req).await?; - let resp: FsResponse = resp_msg.payload()?; + let resp = filesystem_response(resp_msg)?; if !resp.ok { return Err(MicrosandboxError::SandboxFsOps( resp.error.unwrap_or_else(|| "unknown error".into()), @@ -806,7 +817,7 @@ pub(crate) mod agent { }, }; let resp_msg = client.request(MessageType::FsRequest, &req).await?; - let resp: FsResponse = resp_msg.payload()?; + let resp = filesystem_response(resp_msg)?; if !resp.ok { return Err(MicrosandboxError::SandboxFsOps( resp.error.unwrap_or_else(|| "unknown error".into()), @@ -909,7 +920,7 @@ pub(crate) mod agent { op: FsOp::ReadDir { handle, limit }, }; let resp_msg = client.request(MessageType::FsRequest, &req).await?; - let resp: FsResponse = resp_msg.payload()?; + let resp = filesystem_response(resp_msg)?; if !resp.ok { return Err(MicrosandboxError::SandboxFsOps( @@ -933,7 +944,7 @@ pub(crate) mod agent { op: FsOp::FStat { handle }, }; let resp_msg = client.request(MessageType::FsRequest, &req).await?; - let resp: FsResponse = resp_msg.payload()?; + let resp = filesystem_response(resp_msg)?; if !resp.ok { return Err(MicrosandboxError::SandboxFsOps( @@ -985,8 +996,8 @@ pub(crate) mod agent { let chunk: FsData = msg.payload()?; data.extend_from_slice(&chunk.data); } - MessageType::FsResponse => { - let resp: FsResponse = msg.payload()?; + MessageType::FsResponse | MessageType::CoreError => { + let resp = filesystem_response(msg)?; if !resp.ok { return Err(MicrosandboxError::SandboxFsOps( resp.error.unwrap_or_else(|| "unknown error".into()), @@ -1094,7 +1105,7 @@ pub(crate) mod agent { }, }; let resp_msg = client.request(MessageType::FsRequest, &req).await?; - let resp: FsResponse = resp_msg.payload()?; + let resp = filesystem_response(resp_msg)?; if !resp.ok { return Err(MicrosandboxError::SandboxFsOps( @@ -1219,7 +1230,7 @@ pub(crate) mod agent { }, }; let resp_msg = client.request(MessageType::FsRequest, &req).await?; - let resp: FsResponse = resp_msg.payload()?; + let resp = filesystem_response(resp_msg)?; if !resp.ok { return Err(MicrosandboxError::SandboxFsOps( @@ -1266,7 +1277,7 @@ pub(crate) mod agent { }, }; let resp_msg = client.request(MessageType::FsRequest, &req).await?; - let resp: FsResponse = resp_msg.payload()?; + let resp = filesystem_response(resp_msg)?; if !resp.ok { return Err(MicrosandboxError::SandboxFsOps( @@ -1311,7 +1322,7 @@ pub(crate) mod agent { }, }; let resp_msg = client.request(MessageType::FsRequest, &req).await?; - let resp: FsResponse = resp_msg.payload()?; + let resp = filesystem_response(resp_msg)?; if !resp.ok { return Err(MicrosandboxError::SandboxFsOps( @@ -1370,6 +1381,102 @@ pub(crate) mod agent { } } +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use microsandbox_protocol::core::{CoreError, CoreErrorKind}; + + use super::*; + + fn paused_response() -> Message { + Message::with_payload( + MessageType::CoreError, + 1, + &CoreError { + kind: CoreErrorKind::InvalidSession, + message: "sandbox is paused; resume it before starting guest work".into(), + offending_type: None, + workload_failure: None, + }, + ) + .unwrap() + } + + #[test] + fn filesystem_error_preserves_paused_diagnostic() { + let error = filesystem_response(paused_response()).unwrap_err(); + assert!(matches!(error, MicrosandboxError::Runtime(_))); + assert!(error.to_string().contains("sandbox is paused")); + } + + #[test] + fn filesystem_response_rejects_unexpected_envelope() { + let mut message = paused_response(); + message.t = MessageType::Pong; + let error = filesystem_response(message).unwrap_err(); + assert!(error.to_string().contains("agent returned")); + } + + #[test] + fn filesystem_response_retains_normal_success_and_failure() { + for ok in [true, false] { + let response = FsResponse { + ok, + error: (!ok).then(|| "permission denied".to_string()), + data: None, + }; + let message = Message::with_payload(MessageType::FsResponse, 1, &response).unwrap(); + let result = check_response(message); + if ok { + assert!(result.is_ok()); + } else { + assert!(matches!(result, Err(MicrosandboxError::SandboxFsOps(_)))); + } + } + } + + #[tokio::test] + async fn filesystem_read_stream_preserves_rejection_instead_of_eof() { + let (tx, rx) = mpsc::channel(1); + tx.send(paused_response()).await.unwrap(); + let mut stream = FsReadStream { + rx, + client: None, + close_handle: None, + }; + let result = tokio::time::timeout(std::time::Duration::from_secs(1), stream.recv()) + .await + .expect("a read rejection must not be discarded"); + assert!( + result + .unwrap_err() + .to_string() + .contains("sandbox is paused") + ); + } + + #[tokio::test] + async fn filesystem_stream_rejection_does_not_wait_for_channel_close() { + let (tx, mut rx) = mpsc::channel(1); + tx.send(paused_response()).await.unwrap(); + let result = tokio::time::timeout( + std::time::Duration::from_secs(1), + wait_for_ok_response(&mut rx), + ) + .await + .expect("core.error must terminate the stream even with its sender alive"); + assert!( + result + .unwrap_err() + .to_string() + .contains("sandbox is paused") + ); + } +} + //-------------------------------------------------------------------------------------------------- // Re-Exports //-------------------------------------------------------------------------------------------------- From 3c3d9d0be2aef363935b7a2a975b9c1ef0e230de Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Thu, 10 Sep 2026 02:25:18 +0100 Subject: [PATCH 16/29] feat(snapshot)!: organize snapshots into groups with explicit heads Add group-scoped names and immutable member directories while preserving portable snapshot identities across independent imports. Advance heads only through known ancestry and retain concurrent sibling captures. Carry source ancestry through capture, restore and local branch. Serialize publication and retain cursor locks during cancellation. Expose group selection and import options across the CLI and Rust, Python, Node and Go SDKs. Update the path-keyed index, archive aliases and downgrade checks. Cover group publication, duplicate imports, archive dependencies and legacy artifact paths with automated tests and isolated macOS live matrices for managed, flat and tmpfs roots. BREAKING CHANGE: Bare snapshot names now select group heads. Address exact members as group:member or use an explicit artifact path. Installed group members cannot be overwritten with force. --- COMPATIBILITY.md | 6 +- crates/cli/lib/commands/self_cmd.rs | 63 +- crates/cli/lib/commands/snapshot.rs | 155 +++- crates/db/lib/entity/snapshot.rs | 14 +- crates/migration/lib/lib.rs | 2 + .../lib/m20260910_000001_snapshot_groups.rs | 185 ++++ crates/migration/lib/schema_metadata.rs | 11 + docs/sandboxes/snapshots.mdx | 135 ++- docs/sdk/go/snapshots.mdx | 69 +- docs/sdk/python/snapshots.mdx | 76 +- docs/sdk/rust/snapshots.mdx | 106 ++- docs/sdk/typescript/snapshots.mdx | 68 +- docs/snapshot-groups-explained.md | 132 +++ .../microsandbox-types/rust/lib/domain.rs | 20 +- scripts/smoke/cli/dirty-memory-checkpoint.py | 21 +- scripts/smoke/cli/snapshot-groups.py | 194 +++++ .../reports/snapshot-groups-2026-09-10.md | 110 +++ sdk/go/integration/snapshot_test.go | 45 +- sdk/go/internal/ffi/ffi.go | 152 +++- sdk/go/native/microsandbox_go_ffi.h | 17 + sdk/go/native/src/lib.rs | 72 +- sdk/go/snapshot.go | 109 ++- sdk/go/snapshot_test.go | 46 +- sdk/node-ts/native/index.d.ts | 37 +- sdk/node-ts/native/snapshot.rs | 95 +- sdk/node-ts/native/snapshot_builder.rs | 17 +- sdk/node-ts/src/index.ts | 6 +- sdk/node-ts/src/internal/napi.ts | 23 + sdk/node-ts/src/snapshot-handle.ts | 10 +- sdk/node-ts/src/snapshot.ts | 52 +- sdk/node-ts/tests/cow-lifecycle.test.ts | 4 +- .../tests/unit/native-contract.test.ts | 9 + sdk/node-ts/tests/unit/snapshot.test.ts | 43 +- sdk/python/integration/test_snapshots.py | 16 +- sdk/python/microsandbox/_microsandbox.pyi | 14 +- sdk/python/src/snapshot.rs | 93 +- sdk/rust/lib/backend/local/sandbox/create.rs | 1 + sdk/rust/lib/lib.rs | 8 +- sdk/rust/lib/sandbox/branch.rs | 4 + sdk/rust/lib/sandbox/builder.rs | 1 + sdk/rust/lib/sandbox/config.rs | 5 + sdk/rust/lib/sandbox/modify.rs | 110 ++- sdk/rust/lib/snapshot/archive.rs | 342 ++++++-- sdk/rust/lib/snapshot/create.rs | 210 ++++- sdk/rust/lib/snapshot/downgrade.rs | 3 +- sdk/rust/lib/snapshot/group.rs | 819 ++++++++++++++++++ sdk/rust/lib/snapshot/group_tests.rs | 515 +++++++++++ sdk/rust/lib/snapshot/lineage.rs | 357 ++++++++ sdk/rust/lib/snapshot/migration.rs | 15 +- sdk/rust/lib/snapshot/mod.rs | 63 +- sdk/rust/lib/snapshot/store.rs | 505 +++++++---- sdk/rust/tests/snapshot_artifact.rs | 186 +++- 52 files changed, 4768 insertions(+), 603 deletions(-) create mode 100644 crates/migration/lib/m20260910_000001_snapshot_groups.rs create mode 100644 docs/snapshot-groups-explained.md create mode 100644 scripts/smoke/cli/snapshot-groups.py create mode 100644 scripts/smoke/reports/snapshot-groups-2026-09-10.md create mode 100644 sdk/rust/lib/snapshot/group.rs create mode 100644 sdk/rust/lib/snapshot/group_tests.rs create mode 100644 sdk/rust/lib/snapshot/lineage.rs diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index 82c2c7298..6ef5a47a0 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -186,10 +186,14 @@ Parsers and mutators must validate the complete supported feature set before the ## 9. Snapshots, Manifests, and Portable Archives -Snapshot descriptor bytes are identity-bearing: their canonical bytes determine the snapshot ID. Compatibility-sensitive elements include field order, required `null` values, map ordering, duplicate-key handling, tag spellings, schema and integrity identifiers, payload names, parent identities, state/scope/format variants, extension requirements, and translation-graph behavior. +Snapshot descriptors carry a stable random `snap_...` ID; their canonical bytes determine the descriptor digest, not that ID. Compatibility-sensitive elements include field order, required `null` values, map ordering, duplicate-key handling, tag spellings, schema and integrity identifiers, payload names, parent identities, state/scope/format variants, extension requirements, and translation-graph behavior. Archive compatibility includes compression detection, `archive.json`, canonical inventory order, transport digests, accepted path grammar, legacy paths, cache-closure entries, and rejection of duplicate, missing, or escaping paths. +Installed snapshots now live under `snapshots///`. `group.json` selects a head; `group-member.json` stores a local friendly name without changing descriptor identity. Bare selectors mean a group head, and `group:member` selects an exact member. Existing flat artifacts remain readable by explicit path; this change does not silently move their directories. The index keys local artifact paths rather than globally unique portable IDs/digests, so importing the same snapshot into two groups preserves both copies. Downgrade refuses grouped state before rewriting artifacts or rolling back the index. + +Capture records the actual source snapshot lineage in the existing descriptor `parent` field. Per-sandbox cursor publication serializes captures without holding a VM pause; group head publication is locked separately. Automatic head advancement requires known ancestry, not capture timestamps, export dependency bases, or import order. An explicit head selection may rewind or choose a sibling. Missing ancestry may prevent advancement but is not a missing payload dependency. Archives optionally carry friendly names in `msb-snapshot-member-names`; their snapshot IDs, payload paths and descriptor schema are unchanged. + Unreleased #8 incremental exports use `completeness: "dependent"` and the must-understand `msb-snapshot-dependencies-v1` extension. `--since` records omitted physical disk-prefix layers and reusable RAM-object identities; `--last-layers` only omits disk layers. The complete target memory manifest and CPU/device state remain included. Loading and direct archive restore resolve the explicitly supplied base into owned staging before opening the complete target. This replaces the unreleased disk-only dependency encoding without a compatibility shim or snapshot descriptor change. Readers that do not understand this requirement refuse it; ordinary standalone archives are unchanged. Evolution rules: diff --git a/crates/cli/lib/commands/self_cmd.rs b/crates/cli/lib/commands/self_cmd.rs index 33f653b72..690f4b479 100644 --- a/crates/cli/lib/commands/self_cmd.rs +++ b/crates/cli/lib/commands/self_cmd.rs @@ -1102,6 +1102,15 @@ async fn run_downgrade_with_db( } if ctx.operation.phase() < DowngradePhase::DatabaseReverted { + if fresh_plan + .rollback + .iter() + .any(|migration| migration.id == schema_metadata::SNAPSHOT_GROUPS_MIGRATION_ID) + { + // Refuse before any artifact rewrite. A grouped tree cannot be represented + // by the target's flat namespace, even if its rebuildable index is missing. + refuse_snapshot_group_downgrade(ctx.db.inner(), ctx.snapshots_dir).await?; + } let reverses_legacy_snapshots = fresh_plan.rollback.iter().any(|migration| { migration.id == schema_metadata::SNAPSHOT_ARTIFACT_TRANSITION_MIGRATION_ID }); @@ -2512,6 +2521,34 @@ async fn applied_migrations(db: &DatabaseConnection) -> anyhow::Result anyhow::Result<()> { + let grouped = optional_count( + db, + "SELECT COUNT(*) FROM snapshot_index WHERE group_path IS NOT NULL OR group_name IS NOT NULL", + ).await?; + let mut grouped_on_disk = false; + if snapshots_dir.exists() { + for entry in fs::read_dir(snapshots_dir)? { + let path = entry?.path(); + // The metadata file is the on-disk namespace marker. Refuse even an empty or + // unindexed group rather than making it disappear from the older CLI. + if path.join("group.json").exists() { + grouped_on_disk = true; + break; + } + } + } + if grouped > 0 || grouped_on_disk { + anyhow::bail!( + "snapshot groups prevent downgrade: retain this version or export and remove snapshot groups before retrying" + ); + } + Ok(()) +} + async fn user_data_warnings(db: &DatabaseConnection) -> anyhow::Result> { let snapshot_count = optional_count(db, "SELECT COUNT(*) FROM snapshot_index").await?; let disk_volume_count = optional_count( @@ -3497,6 +3534,27 @@ mod tests { assert_eq!(row.try_get_by_index::(0).unwrap(), "wal-value"); } + #[tokio::test] + async fn downgrade_refuses_unindexed_group_before_artifact_mutation() { + let dir = tempfile::tempdir().unwrap(); + let db = sea_orm::Database::connect("sqlite::memory:").await.unwrap(); + Migrator::up(&db, None).await.unwrap(); + let snapshots = dir.path().join("snapshots"); + let group = snapshots.join("unindexed"); + fs::create_dir_all(&group).unwrap(); + let state = br#"{"schema":"microsandbox.snapshot-group/1","head":null}"#; + fs::write(group.join("group.json"), state).unwrap(); + let error = refuse_snapshot_group_downgrade(&db, &snapshots) + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("snapshot groups prevent downgrade") + ); + assert_eq!(fs::read(group.join("group.json")).unwrap(), state); + } + #[tokio::test] async fn rollback_schema_steps_through_latest_migrations() { let dir = tempfile::tempdir().unwrap(); @@ -3510,7 +3568,10 @@ mod tests { .unwrap(); Migrator::up(db.inner(), None).await.unwrap(); - // Stable snapshot identity is the newest migration. With no snapshot + // Empty databases can drop grouped addressing without discarding any instances. + rollback_schema(db.inner(), 1).await.unwrap(); + + // With no snapshot // artifacts to translate, rollback removes its two rebuildable index // projections and leaves the earlier compatibility marker applied. rollback_schema(db.inner(), 1).await.unwrap(); diff --git a/crates/cli/lib/commands/snapshot.rs b/crates/cli/lib/commands/snapshot.rs index 5647b7a2a..7bc7fa2ba 100644 --- a/crates/cli/lib/commands/snapshot.rs +++ b/crates/cli/lib/commands/snapshot.rs @@ -45,21 +45,27 @@ pub enum SnapshotCommands { /// Load a snapshot archive into the snapshots directory. Load(SnapshotLoadArgs), + + /// Read a group's head, or select a member as its head. + Head(SnapshotHeadArgs), } /// Arguments for `msb snapshot create`. #[derive(Debug, Args)] pub struct SnapshotCreateArgs { - /// Snapshot name, resolved under `~/.microsandbox/snapshots//` - /// (or under `--dest-dir` when given). - pub name: String, + /// Snapshot member name (generated when omitted). + pub name: Option, + + /// Snapshot group to create or add to (defaults to the source sandbox name). + #[arg(long, value_name = "GROUP")] + pub group: Option, /// Source sandbox name. Disk capture also supports running and user-paused sources. #[arg(long, value_name = "SANDBOX")] pub from: String, /// Parent directory to create the artifact in, instead of the - /// default snapshots directory. The artifact lands at `DIR/`. + /// default snapshots directory. The group is created under this root. #[arg(long = "dest-dir", value_name = "DIR")] pub dest_dir: Option, @@ -75,7 +81,7 @@ pub struct SnapshotCreateArgs { #[arg(long = "label", value_name = "K=V")] pub labels: Vec, - /// Overwrite an existing artifact at the destination. + /// Overwrite an existing archive file; installed group members are immutable. #[arg(short = 'f', long)] pub force: bool, @@ -186,6 +192,25 @@ pub struct SnapshotLoadArgs { /// Exact base snapshot or standalone base archive for a dependent archive. #[arg(long)] pub base: Option, + + /// Import into this group (generated when omitted). + #[arg(long, value_name = "GROUP")] + pub group: Option, + + /// Select the imported member as head even if it is not a fast-forward. + #[arg(long)] + pub set_head: bool, +} + +/// Arguments for `msb snapshot head`. +#[derive(Debug, Args)] +pub struct SnapshotHeadArgs { + /// Group to read, or GROUP:MEMBER to select a new head. + pub selector: String, + + /// Output format (json). + #[arg(long, value_name = "FORMAT", value_parser = ["json"])] + pub format: Option, } //-------------------------------------------------------------------------------------------------- @@ -203,11 +228,15 @@ pub async fn run(args: SnapshotArgs) -> anyhow::Result<()> { SnapshotCommands::Reindex(args) => reindex(args).await, SnapshotCommands::Save(args) => save(args).await, SnapshotCommands::Load(args) => load(args).await, + SnapshotCommands::Head(args) => head(args).await, } } async fn create(args: SnapshotCreateArgs) -> anyhow::Result<()> { - let mut builder = Snapshot::builder(&args.name).from_sandbox(&args.from); + let mut builder = Snapshot::builder(args.name.unwrap_or_default()).from_sandbox(&args.from); + if let Some(group) = args.group { + builder = builder.group(group); + } if let Some(ref dest_dir) = args.dest_dir { builder = builder.dest_dir(dest_dir); } @@ -247,6 +276,9 @@ async fn create(args: SnapshotCreateArgs) -> anyhow::Result<()> { Ok(snap) => { spinner.finish_success("Snapshotted"); if !args.quiet { + if let Some(update) = snap.head_update() { + report_head_update(update); + } println!("{}", snap.id()); println!("{}", snap.path().display()); } @@ -267,8 +299,10 @@ async fn list(args: SnapshotListArgs) -> anyhow::Result<()> { .iter() .map(|s| { serde_json::json!({ + "snapshot_id": s.id(), "digest": s.digest(), "name": s.name(), + "group": s.group(), "parent_digest": s.parent_digest(), "scope": format_scope(s.scope()), "state_kind": s.state_kind(), @@ -312,7 +346,7 @@ async fn list(args: SnapshotListArgs) -> anyhow::Result<()> { "DIGEST", ]); for s in &snapshots { - let name = s.name().unwrap_or("-").to_string(); + let name = format_member_selector(s.group(), s.name(), s.id()); let size = s .size_bytes() .map(format_size) @@ -469,20 +503,58 @@ async fn save(args: SnapshotSaveArgs) -> anyhow::Result<()> { } async fn load(args: SnapshotLoadArgs) -> anyhow::Result<()> { - let handle = if let Some(base) = args.base.as_deref() { - Snapshot::load_with_base(&args.archive, args.dest.as_deref(), base).await? - } else { - Snapshot::load(&args.archive, args.dest.as_deref()).await? - }; + let handle = Snapshot::load_with_options( + &args.archive, + microsandbox::snapshot::LoadOpts { + dest: args.dest, + base: args.base, + group: args.group, + set_head: args.set_head, + }, + ) + .await?; + if let Some(update) = handle.head_update() { + report_head_update(update); + } println!("{}", handle.digest()); + // Keep the installed path as the final stdout line for shell consumers. println!("{}", handle.path().display()); Ok(()) } +async fn head(args: SnapshotHeadArgs) -> anyhow::Result<()> { + let update = Snapshot::group_head(&args.selector).await?; + if args.format.as_deref() == Some("json") { + println!("{}", serde_json::to_string_pretty(&update)?); + } else { + ui::detail_kv("Group", &update.group); + ui::detail_kv("Previous head", update.previous.as_deref().unwrap_or("-")); + ui::detail_kv("Head", &update.head); + ui::detail_kv("Reason", &format!("{:?}", update.reason)); + } + Ok(()) +} + //-------------------------------------------------------------------------------------------------- // Functions: Helpers //-------------------------------------------------------------------------------------------------- +fn format_member_selector(group: Option<&str>, name: Option<&str>, id: &str) -> String { + let member = name.unwrap_or(id); + // Friendly member names are scoped to one group; qualify them so rows stay distinct. + match group { + Some(group) => format!("{group}:{member}"), + None => member.to_string(), + } +} + +fn report_head_update(update: µsandbox::snapshot::HeadUpdate) { + eprintln!( + "group {}: head {} ({:?})", + update.group, update.head, update.reason + ); +} + fn format_str(f: microsandbox::SnapshotFormat) -> &'static str { match f { microsandbox::SnapshotFormat::Raw => "raw", @@ -570,7 +642,7 @@ mod tests { let SnapshotCommands::Create(args) = args.command else { panic!("expected create command"); }; - assert_eq!(args.name, "clean"); + assert_eq!(args.name.as_deref(), Some("clean")); assert_eq!(args.from, "box"); assert!(args.full); } @@ -645,4 +717,61 @@ mod tests { Some(std::path::Path::new("/tmp/snaps")) ); } + + #[test] + fn create_accepts_generated_member_in_explicit_group() { + let parsed = parse_snapshot_args(&["create", "--from", "box", "--group", "work"]); + let SnapshotCommands::Create(args) = parsed.command else { + panic!("expected create command"); + }; + assert!(args.name.is_none()); + assert_eq!(args.group.as_deref(), Some("work")); + assert_eq!(args.from, "box"); + } + + #[test] + fn load_accepts_group_and_explicit_head_selection() { + let parsed = parse_snapshot_args(&[ + "load", + "changes.msnap", + "--base", + "work:base", + "--group", + "work", + "--set-head", + ]); + let SnapshotCommands::Load(args) = parsed.command else { + panic!("expected load command"); + }; + assert_eq!(args.base.as_deref(), Some("work:base")); + assert_eq!(args.group.as_deref(), Some("work")); + assert!(args.set_head); + } + + #[test] + fn head_accepts_member_selector_and_json_format() { + let parsed = parse_snapshot_args(&["head", "work:baseline", "--format", "json"]); + let SnapshotCommands::Head(args) = parsed.command else { + panic!("expected head command"); + }; + assert_eq!(args.selector, "work:baseline"); + assert_eq!(args.format.as_deref(), Some("json")); + } + + #[test] + fn list_disambiguates_aliases_and_unnamed_members_by_group() { + assert_eq!( + format_member_selector(Some("work"), Some("base"), "snap_1"), + "work:base" + ); + assert_eq!( + format_member_selector(Some("copy"), Some("base"), "snap_1"), + "copy:base" + ); + assert_eq!( + format_member_selector(Some("copy"), None, "snap_1"), + "copy:snap_1" + ); + assert_eq!(format_member_selector(None, None, "snap_1"), "snap_1"); + } } diff --git a/crates/db/lib/entity/snapshot.rs b/crates/db/lib/entity/snapshot.rs index f17f7b849..0155a7021 100644 --- a/crates/db/lib/entity/snapshot.rs +++ b/crates/db/lib/entity/snapshot.rs @@ -14,16 +14,19 @@ use sea_orm::entity::prelude::*; #[derive(Clone, Debug, PartialEq, DeriveEntityModel)] #[sea_orm(table_name = "snapshot_index")] pub struct Model { - /// Released descriptor-digest primary key retained for database compatibility. - #[sea_orm(primary_key, auto_increment = false)] + /// Descriptor digest, shared by identical artifacts in different local groups. pub digest: String, /// Stable opaque snapshot identity. pub snapshot_id: Option, /// SHA-256 of canonical descriptor bytes. pub descriptor_digest: Option, - /// Convenience name (unique when present). NULL for digest-only entries. + /// Convenience name, unique within its group directory. pub name: Option, - /// Manifest digest of the parent snapshot, or NULL for a root. + /// Local group label, absent for explicitly opened ungrouped artifacts. + pub group_name: Option, + /// Absolute group directory, which scopes member aliases across storage roots. + pub group_path: Option, + /// Stable identity of the parent snapshot, or NULL for a root. pub parent_digest: Option, /// Snapshot payload scope (`disk` or `full`). pub scope: String, @@ -40,6 +43,7 @@ pub struct Model { /// Checkpoint-manifest digest for checkpoint state. pub checkpoint_manifest_digest: Option, /// Absolute path to the artifact directory on this host. + #[sea_orm(primary_key, auto_increment = false)] pub artifact_path: String, /// Apparent size of the upper file in bytes. pub size_bytes: Option, @@ -57,7 +61,7 @@ pub struct Model { pub created_at: DateTime, /// When this row was inserted/refreshed. pub indexed_at: DateTime, - /// Number of indexed snapshots whose `parent_digest == self.digest`. + /// Number of distinct child identities whose parent is this snapshot's stable identity. pub child_count: i32, } diff --git a/crates/migration/lib/lib.rs b/crates/migration/lib/lib.rs index 1550e361e..7ce0af370 100644 --- a/crates/migration/lib/lib.rs +++ b/crates/migration/lib/lib.rs @@ -25,6 +25,7 @@ mod m20260810_000001_rebuild_sandbox_labels; mod m20260813_000001_share_cpu_allocations; mod m20260824_000001_mount_owner_config; mod m20260829_000001_split_snapshot_identity; +mod m20260910_000001_snapshot_groups; pub mod schema_metadata; use sea_orm_migration::prelude::*; @@ -75,6 +76,7 @@ impl MigratorTrait for Migrator { Box::new(m20260813_000001_share_cpu_allocations::Migration), Box::new(m20260824_000001_mount_owner_config::Migration), Box::new(m20260829_000001_split_snapshot_identity::Migration), + Box::new(m20260910_000001_snapshot_groups::Migration), ] } } diff --git a/crates/migration/lib/m20260910_000001_snapshot_groups.rs b/crates/migration/lib/m20260910_000001_snapshot_groups.rs new file mode 100644 index 000000000..91f72bccc --- /dev/null +++ b/crates/migration/lib/m20260910_000001_snapshot_groups.rs @@ -0,0 +1,185 @@ +//! Index local snapshot instances separately from portable identities. + +use sea_orm_migration::{ + prelude::*, + sea_orm::{DatabaseBackend, Statement}, +}; + +//-------------------------------------------------------------------------------------------------- +// Constants +//-------------------------------------------------------------------------------------------------- + +const SHARED_COLUMNS: &str = "digest, snapshot_id, descriptor_digest, name, parent_digest, scope, state_kind, image_ref, image_manifest_digest, format, fstype, checkpoint_manifest_digest, artifact_path, size_bytes, locality, storage_binding_id, availability, migration_state, migration_error_code, created_at, indexed_at, child_count"; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +#[derive(DeriveMigrationName)] +pub struct Migration; + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + rebuild_index(manager, true).await + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + let connection = manager.get_connection(); + // Check before rebuilding: older binaries cannot address groups or retain several + // copies of one portable identity. Never silently discard rows to satisfy old keys. + let incompatible = connection + .query_one_raw(Statement::from_string( + DatabaseBackend::Sqlite, + "SELECT EXISTS(SELECT 1 FROM snapshot_index WHERE group_path IS NOT NULL OR group_name IS NOT NULL) OR EXISTS(SELECT 1 FROM snapshot_index GROUP BY digest HAVING COUNT(*) > 1) OR EXISTS(SELECT 1 FROM snapshot_index WHERE snapshot_id IS NOT NULL GROUP BY snapshot_id HAVING COUNT(*) > 1) OR EXISTS(SELECT 1 FROM snapshot_index WHERE name IS NOT NULL GROUP BY name HAVING COUNT(*) > 1) AS incompatible", + )) + .await? + .ok_or_else(|| DbErr::Custom("snapshot group downgrade preflight returned no row".into()))? + .try_get::("", "incompatible")?; + if incompatible != 0 { + return Err(DbErr::Custom( + "snapshot groups prevent downgrade: retain this version or export and remove grouped/duplicate snapshot instances before retrying".into(), + )); + } + rebuild_index(manager, false).await + } +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +async fn rebuild_index(manager: &SchemaManager<'_>, grouped: bool) -> Result<(), DbErr> { + let connection = manager.get_connection(); + connection + .execute_unprepared("ALTER TABLE snapshot_index RENAME TO snapshot_index_group_transition") + .await?; + let digest_key = if grouped { "" } else { " PRIMARY KEY" }; + let path_key = if grouped { " PRIMARY KEY" } else { "" }; + let group_columns = if grouped { + "group_name TEXT, group_path TEXT," + } else { + "" + }; + connection + .execute_unprepared(&format!( + "CREATE TABLE snapshot_index (digest TEXT NOT NULL{digest_key}, snapshot_id TEXT, descriptor_digest TEXT, name TEXT, {group_columns} parent_digest TEXT, scope TEXT NOT NULL, state_kind TEXT NOT NULL, image_ref TEXT NOT NULL, image_manifest_digest TEXT NOT NULL, format TEXT, fstype TEXT, checkpoint_manifest_digest TEXT, artifact_path TEXT NOT NULL{path_key}, size_bytes BIGINT, locality TEXT NOT NULL DEFAULT 'embedded', storage_binding_id TEXT, availability TEXT NOT NULL DEFAULT 'ready', migration_state TEXT NOT NULL DEFAULT 'canonical', migration_error_code TEXT, created_at DATETIME NOT NULL, indexed_at DATETIME NOT NULL, child_count INTEGER NOT NULL DEFAULT 0)" + )) + .await?; + connection + .execute_unprepared(&format!( + "INSERT INTO snapshot_index ({SHARED_COLUMNS}) SELECT {SHARED_COLUMNS} FROM snapshot_index_group_transition" + )) + .await?; + // Dropping the old table also releases its index names before creating replacements. + connection + .execute_unprepared("DROP TABLE snapshot_index_group_transition") + .await?; + let name_index = if grouped { + "CREATE UNIQUE INDEX idx_snapshot_index_name ON snapshot_index (group_path, name) WHERE group_path IS NOT NULL AND name IS NOT NULL" + } else { + "CREATE UNIQUE INDEX idx_snapshot_index_name ON snapshot_index (name) WHERE name IS NOT NULL" + }; + connection.execute_unprepared(name_index).await?; + let identity_unique = if grouped { "" } else { "UNIQUE " }; + connection.execute_unprepared(&format!("CREATE {identity_unique}INDEX idx_snapshot_index_snapshot_id ON snapshot_index (snapshot_id)")).await?; + for (name, column) in [ + ("idx_snapshot_index_digest", "digest"), + ("idx_snapshot_index_descriptor_digest", "descriptor_digest"), + ("idx_snapshot_index_parent", "parent_digest"), + ("idx_snapshot_index_image", "image_manifest_digest"), + ] { + connection + .execute_unprepared(&format!("CREATE INDEX {name} ON snapshot_index ({column})")) + .await?; + } + Ok(()) +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use sea_orm_migration::sea_orm::{Database, DatabaseConnection}; + + use super::*; + use crate::{Migrator, MigratorTrait}; + + async fn prior_database() -> DatabaseConnection { + let db = Database::connect("sqlite::memory:").await.unwrap(); + Migrator::up(&db, Some((Migrator::migrations().len() - 1) as u32)) + .await + .unwrap(); + db.execute_unprepared("INSERT INTO snapshot_index (digest, snapshot_id, descriptor_digest, name, scope, state_kind, image_ref, image_manifest_digest, artifact_path, created_at, indexed_at) VALUES ('sha256:original', 'snap_original', 'sha256:original', 'baseline', 'disk', 'file', 'example', 'sha256:image', '/old/baseline', '2026-09-10 00:00:00', '2026-09-10 00:00:00')").await.unwrap(); + db + } + + #[tokio::test] + async fn preserves_old_rows_and_round_trips_ungrouped_database() { + let db = prior_database().await; + Migrator::up(&db, None).await.unwrap(); + let row = db + .query_one_raw(Statement::from_string( + DatabaseBackend::Sqlite, + "SELECT digest, artifact_path, group_name FROM snapshot_index", + )) + .await + .unwrap() + .unwrap(); + assert_eq!( + row.try_get::("", "digest").unwrap(), + "sha256:original" + ); + assert_eq!( + row.try_get::("", "artifact_path").unwrap(), + "/old/baseline" + ); + assert_eq!( + row.try_get::>("", "group_name").unwrap(), + None + ); + Migrator::down(&db, Some(1)).await.unwrap(); + let row = db + .query_one_raw(Statement::from_string( + DatabaseBackend::Sqlite, + "SELECT digest FROM snapshot_index", + )) + .await + .unwrap() + .unwrap(); + assert_eq!( + row.try_get::("", "digest").unwrap(), + "sha256:original" + ); + } + + #[tokio::test] + async fn permits_duplicate_imports_and_refuses_lossy_downgrade() { + let db = prior_database().await; + Migrator::up(&db, None).await.unwrap(); + for group in ["first", "second"] { + db.execute_unprepared(&format!("INSERT INTO snapshot_index ({SHARED_COLUMNS}, group_name, group_path) SELECT digest, snapshot_id, descriptor_digest, name, parent_digest, scope, state_kind, image_ref, image_manifest_digest, format, fstype, checkpoint_manifest_digest, '/snapshots/{group}/baseline', size_bytes, locality, storage_binding_id, availability, migration_state, migration_error_code, created_at, indexed_at, child_count, '{group}', '/snapshots/{group}' FROM snapshot_index WHERE artifact_path = '/old/baseline'")).await.unwrap(); + } + let error = Migrator::down(&db, Some(1)).await.unwrap_err(); + assert!( + error + .to_string() + .contains("snapshot groups prevent downgrade") + ); + let row = db + .query_one_raw(Statement::from_string( + DatabaseBackend::Sqlite, + "SELECT COUNT(*) AS n FROM snapshot_index", + )) + .await + .unwrap() + .unwrap(); + assert_eq!(row.try_get::("", "n").unwrap(), 3); + } +} diff --git a/crates/migration/lib/schema_metadata.rs b/crates/migration/lib/schema_metadata.rs index c943688f4..bce291a04 100644 --- a/crates/migration/lib/schema_metadata.rs +++ b/crates/migration/lib/schema_metadata.rs @@ -52,6 +52,9 @@ pub const MOUNT_OWNER_CONFIG_MIGRATION_ID: &str = "m20260824_000001_mount_owner_ /// Migration that separates stable snapshot identity from descriptor integrity. pub const SNAPSHOT_IDENTITY_MIGRATION_ID: &str = "m20260829_000001_split_snapshot_identity"; +/// Migration that separates local group membership from portable snapshot identity. +pub const SNAPSHOT_GROUPS_MIGRATION_ID: &str = "m20260910_000001_snapshot_groups"; + /// Frozen migration baseline for the transitional 0.6.0 release. /// /// The released 0.6.0 binary predates `msb __schema-baseline --json`, so @@ -251,6 +254,13 @@ pub const MIGRATION_METADATA: &[MigrationMetadata] = &[ affects_user_data: true, summary: "reverse final snapshot descriptors before dropping identity projections", }, + MigrationMetadata { + id: SNAPSHOT_GROUPS_MIGRATION_ID, + reversible: true, + affects_cache: false, + affects_user_data: true, + summary: "restore the flat snapshot index only when no groups or duplicate identities remain", + }, ]; //-------------------------------------------------------------------------------------------------- @@ -337,6 +347,7 @@ mod tests { #[test] fn canonical_applied_prefix_uses_metadata_order() { let applied = [ + SNAPSHOT_GROUPS_MIGRATION_ID, SNAPSHOT_IDENTITY_MIGRATION_ID, MOUNT_OWNER_CONFIG_MIGRATION_ID, SHARED_CPU_ALLOCATION_MIGRATION_ID, diff --git a/docs/sandboxes/snapshots.mdx b/docs/sandboxes/snapshots.mdx index 3b489be34..646a8ae5b 100644 --- a/docs/sandboxes/snapshots.mdx +++ b/docs/sandboxes/snapshots.mdx @@ -18,7 +18,7 @@ Disk snapshots work with running, paused, stopped, or crashed sandboxes. A runni | Mode | Source | Captured state | Restore behavior | | ---- | ------ | -------------- | ---------------- | -| Disk (default) | Stopped or crashed sandbox | Writable disk closure and pinned image | Cold-boots a fresh VM | +| Disk (default) | Running, paused, stopped, or crashed sandbox | Writable disk closure and pinned image | Cold-boots a fresh VM | | Full | Running or user-paused sandbox | Disk, memory, vCPU, device, and admitted resource state | Resumes the captured execution in a child VM | Both modes produce the same schema-1 `snapshot.json` descriptor. Its closed `state.kind` is either `file` or `checkpoint`, and `root_disk.layout` records `managed`, `flat`, or `tmpfs`. Checkpoint payloads live in a content-addressed closure rather than a standalone disk file. A tmpfs root has no stopped disk snapshot because its writable state exists only in memory, but it is included in a running full snapshot; such a snapshot must be resumed and cannot use `--disk-only`. @@ -30,8 +30,8 @@ On supported Linux, macOS, and Windows hosts, add `--forked` when restoring a fu ```bash msb create alpine --name baseline --root-disk flat:1G msb snapshot create ready --from baseline --full -msb create --name worker-a --from-snapshot ready --forked -msb create --name worker-b --from-snapshot ready --forked +msb create --name worker-a --from-snapshot baseline:ready --forked +msb create --name worker-b --from-snapshot baseline:ready --forked ``` The SDK creation options are Rust `.forked()`, Python `forked=True`, TypeScript `.forked()`, and Go `WithForked()`. The option requires a full snapshot and cannot be combined with `--disk-only` or an image cold boot. @@ -68,17 +68,52 @@ msb stop baseline msb snapshot create after-pip-install --from baseline # 3. Boot a fresh sandbox from the snapshot -msb run --name worker --from-snapshot after-pip-install \ +msb run --name worker --from-snapshot baseline:after-pip-install \ -- python -c "import requests; print(requests.__version__)" ``` -By default, the snapshot lives at `~/.microsandbox/snapshots/after-pip-install/`. That whole directory is the snapshot. +The snapshot belongs to the source's group, `baseline`, and lives at `~/.microsandbox/snapshots/baseline/snap_/`. Its friendly name is `baseline:after-pip-install`. ## Snapshot a sandbox -Snapshot under a bare name, resolved to `~/.microsandbox/snapshots//` by default. The name is a local alias; the descriptor carries a stable `snap_...` identity. Pass a destination directory to create the artifact on a different volume (`DIR/`). Either way the directory is the whole artifact; move it with save/load (or plain `mv`): +Each capture is an immutable member of a snapshot group. The group defaults to the source sandbox's name; use `--group` to choose another. Member names are local aliases, while the descriptor carries a stable `snap_...` identity. Omitting the member name generates one. A destination directory selects another group-store root (`DIR//`). + +## Groups and head selection + +Use a bare group to restore its selected head, or `group:member` for an exact checkpoint: + +```bash +msb snapshot create cp01 --from baseline --group work +msb snapshot create cp02 --from baseline --group work +msb create --name latest --from-snapshot work +msb create --name earlier --from-snapshot work:cp01 + +msb snapshot head work # Show the selected snapshot ID +msb snapshot head work:cp01 # Explicitly select an earlier checkpoint +``` + +The first snapshot initializes the head. Later captures or imports advance it only when known ancestry proves they descend from the current head. Importing an older checkpoint, a sibling, or one with missing history keeps the head unchanged. Both members remain available. Concurrent siblings use the first successful head update; the other capture still succeeds. There is no special main branch and timestamps do not decide the winner. + +```text +work:cp01 ---- work:cp02 ---- work:cp03 <- head + \ + +------ work:experiment +``` + +`msb snapshot head work:experiment` selects the other branch. Import with `--set-head` to explicitly select the imported archive's head. Missing historical checkpoints are allowed if the snapshot's disk and RAM dependencies are complete; missing payload dependencies still require `--base`. + +```bash +msb snapshot load checkpoint.msnap --group work +msb snapshot load experiment.msnap --group work --set-head +``` + +Loading without `--group` creates a fresh generated group. The same archive can be imported into different groups without overwriting either copy. Within a group, an identical ID/descriptor is reusable; conflicting IDs or names fail. A current head cannot be removed while other members remain—select another first. Existing flat snapshot directories can still be opened by explicit path. + +An interrupted call may already have published its snapshot. Inspect the group before retrying; member publication never overwrites an existing snapshot. + +## Capture from the SDK ```rust Rust @@ -86,7 +121,7 @@ use microsandbox::Sandbox; let h = Sandbox::get("baseline").await?; -// Resolves under ~/.microsandbox/snapshots// +// Installs in baseline's group; snap.path() is the exact artifact directory. let snap = h.snapshot("after-pip-install").await?; println!("{}", snap.digest()); // sha256:... @@ -97,7 +132,7 @@ import { Sandbox } from "microsandbox"; const h = await Sandbox.get("baseline"); -// Resolves under ~/.microsandbox/snapshots// +// Installs in baseline's group; snap.path is the exact artifact directory. const snap = await h.snapshot("after-pip-install"); console.log(snap.digest); // sha256:... @@ -108,7 +143,7 @@ from microsandbox import Sandbox h = await Sandbox.get("baseline") -# Resolves under ~/.microsandbox/snapshots// +# Installs in baseline's group; snap.path is the exact artifact directory. snap = await h.snapshot("after-pip-install") print(snap.digest) # sha256:... @@ -120,7 +155,7 @@ if err != nil { return err } -// Resolves under ~/.microsandbox/snapshots// +// Installs in baseline's group; snap.Path() is the exact artifact directory. snap, err := h.Snapshot(ctx, "after-pip-install") fmt.Println(snap.Digest()) // sha256:... @@ -128,9 +163,9 @@ fmt.Println(snap.Digest()) // sha256:... ```bash CLI msb snapshot create after-pip-install --from baseline -msb snapshot create after-pip-install --from baseline --label stage=ready +msb snapshot create ready-with-label --from baseline --label stage=ready -# Create the artifact on another volume: lands at /mnt/big/after-pip-install +# Use another group-store root: /mnt/big/baseline/snap_/ msb snapshot create after-pip-install --from baseline --dest-dir /mnt/big ``` @@ -189,7 +224,7 @@ Repeated full checkpoints add disk layers. You choose when to export changes and ```bash msb snapshot create checkpoint-b --from worker --full -msb snapshot save checkpoint-b changes.msnap --since checkpoint-a +msb snapshot save worker:checkpoint-b changes.msnap --since worker:checkpoint-a msb modify worker --compact --layers 3 --dry-run msb modify worker --compact --layers 3 ``` @@ -199,8 +234,8 @@ msb modify worker --compact --layers 3 `--since` omits disk layers and, for full checkpoints, RAM objects already supplied by the base. The disk base must be an exact physical prefix. Each archive still includes its complete memory map and CPU/device state; it does not replay earlier memory images. Alternatively, `--last-layers 2` selects only disk layers and keeps all required RAM objects. These smaller archives require an explicit base when loading or restoring: ```bash -msb snapshot load changes.msnap --base checkpoint-a -msb create --name child --from-snapshot changes.msnap --snapshot-base checkpoint-a +msb snapshot load changes.msnap --base worker:checkpoint-a --group imported +msb create --name child --from-snapshot changes.msnap --snapshot-base worker:checkpoint-a ``` The base can also be a standalone snapshot archive. Load a dependent base first, then pass the installed path printed by `snapshot load` to the next load or restore. Loads resolve disk and RAM dependencies without starting a VM; only the final sandbox creation resumes execution. Missing or incorrect dependencies fail before execution. Loaded snapshots and restored children own their required files, so removing the base later does not break them. Direct restore skips installing an intermediate snapshot. Add `--disk-only` to cold-boot only disk state. After compaction, export a new standalone baseline before resuming incremental exports: the old physical prefix no longer matches. Do not combine compaction with unrelated `modify` options, or incremental export with `--with-parents`. @@ -277,7 +312,7 @@ A snapshot already pins its image, so booting from one is mutually exclusive wit use microsandbox::Sandbox; let sb = Sandbox::builder("worker") - .from_snapshot("after-pip-install") + .from_snapshot("baseline:after-pip-install") .create() .await?; ``` @@ -286,7 +321,7 @@ let sb = Sandbox::builder("worker") import { Sandbox } from "microsandbox"; const sb = await Sandbox.builder("worker") - .fromSnapshot("after-pip-install") + .fromSnapshot("baseline:after-pip-install") .create(); ``` @@ -294,17 +329,17 @@ const sb = await Sandbox.builder("worker") from microsandbox import Sandbox # `from_snapshot=` is a peer of `image=` and mutually exclusive with it -sb = await Sandbox.create("worker", from_snapshot="after-pip-install") +sb = await Sandbox.create("worker", from_snapshot="baseline:after-pip-install") ``` ```go Go sb, err := m.CreateSandbox(ctx, "worker", - m.WithFromSnapshot("after-pip-install"), + m.WithFromSnapshot("baseline:after-pip-install"), ) ``` ```bash CLI -msb run --name worker --from-snapshot after-pip-install -- python -V +msb run --name worker --from-snapshot baseline:after-pip-install -- python -V ``` @@ -318,7 +353,7 @@ To discard the captured execution and cold-boot only its filesystem state, selec ```rust Rust let sb = Sandbox::builder("worker") - .from_snapshot("worker-checkpoint") + .from_snapshot("worker:worker-checkpoint") .disk_only() .create() .await?; @@ -326,7 +361,7 @@ let sb = Sandbox::builder("worker") ```typescript TypeScript const sb = await Sandbox.builder("worker") - .fromSnapshot("worker-checkpoint") + .fromSnapshot("worker:worker-checkpoint") .diskOnly() .create(); ``` @@ -334,20 +369,20 @@ const sb = await Sandbox.builder("worker") ```python Python sb = await Sandbox.create( "worker", - from_snapshot="worker-checkpoint", + from_snapshot="worker:worker-checkpoint", disk_only=True, ) ``` ```go Go sb, err := m.CreateSandbox(ctx, "worker", - m.WithFromSnapshot("worker-checkpoint"), + m.WithFromSnapshot("worker:worker-checkpoint"), m.WithSnapshotDiskOnly(), ) ``` ```bash CLI -msb run --name worker --from-snapshot worker-checkpoint --disk-only +msb run --name worker --from-snapshot worker:worker-checkpoint --disk-only ``` @@ -360,10 +395,10 @@ Disk-only restore copies only the checkpoint's disk chain, creates a fresh writa use microsandbox::Snapshot; let all = Snapshot::list().await?; // Indexed snapshots -let h = Snapshot::get("after-pip-install").await?; // By name, digest, or path +let h = Snapshot::get("baseline:after-pip-install").await?; // By group/member selector or explicit path println!("{} ({})", h.name().unwrap_or("-"), h.digest()); -Snapshot::remove("after-pip-install", false).await?; +Snapshot::remove("baseline:after-pip-install", false).await?; Snapshot::reindex("/data/snapshots").await?; ``` @@ -371,10 +406,10 @@ Snapshot::reindex("/data/snapshots").await?; import { Snapshot } from "microsandbox"; const all = await Snapshot.list(); // Indexed snapshots -const h = await Snapshot.get("after-pip-install"); // By name, digest, or path +const h = await Snapshot.get("baseline:after-pip-install"); // By group/member selector or explicit path console.log(`${h.name ?? "-"} (${h.digest})`); -await Snapshot.remove("after-pip-install"); +await Snapshot.remove("baseline:after-pip-install"); await Snapshot.reindex(); // Default snapshots directory ``` @@ -382,34 +417,34 @@ await Snapshot.reindex(); // Default snapshots dir from microsandbox import Snapshot all = await Snapshot.list() # Indexed snapshots -h = await Snapshot.get("after-pip-install") # By name, digest, or path +h = await Snapshot.get("baseline:after-pip-install") # By group/member selector or explicit path print(f"{h.name or '-'} ({h.digest})") -await Snapshot.remove("after-pip-install") +await Snapshot.remove("baseline:after-pip-install") await Snapshot.reindex() # Default snapshots directory ``` ```go Go all, err := m.Snapshot.List(ctx) // Indexed snapshots fmt.Printf("%d snapshots\n", len(all)) -h, err := m.Snapshot.Get(ctx, "after-pip-install") // By name, digest, or path +h, err := m.Snapshot.Get(ctx, "baseline:after-pip-install") // By group/member selector or explicit path name := "-" if h.Name() != nil { name = *h.Name() } fmt.Printf("%s (%s)\n", name, h.Digest()) -err = m.Snapshot.Remove(ctx, "after-pip-install", false) +err = m.Snapshot.Remove(ctx, "baseline:after-pip-install", false) _, err = m.Snapshot.Reindex(ctx, "/data/snapshots") ``` ```bash CLI msb snapshots # Also: msb snaps, msb snapshot ls -msb snapshot inspect after-pip-install -msb snapshot rm after-pip-install +msb snapshot inspect baseline:after-pip-install +msb snapshot rm baseline:after-pip-install # Also if it has indexed children -msb snapshot rm after-pip-install --force +msb snapshot rm baseline:after-pip-install --force # Rebuild the index from artifacts on disk msb snapshot reindex @@ -417,7 +452,7 @@ msb snapshot reindex -`list` and `get` use a local index for fast lookup. If the index gets out of sync, `reindex` rebuilds it from the snapshot artifacts on disk. +`list` uses a rebuildable local index; group/head selection comes from the artifacts on disk. If the index gets out of sync, `reindex` rebuilds it. Global ID or digest lookup refuses ambiguity when several local copies exist—use a group selector or explicit path. `--force` does not bypass the current-head removal guard. ## Move snapshots between machines @@ -425,16 +460,16 @@ The snapshot directory is the whole artifact; there is no hidden daemon state. C ```bash # Copy the directory directly with scp (image must be cached or pullable on the target) -scp -r ~/.microsandbox/snapshots/after-pip-install \ - other-host:~/.microsandbox/snapshots/ +scp -r ~/.microsandbox/snapshots/baseline/snap_ other-host:/tmp/saved-snapshot +# Open that copied artifact by its explicit path; use archive load to add it to a group. # Bundle into a .msnap, transport, then load -msb snapshot save after-pip-install /tmp/snap.msnap +msb snapshot save baseline:after-pip-install /tmp/snap.msnap scp /tmp/snap.msnap other-host: ssh other-host msb snapshot load /tmp/snap.msnap # Fully offline: include the OCI image cache so the target needs no network -msb snapshot save after-pip-install /tmp/snap.msnap --with-image +msb snapshot save baseline:after-pip-install /tmp/snap.msnap --with-image ssh other-host msb snapshot load /tmp/snap.msnap ``` @@ -445,11 +480,15 @@ Archives use tar + zstd by default; `.msnap` is the recommended filename extensi Current file snapshots use a stable opaque snapshot ID, a separate SHA-256 descriptor digest, and one ordered physical layer closure. A normal raw ext4 snapshot is still one layer, whether those bytes represent a managed upper or a complete flat root: ```text -after-pip-install/ -├── snapshot.json -├── metadata.json # present when local labels exist -└── layers/ - └── layer_....raw +snapshots/ +└── baseline/ + ├── group.json # selected head ID + └── snap_/ + ├── snapshot.json # stable ID + parent ID + disk/state references + ├── group-member.json # local name: after-pip-install + ├── metadata.json # optional labels + └── layers/ + └── layer_....raw ``` `metadata.json` contains mutable local labels and does not participate in the descriptor digest or snapshot identity. The descriptor names each layer by `DiskLayerId`. Once a full snapshot has rolled the sandbox onto qcow2, a later disk snapshot carries the complete oldest-first raw/qcow2 chain in this same artifact family; archive save/load and direct archive restore preserve every member. @@ -520,8 +559,8 @@ report, err := snap.Verify(ctx) msb snapshot create after-pip-install --from baseline --integrity # Verify a snapshot's recorded integrity on demand -msb snapshot verify after-pip-install -msb snapshot inspect after-pip-install --verify +msb snapshot verify baseline:after-pip-install +msb snapshot inspect baseline:after-pip-install --verify ``` diff --git a/docs/sdk/go/snapshots.mdx b/docs/sdk/go/snapshots.mdx index c6069a1e0..13bf4688f 100644 --- a/docs/sdk/go/snapshots.mdx +++ b/docs/sdk/go/snapshots.mdx @@ -5,7 +5,27 @@ description: Go SDK - Snapshot API reference Local-only -Create disk snapshots of stopped sandboxes and full checkpoints of running sandboxes. See [Snapshots](/sandboxes/snapshots) for usage and lifecycle concepts. +Create disk snapshots of running, paused, stopped, or crashed sandboxes, or full checkpoints of running and paused sandboxes. See [Snapshots](/sandboxes/snapshots) for usage and lifecycle concepts. + +## Snapshot groups + +Installed snapshots belong to a group. A bare group selects its head; `group:member` selects a member by name or stable snapshot ID. Artifact paths remain valid selectors. Creation defaults to the source sandbox's group and generates a member name when empty; imports without a group create a new generated group. `DestDir` and `SnapshotLoadOptions.Dest` select the parent directory containing groups. + +```go +snap, err := m.Snapshot.Create(ctx, m.SnapshotCreateOptions{ + Name: "baseline", FromSandbox: "box", Group: "work", +}) +loaded, err := m.Snapshot.LoadWithOptions(ctx, "changes.msnap", m.SnapshotLoadOptions{ + Base: "work:baseline", Group: "work", +}) +head, err := m.Snapshot.GroupHead(ctx, "work") +selected, err := m.Snapshot.GroupHead(ctx, "work:baseline") +loaded, err = m.Snapshot.LoadWithOptions(ctx, "other.msnap", m.SnapshotLoadOptions{ + Group: "work", SetHead: true, +}) +``` + +`SnapshotLoadOptions` contains `Dest`, `Base`, `Group`, and `SetHead`. `Load` and `LoadWithBase` use generated groups. `GroupHead` returns `SnapshotHeadUpdate` with `Group`, `Previous`, `Head`, `Reason`, and `Changed`; `Previous` and `Head` are stable snapshot IDs. Automatic head updates initialize an empty group or advance to a proven descendant; divergent imports and imports with unknown ancestry retain the current head. Reasons are `initialized`, `fast_forwarded`, `selected`, `unchanged`, `diverged`, and `unknown_ancestry`. Direct archive capture creates no group and rejects a nonempty `Group`. ## Disk maintenance and incremental export @@ -17,12 +37,12 @@ if err != nil { return err } layers := uint32(3) result, err := worker.Compact(ctx, m.DiskCompactionOptions{Layers: &layers}) if err != nil { return err } -err = m.Snapshot.Save(ctx, "checkpoint-b", "changes.msnap", m.SnapshotSaveOptions{Since: "checkpoint-a"}) +err = m.Snapshot.Save(ctx, "worker:checkpoint-b", "changes.msnap", m.SnapshotSaveOptions{Since: "worker:checkpoint-a"}) if err != nil { return err } -snapshot, err := m.Snapshot.LoadWithBase(ctx, "changes.msnap", "", "checkpoint-a") +snapshot, err := m.Snapshot.LoadWithBase(ctx, "changes.msnap", "", "worker:checkpoint-a") ``` -The count includes the oldest base but excludes the writable head. A nil `Layers` selects all sealed layers; `DryRun: true` only resolves the plan. `LastLayers` is an alternative to `Since` for export. For direct restore combine `WithFromSnapshot("changes.msnap")` with `WithSnapshotBase("checkpoint-a")`. See [disk-chain maintenance](/sandboxes/snapshots#export-changes-and-compact-a-disk-chain) for dependency and retention rules. +The count includes the oldest base but excludes the writable head. A nil `Layers` selects all sealed layers; `DryRun: true` only resolves the plan. `LastLayers` is an alternative to `Since` for export. For direct restore combine `WithFromSnapshot("changes.msnap")` with `WithSnapshotBase("worker:checkpoint-a")`. See [disk-chain maintenance](/sandboxes/snapshots#export-changes-and-compact-a-disk-chain) for dependency and retention rules. ## Snapshot @@ -34,7 +54,7 @@ Package-level helpers for snapshot artifacts. Access them through the exported ` func (snapshotFactory) Create(ctx context.Context, opts SnapshotCreateOptions) (*SnapshotArtifact, error) ``` -Create a disk snapshot from a stopped or crashed sandbox, or set `Full` to checkpoint a running sandbox. [`SnapshotCreateOptions.Name`](#snapshotcreateoptionsstruct) and [`SnapshotCreateOptions.FromSandbox`](#snapshotcreateoptionsstruct) are both required. +Create a disk snapshot, or set `Full` to include memory and execution state. [`SnapshotCreateOptions.FromSandbox`](#snapshotcreateoptionsstruct) is required. An empty `Name` is generated; an empty `Group` uses the source sandbox's name.

Parameters

@@ -102,12 +122,12 @@ func (snapshotFactory) Open(ctx context.Context, pathOrName string) (*SnapshotAr ```go -snap, err := m.Snapshot.Open(ctx, "after-pip-install") +snap, err := m.Snapshot.Open(ctx, "baseline:after-pip-install") ``` -Open an existing artifact by bare name or filesystem path. This validates metadata only; call [`s.Verify()`](#s-verify) for content checks. +Open an existing artifact by group head, `group:member`, or filesystem path. This validates metadata only; call [`s.Verify()`](#s-verify) for content checks.

Parameters

@@ -118,7 +138,7 @@ Open an existing artifact by bare name or filesystem path. This validates metada
pathOrNamestring
-
Bare name (resolved under the default snapshots directory) or artifact directory path.
+
Group head or group:member selector or artifact directory path.
@@ -140,12 +160,12 @@ func (snapshotFactory) Get(ctx context.Context, nameOrDigest string) (*SnapshotH ```go -h, err := m.Snapshot.Get(ctx, "after-pip-install") +h, err := m.Snapshot.Get(ctx, "baseline:after-pip-install") ``` -Look up a lightweight handle in the local index by name, digest, or path. +Look up a lightweight handle by group head, `group:member`, stable snapshot ID, descriptor digest, or artifact path. Global IDs and digests must resolve unambiguously.

Parameters

@@ -156,7 +176,7 @@ Look up a lightweight handle in the local index by name, digest, or path.
nameOrDigeststring
-
Bare name, manifest digest, or artifact path.
+
Group head, group:member, stable snapshot ID, digest, or artifact path.
@@ -236,7 +256,7 @@ func (snapshotFactory) Remove(ctx context.Context, pathOrName string, force bool ```go -err := m.Snapshot.Remove(ctx, "after-pip-install", false) +err := m.Snapshot.Remove(ctx, "baseline:after-pip-install", false) ``` @@ -252,7 +272,7 @@ Remove a snapshot artifact and its index row. Refuses to delete a snapshot with
pathOrNamestring
-
Bare name or artifact path.
+
Group head, group:member, or artifact path.
forcebool
@@ -319,7 +339,7 @@ Bundle a snapshot into a `.msnap` archive at `outPath`. Set [`SnapshotSaveOption
nameOrPathstring
-
Bare name or artifact path to save.
+
Group head, group:member, or artifact path to save.
outPathstring
@@ -334,7 +354,7 @@ Bundle a snapshot into a `.msnap` archive at `outPath`. Set [`SnapshotSaveOption ```go -err := m.Snapshot.Save(ctx, "after-pip-install", "/tmp/snap.msnap", +err := m.Snapshot.Save(ctx, "baseline:after-pip-install", "/tmp/snap.msnap", m.SnapshotSaveOptions{WithParents: true}, ) ``` @@ -388,7 +408,7 @@ h, err := m.Snapshot.Load(ctx, "/tmp/snap.msnap", "") ## SandboxHandle -Snapshots are taken from a metadata handle, so stop the sandbox first and then call [`GetSandbox`](/sdk/go/sandbox#getsandbox). +Snapshots are taken from a metadata handle returned by [`GetSandbox`](/sdk/go/sandbox#getsandbox). Disk capture also supports a running or paused sandbox. ```go _ = sb.Stop(ctx) @@ -407,7 +427,7 @@ snap, err := h.Snapshot(ctx, "after-pip-install") func (h *SandboxHandle) Snapshot(ctx context.Context, name string) (*SnapshotArtifact, error) ``` -Snapshot this sandbox under a bare name in the default snapshots directory. The sandbox must be stopped or crashed. To place the artifact elsewhere, use [`Snapshot.Save`](#snapshot-save) / [`Snapshot.Load`](#snapshot-load) or move the self-contained artifact directory. +Snapshot this sandbox into its default group with the given member name. Live disk captures preserve the source's running or paused state. Use the returned artifact path or `sandbox:member` to open it later.

Parameters

@@ -418,7 +438,7 @@ Snapshot this sandbox under a bare name in the default snapshots directory. The
namestring
-
Bare name for the artifact.
+
Member name within the source sandbox's group.
@@ -638,7 +658,7 @@ err := h.Remove(ctx, false) -Remove this snapshot. Equivalent to [`Snapshot.Remove`](#snapshot-remove) on this handle's digest. +Remove this installed snapshot copy by its stored artifact path. Other groups containing the same snapshot ID or digest remain unchanged.

Parameters

@@ -667,7 +687,7 @@ Manifest digest. func (h *SnapshotHandle) Name() *string ``` -Bare-name alias, if the snapshot was indexed with one; otherwise `nil`. Returns a defensive copy. +Member name within its group, or `nil` when no alias is recorded. Returns a defensive copy. #### h.ParentDigest() @@ -749,15 +769,16 @@ Scope of what a snapshot captures: disk-only or disk plus complete VM state.

Accepted by Snapshot.Create()

-Configures [`Snapshot.Create`](#snapshot-create). `Name` and `FromSandbox` are both required. +Configures [`Snapshot.Create`](#snapshot-create). `FromSandbox` is required. An empty `Name` is generated; an empty `Group` uses the source sandbox's name. | Field | Type | Description | |-------|------|-------------| -| Name | `string` | Bare name; always the artifact directory's basename | +| Name | `string` | Member name within its group; generated when empty | +| Group | `string` | Destination snapshot group; defaults to the source sandbox's name | | FromSandbox | `string` | Name of the stopped or crashed sandbox to capture | -| DestDir | `string` | Parent directory for the artifact (`DestDir/`); empty = the default snapshots directory | +| DestDir | `string` | Parent directory containing snapshot groups; empty = the default snapshots directory | | Labels | `map[string]string` | Arbitrary user labels recorded in the manifest | -| Force | `bool` | Overwrite an existing artifact with the same name | +| Force | `bool` | Overwrite a direct archive output; rejected for installed group members | | RecordIntegrity | `bool` | Record content hashes so [`Verify`](#s-verify) can recompute them later | | Full | `bool` | Capture a running sandbox's full checkpoint instead of a stopped-sandbox disk snapshot | diff --git a/docs/sdk/python/snapshots.mdx b/docs/sdk/python/snapshots.mdx index 56d924d16..f253f91a9 100644 --- a/docs/sdk/python/snapshots.mdx +++ b/docs/sdk/python/snapshots.mdx @@ -5,7 +5,22 @@ description: Python SDK - Snapshot API reference Local-only -Create disk snapshots of stopped sandboxes and full checkpoints of running sandboxes. See [Snapshots](/sandboxes/snapshots) for usage and lifecycle concepts. +Create disk snapshots and full checkpoints in local snapshot groups. See [Snapshots](/sandboxes/snapshots) for usage and lifecycle concepts. + +## Snapshot groups + +Installed snapshots belong to a group. A bare group selects its head; `group:member` selects a member by name or stable snapshot ID. Artifact paths remain valid selectors. Creation defaults to the source sandbox's group and generates a member name when omitted; imports without `group=` create a new generated group. `dest_dir` and `dest` select the parent directory containing groups. + +```python +snap = await Snapshot.create("baseline", from_sandbox="box", group="work") +loaded = await Snapshot.load("changes.msnap", base="work:baseline", group="work") +head = await Snapshot.group_head("work") +selected = await Snapshot.group_head("work:baseline") +# Explicitly select an imported member even when it is not a descendant. +loaded = await Snapshot.load("other.msnap", group="work", set_head=True) +``` + +`group_head(selector)` returns a dictionary with `group`, `previous`, `head`, `reason`, and `changed`. `previous` and `head` are stable snapshot IDs. Automatic head updates initialize an empty group or advance to a proven descendant; divergent imports and imports with unknown ancestry retain the current head. Reasons are `initialized`, `fast_forwarded`, `selected`, `unchanged`, `diverged`, and `unknown_ancestry`. Direct archive capture creates no group and rejects `group=`. ## SandboxHandle @@ -15,14 +30,14 @@ Create disk snapshots of stopped sandboxes and full checkpoints of running sandb async def snapshot(self, name: str) -> Snapshot ``` -Snapshot this sandbox under a bare name in the default snapshots directory (`~/.microsandbox/snapshots//`). The sandbox must be stopped or crashed. To place the artifact elsewhere, use [`Snapshot.save()`](#snapshot-save) / [`Snapshot.load()`](#snapshot-load) or move the self-contained artifact directory. Called on a [`SandboxHandle`](/sdk/python/sandbox#sandboxhandle), obtained from [`Sandbox.get()`](/sdk/python/sandbox#sandbox-get). +Snapshot this sandbox into its default group with the given member name. Live disk captures preserve the source's running or paused state. Use the returned artifact path or `sandbox:member` to open it later. Called on a [`SandboxHandle`](/sdk/python/sandbox#sandboxhandle), obtained from [`Sandbox.get()`](/sdk/python/sandbox#sandbox-get).

Parameters

namestr
-
Snapshot name; resolved under the default snapshots directory.
+
Member name within the source sandbox's group.
@@ -68,7 +83,7 @@ Create a sandbox from a snapshot artifact by passing `from_snapshot=` as a peer
from_snapshotstr | os.PathLike | None
-
Snapshot bare name, artifact directory, or archive file to boot from instead of image=.
+
Group head or group:member, artifact directory, or archive file to boot from instead of image=.
disk_onlybool
@@ -89,7 +104,7 @@ Create a sandbox from a snapshot artifact by passing `from_snapshot=` as a peer ```python # Boot from a snapshot -sb = await Sandbox.create("worker", from_snapshot="after-pip-install") +sb = await Sandbox.create("worker", from_snapshot="baseline:after-pip-install") # Or from an image (existing flow, unchanged) sb = await Sandbox.create("worker", image="python:3.12") @@ -107,9 +122,9 @@ Exporting since a base omits its reusable disk layers and RAM objects. Full chec worker = await Sandbox.get("worker") plan = await worker.compact(layers=3, dry_run=True) result = await worker.compact(layers=3) -await Snapshot.save("checkpoint-b", "changes.msnap", since="checkpoint-a") -await Snapshot.load("changes.msnap", base="checkpoint-a") -child = await Sandbox.create("child", from_snapshot="changes.msnap", snapshot_base="checkpoint-a") +await Snapshot.save("worker:checkpoint-b", "changes.msnap", since="worker:checkpoint-a") +await Snapshot.load("changes.msnap", base="worker:checkpoint-a") +child = await Sandbox.create("child", from_snapshot="changes.msnap", snapshot_base="worker:checkpoint-a") ``` The count includes the oldest base but excludes the writable head. Omit `layers` to compact all sealed layers. Use `last_layers=n` instead of `since` to export the newest N sealed layers. Results are dictionaries with `input_layers`, `selected_layers`, `output_layers`, `materialized_bytes`, `total_us`, `pause_us`, and `dry_run`. See [disk-chain maintenance](/sandboxes/snapshots#export-changes-and-compact-a-disk-chain) for dependency and retention rules. @@ -223,9 +238,10 @@ Best-effort source-sandbox name ```python @staticmethod async def create( - name: str, + name: str = "", *, from_sandbox: str, + group: str | None = None, dest_dir: str | os.PathLike[str] | None = None, labels: dict[str, str] | None = None, force: bool = False, @@ -234,14 +250,14 @@ async def create( ) -> Snapshot ``` -Create a disk snapshot from a stopped or crashed sandbox, or set `full=True` to checkpoint a running sandbox. `name` is resolved under the default snapshots directory (`~/.microsandbox/snapshots//`), or under `dest_dir=` when given; `from_sandbox=` names the sandbox to capture and is required. +Create a disk snapshot, or set `full=True` to include memory and execution state. `name` identifies a member within `group`; an omitted name is generated and an omitted group uses the source sandbox's name. `dest_dir` selects the parent directory containing the group. `from_sandbox` names the sandbox to capture and is required.

Parameters

namestr
-
Bare snapshot name; resolved under the default snapshots directory.
+
Snapshot member name; generated when omitted.
from_sandboxstr
@@ -249,7 +265,7 @@ Create a disk snapshot from a stopped or crashed sandbox, or set `full=True` to
dest_dirstr | os.PathLike[str] | None
-
Parent directory to create the artifact in; the artifact lands at dest_dir/<name>. Defaults to the snapshots directory.
+
Parent directory containing snapshot groups. Defaults to the snapshots directory.
labelsdict[str, str] | None
@@ -257,7 +273,7 @@ Create a disk snapshot from a stopped or crashed sandbox, or set `full=True` to
forcebool
-
Overwrite an existing artifact with the same name. Default False.
+
Must remain False for installed group members, which are immutable. Direct archive capture supports overwriting its output file.
record_integritybool
@@ -302,6 +318,7 @@ async def create_archive( archive: str | os.PathLike[str], *, from_sandbox: str, + group: str | None = None, labels: dict[str, str] | None = None, force: bool = False, record_integrity: bool = False, @@ -332,20 +349,20 @@ async def open(path_or_name: str) -> Snapshot ```python -snap = await Snapshot.open("after-pip-install") +snap = await Snapshot.open("baseline:after-pip-install") print(snap.image_ref) ``` -Open an existing artifact by bare name (resolved under the default snapshots directory) or path. Cheap metadata validation only; does **not** read the upper file. Use [`verify()`](#snap-verify) for content checks. +Open an existing artifact by group head, `group:member`, or path. This validates metadata without reading the full upper file. Use [`verify()`](#snap-verify) for content checks.

Parameters

path_or_namestr
-
Bare snapshot name or artifact directory path.
+
Group head, group:member, or artifact directory path.
@@ -368,20 +385,20 @@ async def get(name_or_digest: str) -> SnapshotHandle ```python -h = await Snapshot.get("after-pip-install") +h = await Snapshot.get("baseline:after-pip-install") print(h.digest) ``` -Look up a handle in the local index by name, digest, or path. +Look up a handle by group head, `group:member`, stable snapshot ID, descriptor digest, or artifact path. Global IDs and digests must resolve unambiguously.

Parameters

name_or_digeststr
-
Snapshot name, digest, or path.
+
Group head, group:member, stable snapshot ID, descriptor digest, or artifact path.
@@ -458,7 +475,7 @@ async def remove(path_or_name: str, *, force: bool = False) -> None ```python -await Snapshot.remove("after-pip-install", force=True) +await Snapshot.remove("baseline:after-pip-install", force=True) ``` @@ -470,7 +487,7 @@ Remove a snapshot artifact and its index row. Refuses if the snapshot has indexe
path_or_namestr
-
Bare snapshot name or artifact path.
+
Group head, group:member, or artifact path.
forcebool
@@ -544,7 +561,7 @@ async def save( ```python await Snapshot.save( - "after-pip-install", + "baseline:after-pip-install", "/tmp/after-pip-install.msnap", with_parents=True, ) @@ -559,7 +576,7 @@ Bundle a snapshot into a `.msnap` archive. The existing snapshot manifest is arc
name_or_pathstr
-
Snapshot bare name or artifact path to save.
+
Group head or group:member or artifact path to save.
outstr | os.PathLike
@@ -583,7 +600,7 @@ Bundle a snapshot into a `.msnap` archive. The existing snapshot manifest is arc ```python await Snapshot.save( - "after-pip-install", + "baseline:after-pip-install", "/tmp/after-pip-install.msnap", with_parents=True, ) @@ -606,10 +623,13 @@ async def load( archive: str | os.PathLike, *, dest: str | os.PathLike | None = None, + base: str | None = None, + group: str | None = None, + set_head: bool = False, ) -> SnapshotHandle ``` -Unpack a snapshot archive (`.msnap` or `.tar`) into the snapshots directory. Structural and archive-entry checks run during import; recorded payload integrity is preserved for explicit [`verify()`](#snap-verify). Compression is detected from magic bytes. +Unpack a snapshot archive (`.msnap` or `.tar`) into the selected or generated group. The returned handle's `group` identifies the group and `head_update` reports the head selection outcome. Structural and archive-entry checks run during import; recorded payload integrity is preserved for explicit [`verify()`](#snap-verify). Compression is detected from magic bytes.

Parameters

@@ -799,7 +819,7 @@ async def open(self) -> Snapshot ```python -h = await Snapshot.get("after-pip-install") +h = await Snapshot.get("baseline:after-pip-install") snap = await h.open() print(snap.fstype) ``` @@ -826,13 +846,13 @@ async def remove(self, *, force: bool = False) -> None ```python -h = await Snapshot.get("after-pip-install") +h = await Snapshot.get("baseline:after-pip-install") await h.remove(force=False) ``` -Remove this snapshot artifact and its index row. Refuses if the snapshot has indexed children unless `force=True`. +Remove this installed snapshot copy and its index row using the handle's stored artifact path. Other groups containing the same snapshot ID or digest remain unchanged. Refuses if the snapshot has indexed children unless `force=True`.

Parameters

diff --git a/docs/sdk/rust/snapshots.mdx b/docs/sdk/rust/snapshots.mdx index f41d84af8..2482edcd0 100644 --- a/docs/sdk/rust/snapshots.mdx +++ b/docs/sdk/rust/snapshots.mdx @@ -7,6 +7,26 @@ description: Rust SDK - Snapshot API reference Create disk snapshots of running, paused, stopped, or crashed sandboxes, or full checkpoints of running and paused sandboxes. See [Snapshots](/sandboxes/snapshots) for usage and lifecycle concepts. +## Snapshot groups + +Installed snapshots belong to a group. A bare group selects its head; `group:member` selects a member by name or stable snapshot ID. Artifact paths remain valid selectors. Creation defaults to the source sandbox's group, and an empty builder name generates a member name. Imports without `LoadOpts.group` create a new generated group. `dest_dir` and `LoadOpts.dest` select the parent directory containing groups. + +```rust +use microsandbox::{Snapshot, snapshot::LoadOpts}; +use std::path::Path; + +let snap = Snapshot::builder("baseline").from_sandbox("box").group("work").create().await?; +let loaded = Snapshot::load_with_options(Path::new("changes.msnap"), LoadOpts { + base: Some("work:baseline".into()), + group: Some("work".into()), + ..Default::default() +}).await?; +let head = Snapshot::group_head("work").await?; +let selected = Snapshot::group_head("work:baseline").await?; +``` + +`LoadOpts` contains `dest: Option`, `base: Option`, `group: Option`, and `set_head: bool`. `set_head: true` explicitly selects the imported member even when it is not a descendant. `load` and `load_with_base` use generated groups. `group_head` returns `HeadUpdate` with `group`, `previous`, `head`, `reason`, and `changed`; `previous` and `head` are stable snapshot IDs. Automatic head updates initialize an empty group or advance to a proven descendant; divergent imports and imports with unknown ancestry retain the current head. `Snapshot::head_update()` and `SnapshotHandle::head_update()` expose the create/import outcome. Direct archive capture creates no group and rejects `.group(...)`. + ## Disk maintenance and incremental export Exporting since a base omits its reusable disk layers and RAM objects. Full checkpoints keep the complete memory map and CPU/device state. The last-layers option only selects disk layers and leaves RAM payloads complete. Supply the base when loading or restoring a dependent archive; a dependent base archive must be loaded first. @@ -15,14 +35,14 @@ Exporting since a base omits its reusable disk layers and RAM objects. Full chec let worker = Sandbox::get("worker").await?; let plan = worker.compact().layers(3).dry_run().await?; let result = worker.compact().layers(3).apply().await?; -Snapshot::save("checkpoint-b", Path::new("changes.msnap"), SaveOpts { - since: Some("checkpoint-a".into()), +Snapshot::save("worker:checkpoint-b", Path::new("changes.msnap"), SaveOpts { + since: Some("worker:checkpoint-a".into()), ..Default::default() }).await?; -Snapshot::load_with_base(Path::new("changes.msnap"), None, "checkpoint-a").await?; +Snapshot::load_with_base(Path::new("changes.msnap"), None, "worker:checkpoint-a").await?; ``` -`layers` counts the oldest physical layers including the base, excluding the writable head. Omit it to compact all sealed layers. `last_layers: Some(n)` is an alternative to `since`; they cannot be combined with each other or `with_parents`. For direct restore, use `Sandbox::builder("child").from_snapshot("changes.msnap").snapshot_base("checkpoint-a")`. See [disk-chain maintenance](/sandboxes/snapshots#export-changes-and-compact-a-disk-chain) for dependency and retention rules. +`layers` counts the oldest physical layers including the base, excluding the writable head. Omit it to compact all sealed layers. `last_layers: Some(n)` is an alternative to `since`; they cannot be combined with each other or `with_parents`. For direct restore, use `Sandbox::builder("child").from_snapshot("changes.msnap").snapshot_base("worker:checkpoint-a")`. See [disk-chain maintenance](/sandboxes/snapshots#export-changes-and-compact-a-disk-chain) for dependency and retention rules. ## Snapshot @@ -32,14 +52,14 @@ Snapshot::load_with_base(Path::new("changes.msnap"), None, "checkpoint-a").await fn builder(name: impl Into) -> SnapshotBuilder ``` -Start configuring a new snapshot named `name`, resolved under the default snapshots directory (`~/.microsandbox/snapshots//`) or under [`dest_dir()`](#dest_dir) when set. The source sandbox is set with [`from_sandbox()`](#from_sandbox), which is required; the other setters cover labels and whether to record content integrity before capturing. See [`SnapshotBuilder`](#snapshotbuilder) for all options. +Start configuring a snapshot member named `name`, generated when empty. The default group is the source sandbox's name; `.group(name)` selects another group and [`dest_dir()`](#dest_dir) selects its parent directory. The source sandbox is set with [`from_sandbox()`](#from_sandbox), which is required; the other setters cover labels and whether to record content integrity before capturing. See [`SnapshotBuilder`](#snapshotbuilder) for all options.

Parameters

nameimpl Into<String>
-
Bare snapshot name. Must not be empty, contain /, or start with ..
+
Member name; generated when empty. Names must not contain path separators or start with ..
@@ -135,20 +155,20 @@ async fn open(path_or_name: impl AsRef) -> MicrosandboxResult ```rust -let snap = Snapshot::open("baseline").await?; +let snap = Snapshot::open("api:baseline").await?; println!("{}", snap.manifest().image.reference); ``` -Open an existing artifact by path or bare name. Bare names (no path separator, not starting with `.` or `~`) resolve under the default snapshots directory; anything else is treated as a path. This is a fast metadata operation: it verifies the manifest structure, recomputes the manifest digest, and checks that the upper file exists with the recorded size. It does **not** read the full upper contents; use [`verify()`](#snap-verify) for that. +Open an existing artifact by group head, `group:member`, or path. This is a fast metadata operation: it verifies the manifest structure, recomputes the manifest digest, and checks that the upper file exists with the recorded size. It does **not** read the full upper contents; use [`verify()`](#snap-verify) for that.

Parameters

path_or_nameimpl AsRef<str>
-
Bare snapshot name or filesystem path to an artifact directory.
+
Group head, group:member, or filesystem path to an artifact directory.
@@ -170,20 +190,20 @@ async fn get(name_or_digest: &str) -> MicrosandboxResult ```rust -let h = Snapshot::get("after-pip-install").await?; +let h = Snapshot::get("api:baseline").await?; println!("{} from {}", h.digest(), h.image_ref()); ``` -Look up a lightweight [`SnapshotHandle`](#snapshothandle) in the local index by name, digest (`sha256:`/`sha512:` prefix), or path. +Look up a lightweight [`SnapshotHandle`](#snapshothandle) by group head, `group:member`, stable snapshot ID, descriptor digest, or artifact path. Global IDs and digests must resolve unambiguously.

Parameters

name_or_digest&str
-
Snapshot name, manifest digest, or artifact path.
+
Group head, group:member, stable snapshot ID, descriptor digest, or artifact path.
@@ -258,19 +278,19 @@ async fn remove(path_or_name: &str, force: bool) -> MicrosandboxResult<()> ```rust -Snapshot::remove("after-pip-install", false).await?; +Snapshot::remove("api:baseline", false).await?; ``` -Remove a snapshot artifact (by digest, name, or path) and its index row. Refuses if the snapshot has indexed children unless `force` is set. The artifact directory is deleted on success and the parent's child count is decremented. +Remove a snapshot artifact by group selector, unambiguous ID or digest, or path, along with its index row. Refuses if the snapshot has indexed children unless `force` is set. A group's head cannot be removed while other members remain, even with `force`; select another head first.

Parameters

path_or_name&str
-
Snapshot digest, name, or artifact path.
+
Group head, group:member, unambiguous snapshot ID or digest, or artifact path.
forcebool
@@ -338,7 +358,7 @@ Bundle a snapshot into a `.msnap` archive (or plain `.tar`) at `out`. Recorded p
name_or_path&str
-
Snapshot name or artifact path to save.
+
Group head, group:member, or artifact path to save.
out&Path
@@ -357,7 +377,7 @@ use microsandbox::snapshot::SaveOpts; use std::path::Path; Snapshot::save( - "baseline", + "api:baseline", Path::new("/tmp/baseline.msnap"), SaveOpts { with_parents: true, with_image: true, ..Default::default() }, ).await?; @@ -405,7 +425,7 @@ Unpack a snapshot archive (`.msnap` or `.tar`, detected from magic bytes) into t
-
Handle for the head (last-listed) snapshot.
+
Handle for the archive-declared head snapshot, which may differ from the receiving group's selected head.
@@ -477,7 +497,7 @@ fn manifest(&self) -> &Manifest ```rust -let snap = Snapshot::open("baseline").await?; +let snap = Snapshot::open("api:baseline").await?; let m = snap.manifest(); println!("{} @ {}", m.image.reference, m.image.manifest_digest); ``` @@ -523,7 +543,7 @@ async fn verify(&self) -> MicrosandboxResult ```rust use microsandbox::snapshot::UpperVerifyStatus; -let snap = Snapshot::open("baseline").await?; +let snap = Snapshot::open("api:baseline").await?; match snap.verify().await?.upper { UpperVerifyStatus::Verified { algorithm, .. } => println!("ok via {algorithm}"), UpperVerifyStatus::NotRecorded => println!("no integrity hash recorded"), @@ -565,7 +585,7 @@ Manifest digest (`sha256:hex`), the canonical identity. fn name(&self) -> Option<&str> ``` -Name alias, or `None` for digest-only entries. +Member name within its group, or `None` when no alias is recorded. #### h.parent_digest() @@ -573,7 +593,7 @@ Name alias, or `None` for digest-only entries. fn parent_digest(&self) -> Option<&str> ``` -The parent snapshot's digest, or `None` for a root. Always `None` today; populated once chained snapshots land. +The captured parent snapshot's stable ID, or `None` when no parent is known. The accessor retains its existing `parent_digest` name. --- @@ -646,7 +666,7 @@ async fn open(&self) -> MicrosandboxResult ```rust -let h = Snapshot::get("baseline").await?; +let h = Snapshot::get("api:baseline").await?; let snap = h.open().await?; snap.verify().await?; ``` @@ -673,13 +693,13 @@ async fn remove(&self, force: bool) -> MicrosandboxResult<()> ```rust -let h = Snapshot::get("baseline").await?; +let h = Snapshot::get("api:baseline").await?; h.remove(false).await?; ``` -Remove this snapshot. Delegates to [`Snapshot::remove(self.digest(), force)`](#snapshotremove). +Remove this installed snapshot copy by its stored artifact path. Other groups containing the same snapshot ID or digest remain unchanged.

Parameters

@@ -704,7 +724,7 @@ fn from_snapshot(self, path_or_name: impl Into) -> Self ```rust let sb = Sandbox::builder("api-restored") - .from_snapshot("after-pip-install") + .from_snapshot("api:baseline") .create() .await?; ``` @@ -718,7 +738,7 @@ let sb = Sandbox::builder("api-restored")
path_or_nameimpl Into<String>
-
Bare name resolved under the default snapshots directory, or a path to an artifact directory or archive file.
+
Group head or group:member selector, or a path to an artifact directory or archive file.
@@ -738,14 +758,14 @@ Cold-boot only the disk state carried by a full snapshot. Chain after [`from_sna async fn snapshot(&self, name: &str) -> MicrosandboxResult ``` -`SandboxHandle` method. Snapshot this sandbox's disk under a bare name in the default snapshots directory (`~/.microsandbox/snapshots//`). Live captures are crash-consistent and preserve the source's running/paused state. Local handles only. To place the artifact elsewhere, use [`Snapshot::save()`](#snapshotsave) / [`Snapshot::load()`](#snapshotload) or move the self-contained artifact directory. +`SandboxHandle` method. Snapshot this sandbox's disk into its default group with the given member name. Use the returned artifact path or `sandbox:member` to open it later. Live captures are crash-consistent and preserve the source's running/paused state. Local handles only. To place the artifact elsewhere, use [`Snapshot::save()`](#snapshotsave) / [`Snapshot::load()`](#snapshotload) or move the self-contained artifact directory.

Parameters

name&str
-
Bare snapshot name.
+
Member name within the source sandbox's group.
@@ -758,18 +778,15 @@ async fn snapshot(&self, name: &str) -> MicrosandboxResult
-#### h.snapshot_to() + ```rust -async fn snapshot_to(&self, path: impl AsRef) -> MicrosandboxResult -``` - - - -```rust -let h = Sandbox::get("api").await?; -h.stop().await?; -let snap = h.snapshot_to("/data/snapshots/baseline").await?; +let snap = Snapshot::builder("baseline") + .from_sandbox("api") + .dest_dir("/data/snapshots") + .create() + .await?; +// snap.path() is /data/snapshots/api/. ``` @@ -809,7 +826,7 @@ Set the sandbox to capture. Required; [`build()`](#build) and [`create()`](#crea fn dest_dir(self, dest_dir: impl Into) -> Self ``` -Create the artifact under this parent directory instead of the default snapshots store. The artifact directory is `dest_dir/`; the name stays the snapshot's identity either way. +Create the snapshot group under this parent directory instead of the default snapshots store. Member names are local aliases within the group; stable snapshot IDs identify immutable artifacts.

Parameters

@@ -847,7 +864,7 @@ Add a user label. Can be called multiple times. Labels are sorted by key in the fn force(self) -> Self ``` -Overwrite an existing artifact with the same name. Without this, creation fails with `SnapshotAlreadyExists` if the artifact directory exists. +Overwrite an existing direct archive output file. Installed group members are immutable, so installed creation rejects this option. #### snapshot_builder.record_integrity() @@ -912,11 +929,12 @@ Inputs to create a snapshot. A type alias for `SnapshotSpec`. Usually built via | Field | Type | Description | |-------|------|-------------| -| name | `String` | Bare snapshot name; always the artifact directory's basename | -| dest_dir | `Option` | Parent directory for the artifact; `None` = the default snapshots directory | +| name | `String` | Member name within its group; generated when empty | +| group | `Option` | Destination group; defaults to the source sandbox's name | +| dest_dir | `Option` | Parent directory containing groups; `None` = the default snapshots directory | | source_sandbox | `String` | Name of the source sandbox; disk capture preserves running/paused state | | labels | `Vec<(String, String)>` | User-supplied labels | -| force | `bool` | Overwrite an existing artifact with the same name | +| force | `bool` | Overwrite a direct archive output; rejected for installed group members | | record_integrity | `bool` | Compute and record upper-layer integrity at creation | | full | `bool` | Capture a running sandbox's full checkpoint instead of a stopped-sandbox disk snapshot | diff --git a/docs/sdk/typescript/snapshots.mdx b/docs/sdk/typescript/snapshots.mdx index 2b3acf2fd..56327b050 100644 --- a/docs/sdk/typescript/snapshots.mdx +++ b/docs/sdk/typescript/snapshots.mdx @@ -21,7 +21,7 @@ fromSnapshot(pathOrName: string): SandboxBuilder ```typescript const sb = await Sandbox.builder("worker") - .fromSnapshot("after-pip-install") + .fromSnapshot("baseline:after-pip-install") .create(); ``` @@ -36,7 +36,7 @@ Chain `.diskOnly()` after `.fromSnapshot()` to cold-boot only the disk state fro
pathOrNamestring
-
Bare name (resolved under the default snapshots directory) or filesystem path to an artifact directory or archive file.
+
Group head or group:member selector or filesystem path to an artifact directory or archive file.
@@ -57,14 +57,14 @@ Chain `.diskOnly()` after `.fromSnapshot()` to cold-boot only the disk state fro snapshot(name: string): Promise ``` -Snapshot this sandbox under a bare name in the default snapshots directory (`~/.microsandbox/snapshots//`). Called on a [`SandboxHandle`](/sdk/typescript/sandbox#sandboxhandle). The sandbox must be stopped or crashed; running sandboxes are rejected with a `SnapshotSandboxRunning` error. To place the artifact elsewhere, use [`Snapshot.save()`](#snapshot-save) / [`Snapshot.load()`](#snapshot-load) or move the self-contained artifact directory. +Snapshot this sandbox into its default group with the given member name. Called on a [`SandboxHandle`](/sdk/typescript/sandbox#sandboxhandle). Live disk captures preserve the source's running or paused state. Use the returned artifact path or `sandbox:member` to open it later.

Parameters

namestring
-
Bare name; becomes the artifact directory under the default snapshots dir.
+
Member name within the source sandbox's group.
@@ -89,6 +89,23 @@ const snap = await h.snapshot("after-pip-install"); --- +## Snapshot groups + +Installed snapshots belong to a group. A bare group selects its head; `group:member` selects a member by name or stable snapshot ID. Artifact paths remain valid selectors. Creation defaults to the source sandbox's group and generates a member name when omitted; imports without a group create a new generated group. `destDir` and `LoadOpts.dest` select the parent directory containing groups. + +```typescript +const snap = await Snapshot.builder("baseline").fromSandbox("box").group("work").create(); +const loaded = await Snapshot.loadWithOptions("changes.msnap", { + base: "work:baseline", + group: "work", +}); +const head = await Snapshot.groupHead("work"); +const selected = await Snapshot.groupHead("work:baseline"); +await Snapshot.loadWithOptions("other.msnap", { group: "work", setHead: true }); +``` + +`LoadOpts` contains optional `dest`, `base`, `group`, and `setHead` fields. `Snapshot.load(archive, dest?, base?)` uses a generated group. `Snapshot.groupHead(selector)` returns `HeadUpdate` with `group`, `previous`, `head`, `reason`, and `changed`; `previous` and `head` are stable snapshot IDs. Automatic head updates initialize an empty group or advance to a proven descendant; divergent imports and imports with unknown ancestry retain the current head. Reasons are `initialized`, `fast_forwarded`, `selected`, `unchanged`, `diverged`, and `unknown_ancestry`. Direct archive capture creates no group and rejects `.group(...)`. + ## Disk maintenance and incremental export Exporting since a base omits its reusable disk layers and RAM objects. Full checkpoints keep the complete memory map and CPU/device state. The last-layers option only selects disk layers and leaves RAM payloads complete. Supply the base when loading or restoring a dependent archive; a dependent base archive must be loaded first. @@ -97,10 +114,10 @@ Exporting since a base omits its reusable disk layers and RAM objects. Full chec const worker = await Sandbox.get("worker"); const plan = await worker.compact({ layers: 3, dryRun: true }); const result = await worker.compact({ layers: 3 }); -await Snapshot.save("checkpoint-b", "changes.msnap", { since: "checkpoint-a" }); -await Snapshot.load("changes.msnap", undefined, "checkpoint-a"); +await Snapshot.save("worker:checkpoint-b", "changes.msnap", { since: "worker:checkpoint-a" }); +await Snapshot.load("changes.msnap", undefined, "worker:checkpoint-a"); const child = await Sandbox.builder("child") - .fromSnapshot("changes.msnap").snapshotBase("checkpoint-a").create(); + .fromSnapshot("changes.msnap").snapshotBase("worker:checkpoint-a").create(); ``` The count includes the oldest base but excludes the writable head. Omit `layers` to compact all sealed layers. `lastLayers` selects the newest N sealed export layers instead of `since`. Results expose physical counts, `materializedBytes`, `totalUs`, and `pauseUs`; materialized bytes are not reclaimed space. See [disk-chain maintenance](/sandboxes/snapshots#export-changes-and-compact-a-disk-chain) for dependency and retention rules. @@ -221,17 +238,17 @@ Best-effort source-sandbox name, if recorded. `null` when the manifest has no so #### Snapshot.builder() ```typescript -static builder(name: string): SnapshotBuilder +static builder(name?: string): SnapshotBuilder ``` -Begin building a new snapshot named `name`, resolved under the default snapshots directory (`~/.microsandbox/snapshots//`) or under [`.destDir()`](#destdir) when set. The fluent builder is what powers the CLI internally. The source sandbox is set with [`.fromSandbox()`](#fromsandbox), which is required. See [`SnapshotBuilder`](#snapshotbuilder) for all setters. +Begin building a snapshot member named `name`, generated when omitted. The default group is the source sandbox's name; `.group(name)` selects another group and [`.destDir()`](#destdir) selects its parent directory. The fluent builder is what powers the CLI internally. The source sandbox is set with [`.fromSandbox()`](#fromsandbox), which is required. See [`SnapshotBuilder`](#snapshotbuilder) for all setters.

Parameters

namestring
-
Bare snapshot name; becomes the artifact directory under the default snapshots dir.
+
Member name within its group; generated when omitted.
@@ -283,20 +300,20 @@ static open(pathOrName: string): Promise ```typescript -const snap = await Snapshot.open("after-pip-install"); +const snap = await Snapshot.open("baseline:after-pip-install"); console.log(snap.digest); ``` -Open an existing snapshot artifact. Bare names resolve under the default snapshots directory; anything else is treated as a path. Cheap metadata validation only; it does not read the upper file. Use [`verify()`](#snap-verify) for content checks. +Open an existing snapshot artifact by group head, `group:member`, or path. Cheap metadata validation only; it does not read the upper file. Use [`verify()`](#snap-verify) for content checks.

Parameters

pathOrNamestring
-
Bare name (resolved under the default snapshots dir) or filesystem path.
+
Group head or group:member selector or filesystem path.
@@ -318,7 +335,7 @@ static get(nameOrDigest: string): Promise ```typescript -const h = await Snapshot.get("after-pip-install"); +const h = await Snapshot.get("baseline:after-pip-install"); console.log(h.digest, h.createdAt); ``` @@ -406,19 +423,19 @@ static remove(pathOrName: string, opts?: { force?: boolean }): Promise ```typescript -await Snapshot.remove("after-pip-install", { force: true }); +await Snapshot.remove("baseline:after-pip-install", { force: true }); ``` -Remove a snapshot by path, name, or digest. Refuses if the snapshot has indexed children unless `force` is set. +Remove a snapshot by group selector, unambiguous ID or digest, or path. Refuses if the snapshot has indexed children unless `force` is set. A group's head cannot be removed while other members remain, even with `force`; select another head first.

Parameters

pathOrNamestring
-
Path, name, or digest of the snapshot to remove.
+
Group head, group:member, unambiguous snapshot ID or digest, or artifact path.
opts.forceboolean
@@ -477,7 +494,7 @@ Bundle a snapshot into a `.msnap` archive. The recorded manifest is archived as-
nameOrPathstring
-
Name or path of the snapshot to bundle.
+
Group head, group:member, or artifact path to bundle.
outstring
@@ -492,7 +509,7 @@ Bundle a snapshot into a `.msnap` archive. The recorded manifest is archived as- ```typescript -await Snapshot.save("after-pip-install", "./baseline.msnap", { +await Snapshot.save("baseline:after-pip-install", "./baseline.msnap", { withImage: true, }); ``` @@ -505,7 +522,7 @@ await Snapshot.save("after-pip-install", "./baseline.msnap", {
staticasync
```typescript -static load(archive: string, dest?: string): Promise +static load(archive: string, dest?: string, base?: string): Promise ``` Unpack a snapshot archive (`.msnap` or `.tar`) into the snapshots directory. Structural and archive-entry checks run during import; recorded payload integrity is preserved for explicit [`verify()`](#snap-verify). Compression is detected from magic bytes. @@ -594,7 +611,7 @@ Manifest digest (`sha256:hex`), the canonical identity. `string \| null` -Convenience name; `null` for digest-only entries. +Member name within its group, or `null` when no alias is recorded. #### snapshotHandle.parentDigest @@ -652,10 +669,10 @@ Open and metadata-validate the underlying artifact. Throws if this handle is rea remove(opts?: { force?: boolean }): Promise ``` -Remove the artifact and its index row. Refuses if the snapshot has indexed children unless `force` is set. Throws if this handle is read-only. +Remove this installed snapshot copy and its index row using the handle's stored artifact path. Other groups containing the same snapshot ID or digest remain unchanged. Refuses if the snapshot has indexed children unless `force` is set. Throws if this handle is read-only. ```typescript -const h = await Snapshot.get("after-pip-install"); +const h = await Snapshot.get("baseline:after-pip-install"); const snap = await h.open(); // metadata-validated await h.remove({ force: false }); // refuse if it has children ``` @@ -693,7 +710,7 @@ Set the sandbox to capture. Required; [`.create()`](#create) fails without it. destDir(destDir: string): this ``` -Create the artifact under this parent directory instead of the default snapshots store. The artifact directory is `destDir/`; the name stays the snapshot's identity either way. +Create the snapshot group under this parent directory instead of the default snapshots store. Member names are local aliases within the group; stable snapshot IDs identify immutable artifacts.

Parameters

@@ -731,7 +748,7 @@ Add a `key=value` label to the snapshot manifest. May be called repeatedly. force(): this ``` -Overwrite an existing artifact with the same name instead of failing on conflict. +Overwrite an existing direct archive output file. Installed group members are immutable, so installed creation rejects this option. #### snapshot.recordIntegrity() @@ -765,7 +782,6 @@ create(): Promise ```typescript const snap = await Snapshot.builder("baseline-v2") .fromSandbox("baseline") - .force() .recordIntegrity() .create(); ``` diff --git a/docs/snapshot-groups-explained.md b/docs/snapshot-groups-explained.md new file mode 100644 index 000000000..ae4b3bbbb --- /dev/null +++ b/docs/snapshot-groups-explained.md @@ -0,0 +1,132 @@ +# Microsandbox snapshots: groups, checkpoints, and the head + +This describes the snapshot-group implementation on the development stack, not an already released CLI. + +## Start with the snapshot + +A snapshot is a saved point you can use to create another sandbox: + +```text +Running sandbox + | + +-- disk snapshot ----> new VM boots from the saved disk + | + +-- full snapshot ----> new VM resumes saved RAM, CPUs, devices, and disk +``` + +Disk snapshots also work when the source is paused or stopped. Full snapshots require resident execution state: a running or user-paused VM. Each capture produces a new immutable snapshot, even when unchanged disk layers or RAM objects are reused. Exporting a snapshot packages it as a `.msnap` archive; loading an archive installs it without starting a VM. + +## A group gives those saved points a local home + +A **group** is a namespace containing snapshots and one selected **head**. Member names such as `cp01` are meaningful inside their group. Each snapshot also keeps its portable `snap_...` ID. + +```text +~/.microsandbox/snapshots/ +| ++-- worker/ +| +-- group.json head = snap_B +| +-- snap_A/ +| | +-- snapshot.json ID, parent, disk/state references +| | +-- group-member.json name = cp01 +| | +-- layers/... disk-only payload +| +-- snap_B/ +| +-- snapshot.json parent = snap_A +| +-- group-member.json name = cp02 +| +-- checkpoint/... full checkpoint payload, when captured full +| ++-- imported/ + +-- group.json + +-- snap_A/... a separate local copy of the same snapshot +``` + +IDs above are shortened for readability. A disk-only member uses `layers/`; a full member uses `checkpoint/` with its disk layers, RAM objects, and execution/device state. Optional `metadata.json` stores labels. + +Groups do not magically make random IDs collision-proof. They keep local addresses separate. Within one group, the same ID with the same descriptor is reusable; the same ID with different descriptor bytes is rejected. A name already used by another member is also rejected. Nothing is silently overwritten. If a global ID resolves to multiple local copies, use the group-qualified address instead. + +## Create and restore + +```bash +msb create alpine --name worker --memory 512M + +# Group defaults to the source sandbox's name: worker. +msb snapshot create cp01 --from worker --full +msb snapshot create cp02 --from worker --full + +# A bare group selects its head, currently cp02. +msb create --name latest --from-snapshot worker --forked + +# A qualified name selects an exact checkpoint. +msb create --name earlier --from-snapshot worker:cp01 --forked + +# You can choose a different group, or let a member name be generated. +msb snapshot create --from worker --group experiments --full +``` + +`--forked` shares clean restored RAM pages using copy-on-write; child writes remain private. It does not change which snapshot is selected. Omit `--full` at capture for a disk-only snapshot, and omit `--forked` when cold-booting disk state. + +## The head moves forward, not sideways by surprise + +Snapshots record their actual source ancestry. Neither timestamps, import order, nor an export's `--since` base defines that ancestry. + +```text +worker:cp01 ---- worker:cp02 ---- worker:cp03 <- head + \ + +-------- worker:experiment +``` + +The rules are small: + +- Empty group: the first successful publication initializes its head. +- Known descendant of the current head: advance automatically. +- Same member, older member, sibling, unrelated history, or missing ancestry: keep the current head. The capture/import still succeeds. +- Explicit selection: choose any complete installed member, including an older one. + +```bash +msb snapshot head worker # Read the current head ID +msb snapshot head worker:experiment # Explicitly choose the other branch +msb snapshot head worker:cp01 # Explicitly rewind +``` + +There is no special `main` branch. The head is a selected snapshot, not a rule for guessing which future branch is preferred. + +### What if two siblings arrive together? + +```text + +---- snapshot A +head: cp02 ---------+ + +---- snapshot B + +A publishes first: head cp02 -> A +B publishes next: B is A's sibling, so head stays A + +Result: both snapshots exist. Only the first head update wins. +``` + +Publication checks and head replacement share a per-group lock. The losing sibling is not discarded or reported as a failed capture. If you want B, select it explicitly. Two captures of the *same* source are serialized and record a parent chain; they are not treated as sibling captures. + +## Move a history to another machine + +```bash +# On the source machine: +msb snapshot save worker:cp01 cp01.msnap +msb snapshot save worker:cp02 cp02.msnap --since worker:cp01 + +# On the destination machine: +msb snapshot load cp01.msnap --group received +msb snapshot load cp02.msnap --group received --base received:cp01 +msb create --name restored --from-snapshot received --forked +``` + +`--since` omits disk layers and reusable RAM objects supplied by the explicit base. Loading reconstructs a complete owned snapshot; the target does not depend on replaying earlier VMs. Each archive still includes the target's complete memory map and CPU/device state. The `.msnap` archive does not contain a local group's mutable head file: its declared archive head is the import candidate, and the receiving group applies the rules above. + +Loading without `--group` creates a fresh generated group. The final stdout line is the installed artifact **path**, not its ID; scripts can pass it as the next `--base`. + +Importing an old checkpoint does not rewind an existing group. To deliberately select the imported archive's head: + +```bash +msb snapshot load cp01.msnap --group received --set-head +``` + +Missing historical checkpoints are okay when payload dependencies are complete. But a missing parent may prevent proving a fast-forward. Filling a history hole does not retrospectively select some other retained tip; select that tip explicitly or import it again once its ancestry is known. + +Direct archive capture (`snapshot create --archive`) and direct archive restore still skip installed snapshot directories. `msb branch` still creates a local child without publishing a durable snapshot. Neither operation implicitly moves a group's head; a later explicit capture can join a group using the child's recorded ancestry. diff --git a/packages/microsandbox-types/rust/lib/domain.rs b/packages/microsandbox-types/rust/lib/domain.rs index 686c370e9..9391af151 100644 --- a/packages/microsandbox-types/rust/lib/domain.rs +++ b/packages/microsandbox-types/rust/lib/domain.rs @@ -727,29 +727,31 @@ pub struct SandboxPolicy { /// Inputs to create a snapshot. /// -/// The snapshot's name is its identity; the artifact directory is -/// `dest_dir.join(name)`, with `dest_dir` defaulting to the snapshots -/// store. Archive movement happens through save/load (the artifact -/// directory is also self-contained and safe to move directly). +/// Installed artifacts live at `dest_dir//`. A friendly name +/// is scoped to the group; it does not change the portable snapshot identity. +/// Save/load moves artifacts between stores without starting a VM. #[derive(Debug, Clone, Serialize, Deserialize)] #[cfg_attr(feature = "ts", derive(ts_rs::TS))] pub struct SnapshotSpec { - /// Snapshot name. Always the artifact directory's basename. + /// Friendly member name within a group; empty selects a generated name. pub name: String, - /// Parent directory to create the artifact in. `None` = the default - /// snapshots directory. + /// Local snapshot group; defaults to the source sandbox's name. + #[serde(default)] + pub group: Option, + + /// Group-store root. `None` selects the default snapshots directory. #[serde(default)] #[cfg_attr(feature = "ts", ts(type = "string | null"))] pub dest_dir: Option, - /// Name of the source sandbox. Must be stopped. + /// Source sandbox. Disk capture accepts running, paused, or stopped sources. pub source_sandbox: String, /// User-supplied labels. pub labels: Vec<(String, String)>, - /// Overwrite an existing artifact at the destination. + /// Overwrite a direct archive destination; installed members remain immutable. pub force: bool, /// Compute and record upper-layer content integrity at creation time. diff --git a/scripts/smoke/cli/dirty-memory-checkpoint.py b/scripts/smoke/cli/dirty-memory-checkpoint.py index 98176c8eb..b0b4d3101 100644 --- a/scripts/smoke/cli/dirty-memory-checkpoint.py +++ b/scripts/smoke/cli/dirty-memory-checkpoint.py @@ -45,9 +45,13 @@ def do_GET(self): shared[:8] = b'shared01'; private[:8] = b'private1'; heap[:] = b'heap0001' Path('/dev/shm/latch-marker').write_bytes(b'tmpfs001') dirty = next(int(x.split()[1]) for x in Path('/proc/meminfo').read_text().splitlines() if x.startswith('Dirty:')) + # Poll only the marker: copying 64 MiB per request dirties unrelated heap pages + # and can turn this sparse mutation check into a legitimate dense full capture. + with open('/dirty-cache', 'rb') as disk_file: + disk_cache = disk_file.read(8).decode() body = json.dumps(dict(nonce=nonce, heap=heap.decode(), shared=shared[:8].decode(), private=private[:8].decode(), tmpfs=Path('/dev/shm/latch-marker').read_text(), - disk_cache=Path('/dirty-cache').read_bytes()[:8].decode(), dirty_kib=dirty, + disk_cache=disk_cache, dirty_kib=dirty, clock=time.time())).encode() self.send_response(200); self.send_header('Content-Length', str(len(body))); self.end_headers(); self.wfile.write(body) HTTPServer(('0.0.0.0', 8080), Handler).serve_forever() @@ -139,18 +143,19 @@ def stop(name): stop(source) for mode in ('eager', 'forked'): child = mode; names.append(child) - run('restore-' + mode, 'create', '--name', child, '--from-snapshot', 'dirty-full', + run('restore-' + mode, 'create', '--name', child, '--from-snapshot', 'source:dirty-full', *(['--forked'] if mode == 'forked' else [])) active = child matches(initial) if mode == 'forked': - # A restored child starts a fresh capture lineage. Establish its - # baseline before mutating, then verify the next capture is incremental. + # A restored child starts a fresh dirty-tracking baseline while retaining + # snapshot ancestry. Capture before mutating, then verify an incremental cut. run('child-baseline', 'snapshot', 'create', 'child-baseline', '--from', child, '--full') changed = state('/mutate') assert changed['private'] == 'private1' - run('incremental-dirty', 'snapshot', 'create', 'dirty-incremental', '--from', child, '--full') - checkpoint = home / 'snapshots/dirty-incremental/checkpoint' + captured = run('incremental-dirty', 'snapshot', 'create', 'dirty-incremental', '--from', child, '--full') + # Capture returns the exact installed member path, independent of its alias. + checkpoint = Path(captured.strip().splitlines()[-1]) / 'checkpoint' descriptor = json.loads((checkpoint / 'checkpoint.json').read_text()) algorithm, digest = descriptor['memory'].split(':', 1) memory = json.loads((checkpoint / 'objects' / algorithm / digest[:2] / digest).read_text()) @@ -159,11 +164,11 @@ def stop(name): matches(changed) stop(child) names.append('grandchild') - run('restore-incremental', 'create', '--name', 'grandchild', '--from-snapshot', 'dirty-incremental', '--forked') + run('restore-incremental', 'create', '--name', 'grandchild', '--from-snapshot', 'forked:dirty-incremental', '--forked') active = 'grandchild' matches(changed); stop('grandchild') names.append('disk-only') - run('disk-only', 'create', '--name', 'disk-only', '--from-snapshot', 'dirty-full', '--disk-only') + run('disk-only', 'create', '--name', 'disk-only', '--from-snapshot', 'source:dirty-full', '--disk-only') assert run('persisted-disk-data', 'exec', 'disk-only', '--', 'cat', '/persisted').strip() == 'persisted-before-checkpoint' run('no-tmpfs-in-disk-view', 'exec', 'disk-only', '--', 'test', '!', '-e', '/dev/shm/latch-marker') # Unsynced disk bytes are deliberately not asserted either present or absent. diff --git a/scripts/smoke/cli/snapshot-groups.py b/scripts/smoke/cli/snapshot-groups.py new file mode 100644 index 000000000..a27bee50c --- /dev/null +++ b/scripts/smoke/cli/snapshot-groups.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +"""Opt-in live snapshot-group qualification in an isolated MSB_HOME. + +Set MSB_PATH and GROUP_TEST_OUT; optionally GROUP_TEST_LAYOUT=512M, flat:512M, or tmpfs:128M. +All created VMs are stopped in finally, and every command/timing is retained. +""" + +from concurrent.futures import ThreadPoolExecutor +import json +import os +from pathlib import Path +import subprocess +import time + +binary = os.environ["MSB_PATH"] +root = Path(os.environ["GROUP_TEST_OUT"]) +root.mkdir(parents=True, exist_ok=False) +home = root / "home" +env = dict(os.environ, MSB_HOME=str(home)) +layout = os.environ.get("GROUP_TEST_LAYOUT", "flat:512M") +full_only = layout.startswith("tmpfs:") +rows = [] +names = [] + + +def run(label, *args, fail=False): + started = time.perf_counter() + result = subprocess.run([binary, *map(str, args)], env=env, capture_output=True, + text=True, timeout=180) + row = {"case": label, "ms": round((time.perf_counter() - started) * 1000, 2), + "exit": result.returncode} + rows.append(row) + print(json.dumps(row), flush=True) + (root / f"{label}.stdout").write_text(result.stdout) + (root / f"{label}.stderr").write_text(result.stderr) + if (result.returncode != 0) != fail: + raise RuntimeError(f"{label}: {result.stderr[-3000:]}") + return result.stdout.strip() + + +def guest(label, name, script): + return run(label, "exec", name, "--", "sh", "-ec", script) + + +def create(name, snapshot=None, forked=False): + names.append(name) + args = ["create", "--name", name] + if snapshot: + args += ["--from-snapshot", snapshot] + else: + args += ["alpine", "--root-disk", layout, "--memory", "256M", "--cpus", "2"] + if forked: + args.append("--forked") + run("create-" + name, *args) + + +def capture(label, source, member, group="work", full=False, fail=False): + args = ["snapshot", "create", member, "--from", source, "--group", group] + if full or full_only: + args.append("--full") + output = run(label, *args, fail=fail) + if fail: + return None + path = Path(output.splitlines()[-1]) + descriptor = json.loads((path / "snapshot.json").read_text()) + assert path.parent == home / "snapshots" / group, path + assert path.name == descriptor["snapshot_id"], path + return path, descriptor + + +def head(label, selector): + return json.loads(run(label, "snapshot", "head", selector, "--format", "json")) + + +try: + create("source") + guest("source-one", "source", "echo one > /disk-marker; echo ram-one > /dev/shm/marker; sync") + cp1, d1 = capture("capture-cp1", "source", "cp1") + assert d1["parent"] is None + assert head("initial-head", "work")["head"] == d1["snapshot_id"] + guest("source-two", "source", "echo two > /disk-marker; sync") + cp2, d2 = capture("capture-cp2", "source", "cp2") + assert d2["parent"] == d1["snapshot_id"] + assert head("advanced-head", "work")["head"] == d2["snapshot_id"] + + create("old-child", "work:cp1") + assert guest("read-old-child", "old-child", "cat /disk-marker") == "one" + branch, db = capture("capture-old-child", "old-child", "experiment") + assert db["parent"] == d1["snapshot_id"] + assert head("sibling-keeps-head", "work")["head"] == d2["snapshot_id"] + assert head("select-sibling", "work:experiment")["head"] == db["snapshot_id"] + create("selected-child", "work") + assert guest("read-selected-head", "selected-child", "cat /disk-marker") == "one" + run("stop-selected-child", "stop", "selected-child") + head("select-cp2", "work:cp2") + + # Same source serialization creates ancestry; different children of one head are siblings. + create("race-a", "work:cp2") + create("race-b", "work:cp2") + with ThreadPoolExecutor(max_workers=2) as pool: + pending = [pool.submit(capture, "capture-" + child, child, child) + for child in ("race-a", "race-b")] + siblings = [future.result() for future in pending] + ids = {desc["snapshot_id"] for _, desc in siblings} + assert all(desc["parent"] == d2["snapshot_id"] for _, desc in siblings) + assert head("race-head", "work")["head"] in ids + assert all(path.exists() for path, _ in siblings) + for name in ("old-child", "race-a", "race-b"): + run("stop-" + name, "stop", name) + head("select-cp2-again", "work:cp2") + + # Full capture, direct local branch ancestry, and paused-source preservation. + full1, f1 = capture("capture-full1", "source", "full1", full=True) + assert f1["parent"] == d2["snapshot_id"] + names.append("local-child") + run("local-branch", "branch", "source", "--name", "local-child") + local_snap, dl = capture("capture-local-child", "local-child", "local-child", full=True) + assert dl["parent"] == f1["snapshot_id"] + guest("source-three", "source", "echo three > /disk-marker; echo ram-three > /dev/shm/marker; sync") + run("pause-source", "pause", "source") + full2, f2 = capture("capture-paused-full2", "source", "full2", full=True) + assert f2["parent"] == f1["snapshot_id"] + run("resume-source", "resume", "source") + assert guest("source-still-live", "source", "cat /disk-marker") == "three" + run("stop-local-child", "stop", "local-child") + + # A rejected alias collision cannot replace the artifact or advance the cursor/head. + before = (full2 / "snapshot.json").read_bytes() + current = head("head-before-conflict", "work")["head"] + capture("duplicate-name-refused", "source", "full2", fail=True) + assert (full2 / "snapshot.json").read_bytes() == before + assert head("head-after-conflict", "work")["head"] == current + full3, f3 = capture("capture-after-conflict", "source", "full3", full=True) + assert f3["parent"] == f2["snapshot_id"] + + base_archive = root / "base.msnap" + delta_archive = root / "delta.msnap" + run("export-base", "snapshot", "save", full1, base_archive) + run("export-delta", "snapshot", "save", full2, delta_archive, "--since", full1) + inventory = json.loads(subprocess.check_output(["tar", "-xOf", str(delta_archive), "archive.json"])) + dependent = inventory["completeness"] == "dependent" + # A RAM-only cut can have no reusable objects: --since then emits a standalone archive. + # Require a base exactly when the archive actually omitted required payloads. + run("missing-base" + ("-refused" if dependent else "-not-needed"), "snapshot", "load", + delta_archive, "--group", "missing", fail=dependent) + if dependent: + assert not (home / "snapshots" / "missing").exists() + imported_base = Path(run("import-base", "snapshot", "load", base_archive, + "--group", "received").splitlines()[-1]) + imported_delta = Path(run("import-delta", "snapshot", "load", delta_archive, + "--group", "received", "--base", "received:full1").splitlines()[-1]) + assert imported_base.name == f1["snapshot_id"] and imported_delta.name == f2["snapshot_id"] + assert head("import-advanced-head", "received")["head"] == f2["snapshot_id"] + run("reimport-old", "snapshot", "load", base_archive, "--group", "received") + assert head("old-import-kept-head", "received")["head"] == f2["snapshot_id"] + run("reimport-set-head", "snapshot", "load", base_archive, "--group", "received", "--set-head") + assert head("old-import-explicit-head", "received")["head"] == f1["snapshot_id"] + run("head-removal-refused", "snapshot", "remove", "received:full1", "--force", fail=True) + + duplicate = Path(run("import-second-group", "snapshot", "load", base_archive).splitlines()[-1]) + assert duplicate.name == imported_base.name and duplicate.parent != imported_base.parent + run("ambiguous-id-refused", "snapshot", "inspect", f1["snapshot_id"], fail=True) + run("reindex", "snapshot", "reindex") + for mode in ("eager", "forked"): + child = "restored-" + mode + create(child, "received:full2", forked=mode == "forked") + assert guest("restored-state-" + mode, child, + "cat /disk-marker; cat /dev/shm/marker") == "three\nram-three" + run("stop-" + child, "stop", child) + + # Direct archive capture records ancestry but never creates an installed member. + before_members = sorted(str(p) for p in (home / "snapshots").rglob("snapshot.json")) + direct = root / "direct.msnap" + run("direct-capture", "snapshot", "create", "direct", "--from", "source", "--full", "--archive", direct) + assert sorted(str(p) for p in (home / "snapshots").rglob("snapshot.json")) == before_members + create("direct-restored", str(direct), forked=True) + assert guest("direct-restored-state", "direct-restored", "cat /dev/shm/marker") == "ram-three" + assert sorted(str(p) for p in (home / "snapshots").rglob("snapshot.json")) == before_members + run("stop-direct-restored", "stop", "direct-restored") + run("stop-source", "stop", "source") + if full_only: + capture("stopped-tmpfs-refused", "source", "stopped", fail=True) + else: + stopped, stopped_desc = capture("stopped-capture", "source", "stopped") + assert stopped_desc["parent"] != f3["snapshot_id"], "direct archive did not advance source ancestry" + create("stopped-restored", "work:stopped") + assert guest("stopped-restored-state", "stopped-restored", "cat /disk-marker") == "three" + print(json.dumps({"pass": True, "layout": layout, "commands": len(rows)}), flush=True) +finally: + for name in reversed(names): + result = subprocess.run([binary, "stop", name], env=env, capture_output=True, + text=True, timeout=30) + (root / ("cleanup-" + name + ".log")).write_text(result.stdout + result.stderr) + (root / "results.json").write_text(json.dumps(rows, indent=2)) diff --git a/scripts/smoke/reports/snapshot-groups-2026-09-10.md b/scripts/smoke/reports/snapshot-groups-2026-09-10.md new file mode 100644 index 000000000..7b3d2c062 --- /dev/null +++ b/scripts/smoke/reports/snapshot-groups-2026-09-10.md @@ -0,0 +1,110 @@ +# Snapshot groups qualification — 2026-09-10 + +## Result + +Snapshot groups, scoped selectors, ancestry-aware head advancement, explicit selection, archive import, and CLI/Rust/Python/TypeScript/Go surfaces were initially implemented and qualified in an isolated detached worktree based on `e20269e1b260149e4cf629f43fac142a35e1458b`. The initial qualification below did not edit the concurrent #8 checkout. Subsequent integration checks are recorded separately so these original measurements retain their context. + +The final macOS ARM64/HVF live matrix passed with flat, managed/layered, and tmpfs roots. These runs verify group integration with existing execution state and archive workflows; they are not a fresh Linux or Windows qualification of the stack. + +## Live coverage + +| Check | Flat 512 MiB | Managed 512 MiB | Tmpfs 128 MiB | +| --- | --- | --- | --- | +| Capture, stable ID directory, alias, recorded parent | Pass | Pass | Pass, full only | +| Automatic fast-forward and explicit head selection | Pass | Pass | Pass | +| Restore exact old member versus selected group head | Pass | Pass | Pass | +| Concurrent sibling captures retain both members; one wins head | Pass | Pass | Pass | +| Full capture and capture from user-paused source | Pass | Pass | Pass | +| Direct branch child records source's last captured ancestor | Pass | Pass | Pass | +| Duplicate name rejects without changing existing member/head/cursor | Pass | Pass | Pass | +| Base + `--since` export/load, preserved aliases and IDs | Pass | Pass | Pass, self-contained export | +| Missing required base rejected before group creation | Pass | Pass | No dependency omitted in this workload | +| Old/idempotent import retains head; `--set-head` explicitly rewinds | Pass | Pass | Pass | +| Same archive in another group; ambiguous global ID refused | Pass | Pass | Pass | +| Head deletion protection, even with `--force` | Pass | Pass | Pass | +| Eager and forked full restore reproduce disk/RAM markers | Pass | Pass | Pass | +| Direct full archive capture and restore install no snapshot member | Pass | Pass | Pass | +| Stopped-source disk capture and restore | Pass | Pass | Correctly refused | + +The final matrices ran 66, 66, and 64 CLI commands respectively, including expected negative cases and explicit stops. Test-owned VMs are stopped in `finally`, including failed creates. The tmpfs source's `--since` archive had no reusable object omissions, so its inventory correctly declared `boot-complete`; the fixture checks whether a base is required from the actual inventory rather than assuming every `--since` request produces a dependent archive. RAM dependency reconstruction is independently covered by the 12-checkpoint RAM-only and disk-plus-RAM artifact tests. + +## Timing context + +These are individual end-to-end **debug-build** CLI wall times, not pause duration, release performance, or a regression comparison. The flat/managed matrices overlapped, while the final tmpfs retry ran separately. Sources had two vCPUs and 256 MiB of RAM with small disk/RAM marker files. They are intentionally small correctness fixtures, not representative large working sets. + +| Operation | Flat | Managed | Tmpfs | +| --- | ---: | ---: | ---: | +| Initial group head read | 7.98 ms | 7.73 ms | 10.15 ms | +| Second disk capture | 123.37 ms | 128.96 ms | Not applicable | +| Full capture (`full1`) | 1356.53 ms | 1099.99 ms | 791.07 ms | +| Capture while paused | 1260.54 ms | 1162.57 ms | 770.73 ms | +| Installed eager full restore | 936.21 ms | 725.02 ms | 522.50 ms | +| Installed forked full restore | 920.67 ms | 733.85 ms | 546.03 ms | +| Direct local branch | 922.42 ms | 865.51 ms | 607.58 ms | +| Direct full archive capture | 2279.01 ms | 1863.23 ms | 1493.84 ms | +| Direct forked archive restore | 2017.03 ms | 1650.82 ms | 1257.52 ms | + +Raw results are `/private/tmp/sgf2/results.json`, `/private/tmp/sgm2/results.json`, and `/private/tmp/sgt2/results.json`, with per-command stdout/stderr alongside them. Initial retries documented fixture issues: an omitted firmware override, selecting an older installed firmware without VM Generation ID readiness, a too-long macOS socket path, and the tmpfs dependency assumption above. No failed run is counted as a passing matrix. + +## Automated checks + +- Snapshot library tests: 61 passed, including 15 group tests, five ancestry tests, duplicate IDs, generated-name retry without recapture, cancellation-safe cursor locking, missing history, source replacement, scoped deletion, and RAM-aware archive reconstruction. +- Artifact integration tests: 45 passed, including released-flat import normalization, repeated imports, alias-preserving re-export, and explicit legacy paths. +- Migration tests: 27 passed; downgrade preflight and rollback tests passed separately. +- Same-name/different-backend full-capture routing and post-publication resume-failure handling: passed with local socket permissions. +- CLI snapshot tests: 10 passed. TypeScript snapshot/native contract tests: 11 passed. Go unit tests passed; Go integration tests compiled. Native Rust bindings checked for Python, TypeScript, and Go; TypeScript declarations regenerated and typecheck passed. +- `cargo fmt --all -- --check` and `git diff --check`: passed. +- Strict broad Clippy encountered pre-existing `derivable_impls` in `crates/image/lib/snapshot/manifest.rs`; targeted `--no-deps -D warnings` encountered pre-existing `too_many_arguments` in `build_artifact`. Targeted Clippy passed with only that existing lint allowed via command-line `-A clippy::too_many_arguments`. No unrelated source was changed to silence lints. +- Full Python/Go VM suites, Linux, Windows, workspace-wide tests, and a release-build performance comparison were not rerun for this change. Python stub Ruff retains two unrelated pre-existing line-length findings. + +## Reproduce + +Use a short, fresh output path on macOS: + +```bash +MSB_PATH=/path/to/codesigned/msb \ +MSB_LIBKRUNFW_PATH=/path/to/matching/libkrunfw.5.dylib \ +GROUP_TEST_OUT=/tmp/sg-test \ +GROUP_TEST_LAYOUT=flat:512M \ +python3 scripts/smoke/cli/snapshot-groups.py +``` + +Repeat with `512M` and `tmpfs:128M`. The script owns an isolated `MSB_HOME` under its output directory and never stops unrelated sandboxes. + +The qualified binary SHA-256 was `6b71b08c9d89f87c59a60b5a218903ec048d6ab47066846d9d83c7d674917fa7`; firmware SHA-256 was `ea0d458cdc12a0fa6dac8d192542ddc39717f816da41176582905e31a8bf868c`; embedded agentd SHA-256 was `1425dc4b6974c10983db03e023c2b868c890fc197857ea45d6289a83df593aa8`. This uses the existing #8 guest artifacts; no kernel or agent implementation changes are part of the group feature. + +## Publication and recovery boundaries + +Group/head and per-source ancestry locks are process-held filesystem locks, not a daemon or a distributed authority. Head/ID lookup opens only the selected member; aliases scan local member-name metadata. An interrupted operation may have published a complete member, so callers should inspect before retrying. A process crash between member/head publication and cursor persistence may leave conservative ancestry that needs explicit head selection; it cannot silently overwrite a sibling. Group identity/name conflicts fail before moving staged members. Missing ancestry is distinct from missing payload data. Explicit removal racing an already resolved open may fail that operation; it never silently changes the chosen snapshot or cold-boots instead. + +Friendly names and group heads are local metadata, not new cryptographic identities. Descriptors retain their existing schema and portable IDs. Released flat artifacts remain readable by explicit path; there is no automatic relocation of old directories. + +## Integration onto updated #8 + +The group commit was replayed onto `86873fa68806914a8417cd0fa4e5f6eaa068105b`, retaining both newer commits: `8713ded2` (resident lifecycle performance) and `86873fa6` (Node ownership locking). No newer-main changes were imported. The newer dirty-memory smoke script was updated to use qualified member selectors and the installed path returned by capture. SDK reference examples were checked against the grouped APIs. + +Post-integration checks passed: 61 snapshot tests, 45 artifact tests, 10 CLI snapshot tests, four read-only control-lookup tests, the unindexed-group downgrade refusal test, and the same-name backend capture-routing test. All 27 migration tests also passed during integration preparation. Native bindings checked for Python, Node and Go; TypeScript typecheck and 16 focused unit tests passed; Go unit tests passed. Formatting and targeted Clippy passed with the previously documented `too_many_arguments` allowance. Native checks used the existing isolated SDK bootstrap home after the default prebuilt installer attempted an unavailable network download; no installed runtime was replaced. + +The complete macOS group matrices passed again: 66 commands each for flat and managed roots and 64 for tmpfs. Raw results are `/private/tmp/sgpf/results.json`, `/private/tmp/sgpm/results.json`, and `/private/tmp/sgpt/results.json`. The integrated, codesigned debug binary SHA-256 is `56f901eba49a3e748608cd2c987f9c7495959c4d6e43981bc0e5aa3c47efa067`, retained at `/private/tmp/sg-push-integrated-msb`; firmware and embedded agentd are unchanged from the initial qualification. + +| Integrated debug CLI operation | Flat | Managed | Tmpfs | +| --- | ---: | ---: | ---: | +| Initial group head read | 6.80 ms | 9.00 ms | 9.58 ms | +| Second disk capture | 114.73 ms | 137.68 ms | Not applicable | +| Full capture | 1275.77 ms | 966.86 ms | 814.14 ms | +| Capture while paused | 1260.71 ms | 993.24 ms | 777.49 ms | +| Installed eager restore | 949.10 ms | 697.05 ms | 504.40 ms | +| Installed forked restore | 947.55 ms | 718.54 ms | 534.55 ms | +| Direct branch | 843.45 ms | 777.08 ms | 626.83 ms | +| Direct full archive capture | 2527.27 ms | 1788.07 ms | 1502.21 ms | +| Direct forked archive restore | 2510.29 ms | 1571.97 ms | 1254.42 ms | + +These remain small debug-build correctness fixtures; some runs overlapped other builds or tests. They are not isolated performance comparisons or VM pause measurements. Linux, Windows, and release-performance qualification were not rerun during this push. + +### Existing dirty-memory qualification gap + +The additional `dirty-memory-checkpoint.py` smoke test did **not** pass. Its second full capture from a forked child reported `capture_mode: full` rather than the asserted `incremental`. Source/child RAM checks, direct branching, branch-of-branch, paused captures, and eager/forked restoration passed up to that assertion; the later grandchild and disk-only checks in that script were not reached. These failures do not invalidate the separate complete group matrices above, but they must not be described as a complete dirty-memory qualification. + +The original poll allocated and copied the entire 64 MiB disk-cache file merely to inspect eight bytes. The integration changes limit that read to eight bytes while retaining the dirty 64 MiB shared mapping and every assertion. This reduced observer overhead but did not resolve the full-capture result: both flat and managed reruns still failed. The runtime has an existing density fallback that chooses a full capture at 60% dirty RAM, but the failed runs do not log the decision reason, so that explanation remains unproven. + +Crucially, the exact pre-group #8 code at `86873fa6`, rebuilt separately with the same guest artifacts, reproduces the same assertion using its original script on a managed root. This establishes that the failure exists before the group change; no runtime workaround or relaxed assertion was added. Baseline binary SHA-256: `6d1eb6edaba2794d564c2b85f4c48598bb6a3d95b8726d861fccac97ac8a9c57` (`/private/tmp/sg-push-baseline-msb`). Reports: `/private/tmp/sg-dirty-push-flat/report.json`, `/private/tmp/sg-dirty-push-flat2/report.json`, `/private/tmp/sg-dirty-push-managed2/report.json`, and `/private/tmp/sg-dirty-baseline/report.json`. Every dirty-memory run reported an empty cleanup-error list. diff --git a/sdk/go/integration/snapshot_test.go b/sdk/go/integration/snapshot_test.go index 6fb00400d..ca9789ad6 100644 --- a/sdk/go/integration/snapshot_test.go +++ b/sdk/go/integration/snapshot_test.go @@ -23,11 +23,12 @@ func TestSandboxHandleSnapshotAndWithFromSnapshotFork(t *testing.T) { baseName := uniqueIntegrationName(t, "go-sdk-snapshot-base") forkName := uniqueIntegrationName(t, "go-sdk-snapshot-fork") snapshotName := uniqueIntegrationName(t, "go-sdk-snapshot") + snapshotSelector := baseName + ":" + snapshotName t.Cleanup(func() { removeSandboxBestEffort(forkName) removeSandboxBestEffort(baseName) - removeSnapshotBestEffort(snapshotName) + removeSnapshotBestEffort(snapshotSelector) }) phaseStart := time.Now() @@ -79,7 +80,7 @@ func TestSandboxHandleSnapshotAndWithFromSnapshotFork(t *testing.T) { t.Fatalf("Verify returned incomplete report: %+v", report) } - handle, err := microsandbox.Snapshot.Get(ctx, snapshotName) + handle, err := microsandbox.Snapshot.Get(ctx, snapshotSelector) if err != nil { t.Fatalf("Snapshot.Get: %v", err) } @@ -114,7 +115,7 @@ func TestSandboxHandleSnapshotAndWithFromSnapshotFork(t *testing.T) { } phaseStart = time.Now() - fork, err := createSandbox(t, ctx, forkName, microsandbox.WithFromSnapshot(snapshotName)) + fork, err := createSandbox(t, ctx, forkName, microsandbox.WithFromSnapshot(snapshotSelector)) if err != nil { t.Fatalf("CreateSandbox with WithFromSnapshot: %v", err) } @@ -182,9 +183,18 @@ func TestSnapshotCreateAndSnapshotDirectoryOps(t *testing.T) { } logSnapshotPhase(t, "create snapshot", phaseStart) snapshotDir := artifact.Path() - if filepath.Base(snapshotDir) != snapshotName { - t.Fatalf("Snapshot.Create path = %q, want basename %q", snapshotDir, snapshotName) + if filepath.Base(snapshotDir) != artifact.ID() { + t.Fatalf("Snapshot.Create path = %q, want stable ID basename %q", snapshotDir, artifact.ID()) + } + update := artifact.HeadUpdate() + if update == nil || update.Group != baseName || update.Head != artifact.ID() { + t.Fatalf("Snapshot.Create missing group head outcome: %#v", update) + } + head, err := microsandbox.Snapshot.GroupHead(ctx, baseName) + if err != nil || head.Head != artifact.ID() { + t.Fatalf("Snapshot.GroupHead = %#v, err = %v", head, err) } + t.Cleanup(func() { removeSnapshotBestEffort(snapshotDir) }) opened, err := microsandbox.Snapshot.Open(ctx, snapshotDir) if err != nil { @@ -219,7 +229,7 @@ func TestSnapshotCreateAndSnapshotDirectoryOps(t *testing.T) { archivePath := filepath.Join(t.TempDir(), "snapshot.tar") phaseStart = time.Now() - if err := microsandbox.Snapshot.Save(ctx, snapshotName, archivePath, + if err := microsandbox.Snapshot.Save(ctx, snapshotDir, archivePath, microsandbox.SnapshotSaveOptions{PlainTar: true}); err != nil { t.Fatalf("Snapshot.Save: %v", err) } @@ -243,6 +253,29 @@ func TestSnapshotCreateAndSnapshotDirectoryOps(t *testing.T) { if imported.Digest() != artifact.Digest() { t.Fatalf("Snapshot.Load digest = %q, want %q", imported.Digest(), artifact.Digest()) } + + // Independent imports share an identity and digest. Handle operations must stay + // bound to one installed copy instead of re-resolving an ambiguous digest. + duplicate, err := microsandbox.Snapshot.Load(loadCtx, archivePath, importDir) + if err != nil { + t.Fatalf("Snapshot.Load duplicate: %v", err) + } + t.Cleanup(func() { removeSnapshotBestEffort(duplicate.Path()) }) + if duplicate.Path() == imported.Path() || duplicate.ID() != imported.ID() { + t.Fatalf("imports should be distinct copies of one snapshot: %q / %q", imported.Path(), duplicate.Path()) + } + if err := imported.Remove(loadCtx, false); err != nil { + t.Fatalf("SnapshotHandle.Remove with duplicate copies: %v", err) + } + if _, err := imported.Open(loadCtx); err == nil { + t.Fatal("removed handle unexpectedly reopened a different copy") + } + if _, err := duplicate.Open(loadCtx); err != nil { + t.Fatalf("removing first import affected second copy: %v", err) + } + if _, err := microsandbox.Snapshot.Open(loadCtx, snapshotDir); err != nil { + t.Fatalf("removing an import affected the original snapshot: %v", err) + } } func logSnapshotPhase(t *testing.T, phase string, started time.Time) { diff --git a/sdk/go/internal/ffi/ffi.go b/sdk/go/internal/ffi/ffi.go index dee9de114..c877507f5 100644 --- a/sdk/go/internal/ffi/ffi.go +++ b/sdk/go/internal/ffi/ffi.go @@ -232,6 +232,8 @@ typedef char *(*msb_snapshot_reindex_fn)(uint64_t cancel_id, const char *dir, ui typedef char *(*msb_snapshot_export_fn)(uint64_t cancel_id, const char *name_or_path, const char *out, const char *opts_json, uint8_t *buf, size_t buf_len); typedef char *(*msb_snapshot_import_fn)(uint64_t cancel_id, const char *archive, const char *dest, uint8_t *buf, size_t buf_len); typedef char *(*msb_snapshot_import_with_base_fn)(uint64_t cancel_id, const char *archive, const char *dest, const char *base, uint8_t *buf, size_t buf_len); +typedef char *(*msb_snapshot_import_with_options_fn)(uint64_t cancel_id, const char *archive, const char *opts_json, uint8_t *buf, size_t buf_len); +typedef char *(*msb_snapshot_group_head_fn)(uint64_t cancel_id, const char *selector, uint8_t *buf, size_t buf_len); typedef char *(*msb_sandbox_compact_fn)(uint64_t cancel_id, uint64_t handle, const char *name, const char *opts, uint8_t *buf, size_t buf_len); typedef char *(*msb_fs_read_stream_fn)(uint64_t cancel_id, uint64_t handle, const char *path, uint8_t *buf, size_t buf_len); @@ -396,6 +398,8 @@ static msb_snapshot_reindex_fn ptr_msb_snapshot_reindex = NULL; static msb_snapshot_export_fn ptr_msb_snapshot_export = NULL; static msb_snapshot_import_fn ptr_msb_snapshot_import = NULL; static msb_snapshot_import_with_base_fn ptr_msb_snapshot_import_with_base = NULL; +static msb_snapshot_import_with_options_fn ptr_msb_snapshot_import_with_options = NULL; +static msb_snapshot_group_head_fn ptr_msb_snapshot_group_head = NULL; static msb_sandbox_compact_fn ptr_msb_sandbox_compact = NULL; // dlopen handle — set once by load_microsandbox, never closed. @@ -579,6 +583,8 @@ const char *load_microsandbox(const char *path) { RESOLVE(msb_snapshot_export); RESOLVE(msb_snapshot_import); RESOLVE(msb_snapshot_import_with_base); + RESOLVE(msb_snapshot_import_with_options); + RESOLVE(msb_snapshot_group_head); RESOLVE(msb_sandbox_compact); return NULL; } @@ -1018,6 +1024,12 @@ char *call_msb_snapshot_import(uint64_t cancel_id, const char *archive, const ch char *call_msb_snapshot_import_with_base(uint64_t cancel_id, const char *archive, const char *dest, const char *base, uint8_t *buf, size_t buf_len) { return ptr_msb_snapshot_import_with_base ? ptr_msb_snapshot_import_with_base(cancel_id, archive, dest, base, buf, buf_len) : NULL; } +char *call_msb_snapshot_import_with_options(uint64_t cancel_id, const char *archive, const char *opts_json, uint8_t *buf, size_t buf_len) { + return ptr_msb_snapshot_import_with_options ? ptr_msb_snapshot_import_with_options(cancel_id, archive, opts_json, buf, buf_len) : NULL; +} +char *call_msb_snapshot_group_head(uint64_t cancel_id, const char *selector, uint8_t *buf, size_t buf_len) { + return ptr_msb_snapshot_group_head ? ptr_msb_snapshot_group_head(cancel_id, selector, buf, buf_len) : NULL; +} char *call_msb_sandbox_compact(uint64_t cancel_id, uint64_t handle, const char *name, const char *opts, uint8_t *buf, size_t buf_len) { return ptr_msb_sandbox_compact ? ptr_msb_sandbox_compact(cancel_id, handle, name, opts, buf, buf_len) : NULL; } @@ -4726,28 +4738,29 @@ func ImageSave(ctx context.Context, references []string, outputPath string, form // --------------------------------------------------------------------------- type SnapshotInfo struct { - ID string `json:"id"` - Path string `json:"path"` - Digest string `json:"digest"` - SizeBytes *uint64 `json:"size_bytes"` - ImageRef string `json:"image_ref"` - ImageManifestDigest string `json:"image_manifest_digest"` - Scope string `json:"scope"` - StateKind string `json:"state_kind"` - Format *string `json:"format"` - Fstype *string `json:"fstype"` - UpperFile *string `json:"upper_file"` - UpperIntegrityAlgorithm *string `json:"upper_integrity_algorithm"` - UpperIntegrityDigest *string `json:"upper_integrity_digest"` - UpperIntegrityRoot *string `json:"upper_integrity_root"` - UpperIntegrityLogicalSize *uint64 `json:"upper_integrity_logical_size"` - UpperIntegrityLeafSize *uint32 `json:"upper_integrity_leaf_size"` - CheckpointID *string `json:"checkpoint_id"` - CheckpointManifestDigest *string `json:"checkpoint_manifest_digest"` - Parent *string `json:"parent"` - CreatedAt string `json:"created_at"` - Labels map[string]string `json:"labels"` - SourceSandbox *string `json:"source_sandbox"` + HeadUpdate *SnapshotHeadUpdate `json:"head_update"` + ID string `json:"id"` + Path string `json:"path"` + Digest string `json:"digest"` + SizeBytes *uint64 `json:"size_bytes"` + ImageRef string `json:"image_ref"` + ImageManifestDigest string `json:"image_manifest_digest"` + Scope string `json:"scope"` + StateKind string `json:"state_kind"` + Format *string `json:"format"` + Fstype *string `json:"fstype"` + UpperFile *string `json:"upper_file"` + UpperIntegrityAlgorithm *string `json:"upper_integrity_algorithm"` + UpperIntegrityDigest *string `json:"upper_integrity_digest"` + UpperIntegrityRoot *string `json:"upper_integrity_root"` + UpperIntegrityLogicalSize *uint64 `json:"upper_integrity_logical_size"` + UpperIntegrityLeafSize *uint32 `json:"upper_integrity_leaf_size"` + CheckpointID *string `json:"checkpoint_id"` + CheckpointManifestDigest *string `json:"checkpoint_manifest_digest"` + Parent *string `json:"parent"` + CreatedAt string `json:"created_at"` + Labels map[string]string `json:"labels"` + SourceSandbox *string `json:"source_sandbox"` } type SnapshotArchiveInfo struct { @@ -4757,23 +4770,25 @@ type SnapshotArchiveInfo struct { } type SnapshotHandleInfo struct { - ID string `json:"id"` - Digest string `json:"digest"` - Name *string `json:"name"` - ParentDigest *string `json:"parent_digest"` - ImageRef string `json:"image_ref"` - Scope string `json:"scope"` - StateKind string `json:"state_kind"` - Format *string `json:"format"` - Fstype *string `json:"fstype"` - CheckpointManifestDigest *string `json:"checkpoint_manifest_digest"` - SizeBytes *uint64 `json:"size_bytes"` - Locality string `json:"locality"` - Availability string `json:"availability"` - MigrationState string `json:"migration_state"` - MigrationErrorCode *string `json:"migration_error_code"` - CreatedAtUnix int64 `json:"created_at_unix"` - Path string `json:"path"` + Group *string `json:"group"` + HeadUpdate *SnapshotHeadUpdate `json:"head_update"` + ID string `json:"id"` + Digest string `json:"digest"` + Name *string `json:"name"` + ParentDigest *string `json:"parent_digest"` + ImageRef string `json:"image_ref"` + Scope string `json:"scope"` + StateKind string `json:"state_kind"` + Format *string `json:"format"` + Fstype *string `json:"fstype"` + CheckpointManifestDigest *string `json:"checkpoint_manifest_digest"` + SizeBytes *uint64 `json:"size_bytes"` + Locality string `json:"locality"` + Availability string `json:"availability"` + MigrationState string `json:"migration_state"` + MigrationErrorCode *string `json:"migration_error_code"` + CreatedAtUnix int64 `json:"created_at_unix"` + Path string `json:"path"` } type SnapshotVerifyReport struct { @@ -4792,6 +4807,7 @@ type SnapshotVerifyReport struct { type SnapshotCreateOptions struct { Name string `json:"name,omitempty"` + Group string `json:"group,omitempty"` DestDir string `json:"dest_dir,omitempty"` Labels map[string]string `json:"labels,omitempty"` Force bool `json:"force,omitempty"` @@ -4807,6 +4823,21 @@ type SnapshotSaveOptions struct { PlainTar bool `json:"plain_tar,omitempty"` } +type SnapshotLoadOptions struct { + Dest string `json:"dest,omitempty"` + Base string `json:"base,omitempty"` + Group string `json:"group,omitempty"` + SetHead bool `json:"set_head,omitempty"` +} + +type SnapshotHeadUpdate struct { + Group string `json:"group"` + Previous *string `json:"previous"` + Head string `json:"head"` + Reason string `json:"reason"` + Changed bool `json:"changed"` +} + func SandboxHandleSnapshot(ctx context.Context, sandboxName, snapshotName string) (*SnapshotInfo, error) { if err := ensureLoaded(); err != nil { return nil, err @@ -5067,3 +5098,46 @@ func SnapshotLoadWithBase(ctx context.Context, archive, dest, base string) (*Sna } return &info, nil } + +func SnapshotLoadWithOptions(ctx context.Context, archive string, opts SnapshotLoadOptions) (*SnapshotHandleInfo, error) { + if err := ensureLoaded(); err != nil { + return nil, err + } + payload, err := json.Marshal(opts) + if err != nil { + return nil, err + } + cArchive, cOpts := C.CString(archive), C.CString(string(payload)) + defer C.free(unsafe.Pointer(cArchive)) + defer C.free(unsafe.Pointer(cOpts)) + out, err := call(ctx, func(cancelID C.uint64_t, buf *C.uint8_t, bufLen C.size_t) *C.char { + return C.call_msb_snapshot_import_with_options(cancelID, cArchive, cOpts, buf, bufLen) + }) + if err != nil { + return nil, err + } + var info SnapshotHandleInfo + if err := json.Unmarshal([]byte(out), &info); err != nil { + return nil, fmt.Errorf("parse snapshot load: %w", err) + } + return &info, nil +} + +func SnapshotGroupHead(ctx context.Context, selector string) (*SnapshotHeadUpdate, error) { + if err := ensureLoaded(); err != nil { + return nil, err + } + cSelector := C.CString(selector) + defer C.free(unsafe.Pointer(cSelector)) + out, err := call(ctx, func(cancelID C.uint64_t, buf *C.uint8_t, bufLen C.size_t) *C.char { + return C.call_msb_snapshot_group_head(cancelID, cSelector, buf, bufLen) + }) + if err != nil { + return nil, err + } + var update SnapshotHeadUpdate + if err := json.Unmarshal([]byte(out), &update); err != nil { + return nil, fmt.Errorf("parse snapshot group head: %w", err) + } + return &update, nil +} diff --git a/sdk/go/native/microsandbox_go_ffi.h b/sdk/go/native/microsandbox_go_ffi.h index 45882e689..90bda08c5 100644 --- a/sdk/go/native/microsandbox_go_ffi.h +++ b/sdk/go/native/microsandbox_go_ffi.h @@ -815,6 +815,23 @@ char *msb_snapshot_import_with_base(uint64_t cancel_id, unsigned char *buf, uintptr_t buf_len); +/** + * Import an archive with group selection without changing the existing import ABI. + */ +char *msb_snapshot_import_with_options(uint64_t cancel_id, + const char *archive, + const char *opts_json, + unsigned char *buf, + uintptr_t buf_len); + +/** + * Read a group head, or select a `group:member` as its head. + */ +char *msb_snapshot_group_head(uint64_t cancel_id, + const char *selector, + unsigned char *buf, + uintptr_t buf_len); + /** * Open a streaming read from a guest file. * Returns `{"stream_handle":}`. diff --git a/sdk/go/native/src/lib.rs b/sdk/go/native/src/lib.rs index cd2881e43..a9dfe749d 100644 --- a/sdk/go/native/src/lib.rs +++ b/sdk/go/native/src/lib.rs @@ -1137,6 +1137,7 @@ struct LogStreamOpts { #[derive(serde::Deserialize, Default)] struct SnapshotCreateOpts { name: Option, + group: Option, dest_dir: Option, #[serde(default)] labels: HashMap, @@ -1160,6 +1161,15 @@ struct SnapshotSaveOptsJson { plain_tar: bool, } +#[derive(serde::Deserialize, Default)] +struct SnapshotLoadOptsJson { + dest: Option, + base: Option, + group: Option, + #[serde(default)] + set_head: bool, +} + #[derive(serde::Deserialize, Default)] struct MountSpec { bind: Option, @@ -5882,6 +5892,7 @@ fn snapshot_json(s: &Snapshot) -> serde_json::Value { }; serde_json::json!({ "path": s.path().display().to_string(), + "head_update": s.head_update(), "id": s.id().as_str(), "digest": s.digest(), "size_bytes": s.size_bytes(), @@ -5911,6 +5922,8 @@ fn snapshot_handle_json(h: µsandbox::SnapshotHandle) -> serde_json::Value { "id": h.id(), "digest": h.digest(), "name": h.name(), + "group": h.group(), + "head_update": h.head_update(), "parent_digest": h.parent_digest(), "image_ref": h.image_ref(), "scope": snapshot_scope_str(h.scope()), @@ -5950,10 +5963,10 @@ fn snapshot_builder_from_opts( source_sandbox: String, opts: SnapshotCreateOpts, ) -> Result { - let Some(name) = opts.name else { - return Err(FfiError::invalid_argument("snapshot create requires name")); - }; - let mut builder = Snapshot::builder(name).from_sandbox(source_sandbox); + let mut builder = Snapshot::builder(opts.name.unwrap_or_default()).from_sandbox(source_sandbox); + if let Some(group) = opts.group { + builder = builder.group(group); + } if let Some(dest_dir) = opts.dest_dir { builder = builder.dest_dir(PathBuf::from(dest_dir)); } @@ -6257,6 +6270,57 @@ pub unsafe extern "C" fn msb_snapshot_import_with_base( }) } +/// Import an archive with group selection without changing the existing import ABI. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn msb_snapshot_import_with_options( + cancel_id: u64, + archive: *const c_char, + opts_json: *const c_char, + buf: *mut c_uchar, + buf_len: usize, +) -> *mut c_char { + run_c(cancel_id, buf, buf_len, || { + let archive = PathBuf::from(unsafe { cstr(archive) }?); + let opts_raw = unsafe { cstr(opts_json) }?; + let opts: SnapshotLoadOptsJson = serde_json::from_str(&opts_raw) + .map_err(|error| FfiError::invalid_argument(error.to_string()))?; + Ok(Box::pin(async move { + let h = Snapshot::load_with_options( + &archive, + microsandbox::snapshot::LoadOpts { + dest: opts.dest, + base: opts.base, + group: opts.group, + set_head: opts.set_head, + }, + ) + .await + .map_err(FfiError::from)?; + Ok(snapshot_handle_json(&h).to_string()) + })) + }) +} + +/// Read a group head, or select a `group:member` as its head. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn msb_snapshot_group_head( + cancel_id: u64, + selector: *const c_char, + buf: *mut c_uchar, + buf_len: usize, +) -> *mut c_char { + run_c(cancel_id, buf, buf_len, || { + let selector = unsafe { cstr(selector) }?; + Ok(Box::pin(async move { + let update = Snapshot::group_head(&selector) + .await + .map_err(FfiError::from)?; + serde_json::to_string(&update) + .map_err(|error| FfiError::invalid_argument(error.to_string())) + })) + }) +} + // --------------------------------------------------------------------------- // Filesystem streaming — FsReadStream / FsWriteSink // --------------------------------------------------------------------------- diff --git a/sdk/go/snapshot.go b/sdk/go/snapshot.go index ebbe5a44f..baac361b1 100644 --- a/sdk/go/snapshot.go +++ b/sdk/go/snapshot.go @@ -14,13 +14,14 @@ type snapshotFactory struct{} // SnapshotCreateOptions configures Snapshot.Create. type SnapshotCreateOptions struct { - // Snapshot name, resolved under the default snapshots directory - // (or under DestDir when set). + // Snapshot member name; generated when empty. Name string + // Group to install the member in; defaults to the source sandbox's name. + Group string // Source sandbox to snapshot. Disk capture preserves running/paused state. Required. FromSandbox string // Parent directory to create the artifact in; empty = the default - // snapshots directory. The artifact lands at DestDir/. + // snapshots directory. The group is created under this root. DestDir string Labels map[string]string Force bool @@ -39,6 +40,27 @@ type SnapshotSaveOptions struct { PlainTar bool } +// SnapshotLoadOptions configures importing an archive into a snapshot group. +type SnapshotLoadOptions struct { + // Parent directory containing snapshot groups; empty selects the default. + Dest string + // Exact base snapshot or standalone archive for a dependent archive. + Base string + // Destination group; generated when empty. + Group string + // Select the imported member even when it is not a fast-forward. + SetHead bool +} + +// SnapshotHeadUpdate reports the result of reading or selecting a group head. +type SnapshotHeadUpdate struct { + Group string + Previous *string + Head string + Reason string + Changed bool +} + // SnapshotArchiveOptions configures direct sandbox-to-archive capture. type SnapshotArchiveOptions struct { SnapshotCreateOptions @@ -117,6 +139,7 @@ type SnapshotIntegrity struct { // SnapshotArtifact is a snapshot artifact on disk. type SnapshotArtifact struct { + headUpdate *SnapshotHeadUpdate id string path string digest string @@ -133,6 +156,7 @@ type SnapshotArtifact struct { func snapshotFromInfo(info *ffi.SnapshotInfo) *SnapshotArtifact { return &SnapshotArtifact{ + headUpdate: snapshotHeadUpdateFromInfo(info.HeadUpdate), id: info.ID, path: info.Path, digest: info.Digest, @@ -173,6 +197,11 @@ func (s *SnapshotArtifact) CreatedAt() string { return s.createdAt } func (s *SnapshotArtifact) Labels() map[string]string { return cloneMap(s.labels) } func (s *SnapshotArtifact) SourceSandbox() *string { return cloneStringPtr(s.sourceSandbox) } +// HeadUpdate returns the group head outcome recorded by this capture, if any. +func (s *SnapshotArtifact) HeadUpdate() *SnapshotHeadUpdate { + return cloneSnapshotHeadUpdate(s.headUpdate) +} + // Verify recomputes recorded content integrity for the snapshot. func (s *SnapshotArtifact) Verify(ctx context.Context) (*SnapshotVerifyReport, error) { report, err := ffi.SnapshotVerify(ctx, s.path) @@ -184,6 +213,8 @@ func (s *SnapshotArtifact) Verify(ctx context.Context) (*SnapshotVerifyReport, e // SnapshotHandle is a lightweight handle backed by the snapshot index. type SnapshotHandle struct { + group *string + headUpdate *SnapshotHeadUpdate id string digest string name *string @@ -205,6 +236,8 @@ type SnapshotHandle struct { func snapshotHandleFromInfo(info *ffi.SnapshotHandleInfo) *SnapshotHandle { return &SnapshotHandle{ + group: info.Group, + headUpdate: snapshotHeadUpdateFromInfo(info.HeadUpdate), id: info.ID, digest: info.Digest, name: info.Name, @@ -225,9 +258,17 @@ func snapshotHandleFromInfo(info *ffi.SnapshotHandleInfo) *SnapshotHandle { } } -func (h *SnapshotHandle) ID() string { return h.id } -func (h *SnapshotHandle) Digest() string { return h.digest } -func (h *SnapshotHandle) Name() *string { return cloneStringPtr(h.name) } +func (h *SnapshotHandle) ID() string { return h.id } +func (h *SnapshotHandle) Digest() string { return h.digest } +func (h *SnapshotHandle) Name() *string { return cloneStringPtr(h.name) } + +// Group returns the local group containing this indexed snapshot. +func (h *SnapshotHandle) Group() *string { return cloneStringPtr(h.group) } + +// HeadUpdate returns the group head outcome recorded by this import, if any. +func (h *SnapshotHandle) HeadUpdate() *SnapshotHeadUpdate { + return cloneSnapshotHeadUpdate(h.headUpdate) +} func (h *SnapshotHandle) ParentDigest() *string { return cloneStringPtr(h.parentDigest) } func (h *SnapshotHandle) Scope() string { return h.scope } func (h *SnapshotHandle) ImageRef() string { return h.imageRef } @@ -250,18 +291,17 @@ func (h *SnapshotHandle) Open(ctx context.Context) (*SnapshotArtifact, error) { } func (h *SnapshotHandle) Remove(ctx context.Context, force bool) error { - return Snapshot.Remove(ctx, h.digest, force) + // Copies in different groups share a digest; the handle owns one exact artifact path. + return Snapshot.Remove(ctx, h.path, force) } func (snapshotFactory) Create(ctx context.Context, opts SnapshotCreateOptions) (*SnapshotArtifact, error) { - if opts.Name == "" { - return nil, &Error{Kind: ErrInvalidConfig, Message: "snapshot create requires a non-empty Name"} - } if opts.FromSandbox == "" { return nil, &Error{Kind: ErrInvalidConfig, Message: "snapshot create requires a source sandbox (FromSandbox)"} } info, err := ffi.SnapshotCreate(ctx, opts.FromSandbox, ffi.SnapshotCreateOptions{ Name: opts.Name, + Group: opts.Group, DestDir: opts.DestDir, Labels: opts.Labels, Force: opts.Force, @@ -277,9 +317,6 @@ func (snapshotFactory) Create(ctx context.Context, opts SnapshotCreateOptions) ( // CreateArchive captures a disk or full snapshot directly into one archive file. // It does not create an installed snapshot directory or index row. func (snapshotFactory) CreateArchive(ctx context.Context, opts SnapshotArchiveOptions) (*SnapshotArchive, error) { - if opts.Name == "" { - return nil, &Error{Kind: ErrInvalidConfig, Message: "snapshot archive create requires a non-empty Name"} - } if opts.FromSandbox == "" { return nil, &Error{Kind: ErrInvalidConfig, Message: "snapshot archive create requires a source sandbox (FromSandbox)"} } @@ -290,6 +327,7 @@ func (snapshotFactory) CreateArchive(ctx context.Context, opts SnapshotArchiveOp create.DestDir = "" info, err := ffi.SnapshotCreateArchive(ctx, opts.FromSandbox, opts.ArchivePath, ffi.SnapshotCreateOptions{ Name: create.Name, + Group: create.Group, Labels: create.Labels, Force: create.Force, RecordIntegrity: create.RecordIntegrity, @@ -381,6 +419,51 @@ func (snapshotFactory) LoadWithBase(ctx context.Context, archive, dest, base str return snapshotHandleFromInfo(info), nil } +// LoadWithOptions imports an archive into a selected or generated group. +func (snapshotFactory) LoadWithOptions(ctx context.Context, archive string, opts SnapshotLoadOptions) (*SnapshotHandle, error) { + info, err := ffi.SnapshotLoadWithOptions(ctx, archive, ffi.SnapshotLoadOptions{ + Dest: opts.Dest, + Base: opts.Base, + Group: opts.Group, + SetHead: opts.SetHead, + }) + if err != nil { + return nil, wrapFFI(err) + } + return snapshotHandleFromInfo(info), nil +} + +// GroupHead reads a group head, or selects a group:member as its head. +func (snapshotFactory) GroupHead(ctx context.Context, selector string) (*SnapshotHeadUpdate, error) { + update, err := ffi.SnapshotGroupHead(ctx, selector) + if err != nil { + return nil, wrapFFI(err) + } + return snapshotHeadUpdateFromInfo(update), nil +} + +func snapshotHeadUpdateFromInfo(update *ffi.SnapshotHeadUpdate) *SnapshotHeadUpdate { + if update == nil { + return nil + } + return &SnapshotHeadUpdate{ + Group: update.Group, + Previous: update.Previous, + Head: update.Head, + Reason: update.Reason, + Changed: update.Changed, + } +} + +func cloneSnapshotHeadUpdate(update *SnapshotHeadUpdate) *SnapshotHeadUpdate { + if update == nil { + return nil + } + copy := *update + copy.Previous = cloneStringPtr(update.Previous) + return © +} + func normalizeSnapshotScope(scope string) string { if scope == "" { return SnapshotScopeDisk diff --git a/sdk/go/snapshot_test.go b/sdk/go/snapshot_test.go index 3a89b195c..88d67d554 100644 --- a/sdk/go/snapshot_test.go +++ b/sdk/go/snapshot_test.go @@ -9,16 +9,6 @@ import ( "github.com/superradcompany/microsandbox/sdk/go/internal/ffi" ) -func TestSnapshotCreateEmptyName(t *testing.T) { - _, err := Snapshot.Create(context.Background(), SnapshotCreateOptions{FromSandbox: "baseline"}) - if !IsKind(err, ErrInvalidConfig) { - t.Fatalf("err = %v, want ErrInvalidConfig", err) - } - if !strings.Contains(err.Error(), "Name") { - t.Fatalf("error should name the missing field: %q", err.Error()) - } -} - func TestSnapshotCreateEmptyFromSandbox(t *testing.T) { _, err := Snapshot.Create(context.Background(), SnapshotCreateOptions{Name: "after-pip-install"}) if !IsKind(err, ErrInvalidConfig) { @@ -68,6 +58,42 @@ func TestFFIWireShape_SnapshotCreateDestDir(t *testing.T) { } } +func TestFFIWireShape_SnapshotGroupWithGeneratedName(t *testing.T) { + got := marshalSnapshotCreateOptions(t, ffi.SnapshotCreateOptions{Group: "work"}) + if got["group"] != "work" { + t.Fatalf("group = %v, want work", got["group"]) + } + if _, present := got["name"]; present { + t.Fatal("generated names must be omitted for the Rust builder to assign") + } +} + +func TestFFIWireShape_SnapshotLoadGroupOptions(t *testing.T) { + payload, err := json.Marshal(ffi.SnapshotLoadOptions{ + Dest: "/snapshots", Base: "work:baseline", Group: "work", SetHead: true, + }) + if err != nil { + t.Fatal(err) + } + var got map[string]any + if err := json.Unmarshal(payload, &got); err != nil { + t.Fatal(err) + } + if got["dest"] != "/snapshots" || got["base"] != "work:baseline" || got["group"] != "work" || got["set_head"] != true { + t.Fatalf("unexpected load options: %s", payload) + } +} + +func TestFFIWireShape_SnapshotHeadUpdate(t *testing.T) { + var update ffi.SnapshotHeadUpdate + if err := json.Unmarshal([]byte(`{"group":"work","previous":null,"head":"baseline","reason":"initialized","changed":true}`), &update); err != nil { + t.Fatal(err) + } + if update.Group != "work" || update.Previous != nil || update.Head != "baseline" || update.Reason != "initialized" || !update.Changed { + t.Fatalf("unexpected head update: %#v", update) + } +} + func TestSnapshotStateProjectionDistinguishesMissingAndMerkleIntegrity(t *testing.T) { format := "raw" fstype := "ext4" diff --git a/sdk/node-ts/native/index.d.ts b/sdk/node-ts/native/index.d.ts index 43bcf450f..81d12fb47 100644 --- a/sdk/node-ts/native/index.d.ts +++ b/sdk/node-ts/native/index.d.ts @@ -1509,7 +1509,12 @@ export declare class Snapshot { */ static save(nameOrPath: string, out: string, opts?: SaveOpts | undefined | null): Promise static load(archive: string, dest?: string | undefined | null, base?: string | undefined | null): Promise + static loadWithOptions(archive: string, opts?: LoadOpts | undefined | null): Promise + /** Read a group's head, or select `group:member` as its head. */ + static groupHead(selector: string): Promise get path(): string + /** Outcome of the group head update performed by this capture. */ + get headUpdate(): HeadUpdate | null get id(): string get digest(): string get sizeBytes(): bigint | null @@ -1550,14 +1555,16 @@ export declare class SnapshotBuilder { constructor(name: string) /** * Create the artifact under this parent directory instead of the - * default snapshots store. The artifact lands at `destDir/`. + * default snapshots store. The snapshot group is created under this root. */ destDir(destDir: string): this + /** Install the snapshot in this group (defaults to the source sandbox's name). */ + group(group: string): this /** Set the source sandbox to snapshot. Required. */ fromSandbox(sourceSandbox: string): this /** Attach a key-value label. May be called multiple times. */ label(key: string, value: string): this - /** Overwrite an existing artifact at the destination. */ + /** Overwrite an archive destination; installed group members are immutable. */ force(): this /** Compute and record content integrity at create time. */ recordIntegrity(): this @@ -1582,6 +1589,8 @@ export type JsSnapshotBuilder = SnapshotBuilder /** Lightweight snapshot handle from the local index. */ export declare class SnapshotHandle { + get group(): string | null + get headUpdate(): HeadUpdate | null get id(): string get digest(): string get name(): string | null @@ -1849,6 +1858,15 @@ export interface FsMetadata { created?: number } +/** Outcome of reading or selecting a snapshot group's head. */ +export interface HeadUpdate { + group: string + previous?: string + head: string + reason: string + changed: boolean +} + /** OCI config fields extracted from the database. */ export interface ImageConfigDetail { digest: string @@ -1958,6 +1976,18 @@ export interface JsSandboxPage { nextCursor?: string } +/** Options for importing an archive into a snapshot group. */ +export interface LoadOpts { + /** Parent directory containing snapshot groups. */ + dest?: string + /** Exact base snapshot or standalone archive for a dependent archive. */ + base?: string + /** Destination group (generated when omitted). */ + group?: string + /** Select the imported member even when it is not a fast-forward. */ + setHead?: boolean +} + /** One captured log entry from `exec.log`. */ export interface LogEntry { /** Wall-clock timestamp when the chunk was captured (ms since epoch). */ @@ -2360,6 +2390,7 @@ export declare function setRuntimeMsbPath(path: string): void /** Built snapshot configuration produced by `SnapshotBuilder.build()`. */ export interface SnapshotConfig { name: string + group?: string sourceSandbox?: string destDir?: string labels: Array @@ -2373,6 +2404,8 @@ export interface SnapshotInfo { id: string digest: string name?: string + group?: string + headUpdate?: HeadUpdate parentDigest?: string imageRef: string /** `"disk"` for file state or `"full"` for a complete VM checkpoint. */ diff --git a/sdk/node-ts/native/snapshot.rs b/sdk/node-ts/native/snapshot.rs index 383658413..55c0749b2 100644 --- a/sdk/node-ts/native/snapshot.rs +++ b/sdk/node-ts/native/snapshot.rs @@ -1,7 +1,9 @@ use std::collections::HashMap; use std::path::PathBuf; -use microsandbox::snapshot::SaveOpts as RustSaveOpts; +use microsandbox::snapshot::{ + HeadUpdateReason, LoadOpts as RustLoadOpts, SaveOpts as RustSaveOpts, +}; use microsandbox::{ Snapshot as RustSnapshot, SnapshotArchive as RustSnapshotArchive, SnapshotFormat as RustSnapshotFormat, SnapshotHandle as RustSnapshotHandle, @@ -51,6 +53,30 @@ pub struct JsSaveOpts { pub last_layers: Option, } +/// Options for importing an archive into a snapshot group. +#[derive(Default)] +#[napi(object, js_name = "LoadOpts")] +pub struct JsLoadOpts { + /// Parent directory containing snapshot groups. + pub dest: Option, + /// Exact base snapshot or standalone archive for a dependent archive. + pub base: Option, + /// Destination group (generated when omitted). + pub group: Option, + /// Select the imported member even when it is not a fast-forward. + pub set_head: Option, +} + +/// Outcome of reading or selecting a snapshot group's head. +#[napi(object, js_name = "HeadUpdate")] +pub struct JsHeadUpdate { + pub group: String, + pub previous: Option, + pub head: String, + pub reason: String, + pub changed: bool, +} + /// Result of `Snapshot.verify()`. /// /// `upperKind` is `"notRecorded"` when integrity is absent or `"verified"` @@ -79,6 +105,8 @@ pub struct JsSnapshotInfo { pub id: String, pub digest: String, pub name: Option, + pub group: Option, + pub head_update: Option, pub parent_digest: Option, pub image_ref: String, /// `"disk"` for file state or `"full"` for a complete VM checkpoint. @@ -199,6 +227,35 @@ impl JsSnapshot { Ok(JsSnapshotHandle::from_rust(h)) } + #[napi(js_name = "loadWithOptions")] + pub async fn load_with_options( + archive: String, + opts: Option, + ) -> Result { + let opts = opts.unwrap_or_default(); + let h = RustSnapshot::load_with_options( + &PathBuf::from(archive), + RustLoadOpts { + dest: opts.dest.map(PathBuf::from), + base: opts.base, + group: opts.group, + set_head: opts.set_head.unwrap_or(false), + }, + ) + .await + .map_err(to_napi_error)?; + Ok(JsSnapshotHandle::from_rust(h)) + } + + /// Read a group's head, or select `group:member` as its head. + #[napi(js_name = "groupHead")] + pub async fn group_head(selector: String) -> Result { + let update = RustSnapshot::group_head(&selector) + .await + .map_err(to_napi_error)?; + Ok(head_update_to_js(&update)) + } + //---------------------------------------------------------------------------------------------- // Instance accessors (mirror PyVolume's getter style) //---------------------------------------------------------------------------------------------- @@ -208,6 +265,12 @@ impl JsSnapshot { self.inner.path().display().to_string() } + /// Outcome of the group head update performed by this capture. + #[napi(getter)] + pub fn head_update(&self) -> Option { + self.inner.head_update().map(head_update_to_js) + } + #[napi(getter)] pub fn id(&self) -> String { self.inner.id().to_string() @@ -410,6 +473,15 @@ impl JsSnapshot { #[napi] impl JsSnapshotHandle { + #[napi(getter)] + pub fn group(&self) -> Option { + self.inner.group().map(str::to_string) + } + + #[napi(getter)] + pub fn head_update(&self) -> Option { + self.inner.head_update().map(head_update_to_js) + } #[napi(getter)] pub fn id(&self) -> String { self.inner.id().to_string() @@ -537,6 +609,8 @@ fn snapshot_handle_to_info(h: &RustSnapshotHandle) -> JsSnapshotInfo { id: h.id().to_string(), digest: h.digest().to_string(), name: h.name().map(|s| s.to_string()), + group: h.group().map(str::to_string), + head_update: h.head_update().map(head_update_to_js), parent_digest: h.parent_digest().map(|s| s.to_string()), image_ref: h.image_ref().to_string(), scope: format_scope(h.scope()).into(), @@ -554,6 +628,25 @@ fn snapshot_handle_to_info(h: &RustSnapshotHandle) -> JsSnapshotInfo { } } +fn head_update_to_js(update: µsandbox::snapshot::HeadUpdate) -> JsHeadUpdate { + // Match the stable serde spelling without losing the closed reason variants. + let reason = match update.reason { + HeadUpdateReason::Initialized => "initialized", + HeadUpdateReason::FastForwarded => "fast_forwarded", + HeadUpdateReason::Selected => "selected", + HeadUpdateReason::Unchanged => "unchanged", + HeadUpdateReason::Diverged => "diverged", + HeadUpdateReason::UnknownAncestry => "unknown_ancestry", + }; + JsHeadUpdate { + group: update.group.clone(), + previous: update.previous.clone(), + head: update.head.clone(), + reason: reason.into(), + changed: update.changed, + } +} + fn verify_report_to_js( report: microsandbox::snapshot::SnapshotVerifyReport, ) -> JsSnapshotVerifyReport { diff --git a/sdk/node-ts/native/snapshot_builder.rs b/sdk/node-ts/native/snapshot_builder.rs index b2601cbc5..2805735cc 100644 --- a/sdk/node-ts/native/snapshot_builder.rs +++ b/sdk/node-ts/native/snapshot_builder.rs @@ -14,6 +14,7 @@ use crate::snapshot::{JsSnapshot, JsSnapshotArchive}; #[napi(object, js_name = "SnapshotConfig")] pub struct JsSnapshotConfig { pub name: String, + pub group: Option, pub source_sandbox: Option, pub dest_dir: Option, pub labels: Vec, @@ -35,6 +36,7 @@ pub struct JsSnapshotLabel { pub struct JsSnapshotBuilder { inner: Option, name: String, + group: Option, source_sandbox: Option, dest_dir: Option, labels: Vec<(String, String)>, @@ -54,6 +56,7 @@ impl JsSnapshotBuilder { Self { inner: Some(RustSnapshot::builder(&name)), name, + group: None, source_sandbox: None, dest_dir: None, labels: Vec::new(), @@ -64,7 +67,7 @@ impl JsSnapshotBuilder { } /// Create the artifact under this parent directory instead of the - /// default snapshots store. The artifact lands at `destDir/`. + /// default snapshots store. The snapshot group is created under this root. #[napi(js_name = "destDir")] pub fn dest_dir(&mut self, dest_dir: String) -> &Self { let prev = self.take_inner(); @@ -73,6 +76,15 @@ impl JsSnapshotBuilder { self } + /// Install the snapshot in this group (defaults to the source sandbox's name). + #[napi] + pub fn group(&mut self, group: String) -> &Self { + let prev = self.take_inner(); + self.inner = Some(prev.group(&group)); + self.group = Some(group); + self + } + /// Set the source sandbox to snapshot. Required. // `from_*` normally takes no self, but napi setters mutate in place and // the JS-facing name `fromSandbox` is the contract. @@ -94,7 +106,7 @@ impl JsSnapshotBuilder { self } - /// Overwrite an existing artifact at the destination. + /// Overwrite an archive destination; installed group members are immutable. #[napi] pub fn force(&mut self) -> &Self { let prev = self.take_inner(); @@ -126,6 +138,7 @@ impl JsSnapshotBuilder { pub fn build(&self) -> JsSnapshotConfig { JsSnapshotConfig { name: self.name.clone(), + group: self.group.clone(), source_sandbox: self.source_sandbox.clone(), dest_dir: self.dest_dir.clone(), labels: self diff --git a/sdk/node-ts/src/index.ts b/sdk/node-ts/src/index.ts index 51170aa6f..9f3fa0790 100644 --- a/sdk/node-ts/src/index.ts +++ b/sdk/node-ts/src/index.ts @@ -107,14 +107,16 @@ import { Snapshot as _Snapshot, type SnapshotBuilder as _SnapBT } from "./snapsh */ export const SnapshotBuilder = function SnapshotBuilder( this: unknown, - name: string, + name = "", ) { return _Snapshot.builder(name); -} as unknown as new (name: string) => _SnapBT; +} as unknown as new (name?: string) => _SnapBT; export type SnapshotBuilder = _SnapBT; export { SnapshotHandle } from "./snapshot-handle.js"; export type { SaveOpts, + LoadOpts, + HeadUpdate, SnapshotScope, SnapshotState, SnapshotVerifyReport, diff --git a/sdk/node-ts/src/internal/napi.ts b/sdk/node-ts/src/internal/napi.ts index d301c1941..94cbc2d6e 100644 --- a/sdk/node-ts/src/internal/napi.ts +++ b/sdk/node-ts/src/internal/napi.ts @@ -559,6 +559,23 @@ export interface NapiSnapshotStatic { reindex(dir?: string): Promise; save(name: string, out: string, opts?: NapiSaveOpts): Promise; load(archive: string, dest?: string, base?: string): Promise; + loadWithOptions(archive: string, opts?: NapiLoadOpts): Promise; + groupHead(selector: string): Promise; +} + +export interface NapiLoadOpts { + dest?: string; + base?: string; + group?: string; + setHead?: boolean; +} + +export interface NapiHeadUpdate { + readonly group: string; + readonly previous: string | null | undefined; + readonly head: string; + readonly reason: string; + readonly changed: boolean; } export type NapiSnapshotBuilderCtor = new (name: string) => NapiSnapshotBuilder; @@ -566,6 +583,7 @@ export type NapiSnapshotBuilderCtor = new (name: string) => NapiSnapshotBuilder; export interface NapiSnapshotBuilderSetters { fromSandbox(sourceSandbox: string): this; destDir(destDir: string): this; + group(group: string): this; label(key: string, value: string): this; force(): this; recordIntegrity(): this; @@ -586,6 +604,7 @@ export interface NapiSnapshotArchive { export interface NapiSnapshot { readonly id: string; readonly path: string; + readonly headUpdate: NapiHeadUpdate | null | undefined; readonly digest: string; readonly sizeBytes: bigint | null | undefined; readonly imageRef: string; @@ -612,6 +631,8 @@ export interface NapiSnapshotHandle { readonly id: string; readonly digest: string; readonly name: string | null | undefined; + readonly group: string | null | undefined; + readonly headUpdate: NapiHeadUpdate | null | undefined; readonly parentDigest: string | null | undefined; readonly scope: string; // "disk" | "full" readonly imageRef: string; @@ -634,6 +655,8 @@ export interface NapiSnapshotInfo { readonly id: string; readonly digest: string; readonly name: string | null | undefined; + readonly group: string | null | undefined; + readonly headUpdate: NapiHeadUpdate | null | undefined; readonly parentDigest: string | null | undefined; readonly scope: string; // "disk" | "full" readonly imageRef: string; diff --git a/sdk/node-ts/src/snapshot-handle.ts b/sdk/node-ts/src/snapshot-handle.ts index 1cff7c7c0..97ef2ceb4 100644 --- a/sdk/node-ts/src/snapshot-handle.ts +++ b/sdk/node-ts/src/snapshot-handle.ts @@ -3,7 +3,7 @@ import type { NapiSnapshotHandle, NapiSnapshotInfo, } from "./internal/napi.js"; -import { Snapshot, type SnapshotScope } from "./snapshot.js"; +import { Snapshot, type HeadUpdate, type SnapshotScope } from "./snapshot.js"; const READ_ONLY_MSG = "SnapshotHandle is read-only — fetch a live handle via Snapshot.get(name) for lifecycle methods."; @@ -23,6 +23,10 @@ export class SnapshotHandle { readonly digest: string; /** Convenience name. `null` for digest-only entries. */ readonly name: string | null; + /** Local group containing this indexed snapshot. */ + readonly group: string | null; + /** Outcome of the group head update performed by this import. */ + readonly headUpdate: HeadUpdate | null; /** Manifest digest of the parent snapshot, or `null` for a root. */ readonly parentDigest: string | null; /** Snapshot payload scope. */ @@ -58,6 +62,10 @@ export class SnapshotHandle { this.id = inner.id; this.digest = inner.digest; this.name = (inner.name ?? null) as string | null; + this.group = inner.group ?? null; + this.headUpdate = inner.headUpdate + ? { ...inner.headUpdate, previous: inner.headUpdate.previous ?? null } + : null; this.parentDigest = (inner.parentDigest ?? null) as string | null; this.scope = inner.scope as SnapshotScope; this.imageRef = inner.imageRef; diff --git a/sdk/node-ts/src/snapshot.ts b/sdk/node-ts/src/snapshot.ts index c5486f9ba..8ecdcaac0 100644 --- a/sdk/node-ts/src/snapshot.ts +++ b/sdk/node-ts/src/snapshot.ts @@ -64,6 +64,27 @@ export interface SaveOpts { plainTar?: boolean; } +/** Options for importing an archive into a snapshot group. */ +export interface LoadOpts { + /** Parent directory containing snapshot groups. */ + dest?: string; + /** Exact base snapshot or standalone archive for a dependent archive. */ + base?: string; + /** Destination group; generated when omitted. */ + group?: string; + /** Select the imported member even when it is not a fast-forward. */ + setHead?: boolean; +} + +/** Outcome of reading or selecting a snapshot group's head. */ +export interface HeadUpdate { + readonly group: string; + readonly previous: string | null; + readonly head: string; + readonly reason: string; + readonly changed: boolean; +} + /** Result of an explicit `Snapshot.verify()` call. */ export type SnapshotVerifyReport = | { @@ -133,23 +154,20 @@ export class Snapshot { } /** - * Begin building a snapshot named `name`, stored under the default - * snapshots directory. + * Begin building a snapshot member; an omitted name is generated. * * The source sandbox is required: * `Snapshot.builder("clean").fromSandbox("box").create()`. * - * Use `destDir(dir)` to create the artifact under a different parent - * directory instead; it lands at `destDir/`, and the name stays - * the snapshot's identity either way. + * Use `group(name)` to select a group and `destDir(dir)` to select its + * parent directory. The default group is the source sandbox's name. */ - static builder(name: string): SnapshotBuilder { + static builder(name = ""): SnapshotBuilder { return wrapBuilder(new napi.SnapshotBuilder(name)); } /** - * Open an existing snapshot artifact. Bare names resolve under the - * default snapshots directory; anything else is treated as a path. + * Open a snapshot by path, group head, or `group:member` selector. * * Cheap metadata validation only — does not read the upper file. * Use `verify()` for content checks. @@ -227,6 +245,18 @@ export class Snapshot { return new SnapshotHandle(raw); } + /** Import into a selected or generated group, with optional head selection. */ + static async loadWithOptions(archive: string, opts: LoadOpts = {}): Promise { + const raw = await withMappedErrors(() => napi.Snapshot.loadWithOptions(archive, opts)); + return new SnapshotHandle(raw); + } + + /** Read a group's head, or select `group:member` as its head. */ + static async groupHead(selector: string): Promise { + const update = await withMappedErrors(() => napi.Snapshot.groupHead(selector)); + return { ...update, previous: update.previous ?? null }; + } + //-------------------------------------------------------------------------- // Instance accessors //-------------------------------------------------------------------------- @@ -236,6 +266,12 @@ export class Snapshot { return this.inner.path; } + /** Outcome of the group head update performed by this capture. */ + get headUpdate(): HeadUpdate | null { + const update = this.inner.headUpdate; + return update ? { ...update, previous: update.previous ?? null } : null; + } + /** Canonical content digest (`sha256:hex`). The snapshot's identity. */ get id(): string { return this.inner.id; diff --git a/sdk/node-ts/tests/cow-lifecycle.test.ts b/sdk/node-ts/tests/cow-lifecycle.test.ts index 8cc3dc08a..4f0402fea 100644 --- a/sdk/node-ts/tests/cow-lifecycle.test.ts +++ b/sdk/node-ts/tests/cow-lifecycle.test.ts @@ -15,9 +15,9 @@ it.skipIf(process.env.MSB_COW_LIVE !== "1")("captures a resident pause and resto const branched = await paused.branch(`${name}-paused-branch`); branches.push(branched); expect((await branched.exec("cat", ["/dev/shm/sdk-marker"])).stdout().trim()).toBe("source"); - await Snapshot.builder(`${name}-full`).fromSandbox(name).full().create(); + const snapshot = await Snapshot.builder(`${name}-full`).fromSandbox(name).full().create(); await paused.resume(); - child = await Sandbox.builder(`${name}-child`).fromSnapshot(`${name}-full`).forked().create(); + child = await Sandbox.builder(`${name}-child`).fromSnapshot(snapshot.path).forked().create(); expect((await child.exec("cat", ["/dev/shm/sdk-marker"])).stdout().trim()).toBe("source"); await child.exec("sh", ["-c", "echo child > /dev/shm/sdk-marker"]); const descendant = await child.branch(`${name}-branch`); diff --git a/sdk/node-ts/tests/unit/native-contract.test.ts b/sdk/node-ts/tests/unit/native-contract.test.ts index a6ee8dc4b..8c8c298d5 100644 --- a/sdk/node-ts/tests/unit/native-contract.test.ts +++ b/sdk/node-ts/tests/unit/native-contract.test.ts @@ -68,6 +68,15 @@ describe("native image cache contract", () => { }); describe("native snapshot contract", () => { + it("exports group creation, import, and head selection", () => { + expect(typeof napi.Snapshot.loadWithOptions).toBe("function"); + expect(typeof napi.Snapshot.groupHead).toBe("function"); + expect(typeof napi.SnapshotBuilder.prototype.group).toBe("function"); + const builder = new napi.SnapshotBuilder("").fromSandbox("source").group("work"); + const config = (builder as unknown as { build(): { name: string; group: string } }).build(); + expect(config.name).toBe(""); + expect(config.group).toBe("work"); + }); it("exports the direct archive result used by the TS wrapper", () => { expect(typeof napi.SnapshotArchive).toBe("function"); }); diff --git a/sdk/node-ts/tests/unit/snapshot.test.ts b/sdk/node-ts/tests/unit/snapshot.test.ts index 2d2d5cfef..a0c4ab1bf 100644 --- a/sdk/node-ts/tests/unit/snapshot.test.ts +++ b/sdk/node-ts/tests/unit/snapshot.test.ts @@ -1,5 +1,10 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { Snapshot } from "../../dist/snapshot.js"; +import { napi } from "../../dist/internal/napi.js"; + +vi.mock("../../dist/internal/napi.js", () => ({ + napi: { Snapshot: { loadWithOptions: vi.fn(), groupHead: vi.fn() } }, +})); function projectedSnapshot( overrides: Record = {}, @@ -39,6 +44,42 @@ function projectedSnapshot( } describe("Snapshot native projections", () => { + it("exposes the create outcome with a nullable previous head", () => { + const snapshot = projectedSnapshot({ + headUpdate: { + group: "work", previous: undefined, head: "snapshot-1", reason: "initialized", changed: true, + }, + }); + expect(snapshot.headUpdate).toEqual({ + group: "work", previous: null, head: "snapshot-1", reason: "initialized", changed: true, + }); + expect(projectedSnapshot().headUpdate).toBeNull(); + }); + + it("forwards import options and preserves a retained-head outcome", async () => { + const headUpdate = { + group: "work", previous: "snapshot-1", head: "snapshot-1", reason: "diverged", changed: false, + }; + vi.mocked(napi.Snapshot.loadWithOptions).mockResolvedValue({ + id: "snapshot-2", digest: "sha256:two", group: "work", headUpdate, + name: "other", createdAt: 0, path: "/snapshots/work/snapshot-2", + } as never); + const options = { dest: "/snapshots", base: "work:base", group: "work", setHead: false }; + const handle = await Snapshot.loadWithOptions("other.msnap", options); + expect(napi.Snapshot.loadWithOptions).toHaveBeenCalledWith("other.msnap", options); + expect(handle.group).toBe("work"); + expect(handle.id).toBe("snapshot-2"); + expect(handle.headUpdate).toEqual(headUpdate); + }); + + it("forwards a member selector for explicit head selection", async () => { + vi.mocked(napi.Snapshot.groupHead).mockResolvedValue({ + group: "work", previous: "snapshot-2", head: "snapshot-1", reason: "selected", changed: true, + }); + expect(await Snapshot.groupHead("work:baseline")).toMatchObject({ reason: "selected", changed: true }); + expect(napi.Snapshot.groupHead).toHaveBeenCalledWith("work:baseline"); + }); + it("returns complete file and checkpoint states", () => { expect(projectedSnapshot().state).toMatchObject({ kind: "file", diff --git a/sdk/python/integration/test_snapshots.py b/sdk/python/integration/test_snapshots.py index 285d4cb64..eba4eef98 100644 --- a/sdk/python/integration/test_snapshots.py +++ b/sdk/python/integration/test_snapshots.py @@ -24,7 +24,8 @@ async def test_snapshot_create_open_list_and_boot(sandbox_name): await remove_sandbox(fork_name) await remove_sandbox(base_name) - await remove_snapshot(snapshot_name) + snapshot_selector = f"{base_name}:{snapshot_name}" + await remove_snapshot(snapshot_selector) base = await Sandbox.create(base_name, image=IMAGE, cpus=1, memory=512, replace=True) fork = None @@ -44,11 +45,18 @@ async def test_snapshot_create_open_list_and_boot(sandbox_name): verify_result = await snapshot.verify() assert isinstance(verify_result, dict) - handle = await Snapshot.get(snapshot_name) + handle = await Snapshot.get(snapshot_selector) assert handle.digest == snapshot.digest assert handle.state_kind is SnapshotStateKind.FILE assert handle.format is SnapshotFormat.RAW assert handle.scope is SnapshotScope.DISK + assert handle.group == base_name + assert snapshot.head_update["reason"] == "initialized" + head = await Snapshot.group_head(base_name) + assert head["head"] == snapshot.id + assert head["changed"] is False + selected = await Snapshot.group_head(snapshot_selector) + assert selected["head"] == snapshot.id opened = await handle.open() assert opened.digest == snapshot.digest assert opened.state_kind is SnapshotStateKind.FILE @@ -58,7 +66,7 @@ async def test_snapshot_create_open_list_and_boot(sandbox_name): fork = await Sandbox.create( fork_name, - from_snapshot=snapshot_name, + from_snapshot=snapshot_selector, cpus=1, memory=512, replace=True, @@ -72,4 +80,4 @@ async def test_snapshot_create_open_list_and_boot(sandbox_name): await fork.stop() await remove_sandbox(fork_name) await remove_sandbox(base_name) - await remove_snapshot(snapshot_name) + await remove_snapshot(snapshot_selector) diff --git a/sdk/python/microsandbox/_microsandbox.pyi b/sdk/python/microsandbox/_microsandbox.pyi index 1fcd0a1c2..7666937a8 100644 --- a/sdk/python/microsandbox/_microsandbox.pyi +++ b/sdk/python/microsandbox/_microsandbox.pyi @@ -829,9 +829,10 @@ class ImagePruneReport: class Snapshot: @staticmethod async def create( - name: str, + name: str = "", *, from_sandbox: str, + group: str | None = None, dest_dir: str | os.PathLike[str] | None = None, labels: dict[str, str] | None = None, force: bool = False, @@ -844,6 +845,7 @@ class Snapshot: archive: str | os.PathLike[str], *, from_sandbox: str, + group: str | None = None, labels: dict[str, str] | None = None, force: bool = False, record_integrity: bool = False, @@ -879,7 +881,13 @@ class Snapshot: *, dest: str | os.PathLike[str] | None = None, base: str | None = None, + group: str | None = None, + set_head: bool = False, ) -> SnapshotHandle: ... + @staticmethod + async def group_head(selector: str) -> dict[str, str | bool | None]: ... + @property + def head_update(self) -> dict[str, str | bool | None] | None: ... @property def id(self) -> str: ... @property @@ -923,6 +931,10 @@ class SnapshotArchive: def path(self) -> str: ... class SnapshotHandle: + @property + def group(self) -> str | None: ... + @property + def head_update(self) -> dict[str, str | bool | None] | None: ... @property def id(self) -> str: ... @property diff --git a/sdk/python/src/snapshot.rs b/sdk/python/src/snapshot.rs index 560dc4e8d..4fa96a8d1 100644 --- a/sdk/python/src/snapshot.rs +++ b/sdk/python/src/snapshot.rs @@ -4,7 +4,7 @@ use std::path::PathBuf; use pyo3::prelude::*; use pyo3::types::PyDict; -use microsandbox::snapshot::SaveOpts as RustSaveOpts; +use microsandbox::snapshot::{LoadOpts as RustLoadOpts, SaveOpts as RustSaveOpts}; use microsandbox::{ Snapshot as RustSnapshot, SnapshotArchive as RustSnapshotArchive, SnapshotFormat as RustSnapshotFormat, SnapshotHandle as RustSnapshotHandle, @@ -44,15 +44,17 @@ pub struct PySnapshotHandle { impl PySnapshot { /// Create a disk snapshot, or include memory and execution state with full=True. /// - /// The artifact is created under `~/.microsandbox/snapshots//`, - /// or under `dest_dir=` when given; move artifacts with `save`/`load`. + /// The artifact is installed in a snapshot group under the default snapshots + /// directory or `dest_dir`. Omitted member names are generated; the group + /// defaults to the source sandbox's name. // PyO3 kwargs map one-to-one onto function parameters; the count is the contract. #[allow(clippy::too_many_arguments)] #[staticmethod] #[pyo3(signature = ( - name, + name = "".to_string(), *, from_sandbox, + group = None, dest_dir = None, labels = None, force = false, @@ -63,6 +65,7 @@ impl PySnapshot { py: Python<'py>, name: String, from_sandbox: String, + group: Option, dest_dir: Option, labels: Option>, force: bool, @@ -71,6 +74,9 @@ impl PySnapshot { ) -> PyResult> { pyo3_async_runtimes::tokio::future_into_py(py, async move { let mut builder = RustSnapshot::builder(name).from_sandbox(&from_sandbox); + if let Some(group) = group { + builder = builder.group(group); + } if let Some(dest_dir) = dest_dir { builder = builder.dest_dir(dest_dir); } @@ -101,6 +107,7 @@ impl PySnapshot { archive, *, from_sandbox, + group = None, labels = None, force = false, record_integrity = false, @@ -112,6 +119,7 @@ impl PySnapshot { name: String, archive: PathBuf, from_sandbox: String, + group: Option, labels: Option>, force: bool, record_integrity: bool, @@ -120,6 +128,9 @@ impl PySnapshot { ) -> PyResult> { pyo3_async_runtimes::tokio::future_into_py(py, async move { let mut builder = RustSnapshot::builder(name).from_sandbox(from_sandbox); + if let Some(group) = group { + builder = builder.group(group); + } if let Some(labels) = labels { for (key, value) in labels { builder = builder.label(key, value); @@ -142,7 +153,7 @@ impl PySnapshot { }) } - /// Open an existing snapshot artifact by path or bare name. + /// Open a snapshot by path, group head, or `group:member` selector. /// Cheap metadata validation only — does not read the upper file. #[staticmethod] fn open<'py>(py: Python<'py>, path_or_name: String) -> PyResult> { @@ -273,24 +284,42 @@ impl PySnapshot { /// snapshots directory, preserving recorded integrity for explicit /// verification. #[staticmethod] - #[pyo3(signature = (archive, *, dest = None, base = None))] + #[pyo3(signature = (archive, *, dest = None, base = None, group = None, set_head = false))] fn load<'py>( py: Python<'py>, archive: PathBuf, dest: Option, base: Option, + group: Option, + set_head: bool, ) -> PyResult> { pyo3_async_runtimes::tokio::future_into_py(py, async move { - let h = if let Some(base) = base { - RustSnapshot::load_with_base(&archive, dest.as_deref(), &base).await - } else { - RustSnapshot::load(&archive, dest.as_deref()).await - } + let h = RustSnapshot::load_with_options( + &archive, + RustLoadOpts { + dest, + base, + group, + set_head, + }, + ) + .await .map_err(to_py_err)?; Ok(PySnapshotHandle::from_rust(h)) }) } + /// Read a group's head, or select `group:member` as its head. + #[staticmethod] + fn group_head<'py>(py: Python<'py>, selector: String) -> PyResult> { + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let update = RustSnapshot::group_head(&selector) + .await + .map_err(to_py_err)?; + Python::with_gil(|py| head_update_to_py(py, &update)) + }) + } + //---------------------------------------------------------------------------------------------- // Instance accessors //---------------------------------------------------------------------------------------------- @@ -301,6 +330,15 @@ impl PySnapshot { self.inner.path().display().to_string() } + /// Outcome of the group head update performed by this capture. + #[getter] + fn head_update(&self, py: Python<'_>) -> PyResult>> { + self.inner + .head_update() + .map(|update| head_update_to_py(py, update)) + .transpose() + } + /// Canonical content digest (`sha256:hex`). The snapshot's identity. #[getter] fn id(&self) -> &str { @@ -493,6 +531,20 @@ impl PySnapshot { #[pymethods] impl PySnapshotHandle { + /// Local group containing this indexed snapshot. + #[getter] + fn group(&self) -> Option<&str> { + self.inner.group() + } + + /// Outcome of the group head update performed by this import. + #[getter] + fn head_update(&self, py: Python<'_>) -> PyResult>> { + self.inner + .head_update() + .map(|update| head_update_to_py(py, update)) + .transpose() + } #[getter] fn id(&self) -> &str { self.inner.id() @@ -612,6 +664,25 @@ impl PySnapshotHandle { // Functions: Helpers //-------------------------------------------------------------------------------------------------- +fn head_update_to_py( + py: Python<'_>, + update: µsandbox::snapshot::HeadUpdate, +) -> PyResult> { + let result = PyDict::new(py); + result.set_item("group", &update.group)?; + result.set_item("previous", &update.previous)?; + result.set_item("head", &update.head)?; + // Preserve the stable reason spelling used by all serialized API surfaces. + let reason = serde_json::to_value(update.reason) + .map_err(|error| pyo3::exceptions::PyRuntimeError::new_err(error.to_string()))?; + let reason = reason.as_str().ok_or_else(|| { + pyo3::exceptions::PyRuntimeError::new_err("snapshot head reason is not a string") + })?; + result.set_item("reason", reason)?; + result.set_item("changed", update.changed)?; + Ok(result.unbind()) +} + fn format_str(f: RustSnapshotFormat) -> &'static str { match f { RustSnapshotFormat::Raw => "raw", diff --git a/sdk/rust/lib/backend/local/sandbox/create.rs b/sdk/rust/lib/backend/local/sandbox/create.rs index 479d1c595..81250e65e 100644 --- a/sdk/rust/lib/backend/local/sandbox/create.rs +++ b/sdk/rust/lib/backend/local/sandbox/create.rs @@ -201,6 +201,7 @@ impl LocalBackend { )) .await?; config.spec.image = RootfsSource::oci(materialized.manifest.image.reference.clone()); + config.snapshot_parent = Some(materialized.manifest.snapshot_id.to_string()); config.manifest_digest = Some(materialized.manifest.image.manifest_digest.clone()); crate::sandbox::apply_snapshot_root_layout( &mut config, diff --git a/sdk/rust/lib/lib.rs b/sdk/rust/lib/lib.rs index 6415ea426..3537f3b43 100644 --- a/sdk/rust/lib/lib.rs +++ b/sdk/rust/lib/lib.rs @@ -76,9 +76,9 @@ pub use sandbox::{ SecretEntryConfigPatch, SecretInjection, TlsConfigPatch, }; pub use snapshot::{ - CheckpointSnapshotState, FileSnapshotState, SaveOpts, Snapshot, SnapshotArchive, - SnapshotBuilder, SnapshotConfig, SnapshotDescriptor, SnapshotFormat, SnapshotHandle, - SnapshotRootDisk, SnapshotScope, SnapshotSpec, SnapshotState, SnapshotVerifyReport, - UpperIntegrity, UpperVerifyStatus, + CheckpointSnapshotState, FileSnapshotState, HeadUpdate, HeadUpdateReason, LoadOpts, SaveOpts, + Snapshot, SnapshotArchive, SnapshotBuilder, SnapshotConfig, SnapshotDescriptor, SnapshotFormat, + SnapshotHandle, SnapshotRootDisk, SnapshotScope, SnapshotSpec, SnapshotState, + SnapshotVerifyReport, UpperIntegrity, UpperVerifyStatus, }; pub use volume::{Volume, VolumeConfig, VolumeHandle, VolumeKind, VolumeSpec}; diff --git a/sdk/rust/lib/sandbox/branch.rs b/sdk/rust/lib/sandbox/branch.rs index 5f8e629e0..679e231f8 100644 --- a/sdk/rust/lib/sandbox/branch.rs +++ b/sdk/rust/lib/sandbox/branch.rs @@ -90,6 +90,9 @@ pub(crate) async fn capture_child( source: &str, child: &Path, ) -> MicrosandboxResult { + // Serialize with durable source captures so a child's ancestry describes its actual cut. + let lineage = crate::snapshot::lineage::begin(local, source).await?; + config.snapshot_parent = lineage.parent.as_ref().map(ToString::to_string); let id = format!("branch_{:032x}", rand::random::()); // Acquired before publication: source exit or another capture cannot create an unpinned // eviction window before this caller opens the completed memory file. @@ -109,6 +112,7 @@ pub(crate) async fn capture_child( format!("{}\n", serde_json::to_string(&request)?), ) .await?; + lineage.validate_source(local, source).await?; let closure = child.join(".branch-restore"); if response.branch.as_ref() != Some(&closure) { return Err(MicrosandboxError::Runtime( diff --git a/sdk/rust/lib/sandbox/builder.rs b/sdk/rust/lib/sandbox/builder.rs index cca630ac1..7ec661db3 100644 --- a/sdk/rust/lib/sandbox/builder.rs +++ b/sdk/rust/lib/sandbox/builder.rs @@ -1254,6 +1254,7 @@ impl SandboxBuilder { } let snap = crate::snapshot::Snapshot::open(&snapshot_ref).await?; + self.config.snapshot_parent = Some(snap.id().to_string()); let unsupported = snap.manifest().unsupported_requires(); if !unsupported.is_empty() { return Err(crate::MicrosandboxError::unsupported( diff --git a/sdk/rust/lib/sandbox/config.rs b/sdk/rust/lib/sandbox/config.rs index ef032f0a6..b81bcf79a 100644 --- a/sdk/rust/lib/sandbox/config.rs +++ b/sdk/rust/lib/sandbox/config.rs @@ -213,6 +213,10 @@ pub struct SandboxConfig { #[serde(skip)] pub(crate) snapshot_base: Option, + /// Snapshot from which this sandbox derives. Later captures retain their own local cursor. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) snapshot_parent: Option, + /// Child-owned checkpoint closure for an unfinished restore construction. /// /// The builder initially points this at an installed snapshot. The local create path copies @@ -819,6 +823,7 @@ impl Default for SandboxConfig { snapshot_root_layer_sources: Vec::new(), snapshot_root_virtual_size: None, snapshot_archive_source: None, + snapshot_parent: None, snapshot_base: None, checkpoint_restore: None, branch_source: None, diff --git a/sdk/rust/lib/sandbox/modify.rs b/sdk/rust/lib/sandbox/modify.rs index 95c002d51..d2d20d657 100644 --- a/sdk/rust/lib/sandbox/modify.rs +++ b/sdk/rust/lib/sandbox/modify.rs @@ -747,6 +747,21 @@ pub(super) async fn control_request_for( local: &crate::backend::LocalBackend, name: &str, request: String, +) -> MicrosandboxResult { + let response = control_request_raw_for(local, name, request).await?; + if !response.ok { + return Err(crate::MicrosandboxError::Runtime(format!( + "runtime control refused: {}", + response.error.unwrap_or_else(|| "unknown error".into()) + ))); + } + Ok(response) +} + +async fn control_request_raw_for( + local: &crate::backend::LocalBackend, + name: &str, + request: String, ) -> MicrosandboxResult { let candidates = crate::runtime::sandbox_agent_socket_path_candidates_for(local, name) .into_iter() @@ -759,14 +774,7 @@ pub(super) async fn control_request_for( crate::MicrosandboxError::Runtime("no backend control endpoint".into()) })?) .await?; - let response = control_request_over_stream(stream, &request).await?; - if !response.ok { - return Err(crate::MicrosandboxError::Runtime(format!( - "runtime control refused: {}", - response.error.unwrap_or_else(|| "unknown error".into()) - ))); - } - Ok(response) + control_request_over_stream(stream, &request).await } async fn control_request_raw( @@ -867,12 +875,17 @@ pub(crate) async fn control_disk_compact( /// can finish publishing the requested snapshot. The runtime diagnostic is logged and the source /// remains visibly non-running rather than losing the completed capture. pub(crate) async fn control_checkpoint_create( + local: &crate::backend::LocalBackend, name: &str, checkpoint_id: String, ) -> MicrosandboxResult { - let capabilities = control_capabilities(name).await?; - if !capabilities.checkpoint_create { - return Err(crate::MicrosandboxError::unsupported( + let capabilities = + control_request_for(local, name, "{\"op\":\"capabilities\"}\n".into()).await?; + if !capabilities + .capabilities + .is_some_and(|capabilities| capabilities.checkpoint_create) + { + return Err(MicrosandboxError::unsupported( Operation::SnapshotOps, UnsupportedReason::NotAvailable( "this running sandbox does not support full checkpoint capture".into(), @@ -883,9 +896,19 @@ pub(crate) async fn control_checkpoint_create( checkpoint_id, intent: microsandbox_runtime::control::CheckpointCaptureIntent::FullSnapshot, }; - let mut line = serde_json::to_string(&request)?; - line.push('\n'); - let response = control_request_raw(name, line).await?; + let response = control_request_raw_for( + local, + name, + format!("{}\n", serde_json::to_string(&request)?), + ) + .await?; + checkpoint_response(name, response) +} + +fn checkpoint_response( + name: &str, + response: microsandbox_runtime::control::ControlResponse, +) -> MicrosandboxResult { if let Some(checkpoint) = response.checkpoint { if !response.ok { tracing::warn!( @@ -2500,6 +2523,65 @@ mod tests { use crate::backend::LocalBackend; use crate::size::SizeExt; + #[cfg(unix)] + #[tokio::test] + async fn full_checkpoint_uses_selected_backend_and_retains_post_publish_failure() { + use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; + + let first_home = tempfile::tempdir_in("/tmp").unwrap(); + let second_home = tempfile::tempdir_in("/tmp").unwrap(); + let first = LocalBackend::builder() + .home(first_home.path()) + .build() + .await + .unwrap(); + let second = LocalBackend::builder() + .home(second_home.path()) + .build() + .await + .unwrap(); + let mut servers = Vec::new(); + for (local, label, resume_ok) in [(&first, "first", true), (&second, "second", false)] { + let agent = + crate::runtime::sandbox_agent_socket_path_candidates_for(local, "worker").remove(0); + let path = microsandbox_runtime::control::control_socket_path_for(&agent); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + let listener = tokio::net::UnixListener::bind(path).unwrap(); + servers.push(tokio::spawn(async move { + for request_index in 0..2 { + let (stream, _) = listener.accept().await.unwrap(); + let mut stream = BufReader::new(stream); + let mut line = String::new(); + stream.read_line(&mut line).await.unwrap(); + let request: serde_json::Value = serde_json::from_str(&line).unwrap(); + let response = if request_index == 0 { + assert_eq!(request["op"], "capabilities"); + serde_json::json!({"ok":true,"capabilities":{"checkpoint_create":true,"cpu_resize":false,"memory_resize":false,"secrets_update":false}}) + } else { + assert_eq!(request["op"], "checkpoint_create"); + serde_json::json!({"ok":resume_ok,"error":"source resume failed","checkpoint":{ + "checkpoint_id":request["checkpoint_id"], "checkpoint_root":format!("sha256:{}", "a".repeat(64)), + "path":format!("/capture/{label}"), "memory_mode":"full", "memory_logical_bytes":4096, "memory_emitted_bytes":4096 + }}) + }; + stream.get_mut().write_all(format!("{response}\n").as_bytes()).await.unwrap(); + } + })); + } + let first_capture = control_checkpoint_create(&first, "worker", "first-checkpoint".into()) + .await + .unwrap(); + let second_capture = + control_checkpoint_create(&second, "worker", "second-checkpoint".into()) + .await + .unwrap(); + assert_eq!(first_capture.path, std::path::Path::new("/capture/first")); + assert_eq!(second_capture.path, std::path::Path::new("/capture/second")); + for server in servers { + server.await.unwrap(); + } + } + #[tokio::test] async fn size_setters_accept_bare_mib_and_typed_sizes() { let temp = tempdir().unwrap(); diff --git a/sdk/rust/lib/snapshot/archive.rs b/sdk/rust/lib/snapshot/archive.rs index ff88cb39e..2a53b4343 100644 --- a/sdk/rust/lib/snapshot/archive.rs +++ b/sdk/rust/lib/snapshot/archive.rs @@ -60,6 +60,19 @@ const GNU_EXT_SPARSE_SLOTS: usize = 21; // Types //-------------------------------------------------------------------------------------------------- +/// Options for installing an archive in a local snapshot group. +#[derive(Debug, Clone, Default)] +pub struct LoadOpts { + /// Group-store root; defaults to the configured snapshots directory. + pub dest: Option, + /// Explicit base selector for omitted disk layers and RAM objects. + pub base: Option, + /// Existing/new destination group, or a freshly generated group when omitted. + pub group: Option, + /// Select the imported target even when it is not a fast-forward. + pub set_head: bool, +} + /// Options for [`super::Snapshot::save`]. #[derive(Debug, Clone, Default)] pub struct SaveOpts { @@ -291,13 +304,25 @@ pub(super) async fn save_snapshot( let mut parents: Vec = Vec::new(); if opts.with_parents { - let mut current = head.manifest().parent.clone(); - while let Some(parent_id) = current { - let parent_path = resolve_parent_artifact(local, parent_id.as_str()).await?; + let mut current = head.clone(); + let mut visited = HashSet::from([head.id().to_string()]); + while let Some(parent_id) = current.manifest().parent.clone() { + if !visited.insert(parent_id.to_string()) { + return Err(MicrosandboxError::SnapshotIntegrity(format!( + "snapshot parent chain contains a cycle at {parent_id}" + ))); + } + let parent_path = resolve_parent_artifact(local, ¤t, parent_id.as_str()).await?; let parent = store::open_snapshot(local, parent_path.to_string_lossy().as_ref()).await?; + if parent.id() != &parent_id { + return Err(MicrosandboxError::SnapshotIntegrity(format!( + "snapshot parent path contains {}, expected {parent_id}", + parent.id() + ))); + } parents.push(parent.clone()); - current = parent.manifest().parent.clone(); + current = parent; } } parents.reverse(); @@ -921,12 +946,26 @@ pub(super) async fn load_snapshot_with_base( archive: &Path, dest: Option<&Path>, base: Option<&str>, +) -> MicrosandboxResult { + load_snapshot_with_options( + local, + archive, + LoadOpts { + dest: dest.map(Path::to_path_buf), + base: base.map(str::to_string), + ..Default::default() + }, + ) + .await +} + +pub(super) async fn load_snapshot_with_options( + local: &LocalBackend, + archive: &Path, + opts: LoadOpts, ) -> MicrosandboxResult { let total_started = Instant::now(); - let snapshots_dir = match dest { - Some(d) => d.to_path_buf(), - None => local.snapshots_dir(), - }; + let snapshots_dir = opts.dest.clone().unwrap_or_else(|| local.snapshots_dir()); tokio::fs::create_dir_all(&snapshots_dir).await?; let cache_dir = local.cache_dir(); tokio::fs::create_dir_all(&cache_dir).await?; @@ -979,7 +1018,7 @@ pub(super) async fn load_snapshot_with_base( inventory, snapshot_stage.path(), cache_stage.path(), - base, + opts.base.as_deref(), ) .await?; materialize_inventory_layers(inventory, snapshot_stage.path()).await?; @@ -991,6 +1030,12 @@ pub(super) async fn load_snapshot_with_base( if let Some(inventory) = unpacked.inventory.as_ref() { validate_inventory_snapshot_bindings(inventory, &imported)?; } + // Released flat descriptors are translated by open_snapshot in memory. Install that + // admitted representation only in our owned staging, after checking the original archive + // bindings; group metadata must never point at a descriptor its reader cannot reopen. + for snapshot in &imported { + normalize_imported_descriptor(snapshot).await?; + } let head_index = match unpacked.head.as_deref() { Some(head) => imported .iter() @@ -1000,17 +1045,10 @@ pub(super) async fn load_snapshot_with_base( })?, None => select_head_snapshot(&imported)?, }; - let head_stage_path = imported[head_index].path().to_path_buf(); - let head_relative = head_stage_path - .strip_prefix(snapshot_stage.path()) - .map_err(|_| MicrosandboxError::Custom("imported snapshot escaped staging dir".into()))? - .to_path_buf(); let head_manifest = imported[head_index].manifest().clone(); - let head_path = snapshots_dir.join(&head_relative); let validate_us = validate_started.elapsed().as_micros(); let promote_started = Instant::now(); - ensure_promote_targets_available(snapshot_stage.path(), &snapshots_dir).await?; // Cache installation carries hashing buffers across await points. Keep // that future on the heap so the archive loader remains within Windows' // smaller default worker-thread stack. @@ -1020,12 +1058,38 @@ pub(super) async fn load_snapshot_with_base( &head_manifest, )) .await?; - promote_stage(snapshot_stage.path(), &snapshots_dir).await?; + let group_dir = super::group::ensure(&snapshots_dir, opts.group.as_deref()).await?; + let mut aliases = BTreeMap::new(); + if let Some(inventory) = unpacked.inventory.as_ref() { + if let Some(names) = inventory.extensions.get("msb-snapshot-member-names") { + aliases = serde_json::from_value(names.clone())?; + } + // This older field is only a suggestion. Released archives can carry names that + // predate group alias restrictions; ignore those while keeping explicit aliases strict. + if let Some(name) = inventory + .suggested_name + .as_ref() + .filter(|name| super::group::validate_alias(name).is_ok()) + { + aliases + .entry(head_manifest.snapshot_id.to_string()) + .or_insert_with(|| name.clone()); + } + } + let update = super::group::publish( + &group_dir, + snapshot_stage.path(), + &aliases, + &head_manifest.snapshot_id, + opts.set_head, + ) + .await?; + let head_path = group_dir.join(head_manifest.snapshot_id.as_str()); let snap = store::open_snapshot(local, head_path.to_string_lossy().as_ref()).await?; // Index this and any sibling artifacts that landed in the dest dir. - let _ = store::reindex_dir(local, &snapshots_dir).await; + let _ = store::reindex_dir(local, &group_dir).await; let promote_index_us = promote_started.elapsed().as_micros(); let (state_kind, format, fstype, checkpoint_manifest_digest, size_bytes) = @@ -1046,13 +1110,11 @@ pub(super) async fn load_snapshot_with_base( ), }; let handle = SnapshotHandle { + group: Some(update.group.clone()), + head_update: Some(update), snapshot_id: snap.id().to_string(), digest: snap.digest().to_string(), - name: snap - .path() - .file_name() - .and_then(|s| s.to_str()) - .map(|s| s.to_string()), + name: super::group::member_name(snap.path())?, parent_digest: snap.manifest().parent.as_ref().map(ToString::to_string), scope: snap.manifest().scope, image_ref: snap.manifest().image.reference.clone(), @@ -1462,6 +1524,30 @@ where Ok(()) } +async fn normalize_imported_descriptor(snapshot: &Snapshot) -> MicrosandboxResult<()> { + let path = snapshot.path().join(DESCRIPTOR_FILENAME); + let canonical = snapshot + .manifest() + .to_canonical_bytes() + .map_err(|error| MicrosandboxError::SnapshotIntegrity(error.to_string()))?; + if tokio::fs::read(&path).await? == canonical { + return Ok(()); + } + if let SnapshotState::File(file) = &snapshot.manifest().state { + for layer in &file.layers { + let source = snapshot.layer_path(layer); + let destination = snapshot.path().join(file.layer_path(layer)); + if source != destination { + tokio::fs::create_dir_all(destination.parent().expect("layer has parent")).await?; + tokio::fs::rename(source, destination).await?; + } + } + } + tokio::fs::write(&path, canonical).await?; + tokio::fs::File::open(path).await?.sync_all().await?; + Ok(()) +} + async fn build_archive_inventory( snapshots: &[Snapshot], cache_files: &[(PathBuf, String)], @@ -1588,12 +1674,28 @@ async fn build_archive_inventory( snapshot_members.sort_by(|left, right| left.snapshot_id.cmp(&right.snapshot_id)); entries.sort_by(|left, right| left.path.as_bytes().cmp(right.path.as_bytes())); - let suggested_name = head - .path() - .file_name() - .and_then(|name| name.to_str()) - .filter(|name| !name.is_empty() && name.len() <= 255) - .map(str::to_string); + // Names are local aliases, not descriptor identity. Carry them as optional + // archive metadata so importing a group preserves its useful selectors. + let mut member_names = BTreeMap::new(); + for snapshot in snapshots { + if let Some(name) = super::group::member_name(snapshot.path())? { + member_names.insert(snapshot.id().to_string(), name); + } + } + let suggested_name = member_names.get(head.id().as_str()).cloned().or_else(|| { + head.path() + .file_name() + .and_then(|name| name.to_str()) + .filter(|name| !name.is_empty() && name.len() <= 255) + .map(str::to_string) + }); + let mut extensions = BTreeMap::new(); + if !member_names.is_empty() { + extensions.insert( + "msb-snapshot-member-names".into(), + serde_json::to_value(member_names)?, + ); + } let encoded_bytes = entries.iter().map(|entry| entry.encoded_size).sum(); let apparent_bytes = entries.iter().map(|entry| entry.apparent_size).sum(); Ok(ArchiveInventory { @@ -1608,7 +1710,7 @@ async fn build_archive_inventory( apparent_bytes, }, entries, - extensions: BTreeMap::new(), + extensions, requires: vec![ARCHIVE_MEMBER_TRANSPORT_ALGORITHM.into()], }) } @@ -3612,28 +3714,6 @@ fn select_head_snapshot(snapshots: &[Snapshot]) -> MicrosandboxResult { } } -async fn ensure_promote_targets_available(stage: &Path, dest: &Path) -> MicrosandboxResult<()> { - let mut entries = tokio::fs::read_dir(stage).await?; - while let Some(entry) = entries.next_entry().await? { - let target = dest.join(entry.file_name()); - if tokio::fs::symlink_metadata(&target).await.is_ok() { - return Err(MicrosandboxError::SnapshotAlreadyExists( - target.display().to_string(), - )); - } - } - Ok(()) -} - -async fn promote_stage(stage: &Path, dest: &Path) -> MicrosandboxResult<()> { - let mut entries = tokio::fs::read_dir(stage).await?; - while let Some(entry) = entries.next_entry().await? { - let target = dest.join(entry.file_name()); - tokio::fs::rename(entry.path(), target).await?; - } - Ok(()) -} - async fn install_staged_cache( cache_stage: &Path, cache_dir: &Path, @@ -4007,8 +4087,17 @@ fn file_name_str(p: &Path) -> MicrosandboxResult { async fn resolve_parent_artifact( local: &LocalBackend, + child: &Snapshot, parent_id: &str, ) -> MicrosandboxResult { + // An archive may be installed repeatedly in independent groups. Follow local siblings + // before consulting the global identity index, where multiple copies are ambiguous. + if let Some(directory) = super::group::group_path(child.path()) { + let sibling = directory.join(parent_id); + if tokio::fs::try_exists(&sibling).await? { + return Ok(sibling); + } + } if let Some(handle) = store::lookup_by_digest(local, parent_id).await? { return Ok(handle.artifact_path); } @@ -4053,6 +4142,155 @@ mod tests { use super::*; + fn grouped_archive_manifest(id: u128, parent: Option<&Manifest>) -> Manifest { + let layer_id = DiskLayerId::new(format!("layer_{id:032x}")).unwrap(); + Manifest { + schema: SCHEMA.into(), + snapshot_id: SnapshotId::new(format!("snap_{id:032x}")).unwrap(), + scope: SnapshotScope::Disk, + state: SnapshotState::File(FileSnapshotState { + disk_format: SnapshotFormat::Raw, + filesystem: "ext4".into(), + virtual_size: 4096, + head: layer_id.clone(), + layers: vec![DiskLayer { + layer_id, + format: SnapshotFormat::Raw, + virtual_size: 4096, + backing: None, + payload: LayerPayload { + file_kind: LayerFileKind::Regular, + integrity: None, + }, + }], + }), + capture: SnapshotCapture { + created_at: "2026-09-10T00:00:00Z".into(), + source_lineage: None, + source_checkpoint: None, + consistency: SnapshotConsistency::CrashConsistent, + }, + image: ImageRef { + reference: "docker.io/library/alpine:3.20".into(), + manifest_digest: format!("sha256:{}", "a".repeat(64)), + }, + root_disk: SnapshotRootDisk::Managed, + parent: parent.map(|parent| parent.snapshot_id.clone()), + extensions: BTreeMap::new(), + requires: Vec::new(), + } + } + + fn write_grouped_archive_fixture(path: &Path, manifest: &Manifest) { + std::fs::create_dir_all(path).unwrap(); + std::fs::write( + path.join(DESCRIPTOR_FILENAME), + manifest.to_canonical_bytes().unwrap(), + ) + .unwrap(); + let SnapshotState::File(state) = &manifest.state else { + unreachable!() + }; + for layer in &state.layers { + let payload = path.join(state.layer_path(layer)); + std::fs::create_dir_all(payload.parent().unwrap()).unwrap(); + std::fs::write(payload, vec![42; layer.virtual_size as usize]).unwrap(); + } + } + + #[tokio::test] + async fn with_parents_prefers_group_members_when_global_identities_repeat() { + let home = tempfile::tempdir().unwrap(); + let local = LocalBackend::builder() + .home(home.path()) + .build() + .await + .unwrap(); + let parent = grouped_archive_manifest(1, None); + let child = grouped_archive_manifest(2, Some(&parent)); + for name in ["first", "second"] { + let group = super::super::group::ensure(&local.snapshots_dir(), Some(name)) + .await + .unwrap(); + let stage = tempfile::tempdir().unwrap(); + for manifest in [&parent, &child] { + write_grouped_archive_fixture( + &stage.path().join(manifest.snapshot_id.as_str()), + manifest, + ); + } + let aliases = BTreeMap::from([ + (parent.snapshot_id.to_string(), "base".into()), + (child.snapshot_id.to_string(), "child".into()), + ]); + super::super::group::publish(&group, stage.path(), &aliases, &child.snapshot_id, false) + .await + .unwrap(); + } + store::reindex_dir(&local, &local.snapshots_dir()) + .await + .unwrap(); + assert!( + store::lookup_by_digest(&local, parent.snapshot_id.as_str()) + .await + .is_err() + ); + let archive = home.path().join("group.msnap"); + save_snapshot( + &local, + "first:child", + &archive, + SaveOpts { + with_parents: true, + plain_tar: true, + ..Default::default() + }, + ) + .await + .unwrap(); + let loaded = load_snapshot(&local, &archive, None).await.unwrap(); + let loaded_group = loaded.group().unwrap(); + assert_eq!( + store::get_handle(&local, &format!("{loaded_group}:base")) + .await + .unwrap() + .id(), + parent.snapshot_id.as_str() + ); + assert_eq!( + store::get_handle(&local, &format!("{loaded_group}:child")) + .await + .unwrap() + .id(), + child.snapshot_id.as_str() + ); + } + + #[tokio::test] + async fn legacy_suggested_name_that_is_not_a_group_alias_does_not_block_import() { + let home = tempfile::tempdir().unwrap(); + let local = LocalBackend::builder() + .home(home.path()) + .build() + .await + .unwrap(); + let manifest = grouped_archive_manifest(1, None); + let artifact = home.path().join("legacy name with spaces"); + write_grouped_archive_fixture(&artifact, &manifest); + let archive = home.path().join("legacy.msnap"); + save_snapshot( + &local, + artifact.to_str().unwrap(), + &archive, + SaveOpts::default(), + ) + .await + .unwrap(); + let loaded = load_snapshot(&local, &archive, None).await.unwrap(); + assert_eq!(loaded.id(), manifest.snapshot_id.as_str()); + assert!(loaded.group().is_some()); + } + #[test] fn digest_hex_rejects_uppercase_identity() { let uppercase = format!("sha256:{}", "A".repeat(64)); diff --git a/sdk/rust/lib/snapshot/create.rs b/sdk/rust/lib/snapshot/create.rs index e865f71dd..8c7a01237 100644 --- a/sdk/rust/lib/snapshot/create.rs +++ b/sdk/rust/lib/snapshot/create.rs @@ -70,12 +70,110 @@ impl Drop for SnapshotDiskClosure { //-------------------------------------------------------------------------------------------------- pub(super) async fn create_snapshot( + local: &LocalBackend, + mut config: SnapshotConfig, +) -> MicrosandboxResult { + if config.force { + return Err(MicrosandboxError::InvalidConfig( + "grouped snapshots are immutable; choose another member name or remove the existing member explicitly".into(), + )); + } + let generated_name = config.name.is_empty(); + if generated_name { + config.name = format!("msb-{:08x}", rand::random::()); + } + validate_snapshot_name(&config.name)?; + let lineage = super::lineage::begin(local, &config.source_sandbox).await?; + let root = config + .dest_dir + .take() + .unwrap_or_else(|| local.snapshots_dir()); + let group_name = config + .group + .take() + .unwrap_or_else(|| config.source_sandbox.clone()); + let group_dir = super::group::ensure(&root, Some(&group_name)).await?; + let staging = tempfile::Builder::new() + .prefix(".capture-") + .tempdir_in(&group_dir)?; + let name = config.name.clone(); + let source_sandbox = config.source_sandbox.clone(); + config.dest_dir = Some(staging.path().to_path_buf()); + let mut captured = capture_installed(local, config, lineage.sandbox_id()).await?; + lineage.validate_source(local, &source_sandbox).await?; + // Ancestry belongs to the immutable descriptor, not to the group head or export base. + captured.manifest.parent = lineage.parent.clone(); + captured.digest = captured + .manifest + .digest() + .map_err(|error| MicrosandboxError::SnapshotIntegrity(error.to_string()))?; + let descriptor = captured + .manifest + .to_canonical_bytes() + .map_err(|error| MicrosandboxError::SnapshotIntegrity(error.to_string()))?; + write_descriptor(captured.path(), &descriptor).await?; + // Publication owns its staging and ancestry sequencer. Dropping an SDK future must not + // release the source lock while a blocking group commit is still running in the background. + let captured = tokio::spawn(async move { + let update = publish_with_name_retry( + &group_dir, + staging.path(), + captured.id(), + name, + generated_name, + || format!("msb-{:08x}", rand::random::()), + ).await?; + captured.path = group_dir.join(captured.id().as_str()); + lineage.commit(captured.id()).await?; + tracing::info!(group = %update.group, head = %update.head, reason = ?update.reason, "snapshot group publication"); + captured.head_update = Some(update); + Ok::<_, MicrosandboxError>(captured) + }).await.map_err(|error| MicrosandboxError::Runtime(format!("snapshot publication task: {error}")))??; + if let Err(error) = index_upsert( + local, + captured.path(), + captured.digest(), + captured.manifest(), + ) + .await + { + tracing::warn!(%error, "snapshot index update failed after group publication"); + } + Ok(captured) +} + +/// Retry generated local names against the same captured artifact; explicit names remain strict. +pub(super) async fn publish_with_name_retry( + group_dir: &Path, + staged: &Path, + snapshot_id: &SnapshotId, + mut name: String, + generated_name: bool, + mut next_name: impl FnMut() -> String, +) -> MicrosandboxResult { + loop { + let aliases = BTreeMap::from([(snapshot_id.to_string(), name)]); + match super::group::publish(group_dir, staged, &aliases, snapshot_id, false).await { + Err(MicrosandboxError::SnapshotAlreadyExists(_)) if generated_name => { + // Alias conflicts are preflight errors: no staged payload was moved and the + // descriptor's identity/ancestry remain unchanged, so no recapture is needed. + name = next_name(); + } + result => return result, + } + } +} + +/// Build a complete artifact in operation-owned staging; group publication happens afterward. +async fn capture_installed( local: &LocalBackend, config: SnapshotConfig, + expected_source_id: i32, ) -> MicrosandboxResult { let total_started = Instant::now(); let SnapshotConfig { name, + group: _, dest_dir, source_sandbox, labels, @@ -102,6 +200,11 @@ pub(super) async fn create_snapshot( .await? .ok_or_else(|| MicrosandboxError::SandboxNotFound(source_sandbox.clone()))?; + if model.id != expected_source_id { + return Err(MicrosandboxError::InvalidConfig( + "source sandbox changed before snapshot capture".into(), + )); + } if full { return create_full_snapshot( local, @@ -227,13 +330,6 @@ pub(super) async fn create_snapshot( promote_snapshot_directory(&staging_dir, &dest_dir, force).await?; let promote_us = promote_started.elapsed().as_micros(); - // Best-effort index upsert. Failures are logged, not propagated — - // the artifact on disk is the source of truth. - let index_started = Instant::now(); - if let Err(e) = index_upsert(local, &dest_dir, &digest, &manifest).await { - tracing::warn!(error = %e, snapshot = %digest, "snapshot_index upsert failed"); - } - let index_us = index_started.elapsed().as_micros(); tracing::info!( target: "microsandbox_checkpoint_timing", operation = "snapshot_create_installed_disk", @@ -241,7 +337,6 @@ pub(super) async fn create_snapshot( total_us = total_started.elapsed().as_micros(), artifact_build_us, promote_us, - index_us, "disk snapshot creation timing" ); @@ -273,7 +368,7 @@ async fn create_full_snapshot( // The runtime owns capture and recovery even if this client disappears. Do not allocate an // artifact staging directory while waiting for it: there is nothing to stage until capture // succeeds. The guard also removes partial materialization on ordinary errors/cancellation. - let captured = capture_full_snapshot(source_sandbox, labels, model).await?; + let captured = capture_full_snapshot(local, source_sandbox, labels, model).await?; let capture_us = capture_started.elapsed().as_micros(); let staging = tempfile::Builder::new() .prefix(&format!(".{name}.")) @@ -317,11 +412,6 @@ async fn create_full_snapshot( let promote_started = Instant::now(); promote_snapshot_directory(&staging_dir, dest_dir, force).await?; let promote_us = promote_started.elapsed().as_micros(); - let index_started = Instant::now(); - if let Err(error) = index_upsert(local, dest_dir, &digest, &captured.manifest).await { - tracing::warn!(error = %error, snapshot = %digest, "snapshot_index upsert failed"); - } - let index_us = index_started.elapsed().as_micros(); tracing::info!( target: "microsandbox_checkpoint_timing", operation = "snapshot_create_installed_full", @@ -332,7 +422,6 @@ async fn create_full_snapshot( closure_verify_us, metadata_descriptor_us, promote_us, - index_us, "installed full snapshot creation timing" ); Ok(Snapshot::from_parts( @@ -353,7 +442,8 @@ pub(super) async fn create_snapshot_archive( ) -> MicrosandboxResult { let total_started = Instant::now(); let SnapshotConfig { - name, + mut name, + group, dest_dir, source_sandbox, labels, @@ -361,37 +451,57 @@ pub(super) async fn create_snapshot_archive( record_integrity, full, } = config; - if dest_dir.is_some() { + if dest_dir.is_some() || group.is_some() { return Err(MicrosandboxError::InvalidConfig( - "direct archive capture is mutually exclusive with dest_dir".into(), + "direct archive capture does not install a group; omit group and dest_dir".into(), )); } + if name.is_empty() { + name = format!("msb-{:08x}", rand::random::()); + } validate_snapshot_name(&name)?; + let lineage = super::lineage::begin(local, &source_sandbox).await?; let db = local.db().await?.read(); let model = sandbox_entity::Entity::find() .filter(sandbox_entity::Column::Name.eq(&source_sandbox)) .one(db) .await? .ok_or_else(|| MicrosandboxError::SandboxNotFound(source_sandbox.clone()))?; + if model.id != lineage.sandbox_id() { + return Err(MicrosandboxError::InvalidConfig( + "source sandbox changed before snapshot capture".into(), + )); + } if full { let capture_started = Instant::now(); - let captured = capture_full_snapshot(&source_sandbox, labels, model).await?; + let mut captured = capture_full_snapshot(local, &source_sandbox, labels, model).await?; + lineage.validate_source(local, &source_sandbox).await?; + captured.manifest.parent = lineage.parent.clone(); let capture_us = capture_started.elapsed().as_micros(); let digest = captured .manifest .digest() .map_err(|error| MicrosandboxError::SnapshotIntegrity(error.to_string()))?; let archive_started = Instant::now(); - super::archive::save_direct_checkpoint_snapshot( - &captured.manifest, - &captured.labels, - &name, - &captured.checkpoint_path, - out, - plain_tar, - force, - ) - .await?; + let owned_out = out.to_path_buf(); + let captured = tokio::spawn(async move { + super::archive::save_direct_checkpoint_snapshot( + &captured.manifest, + &captured.labels, + &name, + &captured.checkpoint_path, + &owned_out, + plain_tar, + force, + ) + .await?; + lineage.commit(&captured.manifest.snapshot_id).await?; + Ok::<_, MicrosandboxError>(captured) + }) + .await + .map_err(|error| { + MicrosandboxError::Runtime(format!("snapshot archive publication: {error}")) + })??; let archive_us = archive_started.elapsed().as_micros(); tracing::info!( target: "microsandbox_checkpoint_timing", @@ -465,6 +575,7 @@ pub(super) async fn create_snapshot_archive( &root_disk, ) .await?; + lineage.validate_source(local, &source_sandbox).await?; let integrity_started = Instant::now(); let integrities = vec![None; disk.sources.len()]; let labels: BTreeMap<_, _> = labels.into_iter().collect(); @@ -476,6 +587,7 @@ pub(super) async fn create_snapshot_archive( &source_sandbox, root_disk, )?; + manifest.parent = lineage.parent.clone(); if record_integrity && let SnapshotState::File(file) = &mut manifest.state { for index in 0..file.layers.len() { let source = &disk.sources[index].path; @@ -502,16 +614,30 @@ pub(super) async fn create_snapshot_archive( .iter() .map(|source| source.path.clone()) .collect::>(); - super::archive::save_direct_file_snapshot( - &manifest, - &labels, - &name, - &source_paths, - out, - plain_tar, - force, - ) - .await?; + let owned_out = out.to_path_buf(); + let logical_bytes = disk.virtual_size; + let (manifest, labels) = tokio::spawn(async move { + // A stopped disk remains locked and a live immutable cut remains pinned until the + // background writer finishes, even if the caller stops awaiting this operation. + let _disk = disk; + let _lifecycle_guard = _lifecycle_guard; + super::archive::save_direct_file_snapshot( + &manifest, + &labels, + &name, + &source_paths, + &owned_out, + plain_tar, + force, + ) + .await?; + lineage.commit(&manifest.snapshot_id).await?; + Ok::<_, MicrosandboxError>((manifest, labels)) + }) + .await + .map_err(|error| { + MicrosandboxError::Runtime(format!("snapshot archive publication: {error}")) + })??; let archive_us = archive_started.elapsed().as_micros(); tracing::info!( target: "microsandbox_checkpoint_timing", @@ -519,7 +645,7 @@ pub(super) async fn create_snapshot_archive( source_sandbox, plain_tar, record_integrity, - logical_bytes = disk.virtual_size, + logical_bytes, total_us = total_started.elapsed().as_micros(), integrity_us, archive_us, @@ -538,6 +664,7 @@ pub(super) async fn create_snapshot_archive( /// Installed snapshots and direct archives share this boundary so both publish byte-for-byte the /// same descriptor and checkpoint closure. async fn capture_full_snapshot( + local: &LocalBackend, source_sandbox: &str, labels: Vec<(String, String)>, model: sandbox_entity::Model, @@ -560,7 +687,8 @@ async fn capture_full_snapshot( let checkpoint_id = format!("checkpoint_{:032x}", rand::random::()); let checkpoint = - crate::sandbox::control_checkpoint_create(source_sandbox, checkpoint_id.clone()).await?; + crate::sandbox::control_checkpoint_create(local, source_sandbox, checkpoint_id.clone()) + .await?; if checkpoint.checkpoint_id != checkpoint_id { return Err(MicrosandboxError::SnapshotIntegrity( "runtime returned a checkpoint for another capture attempt".into(), diff --git a/sdk/rust/lib/snapshot/downgrade.rs b/sdk/rust/lib/snapshot/downgrade.rs index 2107cbb4f..b7b96d42c 100644 --- a/sdk/rust/lib/snapshot/downgrade.rs +++ b/sdk/rust/lib/snapshot/downgrade.rs @@ -1753,7 +1753,8 @@ mod tests { "reverse_complete" ); - Migrator::down(pools.write().inner(), Some(1)) + // Reverse both the empty group projection and stable-identity projection. + Migrator::down(pools.write().inner(), Some(2)) .await .unwrap(); let count = pools diff --git a/sdk/rust/lib/snapshot/group.rs b/sdk/rust/lib/snapshot/group.rs new file mode 100644 index 000000000..5f09d5d66 --- /dev/null +++ b/sdk/rust/lib/snapshot/group.rs @@ -0,0 +1,819 @@ +//! Durable local snapshot namespaces and their explicitly selected heads. +//! +//! Group membership is represented by installed artifact directories. Only the head and each +//! member's optional local alias need metadata; immutable descriptors remain authoritative for +//! ancestry. All group operations share one process-held lock, acquired off the async executor. + +use std::collections::{BTreeMap, HashSet}; +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Write}; +#[cfg(unix)] +use std::os::unix::fs::OpenOptionsExt; +use std::path::{Path, PathBuf}; + +use microsandbox_image::snapshot::{ + DESCRIPTOR_FILENAME, MAX_DESCRIPTOR_BYTES, Manifest, SnapshotId, +}; +use microsandbox_utils::process_lock; +use serde::{Deserialize, Serialize}; + +use crate::{MicrosandboxError, MicrosandboxResult}; + +//-------------------------------------------------------------------------------------------------- +// Constants +//-------------------------------------------------------------------------------------------------- + +pub(crate) const GROUP_FILENAME: &str = "group.json"; +const GROUP_SCHEMA: &str = "microsandbox.snapshot-group/1"; +const MEMBER_FILENAME: &str = "group-member.json"; +const MEMBER_SCHEMA: &str = "microsandbox.snapshot-group-member/1"; +const MAX_METADATA_BYTES: usize = 4096; +const MAX_NAME_BYTES: usize = 128; +const MAX_ANCESTRY_DEPTH: usize = 65536; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// Outcome of publishing snapshots into a local group or explicitly selecting its head. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct HeadUpdate { + /// Local group name. + pub group: String, + /// Previously selected stable snapshot ID, if the group had a head. + pub previous: Option, + /// Stable snapshot ID selected after the operation. + pub head: String, + /// Explanation for advancing or retaining the selected head. + pub reason: HeadUpdateReason, + /// Whether the selected head changed. + pub changed: bool, +} + +/// Why a snapshot group's head advanced or remained selected. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum HeadUpdateReason { + /// The first candidate initialized an empty group. + Initialized, + /// The candidate is a proven descendant of the current head. + FastForwarded, + /// The caller explicitly selected an installed member. + Selected, + /// The candidate was already selected, or the caller only read the head. + Unchanged, + /// The candidate is not a descendant of the current head. + Diverged, + /// Missing history prevents proving that the candidate descends from the head. + UnknownAncestry, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct GroupState { + schema: String, + head: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct MemberMetadata { + schema: String, + name: String, +} + +#[derive(Debug)] +struct Member { + path: PathBuf, + digest: String, + parent: Option, + name: Option, +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +/// Resolve a group head or a qualified group member; explicit paths belong to the caller. +pub(super) async fn resolve(root: &Path, selector: &str) -> MicrosandboxResult { + let root = root.to_path_buf(); + let selector = selector.to_owned(); + blocking(move || { + let (name, member) = parse_selector(&selector)?; + let directory = root.join(name); + let _lock = lock_group(&directory)?; + let state = read_group(&directory)?; + let (id, _) = resolve_selected(&directory, &state, member)?; + Ok(directory.join(id)) + }) + .await +} + +/// Open or create an explicitly named group, or create a fresh generated local group. +pub(super) async fn ensure(root: &Path, name: Option<&str>) -> MicrosandboxResult { + let root = root.to_path_buf(); + let name = name.map(str::to_owned); + blocking(move || { + if let Some(name) = &name { + validate_group_name(name)?; + } + fs::create_dir_all(&root)?; + require_directory(&root)?; + // Serialize creation separately: the group lock does not exist until publication. + let creation_lock = process_lock::open_lock_file(&root.join(".groups.lock"))?; + process_lock::lock_exclusive(&creation_lock)?; + let name = match name { + Some(name) => name, + None => loop { + let candidate = format!("msb-{:08x}", rand::random::()); + if !path_exists(&root.join(&candidate))? { + break candidate; + } + }, + }; + let directory = root.join(&name); + if path_exists(&directory)? { + require_directory(&directory)?; + if !path_exists(&directory.join(GROUP_FILENAME))? { + return Err(MicrosandboxError::InvalidConfig(format!( + "'{name}' already names a snapshot directory, not a snapshot group; choose another group name or open the existing snapshot by its explicit path" + ))); + } + read_group(&directory)?; + return Ok(directory); + } + + // The initial head and lock become visible together with the new group directory. + let staging = tempfile::Builder::new() + .prefix(".group-new-") + .tempdir_in(&root)?; + write_group(staging.path(), None)?; + process_lock::create_new_lock_file(&staging.path().join(".group.lock"))?.sync_all()?; + sync_directory(staging.path())?; + fs::rename(staging.path(), &directory)?; + sync_directory(&root)?; + Ok(directory) + }) + .await +} + +/// Publish complete staged artifacts and atomically decide the group's next head. +/// +/// `staged` contains immediate child artifact directories. The caller prepares, validates, and +/// flushes their payloads before calling this function. All descriptor and alias conflicts are +/// checked before publication; existing identical artifacts are never overwritten. +pub(super) async fn publish( + group_dir: &Path, + staged: &Path, + aliases: &BTreeMap, + candidate: &SnapshotId, + set_head: bool, +) -> MicrosandboxResult { + let group_dir = group_dir.to_path_buf(); + let staged = staged.to_path_buf(); + let aliases = aliases.clone(); + let candidate = candidate.to_string(); + blocking(move || { + require_directory(&staged)?; + if fs::canonicalize(&group_dir)?.starts_with(fs::canonicalize(&staged)?) { + return Err(MicrosandboxError::InvalidConfig( + "snapshot staging must not contain the destination group".into(), + )); + } + let incoming = read_members(&staged, false)?; + let _lock = lock_group(&group_dir)?; + let state = read_group(&group_dir)?; + let mut members = read_members(&group_dir, true)?; + validate_head(&state, &members)?; + + for (id, member) in &incoming { + if let Some(existing) = members.get(id) { + if existing.digest != member.digest { + return Err(integrity(format!( + "snapshot ID {id} already exists in this group with a different descriptor" + ))); + } + } else { + // Even an unrecognized file at the destination must not be overwritten. + if path_exists(&group_dir.join(id))? { + return Err(integrity(format!( + "snapshot destination already exists: {}", + group_dir.join(id).display() + ))); + } + members.insert( + id.clone(), + Member { + path: member.path.clone(), + digest: member.digest.clone(), + parent: member.parent.clone(), + name: None, + }, + ); + } + } + if !members.contains_key(&candidate) { + return Err(MicrosandboxError::SnapshotNotFound(candidate)); + } + apply_aliases(&mut members, &aliases)?; + validate_ancestry(&members)?; + let update = head_update( + &group_dir, + state.head.as_deref(), + &candidate, + &members, + set_head, + )?; + + // Complete members are durable before publishing the head. A crash or I/O error can + // leave additional complete members, but never a head pointing at a half-written one. + for (id, member) in &incoming { + let destination = group_dir.join(id); + if !path_exists(&destination)? { + write_member_name(&member.path, members[id].name.as_deref())?; + sync_directory(&member.path)?; + fs::rename(&member.path, destination)?; + } + } + for id in aliases.keys() { + write_member_name(&group_dir.join(id), members[id].name.as_deref())?; + } + sync_directory(&group_dir)?; + if update.changed { + write_group(&group_dir, Some(update.head.clone()))?; + } + Ok(update) + }) + .await +} + +/// Read a bare group's head, or explicitly select a qualified member as its head. +pub(super) async fn select(root: &Path, selector: &str) -> MicrosandboxResult { + let root = root.to_path_buf(); + let selector = selector.to_owned(); + blocking(move || { + let (name, selected) = parse_selector(&selector)?; + let directory = root.join(name); + let _lock = lock_group(&directory)?; + let state = read_group(&directory)?; + let (candidate, member) = resolve_selected(&directory, &state, selected)?; + let members = BTreeMap::from([(candidate.clone(), member)]); + let update = head_update( + &directory, + state.head.as_deref(), + &candidate, + &members, + selected.is_some(), + )?; + if update.changed { + write_group(&directory, Some(update.head.clone()))?; + } + Ok(update) + }) + .await +} + +/// Read a member's optional local friendly name without altering its immutable descriptor. +pub(super) fn member_name(path: &Path) -> MicrosandboxResult> { + let metadata_path = path.join(MEMBER_FILENAME); + if !path_exists(&metadata_path)? { + return Ok(None); + } + let metadata: MemberMetadata = + serde_json::from_slice(&read_regular(&metadata_path, MAX_METADATA_BYTES)?)?; + if metadata.schema != MEMBER_SCHEMA { + return Err(integrity(format!( + "unsupported snapshot member metadata schema: {}", + metadata.schema + ))); + } + validate_alias(&metadata.name)?; + Ok(Some(metadata.name)) +} + +/// Return the containing group when a member's parent has regular group metadata. +pub(super) fn group_path(path: &Path) -> Option { + let parent = path.parent()?; + let metadata = fs::symlink_metadata(parent.join(GROUP_FILENAME)).ok()?; + metadata.file_type().is_file().then(|| parent.to_path_buf()) +} + +/// Remove a grouped member under its publication lock, returning false for ungrouped paths. +pub(super) async fn remove_member(path: &Path) -> MicrosandboxResult { + let path = path.to_path_buf(); + blocking(move || { + let Some(directory) = group_path(&path) else { + return Ok(false); + }; + let _lock = lock_group(&directory)?; + let state = read_group(&directory)?; + let members = read_members(&directory, true)?; + validate_head(&state, &members)?; + let id = path.file_name().and_then(|name| name.to_str()).ok_or_else(|| { + MicrosandboxError::InvalidConfig("snapshot member path has no stable ID".into()) + })?; + if !members.contains_key(id) { + return Err(MicrosandboxError::SnapshotNotFound(id.into())); + } + if state.head.as_deref() == Some(id) { + if members.len() > 1 { + return Err(MicrosandboxError::InvalidConfig(format!( + "cannot remove current head {id}; first select another snapshot with 'msb snapshot head {}:'", + directory.file_name().unwrap_or_default().to_string_lossy() + ))); + } + // Clear first so an interrupted recursive removal cannot strand a dangling head. + // A failed removal is recoverable by explicitly selecting the surviving member. + write_group(&directory, None)?; + } + fs::remove_dir_all(&path).map_err(|error| { + MicrosandboxError::Custom(format!( + "could not fully remove snapshot {}: {error}; inspect the group before retrying", + path.display() + )) + })?; + sync_directory(&directory)?; + Ok(true) + }) + .await +} + +//-------------------------------------------------------------------------------------------------- +// Functions: Helpers +//-------------------------------------------------------------------------------------------------- + +async fn blocking( + work: impl FnOnce() -> MicrosandboxResult + Send + 'static, +) -> MicrosandboxResult { + tokio::task::spawn_blocking(work) + .await + .map_err(|error| MicrosandboxError::Custom(format!("snapshot group operation: {error}")))? +} + +fn parse_selector(selector: &str) -> MicrosandboxResult<(&str, Option<&str>)> { + let (group, member) = match selector.split_once(':') { + Some((group, member)) => (group, Some(member)), + None => (selector, None), + }; + validate_group_name(group)?; + if let Some(member) = member { + validate_name(member, "snapshot selector")?; + } + Ok((group, member)) +} + +fn validate_name(name: &str, kind: &str) -> MicrosandboxResult<()> { + let first = name.as_bytes().first().copied(); + if name.len() > MAX_NAME_BYTES + || !first.is_some_and(|byte| byte.is_ascii_alphanumeric()) + || name.ends_with('.') + || !name + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"-_.".contains(&byte)) + { + return Err(MicrosandboxError::InvalidConfig(format!( + "invalid {kind} '{name}': use 1–{MAX_NAME_BYTES} ASCII letters, digits, '-', '_' or '.', start with a letter or digit, and do not end with '.'" + ))); + } + // Reject device names even on Unix so local selectors remain portable to Windows. + let stem = name.split('.').next().unwrap_or(name).to_ascii_uppercase(); + if matches!(stem.as_str(), "CON" | "PRN" | "AUX" | "NUL") + || (stem.len() == 4 + && (stem.starts_with("COM") || stem.starts_with("LPT")) + && matches!(stem.as_bytes()[3], b'1'..=b'9')) + { + return Err(MicrosandboxError::InvalidConfig(format!( + "invalid {kind} '{name}': reserved device name" + ))); + } + Ok(()) +} + +pub(super) fn validate_alias(name: &str) -> MicrosandboxResult<()> { + validate_name(name, "snapshot name")?; + if SnapshotId::new(name).is_ok() { + return Err(MicrosandboxError::InvalidConfig( + "a snapshot's friendly name must not be a stable snapshot ID".into(), + )); + } + Ok(()) +} + +fn validate_group_name(name: &str) -> MicrosandboxResult<()> { + validate_name(name, "group")?; + if matches!(name, "sha256" | "sha512") || SnapshotId::new(name).is_ok() { + return Err(MicrosandboxError::InvalidConfig(format!( + "invalid group name '{name}': reserved snapshot identifier namespace" + ))); + } + Ok(()) +} + +fn lock_group(directory: &Path) -> MicrosandboxResult { + require_directory(directory)?; + read_group(directory)?; + let lock = process_lock::open_lock_file(&directory.join(".group.lock"))?; + process_lock::lock_exclusive(&lock)?; + Ok(lock) +} + +fn read_group(directory: &Path) -> MicrosandboxResult { + let path = directory.join(GROUP_FILENAME); + if !path_exists(&path)? { + return Err(MicrosandboxError::SnapshotNotFound(format!( + "snapshot group {}", + directory.display() + ))); + } + let state: GroupState = serde_json::from_slice(&read_regular(&path, MAX_METADATA_BYTES)?)?; + if state.schema != GROUP_SCHEMA { + return Err(integrity(format!( + "unsupported snapshot group schema: {}", + state.schema + ))); + } + if let Some(head) = &state.head { + SnapshotId::new(head).map_err(|error| integrity(error.to_string()))?; + } + Ok(state) +} + +fn write_group(directory: &Path, head: Option) -> MicrosandboxResult<()> { + write_json( + directory, + GROUP_FILENAME, + &GroupState { + schema: GROUP_SCHEMA.into(), + head, + }, + ) +} + +fn read_members(directory: &Path, installed: bool) -> MicrosandboxResult> { + let mut members = BTreeMap::new(); + for entry in fs::read_dir(directory)? { + let entry = entry?; + let filename = entry.file_name(); + let name = filename + .to_str() + .ok_or_else(|| integrity("snapshot member directory name is not valid UTF-8".into()))?; + if installed && !name.starts_with("snap_") { + continue; + } + if !entry.file_type()?.is_dir() { + if installed { + return Err(integrity(format!( + "snapshot member is not a regular directory: {}", + entry.path().display() + ))); + } + return Err(integrity(format!( + "snapshot staging contains a non-directory member: {}", + entry.path().display() + ))); + } + let (id, member) = read_member(&entry.path(), installed)?; + if installed && id != name { + return Err(integrity(format!( + "snapshot member directory {name} does not match descriptor ID {id}" + ))); + } + if members.insert(id.clone(), member).is_some() { + return Err(integrity(format!( + "snapshot staging contains duplicate stable ID {id}" + ))); + } + } + Ok(members) +} + +fn apply_aliases( + members: &mut BTreeMap, + aliases: &BTreeMap, +) -> MicrosandboxResult<()> { + for (id, name) in aliases { + validate_alias(name)?; + let member = members + .get_mut(id) + .ok_or_else(|| MicrosandboxError::SnapshotNotFound(id.clone()))?; + if let Some(existing) = &member.name + && existing != name + { + return Err(MicrosandboxError::SnapshotAlreadyExists(format!( + "snapshot {id} already has local name '{existing}', not '{name}'" + ))); + } + member.name = Some(name.clone()); + } + let mut names = BTreeMap::new(); + for (id, member) in members { + if let Some(name) = &member.name + && let Some(previous) = names.insert(name.clone(), id.clone()) + { + return Err(MicrosandboxError::SnapshotAlreadyExists(format!( + "snapshot name '{name}' conflicts between {previous} and {id} in this group" + ))); + } + } + Ok(()) +} + +fn read_member(path: &Path, installed: bool) -> MicrosandboxResult<(String, Member)> { + require_directory(path)?; + let manifest = Manifest::from_bytes(&read_regular( + &path.join(DESCRIPTOR_FILENAME), + MAX_DESCRIPTOR_BYTES, + )?) + .map_err(|error| integrity(error.to_string()))?; + let id = manifest.snapshot_id.to_string(); + let member = Member { + digest: manifest + .digest() + .map_err(|error| integrity(error.to_string()))?, + parent: manifest.parent.map(|parent| parent.to_string()), + name: if installed { member_name(path)? } else { None }, + path: path.to_path_buf(), + }; + Ok((id, member)) +} + +fn resolve_selected( + directory: &Path, + state: &GroupState, + selected: Option<&str>, +) -> MicrosandboxResult<(String, Member)> { + // ID and head lookup touch only the selected descriptor. Large histories do not make the + // normal open path progressively slower, and unrelated artifacts need not be reopened. + let selected_id = match selected { + None => Some(state.head.clone().ok_or_else(|| { + MicrosandboxError::SnapshotNotFound(format!( + "snapshot group {} has no head", + directory.display() + )) + })?), + Some(selected) if SnapshotId::new(selected).is_ok() => Some(selected.to_owned()), + Some(_) => None, + }; + let expected = match selected_id { + Some(id) => id, + None => { + let selected = selected.unwrap(); + let mut matched = None; + for entry in fs::read_dir(directory)? { + let entry = entry?; + let name = entry.file_name(); + let Some(name) = name.to_str().filter(|name| name.starts_with("snap_")) else { + continue; + }; + require_directory(&entry.path())?; + if member_name(&entry.path())?.as_deref() == Some(selected) { + if matched.is_some() { + return Err(integrity(format!( + "snapshot name '{selected}' is ambiguous in this group" + ))); + } + matched = Some(name.to_owned()); + } + } + matched.ok_or_else(|| { + MicrosandboxError::SnapshotNotFound(format!( + "{}:{selected}", + directory.file_name().unwrap_or_default().to_string_lossy() + )) + })? + } + }; + let path = directory.join(&expected); + if !path_exists(&path)? { + return Err(MicrosandboxError::SnapshotNotFound(format!( + "snapshot group member {} is missing", + path.display() + ))); + } + let (id, member) = read_member(&path, true)?; + if id != expected { + return Err(integrity(format!( + "snapshot member directory {expected} does not match descriptor ID {id}" + ))); + } + Ok((id, member)) +} + +fn validate_head(state: &GroupState, members: &BTreeMap) -> MicrosandboxResult<()> { + if let Some(head) = &state.head + && !members.contains_key(head) + { + return Err(integrity(format!( + "snapshot group head {head} is missing; explicitly select an installed member to repair the head" + ))); + } + Ok(()) +} + +fn head_update( + directory: &Path, + previous: Option<&str>, + candidate: &str, + members: &BTreeMap, + explicit: bool, +) -> MicrosandboxResult { + let reason = match previous { + Some(head) if head == candidate => HeadUpdateReason::Unchanged, + _ if explicit => HeadUpdateReason::Selected, + None => HeadUpdateReason::Initialized, + Some(head) => ancestry_reason(candidate, head, members)?, + }; + let changed = matches!( + reason, + HeadUpdateReason::Initialized + | HeadUpdateReason::FastForwarded + | HeadUpdateReason::Selected + ); + Ok(HeadUpdate { + group: directory + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| integrity("snapshot group directory has no valid name".into()))? + .into(), + previous: previous.map(str::to_owned), + head: if changed { + candidate.into() + } else { + previous.unwrap_or(candidate).into() + }, + reason, + changed, + }) +} + +fn ancestry_reason( + candidate: &str, + head: &str, + members: &BTreeMap, +) -> MicrosandboxResult { + let mut current = candidate; + let mut visited = HashSet::new(); + while visited.len() < MAX_ANCESTRY_DEPTH { + if !visited.insert(current) { + return Err(integrity("snapshot ancestry contains a cycle".into())); + } + let Some(member) = members.get(current) else { + return Ok(HeadUpdateReason::UnknownAncestry); + }; + let Some(parent) = member.parent.as_deref() else { + return Ok(HeadUpdateReason::Diverged); + }; + if parent == head { + return Ok(HeadUpdateReason::FastForwarded); + } + current = parent; + } + Err(integrity(format!( + "snapshot ancestry exceeds the {MAX_ANCESTRY_DEPTH}-member traversal limit" + ))) +} + +fn validate_ancestry(members: &BTreeMap) -> MicrosandboxResult<()> { + let mut complete = HashSet::new(); + for id in members.keys() { + let mut current = id.as_str(); + let mut visiting = HashSet::new(); + while !complete.contains(current) { + if !visiting.insert(current) { + return Err(integrity("snapshot ancestry contains a cycle".into())); + } + if visiting.len() > MAX_ANCESTRY_DEPTH { + return Err(integrity(format!( + "snapshot ancestry exceeds the {MAX_ANCESTRY_DEPTH}-member traversal limit" + ))); + } + let Some(parent) = members + .get(current) + .and_then(|member| member.parent.as_deref()) + else { + break; + }; + current = parent; + } + complete.extend(visiting); + } + Ok(()) +} + +fn write_member_name(directory: &Path, name: Option<&str>) -> MicrosandboxResult<()> { + let path = directory.join(MEMBER_FILENAME); + match name { + Some(name) => write_json( + directory, + MEMBER_FILENAME, + &MemberMetadata { + schema: MEMBER_SCHEMA.into(), + name: name.into(), + }, + ), + None => { + // Imported local metadata does not choose names in the receiving namespace. + if path_exists(&path)? { + if !fs::symlink_metadata(&path)?.file_type().is_file() { + return Err(integrity(format!( + "snapshot member metadata is not a regular file: {}", + path.display() + ))); + } + fs::remove_file(path)?; + sync_directory(directory)?; + } + Ok(()) + } + } +} + +fn write_json(directory: &Path, filename: &str, value: &impl Serialize) -> MicrosandboxResult<()> { + let bytes = serde_json::to_vec(value)?; + if bytes.len() > MAX_METADATA_BYTES { + return Err(integrity( + "snapshot group metadata exceeds its size limit".into(), + )); + } + let mut temporary = tempfile::Builder::new() + .prefix(".group-write-") + .tempfile_in(directory)?; + temporary.write_all(&bytes)?; + temporary.as_file().sync_all()?; + temporary + .persist(directory.join(filename)) + .map_err(|error| MicrosandboxError::from(error.error))?; + sync_directory(directory)?; + Ok(()) +} + +fn read_regular(path: &Path, maximum: usize) -> MicrosandboxResult> { + let metadata = fs::symlink_metadata(path)?; + if !metadata.file_type().is_file() || metadata.len() > maximum as u64 { + return Err(integrity(format!( + "snapshot metadata is not a bounded regular file: {}", + path.display() + ))); + } + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + options.custom_flags(libc::O_NOFOLLOW); + let file = options.open(path)?; + if !file.metadata()?.is_file() { + return Err(integrity(format!( + "snapshot metadata is not a regular file: {}", + path.display() + ))); + } + let mut bytes = Vec::new(); + file.take(maximum as u64 + 1).read_to_end(&mut bytes)?; + if bytes.len() > maximum { + return Err(integrity(format!( + "snapshot metadata exceeds its size limit: {}", + path.display() + ))); + } + Ok(bytes) +} + +fn require_directory(path: &Path) -> MicrosandboxResult<()> { + if !fs::symlink_metadata(path)?.file_type().is_dir() { + return Err(integrity(format!( + "snapshot group path is not a regular directory: {}", + path.display() + ))); + } + Ok(()) +} + +fn path_exists(path: &Path) -> MicrosandboxResult { + match fs::symlink_metadata(path) { + Ok(_) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error.into()), + } +} + +fn integrity(message: String) -> MicrosandboxError { + MicrosandboxError::SnapshotIntegrity(message) +} + +#[cfg(unix)] +fn sync_directory(path: &Path) -> std::io::Result<()> { + File::open(path)?.sync_all() +} + +#[cfg(windows)] +fn sync_directory(_path: &Path) -> std::io::Result<()> { + // Match artifact publication: payloads and metadata are flushed, directory rename is atomic. + Ok(()) +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +#[path = "group_tests.rs"] +mod tests; diff --git a/sdk/rust/lib/snapshot/group_tests.rs b/sdk/rust/lib/snapshot/group_tests.rs new file mode 100644 index 000000000..25892843c --- /dev/null +++ b/sdk/rust/lib/snapshot/group_tests.rs @@ -0,0 +1,515 @@ +//! Group publication and selector tests using small, complete file-state artifacts. + +use microsandbox_image::snapshot::{ + DiskLayer, DiskLayerId, FileSnapshotState, ImageRef, LayerFileKind, LayerPayload, SCHEMA, + SnapshotCapture, SnapshotConsistency, SnapshotFormat, SnapshotRootDisk, SnapshotScope, + SnapshotState, +}; + +use super::*; + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +fn id(value: u128) -> SnapshotId { + SnapshotId::new(format!("snap_{value:032x}")).unwrap() +} + +fn descriptor(value: u128, parent: Option) -> Manifest { + let layer_id = DiskLayerId::new(format!("layer_{value:032x}")).unwrap(); + Manifest { + schema: SCHEMA.into(), + snapshot_id: id(value), + scope: SnapshotScope::Disk, + state: SnapshotState::File(FileSnapshotState { + disk_format: SnapshotFormat::Raw, + filesystem: "ext4".into(), + virtual_size: 4, + head: layer_id.clone(), + layers: vec![DiskLayer { + layer_id, + format: SnapshotFormat::Raw, + virtual_size: 4, + backing: None, + payload: LayerPayload { + file_kind: LayerFileKind::Regular, + integrity: None, + }, + }], + }), + capture: SnapshotCapture { + created_at: "2026-09-10T00:00:00Z".into(), + source_lineage: Some("source".into()), + source_checkpoint: None, + consistency: SnapshotConsistency::CrashConsistent, + }, + image: ImageRef { + reference: "docker.io/library/alpine:latest".into(), + manifest_digest: format!("sha256:{}", "a".repeat(64)), + }, + root_disk: SnapshotRootDisk::Managed, + parent: parent.map(id), + requires: Vec::new(), + extensions: BTreeMap::new(), + } +} + +fn stage(root: &Path, manifests: &[Manifest]) -> tempfile::TempDir { + let staging = tempfile::Builder::new() + .prefix(".stage-") + .tempdir_in(root) + .unwrap(); + for manifest in manifests { + let directory = staging.path().join(manifest.snapshot_id.as_str()); + fs::create_dir(&directory).unwrap(); + fs::write( + directory.join(DESCRIPTOR_FILENAME), + manifest.to_canonical_bytes().unwrap(), + ) + .unwrap(); + let SnapshotState::File(state) = &manifest.state else { + unreachable!(); + }; + let layer = directory.join(state.layer_path(&state.layers[0])); + fs::create_dir_all(layer.parent().unwrap()).unwrap(); + fs::write(layer, [0u8; 4]).unwrap(); + } + staging +} + +async fn add(group: &Path, value: u128, parent: Option) -> HeadUpdate { + let staging = stage(group.parent().unwrap(), &[descriptor(value, parent)]); + publish(group, staging.path(), &BTreeMap::new(), &id(value), false) + .await + .unwrap() +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[tokio::test] +async fn initializes_and_fast_forwards_through_multiple_imported_ancestors() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("work")).await.unwrap(); + let first = add(&group, 10, None).await; + assert_eq!(first.reason, HeadUpdateReason::Initialized); + assert_eq!(first.previous, None); + + // Directory order puts the tip first. The designated candidate determines the head. + let staging = stage( + root.path(), + &[descriptor(5, Some(20)), descriptor(20, Some(10))], + ); + let update = publish(&group, staging.path(), &BTreeMap::new(), &id(5), false) + .await + .unwrap(); + assert_eq!(update.reason, HeadUpdateReason::FastForwarded); + assert_eq!(update.previous.as_deref(), Some(id(10).as_str())); + assert_eq!( + resolve(root.path(), "work").await.unwrap(), + group.join(id(5).as_str()) + ); + assert!(group.join(id(20).as_str()).is_dir()); +} + +#[tokio::test] +async fn resolved_identity_stays_fixed_after_the_group_head_advances() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("stable")).await.unwrap(); + add(&group, 1, None).await; + let selected = resolve(root.path(), "stable").await.unwrap(); + add(&group, 2, Some(1)).await; + assert_eq!(selected, group.join(id(1).as_str())); + assert_eq!(read_member(&selected, true).unwrap().0, id(1).as_str()); + assert_eq!( + resolve(root.path(), "stable").await.unwrap(), + group.join(id(2).as_str()) + ); +} + +#[tokio::test] +async fn head_and_id_lookup_do_not_scan_unrelated_descriptors() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("lookup")).await.unwrap(); + add(&group, 1, None).await; + let unrelated = group.join(id(2).as_str()); + fs::create_dir(&unrelated).unwrap(); + fs::write( + unrelated.join(DESCRIPTOR_FILENAME), + "broken unrelated descriptor", + ) + .unwrap(); + assert_eq!( + resolve(root.path(), "lookup").await.unwrap(), + group.join(id(1).as_str()) + ); + assert_eq!( + resolve(root.path(), &format!("lookup:{}", id(1))) + .await + .unwrap(), + group.join(id(1).as_str()), + ); + assert_eq!( + select(root.path(), "lookup").await.unwrap().head, + id(1).as_str() + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn concurrent_siblings_keep_both_artifacts_and_only_one_advances() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("race")).await.unwrap(); + add(&group, 1, None).await; + let left = stage(root.path(), &[descriptor(2, Some(1))]); + let right = stage(root.path(), &[descriptor(3, Some(1))]); + let aliases = BTreeMap::new(); + let left_id = id(2); + let right_id = id(3); + let (left_result, right_result) = tokio::join!( + publish(&group, left.path(), &aliases, &left_id, false), + publish(&group, right.path(), &aliases, &right_id, false), + ); + let left_result = left_result.unwrap(); + let right_result = right_result.unwrap(); + assert_ne!(left_result.changed, right_result.changed); + let (winner, retained) = if left_result.changed { + (left_result, right_result) + } else { + (right_result, left_result) + }; + assert_eq!(winner.reason, HeadUpdateReason::FastForwarded); + assert_eq!(retained.reason, HeadUpdateReason::Diverged); + assert_eq!(retained.head, winner.head); + assert_eq!(select(root.path(), "race").await.unwrap().head, winner.head); + assert!(group.join(id(2).as_str()).is_dir()); + assert!(group.join(id(3).as_str()).is_dir()); +} + +#[tokio::test] +async fn unknown_history_is_retained_without_retroactively_selecting_a_tip() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("history")).await.unwrap(); + add(&group, 1, None).await; + let unknown = add(&group, 3, Some(2)).await; + assert_eq!(unknown.reason, HeadUpdateReason::UnknownAncestry); + assert_eq!(unknown.head, id(1).as_str()); + assert!(group.join(id(3).as_str()).is_dir()); + let intermediate = add(&group, 2, Some(1)).await; + assert_eq!(intermediate.head, id(2).as_str()); + assert_eq!( + select(root.path(), "history").await.unwrap().head, + id(2).as_str() + ); + + let staging = stage(root.path(), &[]); + let retried = publish(&group, staging.path(), &BTreeMap::new(), &id(3), false) + .await + .unwrap(); + assert_eq!(retried.reason, HeadUpdateReason::FastForwarded); + assert_eq!(retried.head, id(3).as_str()); +} + +#[tokio::test] +async fn identical_ids_reuse_members_and_conflicts_fail_before_publication() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("duplicates")).await.unwrap(); + add(&group, 1, None).await; + fs::write(group.join(id(1).as_str()).join("keep"), "unchanged").unwrap(); + let duplicate = add(&group, 1, None).await; + assert!(!duplicate.changed); + assert_eq!(duplicate.reason, HeadUpdateReason::Unchanged); + assert_eq!( + fs::read_to_string(group.join(id(1).as_str()).join("keep")).unwrap(), + "unchanged" + ); + + let mut conflicting = descriptor(1, None); + conflicting.capture.source_lineage = Some("another-source".into()); + let staging = stage(root.path(), &[descriptor(2, Some(1)), conflicting]); + let error = publish(&group, staging.path(), &BTreeMap::new(), &id(2), false) + .await + .unwrap_err(); + assert!(error.to_string().contains("different descriptor")); + assert!(!group.join(id(2).as_str()).exists()); + assert!(staging.path().join(id(2).as_str()).is_dir()); + assert_eq!( + select(root.path(), "duplicates").await.unwrap().head, + id(1).as_str() + ); +} + +#[tokio::test] +async fn aliases_are_local_and_all_conflicts_are_checked_before_moving_members() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("aliases")).await.unwrap(); + let staging = stage(root.path(), &[descriptor(1, None)]); + let aliases = BTreeMap::from([(id(1).to_string(), "clean".into())]); + publish(&group, staging.path(), &aliases, &id(1), false) + .await + .unwrap(); + assert_eq!( + resolve(root.path(), "aliases:clean").await.unwrap(), + group.join(id(1).as_str()) + ); + assert_eq!( + member_name(&group.join(id(1).as_str())).unwrap().as_deref(), + Some("clean") + ); + assert_eq!(group_path(&group.join(id(1).as_str())), Some(group.clone())); + + let staging = stage( + root.path(), + &[descriptor(2, Some(1)), descriptor(3, Some(1))], + ); + let aliases = BTreeMap::from([ + (id(2).to_string(), "other".into()), + (id(3).to_string(), "clean".into()), + ]); + let error = publish(&group, staging.path(), &aliases, &id(2), false) + .await + .unwrap_err(); + assert!(error.to_string().contains("conflicts")); + assert!(!group.join(id(2).as_str()).exists()); + assert!(!group.join(id(3).as_str()).exists()); + assert!(staging.path().join(id(2).as_str()).is_dir()); +} + +#[tokio::test] +async fn generated_names_retry_publication_without_recapturing_the_artifact() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("generated")).await.unwrap(); + let first = stage(root.path(), &[descriptor(1, None)]); + let aliases = BTreeMap::from([(id(1).to_string(), "msb-00000001".into())]); + publish(&group, first.path(), &aliases, &id(1), false) + .await + .unwrap(); + + let captured = stage(root.path(), &[descriptor(2, Some(1))]); + let staged_member = captured.path().join(id(2).as_str()); + let original_descriptor = fs::read(staged_member.join(DESCRIPTOR_FILENAME)).unwrap(); + fs::write(staged_member.join("capture-marker"), b"same capture").unwrap(); + let mut retries = 0; + let update = super::super::create::publish_with_name_retry( + &group, + captured.path(), + &id(2), + "msb-00000001".into(), + true, + || { + retries += 1; + // The conflict was detected before moving or rewriting any captured state. + assert_eq!( + fs::read(staged_member.join(DESCRIPTOR_FILENAME)).unwrap(), + original_descriptor + ); + assert_eq!( + fs::read(staged_member.join("capture-marker")).unwrap(), + b"same capture" + ); + "msb-00000002".into() + }, + ) + .await + .unwrap(); + assert_eq!(retries, 1); + assert_eq!(update.reason, HeadUpdateReason::FastForwarded); + let installed = group.join(id(2).as_str()); + assert_eq!( + member_name(&installed).unwrap().as_deref(), + Some("msb-00000002") + ); + assert_eq!( + fs::read(installed.join(DESCRIPTOR_FILENAME)).unwrap(), + original_descriptor + ); + assert_eq!( + fs::read(installed.join("capture-marker")).unwrap(), + b"same capture" + ); + assert_eq!( + member_name(&group.join(id(1).as_str())).unwrap().as_deref(), + Some("msb-00000001") + ); +} + +#[tokio::test] +async fn explicit_names_report_collision_without_retry_or_staging_changes() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("explicit")).await.unwrap(); + let first = stage(root.path(), &[descriptor(1, None)]); + let aliases = BTreeMap::from([(id(1).to_string(), "chosen".into())]); + publish(&group, first.path(), &aliases, &id(1), false) + .await + .unwrap(); + let captured = stage(root.path(), &[descriptor(2, Some(1))]); + let error = super::super::create::publish_with_name_retry( + &group, + captured.path(), + &id(2), + "chosen".into(), + false, + || panic!("explicit names must not be regenerated"), + ) + .await + .unwrap_err(); + assert!(matches!(error, MicrosandboxError::SnapshotAlreadyExists(_))); + assert!( + captured + .path() + .join(id(2).as_str()) + .join(DESCRIPTOR_FILENAME) + .is_file() + ); + assert!(!group.join(id(2).as_str()).exists()); + assert_eq!( + read_group(&group).unwrap().head.as_deref(), + Some(id(1).as_str()) + ); +} + +#[tokio::test] +async fn explicit_selection_can_choose_a_retained_branch_or_an_older_snapshot() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("selection")).await.unwrap(); + add(&group, 1, None).await; + add(&group, 2, Some(1)).await; + assert_eq!( + add(&group, 3, Some(1)).await.reason, + HeadUpdateReason::Diverged + ); + let side = select(root.path(), &format!("selection:{}", id(3))) + .await + .unwrap(); + assert_eq!(side.reason, HeadUpdateReason::Selected); + assert_eq!(side.previous.as_deref(), Some(id(2).as_str())); + assert_eq!(side.head, id(3).as_str()); + let old = select(root.path(), &format!("selection:{}", id(1))) + .await + .unwrap(); + assert_eq!(old.head, id(1).as_str()); + assert_eq!( + select(root.path(), "selection").await.unwrap().reason, + HeadUpdateReason::Unchanged + ); +} + +#[tokio::test] +async fn removing_a_head_requires_selection_unless_it_is_the_final_member() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("removal")).await.unwrap(); + add(&group, 1, None).await; + add(&group, 2, Some(1)).await; + let error = remove_member(&group.join(id(2).as_str())) + .await + .unwrap_err(); + assert!(error.to_string().contains("first select another")); + assert!(group.join(id(2).as_str()).is_dir()); + assert!(remove_member(&group.join(id(1).as_str())).await.unwrap()); + assert!(remove_member(&group.join(id(2).as_str())).await.unwrap()); + assert_eq!(read_group(&group).unwrap().head, None); + assert!( + resolve(root.path(), "removal") + .await + .unwrap_err() + .to_string() + .contains("has no head") + ); + assert_eq!( + add(&group, 3, None).await.reason, + HeadUpdateReason::Initialized + ); +} + +#[tokio::test] +async fn cycles_are_rejected_even_when_a_group_has_no_head() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("cycle")).await.unwrap(); + let staging = stage( + root.path(), + &[descriptor(1, Some(2)), descriptor(2, Some(1))], + ); + let error = publish(&group, staging.path(), &BTreeMap::new(), &id(1), false) + .await + .unwrap_err(); + assert!(error.to_string().contains("cycle")); + assert_eq!(read_group(&group).unwrap().head, None); + assert!(!group.join(id(1).as_str()).exists()); +} + +#[tokio::test] +async fn generated_groups_are_fresh_and_flat_directories_are_never_migrated() { + let root = tempfile::tempdir().unwrap(); + let (first, second) = tokio::join!(ensure(root.path(), None), ensure(root.path(), None)); + let first = first.unwrap(); + let second = second.unwrap(); + assert_ne!(first, second); + assert!( + first + .file_name() + .unwrap() + .to_str() + .unwrap() + .starts_with("msb-") + ); + let flat = root.path().join("flat"); + fs::create_dir(&flat).unwrap(); + fs::write(flat.join("keep"), "untouched").unwrap(); + let error = ensure(root.path(), Some("flat")).await.unwrap_err(); + assert!(error.to_string().contains("explicit path")); + assert!(!flat.join(GROUP_FILENAME).exists()); + assert_eq!(fs::read_to_string(flat.join("keep")).unwrap(), "untouched"); +} + +#[tokio::test] +async fn selectors_and_group_names_cannot_escape_the_store() { + let root = tempfile::tempdir().unwrap(); + for name in [ + "", + ".", + "..", + "../escape", + "a:b", + "a\\b", + "con", + "trailing.", + ] { + assert!( + ensure(root.path(), Some(name)).await.is_err(), + "accepted {name}" + ); + } + for selector in ["valid:", "valid:../escape", "valid:a:b", "../escape:name"] { + assert!( + resolve(root.path(), selector).await.is_err(), + "accepted {selector}" + ); + } +} + +#[cfg(unix)] +#[tokio::test] +async fn symlinked_group_and_descriptor_paths_are_rejected() { + use std::os::unix::fs::symlink; + + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("real")).await.unwrap(); + symlink(&group, root.path().join("redirect")).unwrap(); + assert!(ensure(root.path(), Some("redirect")).await.is_err()); + let staging = stage(root.path(), &[descriptor(1, None)]); + let path = staging + .path() + .join(id(1).as_str()) + .join(DESCRIPTOR_FILENAME); + let external = root.path().join("external.json"); + fs::rename(&path, &external).unwrap(); + symlink(&external, &path).unwrap(); + assert!( + publish(&group, staging.path(), &BTreeMap::new(), &id(1), false) + .await + .is_err() + ); + assert!(!group.join(id(1).as_str()).exists()); + assert!(external.is_file()); +} diff --git a/sdk/rust/lib/snapshot/lineage.rs b/sdk/rust/lib/snapshot/lineage.rs new file mode 100644 index 000000000..988f90e52 --- /dev/null +++ b/sdk/rust/lib/snapshot/lineage.rs @@ -0,0 +1,357 @@ +//! Capture ancestry independent of group names, dirty tracking, and export dependencies. + +use std::fs::File; +use std::io::Write; +use std::path::PathBuf; +use std::sync::Arc; + +use sea_orm::{ColumnTrait, EntityTrait, QueryFilter}; +use serde::{Deserialize, Serialize}; + +use crate::backend::LocalBackend; +use crate::db::entity::sandbox; +use crate::sandbox::SandboxConfig; +use crate::{MicrosandboxError, MicrosandboxResult}; + +use super::SnapshotId; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Cursor { + sandbox_id: i32, + snapshot_id: String, +} + +/// Holds the per-source capture sequencer without holding a VM pause or database transaction. +pub(crate) struct CaptureLineage { + _lock: Arc, + path: PathBuf, + sandbox_id: i32, + pub(crate) parent: Option, +} + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl CaptureLineage { + /// Source incarnation whose ancestry is protected by this sequencer. + pub(crate) fn sandbox_id(&self) -> i32 { + self.sandbox_id + } + + /// Refuse a completed capture if its named source was removed or replaced meanwhile. + pub(crate) async fn validate_source( + &self, + local: &LocalBackend, + name: &str, + ) -> MicrosandboxResult<()> { + let current = sandbox::Entity::find() + .filter(sandbox::Column::Name.eq(name)) + .one(local.db().await?.read()) + .await?; + if current + .as_ref() + .is_none_or(|model| model.id != self.sandbox_id) + { + return Err(MicrosandboxError::InvalidConfig( + "source sandbox changed during snapshot capture".into(), + )); + } + Ok(()) + } + + /// Advance only after the artifact/archive has been successfully published. + pub(crate) async fn commit(&self, snapshot_id: &SnapshotId) -> MicrosandboxResult<()> { + let path = self.path.clone(); + // A cancelled awaiting task must not release the source sequencer while its blocking + // publication still runs, otherwise an older cursor could replace a newer capture. + let lock = Arc::clone(&self._lock); + let cursor = Cursor { + sandbox_id: self.sandbox_id, + snapshot_id: snapshot_id.to_string(), + }; + tokio::task::spawn_blocking(move || -> MicrosandboxResult<()> { + let _lock = lock; + let parent = path.parent().expect("cursor has a sandbox directory"); + let mut staged = tempfile::NamedTempFile::new_in(parent)?; + staged.write_all(&serde_json::to_vec(&cursor)?)?; + staged.as_file().sync_all()?; + staged + .persist(&path) + .map_err(|error| MicrosandboxError::Io(error.error))?; + #[cfg(unix)] + File::open(parent)?.sync_all()?; + Ok(()) + }) + .await + .map_err(|error| { + MicrosandboxError::Runtime(format!("snapshot ancestry publication: {error}")) + })? + } +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +pub(crate) async fn begin(local: &LocalBackend, name: &str) -> MicrosandboxResult { + let model = sandbox::Entity::find() + .filter(sandbox::Column::Name.eq(name)) + .one(local.db().await?.read()) + .await? + .ok_or_else(|| MicrosandboxError::SandboxNotFound(name.into()))?; + let expected_id = model.id; + let directory = local.sandboxes_dir().join(name); + let (lock, cursor) = tokio::task::spawn_blocking(move || -> MicrosandboxResult<_> { + // Never recreate a removed sandbox merely to record ancestry. + let lock = microsandbox_utils::process_lock::open_lock_file( + &directory.join(".snapshot-lineage.lock"), + )?; + microsandbox_utils::process_lock::lock_exclusive(&lock)?; + let path = directory.join("snapshot-lineage.json"); + let cursor = match std::fs::symlink_metadata(&path) { + Ok(meta) if meta.is_file() && meta.len() <= 4096 => { + Some(serde_json::from_slice::(&std::fs::read(&path)?)?) + } + Ok(_) => { + return Err(MicrosandboxError::SnapshotIntegrity( + "invalid snapshot ancestry cursor".into(), + )); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(error.into()), + }; + Ok((lock, (path, cursor))) + }) + .await + .map_err(|error| MicrosandboxError::Runtime(format!("snapshot ancestry lock: {error}")))??; + let current = sandbox::Entity::find() + .filter(sandbox::Column::Name.eq(name)) + .one(local.db().await?.read()) + .await? + .ok_or_else(|| MicrosandboxError::SandboxNotFound(name.into()))?; + if current.id != expected_id { + return Err(MicrosandboxError::InvalidConfig( + "source sandbox changed while waiting for capture".into(), + )); + } + let config: SandboxConfig = + serde_json::from_str(current.active_config.as_deref().unwrap_or(¤t.config))?; + let parent = match cursor.1 { + Some(cursor) if cursor.sandbox_id == current.id => Some(cursor.snapshot_id), + Some(_) => { + return Err(MicrosandboxError::SnapshotIntegrity( + "snapshot ancestry belongs to another sandbox instance".into(), + )); + } + None => config.snapshot_parent, + } + .map(SnapshotId::new) + .transpose() + .map_err(|error| MicrosandboxError::SnapshotIntegrity(error.to_string()))?; + Ok(CaptureLineage { + _lock: Arc::new(lock), + path: cursor.0, + sandbox_id: current.id, + parent, + }) +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use sea_orm::{ActiveModelTrait, ActiveValue::Set}; + + use super::*; + + async fn source(local: &LocalBackend, name: &str, origin: Option<&SnapshotId>) -> i32 { + let mut config = SandboxConfig::default(); + config.spec.name = name.into(); + config.snapshot_parent = origin.map(ToString::to_string); + std::fs::create_dir_all(local.sandboxes_dir().join(name)).unwrap(); + sandbox::ActiveModel { + name: Set(name.into()), + config: Set(serde_json::to_string(&config).unwrap()), + status: Set(sandbox::SandboxStatus::Stopped), + ephemeral: Set(false), + ..Default::default() + } + .insert(local.db().await.unwrap().write()) + .await + .unwrap() + .id + } + + fn id(value: u128) -> SnapshotId { + SnapshotId::new(format!("snap_{value:032x}")).unwrap() + } + + #[test] + fn cancelled_cursor_wait_retains_lock_until_blocking_publication_finishes() { + // Hold the sole blocking worker so cancellation deterministically lands after commit + // queues publication but before that publication can touch the cursor. + let runtime = tokio::runtime::Builder::new_current_thread() + .max_blocking_threads(1) + .enable_all() + .build() + .unwrap(); + runtime.block_on(async { + let directory = tempfile::tempdir().unwrap(); + let lock_path = directory.path().join(".snapshot-lineage.lock"); + let lock = microsandbox_utils::process_lock::open_lock_file(&lock_path).unwrap(); + microsandbox_utils::process_lock::lock_exclusive(&lock).unwrap(); + let lineage = CaptureLineage { + _lock: Arc::new(lock), + path: directory.path().join("snapshot-lineage.json"), + sandbox_id: 1, + parent: None, + }; + let (release, wait_release) = std::sync::mpsc::channel(); + let (started, wait_started) = tokio::sync::oneshot::channel(); + let blocker = tokio::task::spawn_blocking(move || { + started.send(()).unwrap(); + wait_release.recv().unwrap(); + }); + wait_started.await.unwrap(); + let snapshot = id(4); + let mut publication = Box::pin(lineage.commit(&snapshot)); + assert!(futures::poll!(&mut publication).is_pending()); + drop(publication); + drop(lineage); + let observer = + microsandbox_utils::process_lock::open_existing_lock_file(&lock_path).unwrap(); + let retained = + !microsandbox_utils::process_lock::try_lock_exclusive(&observer).unwrap(); + // Release before asserting so a failed test cannot strand its runtime worker. + release.send(()).unwrap(); + blocker.await.unwrap(); + assert!( + retained, + "cancelled await released an in-flight cursor publication lock" + ); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + while !microsandbox_utils::process_lock::try_lock_exclusive(&observer).unwrap() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let cursor: Cursor = serde_json::from_slice( + &std::fs::read(directory.path().join("snapshot-lineage.json")).unwrap(), + ) + .unwrap(); + assert_eq!(cursor.snapshot_id, snapshot.as_str()); + }); + } + + #[tokio::test] + async fn restored_origin_advances_only_after_successful_publication() { + let home = tempfile::tempdir().unwrap(); + let local = LocalBackend::builder() + .home(home.path()) + .build() + .await + .unwrap(); + let origin = id(1); + let captured = id(2); + let source_id = source(&local, "worker", Some(&origin)).await; + let failed = begin(&local, "worker").await.unwrap(); + assert_eq!(failed.sandbox_id(), source_id); + assert_eq!(failed.parent.as_ref(), Some(&origin)); + drop(failed); + let successful = begin(&local, "worker").await.unwrap(); + assert_eq!(successful.parent.as_ref(), Some(&origin)); + successful.commit(&captured).await.unwrap(); + drop(successful); + assert_eq!( + begin(&local, "worker").await.unwrap().parent, + Some(captured) + ); + } + + #[tokio::test] + async fn same_named_sources_in_different_backends_keep_separate_ancestry() { + let first_home = tempfile::tempdir().unwrap(); + let second_home = tempfile::tempdir().unwrap(); + let first = LocalBackend::builder() + .home(first_home.path()) + .build() + .await + .unwrap(); + let second = LocalBackend::builder() + .home(second_home.path()) + .build() + .await + .unwrap(); + source(&first, "worker", Some(&id(1))).await; + source(&second, "worker", Some(&id(2))).await; + let capture = begin(&first, "worker").await.unwrap(); + capture.commit(&id(3)).await.unwrap(); + drop(capture); + assert_eq!(begin(&first, "worker").await.unwrap().parent, Some(id(3))); + assert_eq!(begin(&second, "worker").await.unwrap().parent, Some(id(2))); + } + + #[tokio::test] + async fn cursor_from_a_different_source_incarnation_is_rejected() { + let home = tempfile::tempdir().unwrap(); + let local = LocalBackend::builder() + .home(home.path()) + .build() + .await + .unwrap(); + let source_id = source(&local, "worker", None).await; + std::fs::write( + local.sandboxes_dir().join("worker/snapshot-lineage.json"), + serde_json::to_vec(&Cursor { + sandbox_id: source_id + 1, + snapshot_id: id(1).to_string(), + }) + .unwrap(), + ) + .unwrap(); + let error = begin(&local, "worker") + .await + .err() + .expect("wrong incarnation must fail"); + assert!(error.to_string().contains("another sandbox instance")); + } + + #[tokio::test] + async fn completed_capture_accepts_status_changes_but_rejects_replacement() { + let home = tempfile::tempdir().unwrap(); + let local = LocalBackend::builder() + .home(home.path()) + .build() + .await + .unwrap(); + let original = source(&local, "worker", None).await; + let lineage = begin(&local, "worker").await.unwrap(); + sandbox::Entity::update_many() + .col_expr( + sandbox::Column::Status, + sea_orm::sea_query::Expr::value("Crashed"), + ) + .filter(sandbox::Column::Id.eq(original)) + .exec(local.db().await.unwrap().write()) + .await + .unwrap(); + lineage.validate_source(&local, "worker").await.unwrap(); + sandbox::Entity::delete_by_id(original) + .exec(local.db().await.unwrap().write()) + .await + .unwrap(); + assert!(lineage.validate_source(&local, "worker").await.is_err()); + let replacement = source(&local, "worker", None).await; + assert_ne!(original, replacement); + assert!(lineage.validate_source(&local, "worker").await.is_err()); + } +} diff --git a/sdk/rust/lib/snapshot/migration.rs b/sdk/rust/lib/snapshot/migration.rs index 6cacb04b7..83f2ddff5 100644 --- a/sdk/rust/lib/snapshot/migration.rs +++ b/sdk/rust/lib/snapshot/migration.rs @@ -831,17 +831,8 @@ async fn publish_index_component( transaction .execute_raw(Statement::from_sql_and_values( DatabaseBackend::Sqlite, - "DELETE FROM snapshot_index WHERE digest = ? OR artifact_path = ?", - [ - candidate - .inspected - .pinned - .source - .source_digest - .clone() - .into(), - path.clone().into(), - ], + "DELETE FROM snapshot_index WHERE artifact_path = ?", + [path.clone().into()], )) .await?; insert_canonical_index_row(&transaction, candidate).await?; @@ -859,7 +850,7 @@ async fn publish_index_component( } transaction .execute_unprepared( - "UPDATE snapshot_index SET child_count = (SELECT COUNT(*) FROM snapshot_index child WHERE child.parent_digest = snapshot_index.snapshot_id)", + "UPDATE snapshot_index SET child_count = (SELECT COUNT(DISTINCT COALESCE(child.snapshot_id, child.digest)) FROM snapshot_index child WHERE child.parent_digest = snapshot_index.snapshot_id)", ) .await?; transaction.commit().await?; diff --git a/sdk/rust/lib/snapshot/mod.rs b/sdk/rust/lib/snapshot/mod.rs index 511662b9e..c1a595971 100644 --- a/sdk/rust/lib/snapshot/mod.rs +++ b/sdk/rust/lib/snapshot/mod.rs @@ -9,6 +9,8 @@ mod archive; mod create; #[doc(hidden)] pub mod downgrade; +pub(crate) mod group; +pub(crate) mod lineage; mod metadata; pub(crate) mod migration; mod restore; @@ -36,6 +38,7 @@ pub struct Snapshot { digest: String, manifest: Manifest, labels: BTreeMap, + head_update: Option, } /// Result of direct sandbox-to-archive capture. @@ -56,6 +59,7 @@ pub struct SnapshotArchive { /// [`from_sandbox`](Self::from_sandbox) and is required. pub struct SnapshotBuilder { name: String, + group: Option, source_sandbox: Option, dest_dir: Option, labels: Vec<(String, String)>, @@ -84,6 +88,7 @@ impl Snapshot { pub fn builder(name: impl Into) -> SnapshotBuilder { SnapshotBuilder { name: name.into(), + group: None, source_sandbox: None, dest_dir: None, labels: Vec::new(), @@ -116,10 +121,10 @@ impl Snapshot { create::create_snapshot_archive(local, config, out.as_ref(), plain_tar).await } - /// Open an existing snapshot artifact by path or bare name. + /// Open an existing snapshot by explicit path, group head, or `group:member`. /// - /// Bare names (no path separator) resolve under the default - /// snapshots directory; anything else is treated as a path. + /// Bare names select the group's head under the default snapshots directory. + /// An exact member selector or explicit path remains fixed if that head advances. /// This is a fast metadata operation: it verifies the manifest /// structure, recomputes the manifest digest, and checks that the /// upper file exists with the recorded size. It does not read the @@ -160,6 +165,11 @@ impl Snapshot { &self.labels } + /// Group publication outcome, present on a newly captured snapshot. + pub fn head_update(&self) -> Option<&HeadUpdate> { + self.head_update.as_ref() + } + /// Apparent size of a file-state upper layer in bytes. pub fn size_bytes(&self) -> Option { self.manifest @@ -274,6 +284,23 @@ impl Snapshot { let local = backend.as_local().ok_or_else(snapshots_require_local)?; archive::load_snapshot_with_base(local, archive_path, dest, Some(base)).await } + + /// Import into a selected or newly generated group with explicit dependency/head policy. + pub async fn load_with_options( + archive_path: &Path, + opts: LoadOpts, + ) -> MicrosandboxResult { + let backend = crate::backend::default_backend(); + let local = backend.as_local().ok_or_else(snapshots_require_local)?; + archive::load_snapshot_with_options(local, archive_path, opts).await + } + + /// Read a group's head, or explicitly select a qualified `group:member`. + pub async fn group_head(selector: &str) -> MicrosandboxResult { + let backend = crate::backend::default_backend(); + let local = backend.as_local().ok_or_else(snapshots_require_local)?; + group::select(&local.snapshots_dir(), selector).await + } } impl SnapshotArchive { @@ -335,6 +362,8 @@ pub(crate) use create::CHECKPOINT_DIRECTORY; /// content verification. #[derive(Debug, Clone)] pub struct SnapshotHandle { + pub(crate) group: Option, + pub(crate) head_update: Option, pub(crate) snapshot_id: String, pub(crate) digest: String, pub(crate) name: Option, @@ -355,6 +384,14 @@ pub struct SnapshotHandle { } impl SnapshotHandle { + /// Group publication outcome, present when this handle was returned by import. + pub fn head_update(&self) -> Option<&HeadUpdate> { + self.head_update.as_ref() + } + /// Local group containing this installed copy, if any. + pub fn group(&self) -> Option<&str> { + self.group.as_deref() + } /// Stable opaque snapshot identity. pub fn id(&self) -> &str { &self.snapshot_id @@ -447,20 +484,25 @@ impl SnapshotHandle { /// Remove this snapshot. See [`Snapshot::remove`]. pub async fn remove(&self, force: bool) -> MicrosandboxResult<()> { - Snapshot::remove(&self.digest, force).await + // A handle denotes this installed copy, not every copy of its portable identity. + Snapshot::remove(self.artifact_path.to_string_lossy().as_ref(), force).await } } impl SnapshotBuilder { + /// Place the new member in this local snapshot group. + pub fn group(mut self, group: impl Into) -> Self { + self.group = Some(group.into()); + self + } /// Set the source sandbox to snapshot. Required. pub fn from_sandbox(mut self, source_sandbox: impl Into) -> Self { self.source_sandbox = Some(source_sandbox.into()); self } - /// Create the artifact under this parent directory instead of the - /// default snapshots store. The artifact directory is - /// `dest_dir/`; the name stays the snapshot's identity. + /// Use this group-store root instead of the default snapshots directory. + /// The artifact is installed at `dest_dir//`. pub fn dest_dir(mut self, dest_dir: impl Into) -> Self { self.dest_dir = Some(dest_dir.into()); self @@ -472,7 +514,7 @@ impl SnapshotBuilder { self } - /// Overwrite an existing artifact at the destination. + /// Overwrite a direct archive destination. Installed group members are immutable. pub fn force(mut self) -> Self { self.force = true; self @@ -503,6 +545,7 @@ impl SnapshotBuilder { })?; Ok(SnapshotConfig { name: self.name, + group: self.group, dest_dir: self.dest_dir, source_sandbox, labels: self.labels, @@ -531,9 +574,10 @@ impl SnapshotBuilder { // Re-Exports //-------------------------------------------------------------------------------------------------- -pub use archive::SaveOpts; #[cfg(feature = "fuzzing")] pub use archive::fuzz_unpack_archive; +pub use archive::{LoadOpts, SaveOpts}; +pub use group::{HeadUpdate, HeadUpdateReason}; pub use microsandbox_image::snapshot::{ CheckpointSnapshotState, DESCRIPTOR_FILENAME, DiskLayer, DiskLayerId, FileSnapshotState, ImageRef, LayerFileKind, LayerPayload, Manifest, SnapshotCapture, SnapshotConsistency, @@ -559,6 +603,7 @@ impl Snapshot { digest, manifest, labels, + head_update: None, } } } diff --git a/sdk/rust/lib/snapshot/store.rs b/sdk/rust/lib/snapshot/store.rs index ac918cca8..62988fe62 100644 --- a/sdk/rust/lib/snapshot/store.rs +++ b/sdk/rust/lib/snapshot/store.rs @@ -24,9 +24,8 @@ use super::{Snapshot, SnapshotFormat, SnapshotHandle, SnapshotScope, UpperIntegr /// Open and validate snapshot artifact metadata. /// -/// `path_or_name` is treated as a path if it contains `/` or starts -/// with `.` or `~`; otherwise as a bare name resolved under the -/// passed-in `local` backend's snapshots directory. +/// Explicit paths remain valid. Bare selectors resolve a group's head; qualified selectors +/// resolve a group member. Global portable identities must identify exactly one local copy. pub(super) async fn open_snapshot( local: &LocalBackend, path_or_name: &str, @@ -37,11 +36,7 @@ pub(super) async fn open_snapshot( )); } - let dir = if looks_like_path(path_or_name) { - PathBuf::from(path_or_name) - } else { - local.snapshots_dir().join(path_or_name) - }; + let dir = resolve_path(local, path_or_name).await?; if !dir.exists() { return Err(MicrosandboxError::SnapshotNotFound( @@ -129,14 +124,22 @@ pub(super) async fn open_snapshot( let labels = super::metadata::read(&dir, &manifest, translated_labels).await?; let snap = Snapshot::from_parts(dir.clone(), digest.clone(), manifest, labels); - // Opportunistic auto-reindex: if the artifact lives under the - // configured snapshots dir but its digest isn't in the local - // index, insert it. Keeps the cache aligned with reality without - // forcing the user to think about it. Best-effort — errors are - // logged, not propagated. + // Published managed members and explicitly opened flat artifacts remain discoverable for + // parent traversal. Archive/capture staging must never replace durable index entries. let snapshots_dir = local.snapshots_dir(); - if dir.parent() == Some(snapshots_dir.as_path()) - && let Ok(None) = lookup_by_digest(local, &digest).await + let managed = dir + .strip_prefix(&snapshots_dir) + .ok() + .is_some_and(|relative| { + relative + .components() + .all(|part| !part.as_os_str().to_string_lossy().starts_with('.')) + }); + if managed + && (super::group::group_path(&dir).is_some() + || dir.parent() == Some(snapshots_dir.as_path())) + && let Ok(existing) = indexed_path(local, &dir).await + && existing.as_ref().is_none_or(|row| row.digest != digest) && let Err(e) = index_upsert(local, snap.path(), snap.digest(), snap.manifest()).await { tracing::debug!(error = %e, snapshot = %digest, "auto-reindex skipped"); @@ -159,42 +162,31 @@ pub(super) async fn index_upsert( .unwrap_or_else(|_| Utc::now().naive_utc()); let indexed_at = Utc::now().naive_utc(); + let artifact_path = canonical_path(artifact_path); let artifact_path_str = artifact_path.display().to_string(); - let artifact_name = artifact_path - .file_name() - .and_then(|s| s.to_str()) - .map(|s| s.to_string()); - - // Delete any prior row for this digest, name, or path, then insert. - // This keeps the rebuildable index aligned when an artifact is - // replaced in-place or when a manifest rewrite changes its digest. - // The superseded rows' parent edges disappear with them, so their - // parents' child_count must come down first; the fresh insert re-adds - // its own edge below. + let group_path = super::group::group_path(&artifact_path); + let group_name = group_path + .as_ref() + .and_then(|path| path.file_name()) + .map(|name| name.to_string_lossy().into_owned()); + let artifact_name = super::group::member_name(&artifact_path)?.or_else(|| { + artifact_path + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + }); + let group_path = group_path.map(|path| path.display().to_string()); + + // Portable identities may occur in multiple groups. Replace only this local address, + // never another copy that happens to share descriptor bytes, identity, or member name. let mut supersede = sea_orm::Condition::any() - .add(snapshot_entity::Column::Digest.eq(digest.to_string())) - .add(snapshot_entity::Column::SnapshotId.eq(manifest.snapshot_id.to_string())) .add(snapshot_entity::Column::ArtifactPath.eq(artifact_path_str.clone())); - if let Some(name) = artifact_name.as_ref() { - supersede = supersede.add(snapshot_entity::Column::Name.eq(name.clone())); + if let (Some(group), Some(name)) = (&group_path, &artifact_name) { + supersede = supersede.add( + sea_orm::Condition::all() + .add(snapshot_entity::Column::GroupPath.eq(group.clone())) + .add(snapshot_entity::Column::Name.eq(name.clone())), + ); } - let superseded = snapshot_entity::Entity::find() - .filter(supersede.clone()) - .all(db) - .await?; - for row in &superseded { - if let Some(parent) = row.parent_digest.as_ref() { - db.execute_unprepared(&format!( - "UPDATE snapshot_index SET child_count = MAX(0, child_count - 1) WHERE snapshot_id = '{}'", - parent.replace('\'', "''") - )) - .await?; - } - } - snapshot_entity::Entity::delete_many() - .filter(supersede) - .exec(db) - .await?; let (state_kind, format, fstype, checkpoint_manifest_digest, size_bytes) = match &manifest.state { @@ -233,6 +225,8 @@ pub(super) async fn index_upsert( snapshot_id: Set(Some(manifest.snapshot_id.to_string())), descriptor_digest: Set(Some(digest.to_string())), name: Set(artifact_name), + group_name: Set(group_name), + group_path: Set(group_path), parent_digest: Set(manifest.parent.as_ref().map(ToString::to_string)), scope: Set(scope_str.into()), state_kind: Set(state_kind.into()), @@ -252,17 +246,20 @@ pub(super) async fn index_upsert( indexed_at: Set(indexed_at), child_count: Set(0), }; - row.insert(db).await?; - - // If this snapshot has a parent, bump the parent's child_count. - if let Some(parent) = manifest.parent.as_ref() { - use sea_orm::ConnectionTrait; - db.execute_unprepared(&format!( - "UPDATE snapshot_index SET child_count = child_count + 1 WHERE snapshot_id = '{}'", - parent.as_str().replace('\'', "''") - )) - .await?; - } + db.transaction::<_, _, _, sea_orm::DbErr>(|transaction| { + let row = row.clone(); + let supersede = supersede.clone(); + async move { + snapshot_entity::Entity::delete_many() + .filter(supersede) + .exec(&transaction) + .await?; + row.insert(&transaction).await?; + recompute_children(&transaction).await?; + Ok((transaction, ())) + } + }) + .await?; Ok(()) } @@ -308,11 +305,11 @@ pub(super) async fn list_dir( if !dir.exists() { return Ok(Vec::new()); } - let mut out = Vec::new(); + let mut candidates = Vec::new(); let mut entries = tokio::fs::read_dir(dir).await?; while let Some(entry) = entries.next_entry().await? { let path = entry.path(); - if !path.is_dir() { + if !entry.file_type().await?.is_dir() { continue; } // Dot-prefixed directories are never artifacts; create() stages @@ -325,6 +322,23 @@ pub(super) async fn list_dir( { continue; } + if path.join(super::group::GROUP_FILENAME).is_file() { + // Groups are exactly one level deep. Do not recursively walk arbitrary folders, + // symlink trees, checkpoint stores, or failed staging directories. + let mut members = tokio::fs::read_dir(&path).await?; + while let Some(member) = members.next_entry().await? { + if member.file_type().await?.is_dir() + && !member.file_name().to_string_lossy().starts_with('.') + { + candidates.push(member.path()); + } + } + } else { + candidates.push(path); + } + } + let mut out = Vec::new(); + for path in candidates { if !path.join(DESCRIPTOR_FILENAME).exists() && !path.join(V066_DESCRIPTOR_FILENAME).exists() { continue; @@ -348,41 +362,12 @@ pub(super) async fn remove_snapshot( let read_db = pools.read(); let write_db = pools.write(); - // Resolve the target row. Accept digest, name, or path. - let (digest, artifact_path) = if path_or_name.starts_with("snap_") { - let row = snapshot_entity::Entity::find() - .filter(snapshot_entity::Column::SnapshotId.eq(path_or_name.to_string())) - .one(read_db) - .await? - .ok_or_else(|| MicrosandboxError::SnapshotNotFound(path_or_name.into()))?; - (row.digest.clone(), PathBuf::from(row.artifact_path)) - } else if path_or_name.starts_with("sha256:") || path_or_name.starts_with("sha512:") { - let row = snapshot_entity::Entity::find_by_id(path_or_name.to_string()) - .one(read_db) - .await? - .ok_or_else(|| MicrosandboxError::SnapshotNotFound(path_or_name.into()))?; - (row.digest.clone(), PathBuf::from(row.artifact_path)) - } else if looks_like_path(path_or_name) { - // Path: open to read the digest, then drop both row and dir. - let snap = open_snapshot(local, path_or_name).await?; - (snap.digest.clone(), snap.path.clone()) - } else { - // Bare name: prefer the index lookup; fall back to default-dir resolution. - let row = snapshot_entity::Entity::find() - .filter(snapshot_entity::Column::Name.eq(path_or_name.to_string())) - .one(read_db) - .await?; - if let Some(row) = row { - (row.digest.clone(), PathBuf::from(row.artifact_path)) - } else { - let dir = local.snapshots_dir().join(path_or_name); - let snap = open_snapshot(local, dir.to_string_lossy().as_ref()).await?; - (snap.digest.clone(), snap.path.clone()) - } - }; + let snapshot = open_snapshot(local, path_or_name).await?; + let artifact_path = canonical_path(snapshot.path()); + let artifact_key = artifact_path.display().to_string(); // Check children unless --force. - let row = snapshot_entity::Entity::find_by_id(digest.clone()) + let row = snapshot_entity::Entity::find_by_id(artifact_key.clone()) .one(read_db) .await?; if let Some(ref row) = row @@ -391,28 +376,20 @@ pub(super) async fn remove_snapshot( { return Err(MicrosandboxError::Custom(format!( "snapshot {} has {} indexed child snapshot(s); pass --force to remove anyway", - digest, row.child_count + snapshot.id(), + row.child_count ))); } - // Drop the index row and decrement parent's child_count if any. - let parent = row.as_ref().and_then(|r| r.parent_digest.clone()); - snapshot_entity::Entity::delete_by_id(digest.clone()) - .exec(write_db) - .await?; - if let Some(p) = parent { - write_db - .execute_unprepared(&format!( - "UPDATE snapshot_index SET child_count = MAX(0, child_count - 1) WHERE snapshot_id = '{}'", - p.replace('\'', "''") - )) - .await?; - } - - // Delete the artifact directory. - if artifact_path.exists() { + // The group helper validates head removal and removes the member under its publication + // lock. Even --force must not leave a group's head dangling while other members remain. + if !super::group::remove_member(&artifact_path).await? && artifact_path.exists() { tokio::fs::remove_dir_all(&artifact_path).await?; } + snapshot_entity::Entity::delete_by_id(artifact_key) + .exec(write_db) + .await?; + recompute_children(write_db).await?; Ok(()) } @@ -429,47 +406,41 @@ pub(super) async fn reindex_dir(local: &LocalBackend, dir: &Path) -> Microsandbo // After upserts, recompute child_count from parent edges in one pass // to keep the cache honest about the current set of artifacts. let db = local.db().await?.write(); - db.execute_unprepared( - "UPDATE snapshot_index SET child_count = (\ - SELECT COUNT(*) FROM snapshot_index AS c \ - WHERE c.parent_digest = snapshot_index.snapshot_id)", - ) - .await?; + recompute_children(db).await?; Ok(indexed) } -/// Look up a snapshot by digest, name, or path in the local index. +/// Resolve a local address and refresh its rebuildable index row before returning a handle. pub(super) async fn get_handle( local: &LocalBackend, needle: &str, ) -> MicrosandboxResult { - let db = local.db().await?.read(); - - let row = if needle.starts_with("snap_") { - snapshot_entity::Entity::find() - .filter(snapshot_entity::Column::SnapshotId.eq(needle.to_string())) - .one(db) - .await? - } else if needle.starts_with("sha256:") || needle.starts_with("sha512:") { - snapshot_entity::Entity::find_by_id(needle.to_string()) - .one(db) - .await? - } else if looks_like_path(needle) { - // Path lookup: match by artifact_path. - let canon = std::fs::canonicalize(needle) - .map(|p| p.to_string_lossy().to_string()) - .unwrap_or_else(|_| needle.to_string()); - snapshot_entity::Entity::find() - .filter(snapshot_entity::Column::ArtifactPath.eq(canon)) - .one(db) - .await? - } else { - snapshot_entity::Entity::find() - .filter(snapshot_entity::Column::Name.eq(needle.to_string())) - .one(db) - .await? - }; - + let snapshot = open_snapshot(local, needle).await?; + let artifact_path = canonical_path(snapshot.path()); + let alias = super::group::member_name(&artifact_path)?.or_else(|| { + artifact_path + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + }); + if let Some(row) = indexed_path(local, &artifact_path).await? + && row.digest == snapshot.digest() + && row.name == alias + && row.group_path + == super::group::group_path(&artifact_path).map(|path| path.display().to_string()) + { + return Ok(handle_from_model(row)); + } + index_upsert( + local, + snapshot.path(), + snapshot.digest(), + snapshot.manifest(), + ) + .await?; + let row = + snapshot_entity::Entity::find_by_id(canonical_path(snapshot.path()).display().to_string()) + .one(local.db().await?.read()) + .await?; row.map(handle_from_model) .ok_or_else(|| MicrosandboxError::SnapshotNotFound(needle.into())) } @@ -480,15 +451,78 @@ pub(super) async fn lookup_by_digest( digest: &str, ) -> MicrosandboxResult> { let db = local.db().await?.read(); - let row = snapshot_entity::Entity::find() + let rows = snapshot_entity::Entity::find() .filter( sea_orm::Condition::any() .add(snapshot_entity::Column::Digest.eq(digest.to_string())) .add(snapshot_entity::Column::SnapshotId.eq(digest.to_string())), ) - .one(db) + .all(db) .await?; - Ok(row.map(handle_from_model)) + unique_identity_match(rows, digest).map(|row| row.map(handle_from_model)) +} + +async fn resolve_path(local: &LocalBackend, selector: &str) -> MicrosandboxResult { + if looks_like_path(selector) { + return Ok(PathBuf::from(selector)); + } + if microsandbox_image::snapshot::SnapshotId::new(selector).is_ok() + || selector.starts_with("sha256:") + || selector.starts_with("sha512:") + { + return lookup_by_digest(local, selector) + .await? + .map(|handle| handle.artifact_path) + .ok_or_else(|| MicrosandboxError::SnapshotNotFound(selector.into())); + } + if !selector.contains(':') { + let flat = local.snapshots_dir().join(selector); + if flat.join(DESCRIPTOR_FILENAME).is_file() || flat.join(V066_DESCRIPTOR_FILENAME).is_file() + { + return Err(MicrosandboxError::InvalidConfig(format!( + "'{selector}' is an ungrouped snapshot; bare names now select group heads, so open this artifact by its explicit path: {}", + flat.display() + ))); + } + } + super::group::resolve(&local.snapshots_dir(), selector).await +} + +fn canonical_path(path: &Path) -> PathBuf { + std::fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()) +} + +async fn indexed_path( + local: &LocalBackend, + path: &Path, +) -> MicrosandboxResult> { + Ok( + snapshot_entity::Entity::find_by_id(canonical_path(path).display().to_string()) + .one(local.db().await?.read()) + .await?, + ) +} + +fn unique_identity_match( + mut rows: Vec, + identity: &str, +) -> MicrosandboxResult> { + if rows.len() > 1 { + return Err(MicrosandboxError::InvalidConfig(format!( + "snapshot identity {identity} has {} local copies; use group:member or an explicit artifact path", + rows.len() + ))); + } + Ok(rows.pop()) +} + +async fn recompute_children(db: &C) -> Result<(), sea_orm::DbErr> { + // Repeated imports are instances, not additional lineage edges. Apply the same number of + // distinct child identities to every local copy of a parent. + db.execute_unprepared( + "UPDATE snapshot_index SET child_count = (SELECT COUNT(DISTINCT COALESCE(c.snapshot_id, c.digest)) FROM snapshot_index c WHERE c.parent_digest = snapshot_index.snapshot_id)", + ).await?; + Ok(()) } fn handle_from_model(m: snapshot_entity::Model) -> SnapshotHandle { @@ -510,6 +544,8 @@ fn handle_from_model(m: snapshot_entity::Model) -> SnapshotHandle { snapshot_id: m.snapshot_id.unwrap_or_else(|| m.digest.clone()), digest: m.digest, name: m.name, + group: m.group_name, + head_update: None, parent_digest: m.parent_digest, scope, image_ref: m.image_ref, @@ -533,7 +569,180 @@ fn handle_from_model(m: snapshot_entity::Model) -> SnapshotHandle { #[cfg(test)] mod tests { - use super::looks_like_path; + use std::collections::BTreeMap; + + use microsandbox_image::snapshot::{ + CheckpointSnapshotState, ImageRef, SCHEMA, SnapshotCapture, SnapshotConsistency, + SnapshotId, SnapshotRootDisk, + }; + + use super::*; + + fn manifest(id: u128, parent: Option<&Manifest>) -> Manifest { + Manifest { + schema: SCHEMA.into(), + snapshot_id: SnapshotId::new(format!("snap_{id:032x}")).unwrap(), + scope: SnapshotScope::Full, + // These tests exercise addressing and indexing, not checkpoint restoration. + state: SnapshotState::Checkpoint(CheckpointSnapshotState { + checkpoint_id: "checkpoint_test".into(), + checkpoint_root: format!("sha256:{}", "a".repeat(64)), + restore_intents: vec!["resume".into()], + requirements_summary: BTreeMap::new(), + }), + capture: SnapshotCapture { + created_at: "2026-09-10T00:00:00Z".into(), + source_lineage: None, + source_checkpoint: None, + consistency: SnapshotConsistency::CrashConsistent, + }, + image: ImageRef { + reference: "docker.io/library/alpine:3.20".into(), + manifest_digest: format!("sha256:{}", "b".repeat(64)), + }, + root_disk: SnapshotRootDisk::Managed, + parent: parent.map(|parent| parent.snapshot_id.clone()), + extensions: BTreeMap::new(), + requires: Vec::new(), + } + } + + async fn install( + local: &LocalBackend, + group: &str, + name: &str, + manifest: &Manifest, + ) -> PathBuf { + let directory = super::super::group::ensure(&local.snapshots_dir(), Some(group)) + .await + .unwrap(); + let stage = tempfile::tempdir().unwrap(); + let artifact = stage.path().join("member"); + std::fs::create_dir(&artifact).unwrap(); + std::fs::write( + artifact.join(DESCRIPTOR_FILENAME), + manifest.to_canonical_bytes().unwrap(), + ) + .unwrap(); + super::super::group::publish( + &directory, + stage.path(), + &BTreeMap::from([(manifest.snapshot_id.to_string(), name.into())]), + &manifest.snapshot_id, + false, + ) + .await + .unwrap(); + directory.join(manifest.snapshot_id.as_str()) + } + + #[tokio::test] + async fn duplicate_identities_keep_group_addresses_and_remove_only_selected_copy() { + let home = tempfile::tempdir().unwrap(); + let local = LocalBackend::builder() + .home(home.path()) + .build() + .await + .unwrap(); + let snapshot = manifest(1, None); + let first = install(&local, "first", "baseline", &snapshot).await; + let second = install(&local, "second", "baseline", &snapshot).await; + assert_eq!( + reindex_dir(&local, &local.snapshots_dir()).await.unwrap(), + 2 + ); + let first_handle = get_handle(&local, "first:baseline").await.unwrap(); + let second_handle = get_handle(&local, "second").await.unwrap(); + assert_eq!(first_handle.digest(), second_handle.digest()); + assert_eq!(first_handle.group(), Some("first")); + assert_eq!(second_handle.group(), Some("second")); + assert!( + get_handle(&local, snapshot.snapshot_id.as_str()) + .await + .unwrap_err() + .to_string() + .contains("local copies") + ); + assert!( + get_handle(&local, first_handle.digest()) + .await + .unwrap_err() + .to_string() + .contains("local copies") + ); + remove_snapshot(&local, "first:baseline", false) + .await + .unwrap(); + assert!(!first.exists()); + assert!(second.exists()); + assert_eq!(list_indexed(&local).await.unwrap().len(), 1); + assert_eq!( + get_handle(&local, "second").await.unwrap().digest(), + snapshot.digest().unwrap() + ); + } + + #[tokio::test] + async fn distinct_child_counts_and_head_guard_survive_reindex() { + let home = tempfile::tempdir().unwrap(); + let local = LocalBackend::builder() + .home(home.path()) + .build() + .await + .unwrap(); + let parent = manifest(1, None); + let child = manifest(2, Some(&parent)); + for group in ["first", "second"] { + install(&local, group, "base", &parent).await; + install(&local, group, "child", &child).await; + } + reindex_dir(&local, &local.snapshots_dir()).await.unwrap(); + reindex_dir(&local, &local.snapshots_dir()).await.unwrap(); + let rows = snapshot_entity::Entity::find() + .filter(snapshot_entity::Column::SnapshotId.eq(parent.snapshot_id.as_str())) + .all(local.db().await.unwrap().read()) + .await + .unwrap(); + assert_eq!(rows.len(), 2); + assert!(rows.iter().all(|row| row.child_count == 1)); + assert!( + remove_snapshot(&local, "first:child", true) + .await + .unwrap_err() + .to_string() + .contains("current head") + ); + assert_eq!( + get_handle(&local, "first").await.unwrap().id(), + child.snapshot_id.as_str() + ); + } + + #[tokio::test] + async fn flat_artifact_requires_explicit_path() { + let home = tempfile::tempdir().unwrap(); + let local = LocalBackend::builder() + .home(home.path()) + .build() + .await + .unwrap(); + let artifact = local.snapshots_dir().join("flat"); + std::fs::create_dir_all(&artifact).unwrap(); + let snapshot = manifest(1, None); + std::fs::write( + artifact.join(DESCRIPTOR_FILENAME), + snapshot.to_canonical_bytes().unwrap(), + ) + .unwrap(); + assert!(open_snapshot(&local, "flat").await.is_err()); + assert_eq!( + open_snapshot(&local, artifact.to_str().unwrap()) + .await + .unwrap() + .id(), + &snapshot.snapshot_id + ); + } #[test] fn bare_names_are_not_paths() { diff --git a/sdk/rust/tests/snapshot_artifact.rs b/sdk/rust/tests/snapshot_artifact.rs index 88bbfe4d5..b27f4fa9a 100644 --- a/sdk/rust/tests/snapshot_artifact.rs +++ b/sdk/rust/tests/snapshot_artifact.rs @@ -833,6 +833,162 @@ async fn save_then_load_round_trips_via_plain_tar() { assert_eq!(handle.digest(), original_digest); } +#[tokio::test] +async fn repeated_loads_preserve_ids_and_resolve_local_group_names() { + let tmp = TempDir::new().unwrap(); + let home = tmp.path().join("home"); + let backend = isolated_backend(&home).await; + let (source, digest) = make_artifact(tmp.path(), "clean", b"group payload"); + let snapshot_id = artifact_id(&source); + let archive = tmp.path().join("group.msnap"); + let reexport = tmp.path().join("renamed.msnap"); + + microsandbox::with_backend(backend, async { + Snapshot::save( + source.to_string_lossy().as_ref(), + &archive, + microsandbox::snapshot::SaveOpts::default(), + ) + .await + .unwrap(); + let options = microsandbox::snapshot::LoadOpts { + group: Some("work".into()), + ..Default::default() + }; + let first = Snapshot::load_with_options(&archive, options.clone()) + .await + .unwrap(); + assert_eq!(first.group(), Some("work")); + assert_eq!(first.name(), Some("clean")); + assert_eq!(first.path(), home.join("snapshots/work").join(&snapshot_id)); + assert_eq!( + first.head_update().unwrap().reason, + microsandbox::snapshot::HeadUpdateReason::Initialized + ); + + // Reimporting the same identity into the same group is idempotent. + let repeated = Snapshot::load_with_options(&archive, options) + .await + .unwrap(); + assert_eq!(repeated.path(), first.path()); + assert_eq!(repeated.id(), snapshot_id); + assert_eq!( + repeated.head_update().unwrap().reason, + microsandbox::snapshot::HeadUpdateReason::Unchanged + ); + assert_eq!(Snapshot::list().await.unwrap().len(), 1); + assert_eq!(Snapshot::open("work").await.unwrap().digest(), digest); + assert_eq!( + Snapshot::open("work:clean").await.unwrap().id().as_str(), + snapshot_id + ); + assert_eq!( + Snapshot::open(format!("work:{snapshot_id}")) + .await + .unwrap() + .digest(), + digest + ); + + // A default import always gets its own local namespace, even for identical bytes. + let fresh = Snapshot::load(&archive, None).await.unwrap(); + let another = Snapshot::load(&archive, None).await.unwrap(); + assert_ne!(fresh.group(), another.group()); + assert_ne!(fresh.group(), Some("work")); + assert_eq!(fresh.id(), first.id()); + assert_eq!(another.id(), first.id()); + assert_eq!(Snapshot::list().await.unwrap().len(), 3); + + Snapshot::save( + "work:clean", + &reexport, + microsandbox::snapshot::SaveOpts::default(), + ) + .await + .unwrap(); + let renamed = Snapshot::load_with_options( + &reexport, + microsandbox::snapshot::LoadOpts { + group: Some("renamed".into()), + ..Default::default() + }, + ) + .await + .unwrap(); + assert_eq!(renamed.name(), Some("clean")); + assert_eq!(renamed.id(), snapshot_id); + assert_eq!( + Snapshot::group_head("renamed").await.unwrap().head, + snapshot_id + ); + + // Removing one installed copy does not erase another group's membership or payload. + Snapshot::remove(&format!("{}:clean", fresh.group().unwrap()), false) + .await + .unwrap(); + assert!(!fresh.path().exists()); + assert!(first.path().is_dir()); + assert!(another.path().is_dir()); + assert!(renamed.path().is_dir()); + assert_eq!(Snapshot::list().await.unwrap().len(), 3); + assert_eq!(Snapshot::open("work:clean").await.unwrap().digest(), digest); + }) + .await; +} + +#[tokio::test] +async fn group_alias_collision_keeps_the_installed_snapshot_and_head() { + let tmp = TempDir::new().unwrap(); + let home = tmp.path().join("home"); + let backend = isolated_backend(&home).await; + let (first, first_digest) = make_artifact(&tmp.path().join("first"), "clean", b"first"); + let (second, _) = make_artifact(&tmp.path().join("second"), "clean", b"second"); + let original_id = artifact_id(&first); + let competing_id = artifact_id(&second); + let archive = tmp.path().join("first.msnap"); + let competing = tmp.path().join("second.msnap"); + microsandbox::with_backend(backend, async { + for (source, destination) in [(&first, &archive), (&second, &competing)] { + Snapshot::save( + source.to_string_lossy().as_ref(), + destination, + microsandbox::snapshot::SaveOpts::default(), + ) + .await + .unwrap(); + } + let options = microsandbox::snapshot::LoadOpts { + group: Some("work".into()), + ..Default::default() + }; + let installed = Snapshot::load_with_options(&archive, options.clone()) + .await + .unwrap(); + let error = Snapshot::load_with_options(&competing, options) + .await + .unwrap_err(); + assert!( + error.to_string().contains("conflicts"), + "unexpected error: {error}" + ); + assert_eq!( + Snapshot::group_head("work").await.unwrap().head, + original_id + ); + assert_eq!( + Snapshot::open("work:clean").await.unwrap().digest(), + first_digest + ); + assert_eq!( + std::fs::read(artifact_payload_path(installed.path())).unwrap(), + b"first" + ); + assert!(!home.join("snapshots/work").join(competing_id).exists()); + assert_eq!(Snapshot::list().await.unwrap().len(), 1); + }) + .await; +} + #[tokio::test] async fn save_sparse_upper_round_trips_and_preserves_holes() { let tmp = TempDir::new().unwrap(); @@ -1408,7 +1564,7 @@ async fn load_selects_child_head_when_parents_are_present() { let (parent_dir, _) = make_artifact(&snapshots_dir, "parent", b"parent"); let parent_id = artifact_id(&parent_dir); let (child_dir, child_digest) = - make_artifact_with_parent(&snapshots_dir, "child", b"child", Some(parent_id)); + make_artifact_with_parent(&snapshots_dir, "child", b"child", Some(parent_id.clone())); let child_id = artifact_id(&child_dir); let archive = tmp.path().join("chain.tar"); let dest = tmp.path().join("imported-chain"); @@ -1438,7 +1594,17 @@ async fn load_selects_child_head_when_parents_are_present() { .await; assert_eq!(handle.digest(), child_digest); assert_eq!(handle.id(), child_id); - assert_eq!(handle.path(), dest.join(child_id)); + let imported_group = dest.join(handle.group().expect("load creates a local group")); + assert_eq!(handle.path(), imported_group.join(&child_id)); + assert_eq!(handle.head_update().unwrap().head, child_id); + assert_eq!(handle.head_update().unwrap().previous, None); + assert!( + imported_group + .join(parent_id) + .join(DESCRIPTOR_FILENAME) + .is_file() + ); + assert_eq!(Snapshot::list_dir(&imported_group).await.unwrap().len(), 2); } #[tokio::test] @@ -1531,8 +1697,8 @@ async fn failed_load_with_conflicting_cache_target_does_not_install_cache_entrie .await; assert!( - !dest.join("src-cache-conflict").exists(), - "failed import promoted staged snapshot" + Snapshot::list_dir(&dest).await.unwrap().is_empty(), + "failed import promoted a grouped snapshot" ); assert_eq!( std::fs::read(&conflicting_metadata).unwrap(), @@ -1602,7 +1768,7 @@ async fn create_full_resolves_source_before_touching_anything() { }) .await; - assert!(!home.join("snapshots").join("warm").exists()); + assert!(!home.join("snapshots").join("box").exists()); } #[tokio::test] @@ -1673,10 +1839,14 @@ async fn replacing_child_in_place_does_not_inflate_parent_child_count() { b"child v2 with different size", Some(parent_id), ); - Snapshot::open("child").await.unwrap(); + Snapshot::open(cdir.to_string_lossy().as_ref()) + .await + .unwrap(); - Snapshot::remove("child", false).await.unwrap(); - Snapshot::remove("parent", false) + Snapshot::remove(cdir.to_string_lossy().as_ref(), false) + .await + .unwrap(); + Snapshot::remove(pdir.to_string_lossy().as_ref(), false) .await .expect("parent should be removable once its only child is gone"); }) From 08cf9d6f9e253191dda2c6abeb73022ba4fa68fe Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Thu, 10 Sep 2026 02:56:57 +0100 Subject: [PATCH 17/29] docs(readme): highlight branching and snapshots Promote Branch & Snapshot with a fork icon and concise CLI examples.\nShow live branching and durable full snapshots in separate code groups. --- README.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0de6dea25..c58936dbc 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ ## - **Hardware Isolation**: Hardware-level isolation with microVM technology. +- **Branch & Snapshot**: Save state. Fork live sandboxes. - **Cross Platform**: Runs on Linux, macOS, and Windows. - **OCI Compatible**: Runs standard container images from Docker Hub, GHCR, or any OCI registry. - **Docker-Like Workflows**: Familiar image, command, shell, and volume workflows. @@ -284,7 +285,7 @@ The SDK lets you create and control sandboxes directly from your application. `S ## cli-darkcli  CLI -The `msb` CLI provides a complete interface for managing sandboxes, images, and volumes. +The `msb` CLI provides a complete interface for managing sandboxes, snapshots, images, and volumes. ####   Run a Command @@ -306,6 +307,19 @@ The `msb` CLI provides a complete interface for managing sandboxes, images, and > ``` > > ```sh +> # Fork a running sandbox. Take a new path. +> msb branch app --name experiment +> msb exec experiment -- python -c "print('An independent copy!')" +> msb branch experiment --name another-experiment +> ``` +> +> ```sh +> # Save now, resume later +> msb snapshot create saved --from app --full +> msb create --name restored --from-snapshot saved +> ``` +> +> ```sh > # Lifecycle > msb stop app > msb start app From dd3bb44061c0f5f8116c0f77185198aafed1e4a3 Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Thu, 10 Sep 2026 02:57:26 +0100 Subject: [PATCH 18/29] docs(readme): qualify the saved snapshot with its group Use app:saved to select the exact checkpoint under the source sandbox's snapshot group, matching the current restore selector API. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c58936dbc..63b69e164 100644 --- a/README.md +++ b/README.md @@ -316,7 +316,7 @@ The `msb` CLI provides a complete interface for managing sandboxes, snapshots, i > ```sh > # Save now, resume later > msb snapshot create saved --from app --full -> msb create --name restored --from-snapshot saved +> msb create --name restored --from-snapshot app:saved > ``` > > ```sh From 5e717c1b3141a2389718f12bb3a4961bdf98729e Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Thu, 10 Sep 2026 04:16:35 +0100 Subject: [PATCH 19/29] feat(snapshot)!: load archive batches into snapshot groups Resolve disk and RAM dependencies across supplied archives and the selected destination group, with an optional external base. Validate complete owned closures before publishing members and select a group head independently of archive order. Expose batch loading through the CLI and Rust, Python, TypeScript, and Go SDKs. Prefer .msb archive names throughout examples and documentation without changing content-based decoding or archive schemas. Record automated coverage and 102 macOS live command checks for full and disk-only batches, including eager and forked restoration. Keep the existing disk-only incremental-export limitation explicit. BREAKING CHANGE: snapshot load takes one or more positional archives; use --dest DIR instead of a positional destination directory. --- COMPATIBILITY.md | 4 + crates/cli/lib/commands/snapshot.rs | 78 ++- docs/sandboxes/snapshots.mdx | 65 ++- docs/sdk/go/snapshots.mdx | 38 +- docs/sdk/python/snapshots.mdx | 45 +- docs/sdk/rust/snapshots.mdx | 36 +- docs/sdk/typescript/snapshots.mdx | 37 +- docs/snapshot-groups-explained.md | 47 +- scripts/smoke/cli/checkpoint-clock.py | 2 +- scripts/smoke/cli/cow-memory-lifecycle.py | 4 +- scripts/smoke/cli/disk-compaction-export.sh | 8 +- scripts/smoke/cli/disk-compaction-negative.sh | 6 +- scripts/smoke/cli/incremental-full-archive.py | 4 +- scripts/smoke/cli/live-disk-snapshot.py | 2 +- scripts/smoke/cli/snapshot-groups.py | 6 +- scripts/smoke/cli/snapshot-load-batch.py | 262 +++++++++++ .../cow-memory-lifecycle-2026-09-07.md | 4 +- .../reports/live-disk-snapshot-2026-09-09.md | 8 +- .../reports/snapshot-load-batch-2026-09-10.md | 116 +++++ sdk/go/integration/snapshot_test.go | 15 + sdk/go/internal/ffi/ffi.go | 38 ++ sdk/go/native/microsandbox_go_ffi.h | 9 + sdk/go/native/src/lib.rs | 34 ++ sdk/go/snapshot.go | 24 +- sdk/node-ts/native/index.d.ts | 8 +- sdk/node-ts/native/snapshot.rs | 32 +- sdk/node-ts/src/internal/napi.ts | 1 + sdk/node-ts/src/snapshot.ts | 12 +- .../tests/unit/native-contract.test.ts | 6 + sdk/node-ts/tests/unit/snapshot.test.ts | 28 +- sdk/python/microsandbox/_microsandbox.pyi | 9 + sdk/python/src/snapshot.rs | 30 ++ sdk/python/tests/test_snapshot_stub.py | 28 ++ sdk/rust/lib/snapshot/archive.rs | 199 +------- sdk/rust/lib/snapshot/archive/batch.rs | 444 ++++++++++++++++++ sdk/rust/lib/snapshot/archive/delta.rs | 422 ++++++++++++++++- sdk/rust/lib/snapshot/archive/delta_tests.rs | 217 ++++++++- sdk/rust/lib/snapshot/group.rs | 131 +++++- sdk/rust/lib/snapshot/group_tests.rs | 325 +++++++++++++ sdk/rust/lib/snapshot/mod.rs | 18 +- sdk/rust/lib/snapshot/verify.rs | 15 +- sdk/rust/tests/snapshot_artifact.rs | 372 ++++++++++++++- 42 files changed, 2834 insertions(+), 355 deletions(-) create mode 100644 scripts/smoke/cli/snapshot-load-batch.py create mode 100644 scripts/smoke/reports/snapshot-load-batch-2026-09-10.md create mode 100644 sdk/python/tests/test_snapshot_stub.py create mode 100644 sdk/rust/lib/snapshot/archive/batch.rs diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index 6ef5a47a0..8e8b966ee 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -194,6 +194,10 @@ Installed snapshots now live under `snapshots///`. `group.js Capture records the actual source snapshot lineage in the existing descriptor `parent` field. Per-sandbox cursor publication serializes captures without holding a VM pause; group head publication is locked separately. Automatic head advancement requires known ancestry, not capture timestamps, export dependency bases, or import order. An explicit head selection may rewind or choose a sibling. Missing ancestry may prevent advancement but is not a missing payload dependency. Archives optionally carry friendly names in `msb-snapshot-member-names`; their snapshot IDs, payload paths and descriptor schema are unchanged. +`snapshot load` accepts multiple archive paths; the former positional destination is now `--dest DIR`. Single-archive SDK methods and their return types remain; batch methods return one handle per input archive head in input order. The batch resolves exact disk-layer and RAM-object dependencies from supplied archives, the explicitly selected destination group, and an optional external base. No archive encoding changes or global snapshot search are involved. Borrowed payloads belong to destination staging and use the existing integrity codecs before publication. A compatible source may contain more layers than the omitted prefix; dependency identities still must match. Direct archive restore retains its explicit-base contract. + +Batch head selection is independent of input order: one proven lineage tip uses existing fast-forward rules; ambiguous tips preserve an existing head or leave a new group headless. `--set-head` refuses an ambiguous batch. IDs, aliases, duplicate labels, and payloads are checked before member publication. An I/O failure during final publication can still leave complete additional members, as with single-archive publication, but never a head pointing at an incomplete member. + Unreleased #8 incremental exports use `completeness: "dependent"` and the must-understand `msb-snapshot-dependencies-v1` extension. `--since` records omitted physical disk-prefix layers and reusable RAM-object identities; `--last-layers` only omits disk layers. The complete target memory manifest and CPU/device state remain included. Loading and direct archive restore resolve the explicitly supplied base into owned staging before opening the complete target. This replaces the unreleased disk-only dependency encoding without a compatibility shim or snapshot descriptor change. Readers that do not understand this requirement refuse it; ordinary standalone archives are unchanged. Evolution rules: diff --git a/crates/cli/lib/commands/snapshot.rs b/crates/cli/lib/commands/snapshot.rs index 7bc7fa2ba..064a8d710 100644 --- a/crates/cli/lib/commands/snapshot.rs +++ b/crates/cli/lib/commands/snapshot.rs @@ -40,7 +40,7 @@ pub enum SnapshotCommands { /// Rebuild the local index from artifacts on disk. Reindex(SnapshotReindexArgs), - /// Save a snapshot into a `.msnap` archive (tar + zstd). + /// Save a snapshot into a `.msb` archive (tar + zstd). Save(SnapshotSaveArgs), /// Load a snapshot archive into the snapshots directory. @@ -157,7 +157,7 @@ pub struct SnapshotSaveArgs { /// Snapshot to save (path, name, or digest). pub snapshot: String, - /// Output archive path (`.msnap` recommended; explicit filenames are preserved). + /// Output archive path (`.msb` recommended; explicit filenames are preserved). pub out: std::path::PathBuf, /// Walk the parent chain and include each ancestor in the archive. @@ -184,12 +184,14 @@ pub struct SnapshotSaveArgs { /// Arguments for `msb snapshot load`. #[derive(Debug, Args)] pub struct SnapshotLoadArgs { - /// Archive to unpack. - pub archive: std::path::PathBuf, + /// Archives to import together; dependencies are resolved regardless of argument order. + #[arg(required = true, num_args = 1.., value_name = "ARCHIVE")] + pub archives: Vec, /// Destination directory (defaults to `~/.microsandbox/snapshots/`). + #[arg(long, value_name = "DIR")] pub dest: Option, - /// Exact base snapshot or standalone base archive for a dependent archive. + /// External base snapshot or standalone archive if batch/group members cannot supply dependencies. #[arg(long)] pub base: Option, @@ -197,7 +199,7 @@ pub struct SnapshotLoadArgs { #[arg(long, value_name = "GROUP")] pub group: Option, - /// Select the imported member as head even if it is not a fast-forward. + /// Select the batch's unique tip as head even if it is not a fast-forward. #[arg(long)] pub set_head: bool, } @@ -503,8 +505,8 @@ async fn save(args: SnapshotSaveArgs) -> anyhow::Result<()> { } async fn load(args: SnapshotLoadArgs) -> anyhow::Result<()> { - let handle = Snapshot::load_with_options( - &args.archive, + let handles = Snapshot::load_many( + &args.archives, microsandbox::snapshot::LoadOpts { dest: args.dest, base: args.base, @@ -513,12 +515,25 @@ async fn load(args: SnapshotLoadArgs) -> anyhow::Result<()> { }, ) .await?; - if let Some(update) = handle.head_update() { + // Every imported member belongs to one batch; report its single head decision once. + if let Some(update) = handles.iter().find_map(|handle| handle.head_update()) { report_head_update(update); + } else if let Some(group) = handles.first().and_then(|handle| handle.group()) { + eprintln!( + "group {group}: imported members without selecting a head; choose a member explicitly" + ); + } + for (index, handle) in handles.iter().enumerate() { + if handles.len() > 1 { + if index > 0 { + println!(); + } + println!("Snapshot: {}", handle.id()); + } + println!("{}", handle.digest()); + // Preserve the single-archive digest/path output consumed by shell scripts. + println!("{}", handle.path().display()); } - println!("{}", handle.digest()); - // Keep the installed path as the final stdout line for shell consumers. - println!("{}", handle.path().display()); Ok(()) } @@ -707,17 +722,50 @@ mod tests { #[test] fn load_parses_args() { - let parsed = parse_snapshot_args(&["load", "bundle.tar", "/tmp/snaps"]); + let parsed = parse_snapshot_args(&["load", "bundle.tar", "--dest", "/tmp/snaps"]); let SnapshotCommands::Load(args) = parsed.command else { panic!("expected load command"); }; - assert_eq!(args.archive, std::path::PathBuf::from("bundle.tar")); + assert_eq!(args.archives, vec![std::path::PathBuf::from("bundle.tar")]); assert_eq!( args.dest.as_deref(), Some(std::path::Path::new("/tmp/snaps")) ); } + #[test] + fn load_parses_multiple_archives_and_a_named_destination() { + let parsed = parse_snapshot_args(&[ + "load", + "changes.msb", + "base.msb", + "--dest", + "/tmp/snaps", + "--group", + "received", + ]); + let SnapshotCommands::Load(args) = parsed.command else { + panic!("expected load command"); + }; + assert_eq!( + args.archives, + vec![ + std::path::PathBuf::from("changes.msb"), + std::path::PathBuf::from("base.msb"), + ] + ); + assert_eq!( + args.dest.as_deref(), + Some(std::path::Path::new("/tmp/snaps")) + ); + assert_eq!(args.group.as_deref(), Some("received")); + } + + #[test] + fn load_requires_at_least_one_archive() { + assert!(TestCli::try_parse_from(["msb", "load", "--group", "received"]).is_err()); + } + #[test] fn create_accepts_generated_member_in_explicit_group() { let parsed = parse_snapshot_args(&["create", "--from", "box", "--group", "work"]); @@ -733,7 +781,7 @@ mod tests { fn load_accepts_group_and_explicit_head_selection() { let parsed = parse_snapshot_args(&[ "load", - "changes.msnap", + "changes.msb", "--base", "work:base", "--group", diff --git a/docs/sandboxes/snapshots.mdx b/docs/sandboxes/snapshots.mdx index 646a8ae5b..0d9edb319 100644 --- a/docs/sandboxes/snapshots.mdx +++ b/docs/sandboxes/snapshots.mdx @@ -6,7 +6,7 @@ icon: "code-branch" Local-only -A snapshot is a portable artifact that can hold either a sandbox's writable disk state or a full checkpoint of a running sandbox. Managed and flat OCI roots are supported; the descriptor preserves the root layout so a flat `rootfs.raw` is never mistaken for a managed OverlayFS upper. Move it with `scp`, archive it as `.msnap`, or create a child sandbox from it. +A snapshot is a portable artifact that can hold either a sandbox's writable disk state or a full checkpoint of a running sandbox. Managed and flat OCI roots are supported; the descriptor preserves the root layout so a flat `rootfs.raw` is never mistaken for a managed OverlayFS upper. Move it with `scp`, archive it as `.msb`, or create a child sandbox from it. Disk snapshots work with running, paused, stopped, or crashed sandboxes. A running disk capture briefly pauses the VM, seals its disk, and resumes it without copying RAM. Full snapshots use `--full` to include memory and execution state. A user-paused source stays paused in either mode. @@ -14,7 +14,7 @@ Disk snapshots work with running, paused, stopped, or crashed sandboxes. A runni ## What gets captured -`.msnap` is a tar + zstd snapshot archive. Older `.tar.zst` and `.tar` names still work, and explicit output filenames are kept as given. Disk-only, full, and incremental exports use the same extension; the archive records what it contains. +`.msb` is the recommended snapshot archive extension; compression defaults to tar + zstd. Archives are recognized by their contents, so existing `.tar.zst`, `.tar`, and other filenames still work. Explicit output filenames are kept as given. Disk-only, full, and incremental exports use the same extension; the archive records what it contains. | Mode | Source | Captured state | Restore behavior | | ---- | ------ | -------------- | ---------------- | @@ -102,17 +102,32 @@ work:cp01 ---- work:cp02 ---- work:cp03 <- head +------ work:experiment ``` -`msb snapshot head work:experiment` selects the other branch. Import with `--set-head` to explicitly select the imported archive's head. Missing historical checkpoints are allowed if the snapshot's disk and RAM dependencies are complete; missing payload dependencies still require `--base`. +`msb snapshot head work:experiment` selects the other branch. Import with `--set-head` to explicitly select the imported archive's head. Missing historical checkpoints are allowed if the snapshot's disk and RAM dependencies are complete. Imports resolve payload dependencies from the supplied archives and an explicitly named destination group; `--base` supplies an external fallback. ```bash -msb snapshot load checkpoint.msnap --group work -msb snapshot load experiment.msnap --group work --set-head +msb snapshot load checkpoint.msb --group work +msb snapshot load experiment.msb --group work --set-head ``` Loading without `--group` creates a fresh generated group. The same archive can be imported into different groups without overwriting either copy. Within a group, an identical ID/descriptor is reusable; conflicting IDs or names fail. A current head cannot be removed while other members remain—select another first. Existing flat snapshot directories can still be opened by explicit path. An interrupted call may already have published its snapshot. Inspect the group before retrying; member publication never overwrites an existing snapshot. +## Load a set of archives + +Import a baseline and its dependent archives together. The shell expands the wildcard; archive order does not matter: + +```bash +msb snapshot load checkpoints/*.msb --group received +msb snapshot load changes.msb base.msb --group received --dest /mnt/snapshots +``` + +`load` accepts one or more archive paths. Use `--dest DIR` for a different group-store root; a trailing positional path is another archive, not a destination. A batch without `--group` creates one generated group for all its archives. Existing single-archive calls still print the digest followed by the installed path. Multi-archive calls print each result and report the batch's head decision once. + +Dependencies are resolved from the batch and exact matching snapshots already installed in the explicitly named destination group. For example, after importing `base.msb` into `received`, `msb snapshot load changes.msb --group received` can reuse that base automatically. Use `--base SOURCE` when a dependency must come from another installed snapshot or standalone archive. Imports do not search unrelated groups or nearby archive files. These operations accept the existing archive format; no format conversion or schema bump is required. + +All requested members are validated before publication. A batch with one tip follows the normal head-update rules. If a batch contains divergent tips, the existing group head stays selected; a new group has no head until you choose a member with `msb snapshot head GROUP:MEMBER`. `--set-head` requires a unique batch tip and rejects an ambiguous batch before publishing members. The final printed result is not necessarily the selected head. + ## Capture from the SDK @@ -171,7 +186,7 @@ msb snapshot create after-pip-install --from baseline --dest-dir /mnt/big -The default disk mode captures only the owned managed or flat root disk. Live captures are crash-consistent: unsaved application buffers and tmpfs contents are not included. Use `--full` when you need memory too. Disk snapshotting preserves the complete raw/qcow2 chain, and every restored child receives a fresh private writable head. Direct `--archive ./saved.msnap` capture works in either mode without installing a snapshot directory. +The default disk mode captures only the owned managed or flat root disk. Live captures are crash-consistent: unsaved application buffers and tmpfs contents are not included. Use `--full` when you need memory too. Disk snapshotting preserves the complete raw/qcow2 chain, and every restored child receives a fresh private writable head. Direct `--archive ./saved.msb` capture works in either mode without installing a snapshot directory. ## Capture a running sandbox @@ -224,21 +239,23 @@ Repeated full checkpoints add disk layers. You choose when to export changes and ```bash msb snapshot create checkpoint-b --from worker --full -msb snapshot save worker:checkpoint-b changes.msnap --since worker:checkpoint-a +msb snapshot save worker:checkpoint-b changes.msb --since worker:checkpoint-a msb modify worker --compact --layers 3 --dry-run msb modify worker --compact --layers 3 ``` `--layers 3` merges the oldest three physical layers, **including the base**. It never includes the writable head, even when stopped. Omit `--layers` to merge all sealed layers; a chain with fewer than two sealed layers is unchanged. Managed and flat roots support compaction, running or stopped. Existing snapshots remain valid and retain their storage until you remove them separately. -`--since` omits disk layers and, for full checkpoints, RAM objects already supplied by the base. The disk base must be an exact physical prefix. Each archive still includes its complete memory map and CPU/device state; it does not replay earlier memory images. Alternatively, `--last-layers 2` selects only disk layers and keeps all required RAM objects. These smaller archives require an explicit base when loading or restoring: +`--since` omits disk layers and, for full checkpoints, RAM objects already supplied by the base. The disk base must be an exact physical prefix. Each archive still includes its complete memory map and CPU/device state; it does not replay earlier memory images. Alternatively, `--last-layers 2` selects only disk layers and keeps all required RAM objects. Import these smaller archives with their bases in the same batch, into a named group containing their dependencies, or with an explicit external base. Direct restore still accepts an explicit base: ```bash -msb snapshot load changes.msnap --base worker:checkpoint-a --group imported -msb create --name child --from-snapshot changes.msnap --snapshot-base worker:checkpoint-a +msb snapshot load changes.msb --base worker:checkpoint-a --group imported +msb create --name child --from-snapshot changes.msb --snapshot-base worker:checkpoint-a ``` -The base can also be a standalone snapshot archive. Load a dependent base first, then pass the installed path printed by `snapshot load` to the next load or restore. Loads resolve disk and RAM dependencies without starting a VM; only the final sandbox creation resumes execution. Missing or incorrect dependencies fail before execution. Loaded snapshots and restored children own their required files, so removing the base later does not break them. Direct restore skips installing an intermediate snapshot. Add `--disk-only` to cold-boot only disk state. After compaction, export a new standalone baseline before resuming incremental exports: the old physical prefix no longer matches. Do not combine compaction with unrelated `modify` options, or incremental export with `--with-parents`. +An explicit external base can also be a standalone snapshot archive. Dependent archives can supply one another within a batch; they need not be loaded in dependency order. Loads resolve disk and RAM dependencies without starting a VM; only the final sandbox creation resumes execution. Missing or incorrect dependencies fail before publication or execution. Loaded snapshots and restored children own their required files, so removing the base later does not break them. Direct restore skips installing an intermediate snapshot. Add `--disk-only` to cold-boot only disk state. After compaction, export a new standalone baseline before resuming incremental exports: the old physical prefix no longer matches. Do not combine compaction with unrelated `modify` options, or incremental export with `--with-parents`. + +Current development limitation: successive disk-only captures reassign layer IDs, so their `--since` export can reject the base even when physical layers were unchanged. Use standalone disk-only exports for now. Full-checkpoint incremental exports and imports are unaffected by this capture issue. ## Capture directly to an archive @@ -250,7 +267,7 @@ use microsandbox::Snapshot; let archive = Snapshot::builder("after-pip-install") .from_sandbox("baseline") - .create_archive("/tmp/after-pip-install.msnap", false) + .create_archive("/tmp/after-pip-install.msb", false) .await?; ``` @@ -259,7 +276,7 @@ import { Snapshot } from "microsandbox"; const archive = await Snapshot.builder("after-pip-install") .fromSandbox("baseline") - .createArchive("/tmp/after-pip-install.msnap"); + .createArchive("/tmp/after-pip-install.msb"); ``` ```python Python @@ -267,7 +284,7 @@ from microsandbox import Snapshot archive = await Snapshot.create_archive( "after-pip-install", - "/tmp/after-pip-install.msnap", + "/tmp/after-pip-install.msb", from_sandbox="baseline", ) ``` @@ -278,14 +295,14 @@ archive, err := m.Snapshot.CreateArchive(ctx, m.SnapshotArchiveOptions{ Name: "after-pip-install", FromSandbox: "baseline", }, - ArchivePath: "/tmp/after-pip-install.msnap", + ArchivePath: "/tmp/after-pip-install.msb", }) ``` ```bash CLI msb snapshot create after-pip-install \ --from baseline \ - --archive /tmp/after-pip-install.msnap + --archive /tmp/after-pip-install.msb ``` @@ -294,7 +311,7 @@ Direct capture publishes only the archive and returns its snapshot ID and path. An explicit archive path can also be used directly as the source of a new sandbox: ```bash -msb run --name worker --from-snapshot ./after-pip-install.msnap -- python -V +msb run --name worker --from-snapshot ./after-pip-install.msb -- python -V ``` The same archive path works with the SDK restore methods shown below. The archive is unpacked into child-owned staging, so no intermediate installed snapshot is loaded into `~/.microsandbox/snapshots`. @@ -463,17 +480,17 @@ The snapshot directory is the whole artifact; there is no hidden daemon state. C scp -r ~/.microsandbox/snapshots/baseline/snap_ other-host:/tmp/saved-snapshot # Open that copied artifact by its explicit path; use archive load to add it to a group. -# Bundle into a .msnap, transport, then load -msb snapshot save baseline:after-pip-install /tmp/snap.msnap -scp /tmp/snap.msnap other-host: -ssh other-host msb snapshot load /tmp/snap.msnap +# Bundle into a .msb, transport, then load +msb snapshot save baseline:after-pip-install /tmp/snap.msb +scp /tmp/snap.msb other-host: +ssh other-host msb snapshot load /tmp/snap.msb # Fully offline: include the OCI image cache so the target needs no network -msb snapshot save baseline:after-pip-install /tmp/snap.msnap --with-image -ssh other-host msb snapshot load /tmp/snap.msnap +msb snapshot save baseline:after-pip-install /tmp/snap.msb --with-image +ssh other-host msb snapshot load /tmp/snap.msb ``` -Archives use tar + zstd by default; `.msnap` is the recommended filename extension. Pass `--plain-tar` for uncompressed tar. SDKs expose the same save and load operations as the CLI. +Archives use tar + zstd by default; `.msb` is the recommended filename extension. Pass `--plain-tar` for uncompressed tar. SDKs expose the same save and load operations as the CLI. ## Artifact identity and layout diff --git a/docs/sdk/go/snapshots.mdx b/docs/sdk/go/snapshots.mdx index 13bf4688f..e0964494b 100644 --- a/docs/sdk/go/snapshots.mdx +++ b/docs/sdk/go/snapshots.mdx @@ -15,21 +15,35 @@ Installed snapshots belong to a group. A bare group selects its head; `group:mem snap, err := m.Snapshot.Create(ctx, m.SnapshotCreateOptions{ Name: "baseline", FromSandbox: "box", Group: "work", }) -loaded, err := m.Snapshot.LoadWithOptions(ctx, "changes.msnap", m.SnapshotLoadOptions{ +loaded, err := m.Snapshot.LoadWithOptions(ctx, "changes.msb", m.SnapshotLoadOptions{ Base: "work:baseline", Group: "work", }) head, err := m.Snapshot.GroupHead(ctx, "work") selected, err := m.Snapshot.GroupHead(ctx, "work:baseline") -loaded, err = m.Snapshot.LoadWithOptions(ctx, "other.msnap", m.SnapshotLoadOptions{ +loaded, err = m.Snapshot.LoadWithOptions(ctx, "other.msb", m.SnapshotLoadOptions{ Group: "work", SetHead: true, }) ``` -`SnapshotLoadOptions` contains `Dest`, `Base`, `Group`, and `SetHead`. `Load` and `LoadWithBase` use generated groups. `GroupHead` returns `SnapshotHeadUpdate` with `Group`, `Previous`, `Head`, `Reason`, and `Changed`; `Previous` and `Head` are stable snapshot IDs. Automatic head updates initialize an empty group or advance to a proven descendant; divergent imports and imports with unknown ancestry retain the current head. Reasons are `initialized`, `fast_forwarded`, `selected`, `unchanged`, `diverged`, and `unknown_ancestry`. Direct archive capture creates no group and rejects a nonempty `Group`. +`SnapshotLoadOptions` contains `Dest`, `Base`, `Group`, and `SetHead`. `Load` and `LoadWithBase` use generated groups. `GroupHead` returns `SnapshotHeadUpdate` with `Group`, `Previous`, `Head`, `Reason`, and `Changed`; `Previous` and `Head` are stable snapshot IDs. Automatic head updates initialize an empty group or advance to a proven descendant; divergent imports and imports with unknown ancestry retain the current head. Reasons are `initialized`, `fast_forwarded`, `selected`, `unchanged`, `diverged`, `unknown_ancestry`, and `ambiguous_candidates`. Direct archive capture creates no group and rejects a nonempty `Group`. + +## Load multiple archives + +```go +handles, err := m.Snapshot.LoadMany(ctx, + []string{"changes.msb", "base.msb"}, + m.SnapshotLoadOptions{Group: "received"}, +) +if err != nil { return err } +``` + +`LoadMany(ctx, archives []string, opts SnapshotLoadOptions)` returns `([]*SnapshotHandle, error)`, with one handle per supplied archive in input order. Dependencies are resolved regardless of argument order from the batch and exact matching snapshots already installed in an explicitly named destination group. `Base` supplies an external snapshot or standalone archive when needed. Single-file `LoadWithOptions` also reuses dependencies from its named destination group. An empty `Group` creates one generated group for the batch. + +All batch members are validated before publication. `SetHead: true` requires a unique tip. With divergent tips and the default `false`, an existing head is retained (`ambiguous_candidates`); a new group has no head and imported handles return `nil` from `HeadUpdate()`. Select a member explicitly to give that group a head. The archive format is unchanged. The CLI equivalent is `msb snapshot load checkpoints/*.msb --group received`, with `--dest DIR` for another group-store root. ## Disk maintenance and incremental export -Exporting since a base omits its reusable disk layers and RAM objects. Full checkpoints keep the complete memory map and CPU/device state. The last-layers option only selects disk layers and leaves RAM payloads complete. Supply the base when loading or restoring a dependent archive; a dependent base archive must be loaded first. +Exporting since a base omits its reusable disk layers and RAM objects. Full checkpoints keep the complete memory map and CPU/device state. The last-layers option only selects disk layers and leaves RAM payloads complete. Imports resolve dependencies from the batch or an explicitly named destination group. Use an external base for missing dependencies; direct restore still accepts an explicit base. ```go worker, err := m.GetSandbox(ctx, "worker") @@ -37,12 +51,12 @@ if err != nil { return err } layers := uint32(3) result, err := worker.Compact(ctx, m.DiskCompactionOptions{Layers: &layers}) if err != nil { return err } -err = m.Snapshot.Save(ctx, "worker:checkpoint-b", "changes.msnap", m.SnapshotSaveOptions{Since: "worker:checkpoint-a"}) +err = m.Snapshot.Save(ctx, "worker:checkpoint-b", "changes.msb", m.SnapshotSaveOptions{Since: "worker:checkpoint-a"}) if err != nil { return err } -snapshot, err := m.Snapshot.LoadWithBase(ctx, "changes.msnap", "", "worker:checkpoint-a") +snapshot, err := m.Snapshot.LoadWithBase(ctx, "changes.msb", "", "worker:checkpoint-a") ``` -The count includes the oldest base but excludes the writable head. A nil `Layers` selects all sealed layers; `DryRun: true` only resolves the plan. `LastLayers` is an alternative to `Since` for export. For direct restore combine `WithFromSnapshot("changes.msnap")` with `WithSnapshotBase("worker:checkpoint-a")`. See [disk-chain maintenance](/sandboxes/snapshots#export-changes-and-compact-a-disk-chain) for dependency and retention rules. +The count includes the oldest base but excludes the writable head. A nil `Layers` selects all sealed layers; `DryRun: true` only resolves the plan. `LastLayers` is an alternative to `Since` for export. For direct restore combine `WithFromSnapshot("changes.msb")` with `WithSnapshotBase("worker:checkpoint-a")`. See [disk-chain maintenance](/sandboxes/snapshots#export-changes-and-compact-a-disk-chain) for dependency and retention rules. ## Snapshot @@ -107,7 +121,7 @@ archive, err := m.Snapshot.CreateArchive(ctx, m.SnapshotArchiveOptions{ Name: "after-pip-install", FromSandbox: "baseline", }, - ArchivePath: "/tmp/after-pip-install.msnap", + ArchivePath: "/tmp/after-pip-install.msb", }) ``` @@ -328,7 +342,7 @@ Walk `dir` and rebuild the local index from the artifacts it finds. func (snapshotFactory) Save(ctx context.Context, nameOrPath, outPath string, opts SnapshotSaveOptions) error ``` -Bundle a snapshot into a `.msnap` archive at `outPath`. Set [`SnapshotSaveOptions.PlainTar`](#snapshotsaveoptionsstruct) to skip compression. +Bundle a snapshot into a `.msb` archive at `outPath`. Set [`SnapshotSaveOptions.PlainTar`](#snapshotsaveoptionsstruct) to skip compression.

Parameters

@@ -354,7 +368,7 @@ Bundle a snapshot into a `.msnap` archive at `outPath`. Set [`SnapshotSaveOption ```go -err := m.Snapshot.Save(ctx, "baseline:after-pip-install", "/tmp/snap.msnap", +err := m.Snapshot.Save(ctx, "baseline:after-pip-install", "/tmp/snap.msb", m.SnapshotSaveOptions{WithParents: true}, ) ``` @@ -401,7 +415,7 @@ Unpack a snapshot archive into the snapshots directory or an explicit `dest` dir ```go -h, err := m.Snapshot.Load(ctx, "/tmp/snap.msnap", "") +h, err := m.Snapshot.Load(ctx, "/tmp/snap.msb", "") ``` @@ -792,7 +806,7 @@ Configures [`Snapshot.Save`](#snapshot-save). |-------|------|-------------| | WithParents | `bool` | Include the snapshot's parent chain in the archive | | WithImage | `bool` | Include the base OCI image in the archive | -| PlainTar | `bool` | Write an uncompressed `.tar` instead of `.msnap` | +| PlainTar | `bool` | Write an uncompressed `.tar` instead of `.msb` | ### SnapshotVerifyReportstruct diff --git a/docs/sdk/python/snapshots.mdx b/docs/sdk/python/snapshots.mdx index f253f91a9..3bc565e84 100644 --- a/docs/sdk/python/snapshots.mdx +++ b/docs/sdk/python/snapshots.mdx @@ -13,14 +13,29 @@ Installed snapshots belong to a group. A bare group selects its head; `group:mem ```python snap = await Snapshot.create("baseline", from_sandbox="box", group="work") -loaded = await Snapshot.load("changes.msnap", base="work:baseline", group="work") +loaded = await Snapshot.load("changes.msb", base="work:baseline", group="work") head = await Snapshot.group_head("work") selected = await Snapshot.group_head("work:baseline") # Explicitly select an imported member even when it is not a descendant. -loaded = await Snapshot.load("other.msnap", group="work", set_head=True) +loaded = await Snapshot.load("other.msb", group="work", set_head=True) ``` -`group_head(selector)` returns a dictionary with `group`, `previous`, `head`, `reason`, and `changed`. `previous` and `head` are stable snapshot IDs. Automatic head updates initialize an empty group or advance to a proven descendant; divergent imports and imports with unknown ancestry retain the current head. Reasons are `initialized`, `fast_forwarded`, `selected`, `unchanged`, `diverged`, and `unknown_ancestry`. Direct archive capture creates no group and rejects `group=`. +`group_head(selector)` returns a dictionary with `group`, `previous`, `head`, `reason`, and `changed`. `previous` and `head` are stable snapshot IDs. Automatic head updates initialize an empty group or advance to a proven descendant; divergent imports and imports with unknown ancestry retain the current head. Reasons are `initialized`, `fast_forwarded`, `selected`, `unchanged`, `diverged`, `unknown_ancestry`, and `ambiguous_candidates`. Direct archive capture creates no group and rejects `group=`. + +## Load multiple archives + +```python +from pathlib import Path + +handles = await Snapshot.load_many( + list(Path("checkpoints").glob("*.msb")), + group="received", +) +``` + +`load_many(archives, *, dest=None, base=None, group=None, set_head=False)` returns one `SnapshotHandle` for each supplied archive, in input order. The batch resolves dependencies regardless of argument order, using its archives and exact matching members already installed in an explicitly named destination group. `base` provides an external snapshot or standalone archive when needed. Single-file `load` also reuses dependencies from its named destination group. Omitting `group` creates one generated group for the whole batch. + +The importer validates the complete batch before publishing members. `set_head=True` requires a unique tip. With divergent tips and the default `False`, an existing head is retained (`ambiguous_candidates`); a new group has no head and imported handles report `head_update=None`. Select a member explicitly to give that group a head. Existing archive formats are unchanged. The CLI equivalent is `msb snapshot load checkpoints/*.msb --group received`, with `--dest DIR` for another group-store root. ## SandboxHandle @@ -116,15 +131,15 @@ sb = await Sandbox.create("worker", image="python:3.12") ## Disk maintenance and incremental export -Exporting since a base omits its reusable disk layers and RAM objects. Full checkpoints keep the complete memory map and CPU/device state. The last-layers option only selects disk layers and leaves RAM payloads complete. Supply the base when loading or restoring a dependent archive; a dependent base archive must be loaded first. +Exporting since a base omits its reusable disk layers and RAM objects. Full checkpoints keep the complete memory map and CPU/device state. The last-layers option only selects disk layers and leaves RAM payloads complete. Imports resolve dependencies from the batch or an explicitly named destination group. Use an external base for missing dependencies; direct restore still accepts an explicit base. ```python worker = await Sandbox.get("worker") plan = await worker.compact(layers=3, dry_run=True) result = await worker.compact(layers=3) -await Snapshot.save("worker:checkpoint-b", "changes.msnap", since="worker:checkpoint-a") -await Snapshot.load("changes.msnap", base="worker:checkpoint-a") -child = await Sandbox.create("child", from_snapshot="changes.msnap", snapshot_base="worker:checkpoint-a") +await Snapshot.save("worker:checkpoint-b", "changes.msb", since="worker:checkpoint-a") +await Snapshot.load("changes.msb", base="worker:checkpoint-a") +child = await Sandbox.create("child", from_snapshot="changes.msb", snapshot_base="worker:checkpoint-a") ``` The count includes the oldest base but excludes the writable head. Omit `layers` to compact all sealed layers. Use `last_layers=n` instead of `since` to export the newest N sealed layers. Results are dictionaries with `input_layers`, `selected_layers`, `output_layers`, `materialized_bytes`, `total_us`, `pause_us`, and `dry_run`. See [disk-chain maintenance](/sandboxes/snapshots#export-changes-and-compact-a-disk-chain) for dependency and retention rules. @@ -332,7 +347,7 @@ Capture directly into an archive without installing a snapshot directory or inde ```python archive = await Snapshot.create_archive( "after-pip-install", - "/tmp/after-pip-install.msnap", + "/tmp/after-pip-install.msb", from_sandbox="baseline", ) ``` @@ -562,14 +577,14 @@ async def save( ```python await Snapshot.save( "baseline:after-pip-install", - "/tmp/after-pip-install.msnap", + "/tmp/after-pip-install.msb", with_parents=True, ) ``` -Bundle a snapshot into a `.msnap` archive. The existing snapshot manifest is archived as-is; create the snapshot with recorded integrity when the archive will cross a trust boundary. +Bundle a snapshot into a `.msb` archive. The existing snapshot manifest is archived as-is; create the snapshot with recorded integrity when the archive will cross a trust boundary.

Parameters

@@ -592,7 +607,7 @@ Bundle a snapshot into a `.msnap` archive. The existing snapshot manifest is arc
plain_tarbool
-
Write an uncompressed .tar instead of .msnap. Default False.
+
Write an uncompressed .tar instead of .msb. Default False.
@@ -601,7 +616,7 @@ Bundle a snapshot into a `.msnap` archive. The existing snapshot manifest is arc ```python await Snapshot.save( "baseline:after-pip-install", - "/tmp/after-pip-install.msnap", + "/tmp/after-pip-install.msb", with_parents=True, ) ``` @@ -629,14 +644,14 @@ async def load( ) -> SnapshotHandle ``` -Unpack a snapshot archive (`.msnap` or `.tar`) into the selected or generated group. The returned handle's `group` identifies the group and `head_update` reports the head selection outcome. Structural and archive-entry checks run during import; recorded payload integrity is preserved for explicit [`verify()`](#snap-verify). Compression is detected from magic bytes. +Unpack a snapshot archive (`.msb` or `.tar`) into the selected or generated group. The returned handle's `group` identifies the group and `head_update` reports the head selection outcome. Structural and archive-entry checks run during import; recorded payload integrity is preserved for explicit [`verify()`](#snap-verify). Compression is detected from magic bytes.

Parameters

archivestr | os.PathLike
-
Archive path (.msnap or .tar).
+
Archive path (.msb or .tar).
deststr | os.PathLike | None
@@ -656,7 +671,7 @@ Unpack a snapshot archive (`.msnap` or `.tar`) into the selected or generated gr ```python -h = await Snapshot.load("/tmp/after-pip-install.msnap") +h = await Snapshot.load("/tmp/after-pip-install.msb") print(h.path) ``` diff --git a/docs/sdk/rust/snapshots.mdx b/docs/sdk/rust/snapshots.mdx index 2482edcd0..874144307 100644 --- a/docs/sdk/rust/snapshots.mdx +++ b/docs/sdk/rust/snapshots.mdx @@ -16,7 +16,7 @@ use microsandbox::{Snapshot, snapshot::LoadOpts}; use std::path::Path; let snap = Snapshot::builder("baseline").from_sandbox("box").group("work").create().await?; -let loaded = Snapshot::load_with_options(Path::new("changes.msnap"), LoadOpts { +let loaded = Snapshot::load_with_options(Path::new("changes.msb"), LoadOpts { base: Some("work:baseline".into()), group: Some("work".into()), ..Default::default() @@ -27,22 +27,36 @@ let selected = Snapshot::group_head("work:baseline").await?; `LoadOpts` contains `dest: Option`, `base: Option`, `group: Option`, and `set_head: bool`. `set_head: true` explicitly selects the imported member even when it is not a descendant. `load` and `load_with_base` use generated groups. `group_head` returns `HeadUpdate` with `group`, `previous`, `head`, `reason`, and `changed`; `previous` and `head` are stable snapshot IDs. Automatic head updates initialize an empty group or advance to a proven descendant; divergent imports and imports with unknown ancestry retain the current head. `Snapshot::head_update()` and `SnapshotHandle::head_update()` expose the create/import outcome. Direct archive capture creates no group and rejects `.group(...)`. +## Load multiple archives + +```rust +let archives = vec!["changes.msb".into(), "base.msb".into()]; +let handles = Snapshot::load_many(&archives, LoadOpts { + group: Some("received".into()), + ..Default::default() +}).await?; +``` + +`Snapshot::load_many(archive_paths: &[PathBuf], opts: LoadOpts)` returns `MicrosandboxResult>`, with one handle per supplied archive in input order. Dependencies are resolved regardless of argument order from the batch and exact matching snapshots already installed in an explicitly named destination group. `opts.base` supplies an external snapshot or standalone archive when needed. Single-file `load_with_options` also reuses dependencies from its named destination group. Omitting `group` creates one generated group for the batch. + +All batch members are validated before publication. `set_head: true` requires a unique tip. With divergent tips and the default `false`, an existing head is retained (`HeadUpdateReason::AmbiguousCandidates`); a new group has no head and imported handles return `None` from `head_update()`. Select a member explicitly to give that group a head. The archive format is unchanged. The CLI equivalent is `msb snapshot load checkpoints/*.msb --group received`, with `--dest DIR` for another group-store root. + ## Disk maintenance and incremental export -Exporting since a base omits its reusable disk layers and RAM objects. Full checkpoints keep the complete memory map and CPU/device state. The last-layers option only selects disk layers and leaves RAM payloads complete. Supply the base when loading or restoring a dependent archive; a dependent base archive must be loaded first. +Exporting since a base omits its reusable disk layers and RAM objects. Full checkpoints keep the complete memory map and CPU/device state. The last-layers option only selects disk layers and leaves RAM payloads complete. Imports resolve dependencies from the batch or an explicitly named destination group. Use an external base for missing dependencies; direct restore still accepts an explicit base. ```rust let worker = Sandbox::get("worker").await?; let plan = worker.compact().layers(3).dry_run().await?; let result = worker.compact().layers(3).apply().await?; -Snapshot::save("worker:checkpoint-b", Path::new("changes.msnap"), SaveOpts { +Snapshot::save("worker:checkpoint-b", Path::new("changes.msb"), SaveOpts { since: Some("worker:checkpoint-a".into()), ..Default::default() }).await?; -Snapshot::load_with_base(Path::new("changes.msnap"), None, "worker:checkpoint-a").await?; +Snapshot::load_with_base(Path::new("changes.msb"), None, "worker:checkpoint-a").await?; ``` -`layers` counts the oldest physical layers including the base, excluding the writable head. Omit it to compact all sealed layers. `last_layers: Some(n)` is an alternative to `since`; they cannot be combined with each other or `with_parents`. For direct restore, use `Sandbox::builder("child").from_snapshot("changes.msnap").snapshot_base("worker:checkpoint-a")`. See [disk-chain maintenance](/sandboxes/snapshots#export-changes-and-compact-a-disk-chain) for dependency and retention rules. +`layers` counts the oldest physical layers including the base, excluding the writable head. Omit it to compact all sealed layers. `last_layers: Some(n)` is an alternative to `since`; they cannot be combined with each other or `with_parents`. For direct restore, use `Sandbox::builder("child").from_snapshot("changes.msb").snapshot_base("worker:checkpoint-a")`. See [disk-chain maintenance](/sandboxes/snapshots#export-changes-and-compact-a-disk-chain) for dependency and retention rules. ## Snapshot @@ -138,7 +152,7 @@ Capture a disk or full snapshot directly into an archive. The operation creates ```rust let archive = Snapshot::create_archive( Snapshot::builder("baseline").from_sandbox("api").build()?, - "/tmp/baseline.msnap", + "/tmp/baseline.msb", false, ).await?; println!("{} {}", archive.id(), archive.path().display()); @@ -351,7 +365,7 @@ println!("indexed {n} snapshots"); async fn save(name_or_path: &str, out: &Path, opts: SaveOpts) -> MicrosandboxResult<()> ``` -Bundle a snapshot into a `.msnap` archive (or plain `.tar`) at `out`. Recorded payload integrity is preserved but not executed implicitly; call [`verify()`](#snap-verify) when an independent content scan is part of your workflow. See [`SaveOpts`](#saveopts) to also include ancestors and the OCI image cache. +Bundle a snapshot into a `.msb` archive (or plain `.tar`) at `out`. Recorded payload integrity is preserved but not executed implicitly; call [`verify()`](#snap-verify) when an independent content scan is part of your workflow. See [`SaveOpts`](#saveopts) to also include ancestors and the OCI image cache.

Parameters

@@ -378,7 +392,7 @@ use std::path::Path; Snapshot::save( "api:baseline", - Path::new("/tmp/baseline.msnap"), + Path::new("/tmp/baseline.msb"), SaveOpts { with_parents: true, with_image: true, ..Default::default() }, ).await?; ``` @@ -399,13 +413,13 @@ async fn load(archive_path: &Path, dest: Option<&Path>) -> MicrosandboxResult -Unpack a snapshot archive (`.msnap` or `.tar`, detected from magic bytes) into the snapshots directory (or `dest`), routing any bundled image-cache entries into the global cache and registering everything found in the index. Structural and archive-entry checks remain mandatory, while recorded payload integrity is preserved for explicit [`verify()`](#snap-verify). Returns a handle for the head snapshot. +Unpack a snapshot archive (`.msb` or `.tar`, detected from magic bytes) into the snapshots directory (or `dest`), routing any bundled image-cache entries into the global cache and registering everything found in the index. Structural and archive-entry checks remain mandatory, while recorded payload integrity is preserved for explicit [`verify()`](#snap-verify). Returns a handle for the head snapshot.

Parameters

@@ -434,7 +448,7 @@ Unpack a snapshot archive (`.msnap` or `.tar`, detected from magic bytes) into t ```rust use std::path::Path; -let h = Snapshot::load(Path::new("/tmp/baseline.msnap"), None).await?; +let h = Snapshot::load(Path::new("/tmp/baseline.msb"), None).await?; println!("loaded {}", h.digest()); ``` diff --git a/docs/sdk/typescript/snapshots.mdx b/docs/sdk/typescript/snapshots.mdx index 56327b050..9de6b86ef 100644 --- a/docs/sdk/typescript/snapshots.mdx +++ b/docs/sdk/typescript/snapshots.mdx @@ -95,29 +95,42 @@ Installed snapshots belong to a group. A bare group selects its head; `group:mem ```typescript const snap = await Snapshot.builder("baseline").fromSandbox("box").group("work").create(); -const loaded = await Snapshot.loadWithOptions("changes.msnap", { +const loaded = await Snapshot.loadWithOptions("changes.msb", { base: "work:baseline", group: "work", }); const head = await Snapshot.groupHead("work"); const selected = await Snapshot.groupHead("work:baseline"); -await Snapshot.loadWithOptions("other.msnap", { group: "work", setHead: true }); +await Snapshot.loadWithOptions("other.msb", { group: "work", setHead: true }); ``` -`LoadOpts` contains optional `dest`, `base`, `group`, and `setHead` fields. `Snapshot.load(archive, dest?, base?)` uses a generated group. `Snapshot.groupHead(selector)` returns `HeadUpdate` with `group`, `previous`, `head`, `reason`, and `changed`; `previous` and `head` are stable snapshot IDs. Automatic head updates initialize an empty group or advance to a proven descendant; divergent imports and imports with unknown ancestry retain the current head. Reasons are `initialized`, `fast_forwarded`, `selected`, `unchanged`, `diverged`, and `unknown_ancestry`. Direct archive capture creates no group and rejects `.group(...)`. +`LoadOpts` contains optional `dest`, `base`, `group`, and `setHead` fields. `Snapshot.load(archive, dest?, base?)` uses a generated group. `Snapshot.groupHead(selector)` returns `HeadUpdate` with `group`, `previous`, `head`, `reason`, and `changed`; `previous` and `head` are stable snapshot IDs. Automatic head updates initialize an empty group or advance to a proven descendant; divergent imports and imports with unknown ancestry retain the current head. Reasons are `initialized`, `fast_forwarded`, `selected`, `unchanged`, `diverged`, `unknown_ancestry`, and `ambiguous_candidates`. Direct archive capture creates no group and rejects `.group(...)`. + +## Load multiple archives + +```typescript +const handles = await Snapshot.loadMany( + ["changes.msb", "base.msb"], + { group: "received" }, +); +``` + +`Snapshot.loadMany(archives: string[], opts?: LoadOpts)` returns `Promise`, with one handle per supplied archive in input order. Dependencies are resolved regardless of argument order from the batch and exact matching snapshots in an explicitly named destination group. `opts.base` supplies an external snapshot or standalone archive when needed. Single-file `loadWithOptions` also reuses dependencies from its named destination group. Omitting `group` creates one generated group for the batch. + +All batch members are validated before publication. `setHead: true` requires a unique tip. With divergent tips and the default `false`, an existing head is retained (`ambiguous_candidates`); a new group has no head and imported handles report `headUpdate: null`. Select a member explicitly to give that group a head. The archive format is unchanged. The CLI equivalent is `msb snapshot load checkpoints/*.msb --group received`, with `--dest DIR` for another group-store root. ## Disk maintenance and incremental export -Exporting since a base omits its reusable disk layers and RAM objects. Full checkpoints keep the complete memory map and CPU/device state. The last-layers option only selects disk layers and leaves RAM payloads complete. Supply the base when loading or restoring a dependent archive; a dependent base archive must be loaded first. +Exporting since a base omits its reusable disk layers and RAM objects. Full checkpoints keep the complete memory map and CPU/device state. The last-layers option only selects disk layers and leaves RAM payloads complete. Imports resolve dependencies from the batch or an explicitly named destination group. Use an external base for missing dependencies; direct restore still accepts an explicit base. ```typescript const worker = await Sandbox.get("worker"); const plan = await worker.compact({ layers: 3, dryRun: true }); const result = await worker.compact({ layers: 3 }); -await Snapshot.save("worker:checkpoint-b", "changes.msnap", { since: "worker:checkpoint-a" }); -await Snapshot.load("changes.msnap", undefined, "worker:checkpoint-a"); +await Snapshot.save("worker:checkpoint-b", "changes.msb", { since: "worker:checkpoint-a" }); +await Snapshot.load("changes.msb", undefined, "worker:checkpoint-a"); const child = await Sandbox.builder("child") - .fromSnapshot("changes.msnap").snapshotBase("worker:checkpoint-a").create(); + .fromSnapshot("changes.msb").snapshotBase("worker:checkpoint-a").create(); ``` The count includes the oldest base but excludes the writable head. Omit `layers` to compact all sealed layers. `lastLayers` selects the newest N sealed export layers instead of `since`. Results expose physical counts, `materializedBytes`, `totalUs`, and `pauseUs`; materialized bytes are not reclaimed space. See [disk-chain maintenance](/sandboxes/snapshots#export-changes-and-compact-a-disk-chain) for dependency and retention rules. @@ -286,7 +299,7 @@ Capture directly into an archive without installing a snapshot directory or inde ```typescript const archive = await Snapshot.builder("after-pip-install") .fromSandbox("baseline") - .createArchive("/tmp/after-pip-install.msnap"); + .createArchive("/tmp/after-pip-install.msb"); ``` --- @@ -487,7 +500,7 @@ Walk the snapshots directory (default: the configured snapshots dir) and rebuild static save(nameOrPath: string, out: string, opts?: SaveOpts): Promise ``` -Bundle a snapshot into a `.msnap` archive. The recorded manifest is archived as-is, so create the snapshot with [`recordIntegrity()`](#recordintegrity) if receivers must verify content. See [`SaveOpts`](#saveopts-interface) for bundling options. +Bundle a snapshot into a `.msb` archive. The recorded manifest is archived as-is, so create the snapshot with [`recordIntegrity()`](#recordintegrity) if receivers must verify content. See [`SaveOpts`](#saveopts-interface) for bundling options.

Parameters

@@ -509,7 +522,7 @@ Bundle a snapshot into a `.msnap` archive. The recorded manifest is archived as- ```typescript -await Snapshot.save("baseline:after-pip-install", "./baseline.msnap", { +await Snapshot.save("baseline:after-pip-install", "./baseline.msb", { withImage: true, }); ``` @@ -525,7 +538,7 @@ await Snapshot.save("baseline:after-pip-install", "./baseline.msnap", { static load(archive: string, dest?: string, base?: string): Promise ``` -Unpack a snapshot archive (`.msnap` or `.tar`) into the snapshots directory. Structural and archive-entry checks run during import; recorded payload integrity is preserved for explicit [`verify()`](#snap-verify). Compression is detected from magic bytes. +Unpack a snapshot archive (`.msb` or `.tar`) into the snapshots directory. Structural and archive-entry checks run during import; recorded payload integrity is preserved for explicit [`verify()`](#snap-verify). Compression is detected from magic bytes.

Parameters

@@ -552,7 +565,7 @@ Unpack a snapshot archive (`.msnap` or `.tar`) into the snapshots directory. Str ```typescript -const h = await Snapshot.load("./baseline.msnap"); +const h = await Snapshot.load("./baseline.msb"); console.log("loaded", h.digest); ``` diff --git a/docs/snapshot-groups-explained.md b/docs/snapshot-groups-explained.md index ae4b3bbbb..c056c2128 100644 --- a/docs/snapshot-groups-explained.md +++ b/docs/snapshot-groups-explained.md @@ -14,11 +14,11 @@ Running sandbox +-- full snapshot ----> new VM resumes saved RAM, CPUs, devices, and disk ``` -Disk snapshots also work when the source is paused or stopped. Full snapshots require resident execution state: a running or user-paused VM. Each capture produces a new immutable snapshot, even when unchanged disk layers or RAM objects are reused. Exporting a snapshot packages it as a `.msnap` archive; loading an archive installs it without starting a VM. +Disk snapshots also work when the source is paused or stopped. Full snapshots require resident execution state: a running or user-paused VM. Each capture produces a new immutable snapshot, even when unchanged disk layers or RAM objects are reused. Exporting a snapshot packages it as a `.msb` archive; loading an archive installs it without starting a VM. ## A group gives those saved points a local home -A **group** is a namespace containing snapshots and one selected **head**. Member names such as `cp01` are meaningful inside their group. Each snapshot also keeps its portable `snap_...` ID. +A **group** is a namespace containing snapshots and a selected **head**. Member names such as `cp01` are meaningful inside their group. Each snapshot also keeps its portable `snap_...` ID. A new group imported from competing branches can temporarily have no selected head; choose one explicitly before restoring by the bare group name. ```text ~/.microsandbox/snapshots/ @@ -76,7 +76,7 @@ worker:cp01 ---- worker:cp02 ---- worker:cp03 <- head The rules are small: -- Empty group: the first successful publication initializes its head. +- Empty group: a single capture or import initializes its head. A batch selects its one provably newest tip, if there is one. - Known descendant of the current head: advance automatically. - Same member, older member, sibling, unrelated history, or missing ancestry: keep the current head. The capture/import still succeeds. - Explicit selection: choose any complete installed member, including an older one. @@ -89,7 +89,7 @@ msb snapshot head worker:cp01 # Explicitly rewind There is no special `main` branch. The head is a selected snapshot, not a rule for guessing which future branch is preferred. -### What if two siblings arrive together? +### What if two separate operations publish siblings concurrently? ```text +---- snapshot A @@ -104,29 +104,54 @@ Result: both snapshots exist. Only the first head update wins. Publication checks and head replacement share a per-group lock. The losing sibling is not discarded or reported as a failed capture. If you want B, select it explicitly. Two captures of the *same* source are serialized and record a parent chain; they are not treated as sibling captures. +A **single batch containing both siblings** is different: neither argument order nor which file finishes first chooses the head. An existing group retains its head; a new group imports both members with no selected head. Then use `msb snapshot head worker:` to choose. + ## Move a history to another machine ```bash # On the source machine: -msb snapshot save worker:cp01 cp01.msnap -msb snapshot save worker:cp02 cp02.msnap --since worker:cp01 +mkdir -p checkpoints +msb snapshot save worker:cp01 checkpoints/cp01.msb +msb snapshot save worker:cp02 checkpoints/cp02.msb --since worker:cp01 # On the destination machine: -msb snapshot load cp01.msnap --group received -msb snapshot load cp02.msnap --group received --base received:cp01 +msb snapshot load checkpoints/*.msb --group received msb create --name restored --from-snapshot received --forked ``` -`--since` omits disk layers and reusable RAM objects supplied by the explicit base. Loading reconstructs a complete owned snapshot; the target does not depend on replaying earlier VMs. Each archive still includes the target's complete memory map and CPU/device state. The `.msnap` archive does not contain a local group's mutable head file: its declared archive head is the import candidate, and the receiving group applies the rules above. +The shell expands `*.msb` into archive paths. Their order and filenames do not determine ancestry or load order. You can also list them explicitly, in any order: + +```bash +msb snapshot load checkpoints/cp02.msb checkpoints/cp01.msb --group received +``` + +Loading unpacks each supplied archive once, matches omitted disk layers and RAM objects to the available payloads, and validates the reconstructed snapshots before publishing members. It looks in the supplied batch first, then the explicitly selected destination group. This works for disk-only and full incremental archives. No intermediate VM runs. + +The same automatic lookup works when archives arrive separately: + +```bash +msb snapshot load checkpoints/cp01.msb --group received +msb snapshot load checkpoints/cp02.msb --group received +``` + +`--base` is only needed when the missing data is elsewhere, such as `--base another-group:cp01` or `--base /path/to/baseline.msb`. It supplies data; it does not define ancestry or select the group head. An external archive supplied as `--base` must be standalone; include dependent archives in the batch instead. Missing dependencies and conflicting IDs, names, or duplicate labels fail before publishing any incoming members. + +Current development limitation: disk-only captures reassign layer IDs, so `--since` between successive disk-only captures can reject the base. Use standalone disk-only exports for that workflow until capture identity preservation is fixed. Full-checkpoint incremental imports were live-tested successfully; the batch loader also handles dependency-correct disk-only archives. -Loading without `--group` creates a fresh generated group. The final stdout line is the installed artifact **path**, not its ID; scripts can pass it as the next `--base`. +`--since` omits disk layers and reusable RAM objects supplied by the explicit base. Loading reconstructs a complete owned snapshot; the target does not depend on replaying earlier VMs. Each archive still includes the target's complete memory map and CPU/device state. The `.msb` archive does not contain a local group's mutable head file: its declared archive head is the import candidate, and the receiving group applies the rules above. + +Loading without `--group` creates one fresh generated group for the whole batch. The CLI prints a digest and installed artifact **path** for each input archive head, in input order. With one archive, the final line remains its installed path. With several archives, the final path is not necessarily the selected group head; use the group selector or `msb snapshot head received` instead. Repeating the same snapshot installs it only once. + +The destination directory is now an explicit `--dest DIR` option, leaving positional arguments for archive paths. Existing snapshot/archive formats are unchanged, including legacy readers. Importing an old checkpoint does not rewind an existing group. To deliberately select the imported archive's head: ```bash -msb snapshot load cp01.msnap --group received --set-head +msb snapshot load checkpoints/cp01.msb --group received --set-head ``` +For a batch, `--set-head` requires one unambiguous tip; it refuses competing tips rather than picking the last argument. Load those members without `--set-head`, then select the one you want. + Missing historical checkpoints are okay when payload dependencies are complete. But a missing parent may prevent proving a fast-forward. Filling a history hole does not retrospectively select some other retained tip; select that tip explicitly or import it again once its ancestry is known. Direct archive capture (`snapshot create --archive`) and direct archive restore still skip installed snapshot directories. `msb branch` still creates a local child without publishing a durable snapshot. Neither operation implicitly moves a group's head; a later explicit capture can join a group using the child's recorded ancestry. diff --git a/scripts/smoke/cli/checkpoint-clock.py b/scripts/smoke/cli/checkpoint-clock.py index 2979ac7a3..678d28c41 100644 --- a/scripts/smoke/cli/checkpoint-clock.py +++ b/scripts/smoke/cli/checkpoint-clock.py @@ -50,7 +50,7 @@ def run(label, *args, check=True, timeout=120): snapshot = prefix + "-next" run("capture-next", "snapshot", "create", snapshot, "--from", source, "--full", "--info") if os.environ.get("CLOCK_ARCHIVE") == "1": - archive = str(out / "clock.msnap") + archive = str(out / "clock.msb") run("archive", "snapshot", "save", snapshot, archive) snapshot = archive run("stop-source", "stop", source) diff --git a/scripts/smoke/cli/cow-memory-lifecycle.py b/scripts/smoke/cli/cow-memory-lifecycle.py index 094075887..1a87855fe 100644 --- a/scripts/smoke/cli/cow-memory-lifecycle.py +++ b/scripts/smoke/cli/cow-memory-lifecycle.py @@ -126,7 +126,7 @@ def run(label, *args, expected=0, timeout=120): run("restore-grandchild", "create", "-n", grandchild, "--from-snapshot", child_snapshot, *restore_flags, "--info") assert run("grandchild-marker", "exec", grandchild, "--", "cat", "/dev/shm/cow-marker").stdout.strip() == "private-a" - archive = str(root / "direct.msnap") + archive = str(root / "direct.msb") run("direct-full", "snapshot", "create", prefix + "-direct", "--from", source, "--full", "--archive", archive, "--info") child = prefix + "-archive" @@ -140,7 +140,7 @@ def run(label, *args, expected=0, timeout=120): run("stop-paused", "stop", source, timeout=20) disk_snapshot = prefix + "-disk" run("stopped-disk-capture", "snapshot", "create", disk_snapshot, "--from", source) - disk_archive = str(root / "disk.msnap") + disk_archive = str(root / "disk.msb") run("disk-archive", "snapshot", "save", disk_snapshot, disk_archive) for label, snapshot in (("installed", disk_snapshot), ("archive", disk_archive)): refused_name = prefix + "-refused-" + label diff --git a/scripts/smoke/cli/disk-compaction-export.sh b/scripts/smoke/cli/disk-compaction-export.sh index 950e2ba22..5e10f62d4 100644 --- a/scripts/smoke/cli/disk-compaction-export.sh +++ b/scripts/smoke/cli/disk-compaction-export.sh @@ -46,10 +46,10 @@ for layout in managed flat; do measure "$layout-save-last" msb snapshot save "$name-4" "$QUAL_ROOT/$layout-last.tar" --last-layers 2 --plain-tar measure "$layout-save-base" msb snapshot save "$name-2" "$QUAL_ROOT/$layout-base.tar.zst" measure "$layout-invalid-last-zero" refuse msb snapshot save "$name-4" "$QUAL_ROOT/invalid.tar" --last-layers 0 - measure "$layout-missing-base" refuse msb snapshot load "$QUAL_ROOT/$layout-delta.tar.zst" "$QUAL_ROOT/$layout-missing" - measure "$layout-wrong-base" refuse msb snapshot load "$QUAL_ROOT/$layout-delta.tar.zst" "$QUAL_ROOT/$layout-wrong" --base "$name-1" - measure "$layout-load-delta" msb snapshot load "$QUAL_ROOT/$layout-delta.tar.zst" "$QUAL_ROOT/$layout-import" --base "$name-2" - measure "$layout-load-base-archive" msb snapshot load "$QUAL_ROOT/$layout-last.tar" "$QUAL_ROOT/$layout-base-import" --base "$QUAL_ROOT/$layout-base.tar.zst" + measure "$layout-missing-base" refuse msb snapshot load "$QUAL_ROOT/$layout-delta.tar.zst" --dest "$QUAL_ROOT/$layout-missing" + measure "$layout-wrong-base" refuse msb snapshot load "$QUAL_ROOT/$layout-delta.tar.zst" --dest "$QUAL_ROOT/$layout-wrong" --base "$name-1" + measure "$layout-load-delta" msb snapshot load "$QUAL_ROOT/$layout-delta.tar.zst" --dest "$QUAL_ROOT/$layout-import" --base "$name-2" + measure "$layout-load-base-archive" msb snapshot load "$QUAL_ROOT/$layout-last.tar" --dest "$QUAL_ROOT/$layout-base-import" --base "$QUAL_ROOT/$layout-base.tar.zst" # A long-running agentd-managed writer remains active across preparation and the switch. msb exec "$name" -- sh -c 'i=0; while [ ! -e /writer-stop ]; do i=$((i+1)); echo "$i" >>/writes; sync; done' >"$QUAL_ROOT/logs/$layout-writer.out" 2>"$QUAL_ROOT/logs/$layout-writer.err" & diff --git a/scripts/smoke/cli/disk-compaction-negative.sh b/scripts/smoke/cli/disk-compaction-negative.sh index 42bb2c0d0..141849ff6 100644 --- a/scripts/smoke/cli/disk-compaction-negative.sh +++ b/scripts/smoke/cli/disk-compaction-negative.sh @@ -18,9 +18,9 @@ msb stop compact-neg-owned refuse msb modify compact-neg-owned --compact msb snapshot save "$QUAL_SOURCE-4" "$QUAL_ROOT/truncated.tar" --last-layers 2 --plain-tar truncate -s 2048 "$QUAL_ROOT/truncated.tar" -refuse msb snapshot load "$QUAL_ROOT/truncated.tar" "$QUAL_ROOT/truncated-import" --base "$QUAL_SOURCE-2" +refuse msb snapshot load "$QUAL_ROOT/truncated.tar" --dest "$QUAL_ROOT/truncated-import" --base "$QUAL_SOURCE-2" msb snapshot save "$QUAL_SOURCE-4" "$QUAL_ROOT/complete-last.tar" --last-layers 4 --plain-tar -msb snapshot load "$QUAL_ROOT/complete-last.tar" "$QUAL_ROOT/complete-last-import" +msb snapshot load "$QUAL_ROOT/complete-last.tar" --dest "$QUAL_ROOT/complete-last-import" msb snapshot save "$QUAL_SOURCE-4" "$QUAL_ROOT/same.tar" --since "$QUAL_SOURCE-4" --plain-tar -msb snapshot load "$QUAL_ROOT/same.tar" "$QUAL_ROOT/same-import" --base "$QUAL_SOURCE-4" +msb snapshot load "$QUAL_ROOT/same.tar" --dest "$QUAL_ROOT/same-import" --base "$QUAL_SOURCE-4" echo 'tmpfs/user-owned rejection, truncated refusal, all-layer standalone and equal-base export PASS' diff --git a/scripts/smoke/cli/incremental-full-archive.py b/scripts/smoke/cli/incremental-full-archive.py index 370380503..a8ba46347 100644 --- a/scripts/smoke/cli/incremental-full-archive.py +++ b/scripts/smoke/cli/incremental-full-archive.py @@ -91,7 +91,7 @@ def restore(label, snapshot, expected, base=None, forked=False): if n == 6: run("resume-source", source_home, "resume", source) artifact = source_home / "snapshots" / name - archive = root / f"cp{n:02}.msnap" + archive = root / f"cp{n:02}.msb" args = ["snapshot", "save", artifact, archive] if previous_source: args += ["--since", previous_source] @@ -134,7 +134,7 @@ def restore(label, snapshot, expected, base=None, forked=False): for mode in ("eager", "forked"): child = restore("installed-" + mode, final_loaded, 12, forked=mode == "forked") run("stop-installed-" + mode, dest_home, "stop", child) - complete = root / "standalone.msnap" + complete = root / "standalone.msb" run("export-standalone", source_home, "snapshot", "save", final_artifact, complete) assert all(e["included"] for e in inventory(complete)["entries"]) archive_rows[-1]["standalone_bytes"] = complete.stat().st_size diff --git a/scripts/smoke/cli/live-disk-snapshot.py b/scripts/smoke/cli/live-disk-snapshot.py index ea886ed95..0dca513ed 100644 --- a/scripts/smoke/cli/live-disk-snapshot.py +++ b/scripts/smoke/cli/live-disk-snapshot.py @@ -70,7 +70,7 @@ def sealed_hashes(root): if mode == "paused": run("pause-" + layout, "pause", source) args = ["snapshot", "create", snap, "--from", source] - archive = out / (snap + (".tar" if mode == "plain" else ".msnap")) + archive = out / (snap + (".tar" if mode == "plain" else ".msb")) installed_before = set((home / "snapshots").glob("*")) if mode in ("archive", "plain"): args += ["--archive", str(archive)] diff --git a/scripts/smoke/cli/snapshot-groups.py b/scripts/smoke/cli/snapshot-groups.py index a27bee50c..341cda209 100644 --- a/scripts/smoke/cli/snapshot-groups.py +++ b/scripts/smoke/cli/snapshot-groups.py @@ -133,8 +133,8 @@ def head(label, selector): full3, f3 = capture("capture-after-conflict", "source", "full3", full=True) assert f3["parent"] == f2["snapshot_id"] - base_archive = root / "base.msnap" - delta_archive = root / "delta.msnap" + base_archive = root / "base.msb" + delta_archive = root / "delta.msb" run("export-base", "snapshot", "save", full1, base_archive) run("export-delta", "snapshot", "save", full2, delta_archive, "--since", full1) inventory = json.loads(subprocess.check_output(["tar", "-xOf", str(delta_archive), "archive.json"])) @@ -170,7 +170,7 @@ def head(label, selector): # Direct archive capture records ancestry but never creates an installed member. before_members = sorted(str(p) for p in (home / "snapshots").rglob("snapshot.json")) - direct = root / "direct.msnap" + direct = root / "direct.msb" run("direct-capture", "snapshot", "create", "direct", "--from", "source", "--full", "--archive", direct) assert sorted(str(p) for p in (home / "snapshots").rglob("snapshot.json")) == before_members create("direct-restored", str(direct), forked=True) diff --git a/scripts/smoke/cli/snapshot-load-batch.py b/scripts/smoke/cli/snapshot-load-batch.py new file mode 100644 index 000000000..6f10b38ff --- /dev/null +++ b/scripts/smoke/cli/snapshot-load-batch.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 +"""Qualify unordered batch imports using already captured real VM checkpoints. + +The fixture root comes from snapshot-groups.py and must contain home/snapshots/work +members full1, full2, full3, local-child, and experiment. No fixture is modified. +Pass --live to restore eager/forked children after imports; all owned VMs are stopped. +Pass --file-only to qualify disk-only cp1/cp2 archives and their cold-boot restore. +""" + +import argparse +import json +import os +from pathlib import Path +import shlex +import subprocess +import time + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--binary", required=True) + parser.add_argument("--fixtures", required=True, type=Path) + parser.add_argument("--image-home", type=Path, + help="Optional home containing the fixture's fully materialized image cache") + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--live", action="store_true") + parser.add_argument("--file-only", action="store_true") + parser.add_argument("--fresh-file", action="store_true", + help="Capture two new disk-only checkpoints before the file-only matrix") + parser.add_argument("--file-standalone", action="store_true", + help="Qualify complete disk archives instead of dependent --since exports") + args = parser.parse_args() + if args.fresh_file and not (args.file_only and args.live): + parser.error("--fresh-file requires --file-only --live") + if args.file_standalone and not args.file_only: + parser.error("--file-standalone requires --file-only") + args.output.mkdir(parents=True, exist_ok=False) + home = args.output / "home" + inputs = args.output / "inputs" + inputs.mkdir() + env = dict(os.environ, MSB_HOME=str(home), MSB_BACKEND="local") + # Flat-only fixtures may not have the layered image cache required by --with-image. + # An explicitly supplied cache home must hold the same pinned image digest; export + # validates that match while reading snapshot artifacts from their original paths. + source_env = dict(env, MSB_HOME=str(args.image_home or args.fixtures / "home")) + rows, names = [], [] + report = dict(status="running", binary=args.binary, fixture=str(args.fixtures), + image_home=source_env["MSB_HOME"], home=str(home), live=args.live, + file_only=args.file_only, fresh_file=args.fresh_file, + file_standalone=args.file_standalone, rows=rows) + + def persist(): + (args.output / "report.json").write_text(json.dumps(report, indent=2)) + + def run(label, *command, fail=False, source=False, shell=False): + started = time.perf_counter() + argv = ["/bin/sh", "-c", command[0]] if shell else [args.binary, *map(str, command)] + result = subprocess.run(argv, + env=source_env if source else env, + capture_output=True, text=True, timeout=180) + row = dict(case=label, ms=round((time.perf_counter() - started) * 1000, 2), + exit=result.returncode, expected_failure=fail, + stdout=result.stdout, stderr=result.stderr) + rows.append(row) + persist() + print(json.dumps({key: row[key] for key in ("case", "ms", "exit")}), flush=True) + assert (result.returncode != 0) == fail, row + return result.stdout.strip() + + def group_head(group): + return json.loads((home / "snapshots" / group / "group.json").read_text())["head"] + + def members(group): + return sorted(path.parent.name for path in (home / "snapshots" / group).glob("*/snapshot.json")) + + def restored(mode, group, marker): + name = "batch-" + mode + names.append(name) + options = ["--forked"] if mode == "forked" else [] + run("restore-" + mode, "create", "--name", name, "--from-snapshot", group, *options) + actual = run("state-" + mode, "exec", name, "--", "sh", "-ec", + "cat /disk-marker; cat /dev/shm/marker") + assert actual == marker, actual + run("stop-" + mode, "stop", name) + + fixtures = {} + for metadata in (args.fixtures / "home/snapshots/work").glob("*/group-member.json"): + name = json.loads(metadata.read_text())["name"] + descriptor = json.loads((metadata.parent / "snapshot.json").read_text()) + fixtures[name] = (metadata.parent, descriptor) + required = (("cp1", "cp2") if args.file_only + else ("full1", "full2", "full3", "local-child", "experiment")) + assert all(name in fixtures for name in required), sorted(fixtures) + ids = [fixtures[name][1]["snapshot_id"] for name in required[:3]] + archives = [inputs / (name + ".msb") for name in ("full1", "full2", "full3")] + branch = inputs / "branch.msb" + unrelated = inputs / "unrelated.msb" + try: + if args.file_only: + if args.fresh_file: + # Start only a disposable child, never the retained fixture sandbox. + seed = inputs / "seed.msb" + run("export-file-seed", "snapshot", "save", fixtures["cp2"][0], seed, + "--with-image", source=True) + run("load-file-seed", "snapshot", "load", seed, "--group", "seed") + names.append("batch-capture") + run("create-file-source", "create", "--name", "batch-capture", "--from-snapshot", "seed") + for member, marker in (("cp1", "one"), ("cp2", "two")): + run("write-file-" + member, "exec", "batch-capture", "--", "sh", "-ec", + "echo " + marker + " > /disk-marker; sync") + output = run("capture-file-" + member, "snapshot", "create", member, + "--from", "batch-capture", "--group", "fresh") + artifact = Path(output.splitlines()[-1]) + fixtures[member] = (artifact, json.loads((artifact / "snapshot.json").read_text())) + run("stop-file-source", "stop", "batch-capture") + ids = [fixtures[name][1]["snapshot_id"] for name in required] + # These are actual disk-only artifacts, not full checkpoints restored with + # --disk-only: their inherited payloads use the file-archive layer pool. + assert all(fixtures[name][1]["state"]["kind"] == "file" for name in required) + first, second = inputs / "cp1.msb", inputs / "cp2.msb" + run("export-file-base", "snapshot", "save", fixtures["cp1"][0], first, + "--with-image", source=True) + delta_options = [] if args.file_standalone else ["--since", fixtures["cp1"][0]] + run("export-file-complete" if args.file_standalone else "export-file-delta", + "snapshot", "save", fixtures["cp2"][0], second, *delta_options, source=True) + inventory = json.loads(subprocess.check_output(["tar", "-xOf", str(second), "archive.json"])) + assert inventory["completeness"] == ("boot-complete" if args.file_standalone else "dependent") + report["file_inventory"] = inventory + if not args.file_standalone: + run("missing-file-base-refused", "snapshot", "load", second, "--group", "missing", fail=True) + assert members("missing") == [] + run("load-file-reverse", "snapshot", "load", second, first, "--group", "file") + assert members("file") == sorted(ids) + assert group_head("file") == ids[1] + for name in required: + run("verify-file-" + name, "snapshot", "verify", "file:" + name) + run("install-file-base", "snapshot", "load", first, "--group", "automatic") + run("file-auto-base", "snapshot", "load", second, "--group", "automatic") + assert group_head("automatic") == ids[1] + # The surviving member must own the full disk closure after inputs and its + # installed historical parent disappear, including borrowed archive layers. + first.unlink() + second.unlink() + run("remove-file-ancestor", "snapshot", "remove", "file:" + ids[0], "--force") + run("verify-owned-file", "snapshot", "verify", "file") + if args.live: + name = "batch-file" + names.append(name) + run("restore-file", "create", "--name", name, "--from-snapshot", "file") + actual = run("state-file", "exec", name, "--", "sh", "-ec", + "cat /disk-marker; test ! -e /dev/shm/marker") + assert actual == "two", actual + run("stop-file", "stop", name) + report["status"] = "passed" + return + # Include the pinned image once; importing the batch remains offline-capable. + run("export-baseline", "snapshot", "save", fixtures["full1"][0], archives[0], + "--with-image", source=True) + for index in (1, 2): + run("export-delta-" + str(index), "snapshot", "save", + fixtures["full" + str(index + 1)][0], archives[index], + "--since", fixtures["full" + str(index)][0], source=True) + run("export-branch", "snapshot", "save", fixtures["local-child"][0], branch, source=True) + run("export-unrelated", "snapshot", "save", fixtures["experiment"][0], unrelated, source=True) + for index in (1, 2): + inventory = json.loads(subprocess.check_output(["tar", "-xOf", str(archives[index]), "archive.json"])) + assert inventory["completeness"] == "dependent", "fixture must omit real dependencies" + report["delta_" + str(index) + "_inventory"] = inventory + + for label, order in (("reverse", (2, 1, 0)), ("shuffled", (1, 0, 2))): + output = run("load-" + label, "snapshot", "load", *(archives[index] for index in order), "--group", label) + # Results correspond to input archive heads; input order does not select the + # group's head. In reverse order, the final printed path is the oldest member. + printed_paths = [Path(line) for line in output.splitlines() if line.startswith(str(home))] + assert [path.name for path in printed_paths] == [ids[index] for index in order] + assert members(label) == sorted(ids) + assert group_head(label) == ids[2] + for name in ("full1", "full2", "full3"): + run("verify-" + label + "-" + name, "snapshot", "verify", label + ":" + name) + + wildcard_inputs = inputs / "wildcard" + wildcard_inputs.mkdir() + for archive in archives: + os.link(archive, wildcard_inputs / archive.name) + # This is a real shell expansion, unlike the explicit argument arrays above. + wildcard_command = (shlex.join([args.binary, "snapshot", "load"]) + " " + + shlex.quote(str(wildcard_inputs)) + "/*.msb --group wildcard") + run("shell-wildcard", wildcard_command, shell=True) + assert members("wildcard") == sorted(ids) + assert group_head("wildcard") == ids[2] + + run("group-base-install", "snapshot", "load", archives[0], "--group", "automatic") + run("group-base-auto", "snapshot", "load", archives[2], archives[1], "--group", "automatic") + assert group_head("automatic") == ids[2] + assert members("automatic") == sorted(ids) + + # A destination is a group-store root, not another archive argument. Automatic + # dependency lookup must also respect an explicitly selected nondefault root. + alternate = args.output / "alternate-store" + run("alternate-base", "snapshot", "load", archives[0], "--dest", alternate, "--group", "work") + run("alternate-auto-base", "snapshot", "load", archives[2], archives[1], + "--dest", alternate, "--group", "work") + assert json.loads((alternate / "work/group.json").read_text())["head"] == ids[2] + run("verify-alternate", "snapshot", "verify", alternate / "work" / ids[2]) + + run("missing-base-refused", "snapshot", "load", archives[2], "--group", "missing", fail=True) + assert members("missing") == [] + run("unrelated-base-refused", "snapshot", "load", archives[2], "--base", unrelated, + "--group", "unrelated", fail=True) + assert members("unrelated") == [] + # A complete external base supplies payloads, not mandatory imported history. + run("external-base", "snapshot", "load", archives[2], "--base", fixtures["full2"][0], + "--group", "hole") + assert members("hole") == [ids[2]] + assert group_head("hole") == ids[2] + run("verify-history-hole", "snapshot", "verify", "hole") + + run("duplicates", "snapshot", "load", archives[0], archives[0], "--group", "duplicates") + assert members("duplicates") == [ids[0]] + run("ambiguous-new", "snapshot", "load", archives[1], branch, archives[0], "--group", "branches") + assert group_head("branches") is None + assert len(members("branches")) == 3 + run("headless-restore-refused", "snapshot", "inspect", "branches", fail=True) + run("explicit-branch-selection", "snapshot", "head", "branches:local-child") + assert group_head("branches") == fixtures["local-child"][1]["snapshot_id"] + + run("ambiguity-base", "snapshot", "load", archives[0], "--group", "retained") + run("ambiguity-retains", "snapshot", "load", archives[1], branch, "--group", "retained") + assert group_head("retained") == ids[0] + run("ambiguous-forced-head-refused", "snapshot", "load", archives[1], branch, archives[0], + "--group", "rejected", "--set-head", fail=True) + assert members("rejected") == [] + + # Delete only this harness's exports, never the retained source fixtures. Loaded + # snapshots must keep working with their own dependency-complete disk/RAM files. + for archive in inputs.rglob("*.msb"): + assert archive.is_file() + archive.unlink() + for index in (0, 1): + run("remove-installed-ancestor-" + str(index), "snapshot", "remove", + "reverse:" + ids[index], "--force") + run("verify-owned-final", "snapshot", "verify", "reverse") + assert members("reverse") == [ids[2]] + if args.live: + restored("eager", "reverse", "three\nram-three") + restored("forked", "reverse", "three\nram-three") + report["status"] = "passed" + except Exception as error: + report.update(status="failed", error=repr(error)) + raise + finally: + cleanup = [] + for name in names: + result = subprocess.run([args.binary, "stop", name], env=env, + capture_output=True, text=True, timeout=30) + cleanup.append(dict(name=name, exit=result.returncode, stderr=result.stderr)) + report["cleanup"] = cleanup + persist() + + +if __name__ == "__main__": + main() diff --git a/scripts/smoke/reports/cow-memory-lifecycle-2026-09-07.md b/scripts/smoke/reports/cow-memory-lifecycle-2026-09-07.md index ebd209bb2..d75301788 100644 --- a/scripts/smoke/reports/cow-memory-lifecycle-2026-09-07.md +++ b/scripts/smoke/reports/cow-memory-lifecycle-2026-09-07.md @@ -1,5 +1,7 @@ # CoW memory and resident lifecycle — 2026-09-07 +Archive names in this report use the current `.msb` convention. Retained raw logs preserve the filenames used in the original runs. + Follow-up: [execution-state fixes and qualification](execution-state-2026-09-07.md) supersedes the Windows post-restore failure status below and adds Linux ARM64 coverage. The observations below describe the earlier backend revision. Status: development integration and live smoke coverage, not full platform or performance qualification. Microsandbox #8 remains stacked directly on #7 `ce04099b`, with libkrun `94d680b21bf7ea7c2bed5262ed211833bd4379bd` and firmware `6cca413ac248f63e65d4ea4748b3bc36cd1b22f3`. The kernel and agentd used below were built from matching development sources on the authorized OVH host, including the ARM64 guest artifacts used on macOS and Windows. @@ -14,7 +16,7 @@ The opt-in language SDK tests are `sdk/python/tests/test_cow_lifecycle.py`, `sdk ## Observed coverage -Linux/KVM x86-64 passed CoW flat, managed, and tmpfs roots, plus a standard-memory flat-root baseline. macOS/HVF ARM64 passed the same root/memory variants. The checks exercise fresh construction, running full capture, idempotent pause/resume, host-observed Paused status, prompt rejection of new guest exec while paused, two successive full captures while retaining pause, installed-snapshot restore into two children, private child writes, direct full `.msnap` capture/restore, survival after input-archive unlink, and stop from paused. Completed runs stopped their test VMs; retained snapshot/cache artifacts remain in the isolated test homes for inspection. +Linux/KVM x86-64 passed CoW flat, managed, and tmpfs roots, plus a standard-memory flat-root baseline. macOS/HVF ARM64 passed the same root/memory variants. The checks exercise fresh construction, running full capture, idempotent pause/resume, host-observed Paused status, prompt rejection of new guest exec while paused, two successive full captures while retaining pause, installed-snapshot restore into two children, private child writes, direct full `.msb` capture/restore, survival after input-archive unlink, and stop from paused. Completed runs stopped their test VMs; retained snapshot/cache artifacts remain in the isolated test homes for inspection. The later Linux flat/managed/tmpfs/standard runs and macOS tmpfs run additionally assert unchanged Linux boot ID across ordinary pause/resume, resumed progress of the original counter workload, and guest wall clock within three seconds of the host after a ten-second pause. These checks do not constitute host-suspend, every clock-failure, or VM Generation ID notification testing. diff --git a/scripts/smoke/reports/live-disk-snapshot-2026-09-09.md b/scripts/smoke/reports/live-disk-snapshot-2026-09-09.md index 1108d4e1d..a3a6b112b 100644 --- a/scripts/smoke/reports/live-disk-snapshot-2026-09-09.md +++ b/scripts/smoke/reports/live-disk-snapshot-2026-09-09.md @@ -1,5 +1,7 @@ # Live disk-only snapshots — 2026-09-09 +Archive names in this report use the current `.msb` convention. Retained raw logs preserve the filenames used in the original runs. + Implemented on the #8 branch above Microsandbox `f68c1329`, using existing pinned libkrun `862d6842`, rust-vmm `f798d4f2`, and matching firmware. No companion changes, dependency overrides, schema change, or new public flags were required. ## Behavior @@ -10,8 +12,8 @@ msb snapshot create saved --from source msb create --name child --from-snapshot saved # Direct archive: no installed snapshot directory or index row. -msb snapshot create exported --from source --archive ./exported.msnap -msb create --name archive-child --from-snapshot ./exported.msnap +msb snapshot create exported --from source --archive ./exported.msb +msb create --name archive-child --from-snapshot ./exported.msb ``` Running and user-paused managed/flat OCI roots use the serialized runtime control executor and a distinct capability-gated `disk_checkpoint_create` operation. Rollover seals the disk and selects a private successor. Running sources resume before SDK packaging; user-paused sources stay paused. Stopped/crashed copies retain their lifecycle lock. The SDK packages only the immutable disk closure, rechecks source identity, and removes consumed staging. @@ -27,7 +29,7 @@ The committed `scripts/smoke/cli/live-disk-snapshot.py` passed on macOS ARM64/HV | Checks, on both root layouts | Mac | Linux | Windows | | --- | --- | --- | --- | | Running installed capture and cold restore; optional integrity | Pass | Pass | Pass | -| Direct compressed `.msnap` and plain `.tar`; no installed intermediate | Pass | Pass | Pass | +| Direct compressed `.msb` and plain `.tar`; no installed intermediate | Pass | Pass | Pass | | User-paused capture remains paused; exec refuses until explicit resume | Pass | Pass | Pass | | Source RAM/boot ID retained; disk child has a new boot ID and no tmpfs marker | Pass | Pass | Pass | | Source/child writes isolated; sealed payload hashes unchanged | Pass | Pass | Pass | diff --git a/scripts/smoke/reports/snapshot-load-batch-2026-09-10.md b/scripts/smoke/reports/snapshot-load-batch-2026-09-10.md new file mode 100644 index 000000000..f7562e239 --- /dev/null +++ b/scripts/smoke/reports/snapshot-load-batch-2026-09-10.md @@ -0,0 +1,116 @@ +# Snapshot batch-load qualification — 2026-09-10 + +Archive names in the original qualification below use the current `.msb` convention. Retained raw logs preserve the filenames used at the time; post-rename qualification is recorded separately. + +## Result + +Batch loading passed on macOS ARM64/HVF with real full-checkpoint archives for both flat and managed root disks, including eager and forked restoration. Real complete disk-only archive batches also passed for both layouts. Fresh disk-only `--since` export is blocked by an existing producer limitation, described below; this is not counted as passing dependent File-archive live coverage. + +The final signed development binary was `/private/tmp/msb-batch-load-final`, SHA-256 `269bcba93e00b1e04b98a92386a301571dc0b59f2f2b0d90e52a2aa46e28541a`. Runtime tests used `/private/tmp/msb-forked-build.8HN0Ie/lib/libkrunfw.5.dylib` and `/private/tmp/msb-cow-8.PIhgYp/build/agentd`. Existing real fixtures were 256 MiB RAM, two vCPUs, and 512 MiB root disks. The full matrices ran in parallel in isolated homes. These are CLI wall times from qualification runs, not release-build performance benchmarks or stop-the-world measurements. + +## Live coverage + +| Check | Flat root | Managed root | +| --- | --- | --- | +| Three full archives, supplied in reverse and shuffled order | Pass | Pass | +| Actual shell-expanded `*.msb` batch | Pass | Pass | +| Returned paths retain input order while group head selects the unique tip | Pass | Pass | +| Automatic dependency resolution from the named destination group | Pass | Pass | +| `--dest` store root and automatic base lookup there | Pass | Pass | +| Missing disk/RAM closure and unrelated external base rejected before member publication | Pass | Pass | +| Complete external base supplies payloads without importing historical ancestors | Pass | Pass | +| Duplicate inputs install once | Pass | Pass | +| Ambiguous new group remains headless; explicit head selection works | Pass | Pass | +| Ambiguous existing group retains its head | Pass | Pass | +| Ambiguous `--set-head` rejected without published members | Pass | Pass | +| Full imported final member survives removal of inputs and its installed ancestors | Pass | Pass | +| Eager/forked restore retains `/disk-marker` and `/dev/shm/marker` | Pass | Pass | +| Complete disk-only archives load in reverse order and select the child head | Pass | Pass | +| Disk-only final member survives removal of inputs and its installed ancestor | Pass | Pass | +| Disk-only cold boot retains disk marker and does not restore tmpfs marker | Pass | Pass | +| Fresh disk-only dependent `--since` archive creation | Blocked: existing layer-ID issue | Blocked: existing layer-ID issue | + +Each final full matrix checked 39 command outcomes, including expected errors. Each complete disk-only matrix checked 12: 102 checked command outcomes in the four successful runs. All six VMs started by those final matrices were stopped, as were the two additional fresh-capture probe VMs. A final process inspection found no matching test harness or VM process remaining. + +## Observed timings + +| CLI operation | Flat root | Managed root | +| --- | ---: | ---: | +| Load three full archives, reverse order | 3,853.53 ms | 2,511.74 ms | +| Load three full archives, shuffled order | 3,604.32 ms | 3,234.55 ms | +| Load full archive shell wildcard | 3,621.72 ms | 2,752.73 ms | +| Load two full deltas using installed group base | 2,070.18 ms | 1,467.89 ms | +| Same automatic dependency lookup under `--dest` | 2,058.32 ms | 1,467.14 ms | +| Load final full delta with explicit complete external base | 1,074.19 ms | 751.28 ms | +| Restore imported final checkpoint, eager | 958.19 ms | 763.53 ms | +| Restore imported final checkpoint, forked | 970.45 ms | 774.82 ms | +| Load two complete disk-only archives, reverse order | 707.11 ms | 474.26 ms | +| Cold boot imported final disk-only snapshot | 422.90 ms | 329.61 ms | + +Restore times cover the `msb create` invocation. Guest marker checks were separate successful `msb exec` commands. No claim is made that these times isolate memory mapping, disk preparation, or guest readiness from the rest of the CLI pipeline. + +## Producer limitations found during qualification + +1. `--with-image` currently requires materialized layered image-cache artifacts, including fsmeta and VMDK, even for a flat snapshot. The old flat-only fixture lacked those files. The successful flat runs exported the original flat snapshot paths using the managed fixture's already populated image cache with the identical pinned image digest. The harness exposes this as `--image-home`; it does not modify either fixture. The first failed attempt is retained in `/private/tmp/sblf/report.json`. +2. Consecutive disk-only snapshots currently receive fresh IDs for every copied disk layer. `build_artifact` and `new_file_manifest` in `sdk/rust/lib/snapshot/create.rs` allocate random layer IDs for the whole source closure. The physical-prefix requirement therefore rejects `msb snapshot save work:child child.msb --since work:parent` even when parent and child were captured consecutively from the same running sandbox. This was reproduced on the final binary with new captures on both layouts, not only old fixtures. No producer implementation was changed in this work, and no archive metadata was fabricated to make the live test pass. The strict dependent-File test remains available and fails at export. Complete File batch import was qualified separately. Synthetic dependent-File import tests cover the loader independently of this existing exporter limitation. + +## Automated checks + +- Snapshot library: 75 tests passed, including a six-archive mixed-codec reverse-order chain, RAM-only and disk-plus-RAM dependency resolution, missing/corrupt RAM, borrowed File payload checks, and dependent File archives in both input orders. +- Snapshot artifact integration: 55 tests passed, covering legacy single load, ordering, duplicates, conflicting aliases/IDs/labels, branch/head behavior, corruption before publication, missing historical ancestors with complete payloads, and independent installed payload ownership. +- CLI snapshot tests: 12 passed. +- Rust native checks: Python, Node, and Go bindings passed. +- Node: native build, 14 focused tests, and type checking passed. +- Python: one stub-surface test and Ruff passed. +- Go: unit/native checks and integration test compilation passed; VM-backed Go integration execution was not run. +- Targeted Microsandbox/CLI Clippy passed with `--no-deps -D warnings -A clippy::too_many_arguments`; formatting and diff checks passed. + +The automated counts above include checks run by the coordinating agent and the SDK agent. Live qualification in this report was macOS-only; Linux and Windows were not rerun for this batch-load change. + +## Reproduction and retained evidence + +Use a fresh output directory for each invocation. The harness never restarts or deletes the retained source fixtures. It removes only its own exported input archives and selected imported ancestor snapshots to verify dependency independence. + +```bash +export MSB_LIBKRUNFW_PATH=/private/tmp/msb-forked-build.8HN0Ie/lib/libkrunfw.5.dylib +export MSB_AGENTD_PATH=/private/tmp/msb-cow-8.PIhgYp/build/agentd + +python3 scripts/smoke/cli/snapshot-load-batch.py \ + --binary /private/tmp/msb-batch-load-final \ + --fixtures /private/tmp/sgpf \ + --image-home /private/tmp/sgpm/home \ + --output /private/tmp/batch-flat-new-run --live + +# Repeat with --fixtures /private/tmp/sgpm and a different output directory. +# Complete File archive coverage: add --file-only --file-standalone. +# Reproduce the producer limitation: add --file-only --fresh-file instead. +``` + +Raw JSON reports contain every command result, full output, timing, archive inventories, and cleanup results: + +| Run | Report | +| --- | --- | +| Final full flat | `/private/tmp/sblf-final/report.json` | +| Final full managed | `/private/tmp/sblm-final/report.json` | +| Final complete File flat | `/private/tmp/sblf-file-standalone-r2/report.json` | +| Final complete File managed | `/private/tmp/sblm-file-standalone-r2/report.json` | +| Fresh File delta producer failure, flat | `/private/tmp/sblf-fresh-file/report.json` | +| Fresh File delta producer failure, managed | `/private/tmp/sblm-fresh-file/report.json` | + +An earlier standalone-File harness attempt incorrectly expected the archive completeness spelling `complete`; the assertion was corrected to the actual `boot-complete` enum before the successful final File runs. No product assertion or dependency check was relaxed. + +## `.msb` extension follow-up + +The source, CLI help, SDK examples, and live harness now use `.msb`. This is only a naming convention: explicit paths are preserved, compressed/plain-tar decoding remains content-based, and the archive and descriptor schemas are unchanged. The direct-archive unit matrix passed all eight compression/filename combinations (`.msb`, `.tar.zst`, `.tar`, and extensionless, each compressed and plain). Snapshot library tests (75), artifact integration tests (55), CLI tests (12), TypeScript tests (14) and type checking, the Python stub test, Go unit tests, formatting, and diff checks passed again. + +The signed follow-up binary is `/private/tmp/msb-extension-final`, SHA-256 `3326644fcc666d55bb72c41b851b1f7f0c28b5bb06ca7095914421a0d28b8e24`. The same fixtures and runtime artifacts were used. Both full matrices and both complete File matrices passed again with newly exported `.msb` archives: 102 checked command outcomes, including actual `*.msb` shell expansion, dependent full imports, eager/forked full restore, and disk-only cold boot. All six test VMs were stopped. Linux and Windows were not rerun for this extension-only change. + +| Follow-up operation | Flat root | Managed root | +| --- | ---: | ---: | +| Load three full archives, reverse order | 3,442.06 ms | 2,441.31 ms | +| Restore imported full checkpoint, eager | 939.44 ms | 764.74 ms | +| Restore imported full checkpoint, forked | 955.37 ms | 789.54 ms | +| Load two complete disk-only archives | 803.58 ms | 477.47 ms | +| Cold boot imported disk-only snapshot | 430.29 ms | 336.07 ms | + +These remain development-build qualification timings, not isolated performance comparisons. Reports: `/private/tmp/sxbf2/report.json` (full flat), `/private/tmp/sxbm2/report.json` (full managed), `/private/tmp/sxbfd2/report.json` (File flat), and `/private/tmp/sxbmd2/report.json` (File managed). Initial attempts under the tool sandbox passed archive checks but were denied VM endpoint creation (`Operation not permitted`); their failed reports remain under `/private/tmp/sxbf`, `/private/tmp/sxbm`, `/private/tmp/sxbfd`, and `/private/tmp/sxbmd`. The passing reruns used host permissions without changing product code or relaxing assertions. The disk-only dependent-export limitation above remains unchanged. diff --git a/sdk/go/integration/snapshot_test.go b/sdk/go/integration/snapshot_test.go index ca9789ad6..071689c48 100644 --- a/sdk/go/integration/snapshot_test.go +++ b/sdk/go/integration/snapshot_test.go @@ -276,6 +276,21 @@ func TestSnapshotCreateAndSnapshotDirectoryOps(t *testing.T) { if _, err := microsandbox.Snapshot.Open(loadCtx, snapshotDir); err != nil { t.Fatalf("removing an import affected the original snapshot: %v", err) } + + // A repeated archive still yields one result per input, but one batch group + // installs the identical snapshot only once. + batch, err := microsandbox.Snapshot.LoadMany(loadCtx, + []string{archivePath, archivePath}, microsandbox.SnapshotLoadOptions{Dest: importDir}) + if err != nil { + t.Fatalf("Snapshot.LoadMany: %v", err) + } + if len(batch) != 2 || batch[0].ID() != artifact.ID() || batch[1].ID() != artifact.ID() { + t.Fatalf("Snapshot.LoadMany returned unexpected handles: %#v", batch) + } + t.Cleanup(func() { removeSnapshotBestEffort(batch[0].Path()) }) + if batch[0].Path() != batch[1].Path() || batch[0].Group() == nil { + t.Fatal("batch should reuse one installed copy in one generated group") + } } func logSnapshotPhase(t *testing.T, phase string, started time.Time) { diff --git a/sdk/go/internal/ffi/ffi.go b/sdk/go/internal/ffi/ffi.go index c877507f5..d9ef6cf71 100644 --- a/sdk/go/internal/ffi/ffi.go +++ b/sdk/go/internal/ffi/ffi.go @@ -233,6 +233,7 @@ typedef char *(*msb_snapshot_export_fn)(uint64_t cancel_id, const char *name_or_ typedef char *(*msb_snapshot_import_fn)(uint64_t cancel_id, const char *archive, const char *dest, uint8_t *buf, size_t buf_len); typedef char *(*msb_snapshot_import_with_base_fn)(uint64_t cancel_id, const char *archive, const char *dest, const char *base, uint8_t *buf, size_t buf_len); typedef char *(*msb_snapshot_import_with_options_fn)(uint64_t cancel_id, const char *archive, const char *opts_json, uint8_t *buf, size_t buf_len); +typedef char *(*msb_snapshot_import_many_fn)(uint64_t cancel_id, const char *archives_json, const char *opts_json, uint8_t *buf, size_t buf_len); typedef char *(*msb_snapshot_group_head_fn)(uint64_t cancel_id, const char *selector, uint8_t *buf, size_t buf_len); typedef char *(*msb_sandbox_compact_fn)(uint64_t cancel_id, uint64_t handle, const char *name, const char *opts, uint8_t *buf, size_t buf_len); @@ -399,6 +400,7 @@ static msb_snapshot_export_fn ptr_msb_snapshot_export = NULL; static msb_snapshot_import_fn ptr_msb_snapshot_import = NULL; static msb_snapshot_import_with_base_fn ptr_msb_snapshot_import_with_base = NULL; static msb_snapshot_import_with_options_fn ptr_msb_snapshot_import_with_options = NULL; +static msb_snapshot_import_many_fn ptr_msb_snapshot_import_many = NULL; static msb_snapshot_group_head_fn ptr_msb_snapshot_group_head = NULL; static msb_sandbox_compact_fn ptr_msb_sandbox_compact = NULL; @@ -584,6 +586,7 @@ const char *load_microsandbox(const char *path) { RESOLVE(msb_snapshot_import); RESOLVE(msb_snapshot_import_with_base); RESOLVE(msb_snapshot_import_with_options); + RESOLVE(msb_snapshot_import_many); RESOLVE(msb_snapshot_group_head); RESOLVE(msb_sandbox_compact); return NULL; @@ -1027,6 +1030,9 @@ char *call_msb_snapshot_import_with_base(uint64_t cancel_id, const char *archive char *call_msb_snapshot_import_with_options(uint64_t cancel_id, const char *archive, const char *opts_json, uint8_t *buf, size_t buf_len) { return ptr_msb_snapshot_import_with_options ? ptr_msb_snapshot_import_with_options(cancel_id, archive, opts_json, buf, buf_len) : NULL; } +char *call_msb_snapshot_import_many(uint64_t cancel_id, const char *archives_json, const char *opts_json, uint8_t *buf, size_t buf_len) { + return ptr_msb_snapshot_import_many ? ptr_msb_snapshot_import_many(cancel_id, archives_json, opts_json, buf, buf_len) : NULL; +} char *call_msb_snapshot_group_head(uint64_t cancel_id, const char *selector, uint8_t *buf, size_t buf_len) { return ptr_msb_snapshot_group_head ? ptr_msb_snapshot_group_head(cancel_id, selector, buf, buf_len) : NULL; } @@ -5123,6 +5129,38 @@ func SnapshotLoadWithOptions(ctx context.Context, archive string, opts SnapshotL return &info, nil } +func SnapshotLoadMany(ctx context.Context, archives []string, opts SnapshotLoadOptions) ([]*SnapshotHandleInfo, error) { + if err := ensureLoaded(); err != nil { + return nil, err + } + // A nil slice is an empty batch, not JSON null; the core validates empty batches. + if archives == nil { + archives = []string{} + } + archivePayload, err := json.Marshal(archives) + if err != nil { + return nil, err + } + optsPayload, err := json.Marshal(opts) + if err != nil { + return nil, err + } + cArchives, cOpts := C.CString(string(archivePayload)), C.CString(string(optsPayload)) + defer C.free(unsafe.Pointer(cArchives)) + defer C.free(unsafe.Pointer(cOpts)) + out, err := call(ctx, func(cancelID C.uint64_t, buf *C.uint8_t, bufLen C.size_t) *C.char { + return C.call_msb_snapshot_import_many(cancelID, cArchives, cOpts, buf, bufLen) + }) + if err != nil { + return nil, err + } + var infos []*SnapshotHandleInfo + if err := json.Unmarshal([]byte(out), &infos); err != nil { + return nil, fmt.Errorf("parse snapshot batch load: %w", err) + } + return infos, nil +} + func SnapshotGroupHead(ctx context.Context, selector string) (*SnapshotHeadUpdate, error) { if err := ensureLoaded(); err != nil { return nil, err diff --git a/sdk/go/native/microsandbox_go_ffi.h b/sdk/go/native/microsandbox_go_ffi.h index 90bda08c5..fdbab92dc 100644 --- a/sdk/go/native/microsandbox_go_ffi.h +++ b/sdk/go/native/microsandbox_go_ffi.h @@ -824,6 +824,15 @@ char *msb_snapshot_import_with_options(uint64_t cancel_id, unsigned char *buf, uintptr_t buf_len); +/** + * Import archives together with dependencies resolved within the batch and destination group. + */ +char *msb_snapshot_import_many(uint64_t cancel_id, + const char *archives_json, + const char *opts_json, + unsigned char *buf, + uintptr_t buf_len); + /** * Read a group head, or select a `group:member` as its head. */ diff --git a/sdk/go/native/src/lib.rs b/sdk/go/native/src/lib.rs index a9dfe749d..c27f27840 100644 --- a/sdk/go/native/src/lib.rs +++ b/sdk/go/native/src/lib.rs @@ -6301,6 +6301,40 @@ pub unsafe extern "C" fn msb_snapshot_import_with_options( }) } +/// Import archives together with dependencies resolved within the batch and destination group. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn msb_snapshot_import_many( + cancel_id: u64, + archives_json: *const c_char, + opts_json: *const c_char, + buf: *mut c_uchar, + buf_len: usize, +) -> *mut c_char { + run_c(cancel_id, buf, buf_len, || { + let archives_raw = unsafe { cstr(archives_json) }?; + let archives: Vec = serde_json::from_str(&archives_raw) + .map_err(|error| FfiError::invalid_argument(error.to_string()))?; + let opts_raw = unsafe { cstr(opts_json) }?; + let opts: SnapshotLoadOptsJson = serde_json::from_str(&opts_raw) + .map_err(|error| FfiError::invalid_argument(error.to_string()))?; + Ok(Box::pin(async move { + let handles = Snapshot::load_many( + &archives, + microsandbox::snapshot::LoadOpts { + dest: opts.dest, + base: opts.base, + group: opts.group, + set_head: opts.set_head, + }, + ) + .await + .map_err(FfiError::from)?; + let values = handles.iter().map(snapshot_handle_json).collect::>(); + Ok(serde_json::Value::Array(values).to_string()) + })) + }) +} + /// Read a group head, or select a `group:member` as its head. #[unsafe(no_mangle)] pub unsafe extern "C" fn msb_snapshot_group_head( diff --git a/sdk/go/snapshot.go b/sdk/go/snapshot.go index baac361b1..807b1f411 100644 --- a/sdk/go/snapshot.go +++ b/sdk/go/snapshot.go @@ -40,15 +40,15 @@ type SnapshotSaveOptions struct { PlainTar bool } -// SnapshotLoadOptions configures importing an archive into a snapshot group. +// SnapshotLoadOptions configures importing one or more archives into a snapshot group. type SnapshotLoadOptions struct { // Parent directory containing snapshot groups; empty selects the default. Dest string - // Exact base snapshot or standalone archive for a dependent archive. + // External snapshot or standalone archive for dependencies absent from the batch/group. Base string // Destination group; generated when empty. Group string - // Select the imported member even when it is not a fast-forward. + // Select the unique imported tip even when it is not a fast-forward. SetHead bool } @@ -433,6 +433,24 @@ func (snapshotFactory) LoadWithOptions(ctx context.Context, archive string, opts return snapshotHandleFromInfo(info), nil } +// LoadMany imports archives together into one group, resolving dependencies regardless of input order. +func (snapshotFactory) LoadMany(ctx context.Context, archives []string, opts SnapshotLoadOptions) ([]*SnapshotHandle, error) { + infos, err := ffi.SnapshotLoadMany(ctx, archives, ffi.SnapshotLoadOptions{ + Dest: opts.Dest, + Base: opts.Base, + Group: opts.Group, + SetHead: opts.SetHead, + }) + if err != nil { + return nil, wrapFFI(err) + } + handles := make([]*SnapshotHandle, len(infos)) + for index, info := range infos { + handles[index] = snapshotHandleFromInfo(info) + } + return handles, nil +} + // GroupHead reads a group head, or selects a group:member as its head. func (snapshotFactory) GroupHead(ctx context.Context, selector string) (*SnapshotHeadUpdate, error) { update, err := ffi.SnapshotGroupHead(ctx, selector) diff --git a/sdk/node-ts/native/index.d.ts b/sdk/node-ts/native/index.d.ts index 81d12fb47..090489c2f 100644 --- a/sdk/node-ts/native/index.d.ts +++ b/sdk/node-ts/native/index.d.ts @@ -1510,6 +1510,8 @@ export declare class Snapshot { static save(nameOrPath: string, out: string, opts?: SaveOpts | undefined | null): Promise static load(archive: string, dest?: string | undefined | null, base?: string | undefined | null): Promise static loadWithOptions(archive: string, opts?: LoadOpts | undefined | null): Promise + /** Import archives together, resolving dependencies within the batch and destination group. */ + static loadMany(archives: Array, opts?: LoadOpts | undefined | null): Promise> /** Read a group's head, or select `group:member` as its head. */ static groupHead(selector: string): Promise get path(): string @@ -1976,15 +1978,15 @@ export interface JsSandboxPage { nextCursor?: string } -/** Options for importing an archive into a snapshot group. */ +/** Options for importing one or more archives into a snapshot group. */ export interface LoadOpts { /** Parent directory containing snapshot groups. */ dest?: string - /** Exact base snapshot or standalone archive for a dependent archive. */ + /** External snapshot or standalone archive for dependencies absent from the batch/group. */ base?: string /** Destination group (generated when omitted). */ group?: string - /** Select the imported member even when it is not a fast-forward. */ + /** Select the unique imported tip even when it is not a fast-forward. */ setHead?: boolean } diff --git a/sdk/node-ts/native/snapshot.rs b/sdk/node-ts/native/snapshot.rs index 55c0749b2..5a7386878 100644 --- a/sdk/node-ts/native/snapshot.rs +++ b/sdk/node-ts/native/snapshot.rs @@ -53,17 +53,17 @@ pub struct JsSaveOpts { pub last_layers: Option, } -/// Options for importing an archive into a snapshot group. +/// Options for importing one or more archives into a snapshot group. #[derive(Default)] #[napi(object, js_name = "LoadOpts")] pub struct JsLoadOpts { /// Parent directory containing snapshot groups. pub dest: Option, - /// Exact base snapshot or standalone archive for a dependent archive. + /// External snapshot or standalone archive for dependencies absent from the batch/group. pub base: Option, /// Destination group (generated when omitted). pub group: Option, - /// Select the imported member even when it is not a fast-forward. + /// Select the unique imported tip even when it is not a fast-forward. pub set_head: Option, } @@ -247,6 +247,31 @@ impl JsSnapshot { Ok(JsSnapshotHandle::from_rust(h)) } + /// Import archives together, resolving dependencies within the batch and destination group. + #[napi(js_name = "loadMany")] + pub async fn load_many( + archives: Vec, + opts: Option, + ) -> Result> { + let opts = opts.unwrap_or_default(); + let paths = archives.into_iter().map(PathBuf::from).collect::>(); + let handles = RustSnapshot::load_many( + &paths, + RustLoadOpts { + dest: opts.dest.map(PathBuf::from), + base: opts.base, + group: opts.group, + set_head: opts.set_head.unwrap_or(false), + }, + ) + .await + .map_err(to_napi_error)?; + Ok(handles + .into_iter() + .map(JsSnapshotHandle::from_rust) + .collect()) + } + /// Read a group's head, or select `group:member` as its head. #[napi(js_name = "groupHead")] pub async fn group_head(selector: String) -> Result { @@ -637,6 +662,7 @@ fn head_update_to_js(update: µsandbox::snapshot::HeadUpdate) -> JsHeadUpdat HeadUpdateReason::Unchanged => "unchanged", HeadUpdateReason::Diverged => "diverged", HeadUpdateReason::UnknownAncestry => "unknown_ancestry", + HeadUpdateReason::AmbiguousCandidates => "ambiguous_candidates", }; JsHeadUpdate { group: update.group.clone(), diff --git a/sdk/node-ts/src/internal/napi.ts b/sdk/node-ts/src/internal/napi.ts index 94cbc2d6e..44a0f46c9 100644 --- a/sdk/node-ts/src/internal/napi.ts +++ b/sdk/node-ts/src/internal/napi.ts @@ -560,6 +560,7 @@ export interface NapiSnapshotStatic { save(name: string, out: string, opts?: NapiSaveOpts): Promise; load(archive: string, dest?: string, base?: string): Promise; loadWithOptions(archive: string, opts?: NapiLoadOpts): Promise; + loadMany(archives: string[], opts?: NapiLoadOpts): Promise; groupHead(selector: string): Promise; } diff --git a/sdk/node-ts/src/snapshot.ts b/sdk/node-ts/src/snapshot.ts index 8ecdcaac0..2f9c02100 100644 --- a/sdk/node-ts/src/snapshot.ts +++ b/sdk/node-ts/src/snapshot.ts @@ -64,15 +64,15 @@ export interface SaveOpts { plainTar?: boolean; } -/** Options for importing an archive into a snapshot group. */ +/** Options for importing one or more archives into a snapshot group. */ export interface LoadOpts { /** Parent directory containing snapshot groups. */ dest?: string; - /** Exact base snapshot or standalone archive for a dependent archive. */ + /** External snapshot or standalone archive for dependencies absent from the batch/group. */ base?: string; /** Destination group; generated when omitted. */ group?: string; - /** Select the imported member even when it is not a fast-forward. */ + /** Select the unique imported tip even when it is not a fast-forward. */ setHead?: boolean; } @@ -251,6 +251,12 @@ export class Snapshot { return new SnapshotHandle(raw); } + /** Import archives together into one group, resolving dependencies regardless of input order. */ + static async loadMany(archives: string[], opts: LoadOpts = {}): Promise { + const raw = await withMappedErrors(() => napi.Snapshot.loadMany(archives, opts)); + return raw.map((handle) => new SnapshotHandle(handle)); + } + /** Read a group's head, or select `group:member` as its head. */ static async groupHead(selector: string): Promise { const update = await withMappedErrors(() => napi.Snapshot.groupHead(selector)); diff --git a/sdk/node-ts/tests/unit/native-contract.test.ts b/sdk/node-ts/tests/unit/native-contract.test.ts index 8c8c298d5..be6d5411b 100644 --- a/sdk/node-ts/tests/unit/native-contract.test.ts +++ b/sdk/node-ts/tests/unit/native-contract.test.ts @@ -70,6 +70,7 @@ describe("native image cache contract", () => { describe("native snapshot contract", () => { it("exports group creation, import, and head selection", () => { expect(typeof napi.Snapshot.loadWithOptions).toBe("function"); + expect(typeof napi.Snapshot.loadMany).toBe("function"); expect(typeof napi.Snapshot.groupHead).toBe("function"); expect(typeof napi.SnapshotBuilder.prototype.group).toBe("function"); const builder = new napi.SnapshotBuilder("").fromSandbox("source").group("work"); @@ -80,4 +81,9 @@ describe("native snapshot contract", () => { it("exports the direct archive result used by the TS wrapper", () => { expect(typeof napi.SnapshotArchive).toBe("function"); }); + it("passes empty batches to core validation", async () => { + await expect(napi.Snapshot.loadMany([])).rejects.toThrow( + "snapshot load requires at least one archive", + ); + }); }); diff --git a/sdk/node-ts/tests/unit/snapshot.test.ts b/sdk/node-ts/tests/unit/snapshot.test.ts index a0c4ab1bf..5c172cbd6 100644 --- a/sdk/node-ts/tests/unit/snapshot.test.ts +++ b/sdk/node-ts/tests/unit/snapshot.test.ts @@ -3,7 +3,7 @@ import { Snapshot } from "../../dist/snapshot.js"; import { napi } from "../../dist/internal/napi.js"; vi.mock("../../dist/internal/napi.js", () => ({ - napi: { Snapshot: { loadWithOptions: vi.fn(), groupHead: vi.fn() } }, + napi: { Snapshot: { loadWithOptions: vi.fn(), loadMany: vi.fn(), groupHead: vi.fn() } }, })); function projectedSnapshot( @@ -65,8 +65,8 @@ describe("Snapshot native projections", () => { name: "other", createdAt: 0, path: "/snapshots/work/snapshot-2", } as never); const options = { dest: "/snapshots", base: "work:base", group: "work", setHead: false }; - const handle = await Snapshot.loadWithOptions("other.msnap", options); - expect(napi.Snapshot.loadWithOptions).toHaveBeenCalledWith("other.msnap", options); + const handle = await Snapshot.loadWithOptions("other.msb", options); + expect(napi.Snapshot.loadWithOptions).toHaveBeenCalledWith("other.msb", options); expect(handle.group).toBe("work"); expect(handle.id).toBe("snapshot-2"); expect(handle.headUpdate).toEqual(headUpdate); @@ -80,6 +80,28 @@ describe("Snapshot native projections", () => { expect(napi.Snapshot.groupHead).toHaveBeenCalledWith("work:baseline"); }); + it("loads a batch once and preserves input-order handles and headless outcomes", async () => { + vi.mocked(napi.Snapshot.loadMany).mockResolvedValue([ + { id: "snapshot-tip", digest: "sha256:tip", path: "/snapshots/received/tip", group: "received", createdAt: 0 }, + { id: "snapshot-base", digest: "sha256:base", path: "/snapshots/received/base", group: "received", createdAt: 0 }, + ] as never); + const archives = ["changes.msb", "base.msb"]; + const options = { group: "received", dest: "/snapshots" }; + const handles = await Snapshot.loadMany(archives, options); + expect(napi.Snapshot.loadMany).toHaveBeenCalledWith(archives, options); + expect(handles.map((handle) => handle.id)).toEqual(["snapshot-tip", "snapshot-base"]); + expect(handles.map((handle) => handle.group)).toEqual(["received", "received"]); + expect(handles.every((handle) => handle.headUpdate === null)).toBe(true); + }); + + it("passes explicit batch head selection to the native importer", async () => { + vi.mocked(napi.Snapshot.loadMany).mockResolvedValue([]); + await Snapshot.loadMany(["tip.msb", "base.msb"], { group: "received", base: "outside:base", setHead: true }); + expect(napi.Snapshot.loadMany).toHaveBeenLastCalledWith( + ["tip.msb", "base.msb"], { group: "received", base: "outside:base", setHead: true }, + ); + }); + it("returns complete file and checkpoint states", () => { expect(projectedSnapshot().state).toMatchObject({ kind: "file", diff --git a/sdk/python/microsandbox/_microsandbox.pyi b/sdk/python/microsandbox/_microsandbox.pyi index 7666937a8..6a9d42804 100644 --- a/sdk/python/microsandbox/_microsandbox.pyi +++ b/sdk/python/microsandbox/_microsandbox.pyi @@ -885,6 +885,15 @@ class Snapshot: set_head: bool = False, ) -> SnapshotHandle: ... @staticmethod + async def load_many( + archives: Sequence[str | os.PathLike[str]], + *, + dest: str | os.PathLike[str] | None = None, + base: str | None = None, + group: str | None = None, + set_head: bool = False, + ) -> list[SnapshotHandle]: ... + @staticmethod async def group_head(selector: str) -> dict[str, str | bool | None]: ... @property def head_update(self) -> dict[str, str | bool | None] | None: ... diff --git a/sdk/python/src/snapshot.rs b/sdk/python/src/snapshot.rs index 4fa96a8d1..9bd5f09b6 100644 --- a/sdk/python/src/snapshot.rs +++ b/sdk/python/src/snapshot.rs @@ -309,6 +309,36 @@ impl PySnapshot { }) } + /// Import archives together, resolving dependencies within the batch and destination group. + #[staticmethod] + #[pyo3(signature = (archives, *, dest = None, base = None, group = None, set_head = false))] + fn load_many<'py>( + py: Python<'py>, + archives: Vec, + dest: Option, + base: Option, + group: Option, + set_head: bool, + ) -> PyResult> { + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let handles = RustSnapshot::load_many( + &archives, + RustLoadOpts { + dest, + base, + group, + set_head, + }, + ) + .await + .map_err(to_py_err)?; + Ok(handles + .into_iter() + .map(PySnapshotHandle::from_rust) + .collect::>()) + }) + } + /// Read a group's head, or select `group:member` as its head. #[staticmethod] fn group_head<'py>(py: Python<'py>, selector: String) -> PyResult> { diff --git a/sdk/python/tests/test_snapshot_stub.py b/sdk/python/tests/test_snapshot_stub.py new file mode 100644 index 000000000..e23ca6a1d --- /dev/null +++ b/sdk/python/tests/test_snapshot_stub.py @@ -0,0 +1,28 @@ +"""Verify the typed single-archive and batch snapshot import contracts.""" + +from __future__ import annotations + +import ast +from pathlib import Path + + +def test_batch_load_preserves_single_load_options_and_returns_handles() -> None: + stub = Path(__file__).parent.parent / "microsandbox" / "_microsandbox.pyi" + tree = ast.parse(stub.read_text()) + snapshot = next( + node for node in tree.body if isinstance(node, ast.ClassDef) and node.name == "Snapshot" + ) + methods = { + node.name: node for node in snapshot.body if isinstance(node, ast.AsyncFunctionDef) + } + single, batch = methods["load"], methods["load_many"] + assert [arg.arg for arg in batch.args.args] == ["archives"] + assert ast.unparse(batch.args.args[0].annotation) == "Sequence[str | os.PathLike[str]]" + assert [arg.arg for arg in batch.args.kwonlyargs] == [ + arg.arg for arg in single.args.kwonlyargs + ] == ["dest", "base", "group", "set_head"] + assert [ast.dump(value) for value in batch.args.kw_defaults] == [ + ast.dump(value) for value in single.args.kw_defaults + ] + assert ast.unparse(single.returns) == "SnapshotHandle" + assert ast.unparse(batch.returns) == "list[SnapshotHandle]" diff --git a/sdk/rust/lib/snapshot/archive.rs b/sdk/rust/lib/snapshot/archive.rs index 2a53b4343..596b67d37 100644 --- a/sdk/rust/lib/snapshot/archive.rs +++ b/sdk/rust/lib/snapshot/archive.rs @@ -1,4 +1,4 @@ -//! Snapshot save / load via `.msnap` bundles (tar + zstd, or explicit plain tar). +//! Snapshot save / load via `.msb` bundles (tar + zstd, or explicit plain tar). //! Encoding is detected from contents; legacy suffixes and extensionless inputs remain valid. //! //! Default archive format is zstd-compressed tar. Regular files with holes, notably the sparse `upper.ext4` whose logical size is the configured upper cap rather than the data @@ -9,6 +9,7 @@ //! depths, produced by our own save path), and owning the walk lets sparse entries be restored map-driven: data runs copied straight off the wire, holes never written and kept //! unallocated per platform ([`extent::mark_sparse`] on NTFS, [`extent::punch_hole_aligned`] on APFS). `tokio_tar` remains the header codec and the dense-entry writer. +mod batch; mod delta; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; @@ -964,187 +965,17 @@ pub(super) async fn load_snapshot_with_options( archive: &Path, opts: LoadOpts, ) -> MicrosandboxResult { - let total_started = Instant::now(); - let snapshots_dir = opts.dest.clone().unwrap_or_else(|| local.snapshots_dir()); - tokio::fs::create_dir_all(&snapshots_dir).await?; - let cache_dir = local.cache_dir(); - tokio::fs::create_dir_all(&cache_dir).await?; - - let snapshot_stage = tempfile::Builder::new() - .prefix(".msb-snapshot-import-") - .tempdir_in(&snapshots_dir)?; - let cache_tmp_dir = cache_dir.join("tmp"); - tokio::fs::create_dir_all(&cache_tmp_dir).await?; - let cache_stage = tempfile::Builder::new() - .prefix("snapshot-import-") - .tempdir_in(&cache_tmp_dir)?; - - // Stream rather than slurp — archives carry the full upper layer and are - // routinely multi-GB. - let file = tokio::fs::File::open(archive).await?; - let mut buf = BufReader::with_capacity(1024 * 1024, file); - let is_zstd = { - let bytes = buf.fill_buf().await?; - bytes.starts_with(&[0x28, 0xb5, 0x2f, 0xfd]) - }; - - let unpack_started = Instant::now(); - let unpacked = if is_zstd { - let decoder = ZstdDecoder::new(buf); - // The decoder and archive walker both carry sizeable buffers across - // await points. Keep their combined future off Tokio's worker stack. - Box::pin(unpack_archive( - decoder, - snapshot_stage.path(), - cache_stage.path(), - )) - .await? - } else { - Box::pin(unpack_archive( - buf, - snapshot_stage.path(), - cache_stage.path(), - )) - .await? - }; - let unpack_us = unpack_started.elapsed().as_micros(); + let mut loaded = batch::load(local, &[archive.to_path_buf()], opts).await?; + Ok(loaded.remove(0)) +} - let validate_started = Instant::now(); - if unpacked.inventory.is_none() { - super::migration::normalize_staged(local.db().await?, &unpacked.manifest_dirs).await?; - } else if let Some(inventory) = unpacked.inventory.as_ref() { - delta::resolve( - local, - inventory, - snapshot_stage.path(), - cache_stage.path(), - opts.base.as_deref(), - ) - .await?; - materialize_inventory_layers(inventory, snapshot_stage.path()).await?; - } - let imported = verify_imported_snapshots(local, &unpacked.manifest_dirs).await?; - for snapshot in &imported { - super::metadata::write(snapshot.path(), snapshot.labels()).await?; - } - if let Some(inventory) = unpacked.inventory.as_ref() { - validate_inventory_snapshot_bindings(inventory, &imported)?; - } - // Released flat descriptors are translated by open_snapshot in memory. Install that - // admitted representation only in our owned staging, after checking the original archive - // bindings; group metadata must never point at a descriptor its reader cannot reopen. - for snapshot in &imported { - normalize_imported_descriptor(snapshot).await?; - } - let head_index = match unpacked.head.as_deref() { - Some(head) => imported - .iter() - .position(|snapshot| snapshot.id().as_str() == head) - .ok_or_else(|| { - MicrosandboxError::Custom(format!("archive inventory head {head} was not imported")) - })?, - None => select_head_snapshot(&imported)?, - }; - let head_manifest = imported[head_index].manifest().clone(); - let validate_us = validate_started.elapsed().as_micros(); - - let promote_started = Instant::now(); - // Cache installation carries hashing buffers across await points. Keep - // that future on the heap so the archive loader remains within Windows' - // smaller default worker-thread stack. - Box::pin(install_staged_cache( - cache_stage.path(), - &cache_dir, - &head_manifest, - )) - .await?; - let group_dir = super::group::ensure(&snapshots_dir, opts.group.as_deref()).await?; - let mut aliases = BTreeMap::new(); - if let Some(inventory) = unpacked.inventory.as_ref() { - if let Some(names) = inventory.extensions.get("msb-snapshot-member-names") { - aliases = serde_json::from_value(names.clone())?; - } - // This older field is only a suggestion. Released archives can carry names that - // predate group alias restrictions; ignore those while keeping explicit aliases strict. - if let Some(name) = inventory - .suggested_name - .as_ref() - .filter(|name| super::group::validate_alias(name).is_ok()) - { - aliases - .entry(head_manifest.snapshot_id.to_string()) - .or_insert_with(|| name.clone()); - } - } - let update = super::group::publish( - &group_dir, - snapshot_stage.path(), - &aliases, - &head_manifest.snapshot_id, - opts.set_head, - ) - .await?; - let head_path = group_dir.join(head_manifest.snapshot_id.as_str()); - - let snap = store::open_snapshot(local, head_path.to_string_lossy().as_ref()).await?; - - // Index this and any sibling artifacts that landed in the dest dir. - let _ = store::reindex_dir(local, &group_dir).await; - let promote_index_us = promote_started.elapsed().as_micros(); - - let (state_kind, format, fstype, checkpoint_manifest_digest, size_bytes) = - match &snap.manifest().state { - SnapshotState::File(state) => ( - "file".to_string(), - Some(state.disk_format), - Some(state.filesystem.clone()), - None, - Some(state.virtual_size), - ), - SnapshotState::Checkpoint(state) => ( - "checkpoint".to_string(), - None, - None, - Some(state.checkpoint_root.clone()), - None, - ), - }; - let handle = SnapshotHandle { - group: Some(update.group.clone()), - head_update: Some(update), - snapshot_id: snap.id().to_string(), - digest: snap.digest().to_string(), - name: super::group::member_name(snap.path())?, - parent_digest: snap.manifest().parent.as_ref().map(ToString::to_string), - scope: snap.manifest().scope, - image_ref: snap.manifest().image.reference.clone(), - state_kind, - format, - fstype, - checkpoint_manifest_digest, - size_bytes, - locality: "embedded".into(), - availability: "ready".into(), - migration_state: "canonical".into(), - migration_error_code: None, - created_at: chrono::DateTime::parse_from_rfc3339(&snap.manifest().capture.created_at) - .map(|d| d.naive_utc()) - .unwrap_or_else(|_| chrono::Utc::now().naive_utc()), - artifact_path: snap.path().to_path_buf(), - }; - let archive_bytes = tokio::fs::metadata(archive).await?.len(); - tracing::info!( - target: "microsandbox_checkpoint_timing", - operation = "snapshot_load_archive", - zstd = is_zstd, - archive_bytes, - total_us = total_started.elapsed().as_micros(), - unpack_us, - validate_us, - promote_index_us, - "snapshot archive load timing" - ); - Ok(handle) +/// Resolve all supplied archives together, publishing their members into one group. +pub(super) async fn load_snapshots( + local: &LocalBackend, + archives: &[PathBuf], + opts: LoadOpts, +) -> MicrosandboxResult> { + batch::load(local, archives, opts).await } /// Consume a current archive directly into a child sandbox's staging directory. @@ -4235,7 +4066,7 @@ mod tests { .await .is_err() ); - let archive = home.path().join("group.msnap"); + let archive = home.path().join("group.msb"); save_snapshot( &local, "first:child", @@ -4277,7 +4108,7 @@ mod tests { let manifest = grouped_archive_manifest(1, None); let artifact = home.path().join("legacy name with spaces"); write_grouped_archive_fixture(&artifact, &manifest); - let archive = home.path().join("legacy.msnap"); + let archive = home.path().join("legacy.msb"); save_snapshot( &local, artifact.to_str().unwrap(), @@ -4432,7 +4263,7 @@ mod tests { let archive_dir = directory.path().join(plain_tar.to_string()); std::fs::create_dir(&archive_dir).unwrap(); for name in [ - "snapshot.msnap", + "snapshot.msb", "snapshot.tar.zst", "snapshot.tar", "snapshot", diff --git a/sdk/rust/lib/snapshot/archive/batch.rs b/sdk/rust/lib/snapshot/archive/batch.rs new file mode 100644 index 000000000..dce629fe6 --- /dev/null +++ b/sdk/rust/lib/snapshot/archive/batch.rs @@ -0,0 +1,444 @@ +//! One-pass archive staging and dependency resolution for a local group import. + +use super::*; +use microsandbox_image::snapshot::{Manifest, SnapshotId}; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +struct StagedArchive { + source: PathBuf, + snapshots: tempfile::TempDir, + cache: tempfile::TempDir, + unpacked: UnpackedArchive, +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +pub(super) async fn load( + local: &LocalBackend, + archives: &[PathBuf], + opts: LoadOpts, +) -> MicrosandboxResult> { + if archives.is_empty() { + return Err(MicrosandboxError::InvalidConfig( + "snapshot load requires at least one archive".into(), + )); + } + let started = Instant::now(); + let snapshots_dir = opts.dest.clone().unwrap_or_else(|| local.snapshots_dir()); + // Validate the requested namespace before doing expensive I/O. This read does not create a + // group; a bad or incomplete batch must not publish any of its snapshot members. + let existing = match opts.group.as_deref() { + Some(group) => super::super::group::dependency_members(&snapshots_dir, group).await?, + None => Vec::new(), + }; + tokio::fs::create_dir_all(&snapshots_dir).await?; + let cache_dir = local.cache_dir(); + let cache_tmp = cache_dir.join("tmp"); + tokio::fs::create_dir_all(&cache_tmp).await?; + + let mut staged = Vec::with_capacity(archives.len()); + for archive in archives { + staged.push(unpack(archive, &snapshots_dir, &cache_tmp).await?); + } + let unpack_us = started.elapsed().as_micros(); + let has_dependencies = staged.iter().try_fold(false, |found, item| { + Ok::<_, MicrosandboxError>( + found + | match item.unpacked.inventory.as_ref() { + Some(inventory) => delta::validate(inventory)?.is_some(), + None => false, + }, + ) + })?; + + let mut sources = delta::Sources::default(); + // Legacy archives have no payload omissions. Translate only owned staging, as in a single + // import, before offering their canonical layers as potential sources for another archive. + for item in &staged { + if item.unpacked.inventory.is_none() { + super::super::migration::normalize_staged( + local.db().await?, + &item.unpacked.manifest_dirs, + ) + .await?; + for snapshot in verify_imported_snapshots(local, &item.unpacked.manifest_dirs).await? { + super::super::metadata::write(snapshot.path(), snapshot.labels()).await?; + normalize_imported_descriptor(&snapshot).await?; + } + } + if has_dependencies { + for directory in &item.unpacked.manifest_dirs { + let manifest = read_manifest(directory).await?; + let shared = item + .unpacked + .inventory + .as_ref() + .map(|_| item.snapshots.path()); + sources.add(&manifest, directory, shared).await?; + } + } + } + + // No ambient/global search: only the explicitly selected group and optional external base + // augment the supplied batch. Keep an unpacked base alive until all copies are destination-owned. + let complete = |sources: &delta::Sources| { + staged.iter().all(|item| { + item.unpacked + .inventory + .as_ref() + .is_none_or(|inventory| sources.require(inventory).is_ok()) + }) + }; + let external = if has_dependencies && !complete(&sources) { + for directory in &existing { + if complete(&sources) { + break; + } + // An unrelated damaged checkpoint must not block a load whose dependencies are + // available elsewhere. A missing required identity is still reported below, and any + // chosen payload is verified after copying into this operation's owned staging. + let inspected = async { + let manifest = read_manifest(directory).await?; + sources.add(&manifest, directory, None).await + } + .await; + if let Err(error) = inspected { + tracing::debug!(path = %directory.display(), %error, "skipping unavailable group dependency source"); + } + } + match opts.base.as_deref().filter(|_| !complete(&sources)) { + Some(base) => { + let opened = Box::pin(delta::open_base(local, base)).await?; + sources + .add(opened.snapshot.manifest(), opened.snapshot.path(), None) + .await?; + Some(opened) + } + None => None, + } + } else { + None + }; + // Plan every omission before copying any of them. Missing dependencies identify their target + // archive and payload, rather than guessing ancestry or requiring a particular filename. + for item in &staged { + if let Some(inventory) = &item.unpacked.inventory { + sources + .require(inventory) + .map_err(|error| archive_error(&item.source, error))?; + } + } + + for item in &staged { + if let Some(inventory) = &item.unpacked.inventory { + delta::resolve_sources( + local, + inventory, + item.snapshots.path(), + item.cache.path(), + &sources, + ) + .await + .map_err(|error| archive_error(&item.source, error))?; + } + } + // Keep all original source paths intact until every borrowing read is done. Materializing + // file snapshots below consumes their archive-shared layer directories. + let mut imported = Vec::new(); + let mut candidates = Vec::with_capacity(archives.len()); + let mut aliases = BTreeMap::new(); + let mut identities = BTreeMap::new(); + for item in &staged { + if let Some(inventory) = &item.unpacked.inventory { + materialize_inventory_layers(inventory, item.snapshots.path()).await?; + } + let snapshots = verify_imported_snapshots(local, &item.unpacked.manifest_dirs).await?; + if let Some(inventory) = &item.unpacked.inventory { + validate_inventory_snapshot_bindings(inventory, &snapshots)?; + } + let head_index = match item.unpacked.head.as_deref() { + Some(head) => snapshots + .iter() + .position(|snapshot| snapshot.id().as_str() == head) + .ok_or_else(|| { + MicrosandboxError::SnapshotIntegrity(format!( + "archive {} head {head} is not an imported member", + item.source.display() + )) + })?, + None => select_head_snapshot(&snapshots)?, + }; + let head = snapshots[head_index].id().clone(); + if let Some(inventory) = &item.unpacked.inventory { + merge_aliases(&mut aliases, inventory, &head)?; + } + candidates.push(head); + for snapshot in &snapshots { + if let Some((digest, labels)) = identities.insert( + snapshot.id().clone(), + (snapshot.digest().to_string(), snapshot.labels().clone()), + ) { + if digest != snapshot.digest() { + return Err(MicrosandboxError::SnapshotIntegrity(format!( + "snapshot ID {} has conflicting descriptors in this batch", + snapshot.id() + ))); + } + if &labels != snapshot.labels() { + return Err(MicrosandboxError::InvalidConfig(format!( + "snapshot {} has conflicting labels in this batch", + snapshot.id() + ))); + } + } + super::super::metadata::write(snapshot.path(), snapshot.labels()).await?; + normalize_imported_descriptor(snapshot).await?; + } + imported.push(snapshots); + } + drop(external); + let validate_us = started.elapsed().as_micros() - unpack_us; + + let publication = tempfile::Builder::new() + .prefix(".msb-snapshot-batch-") + .tempdir_in(&snapshots_dir)?; + for ((item, snapshots), candidate) in staged.iter().zip(&imported).zip(&candidates) { + let head = snapshots + .iter() + .find(|snapshot| snapshot.id() == candidate) + .expect("validated archive head"); + Box::pin(install_staged_cache( + item.cache.path(), + &cache_dir, + head.manifest(), + )) + .await?; + } + // Resolve all cross-archive reads before moving any source directory. Same-ID/same-descriptor + // duplicates were independently validated above and are published only once. + for snapshots in &imported { + for snapshot in snapshots { + let target = publication.path().join(snapshot.id().as_str()); + if !target.exists() { + tokio::fs::rename(snapshot.path(), target).await?; + } + } + } + let group_dir = super::super::group::ensure(&snapshots_dir, opts.group.as_deref()).await?; + let update = super::super::group::publish_batch( + &group_dir, + publication.path(), + &aliases, + &candidates, + opts.set_head, + ) + .await?; + let group = group_dir + .file_name() + .and_then(|name| name.to_str()) + .expect("validated group name"); + let mut handles = Vec::with_capacity(candidates.len()); + for id in &candidates { + let path = group_dir.join(id.as_str()); + let snapshot = store::open_snapshot(local, path.to_string_lossy().as_ref()).await?; + handles.push(handle(&snapshot, group, update.clone())?); + } + let _ = store::reindex_dir(local, &group_dir).await; + tracing::info!( + target: "microsandbox_checkpoint_timing", + operation = "snapshot_load_batch", + archives = archives.len(), + total_us = started.elapsed().as_micros(), + unpack_us, + validate_us, + "snapshot batch load timing" + ); + Ok(handles) +} + +async fn unpack( + archive: &Path, + root: &Path, + cache_tmp: &Path, +) -> MicrosandboxResult { + let snapshots = tempfile::Builder::new() + .prefix(".msb-snapshot-import-") + .tempdir_in(root)?; + let cache = tempfile::Builder::new() + .prefix("snapshot-import-") + .tempdir_in(cache_tmp)?; + let file = tokio::fs::File::open(archive).await?; + let mut reader = BufReader::with_capacity(1024 * 1024, file); + let compressed = reader + .fill_buf() + .await? + .starts_with(&[0x28, 0xb5, 0x2f, 0xfd]); + let unpacked = if compressed { + Box::pin(unpack_archive( + ZstdDecoder::new(reader), + snapshots.path(), + cache.path(), + )) + .await? + } else { + Box::pin(unpack_archive(reader, snapshots.path(), cache.path())).await? + }; + Ok(StagedArchive { + source: archive.to_path_buf(), + snapshots, + cache, + unpacked, + }) +} + +async fn read_manifest(directory: &Path) -> MicrosandboxResult { + let bytes = tokio::fs::read(directory.join(DESCRIPTOR_FILENAME)).await?; + Manifest::from_bytes(&bytes) + .map_err(|error| MicrosandboxError::SnapshotIntegrity(error.to_string())) +} + +fn archive_error(archive: &Path, error: MicrosandboxError) -> MicrosandboxError { + MicrosandboxError::SnapshotIntegrity(format!("archive {}: {error}", archive.display())) +} + +fn merge_aliases( + aliases: &mut BTreeMap, + inventory: &ArchiveInventory, + head: &SnapshotId, +) -> MicrosandboxResult<()> { + let mut incoming: BTreeMap = inventory + .extensions + .get("msb-snapshot-member-names") + .map(|value| serde_json::from_value(value.clone())) + .transpose()? + .unwrap_or_default(); + if let Some(name) = inventory + .suggested_name + .as_ref() + .filter(|name| super::super::group::validate_alias(name).is_ok()) + { + incoming + .entry(head.to_string()) + .or_insert_with(|| name.clone()); + } + for (id, name) in incoming { + if !inventory + .members + .iter() + .any(|member| member.snapshot_id == id) + { + return Err(MicrosandboxError::SnapshotIntegrity(format!( + "archive member name refers to snapshot {id} outside that archive" + ))); + } + if let Some(previous) = aliases.get(&id) + && previous != &name + { + return Err(MicrosandboxError::InvalidConfig(format!( + "snapshot {id} has conflicting batch member names '{previous}' and '{name}'" + ))); + } + aliases.insert(id, name); + } + Ok(()) +} + +fn handle( + snap: &Snapshot, + group: &str, + head_update: Option, +) -> MicrosandboxResult { + let (state_kind, format, fstype, checkpoint_manifest_digest, size_bytes) = + match &snap.manifest().state { + SnapshotState::File(state) => ( + "file", + Some(state.disk_format), + Some(state.filesystem.clone()), + None, + Some(state.virtual_size), + ), + SnapshotState::Checkpoint(state) => ( + "checkpoint", + None, + None, + Some(state.checkpoint_root.clone()), + None, + ), + }; + Ok(SnapshotHandle { + group: Some(group.into()), + head_update, + snapshot_id: snap.id().to_string(), + digest: snap.digest().to_string(), + name: super::super::group::member_name(snap.path())?, + parent_digest: snap.manifest().parent.as_ref().map(ToString::to_string), + scope: snap.manifest().scope, + image_ref: snap.manifest().image.reference.clone(), + state_kind: state_kind.into(), + format, + fstype, + checkpoint_manifest_digest, + size_bytes, + locality: "embedded".into(), + availability: "ready".into(), + migration_state: "canonical".into(), + migration_error_code: None, + created_at: chrono::DateTime::parse_from_rfc3339(&snap.manifest().capture.created_at) + .map(|date| date.naive_utc()) + .unwrap_or_else(|_| chrono::Utc::now().naive_utc()), + artifact_path: snap.path().to_path_buf(), + }) +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn member_alias_cannot_refer_to_snapshot_outside_its_archive() { + let existing = SnapshotId::new(format!("snap_{:032x}", 1)).unwrap(); + let incoming = SnapshotId::new(format!("snap_{:032x}", 2)).unwrap(); + let inventory = ArchiveInventory { + schema: "microsandbox.snapshot-archive/1".into(), + head: incoming.to_string(), + suggested_name: None, + completeness: "boot-complete".into(), + members: vec![ArchiveSnapshot { + snapshot_id: incoming.to_string(), + descriptor_path: format!("snapshots/{incoming}/{DESCRIPTOR_FILENAME}"), + descriptor_digest: format!("sha256:{}", "0".repeat(64)), + }], + entries: Vec::new(), + limits: ArchiveLimits { + entry_count: 0, + encoded_bytes: 0, + apparent_bytes: 0, + }, + extensions: BTreeMap::from([( + "msb-snapshot-member-names".into(), + serde_json::to_value(BTreeMap::from([(existing.to_string(), "renamed-existing")])) + .unwrap(), + )]), + requires: Vec::new(), + }; + // Knowing this ID from the destination or another supplied archive does not authorize + // this inventory to rename it. Membership is checked before merging local aliases. + let original = BTreeMap::from([(existing.to_string(), "existing-name".into())]); + let mut aliases = original.clone(); + let error = merge_aliases(&mut aliases, &inventory, &incoming).unwrap_err(); + assert!( + error.to_string().contains("outside that archive"), + "{error}" + ); + assert_eq!(aliases, original); + } +} diff --git a/sdk/rust/lib/snapshot/archive/delta.rs b/sdk/rust/lib/snapshot/archive/delta.rs index 41665886b..8f12705b2 100644 --- a/sdk/rust/lib/snapshot/archive/delta.rs +++ b/sdk/rust/lib/snapshot/archive/delta.rs @@ -1,6 +1,8 @@ //! Explicit disk-prefix and immutable RAM-object dependencies for incremental exports. -use microsandbox_image::checkpoint::{DiskLayerExportPlan, DiskLayerRef}; +use microsandbox_image::checkpoint::{ + DiskGenerationManifest, DiskLayerExportPlan, DiskLayerRef, MemoryManifest, +}; use microsandbox_image::snapshot::{DiskLayer, Manifest}; use super::*; @@ -10,6 +12,7 @@ use super::*; //-------------------------------------------------------------------------------------------------- pub(super) const REQUIREMENT: &str = "msb-snapshot-dependencies-v1"; +const MAX_METADATA: u64 = 8 * 1024 * 1024; //-------------------------------------------------------------------------------------------------- // Types @@ -46,16 +49,243 @@ struct PhysicalLayer { source: PathBuf, } -struct BaseSnapshot { - snapshot: Snapshot, +pub(super) struct BaseSnapshot { + pub(super) snapshot: Snapshot, // Keep archive staging alive until all required payloads belong to the destination. _stage: Option, } +/// Sources are scoped to this load, never persisted or discovered through global path scans. +/// Payload identity, not parentage, connects archives that can reconstruct one another. +#[derive(Default)] +pub(super) struct Sources { + disks: BTreeMap, + memory: BTreeMap, +} + +type SourceIndex = (Vec<(String, PathBuf)>, Vec<(ObjectId, PathBuf)>); + +//-------------------------------------------------------------------------------------------------- +// Methods +//-------------------------------------------------------------------------------------------------- + +impl Sources { + pub(super) async fn add( + &mut self, + manifest: &Manifest, + directory: &Path, + archive_stage: Option<&Path>, + ) -> MicrosandboxResult<()> { + let manifest = manifest.clone(); + let directory = directory.to_path_buf(); + let archive_stage = archive_stage.map(Path::to_path_buf); + let (disks, memory) = tokio::task::spawn_blocking(move || { + inspect_sources(&manifest, &directory, archive_stage.as_deref()) + }) + .await + .map_err(|error| { + MicrosandboxError::Runtime(format!("snapshot source inspection: {error}")) + })??; + // Prefer batch bytes over destination-group copies. Every borrowed payload is checked in + // its owned destination, so this index is a location hint, not an integrity receipt. + for (identity, path) in disks { + self.disks.entry(identity).or_insert(path); + } + for (identity, path) in memory { + self.memory.entry(identity).or_insert(path); + } + Ok(()) + } + + pub(super) fn require(&self, inventory: &ArchiveInventory) -> MicrosandboxResult<()> { + let Some(dependencies) = validate(inventory)? else { + return Ok(()); + }; + let mut missing = Vec::new(); + for layer in &dependencies.disks { + if !self + .disks + .contains_key(&serde_json::to_string(&layer.identity)?) + { + missing.push(format!("disk layer {}", layer.path)); + } + } + for object in &dependencies.memory { + if !self.memory.contains_key(object) { + missing.push(format!("RAM object {object}")); + } + } + if !missing.is_empty() { + return Err(MicrosandboxError::SnapshotIntegrity(format!( + "snapshot {} is missing dependencies: {}; supply the missing archives or an external --base", + inventory.head, + missing.join(", ") + ))); + } + Ok(()) + } +} + //-------------------------------------------------------------------------------------------------- // Functions //-------------------------------------------------------------------------------------------------- +/// Inspect only descriptors and small identity-verified manifests. Missing payloads in dependent +/// archives are expected; admission still happens after filling their complete target closure. +fn inspect_sources( + manifest: &Manifest, + directory: &Path, + archive_stage: Option<&Path>, +) -> MicrosandboxResult { + let mut disks = Vec::new(); + let mut memory = Vec::new(); + match &manifest.state { + SnapshotState::File(file) => { + for layer in &file.layers { + let relative = file.layer_path(layer); + let path = match archive_stage { + Some(root) => root + .join(".archive-layers") + .join(relative.file_name().expect("canonical layer filename")), + None => directory.join(relative), + }; + if regular_source(&path)? { + disks.push(( + serde_json::to_string(&LayerIdentity::File(layer.clone()))?, + path, + )); + } + } + } + SnapshotState::Checkpoint(state) => { + let root = directory.join(CHECKPOINT_DIRECTORY); + let expected = ObjectId::new(&state.checkpoint_root).map_err(source_error)?; + let checkpoint = CheckpointClosure::inspect_manifest(&root, Some(&expected)) + .map_err(source_error)?; + let ram = MemoryManifest::from_bytes(&read_metadata_object(&root, &checkpoint.memory)?) + .map_err(source_error)?; + let mut metadata = BTreeSet::from([ + checkpoint.memory.clone(), + checkpoint.execution_state.clone(), + ]); + metadata.extend(checkpoint.disks.iter().cloned()); + metadata.extend(checkpoint.devices.iter().map(|device| device.state.clone())); + let objects: BTreeSet<_> = ram + .extents + .iter() + .filter_map(|extent| match &extent.content { + MemoryExtentContent::Object(content) if !metadata.contains(&content.object) => { + Some(content.object.clone()) + } + _ => None, + }) + .collect(); + for object in objects { + let path = checkpoint_object_path(&root, &object); + if regular_source(&path)? { + memory.push((object, path)); + } + } + for disk in &checkpoint.disks { + let disk = DiskGenerationManifest::from_bytes(&read_metadata_object(&root, disk)?) + .map_err(source_error)?; + for layer in disk.layers { + let path = root + .join("layers") + .join(format!("{}.{}", layer.layer_id, layer.format)); + if regular_source(&path)? { + disks.push(( + serde_json::to_string(&LayerIdentity::Checkpoint(layer))?, + path, + )); + } + } + } + } + } + Ok((disks, memory)) +} + +fn regular_source(path: &Path) -> MicrosandboxResult { + match std::fs::symlink_metadata(path) { + Ok(metadata) if metadata.file_type().is_file() => Ok(true), + Ok(_) => Err(MicrosandboxError::SnapshotIntegrity(format!( + "snapshot payload is not a regular file: {}", + path.display() + ))), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error.into()), + } +} + +fn read_metadata_object(root: &Path, id: &ObjectId) -> MicrosandboxResult> { + use std::io::Read; + let path = checkpoint_object_path(root, id); + if !regular_source(&path)? { + return Err(MicrosandboxError::SnapshotIntegrity(format!( + "missing checkpoint metadata {id}" + ))); + } + // Match the checkpoint resolver's 8 MiB metadata limit; a changing file cannot cause an + // unbounded allocation. This is not a RAM-payload read or a new admission format. + let mut bytes = Vec::new(); + std::fs::File::open(path)? + .take(MAX_METADATA + 1) + .read_to_end(&mut bytes)?; + if bytes.len() as u64 > MAX_METADATA + || &ObjectId::from_bytes(&bytes).map_err(source_error)? != id + { + return Err(MicrosandboxError::SnapshotIntegrity(format!( + "invalid checkpoint metadata {id}" + ))); + } + Ok(bytes) +} + +fn source_error(error: impl std::fmt::Display) -> MicrosandboxError { + MicrosandboxError::SnapshotIntegrity(error.to_string()) +} + +pub(super) async fn resolve_sources( + local: &LocalBackend, + inventory: &ArchiveInventory, + snapshots_dir: &Path, + cache_dir: &Path, + sources: &Sources, +) -> MicrosandboxResult<()> { + let Some(dependencies) = validate(inventory)? else { + return Ok(()); + }; + sources.require(inventory)?; + for layer in &dependencies.disks { + let source = &sources.disks[&serde_json::to_string(&layer.identity)?]; + let target = inventory_entry_target(&layer.path, snapshots_dir, cache_dir)?; + copy_dependency(source, &target).await?; + if let LayerIdentity::File(layer) = &layer.identity { + super::super::verify::verify_file_payload(&target, layer.payload.integrity.as_ref()) + .await?; + } + } + for id in &dependencies.memory { + let target = inventory_entry_target( + &memory_archive_path(&inventory.head, id), + snapshots_dir, + cache_dir, + )?; + copy_dependency(&sources.memory[id], &target).await?; + let actual = format!( + "sha256:{}", + hex::encode(Box::pin(file_sha256(&target)).await?) + ); + if actual != id.as_str() { + return Err(MicrosandboxError::SnapshotIntegrity(format!( + "borrowed RAM object content does not match {id}" + ))); + } + } + validate_resolved(local, inventory, snapshots_dir, cache_dir, &dependencies).await +} + pub(super) async fn selection( local: &LocalBackend, head: &Snapshot, @@ -315,6 +545,16 @@ pub(super) async fn resolve( } } + validate_resolved(local, inventory, snapshots_dir, cache_dir, &dependencies).await +} + +async fn validate_resolved( + local: &LocalBackend, + inventory: &ArchiveInventory, + snapshots_dir: &Path, + cache_dir: &Path, + dependencies: &Dependencies, +) -> MicrosandboxResult<()> { // Open the complete target only after filling omissions. This retains its normal metadata, // range, epoch and disk-integrity validation instead of introducing a partial-closure mode. let artifact = snapshots_dir.join(&inventory.head); @@ -501,7 +741,10 @@ fn physical_layers( } } -async fn open_base(local: &LocalBackend, input: &str) -> MicrosandboxResult { +pub(super) async fn open_base( + local: &LocalBackend, + input: &str, +) -> MicrosandboxResult { let path = Path::new(input); if !path.is_file() { let snapshot = store::open_snapshot(local, input).await?; @@ -580,6 +823,143 @@ mod tests { SnapshotConsistency, SnapshotFormat, SnapshotId, SnapshotRootDisk, SnapshotScope, }; + #[tokio::test] + async fn batch_rejects_corrupt_borrowed_file_layer_before_publication() { + let temp = tempfile::tempdir().unwrap(); + let local = LocalBackend::builder() + .home(temp.path().join("home")) + .build() + .await + .unwrap(); + let base_dir = temp.path().join("base-source"); + let child_dir = temp.path().join("child-source"); + std::fs::create_dir_all(base_dir.join("layers")).unwrap(); + std::fs::create_dir_all(child_dir.join("layers")).unwrap(); + let mut base_layer = DiskLayer { + layer_id: DiskLayerId::new("layer_00000000000000000000000000000001").unwrap(), + format: SnapshotFormat::Raw, + virtual_size: 65536, + backing: None, + payload: LayerPayload { + file_kind: LayerFileKind::Regular, + integrity: None, + }, + }; + let base_path = + microsandbox_image::snapshot::layer_path(&base_layer.layer_id, base_layer.format); + std::fs::write(base_dir.join(&base_path), vec![91u8; 65536]).unwrap(); + std::fs::copy(base_dir.join(&base_path), child_dir.join(&base_path)).unwrap(); + base_layer.payload.integrity = Some( + super::super::super::verify::compute_merkle_integrity(&base_dir.join(&base_path)) + .await + .unwrap(), + ); + let top = DiskLayer { + layer_id: DiskLayerId::new("layer_00000000000000000000000000000002").unwrap(), + format: SnapshotFormat::Qcow2, + virtual_size: 65536, + backing: Some(base_layer.layer_id.clone()), + payload: LayerPayload { + file_kind: LayerFileKind::Regular, + integrity: None, + }, + }; + let top_path = microsandbox_image::snapshot::layer_path(&top.layer_id, top.format); + microsandbox_image::checkpoint::create_qcow2_overlay( + &child_dir.join(top_path), + 65536, + &child_dir.join(&base_path), + "raw", + ) + .await + .unwrap(); + let descriptor = |value: u128, layers: Vec, parent| Manifest { + schema: "microsandbox.snapshot/1".into(), + snapshot_id: SnapshotId::new(format!("snap_{value:032x}")).unwrap(), + scope: SnapshotScope::Disk, + root_disk: SnapshotRootDisk::Managed, + state: SnapshotState::File(FileSnapshotState { + disk_format: layers.last().unwrap().format, + filesystem: "ext4".into(), + virtual_size: 65536, + head: layers.last().unwrap().layer_id.clone(), + layers, + }), + capture: SnapshotCapture { + created_at: "2026-09-10T00:00:00Z".into(), + source_lineage: None, + source_checkpoint: None, + consistency: SnapshotConsistency::CrashConsistent, + }, + image: ImageRef { + reference: "docker.io/library/alpine:3.20".into(), + manifest_digest: format!("sha256:{}", "0".repeat(64)), + }, + parent, + extensions: BTreeMap::new(), + requires: Vec::new(), + }; + let base = descriptor(1, vec![base_layer.clone()], None); + let child = descriptor(2, vec![base_layer, top], Some(base.snapshot_id.clone())); + for (directory, manifest) in [(&base_dir, &base), (&child_dir, &child)] { + std::fs::write( + directory.join(DESCRIPTOR_FILENAME), + manifest.to_canonical_bytes().unwrap(), + ) + .unwrap(); + } + let base_archive = temp.path().join("base.msb"); + save_snapshot( + &local, + base_dir.to_str().unwrap(), + &base_archive, + SaveOpts::default(), + ) + .await + .unwrap(); + let child_archive = temp.path().join("child.msb"); + save_snapshot( + &local, + child_dir.to_str().unwrap(), + &child_archive, + SaveOpts { + since: Some(base_dir.to_string_lossy().into_owned()), + ..Default::default() + }, + ) + .await + .unwrap(); + let options = || LoadOpts { + group: Some("corrupt-source".into()), + ..Default::default() + }; + let installed = load_snapshots(&local, &[base_archive], options()) + .await + .unwrap(); + let group_dir = installed[0].path().parent().unwrap().to_path_buf(); + let previous_head = std::fs::read(group_dir.join("group.json")).unwrap(); + // Metadata and length still agree: only payload verification can detect this change. + std::fs::write(installed[0].path().join(&base_path), vec![92u8; 65536]).unwrap(); + let error = load_snapshots(&local, &[child_archive], options()) + .await + .unwrap_err(); + assert!(error.to_string().contains("integrity mismatch"), "{error}"); + assert!(!group_dir.join(child.snapshot_id.as_str()).exists()); + assert_eq!( + std::fs::read(group_dir.join("group.json")).unwrap(), + previous_head + ); + assert_eq!( + super::super::super::group::dependency_members( + &local.snapshots_dir(), + "corrupt-source", + ) + .await + .unwrap(), + vec![installed[0].path().to_path_buf()] + ); + } + #[tokio::test] async fn delta_load_and_direct_restore_require_exact_base_and_own_their_closure() { let temp = tempfile::tempdir().unwrap(); @@ -685,11 +1065,12 @@ mod tests { .await .unwrap(); assert!(load_snapshot(&local, &archive, None).await.is_err()); - assert!( - load_snapshot_with_base(&local, &archive, None, Some(head_name)) - .await - .is_err() - ); + // Loading now resolves exact payload identities from a source pool. A complete newer + // snapshot can supply the required prefix even when it contains additional layers. + let supplied_by_newer = load_snapshot_with_base(&local, &archive, None, Some(head_name)) + .await + .unwrap(); + assert!(supplied_by_newer.path().join(&base_path).exists()); let loaded = load_snapshot_with_base(&local, &archive, None, Some(base_name)) .await .unwrap(); @@ -739,6 +1120,29 @@ mod tests { ) .await .unwrap(); + // File-state archives use a shared layers directory, unlike full checkpoint payloads. + // Both input orders must finish all borrowing reads before consuming those directories. + for (group, inputs) in [ + ("file-reverse", vec![archive.clone(), base_archive.clone()]), + ("file-forward", vec![base_archive.clone(), archive.clone()]), + ] { + let batch = load_snapshots( + &local, + &inputs, + LoadOpts { + group: Some(group.into()), + ..Default::default() + }, + ) + .await + .unwrap(); + for snapshot in &batch { + assert_eq!( + std::fs::read(snapshot.path().join(&base_path)).unwrap(), + vec![91u8; 65536] + ); + } + } std::fs::remove_dir_all(&base_dir).unwrap(); assert_eq!( std::fs::read(loaded.path().join(&base_path)).unwrap(), diff --git a/sdk/rust/lib/snapshot/archive/delta_tests.rs b/sdk/rust/lib/snapshot/archive/delta_tests.rs index 1c10d11db..1733af9bd 100644 --- a/sdk/rust/lib/snapshot/archive/delta_tests.rs +++ b/sdk/rust/lib/snapshot/archive/delta_tests.rs @@ -174,7 +174,7 @@ async fn fixture( reference: "docker.io/library/alpine:3.20".into(), manifest_digest: format!("sha256:{}", "0".repeat(64)), }, - parent: None, + parent: previous.map(|snapshot| snapshot.id().clone()), extensions: BTreeMap::new(), requires: Vec::new(), }; @@ -243,7 +243,7 @@ async fn chain(disk: bool) { disk, ) .await; - let archive = temp.path().join(format!("cp{generation:02}.msnap")); + let archive = temp.path().join(format!("cp{generation:02}.msb")); save_snapshot( &local, source.path().to_str().unwrap(), @@ -320,7 +320,7 @@ async fn chain(disk: bool) { previous = Some(source); } let final_snapshot = loaded.unwrap(); - let standalone = temp.path().join("standalone.msnap"); + let standalone = temp.path().join("standalone.msb"); save_snapshot( &local, final_snapshot.path().to_str().unwrap(), @@ -339,6 +339,209 @@ async fn chain(disk: bool) { // Tests //-------------------------------------------------------------------------------------------------- +#[tokio::test] +async fn unordered_batch_resolves_disk_and_ram_from_all_supplied_archives() { + for disk in [false, true] { + let temp = tempfile::tempdir().unwrap(); + let local = LocalBackend::builder() + .home(temp.path().join("home")) + .build() + .await + .unwrap(); + let mut previous = None; + let mut archives = Vec::new(); + for generation in 1..=6 { + let source = fixture( + &local, + &temp.path().join(format!("source-{generation}")), + generation, + previous.as_ref(), + disk, + ) + .await; + let archive = temp.path().join(format!("cp{generation}.msb")); + save_snapshot( + &local, + source.path().to_str().unwrap(), + &archive, + SaveOpts { + since: previous + .as_ref() + .map(|snapshot: &Snapshot| snapshot.path().to_string_lossy().into_owned()), + plain_tar: generation % 2 == 0, + ..Default::default() + }, + ) + .await + .unwrap(); + archives.push(archive); + previous = Some(source); + } + archives.reverse(); + let loaded = load_snapshots( + &local, + &archives, + LoadOpts { + group: Some("received".into()), + ..Default::default() + }, + ) + .await + .unwrap(); + for (offset, snapshot) in loaded.iter().enumerate() { + assert_ram(snapshot.path(), 6 - offset as u64); + } + assert_eq!(loaded[0].head_update().unwrap().head, loaded[0].snapshot_id); + // Neither source files, archive bytes, nor other installed generations are needed by + // the final snapshot after the batch has reconstructed destination-owned closures. + for generation in 1..=6 { + std::fs::remove_dir_all(temp.path().join(format!("source-{generation}"))).unwrap(); + } + for archive in &archives { + std::fs::remove_file(archive).unwrap(); + } + for snapshot in &loaded[1..] { + std::fs::remove_dir_all(snapshot.path()).unwrap(); + } + assert_ram(loaded[0].path(), 6); + } +} + +#[tokio::test] +async fn automatic_group_sources_fill_ram_and_disks_without_base_flag() { + for disk in [false, true] { + let temp = tempfile::tempdir().unwrap(); + let local = LocalBackend::builder() + .home(temp.path().join("home")) + .build() + .await + .unwrap(); + let base = fixture(&local, &temp.path().join("base"), 1, None, disk).await; + let target = fixture(&local, &temp.path().join("target"), 3, Some(&base), disk).await; + let baseline = temp.path().join("base.msb"); + let delta = temp.path().join("delta.msb"); + save_snapshot( + &local, + base.path().to_str().unwrap(), + &baseline, + SaveOpts::default(), + ) + .await + .unwrap(); + save_snapshot( + &local, + target.path().to_str().unwrap(), + &delta, + SaveOpts { + since: Some(base.path().to_string_lossy().into_owned()), + ..Default::default() + }, + ) + .await + .unwrap(); + let opts = LoadOpts { + group: Some("received".into()), + ..Default::default() + }; + let installed_base = load_snapshot_with_options(&local, &baseline, opts.clone()) + .await + .unwrap(); + // The same dependent archive cannot search a different, unnamed group implicitly. + let error = load_snapshot_with_options(&local, &delta, LoadOpts::default()) + .await + .unwrap_err(); + assert!( + error.to_string().contains("missing dependencies"), + "{error}" + ); + assert!(!local.snapshots_dir().join("elsewhere").exists()); + let imported = load_snapshot_with_options(&local, &delta, opts) + .await + .unwrap(); + assert_ram(imported.path(), 3); + std::fs::remove_dir_all(installed_base.path()).unwrap(); + assert_ram(imported.path(), 3); + } +} + +#[tokio::test] +async fn missing_or_corrupt_borrowed_ram_never_publishes_target() { + let temp = tempfile::tempdir().unwrap(); + let local = LocalBackend::builder() + .home(temp.path().join("home")) + .build() + .await + .unwrap(); + let base = fixture(&local, &temp.path().join("base"), 1, None, false).await; + let target = fixture(&local, &temp.path().join("target"), 3, Some(&base), false).await; + let baseline = temp.path().join("base.msb"); + let delta = temp.path().join("delta.msb"); + save_snapshot( + &local, + base.path().to_str().unwrap(), + &baseline, + SaveOpts::default(), + ) + .await + .unwrap(); + save_snapshot( + &local, + target.path().to_str().unwrap(), + &delta, + SaveOpts { + since: Some(base.path().to_string_lossy().into_owned()), + ..Default::default() + }, + ) + .await + .unwrap(); + let opts = LoadOpts { + group: Some("received".into()), + ..Default::default() + }; + let installed = load_snapshot_with_options(&local, &baseline, opts.clone()) + .await + .unwrap(); + let id = memory_objects(&base) + .unwrap() + .intersection(&memory_objects(&target).unwrap()) + .next() + .unwrap() + .clone(); + let path = checkpoint_object_path(&installed.path().join(CHECKPOINT_DIRECTORY), &id); + let original = std::fs::read(&path).unwrap(); + std::fs::remove_file(&path).unwrap(); + let missing = load_snapshot_with_options(&local, &delta, opts.clone()) + .await + .unwrap_err(); + assert!( + missing.to_string().contains("missing dependencies"), + "{missing}" + ); + std::fs::write(&path, vec![0x42; original.len()]).unwrap(); + let corrupt = load_snapshot_with_options(&local, &delta, opts) + .await + .unwrap_err(); + assert!( + corrupt + .to_string() + .contains("RAM object content does not match"), + "{corrupt}" + ); + assert!( + !installed + .path() + .parent() + .unwrap() + .join(target.id().as_str()) + .exists() + ); + let head = super::super::super::group::select(&local.snapshots_dir(), "received") + .await + .unwrap(); + assert_eq!(head.head, base.id().as_str()); +} + #[tokio::test] async fn twelve_ram_only_archives_resolve_without_intermediate_vms() { chain(false).await; @@ -371,7 +574,7 @@ async fn last_layers_keeps_ram_complete_and_wrong_ram_base_fails() { .unwrap() .unwrap(); assert!(selection.memory.is_empty()); - let archive = temp.path().join("delta.msnap"); + let archive = temp.path().join("delta.msb"); save_snapshot( &local, target.path().to_str().unwrap(), @@ -527,7 +730,7 @@ async fn memory_dependency_validation_rejects_incomplete_and_misbound_inventorie ); assert!(!local.snapshots_dir().join(target.id().as_str()).exists()); - let truncated = temp.path().join("truncated.msnap"); + let truncated = temp.path().join("truncated.msb"); let bytes = std::fs::read(&archive).unwrap(); std::fs::write(&truncated, &bytes[..bytes.len() / 2]).unwrap(); assert!( @@ -553,7 +756,7 @@ async fn standalone_base_archive_resolves_ram_but_dependent_base_archive_is_refu .unwrap(); let base = fixture(&local, &temp.path().join("base"), 1, None, false).await; let target = fixture(&local, &temp.path().join("target"), 4, None, false).await; - let base_archive = temp.path().join("base.msnap"); + let base_archive = temp.path().join("base.msb"); save_snapshot( &local, base.path().to_str().unwrap(), @@ -562,7 +765,7 @@ async fn standalone_base_archive_resolves_ram_but_dependent_base_archive_is_refu ) .await .unwrap(); - let delta = temp.path().join("delta.msnap"); + let delta = temp.path().join("delta.msb"); save_snapshot( &local, target.path().to_str().unwrap(), diff --git a/sdk/rust/lib/snapshot/group.rs b/sdk/rust/lib/snapshot/group.rs index 5f09d5d66..9723f5fb4 100644 --- a/sdk/rust/lib/snapshot/group.rs +++ b/sdk/rust/lib/snapshot/group.rs @@ -4,7 +4,7 @@ //! member's optional local alias need metadata; immutable descriptors remain authoritative for //! ancestry. All group operations share one process-held lock, acquired off the async executor. -use std::collections::{BTreeMap, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::fs::{self, File, OpenOptions}; use std::io::{Read, Write}; #[cfg(unix)] @@ -66,6 +66,8 @@ pub enum HeadUpdateReason { Diverged, /// Missing history prevents proving that the candidate descends from the head. UnknownAncestry, + /// Supplied archive heads have multiple tips that known ancestry cannot order. + AmbiguousCandidates, } #[derive(Debug, Serialize, Deserialize)] @@ -169,10 +171,35 @@ pub(super) async fn publish( candidate: &SnapshotId, set_head: bool, ) -> MicrosandboxResult { + publish_batch( + group_dir, + staged, + aliases, + std::slice::from_ref(candidate), + set_head, + ) + .await? + .ok_or_else(|| integrity("single snapshot publication did not choose a head".into())) +} + +/// Publish a validated batch under one lock, selecting a head only when supplied candidates +/// have one tip that is a known descendant of every other candidate. +pub(super) async fn publish_batch( + group_dir: &Path, + staged: &Path, + aliases: &BTreeMap, + candidates: &[SnapshotId], + set_head: bool, +) -> MicrosandboxResult> { + if candidates.is_empty() { + return Err(MicrosandboxError::InvalidConfig( + "snapshot batch must contain at least one candidate head".into(), + )); + } let group_dir = group_dir.to_path_buf(); let staged = staged.to_path_buf(); let aliases = aliases.clone(); - let candidate = candidate.to_string(); + let candidates: BTreeSet = candidates.iter().map(ToString::to_string).collect(); blocking(move || { require_directory(&staged)?; if fs::canonicalize(&group_dir)?.starts_with(fs::canonicalize(&staged)?) { @@ -212,15 +239,17 @@ pub(super) async fn publish( ); } } - if !members.contains_key(&candidate) { - return Err(MicrosandboxError::SnapshotNotFound(candidate)); + for candidate in &candidates { + if !members.contains_key(candidate) { + return Err(MicrosandboxError::SnapshotNotFound(candidate.clone())); + } } apply_aliases(&mut members, &aliases)?; validate_ancestry(&members)?; - let update = head_update( + let update = batch_head_update( &group_dir, state.head.as_deref(), - &candidate, + &candidates, &members, set_head, )?; @@ -239,7 +268,9 @@ pub(super) async fn publish( write_member_name(&group_dir.join(id), members[id].name.as_deref())?; } sync_directory(&group_dir)?; - if update.changed { + if let Some(update) = &update + && update.changed + { write_group(&group_dir, Some(update.head.clone()))?; } Ok(update) @@ -247,6 +278,38 @@ pub(super) async fn publish( .await } +/// List installed members available as dependency sources without creating a destination group. +/// Callers must still validate the physical payloads they borrow from these artifact paths. +pub(super) async fn dependency_members( + root: &Path, + name: &str, +) -> MicrosandboxResult> { + let root = root.to_path_buf(); + let name = name.to_owned(); + blocking(move || { + validate_group_name(&name)?; + if !path_exists(&root)? { + return Ok(Vec::new()); + } + require_directory(&root)?; + let directory = root.join(name); + if !path_exists(&directory)? { + return Ok(Vec::new()); + } + require_directory(&directory)?; + read_group(&directory)?; + // Preflight must not create even a lock file in an existing malformed namespace. + let lock = process_lock::open_existing_lock_file(&directory.join(".group.lock"))?; + process_lock::lock_exclusive(&lock)?; + let state = read_group(&directory)?; + let members = read_members(&directory, true)?; + validate_head(&state, &members)?; + validate_ancestry(&members)?; + Ok(members.into_values().map(|member| member.path).collect()) + }) + .await +} + /// Read a bare group's head, or explicitly select a qualified member as its head. pub(super) async fn select(root: &Path, selector: &str) -> MicrosandboxResult { let root = root.to_path_buf(); @@ -547,8 +610,9 @@ fn resolve_selected( let selected_id = match selected { None => Some(state.head.clone().ok_or_else(|| { MicrosandboxError::SnapshotNotFound(format!( - "snapshot group {} has no head", - directory.display() + "snapshot group {} has no head selected; choose an installed member with 'msb snapshot head {}:'", + directory.display(), + directory.file_name().unwrap_or_default().to_string_lossy() )) })?), Some(selected) if SnapshotId::new(selected).is_ok() => Some(selected.to_owned()), @@ -646,6 +710,55 @@ fn head_update( }) } +fn batch_head_update( + directory: &Path, + previous: Option<&str>, + candidates: &BTreeSet, + members: &BTreeMap, + explicit: bool, +) -> MicrosandboxResult> { + // Remove supplied heads that are proven ancestors of another supplied head. Shared paths + // need be traversed only once: their candidate ancestors were already marked on first visit. + let mut ancestors = HashSet::new(); + let mut visited = HashSet::new(); + for candidate in candidates { + let mut current = members[candidate].parent.as_deref(); + while let Some(parent) = current { + if !visited.insert(parent) { + break; + } + if candidates.contains(parent) { + ancestors.insert(parent); + } + current = members + .get(parent) + .and_then(|member| member.parent.as_deref()); + } + } + let mut tips = candidates + .iter() + .filter(|candidate| !ancestors.contains(candidate.as_str())); + let candidate = tips + .next() + .ok_or_else(|| integrity("snapshot batch has no candidate tip".into()))?; + if tips.next().is_none() { + return head_update(directory, previous, candidate, members, explicit).map(Some); + } + if explicit { + return Err(MicrosandboxError::InvalidConfig( + "--set-head cannot choose between multiple snapshot archive heads with incomparable or unknown ancestry; load without --set-head, then use 'msb snapshot head :'".into(), + )); + } + // A fresh group may contain several branches without claiming that one is current. + previous + .map(|head| { + let mut update = head_update(directory, Some(head), head, members, false)?; + update.reason = HeadUpdateReason::AmbiguousCandidates; + Ok(update) + }) + .transpose() +} + fn ancestry_reason( candidate: &str, head: &str, diff --git a/sdk/rust/lib/snapshot/group_tests.rs b/sdk/rust/lib/snapshot/group_tests.rs index 25892843c..fb6680af8 100644 --- a/sdk/rust/lib/snapshot/group_tests.rs +++ b/sdk/rust/lib/snapshot/group_tests.rs @@ -89,6 +89,331 @@ async fn add(group: &Path, value: u128, parent: Option) -> HeadUpdate { // Tests //-------------------------------------------------------------------------------------------------- +#[tokio::test] +async fn batch_head_is_independent_of_archive_and_staging_order() { + let root = tempfile::tempdir().unwrap(); + for (index, order) in [ + [1, 2, 3], + [1, 3, 2], + [2, 1, 3], + [2, 3, 1], + [3, 1, 2], + [3, 2, 1], + ] + .into_iter() + .enumerate() + { + let group = ensure(root.path(), Some(&format!("order-{index}"))) + .await + .unwrap(); + let manifests = order + .iter() + .map(|value| descriptor(*value, (*value > 1).then_some(*value - 1))) + .collect::>(); + let staged = stage(root.path(), &manifests); + let candidates = order.into_iter().map(id).collect::>(); + let update = publish_batch(&group, staged.path(), &BTreeMap::new(), &candidates, false) + .await + .unwrap() + .unwrap(); + assert_eq!(update.head, id(3).as_str()); + assert_eq!(update.reason, HeadUpdateReason::Initialized); + assert_eq!(read_members(&group, true).unwrap().len(), 3); + } +} + +#[tokio::test] +async fn batch_uses_known_destination_intermediates_to_prove_one_tip() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("intermediate")).await.unwrap(); + add(&group, 1, None).await; + add(&group, 2, Some(1)).await; + let staged = stage(root.path(), &[descriptor(3, Some(2))]); + let update = publish_batch( + &group, + staged.path(), + &BTreeMap::new(), + &[id(3), id(1), id(3)], + false, + ) + .await + .unwrap() + .unwrap(); + assert_eq!(update.reason, HeadUpdateReason::FastForwarded); + assert_eq!(update.previous.as_deref(), Some(id(2).as_str())); + assert_eq!(update.head, id(3).as_str()); +} + +#[tokio::test] +async fn batch_unique_tip_still_respects_existing_head_ancestry() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("retained-head")).await.unwrap(); + add(&group, 1, None).await; + for (parent, reason) in [ + (None, HeadUpdateReason::Diverged), + (Some(9), HeadUpdateReason::UnknownAncestry), + ] { + let first = if parent.is_none() { 2 } else { 4 }; + let staged = stage( + root.path(), + &[ + descriptor(first, parent), + descriptor(first + 1, Some(first)), + ], + ); + let update = publish_batch( + &group, + staged.path(), + &BTreeMap::new(), + &[id(first), id(first + 1)], + false, + ) + .await + .unwrap() + .unwrap(); + assert_eq!(update.reason, reason); + assert!(!update.changed); + assert_eq!(update.head, id(1).as_str()); + assert!(group.join(id(first + 1).as_str()).is_dir()); + } +} + +#[tokio::test] +async fn batch_branches_preserve_existing_head_or_leave_new_group_unselected() { + let root = tempfile::tempdir().unwrap(); + for existing in [false, true] { + for order in [[2, 3], [3, 2]] { + let name = format!("branches-{existing}-{}", order[0]); + let group = ensure(root.path(), Some(&name)).await.unwrap(); + if existing { + add(&group, 1, None).await; + } + let staged = stage( + root.path(), + &[ + descriptor(1, None), + descriptor(2, Some(1)), + descriptor(3, Some(1)), + ], + ); + let candidates = order.map(id); + let update = publish_batch(&group, staged.path(), &BTreeMap::new(), &candidates, false) + .await + .unwrap(); + if existing { + let update = update.unwrap(); + assert_eq!(update.reason, HeadUpdateReason::AmbiguousCandidates); + assert!(!update.changed); + assert_eq!(update.head, id(1).as_str()); + } else { + assert_eq!(update, None); + assert_eq!(read_group(&group).unwrap().head, None); + let error = resolve(root.path(), &name).await.unwrap_err().to_string(); + assert!(error.contains("no head selected")); + assert!(error.contains("msb snapshot head")); + assert_eq!( + resolve(root.path(), &format!("{name}:{}", id(3))) + .await + .unwrap(), + group.join(id(3).as_str()) + ); + } + assert_eq!(read_members(&group, true).unwrap().len(), 3); + } + } +} + +#[tokio::test] +async fn batch_unknown_history_does_not_guess_a_candidate_order() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("holes")).await.unwrap(); + add(&group, 1, None).await; + // Snapshot 3 may descend from 1, but absent snapshot 2 prevents proving the relationship. + let staged = stage(root.path(), &[descriptor(3, Some(2))]); + let update = publish_batch( + &group, + staged.path(), + &BTreeMap::new(), + &[id(3), id(1)], + false, + ) + .await + .unwrap() + .unwrap(); + assert_eq!(update.reason, HeadUpdateReason::AmbiguousCandidates); + assert_eq!(update.head, id(1).as_str()); + let staged = stage(root.path(), &[descriptor(2, Some(1))]); + // Repeating the same candidates is now conclusive, even though their intermediate is not + // itself a supplied archive head. + let update = publish_batch( + &group, + staged.path(), + &BTreeMap::new(), + &[id(1), id(3)], + false, + ) + .await + .unwrap() + .unwrap(); + assert_eq!(update.reason, HeadUpdateReason::FastForwarded); + assert_eq!(update.head, id(3).as_str()); +} + +#[tokio::test] +async fn batch_set_head_requires_one_candidate_tip_before_any_publication() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("explicit-batch")).await.unwrap(); + add(&group, 1, None).await; + let staged = stage( + root.path(), + &[descriptor(2, Some(1)), descriptor(3, Some(1))], + ); + let error = publish_batch( + &group, + staged.path(), + &BTreeMap::new(), + &[id(2), id(3)], + true, + ) + .await + .unwrap_err(); + assert!(error.to_string().contains("--set-head cannot choose")); + assert!(error.to_string().contains("msb snapshot head")); + for candidate in [2, 3] { + assert!(!group.join(id(candidate).as_str()).exists()); + assert!(staged.path().join(id(candidate).as_str()).is_dir()); + } + assert_eq!( + read_group(&group).unwrap().head.as_deref(), + Some(id(1).as_str()) + ); + let staged = stage(root.path(), &[descriptor(4, None), descriptor(5, Some(4))]); + let update = publish_batch( + &group, + staged.path(), + &BTreeMap::new(), + &[id(5), id(4)], + true, + ) + .await + .unwrap() + .unwrap(); + assert_eq!(update.reason, HeadUpdateReason::Selected); + assert_eq!(update.head, id(5).as_str()); +} + +#[tokio::test] +async fn batch_descriptor_and_alias_conflicts_do_not_partly_publish() { + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("batch-conflicts")).await.unwrap(); + let staged = stage(root.path(), &[descriptor(1, None)]); + publish( + &group, + staged.path(), + &BTreeMap::from([(id(1).to_string(), "base".into())]), + &id(1), + false, + ) + .await + .unwrap(); + let mut conflict = descriptor(1, None); + conflict.capture.source_lineage = Some("different-source".into()); + let staged = stage(root.path(), &[descriptor(2, Some(1)), conflict]); + assert!( + publish_batch( + &group, + staged.path(), + &BTreeMap::new(), + &[id(2), id(1)], + false + ) + .await + .unwrap_err() + .to_string() + .contains("different descriptor") + ); + assert!(!group.join(id(2).as_str()).exists()); + assert!(staged.path().join(id(2).as_str()).is_dir()); + let staged = stage( + root.path(), + &[descriptor(2, Some(1)), descriptor(3, Some(2))], + ); + let aliases = BTreeMap::from([ + (id(2).to_string(), "other".into()), + (id(3).to_string(), "base".into()), + ]); + assert!( + publish_batch(&group, staged.path(), &aliases, &[id(2), id(3)], false) + .await + .unwrap_err() + .to_string() + .contains("conflicts") + ); + for candidate in [2, 3] { + assert!(!group.join(id(candidate).as_str()).exists()); + assert!(staged.path().join(id(candidate).as_str()).is_dir()); + } + assert_eq!( + read_group(&group).unwrap().head.as_deref(), + Some(id(1).as_str()) + ); +} + +#[tokio::test] +async fn dependency_lookup_reads_only_existing_installed_group_members() { + let root = tempfile::tempdir().unwrap(); + let missing_root = root.path().join("missing-root"); + assert!( + dependency_members(&missing_root, "work") + .await + .unwrap() + .is_empty() + ); + assert!(!missing_root.exists()); + assert!( + dependency_members(root.path(), "work") + .await + .unwrap() + .is_empty() + ); + assert!(!root.path().join("work").exists()); + assert!(!root.path().join(".groups.lock").exists()); + assert!( + dependency_members(&missing_root, "../escape") + .await + .is_err() + ); + let group = ensure(root.path(), Some("work")).await.unwrap(); + add(&group, 1, None).await; + let _incomplete = stage(&group, &[descriptor(2, Some(1))]); + assert_eq!( + dependency_members(root.path(), "work").await.unwrap(), + vec![group.join(id(1).as_str())] + ); + let malformed = root.path().join("malformed"); + fs::create_dir(&malformed).unwrap(); + write_group(&malformed, None).unwrap(); + assert!(dependency_members(root.path(), "malformed").await.is_err()); + assert!(!malformed.join(".group.lock").exists()); +} + +#[cfg(unix)] +#[tokio::test] +async fn dependency_lookup_rejects_symlinked_roots_and_groups() { + use std::os::unix::fs::symlink; + + let root = tempfile::tempdir().unwrap(); + let group = ensure(root.path(), Some("work")).await.unwrap(); + symlink(&group, root.path().join("redirect")).unwrap(); + assert!(dependency_members(root.path(), "redirect").await.is_err()); + symlink(root.path(), root.path().join("root-link")).unwrap(); + assert!( + dependency_members(&root.path().join("root-link"), "work") + .await + .is_err() + ); +} + #[tokio::test] async fn initializes_and_fast_forwards_through_multiple_imported_ancestors() { let root = tempfile::tempdir().unwrap(); diff --git a/sdk/rust/lib/snapshot/mod.rs b/sdk/rust/lib/snapshot/mod.rs index c1a595971..b91b52152 100644 --- a/sdk/rust/lib/snapshot/mod.rs +++ b/sdk/rust/lib/snapshot/mod.rs @@ -250,7 +250,7 @@ impl Snapshot { store::reindex_dir(local, dir.as_ref()).await } - /// Bundle a snapshot into a `.msnap` archive (tar + zstd by default). + /// Bundle a snapshot into a `.msb` archive (tar + zstd by default). /// The explicit output path is preserved; legacy suffixes remain supported. pub async fn save( name_or_path: &str, @@ -262,7 +262,7 @@ impl Snapshot { archive::save_snapshot(local, name_or_path, out, opts).await } - /// Unpack a snapshot archive (`.msnap`, `.tar.zst`, or `.tar`) into the + /// Unpack a snapshot archive (`.msb`, `.tar.zst`, or `.tar`) into the /// snapshots dir, registering anything found in the index. pub async fn load( archive_path: &Path, @@ -295,6 +295,20 @@ impl Snapshot { archive::load_snapshot_with_options(local, archive_path, opts).await } + /// Load archives together, resolving omitted payloads from the batch, destination group, + /// and optional external base. Input order never chooses the group's head. + /// + /// Returns one handle per input archive head, in input order. Repeated snapshots are + /// installed once; all inputs are validated before publishing any snapshot members. + pub async fn load_many( + archive_paths: &[std::path::PathBuf], + opts: LoadOpts, + ) -> MicrosandboxResult> { + let backend = crate::backend::default_backend(); + let local = backend.as_local().ok_or_else(snapshots_require_local)?; + archive::load_snapshots(local, archive_paths, opts).await + } + /// Read a group's head, or explicitly select a qualified `group:member`. pub async fn group_head(selector: &str) -> MicrosandboxResult { let backend = crate::backend::default_backend(); diff --git a/sdk/rust/lib/snapshot/verify.rs b/sdk/rust/lib/snapshot/verify.rs index d80fe62f7..229ee3b68 100644 --- a/sdk/rust/lib/snapshot/verify.rs +++ b/sdk/rust/lib/snapshot/verify.rs @@ -164,12 +164,19 @@ async fn verify_file_layer( snap: &Snapshot, layer: µsandbox_image::snapshot::DiskLayer, ) -> MicrosandboxResult { - let Some(expected) = layer.payload.integrity.as_ref() else { + verify_file_payload(&snap.layer_path(layer), layer.payload.integrity.as_ref()).await +} + +/// Verify an owned imported layer with the same codecs used by explicit snapshot verification. +pub(super) async fn verify_file_payload( + upper_path: &Path, + expected: Option<&UpperIntegrity>, +) -> MicrosandboxResult { + let Some(expected) = expected else { return Ok(UpperVerifyStatus::NotRecorded); }; - let upper_path = snap.layer_path(layer); - let payload = open_verification_source(&upper_path)?; + let payload = open_verification_source(upper_path)?; let before = verification_source_identity(&payload.metadata()?); let actual = match expected { UpperIntegrity::Sha256 { .. } => { @@ -182,7 +189,7 @@ async fn verify_file_layer( compute_merkle_integrity_from_file(payload.try_clone()?).await? } }; - ensure_verification_source_unchanged(&payload, &upper_path, &before)?; + ensure_verification_source_unchanged(&payload, upper_path, &before)?; if actual != *expected { return Err(MicrosandboxError::SnapshotIntegrity(format!( diff --git a/sdk/rust/tests/snapshot_artifact.rs b/sdk/rust/tests/snapshot_artifact.rs index b27f4fa9a..e7c8068b1 100644 --- a/sdk/rust/tests/snapshot_artifact.rs +++ b/sdk/rust/tests/snapshot_artifact.rs @@ -7,7 +7,7 @@ use std::collections::BTreeMap; use std::io::Cursor; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::Arc; use microsandbox::Snapshot; @@ -532,6 +532,52 @@ async fn isolated_backend(home: &Path) -> Arc { Arc::new(LocalBackend::builder().home(home).build().await.unwrap()) } +/// Export complete synthetic artifacts independently: their parent edges describe history, +/// not a requirement to have every ancestor present merely to read the disk payload. +async fn save_batch_fixtures(parent: &Path, artifacts: &[PathBuf]) -> Vec { + let mut archives = Vec::new(); + for (index, artifact) in artifacts.iter().enumerate() { + let archive = parent.join(format!("batch-{index}.msb")); + Snapshot::save( + artifact.to_string_lossy().as_ref(), + &archive, + microsandbox::snapshot::SaveOpts { + plain_tar: true, + ..Default::default() + }, + ) + .await + .unwrap(); + archives.push(archive); + } + archives +} + +fn batch_group_options(group: &str) -> microsandbox::snapshot::LoadOpts { + microsandbox::snapshot::LoadOpts { + group: Some(group.into()), + ..Default::default() + } +} + +/// A failed batch may leave an empty group/staging directory, but no immutable member +/// may become visible before every input archive and publication conflict is checked. +fn assert_no_batch_members(root: &Path) { + let mut pending = vec![root.to_path_buf()]; + while let Some(path) = pending.pop() { + if !path.exists() { + continue; + } + for entry in std::fs::read_dir(path).unwrap() { + let entry = entry.unwrap(); + assert_ne!(entry.file_name(), DESCRIPTOR_FILENAME); + if entry.file_type().unwrap().is_dir() { + pending.push(entry.path()); + } + } + } +} + //-------------------------------------------------------------------------------------------------- // Tests //-------------------------------------------------------------------------------------------------- @@ -840,8 +886,8 @@ async fn repeated_loads_preserve_ids_and_resolve_local_group_names() { let backend = isolated_backend(&home).await; let (source, digest) = make_artifact(tmp.path(), "clean", b"group payload"); let snapshot_id = artifact_id(&source); - let archive = tmp.path().join("group.msnap"); - let reexport = tmp.path().join("renamed.msnap"); + let archive = tmp.path().join("group.msb"); + let reexport = tmp.path().join("renamed.msb"); microsandbox::with_backend(backend, async { Snapshot::save( @@ -945,8 +991,8 @@ async fn group_alias_collision_keeps_the_installed_snapshot_and_head() { let (second, _) = make_artifact(&tmp.path().join("second"), "clean", b"second"); let original_id = artifact_id(&first); let competing_id = artifact_id(&second); - let archive = tmp.path().join("first.msnap"); - let competing = tmp.path().join("second.msnap"); + let archive = tmp.path().join("first.msb"); + let competing = tmp.path().join("second.msb"); microsandbox::with_backend(backend, async { for (source, destination) in [(&first, &archive), (&second, &competing)] { Snapshot::save( @@ -989,6 +1035,322 @@ async fn group_alias_collision_keeps_the_installed_snapshot_and_head() { .await; } +#[tokio::test] +async fn load_many_selects_lineage_tip_independently_of_input_order() { + let tmp = TempDir::new().unwrap(); + let home = tmp.path().join("home"); + let backend = isolated_backend(&home).await; + let source = tmp.path().join("source-artifacts"); + let (first, _) = make_artifact(&source, "cp01", b"first disk"); + let first_id = artifact_id(&first); + let (second, _) = + make_artifact_with_parent(&source, "cp02", b"second disk", Some(first_id.clone())); + let second_id = artifact_id(&second); + let (third, _) = + make_artifact_with_parent(&source, "cp03", b"third disk", Some(second_id.clone())); + let third_id = artifact_id(&third); + microsandbox::with_backend(backend, async { + let archives = save_batch_fixtures(tmp.path(), &[first, second, third]).await; + for (group, order) in [("reverse", [2, 1, 0]), ("shuffled", [1, 0, 2])] { + let input = order.map(|index| archives[index].clone()); + let handles = Snapshot::load_many(&input, batch_group_options(group)) + .await + .unwrap(); + let expected_ids = [&first_id, &second_id, &third_id]; + assert_eq!(handles.len(), input.len()); + for (handle, index) in handles.iter().zip(order) { + assert_eq!(handle.id(), expected_ids[index]); + assert_eq!(handle.group(), Some(group)); + } + assert_eq!(Snapshot::group_head(group).await.unwrap().head, third_id); + assert_eq!( + Snapshot::open(format!("{group}:cp01")) + .await + .unwrap() + .id() + .as_str(), + first_id + ); + } + // Loading owns the reconstructed artifacts, never a path into an input archive + // or the sender's snapshot directory. + std::fs::remove_dir_all(&source).unwrap(); + for archive in archives { + std::fs::remove_file(archive).unwrap(); + } + for (group, name, expected) in [ + ("reverse", "cp01", b"first disk".as_slice()), + ("reverse", "cp02", b"second disk".as_slice()), + ("shuffled", "cp03", b"third disk".as_slice()), + ] { + let artifact = Snapshot::open(format!("{group}:{name}")).await.unwrap(); + assert_eq!( + std::fs::read(artifact_payload_path(artifact.path())).unwrap(), + expected + ); + } + }) + .await; +} + +#[tokio::test] +async fn load_many_duplicate_inputs_return_input_heads_but_install_once() { + let tmp = TempDir::new().unwrap(); + let home = tmp.path().join("home"); + let backend = isolated_backend(&home).await; + let (source, _) = make_artifact(tmp.path(), "baseline", b"owned bytes"); + microsandbox::with_backend(backend, async { + let archive = save_batch_fixtures(tmp.path(), &[source]).await.remove(0); + let copied_archive = tmp.path().join("identical-copy.msb"); + std::fs::copy(&archive, &copied_archive).unwrap(); + let handles = Snapshot::load_many( + &[archive.clone(), copied_archive, archive], + batch_group_options("work"), + ) + .await + .unwrap(); + assert_eq!(handles.len(), 3); + assert!( + handles + .iter() + .all(|handle| handle.path() == handles[0].path()) + ); + assert_eq!(Snapshot::list().await.unwrap().len(), 1); + }) + .await; +} + +#[tokio::test] +async fn load_many_sibling_batch_preserves_existing_head_and_leaves_new_group_headless() { + let tmp = TempDir::new().unwrap(); + let home = tmp.path().join("home"); + let backend = isolated_backend(&home).await; + let (base, _) = make_artifact(tmp.path(), "base", b"base"); + let base_id = artifact_id(&base); + let (left, _) = make_artifact_with_parent(tmp.path(), "left", b"left", Some(base_id.clone())); + let left_id = artifact_id(&left); + let (right, _) = + make_artifact_with_parent(tmp.path(), "right", b"right", Some(base_id.clone())); + microsandbox::with_backend(backend, async { + let archives = save_batch_fixtures(tmp.path(), &[base, left, right]).await; + Snapshot::load_with_options(&archives[0], batch_group_options("existing")) + .await + .unwrap(); + let siblings = [archives[2].clone(), archives[1].clone()]; + Snapshot::load_many(&siblings, batch_group_options("existing")) + .await + .unwrap(); + assert_eq!( + Snapshot::group_head("existing").await.unwrap().head, + base_id + ); + let handles = Snapshot::load_many(&siblings, batch_group_options("fresh")) + .await + .unwrap(); + assert_eq!(handles.len(), 2); + assert!(handles.iter().all(|handle| handle.head_update().is_none())); + assert!(Snapshot::open("fresh").await.is_err()); + assert_eq!( + Snapshot::open("fresh:left").await.unwrap().id().as_str(), + left_id + ); + assert_eq!( + Snapshot::group_head("fresh:left").await.unwrap().head, + left_id + ); + assert_eq!( + Snapshot::open("fresh").await.unwrap().id().as_str(), + left_id + ); + }) + .await; +} + +#[tokio::test] +async fn load_many_ambiguous_set_head_rejects_before_publishing_members() { + let tmp = TempDir::new().unwrap(); + let home = tmp.path().join("home"); + let backend = isolated_backend(&home).await; + let (left, _) = make_artifact(tmp.path(), "left", b"left"); + let (right, _) = make_artifact(tmp.path(), "right", b"right"); + microsandbox::with_backend(backend, async { + let archives = save_batch_fixtures(tmp.path(), &[left, right]).await; + let mut options = batch_group_options("ambiguous"); + options.set_head = true; + assert!(Snapshot::load_many(&archives, options).await.is_err()); + assert_no_batch_members(&home.join("snapshots/ambiguous")); + assert!(Snapshot::list().await.unwrap().is_empty()); + }) + .await; +} + +#[tokio::test] +async fn load_many_accepts_complete_payload_with_missing_historical_parent() { + let tmp = TempDir::new().unwrap(); + let home = tmp.path().join("home"); + let backend = isolated_backend(&home).await; + let missing = format!("snap_{:032x}", 7); + let (source, _) = + make_artifact_with_parent(tmp.path(), "complete", b"complete payload", Some(missing)); + let id = artifact_id(&source); + microsandbox::with_backend(backend, async { + let archives = save_batch_fixtures(tmp.path(), &[source]).await; + let handles = Snapshot::load_many(&archives, batch_group_options("work")) + .await + .unwrap(); + assert_eq!(handles[0].id(), id); + assert_eq!(Snapshot::group_head("work").await.unwrap().head, id); + assert_eq!( + std::fs::read(artifact_payload_path(handles[0].path())).unwrap(), + b"complete payload" + ); + }) + .await; +} + +#[tokio::test] +async fn load_many_conflicting_aliases_rejects_before_any_member_is_published() { + let tmp = TempDir::new().unwrap(); + let home = tmp.path().join("home"); + let backend = isolated_backend(&home).await; + let (first, _) = make_artifact(&tmp.path().join("one"), "same-name", b"one"); + let (second, _) = make_artifact(&tmp.path().join("two"), "same-name", b"two"); + microsandbox::with_backend(backend, async { + let archives = save_batch_fixtures(tmp.path(), &[first, second]).await; + let error = Snapshot::load_many(&archives, batch_group_options("work")) + .await + .unwrap_err(); + assert!( + error.to_string().contains("conflict"), + "unexpected error: {error}" + ); + assert_no_batch_members(&home.join("snapshots/work")); + }) + .await; +} + +#[tokio::test] +async fn load_many_duplicate_ids_with_conflicting_labels_rejects_before_publication() { + let tmp = TempDir::new().unwrap(); + let home = tmp.path().join("home"); + let backend = isolated_backend(&home).await; + // Keep both descriptor bytes and suggested aliases identical. Labels are the only conflict, + // and reversing the archive order must not silently choose either local metadata sidecar. + let (first, digest) = make_artifact(&tmp.path().join("one"), "same", b"same disk"); + let (second, _) = make_artifact(&tmp.path().join("two"), "same", b"same disk"); + std::fs::copy( + first.join(DESCRIPTOR_FILENAME), + second.join(DESCRIPTOR_FILENAME), + ) + .unwrap(); + for (artifact, label) in [(&first, "first"), (&second, "second")] { + std::fs::write( + artifact.join("metadata.json"), + serde_json::to_vec(&serde_json::json!({ + "schema": "microsandbox.snapshot-metadata/1", + "labels": {"stage": label}, + })) + .unwrap(), + ) + .unwrap(); + } + microsandbox::with_backend(backend, async { + for artifact in [&first, &second] { + assert_eq!( + Snapshot::open(artifact.to_string_lossy().as_ref()) + .await + .unwrap() + .digest(), + digest + ); + } + let archives = save_batch_fixtures(tmp.path(), &[first, second]).await; + for (group, order) in [("forward", [0, 1]), ("reverse", [1, 0])] { + let inputs = order.map(|index| archives[index].clone()); + let error = Snapshot::load_many(&inputs, batch_group_options(group)) + .await + .unwrap_err(); + assert!(error.to_string().contains("conflicting labels"), "{error}"); + assert_no_batch_members(&home.join("snapshots").join(group)); + } + assert!(Snapshot::list().await.unwrap().is_empty()); + }) + .await; +} + +#[tokio::test] +async fn load_many_conflicting_ids_rejects_before_any_member_is_published() { + let tmp = TempDir::new().unwrap(); + let home = tmp.path().join("home"); + let backend = isolated_backend(&home).await; + let (first, _) = make_artifact(tmp.path(), "first", b"one"); + let (second, _) = make_artifact(tmp.path(), "second", b"two"); + let mut descriptor = + Manifest::from_bytes(&std::fs::read(second.join(DESCRIPTOR_FILENAME)).unwrap()).unwrap(); + descriptor.snapshot_id = SnapshotId::new(artifact_id(&first)).unwrap(); + std::fs::write( + second.join(DESCRIPTOR_FILENAME), + descriptor.to_canonical_bytes().unwrap(), + ) + .unwrap(); + microsandbox::with_backend(backend, async { + let archives = save_batch_fixtures(tmp.path(), &[first, second]).await; + assert!( + Snapshot::load_many(&archives, batch_group_options("work")) + .await + .is_err() + ); + assert_no_batch_members(&home.join("snapshots/work")); + }) + .await; +} + +#[tokio::test] +async fn load_many_corrupt_later_archive_never_publishes_valid_earlier_member() { + let tmp = TempDir::new().unwrap(); + let home = tmp.path().join("home"); + let backend = isolated_backend(&home).await; + let (first, _) = make_artifact(tmp.path(), "first", b"one"); + let (second, _) = make_artifact(tmp.path(), "second", b"two"); + microsandbox::with_backend(backend, async { + let archives = save_batch_fixtures(tmp.path(), &[first, second]).await; + corrupt_dense_tar_member(&archives[1], ".raw"); + let error = Snapshot::load_many(&archives, batch_group_options("work")) + .await + .unwrap_err(); + assert!( + error.to_string().contains("integrity"), + "unexpected error: {error}" + ); + assert_no_batch_members(&home.join("snapshots/work")); + }) + .await; +} + +#[tokio::test] +async fn load_many_single_legacy_archive_preserves_single_load_compatibility() { + let tmp = TempDir::new().unwrap(); + let home = tmp.path().join("home"); + let backend = isolated_backend(&home).await; + let archive = tmp.path().join("legacy.tar"); + write_v066_archive(&archive, "sha256-0123456789abcdef", b"legacy disk"); + microsandbox::with_backend(backend, async { + let batch = Snapshot::load_many(&[archive.clone()], batch_group_options("batch")) + .await + .unwrap(); + let single = Snapshot::load_with_options(&archive, batch_group_options("single")) + .await + .unwrap(); + assert_eq!(batch.len(), 1); + assert_eq!(batch[0].id(), single.id()); + assert_eq!( + std::fs::read(artifact_payload_path(batch[0].path())).unwrap(), + b"legacy disk" + ); + }) + .await; +} + #[tokio::test] async fn save_sparse_upper_round_trips_and_preserves_holes() { let tmp = TempDir::new().unwrap(); From 1c78269b9a046136aaff9195d0d713525101cc56 Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Thu, 10 Sep 2026 04:57:31 +0100 Subject: [PATCH 20/29] refactor(cli)!: name the snapshot source flag explicitly Use --from-sandbox for snapshot capture, matching the existing SDK source-sandbox naming. Update README, documentation, group examples, and the branch, incremental, disk, and restore smoke scripts. Test required source selection and rejection of --from, including full capture, direct archives, and generated group members. All 13 snapshot parser tests pass. BREAKING CHANGE: msb snapshot create requires --from-sandbox instead of --from. No compatibility alias is provided. --- README.md | 2 +- crates/cli/lib/commands/snapshot.rs | 42 ++++++++++++++----- docs/changelog/2026-05-15.mdx | 6 +-- docs/changelog/2026-07-17.mdx | 2 +- docs/examples/data/migration-rehearsal.mdx | 4 +- docs/examples/file-processing/ffmpeg.mdx | 2 +- .../file-processing/libreoffice-pdf.mdx | 2 +- docs/examples/plugins/terraform.mdx | 4 +- docs/examples/sandboxing/warm-workers.mdx | 6 +-- docs/examples/web-automation/scrapy.mdx | 4 +- docs/sandboxes/lifecycle.mdx | 2 +- docs/sandboxes/snapshots.mdx | 22 +++++----- docs/snapshot-groups-explained.md | 6 +-- scripts/smoke/cli/checkpoint-clock.py | 4 +- scripts/smoke/cli/checkpoint-clock.sh | 4 +- scripts/smoke/cli/checkpoint-cpu-state.py | 2 +- scripts/smoke/cli/cow-memory-lifecycle.py | 12 +++--- scripts/smoke/cli/direct-branch.py | 6 +-- scripts/smoke/cli/dirty-memory-checkpoint.py | 8 ++-- scripts/smoke/cli/disk-compaction-depth.ps1 | 2 +- scripts/smoke/cli/disk-compaction-depth.sh | 2 +- scripts/smoke/cli/disk-compaction-export.ps1 | 6 +-- scripts/smoke/cli/disk-compaction-export.sh | 6 +-- scripts/smoke/cli/disk-compaction-negative.sh | 2 +- scripts/smoke/cli/failed-restore.py | 4 +- scripts/smoke/cli/incremental-full-archive.py | 2 +- scripts/smoke/cli/live-disk-snapshot.py | 16 +++---- scripts/smoke/cli/root-disk-growth.py | 6 +-- scripts/smoke/cli/root-disk-large-growth.py | 4 +- scripts/smoke/cli/snapshot-groups.py | 4 +- scripts/smoke/cli/snapshot-load-batch.py | 2 +- .../reports/live-disk-snapshot-2026-09-09.md | 4 +- 32 files changed, 111 insertions(+), 89 deletions(-) diff --git a/README.md b/README.md index 63b69e164..9b6fd92b2 100644 --- a/README.md +++ b/README.md @@ -315,7 +315,7 @@ The `msb` CLI provides a complete interface for managing sandboxes, snapshots, i > > ```sh > # Save now, resume later -> msb snapshot create saved --from app --full +> msb snapshot create saved --from-sandbox app --full > msb create --name restored --from-snapshot app:saved > ``` > diff --git a/crates/cli/lib/commands/snapshot.rs b/crates/cli/lib/commands/snapshot.rs index 064a8d710..30c916a9e 100644 --- a/crates/cli/lib/commands/snapshot.rs +++ b/crates/cli/lib/commands/snapshot.rs @@ -62,7 +62,7 @@ pub struct SnapshotCreateArgs { /// Source sandbox name. Disk capture also supports running and user-paused sources. #[arg(long, value_name = "SANDBOX")] - pub from: String, + pub from_sandbox: String, /// Parent directory to create the artifact in, instead of the /// default snapshots directory. The group is created under this root. @@ -235,7 +235,8 @@ pub async fn run(args: SnapshotArgs) -> anyhow::Result<()> { } async fn create(args: SnapshotCreateArgs) -> anyhow::Result<()> { - let mut builder = Snapshot::builder(args.name.unwrap_or_default()).from_sandbox(&args.from); + let mut builder = + Snapshot::builder(args.name.unwrap_or_default()).from_sandbox(&args.from_sandbox); if let Some(group) = args.group { builder = builder.group(group); } @@ -261,7 +262,7 @@ async fn create(args: SnapshotCreateArgs) -> anyhow::Result<()> { let spinner = if args.quiet { ui::Spinner::quiet() } else { - ui::Spinner::start("Snapshotting", &args.from) + ui::Spinner::start("Snapshotting", &args.from_sandbox) }; if let Some(archive_path) = args.archive.as_ref() { @@ -651,21 +652,42 @@ mod tests { TestCli::parse_from(std::iter::once("msb").chain(args.iter().copied())).args } + #[test] + fn create_requires_explicit_source_sandbox_flag() { + let error = TestCli::try_parse_from(["msb", "create", "clean"]).unwrap_err(); + assert_eq!( + error.kind(), + clap::error::ErrorKind::MissingRequiredArgument + ); + assert!(error.to_string().contains("--from-sandbox ")); + + // This is a clean rename, not an alias: reject the old ambiguous spelling. + let error = + TestCli::try_parse_from(["msb", "create", "clean", "--from", "box"]).unwrap_err(); + assert_eq!(error.kind(), clap::error::ErrorKind::UnknownArgument); + } + #[test] fn create_parses_full_capture_flag() { - let args = parse_snapshot_args(&["create", "clean", "--from", "box", "--full"]); + let args = parse_snapshot_args(&["create", "clean", "--from-sandbox", "box", "--full"]); let SnapshotCommands::Create(args) = args.command else { panic!("expected create command"); }; assert_eq!(args.name.as_deref(), Some("clean")); - assert_eq!(args.from, "box"); + assert_eq!(args.from_sandbox, "box"); assert!(args.full); } #[test] fn create_parses_dest_dir() { - let args = - parse_snapshot_args(&["create", "clean", "--from", "box", "--dest-dir", "/mnt/big"]); + let args = parse_snapshot_args(&[ + "create", + "clean", + "--from-sandbox", + "box", + "--dest-dir", + "/mnt/big", + ]); let SnapshotCommands::Create(args) = args.command else { panic!("expected create command"); }; @@ -680,7 +702,7 @@ mod tests { let args = parse_snapshot_args(&[ "create", "clean", - "--from", + "--from-sandbox", "box", "--archive", "/tmp/clean.tar", @@ -768,13 +790,13 @@ mod tests { #[test] fn create_accepts_generated_member_in_explicit_group() { - let parsed = parse_snapshot_args(&["create", "--from", "box", "--group", "work"]); + let parsed = parse_snapshot_args(&["create", "--from-sandbox", "box", "--group", "work"]); let SnapshotCommands::Create(args) = parsed.command else { panic!("expected create command"); }; assert!(args.name.is_none()); assert_eq!(args.group.as_deref(), Some("work")); - assert_eq!(args.from, "box"); + assert_eq!(args.from_sandbox, "box"); } #[test] diff --git a/docs/changelog/2026-05-15.mdx b/docs/changelog/2026-05-15.mdx index 2a0b0ba5a..70c0b1c90 100644 --- a/docs/changelog/2026-05-15.mdx +++ b/docs/changelog/2026-05-15.mdx @@ -31,11 +31,11 @@ See the [Go SDK reference](/sdk/go/sandbox). **File-first disk snapshots** -Stopped sandboxes can be snapshotted to a content-addressed directory and used to boot fresh sandboxes on any compatible host. Snapshots stay sparse via reflink and `SEEK_DATA`/`SEEK_HOLE` copying, so create and inspect stay fast. The CLI ships `msb snapshot create | open | list | list-dir | remove | reindex | export | import | verify`, and `msb run --from ` boots a fork. The same surface is available in all four SDKs. +Stopped sandboxes can be snapshotted to a content-addressed directory and used to boot fresh sandboxes on any compatible host. Snapshots stay sparse via reflink and `SEEK_DATA`/`SEEK_HOLE` copying, so create and inspect stay fast. The CLI ships `msb snapshot create | open | list | list-dir | remove | reindex | export | import | verify`, and `msb run --from-snapshot ` boots a fork. The same surface is available in all four SDKs. ```bash -msb snapshot create my-sandbox --name baseline -msb run alpine --from baseline --name fork +msb snapshot create baseline --from-sandbox my-sandbox +msb run --from-snapshot baseline --name fork ``` See [Sandbox snapshots](/sandboxes/snapshots). diff --git a/docs/changelog/2026-07-17.mdx b/docs/changelog/2026-07-17.mdx index ea7186dd7..941c9c69d 100644 --- a/docs/changelog/2026-07-17.mdx +++ b/docs/changelog/2026-07-17.mdx @@ -44,7 +44,7 @@ The snapshot surface is now locked in across the CLI and every SDK ahead of 1.0. Booting from a snapshot is spelled the same way everywhere: `--from-snapshot`, `from_snapshot=`, `fromSnapshot`, and Go `WithFromSnapshot`. ```bash -msb snapshot create clean --from box --dest-dir /mnt/big +msb snapshot create clean --from-sandbox box --dest-dir /mnt/big msb snapshot save clean /tmp/clean.tar.zst --with-image msb snapshot load /tmp/clean.tar.zst msb run --name worker --from-snapshot clean -- python -V diff --git a/docs/examples/data/migration-rehearsal.mdx b/docs/examples/data/migration-rehearsal.mdx index d54905ac7..48cf9e6c1 100644 --- a/docs/examples/data/migration-rehearsal.mdx +++ b/docs/examples/data/migration-rehearsal.mdx @@ -77,13 +77,13 @@ msb stop migration-base ```sh macOS & Linux msb snapshot create postgres-before-migration \ - --from migration-base \ + --from-sandbox migration-base \ --integrity ``` ```powershell Windows msb snapshot create postgres-before-migration ` - --from migration-base ` + --from-sandbox migration-base ` --integrity ``` diff --git a/docs/examples/file-processing/ffmpeg.mdx b/docs/examples/file-processing/ffmpeg.mdx index 08e45649b..c311c2d8d 100644 --- a/docs/examples/file-processing/ffmpeg.mdx +++ b/docs/examples/file-processing/ffmpeg.mdx @@ -87,7 +87,7 @@ msb run --name ffmpeg-base --replace ` Capture the prepared toolchain: ```sh -msb snapshot create ffmpeg-tools --from ffmpeg-base --integrity --force +msb snapshot create ffmpeg-tools --from-sandbox ffmpeg-base --integrity --force ``` Verify the snapshot before using it: diff --git a/docs/examples/file-processing/libreoffice-pdf.mdx b/docs/examples/file-processing/libreoffice-pdf.mdx index 60b500938..ffcea2bfa 100644 --- a/docs/examples/file-processing/libreoffice-pdf.mdx +++ b/docs/examples/file-processing/libreoffice-pdf.mdx @@ -85,7 +85,7 @@ msb run --name office-base --replace ` Capture the prepared toolchain: ```sh -msb snapshot create office-tools --from office-base --integrity --force +msb snapshot create office-tools --from-sandbox office-base --integrity --force ``` Verify the snapshot before using it: diff --git a/docs/examples/plugins/terraform.mdx b/docs/examples/plugins/terraform.mdx index 275e0dee5..880a84e9f 100644 --- a/docs/examples/plugins/terraform.mdx +++ b/docs/examples/plugins/terraform.mdx @@ -69,12 +69,12 @@ Capture the downloaded provider: ```sh macOS & Linux msb snapshot create terraform-runtime \ - --from terraform-base --integrity --force + --from-sandbox terraform-base --integrity --force ``` ```powershell Windows msb snapshot create terraform-runtime ` - --from terraform-base --integrity --force + --from-sandbox terraform-base --integrity --force ``` diff --git a/docs/examples/sandboxing/warm-workers.mdx b/docs/examples/sandboxing/warm-workers.mdx index 8c8d6dd97..26bda833a 100644 --- a/docs/examples/sandboxing/warm-workers.mdx +++ b/docs/examples/sandboxing/warm-workers.mdx @@ -72,14 +72,14 @@ When the command exits, `agent-base` is stopped and ready to snapshot. ```sh macOS & Linux msb snapshot create coding-agent-base \ - --from agent-base \ + --from-sandbox agent-base \ --integrity \ --force ``` ```powershell Windows msb snapshot create coding-agent-base ` - --from agent-base ` + --from-sandbox agent-base ` --integrity ` --force ``` @@ -238,7 +238,7 @@ msb snapshot rm coding-agent-base Snapshots are immutable. To update packages, recreate `agent-base`, then overwrite the named snapshot intentionally: ```sh -msb snapshot create coding-agent-base --from agent-base --integrity --force +msb snapshot create coding-agent-base --from-sandbox agent-base --integrity --force ``` See [Snapshots](/sandboxes/snapshots) for archive, integrity, and portability details. diff --git a/docs/examples/web-automation/scrapy.mdx b/docs/examples/web-automation/scrapy.mdx index 2c8928899..bacd9912f 100644 --- a/docs/examples/web-automation/scrapy.mdx +++ b/docs/examples/web-automation/scrapy.mdx @@ -69,12 +69,12 @@ Capture the prepared environment: ```sh macOS & Linux msb snapshot create scrapy-runtime \ - --from scrapy-base --integrity --force + --from-sandbox scrapy-base --integrity --force ``` ```powershell Windows msb snapshot create scrapy-runtime ` - --from scrapy-base --integrity --force + --from-sandbox scrapy-base --integrity --force ``` diff --git a/docs/sandboxes/lifecycle.mdx b/docs/sandboxes/lifecycle.mdx index bec17de40..6d5a32d29 100644 --- a/docs/sandboxes/lifecycle.mdx +++ b/docs/sandboxes/lifecycle.mdx @@ -99,7 +99,7 @@ Pause keeps the local VM and its RAM allocated without creating a snapshot. Resu ```bash CLI msb pause worker -msb snapshot create paused-state --from worker --full +msb snapshot create paused-state --from-sandbox worker --full msb resume worker ``` diff --git a/docs/sandboxes/snapshots.mdx b/docs/sandboxes/snapshots.mdx index 0d9edb319..8ba436989 100644 --- a/docs/sandboxes/snapshots.mdx +++ b/docs/sandboxes/snapshots.mdx @@ -29,7 +29,7 @@ On supported Linux, macOS, and Windows hosts, add `--forked` when restoring a fu ```bash msb create alpine --name baseline --root-disk flat:1G -msb snapshot create ready --from baseline --full +msb snapshot create ready --from-sandbox baseline --full msb create --name worker-a --from-snapshot baseline:ready --forked msb create --name worker-b --from-snapshot baseline:ready --forked ``` @@ -65,7 +65,7 @@ msb exec baseline -- pip install requests msb stop baseline # 2. Snapshot the stopped sandbox -msb snapshot create after-pip-install --from baseline +msb snapshot create after-pip-install --from-sandbox baseline # 3. Boot a fresh sandbox from the snapshot msb run --name worker --from-snapshot baseline:after-pip-install \ @@ -85,8 +85,8 @@ Each capture is an immutable member of a snapshot group. The group defaults to t Use a bare group to restore its selected head, or `group:member` for an exact checkpoint: ```bash -msb snapshot create cp01 --from baseline --group work -msb snapshot create cp02 --from baseline --group work +msb snapshot create cp01 --from-sandbox baseline --group work +msb snapshot create cp02 --from-sandbox baseline --group work msb create --name latest --from-snapshot work msb create --name earlier --from-snapshot work:cp01 @@ -177,11 +177,11 @@ fmt.Println(snap.Digest()) // sha256:... ``` ```bash CLI -msb snapshot create after-pip-install --from baseline -msb snapshot create ready-with-label --from baseline --label stage=ready +msb snapshot create after-pip-install --from-sandbox baseline +msb snapshot create ready-with-label --from-sandbox baseline --label stage=ready # Use another group-store root: /mnt/big/baseline/snap_/ -msb snapshot create after-pip-install --from baseline --dest-dir /mnt/big +msb snapshot create after-pip-install --from-sandbox baseline --dest-dir /mnt/big ``` @@ -225,7 +225,7 @@ snap, err := m.Snapshot.Create(ctx, m.SnapshotCreateOptions{ ``` ```bash CLI -msb snapshot create worker-checkpoint --from worker --full +msb snapshot create worker-checkpoint --from-sandbox worker --full ``` @@ -238,7 +238,7 @@ Creating a sandbox from the resulting snapshot restores into a child-owned disk Repeated full checkpoints add disk layers. You choose when to export changes and when to compact; neither happens automatically. ```bash -msb snapshot create checkpoint-b --from worker --full +msb snapshot create checkpoint-b --from-sandbox worker --full msb snapshot save worker:checkpoint-b changes.msb --since worker:checkpoint-a msb modify worker --compact --layers 3 --dry-run msb modify worker --compact --layers 3 @@ -301,7 +301,7 @@ archive, err := m.Snapshot.CreateArchive(ctx, m.SnapshotArchiveOptions{ ```bash CLI msb snapshot create after-pip-install \ - --from baseline \ + --from-sandbox baseline \ --archive /tmp/after-pip-install.msb ``` @@ -573,7 +573,7 @@ report, err := snap.Verify(ctx) ```bash CLI # Compute and record an integrity hash at create time -msb snapshot create after-pip-install --from baseline --integrity +msb snapshot create after-pip-install --from-sandbox baseline --integrity # Verify a snapshot's recorded integrity on demand msb snapshot verify baseline:after-pip-install diff --git a/docs/snapshot-groups-explained.md b/docs/snapshot-groups-explained.md index c056c2128..0f381d5b7 100644 --- a/docs/snapshot-groups-explained.md +++ b/docs/snapshot-groups-explained.md @@ -49,8 +49,8 @@ Groups do not magically make random IDs collision-proof. They keep local address msb create alpine --name worker --memory 512M # Group defaults to the source sandbox's name: worker. -msb snapshot create cp01 --from worker --full -msb snapshot create cp02 --from worker --full +msb snapshot create cp01 --from-sandbox worker --full +msb snapshot create cp02 --from-sandbox worker --full # A bare group selects its head, currently cp02. msb create --name latest --from-snapshot worker --forked @@ -59,7 +59,7 @@ msb create --name latest --from-snapshot worker --forked msb create --name earlier --from-snapshot worker:cp01 --forked # You can choose a different group, or let a member name be generated. -msb snapshot create --from worker --group experiments --full +msb snapshot create --from-sandbox worker --group experiments --full ``` `--forked` shares clean restored RAM pages using copy-on-write; child writes remain private. It does not change which snapshot is selected. Omit `--full` at capture for a disk-only snapshot, and omit `--forked` when cold-booting disk state. diff --git a/scripts/smoke/cli/checkpoint-clock.py b/scripts/smoke/cli/checkpoint-clock.py index 678d28c41..ef058aa75 100644 --- a/scripts/smoke/cli/checkpoint-clock.py +++ b/scripts/smoke/cli/checkpoint-clock.py @@ -45,10 +45,10 @@ def run(label, *args, check=True, timeout=120): time.sleep(0.05) else: raise RuntimeError("guest clock fixture did not start") - run("capture", "snapshot", "create", snapshot, "--from", source, "--full", "--info") + run("capture", "snapshot", "create", snapshot, "--from-sandbox", source, "--full", "--info") if os.environ.get("CLOCK_INCREMENTAL") == "1": snapshot = prefix + "-next" - run("capture-next", "snapshot", "create", snapshot, "--from", source, "--full", "--info") + run("capture-next", "snapshot", "create", snapshot, "--from-sandbox", source, "--full", "--info") if os.environ.get("CLOCK_ARCHIVE") == "1": archive = str(out / "clock.msb") run("archive", "snapshot", "save", snapshot, archive) diff --git a/scripts/smoke/cli/checkpoint-clock.sh b/scripts/smoke/cli/checkpoint-clock.sh index 2285461f7..bf4253aaa 100644 --- a/scripts/smoke/cli/checkpoint-clock.sh +++ b/scripts/smoke/cli/checkpoint-clock.sh @@ -17,10 +17,10 @@ for _ in $(seq 1 30); do if msb exec "$source_name" -- test -s /tmp/clock-records.csv; then break; fi sleep 0.05 done -msb snapshot create "$CLOCK_PREFIX-full" --from "$source_name" --full --info >"$CLOCK_OUT/capture.out" 2>"$CLOCK_OUT/capture.err" +msb snapshot create "$CLOCK_PREFIX-full" --from-sandbox "$source_name" --full --info >"$CLOCK_OUT/capture.out" 2>"$CLOCK_OUT/capture.err" snapshot="$CLOCK_PREFIX-full" if [ "${CLOCK_INCREMENTAL:-0}" = 1 ]; then - msb snapshot create "$CLOCK_PREFIX-next" --from "$source_name" --full --info >"$CLOCK_OUT/capture-next.out" 2>"$CLOCK_OUT/capture-next.err" + msb snapshot create "$CLOCK_PREFIX-next" --from-sandbox "$source_name" --full --info >"$CLOCK_OUT/capture-next.out" 2>"$CLOCK_OUT/capture-next.err" snapshot="$CLOCK_PREFIX-next" fi if [ "${CLOCK_ARCHIVE:-0}" = 1 ]; then diff --git a/scripts/smoke/cli/checkpoint-cpu-state.py b/scripts/smoke/cli/checkpoint-cpu-state.py index 68aef4103..993bcb3a0 100644 --- a/scripts/smoke/cli/checkpoint-cpu-state.py +++ b/scripts/smoke/cli/checkpoint-cpu-state.py @@ -39,7 +39,7 @@ def run(label, *args, check=True): run("cpu1-before", "exec", source, "--", "/cpu-probe", "1") run("offline", "exec", source, "--", "sh", "-c", "echo 0 > /sys/devices/system/cpu/cpu1/online") assert run("offline-before", "exec", source, "--", "cat", "/sys/devices/system/cpu/cpu1/online") == b"0" - run("capture", "snapshot", "create", prefix + "-full", "--from", source, "--full", "--info") + run("capture", "snapshot", "create", prefix + "-full", "--from-sandbox", source, "--full", "--info") run("restore", "create", "-n", child, "--from-snapshot", prefix + "-full", *(["--forked"] if os.environ.get("CPU_FORKED") == "1" else []), "--info") assert run("offline-after", "exec", child, "--", "cat", "/sys/devices/system/cpu/cpu1/online") == b"0" diff --git a/scripts/smoke/cli/cow-memory-lifecycle.py b/scripts/smoke/cli/cow-memory-lifecycle.py index 1a87855fe..d1e800de3 100644 --- a/scripts/smoke/cli/cow-memory-lifecycle.py +++ b/scripts/smoke/cli/cow-memory-lifecycle.py @@ -87,15 +87,15 @@ def run(label, *args, expected=0, timeout=120): time.sleep(0.1) assert run(f"memory-marker-{step}", "exec", source, "--", "cat", "/dev/shm/cow-marker").stdout.strip() == "captured" snap = prefix + "-full" - run("first-full", "snapshot", "create", snap, "--from", source, "--full", "--info") + run("first-full", "snapshot", "create", snap, "--from-sandbox", source, "--full", "--info") run("pause", "pause", source) run("pause-idempotent", "pause", source) inspected = run("paused-inspect", "inspect", source, "--format", "json") assert json.loads(inspected.stdout)["status"] == "Paused" refusal = run("paused-exec", "exec", source, "--", "true", expected=None, timeout=10) assert refusal.returncode != 0, "paused exec must fail promptly" - run("paused-full-1", "snapshot", "create", prefix + "-paused1", "--from", source, "--full", "--info") - run("paused-full-2", "snapshot", "create", prefix + "-paused2", "--from", source, "--full", "--info") + run("paused-full-1", "snapshot", "create", prefix + "-paused1", "--from-sandbox", source, "--full", "--info") + run("paused-full-2", "snapshot", "create", prefix + "-paused2", "--from-sandbox", source, "--full", "--info") time.sleep(float(os.environ.get("STACK8_PAUSE_SECONDS", "5"))) run("resume", "resume", source) run("resume-idempotent", "resume", source) @@ -120,14 +120,14 @@ def run(label, *args, expected=0, timeout=120): # A restored child remains a normal capture source; no creation-time memory opt-in exists. child_snapshot = prefix + "-child-full" run("capture-restored-child", "snapshot", "create", child_snapshot, - "--from", prefix + "-a", "--full", "--info") + "--from-sandbox", prefix + "-a", "--full", "--info") grandchild = prefix + "-grandchild" names.append(grandchild) run("restore-grandchild", "create", "-n", grandchild, "--from-snapshot", child_snapshot, *restore_flags, "--info") assert run("grandchild-marker", "exec", grandchild, "--", "cat", "/dev/shm/cow-marker").stdout.strip() == "private-a" archive = str(root / "direct.msb") - run("direct-full", "snapshot", "create", prefix + "-direct", "--from", source, + run("direct-full", "snapshot", "create", prefix + "-direct", "--from-sandbox", source, "--full", "--archive", archive, "--info") child = prefix + "-archive" names.append(child) @@ -139,7 +139,7 @@ def run(label, *args, expected=0, timeout=120): run("pause-for-stop", "pause", source) run("stop-paused", "stop", source, timeout=20) disk_snapshot = prefix + "-disk" - run("stopped-disk-capture", "snapshot", "create", disk_snapshot, "--from", source) + run("stopped-disk-capture", "snapshot", "create", disk_snapshot, "--from-sandbox", source) disk_archive = str(root / "disk.msb") run("disk-archive", "snapshot", "save", disk_snapshot, disk_archive) for label, snapshot in (("installed", disk_snapshot), ("archive", disk_archive)): diff --git a/scripts/smoke/cli/direct-branch.py b/scripts/smoke/cli/direct-branch.py index 0d98630d7..ea665f093 100644 --- a/scripts/smoke/cli/direct-branch.py +++ b/scripts/smoke/cli/direct-branch.py @@ -52,7 +52,7 @@ def benchmark(source): saved = prefix + "-warm" if os.environ.get("STACK8_BENCH_COMPACT") == "1": run("setup-compact-snapshot", "modify", source, "--compact", "--format", "json") - run("capture-warm-source", "snapshot", "create", saved, "--from", source, "--full") + run("capture-warm-source", "snapshot", "create", saved, "--from-sandbox", source, "--full") for mode in ("forked", "eager"): for i in range(8): child = prefix + f"-{mode}-{i}" @@ -67,7 +67,7 @@ def benchmark(source): saved = prefix + f"-full-{i}" child = prefix + f"-durable-{i}" names.append(child) - run(f"pipeline-capture-{i}", "snapshot", "create", saved, "--from", source, "--full") + run(f"pipeline-capture-{i}", "snapshot", "create", saved, "--from-sandbox", source, "--full") run(f"pipeline-restore-{i}", "create", "--name", child, "--from-snapshot", saved, "--forked") assert exec_guest(child, "cat /dev/shm/branch-marker", f"pipeline-ready-{i}") == "source" run(f"stop-pipeline-{i}", "stop", child) @@ -113,7 +113,7 @@ def benchmark(source): # Compare durable capture+forked-child against the same source and readiness endpoint. for i in range(3): snap = prefix + f"-saved-{i}" - run(f"full-capture-{i}", "snapshot", "create", snap, "--from", source, "--full") + run(f"full-capture-{i}", "snapshot", "create", snap, "--from-sandbox", source, "--full") name = prefix + f"-restored-{i}" names.append(name) run(f"forked-restore-{i}", "create", "--name", name, "--from-snapshot", snap, "--forked") diff --git a/scripts/smoke/cli/dirty-memory-checkpoint.py b/scripts/smoke/cli/dirty-memory-checkpoint.py index b0b4d3101..de68a0ba9 100644 --- a/scripts/smoke/cli/dirty-memory-checkpoint.py +++ b/scripts/smoke/cli/dirty-memory-checkpoint.py @@ -113,7 +113,7 @@ def stop(name): time.sleep(.1) assert initial['dirty_kib'] >= 32 * 1024, initial report['initial'] = initial - run('full-dirty', 'snapshot', 'create', 'dirty-full', '--from', source, '--full') + run('full-dirty', 'snapshot', 'create', 'dirty-full', '--from-sandbox', source, '--full') matches(initial) names.append('running-branch') run('branch-dirty', 'branch', source, '--name', 'running-branch') @@ -130,7 +130,7 @@ def stop(name): stop('running-branch'); matches(initial) run('pause', 'pause', source) for suffix in ('one', 'two'): - run('capture-paused-' + suffix, 'snapshot', 'create', 'paused-' + suffix, '--from', source, '--full') + run('capture-paused-' + suffix, 'snapshot', 'create', 'paused-' + suffix, '--from-sandbox', source, '--full') assert json.loads(run('inspect-paused-' + suffix, 'inspect', source, '--format', 'json'))['status'] == 'Paused' names.append('paused-branch') run('branch-paused-dirty', 'branch', source, '--name', 'paused-branch') @@ -150,10 +150,10 @@ def stop(name): if mode == 'forked': # A restored child starts a fresh dirty-tracking baseline while retaining # snapshot ancestry. Capture before mutating, then verify an incremental cut. - run('child-baseline', 'snapshot', 'create', 'child-baseline', '--from', child, '--full') + run('child-baseline', 'snapshot', 'create', 'child-baseline', '--from-sandbox', child, '--full') changed = state('/mutate') assert changed['private'] == 'private1' - captured = run('incremental-dirty', 'snapshot', 'create', 'dirty-incremental', '--from', child, '--full') + captured = run('incremental-dirty', 'snapshot', 'create', 'dirty-incremental', '--from-sandbox', child, '--full') # Capture returns the exact installed member path, independent of its alias. checkpoint = Path(captured.strip().splitlines()[-1]) / 'checkpoint' descriptor = json.loads((checkpoint / 'checkpoint.json').read_text()) diff --git a/scripts/smoke/cli/disk-compaction-depth.ps1 b/scripts/smoke/cli/disk-compaction-depth.ps1 index 1145bf66f..f87a6f70d 100644 --- a/scripts/smoke/cli/disk-compaction-depth.ps1 +++ b/scripts/smoke/cli/disk-compaction-depth.ps1 @@ -27,7 +27,7 @@ try { M @('exec',$name,'--','sh','-c','dd if=/dev/urandom of=/payload bs=1048576 count=4 2>/dev/null; sha256sum /payload >/expected; sync') foreach ($generation in 1..64) { M @('exec',$name,'--','sh','-c',"echo $generation >/version; sync") - M @('snapshot','create',"$name-$generation",'--from',$name,'--full') *> "$output\$layout-$generation.log" + M @('snapshot','create',"$name-$generation",'--from-sandbox',$name,'--full') *> "$output\$layout-$generation.log" if ($generation -in @(1,4,16,64)) { M @('modify',$name,'--compact','--dry-run','--format','json') > "$output\$layout-depth-$generation.json" } } } diff --git a/scripts/smoke/cli/disk-compaction-depth.sh b/scripts/smoke/cli/disk-compaction-depth.sh index 9fefbe5a5..f9ef6a46e 100644 --- a/scripts/smoke/cli/disk-compaction-depth.sh +++ b/scripts/smoke/cli/disk-compaction-depth.sh @@ -13,7 +13,7 @@ for layout in managed flat; do msb exec "$name" -- sh -c 'dd if=/dev/urandom of=/payload bs=1048576 count=4 2>/dev/null; sha256sum /payload >/expected; sync' for generation in $(seq 1 64); do msb exec "$name" -- sh -c "echo $generation >/version; sync" >/dev/null - msb snapshot create "$name-$generation" --from "$name" --full >"$QUAL_ROOT/$layout-$generation.out" 2>&1 + msb snapshot create "$name-$generation" --from-sandbox "$name" --full >"$QUAL_ROOT/$layout-$generation.out" 2>&1 case $generation in 1|4|16|64) msb modify "$name" --compact --dry-run --format json >"$QUAL_ROOT/$layout-depth-$generation.json" msb exec "$name" -- sh -c 'sha256sum -c /expected' >/dev/null;; diff --git a/scripts/smoke/cli/disk-compaction-export.ps1 b/scripts/smoke/cli/disk-compaction-export.ps1 index 4b5eb5ecd..08ecfc686 100644 --- a/scripts/smoke/cli/disk-compaction-export.ps1 +++ b/scripts/smoke/cli/disk-compaction-export.ps1 @@ -34,7 +34,7 @@ try { Measure-Msb "$layout-seed" @('exec',$name,'--','sh','-c','dd if=/dev/urandom of=/payload bs=1048576 count=8 2>/dev/null; sha256sum /payload >/expected; mkdir -p /dev/shm; echo volatile >/dev/shm/ram-marker; sync') foreach ($generation in 1..4) { Measure-Msb "$layout-write-$generation" @('exec',$name,'--','sh','-c',"echo $generation >/version; sync") - Measure-Msb "$layout-checkpoint-$generation" @('snapshot','create',"$name-$generation",'--from',$name,'--full') + Measure-Msb "$layout-checkpoint-$generation" @('snapshot','create',"$name-$generation",'--from-sandbox',$name,'--full') } Measure-Msb "$layout-dry-run" @('modify',$name,'--compact','--layers','3','--dry-run','--format','json') $plan = Get-Content "$output\logs\$layout-dry-run.out" -Raw | ConvertFrom-Json @@ -53,11 +53,11 @@ try { Measure-Msb "$layout-data-after" @('exec',$name,'--','sh','-c','sha256sum -c /expected && test $(cat /version) = 4 && echo after >/after && sync') Measure-Msb "$layout-stop" @('stop',$name) Measure-Msb "$layout-offline-compact" @('modify',$name,'--compact','--format','json') - Measure-Msb "$layout-stopped-snapshot" @('snapshot','create',"$name-stopped",'--from',$name,'--integrity') + Measure-Msb "$layout-stopped-snapshot" @('snapshot','create',"$name-stopped",'--from-sandbox',$name,'--integrity') Measure-Msb "$layout-stopped-verify" @('snapshot','verify',"$name-stopped") Measure-Msb "$layout-restart" @('start',$name) Measure-Msb "$layout-restarted-data" @('exec',$name,'--','sh','-c','sha256sum -c /expected && test $(cat /version) = 4 && test $(cat /after) = after') - Measure-Msb "$layout-post-compact-checkpoint" @('snapshot','create',"$name-new",'--from',$name,'--full') + Measure-Msb "$layout-post-compact-checkpoint" @('snapshot','create',"$name-new",'--from-sandbox',$name,'--full') Measure-Msb "$layout-old-prefix-rejected" @('snapshot','save',"$name-new","$output\invalid.tar",'--since',"$name-4") $true Measure-Msb "$layout-stop-source" @('stop',$name) foreach ($variant in @('old','full','disk','stopped')) { diff --git a/scripts/smoke/cli/disk-compaction-export.sh b/scripts/smoke/cli/disk-compaction-export.sh index 5e10f62d4..a03b29908 100644 --- a/scripts/smoke/cli/disk-compaction-export.sh +++ b/scripts/smoke/cli/disk-compaction-export.sh @@ -36,7 +36,7 @@ for layout in managed flat; do measure "$layout-seed" guest "$name" 'dd if=/dev/urandom of=/payload bs=1048576 count=8 2>/dev/null; sha256sum /payload >/expected; printf 1 >/version; mkdir -p /dev/shm; echo volatile >/dev/shm/ram-marker; sync' for generation in 1 2 3 4; do measure "$layout-write-$generation" guest "$name" "printf $generation >/version; sync" - measure "$layout-checkpoint-$generation" msb snapshot create "$name-$generation" --from "$name" --full + measure "$layout-checkpoint-$generation" msb snapshot create "$name-$generation" --from-sandbox "$name" --full done measure "$layout-dry-run" msb modify "$name" --compact --layers 3 --dry-run --format json measure "$layout-dry-run-counts" jq -e '.dry_run and .input_layers == 5 and .selected_layers == 3 and .output_layers == 3' "$QUAL_ROOT/logs/$layout-dry-run.out" @@ -61,11 +61,11 @@ for layout in managed flat; do measure "$layout-online-counts" jq -e '.input_layers == 5 and .output_layers == 3 and .selected_layers == 3' "$QUAL_ROOT/logs/$layout-online-compact.out" measure "$layout-stop" msb stop "$name" measure "$layout-offline-compact" msb modify "$name" --compact --format json - measure "$layout-stopped-snapshot" msb snapshot create "$name-stopped" --from "$name" --integrity + measure "$layout-stopped-snapshot" msb snapshot create "$name-stopped" --from-sandbox "$name" --integrity measure "$layout-stopped-verify" msb snapshot verify "$name-stopped" measure "$layout-restart" msb start "$name" measure "$layout-restarted-data" guest "$name" 'sha256sum -c /expected && test "$(cat /version)" = 4' - measure "$layout-post-compact-checkpoint" msb snapshot create "$name-new" --from "$name" --full + measure "$layout-post-compact-checkpoint" msb snapshot create "$name-new" --from-sandbox "$name" --full measure "$layout-old-prefix-rejected" refuse msb snapshot save "$name-new" "$QUAL_ROOT/invalid.tar" --since "$name-4" measure "$layout-stop-source" msb stop "$name" diff --git a/scripts/smoke/cli/disk-compaction-negative.sh b/scripts/smoke/cli/disk-compaction-negative.sh index 141849ff6..21ffbbbf6 100644 --- a/scripts/smoke/cli/disk-compaction-negative.sh +++ b/scripts/smoke/cli/disk-compaction-negative.sh @@ -8,7 +8,7 @@ trap cleanup EXIT refuse() { if "$@"; then echo 'unexpected success' >&2; exit 1; fi; } msb create -n compact-neg-tmpfs --root-disk tmpfs:128M -m 256M --max-duration 5m alpine refuse msb modify compact-neg-tmpfs --compact -msb snapshot create compact-neg-tmpfs-snap --from compact-neg-tmpfs --full +msb snapshot create compact-neg-tmpfs-snap --from-sandbox compact-neg-tmpfs --full refuse msb snapshot save compact-neg-tmpfs-snap "$QUAL_ROOT/tmpfs.tar" --last-layers 1 msb stop compact-neg-tmpfs cp "$MSB_HOME/sandboxes/$QUAL_SOURCE/upper.ext4" "$QUAL_ROOT/owned.ext4" diff --git a/scripts/smoke/cli/failed-restore.py b/scripts/smoke/cli/failed-restore.py index 165647a06..5ad94035a 100644 --- a/scripts/smoke/cli/failed-restore.py +++ b/scripts/smoke/cli/failed-restore.py @@ -48,7 +48,7 @@ def layers(): try: call("source", "create", "alpine", "--name", source, "--root-disk", os.environ.get("STACK8_LAYOUT", "flat:512M"), "--memory", "256M") call("marker", "exec", source, "--", "sh", "-c", "echo preserved > /dev/shm/restore-marker; echo disk-preserved > /restore-disk-marker") - call("capture", "snapshot", "create", snapshot, "--from", source, "--full") + call("capture", "snapshot", "create", snapshot, "--from-sandbox", source, "--full") before = layers() # Trigger a real host I/O error during memory installation, after child staging # and DB insertion. No production test-only failure hook is necessary. @@ -67,7 +67,7 @@ def layers(): for label, args in [("start", ["start", child]), ("exec", ["exec", child, "--", "true"]), ("modify", ["modify", child, "--root-disk", "8G"]), ("compact", ["modify", child, "--compact"]), - ("snapshot", ["snapshot", "create", prefix+'-invalid', "--from", child])]: + ("snapshot", ["snapshot", "create", prefix+'-invalid', "--from-sandbox", child])]: refused = call(label + "-refused", *args, expected=None) assert refused.returncode != 0 and "incomplete restore" in refused.stderr, rows[-1] assert layers() == before, "failed restore or later lifecycle mutated sealed disk bytes" diff --git a/scripts/smoke/cli/incremental-full-archive.py b/scripts/smoke/cli/incremental-full-archive.py index a8ba46347..fa7a9d8bb 100644 --- a/scripts/smoke/cli/incremental-full-archive.py +++ b/scripts/smoke/cli/incremental-full-archive.py @@ -87,7 +87,7 @@ def restore(label, snapshot, expected, base=None, forked=False): if n == 6: run("pause-source", source_home, "pause", source) name = f"cp{n:02}" - run(f"capture-{n}", source_home, "snapshot", "create", name, "--from", source, "--full") + run(f"capture-{n}", source_home, "snapshot", "create", name, "--from-sandbox", source, "--full") if n == 6: run("resume-source", source_home, "resume", source) artifact = source_home / "snapshots" / name diff --git a/scripts/smoke/cli/live-disk-snapshot.py b/scripts/smoke/cli/live-disk-snapshot.py index 0dca513ed..e55d1b0b0 100644 --- a/scripts/smoke/cli/live-disk-snapshot.py +++ b/scripts/smoke/cli/live-disk-snapshot.py @@ -69,7 +69,7 @@ def sealed_hashes(root): run("reset-" + snap, "exec", source, "--", "sh", "-c", "echo before > /disk-marker; sync") if mode == "paused": run("pause-" + layout, "pause", source) - args = ["snapshot", "create", snap, "--from", source] + args = ["snapshot", "create", snap, "--from-sandbox", source] archive = out / (snap + (".tar" if mode == "plain" else ".msb")) installed_before = set((home / "snapshots").glob("*")) if mode in ("archive", "plain"): @@ -124,7 +124,7 @@ def sealed_hashes(root): "for i in $(seq 1 100); do [ -s /counter ] && exit 0; sleep 0.02; done; exit 1") start_counter = int(run("counter-before-" + layout, "exec", source, "--", "cat", "/counter").stdout.strip()) run("counter-baseline-sync-" + layout, "exec", source, "--", "sync") - run("busy-capture-" + layout, "snapshot", "create", source + "-busy", "--from", source) + run("busy-capture-" + layout, "snapshot", "create", source + "-busy", "--from-sandbox", source) run("counter-progress-" + layout, "exec", source, "--", "sh", "-c", f"for i in $(seq 1 100); do [ $(cat /counter) -gt {start_counter} ] && exit 0; sleep 0.02; done; exit 1") run("stop-writer-" + layout, "exec", source, "--", "touch", "/stop-counter") @@ -137,31 +137,31 @@ def sealed_hashes(root): run("stop-busy-child-" + layout, "stop", busy_child) # A later full checkpoint must still work after disk-only generations. full = source + "-full" - run("full-after-disk-" + layout, "snapshot", "create", full, "--from", source, "--full") + run("full-after-disk-" + layout, "snapshot", "create", full, "--from-sandbox", source, "--full") full_child = source + "-full-child" names.append(full_child) run("full-restore-" + layout, "create", "--name", full_child, "--from-snapshot", full) assert run("full-ram-" + layout, "exec", full_child, "--", "cat", "/dev/shm/ram-marker").stdout.strip() == "ram-only" # A disk-only cut between full captures must not consume/advance the RAM baseline. memory_before = files(runtime / "checkpoint-store") - run("disk-between-full-" + layout, "snapshot", "create", source + "-between", "--from", source) + run("disk-between-full-" + layout, "snapshot", "create", source + "-between", "--from-sandbox", source) assert files(runtime / "checkpoint-store") == memory_before run("change-ram-" + layout, "exec", source, "--", "sh", "-c", "echo updated > /dev/shm/ram-marker") - run("second-full-" + layout, "snapshot", "create", full + "-next", "--from", source, "--full") + run("second-full-" + layout, "snapshot", "create", full + "-next", "--from-sandbox", source, "--full") next_child = source + "-next-child" names.append(next_child) run("second-full-restore-" + layout, "create", "--name", next_child, "--from-snapshot", full + "-next") assert run("second-full-ram-" + layout, "exec", next_child, "--", "cat", "/dev/shm/ram-marker").stdout.strip() == "updated" # A name collision must fail before publication and leave both the source and snapshot usable. - refused = run("duplicate-refused-" + layout, "snapshot", "create", source + "-between", "--from", source, ok=False) + refused = run("duplicate-refused-" + layout, "snapshot", "create", source + "-between", "--from-sandbox", source, ok=False) assert refused.returncode != 0 run("source-after-refusal-" + layout, "exec", source, "--", "true") run("stop-source-" + layout, "stop", source) - run("stopped-after-live-" + layout, "snapshot", "create", source + "-stopped", "--from", source) + run("stopped-after-live-" + layout, "snapshot", "create", source + "-stopped", "--from-sandbox", source) tmpfs = prefix + "-tmpfs" names.append(tmpfs) run("tmpfs-create", "create", "alpine", "--name", tmpfs, "--root-disk", "tmpfs:128M", "--memory", "256M") - refused = run("tmpfs-refused", "snapshot", "create", tmpfs + "-bad", "--from", tmpfs, ok=False) + refused = run("tmpfs-refused", "snapshot", "create", tmpfs + "-bad", "--from-sandbox", tmpfs, ok=False) assert refused.returncode != 0 and "tmpfs" in refused.stderr run("tmpfs-still-running", "exec", tmpfs, "--", "true") print(json.dumps({"result": "pass", "layouts": ["flat", "managed"], "modes": ["installed", "integrity", "archive", "plain", "paused"]})) diff --git a/scripts/smoke/cli/root-disk-growth.py b/scripts/smoke/cli/root-disk-growth.py index 60d997c15..9adc913b3 100644 --- a/scripts/smoke/cli/root-disk-growth.py +++ b/scripts/smoke/cli/root-disk-growth.py @@ -76,7 +76,7 @@ def file_hash(path): guest(f"{layout}-seed", name, "dd if=/dev/urandom of=/payload bs=1048576 count=8 2>/dev/null; sha256sum /payload >/expected; echo ram >/dev/shm/grow-marker; sync") phase_grow(f"{layout}-raw-live", name, 768) guest(f"{layout}-new-space", name, "dd if=/dev/zero of=/space bs=1048576 count=600 conv=fsync && test $(stat -c %s /space) = 629145600 && rm /space && sha256sum -c /expected") - run(f"{layout}-old-snapshot", "snapshot", "create", f"{name}-old", "--from", name, "--full") + run(f"{layout}-old-snapshot", "snapshot", "create", f"{name}-old", "--from-sandbox", name, "--full") before = journal(name) ancestor = Path(before["layers"][0]["path"]) ancestor_before = file_hash(ancestor) @@ -85,7 +85,7 @@ def file_hash(path): assert len(journal(name)["layers"]) == len(before["layers"]) assert file_hash(ancestor) == ancestor_before run(f"{layout}-shrink-refused", "modify", name, "--root-disk", "512M", refuse=True) - run(f"{layout}-new-snapshot", "snapshot", "create", f"{name}-new", "--from", name, "--full") + run(f"{layout}-new-snapshot", "snapshot", "create", f"{name}-new", "--from-sandbox", name, "--full") run(f"{layout}-compact", "modify", name, "--compact", "--format", "json") phase_grow(f"{layout}-compacted-live", name, 1536) phase_grow(f"{layout}-partial-group-live", name, 1700) @@ -98,7 +98,7 @@ def file_hash(path): run(f"{layout}-defer-start", "start", name) guest(f"{layout}-defer-space", name, "dd if=/dev/zero of=/space bs=1048576 count=1800 conv=fsync && rm /space && sha256sum -c /expected") run(f"{layout}-final-stop", "stop", name) - run(f"{layout}-stopped-snapshot", "snapshot", "create", f"{name}-stopped", "--from", name, "--integrity") + run(f"{layout}-stopped-snapshot", "snapshot", "create", f"{name}-stopped", "--from-sandbox", name, "--integrity") run(f"{layout}-verify", "snapshot", "verify", f"{name}-stopped") for suffix, capacity in (("old", 768), ("new", 1280)): child = f"{name}-{suffix}-child" diff --git a/scripts/smoke/cli/root-disk-large-growth.py b/scripts/smoke/cli/root-disk-large-growth.py index 7a50a85f6..fadcc0b6d 100644 --- a/scripts/smoke/cli/root-disk-large-growth.py +++ b/scripts/smoke/cli/root-disk-large-growth.py @@ -90,7 +90,7 @@ def case(layout, backing, mode, target): "cat /proc/sys/kernel/random/boot_id >/boot-before; sync") if backing == "qcow2": run(label + "-old-snapshot", "snapshot", "create", name + "-old", - "--from", name, "--full") + "--from-sandbox", name, "--full") before = state(name) check(label + "-initial-capacity", capacity(before["layers"][-1]) == 512 * MIB) check(label + "-initial-format", @@ -133,7 +133,7 @@ def case(layout, backing, mode, target): f"dd if=/far bs=1048576 skip={target - 256} count=8 2>/dev/null | cmp - /payload; " "df -k /; du -k /large") # Snapshot the allocated file, not just metadata or an empty resized filesystem. - run(label + "-new-snapshot", "snapshot", "create", name + "-new", "--from", name, "--full") + run(label + "-new-snapshot", "snapshot", "create", name + "-new", "--from-sandbox", name, "--full") run(label + "-source-stop", "stop", name) for suffix, expected in [("new", target)] + ([("old", 512)] if backing == "qcow2" else []): child = name + "-" + suffix + "-child" diff --git a/scripts/smoke/cli/snapshot-groups.py b/scripts/smoke/cli/snapshot-groups.py index 341cda209..59e3021f6 100644 --- a/scripts/smoke/cli/snapshot-groups.py +++ b/scripts/smoke/cli/snapshot-groups.py @@ -55,7 +55,7 @@ def create(name, snapshot=None, forked=False): def capture(label, source, member, group="work", full=False, fail=False): - args = ["snapshot", "create", member, "--from", source, "--group", group] + args = ["snapshot", "create", member, "--from-sandbox", source, "--group", group] if full or full_only: args.append("--full") output = run(label, *args, fail=fail) @@ -171,7 +171,7 @@ def head(label, selector): # Direct archive capture records ancestry but never creates an installed member. before_members = sorted(str(p) for p in (home / "snapshots").rglob("snapshot.json")) direct = root / "direct.msb" - run("direct-capture", "snapshot", "create", "direct", "--from", "source", "--full", "--archive", direct) + run("direct-capture", "snapshot", "create", "direct", "--from-sandbox", "source", "--full", "--archive", direct) assert sorted(str(p) for p in (home / "snapshots").rglob("snapshot.json")) == before_members create("direct-restored", str(direct), forked=True) assert guest("direct-restored-state", "direct-restored", "cat /dev/shm/marker") == "ram-three" diff --git a/scripts/smoke/cli/snapshot-load-batch.py b/scripts/smoke/cli/snapshot-load-batch.py index 6f10b38ff..44af10d45 100644 --- a/scripts/smoke/cli/snapshot-load-batch.py +++ b/scripts/smoke/cli/snapshot-load-batch.py @@ -109,7 +109,7 @@ def restored(mode, group, marker): run("write-file-" + member, "exec", "batch-capture", "--", "sh", "-ec", "echo " + marker + " > /disk-marker; sync") output = run("capture-file-" + member, "snapshot", "create", member, - "--from", "batch-capture", "--group", "fresh") + "--from-sandbox", "batch-capture", "--group", "fresh") artifact = Path(output.splitlines()[-1]) fixtures[member] = (artifact, json.loads((artifact / "snapshot.json").read_text())) run("stop-file-source", "stop", "batch-capture") diff --git a/scripts/smoke/reports/live-disk-snapshot-2026-09-09.md b/scripts/smoke/reports/live-disk-snapshot-2026-09-09.md index a3a6b112b..8ed0b7588 100644 --- a/scripts/smoke/reports/live-disk-snapshot-2026-09-09.md +++ b/scripts/smoke/reports/live-disk-snapshot-2026-09-09.md @@ -8,11 +8,11 @@ Implemented on the #8 branch above Microsandbox `f68c1329`, using existing pinne ```sh # Source stays running. No RAM checkpoint or --full workaround. -msb snapshot create saved --from source +msb snapshot create saved --from-sandbox source msb create --name child --from-snapshot saved # Direct archive: no installed snapshot directory or index row. -msb snapshot create exported --from source --archive ./exported.msb +msb snapshot create exported --from-sandbox source --archive ./exported.msb msb create --name archive-child --from-snapshot ./exported.msb ``` From 24152183b0eace990798e31f1a53f024980f938f Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Thu, 10 Sep 2026 05:10:13 +0100 Subject: [PATCH 21/29] test(snapshot): add focused checks and live branch smoke Add a fast host-side snapshot/checkpoint test recipe and a bounded live smoke suite for managed and flat disks. Cover branching, paused capture, disk-only capture, grouped archive loading, eager/forked restore, and source/child isolation. Verify runtime exit and reclaim successful test homes while retaining reports and failure diagnostics. Wire the smoke suite into Linux/KVM CI, document the commands, align existing SDK/CLI tests with snapshot groups, and remove the extra branching slogan from the README. --- .github/workflows/check.yml | 34 ++ DEVELOPMENT.md | 22 ++ README.md | 2 +- justfile | 24 ++ scripts/smoke/cli/direct-branch.py | 20 +- scripts/smoke/cli/snapshot-branch.py | 393 +++++++++++++++++++ scripts/smoke/cli/test_snapshot_branch.py | 446 ++++++++++++++++++++++ sdk/go/cow_lifecycle_test.go | 10 +- sdk/python/tests/test_cow_lifecycle.py | 7 +- 9 files changed, 947 insertions(+), 11 deletions(-) create mode 100644 scripts/smoke/cli/snapshot-branch.py create mode 100644 scripts/smoke/cli/test_snapshot_branch.py diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index be8550630..605817a8d 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -1392,6 +1392,40 @@ jobs: scripts/smoke/cli/image-archive.sh scripts/smoke/cli/split-irqchip-bind-net.sh + - name: Snapshot smoke runner unit tests + run: python3 -m unittest discover -s scripts/smoke/cli -p test_snapshot_branch.py + + - name: Snapshot and branch live smoke + # The short operation deadline excludes image setup and bounded cleanup. + # Leave both layouts enough failure-path time to stop VMs and save evidence. + timeout-minutes: 15 + env: + MSB_LIBKRUNFW_PATH: ${{ github.workspace }}/build/libkrunfw.so.${{ env.LIBKRUNFW_VERSION }} + MSB_AGENTD_PATH: ${{ github.workspace }}/build/agentd + LD_LIBRARY_PATH: ${{ github.workspace }}/build + run: | + status=0 + # Short isolated homes avoid Unix socket limits and cross-layout reuse. + # Run both even if one fails so the artifact reports retain both outcomes. + for layout in managed flat; do + python3 scripts/smoke/cli/snapshot-branch.py \ + --binary "${{ github.workspace }}/build/msb" \ + --output "/tmp/msb-smoke-${{ github.run_id }}-${{ github.run_attempt }}-$layout" \ + --layout "$layout" || status=$? + done + exit "$status" + + - name: Upload snapshot smoke reports and logs + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: snapshot-branch-smoke-linux-x86_64 + # Keep text evidence, not the potentially large guest RAM/disk artifacts. + path: | + /tmp/msb-smoke-${{ github.run_id }}-${{ github.run_attempt }}-*/report.json + /tmp/msb-smoke-${{ github.run_id }}-${{ github.run_attempt }}-*/logs/*.log + if-no-files-found: warn + - name: Disk usage if: always() run: scripts/ci/clean-runner-disk.sh diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index a3d113902..27d3ddc6c 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -186,6 +186,28 @@ Run a specific test: cargo test -p microsandbox test_name ``` +### Snapshot and branch checks + +Run the focused logic suite without starting VMs: + +```bash +just test-snapshot +``` + +This covers snapshot archives/groups, dependency validation, checkpoint logic, snapshot CLI parsing, and the live-smoke runner's own unit tests. The Rust tests already run in the normal Linux workspace CI lane. Cached test execution is much shorter than a first build; Cargo compilation and dependency setup are additional costs, not snapshot-operation timings. + +For a compact end-to-end check, build a matching runtime bundle with `just build`, then run: + +```bash +just test-snapshot-live +just test-snapshot-live --layout flat +just test-snapshot-live --binary /path/to/msb --output /tmp/snapshot-smoke-new +``` + +The live check requires working virtualization and Python (`python3` on Linux/macOS, `python` on Windows). macOS binaries must be codesigned with `msb-entitlements.plist`; `just build` does this. It uses a new isolated `MSB_HOME`, stops its own VMs, verifies host-process exit, and retains a report and logs in the printed output directory. Successful runs remove their temporary RAM/disk artifacts; failed runs retain their home for investigation. An explicit `--output` directory must not exist; choose a short path under `/tmp` on Unix to stay within socket-path limits. Use `--help` for image and timeout options. + +The warm live target is under 60 seconds per layout, excluding compilation and image-pull setup; this is a target, not a guarantee or a performance benchmark. Per-command and suite deadlines bound failures separately. The existing Linux/KVM CLI smoke CI job runs managed and flat layouts and uploads reports/logs even on failure. This compact check complements, rather than replaces, the larger live invariant and benchmark matrices under `scripts/smoke/cli/`. + ## Benchmarking The benchmark suite lives in its own repository: diff --git a/README.md b/README.md index 9b6fd92b2..5eabf2702 100644 --- a/README.md +++ b/README.md @@ -307,7 +307,7 @@ The `msb` CLI provides a complete interface for managing sandboxes, snapshots, i > ``` > > ```sh -> # Fork a running sandbox. Take a new path. +> # Fork a running sandbox. > msb branch app --name experiment > msb exec experiment -- python -c "print('An independent copy!')" > msb branch experiment --name another-experiment diff --git a/justfile b/justfile index 4baeb04fd..b36821264 100644 --- a/justfile +++ b/justfile @@ -218,6 +218,30 @@ build mode="debug": (build-msb mode) _ensure-libkrunfw [windows] build mode="debug": (build-msb mode) _ensure-libkrunfw +# Run snapshot/archive/group and checkpoint logic tests without starting VMs. +test-snapshot: + cargo test -p microsandbox --lib snapshot:: + cargo test -p microsandbox --test snapshot_artifact + cargo test -p microsandbox-runtime --lib checkpoint:: + cargo test -p microsandbox-cli --lib commands::snapshot::tests + {{ if os_family() == "windows" { "python" } else { "python3" } }} -m unittest discover -s scripts/smoke/cli -p test_snapshot_branch.py + +# Run the compact live snapshot/branch smoke. Forward arguments without shell re-parsing. +[unix] +[script("python3")] +[positional-arguments] +test-snapshot-live *args: + import runpy + runpy.run_path("scripts/smoke/cli/snapshot-branch.py", run_name="__main__") + +# Run the same smoke with the native Windows Python launcher. +[windows] +[script("python")] +[positional-arguments] +test-snapshot-live *args: + import runpy + runpy.run_path("scripts/smoke/cli/snapshot-branch.py", run_name="__main__") + # Install msb and libkrunfw to ~/.microsandbox/{bin,lib}/ and configure shell paths. Requires: just build. [linux] install: diff --git a/scripts/smoke/cli/direct-branch.py b/scripts/smoke/cli/direct-branch.py index ea665f093..44fafce1d 100644 --- a/scripts/smoke/cli/direct-branch.py +++ b/scripts/smoke/cli/direct-branch.py @@ -40,6 +40,14 @@ def branch(source, child, label): assert exec_guest(child, "cat /dev/shm/branch-marker", label + "-ready") == "source" +def capture(source, member, label): + result = run(label, "snapshot", "create", member, "--from-sandbox", source, "--full") + # A member's bare alias no longer identifies its installed group; use the returned path. + path = Path(result.stdout.strip().splitlines()[-1]) + assert (path / "snapshot.json").is_file(), path + return str(path) + + def benchmark(source): # One source and at most one measured child: do not let accumulating VMs distort later # samples. Each CLI return includes activation; first guest command is recorded separately. @@ -52,12 +60,12 @@ def benchmark(source): saved = prefix + "-warm" if os.environ.get("STACK8_BENCH_COMPACT") == "1": run("setup-compact-snapshot", "modify", source, "--compact", "--format", "json") - run("capture-warm-source", "snapshot", "create", saved, "--from-sandbox", source, "--full") + saved_path = capture(source, saved, "capture-warm-source") for mode in ("forked", "eager"): for i in range(8): child = prefix + f"-{mode}-{i}" names.append(child) - run(f"{mode}-restore-{i}", "create", "--name", child, "--from-snapshot", saved, + run(f"{mode}-restore-{i}", "create", "--name", child, "--from-snapshot", saved_path, *(["--forked"] if mode == "forked" else [])) assert exec_guest(child, "cat /dev/shm/branch-marker", f"{mode}-ready-{i}") == "source" run(f"stop-{mode}-{i}", "stop", child) @@ -67,8 +75,8 @@ def benchmark(source): saved = prefix + f"-full-{i}" child = prefix + f"-durable-{i}" names.append(child) - run(f"pipeline-capture-{i}", "snapshot", "create", saved, "--from-sandbox", source, "--full") - run(f"pipeline-restore-{i}", "create", "--name", child, "--from-snapshot", saved, "--forked") + saved_path = capture(source, saved, f"pipeline-capture-{i}") + run(f"pipeline-restore-{i}", "create", "--name", child, "--from-snapshot", saved_path, "--forked") assert exec_guest(child, "cat /dev/shm/branch-marker", f"pipeline-ready-{i}") == "source" run(f"stop-pipeline-{i}", "stop", child) @@ -113,10 +121,10 @@ def benchmark(source): # Compare durable capture+forked-child against the same source and readiness endpoint. for i in range(3): snap = prefix + f"-saved-{i}" - run(f"full-capture-{i}", "snapshot", "create", snap, "--from-sandbox", source, "--full") + snap_path = capture(source, snap, f"full-capture-{i}") name = prefix + f"-restored-{i}" names.append(name) - run(f"forked-restore-{i}", "create", "--name", name, "--from-snapshot", snap, "--forked") + run(f"forked-restore-{i}", "create", "--name", name, "--from-snapshot", snap_path, "--forked") assert exec_guest(name, "cat /dev/shm/branch-marker", f"restore-ready-{i}") == "source" if os.environ.get("STACK8_MAINTENANCE") == "1" and not layout.startswith("tmpfs"): run("grow-source", "modify", source, "--root-disk", "768M", "--format", "json") diff --git a/scripts/smoke/cli/snapshot-branch.py b/scripts/smoke/cli/snapshot-branch.py new file mode 100644 index 000000000..4e8523deb --- /dev/null +++ b/scripts/smoke/cli/snapshot-branch.py @@ -0,0 +1,393 @@ +#!/usr/bin/env python3 +"""Small live snapshot/branch regression suite; no benchmark repetitions or shared VM state.""" + +import argparse +from contextlib import contextmanager +import csv +import io +import json +import os +from pathlib import Path +import signal +import shutil +import sqlite3 +import subprocess +import sys +import tempfile +import time + + +REPOSITORY = Path(__file__).resolve().parents[3] + + +def positive_seconds(value): + seconds = float(value) + if not 0 < seconds < float("inf"): + raise argparse.ArgumentTypeError("timeout must be finite and positive") + return seconds + + +def output_text(value): + # TimeoutExpired carries bytes even when subprocess.run requested text output. + return value.decode(errors="replace") if isinstance(value, bytes) else value or "" + + +@contextmanager +def defer_interrupts(): + # A second Ctrl-C/SIGTERM must not abandon the remaining bounded stop attempts. + received = [] + previous = {sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM)} + try: + for sig in previous: + signal.signal(sig, lambda signum, _frame: received.append(signum)) + yield received + finally: + for sig, handler in previous.items(): + signal.signal(sig, handler) + + +class Smoke: + def __init__(self, args): + self.args = args + self.binary = args.binary.expanduser().resolve(strict=True) + if not self.binary.is_file() or not os.access(self.binary, os.X_OK): + raise ValueError(f"msb binary is not executable: {self.binary}") + if args.output: + self.root = args.output.expanduser().resolve() + self.root.mkdir(mode=0o700, parents=True, exist_ok=False) + else: + # macOS's default per-user temp path is too long for the runtime's Unix sockets. + self.root = Path(tempfile.mkdtemp(prefix="msb-smoke-", dir="/tmp" if os.name == "posix" else None)) + self.logs = self.root / "logs" + self.logs.mkdir() + self.home = self.root / "home" + # Never borrow the caller's catalog, cloud backend, project config, or VM names. + # Explicit matching runtime/firmware overrides remain available to development builds. + self.env = dict(os.environ) + for key in ("MSB_HOME", "MSB_CONFIG_PATH", "MSB_BACKEND", "MSB_PROFILE"): + self.env.pop(key, None) + self.env.update(MSB_HOME=str(self.home), MSB_BACKEND="local", NO_COLOR="1") + self.names = [] + self.active = [] + self.deadline = None + self.report = { + "status": "running", "binary": str(self.binary), "home": str(self.home), + "layout": args.layout, "image": args.image, "commands": [], "cleanup": [], + "setup_ms": 0, "operations_ms": 0, "cleanup_ms": 0, + } + self.persist() + + def persist(self): + (self.root / "report.json").write_text(json.dumps(self.report, indent=2) + "\n") + + def run(self, case, *arguments, expected_failure=False, phase="operations", timeout=None): + limit = self.args.timeout if timeout is None else timeout + if phase == "operations" and self.deadline is not None: + remaining = self.deadline - time.monotonic() + if remaining <= 0: + raise RuntimeError(f"suite deadline exceeded before {case}") + limit = min(limit, remaining) + command = [str(self.binary), *map(str, arguments)] + started = time.monotonic() + stdout, stderr, code, timed_out = "", "", None, False + try: + result = subprocess.run(command, env=self.env, cwd=self.root, capture_output=True, + text=True, timeout=limit) + stdout, stderr, code = result.stdout, result.stderr, result.returncode + except subprocess.TimeoutExpired as error: + stdout, stderr = output_text(error.stdout), output_text(error.stderr) + timed_out = True + finally: + elapsed = round((time.monotonic() - started) * 1000, 2) + prefix = f"{len(self.report['commands']):03d}-{case}" + (self.logs / f"{prefix}.stdout.log").write_text(stdout) + (self.logs / f"{prefix}.stderr.log").write_text(stderr) + row = dict(case=case, phase=phase, argv=command[1:], ms=elapsed, exit=code, + expected_failure=expected_failure, timed_out=timed_out) + self.report["commands"].append(row) + self.persist() + print(f"{case}: {elapsed:.2f} ms (exit={code})", flush=True) + if timed_out: + raise RuntimeError(f"{case} timed out after {limit:.2f}s") + # A crash is not proof that a deliberately unsupported operation was refused cleanly. + if code is None or code < 0 or code > 255 or (code != 0) != expected_failure: + raise RuntimeError(f"{case} failed (exit={code}): {stderr[-2000:]}") + return stdout.strip(), stderr.strip() + + def guest(self, case, name, script): + return self.run(case, "exec", name, "--", "sh", "-ec", script)[0] + + def remember(self, name): + # Register before create/branch: a timed-out client may have started a detached VM. + self.names.append(name) + self.active.append(name) + + def create(self, name, *options): + self.remember(name) + self.run("create-" + name, "create", "--name", name, *options) + + def branch(self, source, child): + self.remember(child) + self.run("branch-" + child, "branch", source, "--name", child) + + def stop(self, name): + self.run("stop-" + name, "stop", name, "--timeout", "5") + self.active.remove(name) + + def check_markers(self, name, value, ram=True): + script = "cat /smoke-marker; " + ( + "cat /dev/shm/smoke-marker" if ram else "test ! -e /dev/shm/smoke-marker") + actual = self.guest("markers-" + name, name, script) + expected = value + "\n" + value if ram else value + if actual != expected: + raise RuntimeError(f"{name}: expected {expected!r}, got {actual!r}") + + def check_status(self, name, expected): + entries = json.loads(self.run("status-" + name, "list", "--format", "json")[0]) + actual = {entry["name"]: entry["status"] for entry in entries}.get(name) + if actual != expected: + raise RuntimeError(f"{name}: expected status {expected}, got {actual}") + + def capture(self, member, full=False): + options = ["--full"] if full else [] + output = self.run("capture-" + member, "snapshot", "create", member, + "--from-sandbox", "source", "--group", "work", *options)[0] + path = Path(output.splitlines()[-1]) + descriptor = json.loads((path / "snapshot.json").read_text()) + if (path.parent != self.home / "snapshots/work" + or path.name != descriptor["snapshot_id"] + or descriptor["state"]["kind"] != ("checkpoint" if full else "file")): + raise RuntimeError(f"unexpected captured artifact: {path}") + return descriptor + + def exercise(self): + layout = "flat:512M" if self.args.layout == "flat" else "512M" + self.create("source", self.args.image, "--root-disk", layout, + "--memory", "256M", "--cpus", "2") + self.guest("seed-source", "source", + "echo source > /smoke-marker; echo source > /dev/shm/smoke-marker; sync") + before = set((self.home / "snapshots").rglob("snapshot.json")) + self.branch("source", "child") + self.check_markers("child", "source") + self.guest("write-private-child", "child", + "echo child > /smoke-marker; echo child > /dev/shm/smoke-marker") + self.check_markers("source", "source") + self.branch("child", "grandchild") + self.check_markers("grandchild", "child") + if before != set((self.home / "snapshots").rglob("snapshot.json")): + raise RuntimeError("direct branching installed a durable snapshot") + _, error = self.run("duplicate-child-refused", "branch", "source", "--name", "child", + expected_failure=True) + if "already exists" not in error.lower(): + raise RuntimeError(f"duplicate branch failed for an unrelated reason: {error}") + self.check_markers("child", "child") + self.stop("child") + self.check_markers("grandchild", "child") + self.stop("grandchild") + + self.run("pause-source", "pause", "source") + self.check_status("source", "Paused") + self.branch("source", "paused-child") + self.check_markers("paused-child", "source") + self.stop("paused-child") + full = self.capture("full", full=True) + self.check_status("source", "Paused") + _, error = self.run("paused-exec-refused", "exec", "source", "--", "true", + expected_failure=True) + if "paused" not in error.lower(): + raise RuntimeError(f"paused exec failed for an unrelated reason: {error}") + self.run("resume-source", "resume", "source") + self.check_markers("source", "source") + + # A full snapshot preserves tmpfs; disk-only cold boot must not restore it. + disk = self.capture("disk") + self.check_status("source", "Running") + if disk["parent"] != full["snapshot_id"]: + raise RuntimeError("capture lineage was not retained") + self.create("disk-child", "--from-snapshot", "work:disk") + self.check_markers("disk-child", "source", ram=False) + self.stop("disk-child") + + full_archive, disk_archive = self.root / "full.msb", self.root / "disk.msb" + self.run("save-full", "snapshot", "save", "work:full", full_archive) + self.run("save-disk", "snapshot", "save", "work:disk", disk_archive) + self.run("load-batch-reversed", "snapshot", "load", disk_archive, full_archive, + "--group", "received") + head = json.loads(self.run("batch-head", "snapshot", "head", "received", + "--format", "json")[0]) + if head["head"] != disk["snapshot_id"]: + raise RuntimeError("batch head followed argument order instead of ancestry") + self.run("verify-imported-full", "snapshot", "verify", "received:full") + self.create("eager", "--from-snapshot", "received:full") + self.check_markers("eager", "source") + self.run("pause-eager", "pause", "eager") + self.stop("eager") + + before = set((self.home / "snapshots").rglob("snapshot.json")) + self.create("forked", "--from-snapshot", full_archive, "--forked") + if before != set((self.home / "snapshots").rglob("snapshot.json")): + raise RuntimeError("direct archive restore installed an intermediate snapshot") + # Unlink only archives created by this test, after the child is ready. + full_archive.unlink() + disk_archive.unlink() + self.check_markers("forked", "source") + self.guest("write-private-forked", "forked", + "echo forked > /smoke-marker; echo forked > /dev/shm/smoke-marker") + self.check_markers("source", "source") + self.stop("source") + self.check_markers("forked", "forked") + self.stop("forked") + + def runtime_pids(self): + # The CLI catalog can become terminal before the host process has exited. Read only + # this test's private run history, including VMs stopped earlier in the suite. + database = self.home / "db/msb.db" + if not database.exists(): + return [] + with sqlite3.connect(database.as_uri() + "?mode=ro", uri=True, timeout=2) as db: + pids = sorted({row[0] for row in db.execute('SELECT pid FROM "run" WHERE pid > 0')}) + remaining = [] + for pid in pids: + if os.name == "nt": + result = subprocess.run(["tasklist", "/FI", f"PID eq {pid}", "/FO", "CSV", "/NH"], + capture_output=True, text=True, timeout=2, check=True) + alive = any(len(row) > 1 and row[1] == str(pid) + for row in csv.reader(io.StringIO(result.stdout))) + else: + result = subprocess.run(["ps", "-p", str(pid), "-o", "stat="], + capture_output=True, text=True, timeout=2) + if result.returncode not in (0, 1): + raise RuntimeError(f"could not inspect runtime PID {pid}: {result.stderr}") + alive = bool(result.stdout.strip()) and not result.stdout.strip().startswith("Z") + if alive: + remaining.append(pid) + # Never signal bare recorded PIDs: PID reuse must not endanger another process. + return remaining + + def cleanup(self): + with defer_interrupts() as interrupted: + errors = self.cleanup_owned() + if interrupted: + errors.append("interrupted during cleanup; completed bounded stop attempts") + return errors + + def cleanup_owned(self): + errors = [] + for name in reversed(self.active): + try: + self.run("cleanup-" + name, "stop", name, "--timeout", "5", + phase="cleanup", timeout=10) + self.report["cleanup"].append(dict(name=name, stopped=True)) + except Exception as error: + # A failed create may not have a catalog row. Preserve those diagnostics, then + # check both the catalog and process history after a bounded force-stop attempt. + self.report["cleanup"].append(dict(name=name, error=str(error))) + try: + self.run("force-stop-" + name, "stop", name, "--force", + phase="cleanup", timeout=10) + except Exception as forced: + errors.append(str(forced)) + try: + entries = json.loads(self.run("cleanup-inventory", "list", "--format", "json", + phase="cleanup", timeout=10)[0]) + resident = [entry for entry in entries if entry["status"] not in ("Stopped", "Crashed")] + self.report["remaining_sandboxes"] = resident + if resident: + errors.append(f"test sandboxes remain resident: {resident}") + except Exception as error: + errors.append(f"could not verify VM cleanup: {error}") + try: + deadline = time.monotonic() + 5 + while True: + remaining = self.runtime_pids() + if not remaining or time.monotonic() >= deadline: + break + time.sleep(0.1) + self.report["remaining_runtime_pids"] = remaining + if remaining: + errors.append(f"recorded runtime PIDs are still alive: {remaining}") + except Exception as error: + errors.append(f"could not verify runtime process exit: {error}") + return errors + + def execute(self): + started = time.monotonic() + failure = None + operations_started = None + try: + # Build/download/materialization time is not snapshot latency. Preparing both layouts + # also avoids deferred image work becoming part of the first measured create. + self.run("prepare-image", "pull", self.args.image, "--materialize", "all", + phase="setup", timeout=180) + self.report["setup_ms"] = round((time.monotonic() - started) * 1000, 2) + operations_started = time.monotonic() + self.deadline = operations_started + self.args.suite_timeout + self.exercise() + except (Exception, KeyboardInterrupt) as error: + failure = str(error) or "interrupted" + finally: + if operations_started is not None: + self.report["operations_ms"] = round( + (time.monotonic() - operations_started) * 1000, 2) + else: + self.report["setup_ms"] = round((time.monotonic() - started) * 1000, 2) + cleanup_started = time.monotonic() + # Cleanup is independent of the expired operation deadline and tries every owned VM. + cleanup_errors = self.cleanup() + if failure or cleanup_errors: + for name in self.names: + try: + self.run("diagnostics-" + name, "logs", "--source", "system", name, + phase="diagnostics", timeout=5) + except Exception: + pass + self.report["home_removed"] = False + if not failure and not cleanup_errors and self.home.exists(): + # This home was created beneath our exclusive output directory. Once both + # catalog and process checks pass, retain text evidence, not large RAM/disks. + try: + shutil.rmtree(self.home) + self.report["home_removed"] = True + except OSError as error: + cleanup_errors.append(f"could not remove owned test home: {error}") + self.report.update( + status="failed" if failure or cleanup_errors else "passed", error=failure, + cleanup_errors=cleanup_errors, + cleanup_ms=round((time.monotonic() - cleanup_started) * 1000, 2), + total_ms=round((time.monotonic() - started) * 1000, 2), + ) + self.persist() + print(json.dumps({key: self.report[key] for key in + ("status", "setup_ms", "operations_ms", "cleanup_ms", "total_ms")}), + flush=True) + print(f"Report: {self.root / 'report.json'}", flush=True) + if self.report["error"] or self.report["cleanup_errors"]: + print(self.report["error"] or "; ".join(self.report["cleanup_errors"]), file=sys.stderr) + return 0 if self.report["status"] == "passed" else 1 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--binary", type=Path, + default=REPOSITORY / "build" / ("msb.exe" if os.name == "nt" else "msb")) + parser.add_argument("--output", type=Path, help="New directory for an isolated home and logs") + parser.add_argument("--layout", choices=("managed", "flat"), default="managed") + parser.add_argument("--image", default="mirror.gcr.io/library/alpine:3.20") + parser.add_argument("--timeout", type=positive_seconds, default=30, + help="Per-operation timeout in seconds (default: 30)") + parser.add_argument("--suite-timeout", type=positive_seconds, default=120, + help="Operation-suite deadline, excluding image setup/cleanup (default: 120)") + args = parser.parse_args() + + def interrupted(_signum, _frame): + raise KeyboardInterrupt("interrupted; cleaning up owned test VMs") + + signal.signal(signal.SIGTERM, interrupted) + try: + return Smoke(args).execute() + except (OSError, ValueError) as error: + parser.exit(1, f"snapshot smoke: {error}\n") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/smoke/cli/test_snapshot_branch.py b/scripts/smoke/cli/test_snapshot_branch.py new file mode 100644 index 000000000..fde5afa6d --- /dev/null +++ b/scripts/smoke/cli/test_snapshot_branch.py @@ -0,0 +1,446 @@ +#!/usr/bin/env python3 +"""VM-free tests for snapshot-branch smoke isolation, reporting, and cleanup.""" + +from __future__ import annotations + +import argparse +import contextlib +import importlib.util +import io +import json +import os +from pathlib import Path +import signal +import sqlite3 +import subprocess +import sys +import tempfile +import time +from types import SimpleNamespace +import unittest +from unittest import mock + + +SPEC = importlib.util.spec_from_file_location( + "snapshot_branch_smoke", Path(__file__).with_name("snapshot-branch.py") +) +assert SPEC is not None and SPEC.loader is not None +HARNESS = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(HARNESS) + + +class SnapshotBranchSmokeTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix="snapshot-branch-unit-") + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + contexts = contextlib.ExitStack() + self.addCleanup(contexts.close) + contexts.enter_context(contextlib.redirect_stdout(io.StringIO())) + contexts.enter_context(contextlib.redirect_stderr(io.StringIO())) + # Patch the process boundary globally for each test: no accidental command can boot a VM. + self.process = contexts.enter_context(mock.patch.object(HARNESS.subprocess, "run")) + self.process.side_effect = self.successful_process + + @staticmethod + def successful_process(command, **_kwargs): + stdout = "[]" if command[1] == "list" else "" + return subprocess.CompletedProcess(command, 0, stdout, "") + + def smoke(self, output="run"): + return HARNESS.Smoke(argparse.Namespace( + binary=Path(sys.executable), output=self.root / output, + layout="managed", image="test-image", timeout=30, suite_timeout=120, + )) + + @staticmethod + def report(smoke): + return json.loads((smoke.root / "report.json").read_text()) + + def commands(self): + return [call.args[0][1:] for call in self.process.call_args_list] + + @staticmethod + def run_history(smoke, rows): + database = smoke.home / "db/msb.db" + database.parent.mkdir(parents=True) + with sqlite3.connect(database) as db: + db.execute('CREATE TABLE "run" (pid INTEGER, status TEXT)') + db.executemany('INSERT INTO "run" (pid, status) VALUES (?, ?)', rows) + return database + + def test_defer_interrupts_records_signals_and_restores_handlers_on_exception(self): + original = {signal.SIGINT: mock.Mock(), signal.SIGTERM: mock.Mock()} + handlers = dict(original) + with mock.patch.object(HARNESS.signal, "getsignal", side_effect=handlers.__getitem__), \ + mock.patch.object(HARNESS.signal, "signal", side_effect=handlers.__setitem__): + with self.assertRaisesRegex(RuntimeError, "cleanup failed"): + with HARNESS.defer_interrupts() as received: + handlers[signal.SIGINT](signal.SIGINT, None) + handlers[signal.SIGTERM](signal.SIGTERM, None) + self.assertEqual(received, [signal.SIGINT, signal.SIGTERM]) + raise RuntimeError("cleanup failed") + self.assertEqual(handlers, original) + for handler in original.values(): + handler.assert_not_called() + + def test_isolates_home_backend_config_and_command_working_directory(self): + ambient = { + "MSB_HOME": str(self.root / "caller-home"), + "MSB_CONFIG_PATH": str(self.root / "caller-config.json"), + "MSB_BACKEND": "cloud", "MSB_PROFILE": "production", + "MSB_AGENTD_PATH": "/matching/agentd", "NO_COLOR": "0", + } + with mock.patch.dict(os.environ, ambient): + smoke = self.smoke() + smoke.run("probe", "list", "--format", "json") + self.assertEqual(os.environ["MSB_HOME"], ambient["MSB_HOME"]) + passed = self.process.call_args.kwargs + self.assertEqual(passed["cwd"], smoke.root) + self.assertEqual(passed["env"]["MSB_HOME"], str(smoke.root / "home")) + self.assertEqual(passed["env"]["MSB_BACKEND"], "local") + self.assertEqual(passed["env"]["NO_COLOR"], "1") + self.assertEqual(passed["env"]["MSB_AGENTD_PATH"], "/matching/agentd") + self.assertNotIn("MSB_CONFIG_PATH", passed["env"]) + self.assertNotIn("MSB_PROFILE", passed["env"]) + self.assertEqual(self.report(smoke)["home"], str(smoke.home)) + self.assertFalse((self.root / "caller-home").exists()) + + def test_refuses_existing_output_without_overwriting_it(self): + output = self.root / "existing" + output.mkdir() + sentinel = output / "report.json" + sentinel.write_text("keep existing report") + with self.assertRaises(FileExistsError): + self.smoke("existing") + self.assertEqual(sentinel.read_text(), "keep existing report") + self.process.assert_not_called() + + def test_records_success_and_clean_expected_failures(self): + smoke = self.smoke() + for code, expected_failure in [(0, False), (1, True), (2, True), (255, True)]: + with self.subTest(code=code, expected_failure=expected_failure): + self.process.side_effect = None + self.process.return_value = subprocess.CompletedProcess( + [], code, " output\n", " diagnostic\n" + ) + self.assertEqual( + smoke.run(f"exit-{code}", "probe", expected_failure=expected_failure), + ("output", "diagnostic"), + ) + row = self.report(smoke)["commands"][-1] + self.assertEqual(row["exit"], code) + self.assertEqual(row["expected_failure"], expected_failure) + self.assertFalse(row["timed_out"]) + + def test_unexpected_success_failure_and_crashes_are_not_accepted(self): + smoke = self.smoke() + # Windows exception statuses are positive; they must not pass as ordinary refusals. + for code, expected_failure in [ + (0, True), (1, False), (-9, False), (-9, True), + (256, True), (0xC0000005, False), (0xC0000005, True), + ]: + with self.subTest(code=code, expected_failure=expected_failure): + self.process.side_effect = None + self.process.return_value = subprocess.CompletedProcess([], code, "", "failed") + with self.assertRaisesRegex(RuntimeError, "failed"): + smoke.run("rejected", "probe", expected_failure=expected_failure) + self.assertEqual(self.report(smoke)["commands"][-1]["exit"], code) + + def test_timeout_preserves_partial_bytes_in_logs_and_report(self): + smoke = self.smoke() + self.process.side_effect = subprocess.TimeoutExpired( + [sys.executable, "probe"], 30, output=b"partial\xff\n", stderr=b"waiting\xfe" + ) + with self.assertRaisesRegex(RuntimeError, "timed out"): + smoke.run("slow", "probe", expected_failure=True) + row = self.report(smoke)["commands"][-1] + self.assertTrue(row["timed_out"]) + self.assertIsNone(row["exit"]) + self.assertEqual(next(smoke.logs.glob("*slow.stdout.log")).read_text(), "partial\ufffd\n") + self.assertEqual(next(smoke.logs.glob("*slow.stderr.log")).read_text(), "waiting\ufffd") + + def test_expired_suite_deadline_still_allows_bounded_cleanup(self): + smoke = self.smoke() + smoke.remember("owned") + smoke.deadline = time.monotonic() - 1 + with self.assertRaisesRegex(RuntimeError, "suite deadline exceeded"): + smoke.run("too-late", "probe") + self.process.assert_not_called() + self.assertEqual(smoke.cleanup(), []) + self.assertIn(["stop", "owned", "--timeout", "5"], self.commands()) + self.assertEqual(self.commands()[-1], ["list", "--format", "json"]) + self.assertTrue(all(call.kwargs["timeout"] > 0 for call in self.process.call_args_list)) + self.assertTrue(all(row["phase"] == "cleanup" for row in smoke.report["commands"])) + + def test_failed_create_is_still_registered_for_cleanup(self): + smoke = self.smoke() + self.process.side_effect = subprocess.TimeoutExpired([sys.executable, "create"], 30) + with self.assertRaisesRegex(RuntimeError, "timed out"): + smoke.create("possibly-started", "test-image") + self.process.side_effect = self.successful_process + self.assertEqual(smoke.cleanup(), []) + self.assertIn(["stop", "possibly-started", "--timeout", "5"], self.commands()) + + def test_cleanup_attempts_every_vm_and_force_fallback_despite_stop_failures(self): + smoke = self.smoke() + for name in ("first", "second", "third"): + smoke.remember(name) + + def process(command, **kwargs): + if command[1] == "stop": + name, forced = command[2], "--force" in command + # One fallback succeeds; another fails. Neither may prevent the next VM's stop. + code = int(name == "third" or (name == "second" and not forced)) + return subprocess.CompletedProcess(command, code, "", "stop refused") + return self.successful_process(command, **kwargs) + + self.process.side_effect = process + errors = smoke.cleanup() + for name in ("first", "second", "third"): + self.assertIn(["stop", name, "--timeout", "5"], self.commands()) + for name in ("second", "third"): + self.assertIn(["stop", name, "--force"], self.commands()) + self.assertEqual(self.commands()[-1], ["list", "--format", "json"]) + self.assertEqual(len(smoke.report["cleanup"]), 3) + self.assertTrue(any("third" in error for error in errors)) + + def test_cleanup_rejects_resident_inventory_but_accepts_terminal_entries(self): + smoke = self.smoke() + resident = [{"name": "running", "status": "Running"}, + {"name": "paused", "status": "Paused"}] + terminal = [{"name": "stopped", "status": "Stopped"}, + {"name": "crashed", "status": "Crashed"}] + self.process.side_effect = None + self.process.return_value = subprocess.CompletedProcess( + [], 0, json.dumps(resident + terminal), "" + ) + self.assertTrue(any("remain resident" in error for error in smoke.cleanup())) + self.assertEqual(smoke.report["remaining_sandboxes"], resident) + self.process.return_value = subprocess.CompletedProcess([], 0, json.dumps(terminal), "") + self.assertEqual(smoke.cleanup(), []) + self.assertEqual(smoke.report["remaining_sandboxes"], []) + + def test_cleanup_inventory_failure_is_reported(self): + smoke = self.smoke() + self.process.side_effect = None + self.process.return_value = subprocess.CompletedProcess([], 0, "not JSON", "") + self.assertTrue(any("could not verify VM cleanup" in error for error in smoke.cleanup())) + + def test_runtime_pids_checks_stopped_history_and_ignores_zombies_and_absent_processes(self): + smoke = self.smoke() + database = self.run_history(smoke, [ + (41001, "Stopped"), (41002, "Stopped"), (41003, "Stopped"), + (41003, "Crashed"), (0, "Running"), (-1, "Running"), (None, "Stopped"), + ]) + before = database.read_bytes() + states = {41001: (0, "Z+\n"), 41002: (1, ""), 41003: (0, "S+\n")} + + def process(command, **_kwargs): + self.assertEqual(command[0], "ps") + code, state = states[int(command[2])] + return subprocess.CompletedProcess(command, code, state, "") + + self.process.side_effect = process + platform = SimpleNamespace(name="posix", kill=mock.Mock()) + with mock.patch.object(HARNESS, "os", platform), \ + mock.patch.object(HARNESS.sqlite3, "connect", wraps=sqlite3.connect) as connect: + self.assertEqual(smoke.runtime_pids(), [41003]) + self.assertEqual(connect.call_args.args[0], database.as_uri() + "?mode=ro") + self.assertTrue(connect.call_args.kwargs["uri"]) + self.assertEqual([int(call.args[0][2]) for call in self.process.call_args_list], + [41001, 41002, 41003]) + self.assertEqual(database.read_bytes(), before) + platform.kill.assert_not_called() + + def test_runtime_pids_reads_windows_tasklist_csv_and_requires_exact_pid(self): + smoke = self.smoke() + self.run_history(smoke, [(42001, "Stopped"), (42002, "Stopped"), (42003, "Stopped")]) + outputs = { + 42001: '"msb.exe","42001","Console","1","8,192 K"\r\n', + 42002: "INFO: No tasks are running which match the specified criteria.\r\n", + 42003: '"msb.exe","420030","Console","1","4,096 K"\r\n', + } + + def process(command, **kwargs): + self.assertEqual(command[0], "tasklist") + self.assertTrue(kwargs["check"]) + pid = int(command[2].split()[-1]) + return subprocess.CompletedProcess(command, 0, outputs[pid], "") + + self.process.side_effect = process + platform = SimpleNamespace(name="nt", kill=mock.Mock()) + with mock.patch.object(HARNESS, "os", platform): + self.assertEqual(smoke.runtime_pids(), [42001]) + self.assertEqual(self.process.call_count, 3) + platform.kill.assert_not_called() + + def test_runtime_pids_does_not_create_a_missing_database(self): + smoke = self.smoke() + self.assertEqual(smoke.runtime_pids(), []) + self.assertFalse((smoke.home / "db/msb.db").exists()) + self.process.assert_not_called() + + def test_runtime_pids_reports_process_inspection_failure(self): + smoke = self.smoke() + self.run_history(smoke, [(43001, "Stopped")]) + self.process.side_effect = None + self.process.return_value = subprocess.CompletedProcess([], 2, "", "ps unavailable") + with mock.patch.object(HARNESS, "os", SimpleNamespace(name="posix")): + with self.assertRaisesRegex(RuntimeError, "could not inspect runtime PID 43001"): + smoke.runtime_pids() + + def test_cleanup_detects_live_runtime_pid_after_catalog_is_stopped_without_real_wait(self): + smoke = self.smoke() + self.run_history(smoke, [(44001, "Stopped")]) + + def process(command, **_kwargs): + if command[0] == "ps": + return subprocess.CompletedProcess(command, 0, "S\n", "") + return subprocess.CompletedProcess( + command, 0, json.dumps([{"name": "owned", "status": "Stopped"}]), "" + ) + + self.process.side_effect = process + # Advance a synthetic clock on each read, so the real grace period costs no wall time. + with mock.patch.object(HARNESS, "os", SimpleNamespace(name="posix")), \ + mock.patch.object(HARNESS.time, "monotonic", side_effect=iter(range(0, 100, 2))), \ + mock.patch.object(HARNESS.time, "sleep") as sleep: + errors = smoke.cleanup() + sleep.assert_called() + self.assertEqual(smoke.report["remaining_sandboxes"], []) + self.assertEqual(smoke.report["remaining_runtime_pids"], [44001]) + self.assertTrue(any("runtime PIDs are still alive" in error for error in errors)) + + def test_cleanup_succeeds_when_runtime_exits_during_grace_period(self): + smoke = self.smoke() + self.run_history(smoke, [(45001, "Stopped")]) + states = iter([(0, "S\n"), (1, "")]) + + def process(command, **kwargs): + if command[0] == "ps": + code, state = next(states) + return subprocess.CompletedProcess(command, code, state, "") + return self.successful_process(command, **kwargs) + + self.process.side_effect = process + with mock.patch.object(HARNESS, "os", SimpleNamespace(name="posix")), \ + mock.patch.object(HARNESS.time, "sleep") as sleep: + self.assertEqual(smoke.cleanup(), []) + sleep.assert_called_once() + self.assertEqual(smoke.report["remaining_runtime_pids"], []) + + def test_cleanup_defers_interrupt_and_still_attempts_all_owned_vms(self): + smoke = self.smoke() + smoke.remember("first") + smoke.remember("second") + original = {signal.SIGINT: mock.Mock(), signal.SIGTERM: mock.Mock()} + handlers = dict(original) + + def process(command, **kwargs): + if command[1] == "stop" and command[2] == "second": + handlers[signal.SIGTERM](signal.SIGTERM, None) + return self.successful_process(command, **kwargs) + + self.process.side_effect = process + with mock.patch.object(HARNESS.signal, "getsignal", side_effect=handlers.__getitem__), \ + mock.patch.object(HARNESS.signal, "signal", side_effect=handlers.__setitem__): + errors = smoke.cleanup() + self.assertEqual(handlers, original) + self.assertTrue(any("interrupted during cleanup" in error for error in errors)) + for name in ("first", "second"): + self.assertIn(["stop", name, "--timeout", "5"], self.commands()) + self.assertIn(["list", "--format", "json"], self.commands()) + + def test_execute_cleans_and_persists_failure_even_when_interrupted(self): + for index, error in enumerate((RuntimeError("operation failed"), KeyboardInterrupt())): + with self.subTest(error=type(error).__name__): + smoke = self.smoke(f"failure-{index}") + self.process.reset_mock() + + def exercise(): + smoke.remember("possibly-started") + raise error + + with mock.patch.object(smoke, "exercise", side_effect=exercise): + self.assertEqual(smoke.execute(), 1) + report = self.report(smoke) + self.assertEqual(report["status"], "failed") + self.assertEqual(report["error"], str(error) or "interrupted") + self.assertIn(["stop", "possibly-started", "--timeout", "5"], self.commands()) + self.assertIn(["list", "--format", "json"], self.commands()) + self.assertTrue(any(command[0] == "logs" for command in self.commands())) + self.assertEqual(report["cleanup_errors"], []) + + def test_execute_setup_failure_still_verifies_cleanup(self): + smoke = self.smoke() + + def process(command, **kwargs): + if command[1] == "pull": + return subprocess.CompletedProcess(command, 1, "", "image preparation failed") + return self.successful_process(command, **kwargs) + + self.process.side_effect = process + with mock.patch.object(smoke, "exercise") as exercise: + self.assertEqual(smoke.execute(), 1) + exercise.assert_not_called() + report = self.report(smoke) + self.assertEqual(report["status"], "failed") + self.assertIn("prepare-image", report["error"]) + self.assertEqual(report["operations_ms"], 0) + self.assertIn(["list", "--format", "json"], self.commands()) + + def test_execute_fails_if_cleanup_leaves_a_resident_vm(self): + smoke = self.smoke() + + def process(command, **kwargs): + if command[1] == "list": + return subprocess.CompletedProcess( + command, 0, json.dumps([{"name": "owned", "status": "Running"}]), "" + ) + return self.successful_process(command, **kwargs) + + self.process.side_effect = process + with mock.patch.object(smoke, "exercise", side_effect=lambda: smoke.remember("owned")): + self.assertEqual(smoke.execute(), 1) + report = self.report(smoke) + self.assertEqual(report["status"], "failed") + self.assertIsNone(report["error"]) + self.assertTrue(report["cleanup_errors"]) + + def test_execute_reports_success_only_after_clean_inventory(self): + smoke = self.smoke() + with mock.patch.object(smoke, "exercise"): + self.assertEqual(smoke.execute(), 0) + report = self.report(smoke) + self.assertEqual(report["status"], "passed") + self.assertIsNone(report["error"]) + self.assertEqual(report["cleanup_errors"], []) + self.assertEqual(self.commands()[-1], ["list", "--format", "json"]) + + def test_success_removes_only_owned_home_and_keeps_text_evidence(self): + smoke = self.smoke() + smoke.home.mkdir() + (smoke.home / "test-ram").write_bytes(b"fixture") + sibling = self.root / "unrelated" + sibling.write_text("keep") + with mock.patch.object(smoke, "exercise"): + self.assertEqual(smoke.execute(), 0) + self.assertFalse(smoke.home.exists()) + self.assertTrue(self.report(smoke)["home_removed"]) + self.assertTrue(any(smoke.logs.iterdir())) + self.assertEqual(sibling.read_text(), "keep") + + def test_failure_retains_owned_home_for_investigation(self): + smoke = self.smoke() + smoke.home.mkdir() + fixture = smoke.home / "test-ram" + fixture.write_bytes(b"fixture") + with mock.patch.object(smoke, "exercise", side_effect=RuntimeError("injected failure")): + self.assertEqual(smoke.execute(), 1) + self.assertEqual(fixture.read_bytes(), b"fixture") + self.assertFalse(self.report(smoke)["home_removed"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/sdk/go/cow_lifecycle_test.go b/sdk/go/cow_lifecycle_test.go index f54778330..4e56f2526 100644 --- a/sdk/go/cow_lifecycle_test.go +++ b/sdk/go/cow_lifecycle_test.go @@ -6,6 +6,7 @@ import ( "context" "fmt" "os" + "path/filepath" "strings" "testing" "time" @@ -58,13 +59,18 @@ func TestCowResidentCapture(t *testing.T) { if strings.TrimSpace(branchResult.Stdout()) != "source" { t.Fatal("branch lost captured RAM") } - if _, err := Snapshot.Create(ctx, SnapshotCreateOptions{Name: name + "-full", FromSandbox: name, Full: true}); err != nil { + snapshot, err := Snapshot.Create(ctx, SnapshotCreateOptions{Name: name + "-full", FromSandbox: name, Full: true}) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(snapshot.Path(), "snapshot.json")); err != nil { t.Fatal(err) } if err := paused.Resume(ctx); err != nil { t.Fatal(err) } - child, err := CreateSandbox(ctx, name+"-child", WithFromSnapshot(name+"-full"), WithForked()) + // The returned artifact path selects the exact member in its snapshot group. + child, err := CreateSandbox(ctx, name+"-child", WithFromSnapshot(snapshot.Path()), WithForked()) if err != nil { t.Fatal(err) } diff --git a/sdk/python/tests/test_cow_lifecycle.py b/sdk/python/tests/test_cow_lifecycle.py index 82b66e6ec..77b6d1313 100644 --- a/sdk/python/tests/test_cow_lifecycle.py +++ b/sdk/python/tests/test_cow_lifecycle.py @@ -1,6 +1,7 @@ """Opt-in live CoW lifecycle check using a matching runtime/kernel bundle.""" import os +from pathlib import Path import pytest @@ -24,10 +25,12 @@ async def test_cow_resident_capture_and_child_isolation(): branched = await paused.branch(f"{name}-paused-branch") branches.append(branched) assert (await branched.exec("cat", ["/dev/shm/sdk-marker"])).stdout_text.strip() == "source" - await Snapshot.create(f"{name}-full", from_sandbox=name, full=True) + snapshot = await Snapshot.create(f"{name}-full", from_sandbox=name, full=True) + assert (Path(snapshot.path) / "snapshot.json").is_file() await paused.resume() + # The returned artifact path selects the exact member in its snapshot group. child = await Sandbox.create( - f"{name}-child", from_snapshot=f"{name}-full", forked=True + f"{name}-child", from_snapshot=snapshot.path, forked=True ) result = await child.exec("cat", ["/dev/shm/sdk-marker"]) assert result.stdout_text.strip() == "source" From 71b399b5d8df93c95d73c32c5543bebc3e0fac30 Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Thu, 10 Sep 2026 13:03:03 +0100 Subject: [PATCH 22/29] build(runtime): use msb_krun 0.1.35 Pin msb_krun and msb_krun_utils to the published 0.1.35 release and refresh only their matching lockfile entries. Pick up runtime resolution of optional macOS GIC APIs without changing the snapshot format, guest protocol, or stack ancestry. --- Cargo.lock | 44 ++++++++++++++++++++++---------------------- Cargo.toml | 4 ++-- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 18d8a1f55..ba1b70887 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4379,9 +4379,9 @@ dependencies = [ [[package]] name = "msb_krun" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6a401c1ec192a3dfbc382901d8c0275f42c36097991a6426f5502a30f017ff0" +checksum = "1d82b78e6694529738582317c2db84b775f6b2f6e76e8429336d40a9ada9dc7e" dependencies = [ "crossbeam-channel", "kvm-bindings", @@ -4399,9 +4399,9 @@ dependencies = [ [[package]] name = "msb_krun_arch" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7362283e6bc924947c2a2415d7cd9025616fa45ea61bc4ec3ea00bddf8df36df" +checksum = "234c8c322b99007c2ad7e60a705ae4b89c16c0b69906259425e32671dca1a32e" dependencies = [ "kvm-bindings", "kvm-ioctls", @@ -4414,15 +4414,15 @@ dependencies = [ [[package]] name = "msb_krun_arch_gen" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4101f8a221a95e8e747684c65d001b6d49899f88e81b41122f9de0fba3eefc3" +checksum = "a8d6558b6564ad5fceadd7291b895fd4ba8d317c8d51a5a89a27117c3bd19837" [[package]] name = "msb_krun_cpuid" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dacf8d69bc3a568970bfc98918f2f60786a5438f8b5701b16dc0ac9f40af6f58" +checksum = "5fae1941e85a4ba925807c45956c1014ccb39e7364dd4e416ab189bf06d3f950" dependencies = [ "kvm-bindings", "kvm-ioctls", @@ -4431,9 +4431,9 @@ dependencies = [ [[package]] name = "msb_krun_devices" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe443bd0f03c68eaf33d63da12e8b3084ec080a6c105d8768d711cab062f2c60" +checksum = "69b2fdcb2c1573272ca936d009796b77072e38e2812a574a1334c9c21897d3b8" dependencies = [ "bincode", "bitflags 1.3.2", @@ -4463,9 +4463,9 @@ dependencies = [ [[package]] name = "msb_krun_hvf" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f5833d1d7e6404ae1e3cc3be50a5ad3eceb1c61c2bdc3bb296f223ac0615a2a" +checksum = "454e8f0088ae171a55dfbcfe69542d1d0c12f7f3b43d3281d9d0124016a3d8ec" dependencies = [ "crossbeam-channel", "libloading 0.8.9", @@ -4476,9 +4476,9 @@ dependencies = [ [[package]] name = "msb_krun_kernel" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "189508d48404ed0768d882f7e93e8ca1a093a77621462c4aa96f0637a02792be" +checksum = "5ecbe5ba13631934f84b57b231c1e7bef371e1d42d2ec940242f9629d28fb196" dependencies = [ "msb-vm-memory", "msb_krun_utils", @@ -4486,9 +4486,9 @@ dependencies = [ [[package]] name = "msb_krun_polly" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e849910d2531272df93806a1e2f5830145d76e64c2d0161aa75a69dcc24ead8" +checksum = "50d0d1bf51ad1fd68cf61d41f3c17d51abb8287e72e1d9a8da0419579a81ba92" dependencies = [ "libc", "msb_krun_utils", @@ -4496,18 +4496,18 @@ dependencies = [ [[package]] name = "msb_krun_smbios" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cf963ea3fcf2e564dd8982ae663339629eaf19a0ec1a4bd161d42c43e23a4ad" +checksum = "f7e5b6e6bd33e9097e882d359abeaa325553edfcf44cf51baa7a8cc827f6dcc7" dependencies = [ "msb-vm-memory", ] [[package]] name = "msb_krun_utils" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "078445461df117b827681679390565e6f1ecf06a8d1f309d641dc6a1e390531c" +checksum = "d9e68f68e5d3c953e5d71e89f6c2cf4a2a1c69fcafa6b856521eee4bb6485646" dependencies = [ "bitflags 1.3.2", "crossbeam-channel", @@ -4521,9 +4521,9 @@ dependencies = [ [[package]] name = "msb_krun_vmm" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "223013c35221736169599da5a71fb5a801346c7a5e4439f21461bf5e503f16bb" +checksum = "099b488e7e337eee46a77a67ba3a99a280ceada4b621639b81a6ab8f7b9c81ba" dependencies = [ "bincode", "bzip2", diff --git a/Cargo.toml b/Cargo.toml index 7d4bd3563..9c02b323d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -91,8 +91,8 @@ microsandbox-protocol = { version = "=0.6.18", path = "crates/protocol" } microsandbox-runtime = { version = "=0.6.18", path = "crates/runtime", default-features = false } microsandbox-utils = { version = "=0.6.18", path = "crates/utils" } microsandbox-vsock = { version = "=0.6.18", path = "crates/vsock" } -msb_krun = "=0.1.34" -msb_krun_utils = "=0.1.34" +msb_krun = "=0.1.35" +msb_krun_utils = "=0.1.35" test-macros = { path = "crates/testing/macros" } test-utils = { path = "crates/testing/utils" } From a827f43075314f146461b02d7024eefa1340810f Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Thu, 10 Sep 2026 14:01:34 +0100 Subject: [PATCH 23/29] fix(runtime): restore Windows stdio inheritance on failure Keep stdio guard construction in the I/O error domain so the runtime spawn helper compiles on Windows. Own successful flag changes immediately and roll them back if a later handle update fails. Add Windows regression tests for partial failure, duplicate handles and unchanged non-inheritable handles, and run them in both Windows CI jobs. Account for Unix-only control-link parameters on Windows so the strict Clippy gate remains usable. --- .github/workflows/check.yml | 8 ++ crates/runtime/lib/runner/vm.rs | 4 + sdk/rust/lib/runtime/spawn.rs | 135 +++++++++++++++++++++++++++++--- 3 files changed, 137 insertions(+), 10 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index b4a11d1a9..a6b2546bf 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -569,6 +569,14 @@ jobs: Set-MsvcEnvironment -Architecture ${{ matrix.vs_arch }} -HostArchitecture ${{ matrix.vs_host_arch }} cargo +stable test --no-default-features --features local,net -p microsandbox --lib --target ${{ matrix.rust_target }} sandbox::patch::tests::bind_patch_ + - name: Test Windows stdio inheritance cleanup + shell: pwsh + run: | + $ErrorActionPreference = "Stop" + . "$env:GITHUB_WORKSPACE\vendor\libkrunfw\scripts\msvc-env.ps1" + Set-MsvcEnvironment -Architecture ${{ matrix.vs_arch }} -HostArchitecture ${{ matrix.vs_host_arch }} + cargo +stable test --no-default-features --features local,net -p microsandbox --lib --target ${{ matrix.rust_target }} runtime::spawn::tests::windows_stdio_guard_ + - name: Test Windows DNS resolver shell: pwsh run: | diff --git a/crates/runtime/lib/runner/vm.rs b/crates/runtime/lib/runner/vm.rs index 57da91187..731a449c5 100644 --- a/crates/runtime/lib/runner/vm.rs +++ b/crates/runtime/lib/runner/vm.rs @@ -2288,6 +2288,10 @@ fn publish_control_endpoint( run_dir: &Path, sandbox_name: &str, ) -> RuntimeResult<()> { + // Windows uses named pipes and has no legacy Unix socket link to publish. + #[cfg(not(unix))] + let _ = (run_dir, sandbox_name); + match super::control::spawn_control_listener(control_sock_path.clone(), context) { Ok(()) => { #[cfg(unix)] diff --git a/sdk/rust/lib/runtime/spawn.rs b/sdk/rust/lib/runtime/spawn.rs index 8ed14212d..aac7c6a85 100644 --- a/sdk/rust/lib/runtime/spawn.rs +++ b/sdk/rust/lib/runtime/spawn.rs @@ -207,15 +207,32 @@ impl EnsuredNamedVolumes { #[cfg(windows)] impl StdioInheritGuard { - fn new() -> MicrosandboxResult { - let mut states = Vec::new(); + fn new() -> std::io::Result { + let handles = [STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, STD_ERROR_HANDLE] + .map(|std_handle| unsafe { GetStdHandle(std_handle) }); + Self::from_handles(handles, |handle| { + if unsafe { SetHandleInformation(handle, HANDLE_FLAG_INHERIT, 0) } == 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } + }) + } + + fn from_handles( + handles: impl IntoIterator, + mut clear_inherit: impl FnMut(HANDLE) -> std::io::Result<()>, + ) -> std::io::Result { + // Own each successful change immediately so an error on a later handle + // restores the earlier handles through Drop before returning. + let mut guard = Self { states: Vec::new() }; - for std_handle in [STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, STD_ERROR_HANDLE] { - let handle = unsafe { GetStdHandle(std_handle) }; + for handle in handles { if handle.is_null() || handle == INVALID_HANDLE_VALUE { continue; } - if states + if guard + .states .iter() .any(|state: &HandleInheritState| state.handle == handle) { @@ -233,13 +250,11 @@ impl StdioInheritGuard { // A redirected `msb create` can receive inheritable stdout/stderr // pipe handles from its own parent. Detached sandbox children must // not keep those pipes alive after the launcher exits. - if unsafe { SetHandleInformation(handle, HANDLE_FLAG_INHERIT, 0) } == 0 { - return Err(std::io::Error::last_os_error().into()); - } - states.push(HandleInheritState { handle, flags }); + clear_inherit(handle)?; + guard.states.push(HandleInheritState { handle, flags }); } - Ok(Self { states }) + Ok(guard) } } @@ -3009,6 +3024,106 @@ mod tests { assert!(child.wait().await.unwrap().success()); } + #[cfg(windows)] + fn windows_handle_flags(handle: super::HANDLE) -> u32 { + let mut flags = 0; + assert_ne!( + unsafe { super::GetHandleInformation(handle, &mut flags) }, + 0 + ); + flags + } + + #[cfg(windows)] + fn windows_set_handle_inherit(handle: super::HANDLE, inherit: bool) { + assert_ne!( + unsafe { + super::SetHandleInformation( + handle, + super::HANDLE_FLAG_INHERIT, + if inherit { + super::HANDLE_FLAG_INHERIT + } else { + 0 + }, + ) + }, + 0 + ); + } + + #[cfg(windows)] + #[test] + fn windows_stdio_guard_deduplicates_and_restores_handles() { + use std::os::windows::io::AsRawHandle; + + // Use private file handles; never change the test runner's process-wide stdio. + let file = tempfile::tempfile().unwrap(); + let untouched = tempfile::tempfile().unwrap(); + let handle = file.as_raw_handle(); + let untouched_handle = untouched.as_raw_handle(); + windows_set_handle_inherit(handle, true); + windows_set_handle_inherit(untouched_handle, false); + let original = windows_handle_flags(handle); + let mut calls = 0; + let guard = super::StdioInheritGuard::from_handles( + [ + std::ptr::null_mut(), + super::INVALID_HANDLE_VALUE, + handle, + handle, + untouched_handle, + ], + |handle| { + calls += 1; + windows_set_handle_inherit(handle, false); + Ok(()) + }, + ) + .unwrap(); + assert_eq!(calls, 1); + assert_eq!(windows_handle_flags(handle) & super::HANDLE_FLAG_INHERIT, 0); + assert_eq!( + windows_handle_flags(untouched_handle) & super::HANDLE_FLAG_INHERIT, + 0 + ); + drop(guard); + assert_eq!(windows_handle_flags(handle), original); + assert_eq!( + windows_handle_flags(untouched_handle) & super::HANDLE_FLAG_INHERIT, + 0 + ); + } + + #[cfg(windows)] + #[test] + fn windows_stdio_guard_rolls_back_partial_failure() { + use std::os::windows::io::AsRawHandle; + + let first = tempfile::tempfile().unwrap(); + let second = tempfile::tempfile().unwrap(); + let handles = [first.as_raw_handle(), second.as_raw_handle()]; + for handle in handles { + windows_set_handle_inherit(handle, true); + } + let original = handles.map(windows_handle_flags); + let mut calls = 0; + let result = super::StdioInheritGuard::from_handles(handles, |handle| { + calls += 1; + if calls == 2 { + return Err(std::io::Error::from_raw_os_error(5)); + } + windows_set_handle_inherit(handle, false); + Ok(()) + }); + let error = match result { + Ok(_) => panic!("expected the second handle update to fail"), + Err(error) => error, + }; + assert_eq!(error.raw_os_error(), Some(5)); + assert_eq!(handles.map(windows_handle_flags), original); + } + #[test] #[cfg(unix)] fn test_inherited_fd_source_needs_spare_for_cross_reserved_fd() { From f0141064556efc5c169a17ee52f4df12a682325b Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Thu, 10 Sep 2026 17:04:45 +0100 Subject: [PATCH 24/29] fix(runtime): unblock restored control and preserve transport progress Separate retained input debt from metadata credit and allow bounded independent control progress without violating stream or lifecycle fences. Yield guest reads by actual work, preserve TCP half-close tails, and retain shared-memory payload ownership on combined transports. Refresh maintenance clocks at transport admission, keep unrelated cleanup from fencing fresh commands, and strengthen source/child readiness checks under autonomous input pressure. Refuse incompatible unreleased transport barriers explicitly. Validated the frozen patch with macOS and Linux live transport suites and benchmarks, plus protocol, guest, runtime and harness regression tests. Final stack integration with the newer registry dependency is checked separately. --- COMPATIBILITY.md | 6 + crates/agentd/lib/agent.rs | 633 ++++++- crates/agentd/lib/serial.rs | 17 +- crates/agentd/lib/tcp.rs | 336 +++- crates/protocol/VERSIONING.md | 4 +- crates/protocol/lib/core.rs | 35 +- crates/protocol/lib/message.rs | 41 + crates/runtime/lib/checkpoint/coordinator.rs | 4 +- crates/runtime/lib/checkpoint/restore.rs | 26 +- crates/runtime/lib/runner/clock.rs | 32 +- crates/runtime/lib/runner/relay.rs | 1478 +++++++++++++++-- crates/runtime/lib/runner/workload_control.rs | 59 +- .../smoke/cli/test_transport_checkpoint.py | 149 ++ scripts/smoke/cli/transport-checkpoint.py | 50 +- 14 files changed, 2677 insertions(+), 193 deletions(-) diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index dc5eedff2..2f51e5d65 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -206,6 +206,10 @@ Unreleased #8 incremental exports use `completeness: "dependent"` and the must-u Full checkpoints and local branches now retain `transport_host_input`, `transport_input_credit`, and `transport_guest_bulk_bytes` in the existing `guest:agentd` resource binding. These are complete-frame cumulative positions and absolute grants, including credit still owned by pending captured input. Restore validates and seeds them before guest activation; resetting them would incorrectly grant capacity twice. Older unreleased development full snapshots missing this state are refused, and new full captures require their matching host/guest implementation. This is an approved replacement of unreleased state, not a snapshot schema bump or migration; released disk-only snapshots are unaffected. +The finalized private transport-credit contract charges stdin, inline filesystem/TCP payloads, and ordered EOF to the existing logical data (`bulk_*`) counters on either physical port. Command/control counters remain available when captured input is still awaiting consumption. Ready advertises barrier contract `2`; the superseded development contract `1` is not translated or restored. The outer frame, generation-8 data format, snapshot descriptor schema, and public SDK requests are unchanged. The ordinary writer retains bounded admission permits until physical delivery, permits unrelated metadata to pass credit-blocked payloads, and preserves per-correlation and client-disconnect ordering. Guest input processing also yields to the runtime after bounded actual reads, including partial records; this does not shrink wire records or change snapshot boundaries. + +Routine host clock maintenance is independent of unrelated correlation input, but stays ordered with other clocks and true global lifecycle fences. Its timestamp is sampled at console admission, not when queued; disconnect cleanup signals fence their own session only. Maintenance remains subject to the pause gate. This bounds host-queue timestamp age, not subsequent aging of already-admitted bytes during arbitrary host suspension or the kernel-only pause fallback when the workload freezer is unavailable. + Evolution rules: - Do not make semantically harmless serialization changes to identity-bearing bytes without treating them as an identity format change. @@ -297,6 +301,8 @@ Compatibility-sensitive ordering includes: Sources: [`crates/runtime/lib/client/ipc.rs`](crates/runtime/lib/client/ipc.rs), [`sdk/rust/lib/backend/local/mod.rs`](sdk/rust/lib/backend/local/mod.rs), [`sdk/rust/lib/runtime/handle.rs`](sdk/rust/lib/runtime/handle.rs), and artifact-specific migration and publication modules. +TCP completion follows both ordered half-closes; the first EOF alone keeps the opposite direction usable. In combined-port mode, validated guest-to-host TCP credit may pass queued host-to-guest raw data and its finish marker: it services the opposite direction without reordering input data or EOF. Opening, cancellation, ownership, and global lifecycle fences still constrain it. Raw TCP output may still be draining on the dedicated lane after its producer finishes. Decoded credit updates for a finished producer or absent TCP session are therefore no-ops, not cancellation: they cannot enable further output, and must not discard the queued tail or create a second terminal response. Active producers retain credit validation; data and finish messages retain their existing validation. + Review concurrency and crash points explicitly. A same-version happy-path test does not establish cross-version or crash compatibility. ## Review Triggers in Diffs diff --git a/crates/agentd/lib/agent.rs b/crates/agentd/lib/agent.rs index 96e12ba2e..d3d5bdacf 100644 --- a/crates/agentd/lib/agent.rs +++ b/crates/agentd/lib/agent.rs @@ -90,6 +90,11 @@ const MAX_INPUT_BUF_SIZE: usize = MAX_FRAME_SIZE as usize + 4; /// Dedicated records additionally carry the transport-level client incarnation. const MAX_BULK_INPUT_BUF_SIZE: usize = MAX_INPUT_BUF_SIZE + CLIENT_INCARNATION_SIZE; +/// Bound actual console work between runtime scheduling points, independently of wire records. +/// Partial records count too: a readable bulk port must not hide fresh primary/PTY readiness. +const AGENT_READ_QUANTUM_BYTES: usize = 256 * 1024; +const AGENT_READ_QUANTUM_CALLS: usize = 64; + /// Maximum time to wait for the host to acknowledge the init context. const INIT_ACK_TIMEOUT_SECS: u64 = 60; @@ -229,6 +234,13 @@ struct BulkInputState { input: BytesMut, } +/// Shared primary/bulk work retained across select turns, including incomplete wire records. +#[derive(Default)] +struct AgentReadBudget { + bytes: usize, + calls: usize, +} + /// One correlation's pending records in the guest-to-host DRR scheduler. struct BulkWriteFlow { queue: VecDeque, @@ -362,14 +374,19 @@ impl BulkInputState { } /// Read at most once, then return one bounded batch already available to the actor. - async fn read_turn(&mut self) -> AgentdResult> { + async fn read_turn( + &mut self, + read_budget: &mut AgentReadBudget, + ) -> AgentdResult> { let buffered = self.drain_turn()?; if !buffered.is_empty() { return Ok(buffered); } let mut guard = self.port.readable().await?; - match guard.try_io(|inner| read_from_fd(inner.get_ref().as_raw_fd(), &mut self.read_buf)) { + match guard + .try_io(|inner| read_budget.read_fd(inner.get_ref().as_raw_fd(), &mut self.read_buf)) + { Ok(Ok(0)) => { return Err(AgentdError::ExecSession( "dedicated bulk port closed".into(), @@ -417,6 +434,33 @@ impl BulkInputState { } } +impl AgentReadBudget { + fn read_fd(&mut self, fd: i32, buf: &mut [u8]) -> std::io::Result { + let result = read_from_fd(fd, buf); + self.record_read(result.as_ref().copied().unwrap_or(0)); + result + } + + fn record_read(&mut self, bytes: usize) { + self.calls = self.calls.saturating_add(1); + self.bytes = self.bytes.saturating_add(bytes); + } + + fn exhausted(&self) -> bool { + self.bytes >= AGENT_READ_QUANTUM_BYTES || self.calls >= AGENT_READ_QUANTUM_CALLS + } + + /// Call outside the competing select futures, after decoded records have an owning queue. + /// AsyncFd::readable can stay immediately ready without spending Tokio's cooperative budget; + /// returning to select alone does not let the driver discover a writable PTY or new control IO. + async fn yield_if_exhausted(&mut self) { + if self.exhausted() { + tokio::task::yield_now().await; + *self = Self::default(); + } + } +} + //-------------------------------------------------------------------------------------------------- // Functions //-------------------------------------------------------------------------------------------------- @@ -534,9 +578,13 @@ pub async fn run( let input_refunds = state.input_window.clone(); let mut credit_deadline = None; let mut last_input_credit = state.input_window.credit()?; + let mut read_budget = AgentReadBudget::default(); // Main loop. 'agent: loop { + // All consumed bytes are now in persistent input buffers or destination-owned records. + // Never yield after decoding inside read_turn: select cancellation could drop its frames. + read_budget.yield_if_exhausted().await; if state.resume_output_after_flush { // The private Thawed reply crosses the primary lane before either ordinary writer // becomes eligible again. A partial old record is never abandoned by this release. @@ -733,7 +781,7 @@ pub async fn run( bulk_input .as_mut() .expect("guarded dedicated bulk input") - .read_turn() + .read_turn(&mut read_budget) .await }, if bulk_input.is_some() && pending_bulk_inputs.is_empty() => { let frames = match turn { @@ -830,7 +878,9 @@ pub async fn run( let mut combined_turn_exhausted = false; loop { - match guard.try_io(|inner| read_from_fd(inner.get_ref().as_raw_fd(), &mut read_buf)) { + match guard.try_io(|inner| { + read_budget.read_fd(inner.get_ref().as_raw_fd(), &mut read_buf) + }) { Ok(Ok(0)) => { // EOF on serial — host disconnected. if !handoff::is_pid_1() { @@ -944,7 +994,12 @@ pub async fn run( if state.output_parked { return Err(AgentdError::ExecSession("ordinary input crossed the frozen transport cut".into())); } - Some(state.input_window.admit(InputLane::Control, wire_bytes)?) + let lane = if msg.t.uses_workload_data_credit() { + InputLane::Bulk + } else { + InputLane::Control + }; + Some(state.input_window.admit(lane, wire_bytes)?) }; if msg.flags != msg.t.flags() { let out_before = serial_out_buf.len(); @@ -1002,11 +1057,19 @@ pub async fn run( // Thawed has now crossed the wire. A host can immediately resume // ordinary input, so release our parked writers at the outer-loop // boundary before draining another readable batch. - if combined_turn_exhausted || state.resume_output_after_flush { + if combined_turn_exhausted + || state.resume_output_after_flush + || read_budget.exhausted() + { break; } } - Ok(Err(e)) if e.kind() == std::io::ErrorKind::Interrupted => continue, + Ok(Err(e)) if e.kind() == std::io::ErrorKind::Interrupted => { + if read_budget.exhausted() { + break; + } + continue; + } Ok(Err(_)) if !handoff::is_pid_1() => { guard.clear_ready(); drop(guard); @@ -2841,7 +2904,10 @@ async fn handle_message_with_charge( BulkKind::Tcp => { let result = match state.tcp_sessions.get(&msg.id) { Some(session) => session.apply_credit(credit).await, - None => Err(format!("unknown TCP session: {}", msg.id)), + // A terminal may retire the producer before the other physical lane + // finishes delivering its output. Late credit has no recipient and must + // not cancel that tail or manufacture a second terminal response. + None => Ok(()), }; if let Err(error) = result { encode_bulk_tcp_failure(msg.id, error, out_buf)?; @@ -4246,6 +4312,345 @@ mod tests { drop(charge); } + #[tokio::test] + async fn mixed_primary_data_cut_waits_for_the_complete_dedicated_prefix() { + let mut state = AgentState::default(); + let (mut sender, _output) = SessionOutputSender::channel(); + let mut activity = ActivityTracker::new(); + let config = AgentdConfig { + user: None, + security_profile: Default::default(), + default_cwd: None, + default_env: Vec::new(), + }; + let heartbeat = heartbeat::HeartbeatControl::default(); + let mut workload = crate::workload::tests::fake_latch(); + + // Primary stdin and its empty EOF share the logical data ledger with the dedicated + // port. Retain every charge to model a consumer that has not accepted any input yet. + let mut charges = Vec::new(); + for data in [vec![0x31; 1024], Vec::new()] { + let message = + Message::with_payload(MessageType::ExecStdin, 1, &ExecStdin { data }).unwrap(); + assert!(message.t.uses_workload_data_credit()); + let mut wire = Vec::new(); + codec::encode_to_buf(&message, &mut wire).unwrap(); + charges.push( + state + .input_window + .admit(InputLane::Bulk, wire.len()) + .unwrap(), + ); + } + let primary_position = state.input_window.position(); + assert_eq!(primary_position.control_bytes, 0); + assert_eq!(primary_position.control_frames, 0); + assert_eq!(primary_position.bulk_frames, 2); + + let incarnation = [0x43; CLIENT_INCARNATION_SIZE]; + let record = BulkRecord { + id: 2, + kind: BulkKind::Filesystem, + flow: BulkFlow::HostToGuest, + offset: 0, + payload: Bytes::from(vec![0x52; 512]), + }; + let mut dedicated_wire = incarnation.to_vec(); + codec::encode_bulk_to_buf(&record, &mut dedicated_wire).unwrap(); + let target = WorkloadTransportPosition { + bulk_bytes: primary_position.bulk_bytes + dedicated_wire.len() as u64, + bulk_frames: primary_position.bulk_frames + 1, + ..primary_position + }; + let freeze = Message::with_payload( + MessageType::WorkloadFreeze, + u32::MAX, + &WorkloadFreeze { + attempt_id: "mixed-prefix".into(), + host_input: target, + }, + ) + .unwrap(); + + let mut dedicated_input = BytesMut::new(); + for prefix in [ + &dedicated_wire[..0], + &dedicated_wire[..dedicated_wire.len() - 1], + ] { + dedicated_input.extend_from_slice(prefix); + assert!( + try_decode_incarnated_bulk_from_bytes(&mut dedicated_input) + .unwrap() + .is_none() + ); + let mut out = Vec::new(); + handle_message( + freeze.clone(), + &mut state, + &mut activity, + &mut sender, + &mut out, + &config, + &mut workload, + &heartbeat, + ) + .await + .unwrap(); + assert!( + out.is_empty(), + "primary data cannot cover missing dedicated bytes" + ); + assert!(!workload.is_frozen()); + assert!(!state.output_parked); + assert!(state.pending_freeze.is_some()); + assert_eq!(state.input_window.position(), primary_position); + } + + dedicated_input.extend_from_slice(&dedicated_wire[dedicated_wire.len() - 1..]); + let decoded = try_decode_incarnated_bulk_from_bytes(&mut dedicated_input) + .unwrap() + .unwrap(); + assert_eq!(decoded.incarnation, incarnation); + assert_eq!(decoded.record, record); + assert!(dedicated_input.is_empty()); + charges.push( + state + .input_window + .admit(InputLane::Bulk, bulk_wire_bytes(&decoded.record, true)) + .unwrap(), + ); + assert_eq!(state.input_window.position(), target); + + // Resume the retained request as the outer actor does once both cumulative counters + // reach the cut. Decoding suffices; none of the primary or dedicated data is consumed. + let pending = state.pending_freeze.take().unwrap(); + let mut out = Vec::new(); + handle_message( + pending, + &mut state, + &mut activity, + &mut sender, + &mut out, + &config, + &mut workload, + &heartbeat, + ) + .await + .unwrap(); + let mut bytes = BytesMut::from(out.as_slice()); + let reply = decode_reply_skipping_credit(&mut bytes); + let frozen = reply.payload::().unwrap(); + assert_eq!(reply.t, MessageType::WorkloadFrozen); + assert_eq!(reply.id, u32::MAX); + assert_eq!(frozen.attempt_id, "mixed-prefix"); + assert_eq!(frozen.input_credit, initial_input_credit()); + assert_eq!(state.frozen_host_input, Some(target)); + assert!(workload.is_frozen()); + assert!(state.output_parked); + assert!(state.pending_freeze.is_none()); + assert!(bytes.is_empty()); + drop(charges); + } + + #[tokio::test] + async fn late_tcp_credit_preserves_raw_tail_before_and_after_terminal_retirement() { + use std::collections::hash_map::DefaultHasher; + use std::hash::Hasher; + + use microsandbox_protocol::bulk::BulkOffer; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpListener; + + tokio::time::timeout(Duration::from_secs(10), async { + let mut state = AgentState::default(); + let incarnation = [0x57; CLIENT_INCARNATION_SIZE]; + establish_relay_client( + &mut state, + RelayClientConnected { + id_start: 1, + id_end_exclusive: microsandbox_protocol::AGENT_RELAY_ID_RANGE_STEP, + incarnation, + }, + ) + .unwrap(); + // Leave the dedicated scheduler's input queued: primary completion is allowed to + // overtake these bytes, but no late credit may send DropFlow to discard their tail. + let (mut sender, mut control, mut bulk, mut scheduler_commands) = + SessionOutputSender::split_channel(); + let producer = sender.with_incarnation(Some(incarnation)); + let listener = TcpListener::bind(("127.0.0.1", 0)).await.unwrap(); + state.tcp_sessions.insert( + 1, + TcpSession::open( + 1, + TcpConnect { + host: "127.0.0.1".into(), + port: listener.local_addr().unwrap().port(), + bulk: Some(BulkOffer::tcp()), + }, + &producer, + ), + ); + let (mut peer, _) = listener.accept().await.unwrap(); + for expected in [MessageType::TcpConnected, MessageType::BulkAccepted] { + let envelope = control.recv().await.unwrap(); + let SessionOutput::Raw(mut output) = envelope.output else { + panic!() + }; + assert_eq!( + codec::try_decode_from_buf(&mut output.frame) + .unwrap() + .unwrap() + .t, + expected + ); + } + state + .tcp_sessions + .get(&1) + .unwrap() + .finish_bulk(BulkFinish { + kind: BulkKind::Tcp, + flow: BulkFlow::HostToGuest, + final_offset: 0, + }) + .await + .unwrap(); + assert_eq!(peer.read(&mut [0]).await.unwrap(), 0); + let payload = Bytes::from( + (0..6 * 1024 * 1024) + .map(|index| (index % 251) as u8) + .collect::>(), + ); + let peer_payload = payload.clone(); + let peer_task = tokio::spawn(async move { + peer.write_all(&peer_payload).await.unwrap(); + peer.shutdown().await.unwrap(); + }); + let mut received = Vec::new(); + let mut consumed = 0; + while consumed < 4 * 1024 * 1024 { + let envelope = bulk.recv().await.unwrap(); + let SessionOutput::Bulk(output) = &envelope.output else { + panic!() + }; + consumed += output.record.payload.len(); + received.push(envelope); + } + peer_task.await.unwrap(); + while !state.tcp_sessions.get(&1).unwrap().is_finished() { + tokio::task::yield_now().await; + } + assert!(consumed < payload.len()); + assert!( + !bulk.is_empty(), + "the dedicated output tail must still be queued" + ); + let credit = BulkCredit { + kind: BulkKind::Tcp, + flow: BulkFlow::GuestToHost, + consumed_offset: consumed as u64, + credit_limit: consumed as u64 + DEFAULT_BULK_WINDOW, + }; + let mut activity = ActivityTracker::new(); + let config = AgentdConfig { + user: None, + security_profile: Default::default(), + default_cwd: None, + default_env: Vec::new(), + }; + let mut workload = crate::workload::tests::fake_latch(); + let heartbeat = heartbeat::HeartbeatControl::default(); + for retired in [false, true] { + if retired { + for expected in [MessageType::BulkFinish, MessageType::TcpClosed] { + let envelope = control.try_recv().unwrap(); + let SessionOutput::Raw(mut output) = envelope.output else { + panic!() + }; + let message = codec::try_decode_from_buf(&mut output.frame) + .unwrap() + .unwrap(); + assert_eq!(message.t, expected); + if expected == MessageType::BulkFinish { + assert_eq!( + message.payload::().unwrap().final_offset, + payload.len() as u64 + ); + } else { + assert_ne!( + message.flags & microsandbox_protocol::message::FLAG_TERMINAL, + 0 + ); + assert!(matches!(output.completion, Some(RawSessionCompletion::Tcp))); + complete_raw_session( + 1, + output.completion, + &mut state.read_sessions, + &mut state.tcp_sessions, + ); + clear_bulk_receive_state(&mut state, 1); + } + } + } + assert_eq!(state.tcp_sessions.contains_key(&1), !retired); + let mut out = Vec::new(); + handle_message( + Message::with_payload(MessageType::BulkCredit, 1, &credit).unwrap(), + &mut state, + &mut activity, + &mut sender, + &mut out, + &config, + &mut workload, + &heartbeat, + ) + .await + .unwrap(); + assert!( + out.is_empty(), + "late credit must not emit cancellation or another terminal" + ); + assert!( + matches!( + scheduler_commands.try_recv(), + Err(tokio::sync::mpsc::error::TryRecvError::Empty) + ), + "late credit must not purge raw output" + ); + assert!(!bulk.is_empty()); + } + assert!(matches!( + control.try_recv(), + Err(tokio::sync::mpsc::error::TryRecvError::Empty) + )); + while let Ok(envelope) = bulk.try_recv() { + received.push(envelope); + } + let mut offset = 0; + let mut actual_hash = DefaultHasher::new(); + let mut expected_hash = DefaultHasher::new(); + expected_hash.write(&payload); + for envelope in received { + assert_eq!(envelope.id, 1); + assert_eq!(envelope.incarnation, Some(incarnation)); + let SessionOutput::Bulk(output) = envelope.output else { + panic!() + }; + let record = output.record; + assert_eq!(record.offset, offset as u64); + let end = offset + record.payload.len(); + assert_eq!(record.payload.as_ref(), &payload[offset..end]); + actual_hash.write(&record.payload); + offset = end; + } + assert_eq!(offset, payload.len()); + assert_eq!(actual_hash.finish(), expected_hash.finish()); + }) + .await + .expect("late-credit TCP tail did not complete"); + } + #[tokio::test] async fn dedicated_bulk_park_finishes_one_record_and_retains_the_next() { use std::os::fd::OwnedFd; @@ -4481,6 +4886,208 @@ mod tests { )); } + #[test] + fn agent_read_budget_counts_bytes_and_calls_independently() { + let mut bytes = AgentReadBudget::default(); + bytes.record_read(AGENT_READ_QUANTUM_BYTES - 1); + assert!(!bytes.exhausted()); + bytes.record_read(1); + assert!(bytes.exhausted()); + + let mut calls = AgentReadBudget::default(); + for _ in 1..AGENT_READ_QUANTUM_CALLS { + assert!(calls.read_fd(-1, &mut [0]).is_err()); + assert!(!calls.exhausted()); + } + assert!(calls.read_fd(-1, &mut [0]).is_err()); + assert!(calls.exhausted(), "failed reads also bound a retry loop"); + assert_eq!(calls.bytes, 0); + + let mut large = AgentReadBudget::default(); + large.record_read(BULK_SERIAL_READ_BUF_SIZE); + assert!( + large.exhausted(), + "a large read is not a smaller wire record" + ); + assert_eq!(large.bytes, BULK_SERIAL_READ_BUF_SIZE); + } + + #[tokio::test(flavor = "current_thread")] + async fn agent_read_budget_services_driver_during_partial_bulk_records() { + // The control demonstrates the failure without relying on wall-clock delays: cached + // readable bulk input never lets the runtime observe the newly readable/writable peers. + assert!(!observe_driver_during_partial_bulk(false).await); + assert!(observe_driver_during_partial_bulk(true).await); + } + + async fn observe_driver_during_partial_bulk(yield_at_boundary: bool) -> bool { + use std::io::Write; + use std::os::fd::OwnedFd; + use std::os::unix::net::UnixStream; + use std::sync::atomic::AtomicUsize; + + let incarnation = [0x42; CLIENT_INCARNATION_SIZE]; + let first = BulkRecord { + id: 1, + kind: BulkKind::Filesystem, + flow: BulkFlow::HostToGuest, + offset: 0, + payload: Bytes::from(vec![0xa5; 512]), + }; + let second = BulkRecord { + offset: first.payload.len() as u64, + payload: Bytes::from(vec![0x5a; 512]), + ..first.clone() + }; + let mut wire = Vec::new(); + for record in [&first, &second] { + wire.extend_from_slice(&incarnation); + codec::encode_bulk_to_buf(record, &mut wire).unwrap(); + } + let (mut source, input) = UnixStream::pair().unwrap(); + input.set_nonblocking(true).unwrap(); + source.write_all(&wire).unwrap(); + let mut input = BulkInputState::new(File::from(OwnedFd::from(input))).unwrap(); + // Model fragmented console reads while keeping the complete wire records unchanged. + input.read_buf.truncate(1); + + let (mut control_source, control) = UnixStream::pair().unwrap(); + control.set_nonblocking(true).unwrap(); + let control = AsyncFd::new(control).unwrap(); + let (pipe_reader, pipe_writer) = nix::unistd::pipe2(nix::fcntl::OFlag::O_NONBLOCK).unwrap(); + let pipe_writer = AsyncFd::new(pipe_writer).unwrap(); + // Prime the reactor, then clear the pipe's cached writable event with a real EAGAIN. + let mut writable = pipe_writer.writable().await.unwrap(); + loop { + match writable.try_io(|fd| write_to_fd(fd.get_ref().as_raw_fd(), &[0; 4096])) { + Ok(Ok(count)) => assert!(count > 0), + Ok(Err(error)) => panic!("fill stdin pipe: {error}"), + Err(_) => break, + } + } + drop(writable); + let mut drained = [0; 4096]; + loop { + match read_from_fd(pipe_reader.as_raw_fd(), &mut drained) { + Ok(count) => assert!(count > 0), + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => break, + Err(error) => panic!("drain stdin pipe: {error}"), + } + } + control_source.write_all(b"c").unwrap(); + let serviced = Arc::new(AtomicUsize::new(0)); + let control_serviced = Arc::clone(&serviced); + let control_task = tokio::spawn(async move { + loop { + let mut ready = control.readable().await.unwrap(); + let mut byte = [0]; + match ready.try_io(|fd| read_from_fd(fd.get_ref().as_raw_fd(), &mut byte)) { + Ok(Ok(1)) => { + assert_eq!(byte, *b"c"); + control_serviced.fetch_or(1, Ordering::Relaxed); + break; + } + Ok(result) => panic!("read control marker: {result:?}"), + Err(_) => continue, + } + } + }); + let stdin_serviced = Arc::clone(&serviced); + let stdin_task = tokio::spawn(async move { + std::future::poll_fn(|cx| { + loop { + let mut ready = std::task::ready!(pipe_writer.poll_write_ready(cx)).unwrap(); + match ready.try_io(|fd| write_to_fd(fd.get_ref().as_raw_fd(), b"i")) { + Ok(result) => return Poll::Ready(result), + Err(_) => continue, + } + } + }) + .await + .unwrap(); + stdin_serviced.fetch_or(2, Ordering::Relaxed); + }); + + let mut budget = AgentReadBudget::default(); + let mut received = Vec::new(); + let mut serviced_before_first_record = false; + while received.len() < 2 { + let frames = tokio::select! { + frames = input.read_turn(&mut budget) => frames.unwrap(), + _ = std::future::pending::<()>() => unreachable!(), + }; + for frame in frames { + assert_eq!(frame.incarnation, incarnation); + received.push(frame.record); + } + // This is the production cancellation-safe boundary: no decoded frames are local to + // a competing select future, and even partial-record reads have spent the budget. + if yield_at_boundary { + budget.yield_if_exhausted().await; + } + if received.is_empty() && serviced.load(Ordering::Relaxed) == 3 { + serviced_before_first_record = true; + } + } + assert_eq!( + received, + [first, second], + "yield preserves record bytes and FIFO" + ); + assert!(input.input.is_empty()); + control_task.await.unwrap(); + stdin_task.await.unwrap(); + let mut marker = [0]; + assert_eq!( + read_from_fd(pipe_reader.as_raw_fd(), &mut marker).unwrap(), + 1 + ); + assert_eq!(marker, *b"i"); + serviced_before_first_record + } + + #[tokio::test(flavor = "current_thread")] + async fn cancelled_bulk_read_keeps_partial_record_and_read_budget() { + use std::io::Write; + use std::os::fd::OwnedFd; + use std::os::unix::net::UnixStream; + + let incarnation = [0x29; CLIENT_INCARNATION_SIZE]; + let record = BulkRecord { + id: 1, + kind: BulkKind::Filesystem, + flow: BulkFlow::HostToGuest, + offset: 0, + payload: Bytes::from(vec![0x51; 512]), + }; + let mut wire = incarnation.to_vec(); + codec::encode_bulk_to_buf(&record, &mut wire).unwrap(); + let (mut source, input) = UnixStream::pair().unwrap(); + input.set_nonblocking(true).unwrap(); + let mut input = BulkInputState::new(File::from(OwnedFd::from(input))).unwrap(); + let mut budget = AgentReadBudget::default(); + source.write_all(&wire[..40]).unwrap(); + assert!(input.read_turn(&mut budget).await.unwrap().is_empty()); + // Consume the stale readable hint so the following select truly cancels a pending read. + assert!(input.read_turn(&mut budget).await.unwrap().is_empty()); + let calls = budget.calls; + tokio::select! { + biased; + result = input.read_turn(&mut budget) => panic!("unexpected read: {result:?}"), + _ = std::future::ready(()) => {}, + } + assert_eq!(input.input.as_ref(), &wire[..40]); + assert_eq!(budget.bytes, 40); + assert_eq!(budget.calls, calls); + source.write_all(&wire[40..]).unwrap(); + let frames = input.read_turn(&mut budget).await.unwrap(); + assert_eq!(frames.len(), 1); + assert_eq!(frames[0].record, record); + assert_eq!(frames[0].incarnation, incarnation); + assert!(input.input.is_empty()); + assert_eq!(budget.bytes, wire.len()); + } + #[test] fn disconnect_cleanup_keeps_other_clients_bulk_offsets() { let mut state = AgentState::default(); @@ -5059,10 +5666,10 @@ mod tests { let ledger = state.input_window.clone(); let initial = ledger.credit().unwrap(); let data = vec![0x61; 1024 * 1024]; - let charge = ledger.admit(InputLane::Control, data.len() + 32).unwrap(); + let charge = ledger.admit(InputLane::Bulk, data.len() + 32).unwrap(); let mut stdin = Message::with_payload(MessageType::ExecStdin, 1, &ExecStdin { data }).unwrap(); - // A generation-8 SDK still travels over the bundled capability-1 host transport. + // A generation-8 SDK still travels over the bundled private transport contract. // Dispatch must not fall back to blocking writes based on the client message version. stdin.v = 8; time::timeout( @@ -5085,7 +5692,7 @@ mod tests { assert!(encoded.is_empty()); assert!(state.sessions[&1].has_pending_stdin()); if accepted_eof { - let charge = ledger.admit(InputLane::Control, 32).unwrap(); + let charge = ledger.admit(InputLane::Bulk, 32).unwrap(); let mut eof = Message::with_payload( MessageType::ExecStdin, 1, @@ -5177,8 +5784,8 @@ mod tests { .expect("accepted stdin or ordered EOF did not drain"); assert_eq!(received, vec![0x61; 1024 * 1024]); assert_eq!( - ledger.credit().unwrap().control_bytes, - initial.control_bytes + position.control_bytes + ledger.credit().unwrap().bulk_bytes, + initial.bulk_bytes + position.bulk_bytes ); if !tty && mode == Continue && !accepted_eof { // Source Continue must not synthesize EOF. The same owner can still send input. diff --git a/crates/agentd/lib/serial.rs b/crates/agentd/lib/serial.rs index 663cfd98a..5b258c27c 100644 --- a/crates/agentd/lib/serial.rs +++ b/crates/agentd/lib/serial.rs @@ -22,8 +22,8 @@ pub use microsandbox_protocol::{AGENT_BULK_PORT_NAME, AGENT_PORT_NAME}; // Types //-------------------------------------------------------------------------------------------------- -/// The frame class whose complete wire bytes consume admission capacity. Raw records use Bulk -/// even on a combined physical port, while ordinary CBOR messages and leases use Control. +/// Logical admission class, independent of the physical port. Raw records, stdin and inline +/// FS/TCP payloads use Bulk; command metadata and leases retain separate Control capacity. #[derive(Clone, Copy, Debug)] pub(crate) enum InputLane { Control, @@ -256,20 +256,23 @@ mod tests { #[test] fn inherited_input_refunds_the_same_cumulative_ledger_after_restore() { let window = window(); - let inherited = window.admit(InputLane::Control, 7).unwrap(); + let inherited = window.admit(InputLane::Bulk, 15).unwrap(); let source_position = window.position(); let restored_view = window.clone(); // Moving a retained session to the detached table must not grant a fresh window. assert_eq!(restored_view.position(), source_position); - assert!(restored_view.admit(InputLane::Control, 2).is_err()); + assert!(restored_view.admit(InputLane::Bulk, 2).is_err()); + // Captured input keeps its debt, but cannot prevent the fresh host from starting an exec. + let command = restored_view.admit(InputLane::Control, 8).unwrap(); + drop(command); drop(inherited); - let fresh = restored_view.admit(InputLane::Control, 8).unwrap(); - assert_eq!(restored_view.position().control_bytes, 15); + let fresh = restored_view.admit(InputLane::Bulk, 16).unwrap(); + assert_eq!(restored_view.position().bulk_bytes, 31); drop(fresh); } #[test] - fn physical_lanes_are_independent_and_counter_overflow_fails_closed() { + fn logical_classes_are_independent_and_counter_overflow_fails_closed() { let window = window(); let control = window.admit(InputLane::Control, 8).unwrap(); let bulk = window.admit(InputLane::Bulk, 16).unwrap(); diff --git a/crates/agentd/lib/tcp.rs b/crates/agentd/lib/tcp.rs index 044d06c08..68278c961 100644 --- a/crates/agentd/lib/tcp.rs +++ b/crates/agentd/lib/tcp.rs @@ -169,7 +169,9 @@ impl TcpSession { .as_ref() .ok_or_else(|| "TCP bulk control path is unavailable".to_string())?; if control.credit.is_closed() { - return Err("TCP session is closed".into()); + // Sink consumption can return credit after the producer queued its final output. + // There is no sender left to enable; failing here would cancel its queued raw tail. + return Ok(()); } control.credit.send_replace(Some(credit)); Ok(()) @@ -484,6 +486,12 @@ async fn relay_tcp_session( let mut read_eof = false; loop { + // One EOF leaves the opposite half usable. Once both halves finish, all ordered + // writes have completed and the peer's final output/EOF is already queued. Exit so + // the existing terminal frame releases the host route and guest session together. + if read_eof && write_shutdown { + break; + } let read_limit = bulk.as_ref().map_or(TCP_CHUNK_SIZE, |state| { state .send @@ -948,9 +956,7 @@ mod tests { #[tokio::test] async fn blocked_tcp_retains_data_and_eof_credit_until_consumption_or_cancel() { use crate::serial::{InputLane, InputWindow}; - use microsandbox_protocol::core::{ - WORKLOAD_TRANSPORT_CONTROL_BYTES, WorkloadTransportCredit, - }; + use microsandbox_protocol::core::WorkloadTransportCredit; use std::os::fd::AsRawFd; for cancel in [false, true] { @@ -983,15 +989,15 @@ mod tests { let (mut peer, _) = listener.accept().await.unwrap(); assert_eq!(recv_message(&mut output).await.t, MessageType::TcpConnected); let initial = WorkloadTransportCredit { - control_bytes: WORKLOAD_TRANSPORT_CONTROL_BYTES, + control_bytes: 64, control_frames: 2, - bulk_bytes: 0, - bulk_frames: 0, + bulk_bytes: 8 * 1024 * 1024, + bulk_frames: 2, }; let ledger = InputWindow::new(initial); - let payload_len = WORKLOAD_TRANSPORT_CONTROL_BYTES as usize - 64; - let data_charge = ledger.admit(InputLane::Control, payload_len + 32).unwrap(); - let eof_charge = ledger.admit(InputLane::Control, 32).unwrap(); + let payload_len = initial.bulk_bytes as usize - 64; + let data_charge = ledger.admit(InputLane::Bulk, payload_len + 32).unwrap(); + let eof_charge = ledger.admit(InputLane::Bulk, 32).unwrap(); tokio::time::timeout(Duration::from_millis(100), async { session .write_data_charged(vec![0x5c; payload_len], Some(data_charge)) @@ -1003,7 +1009,7 @@ mod tests { .expect("admitted input waited for a blocked TCP consumer"); tokio::time::sleep(Duration::from_millis(20)).await; assert_eq!(ledger.credit().unwrap(), initial); - assert!(ledger.admit(InputLane::Control, 1).is_err()); + assert!(ledger.admit(InputLane::Bulk, 1).is_err()); if cancel { session.close(); wait_finished(&session).await; @@ -1018,11 +1024,12 @@ mod tests { session.close(); wait_finished(&session).await; } + assert_eq!(ledger.credit().unwrap().bulk_bytes, initial.bulk_bytes * 2); + assert_eq!(ledger.credit().unwrap().bulk_frames, 4); assert_eq!( ledger.credit().unwrap().control_bytes, - initial.control_bytes * 2 + initial.control_bytes ); - assert_eq!(ledger.credit().unwrap().control_frames, 4); } } @@ -1132,6 +1139,227 @@ mod tests { accept_task.await.unwrap(); } + #[tokio::test] + async fn active_raw_credit_validation_and_inline_negotiation_still_apply() { + for raw in [false, true] { + let listener = TcpListener::bind(("127.0.0.1", 0)).await.unwrap(); + let (tx, mut rx) = SessionOutputSender::channel(); + let session = TcpSession::open( + 41, + TcpConnect { + host: "127.0.0.1".into(), + port: listener.local_addr().unwrap().port(), + bulk: raw.then(BulkOffer::tcp), + }, + &tx, + ); + let (_peer, _) = listener.accept().await.unwrap(); + assert_eq!(recv_message(&mut rx).await.t, MessageType::TcpConnected); + if raw { + assert_eq!(recv_message(&mut rx).await.t, MessageType::BulkAccepted); + } + let result = session + .apply_credit(BulkCredit { + kind: BulkKind::Tcp, + flow: BulkFlow::GuestToHost, + consumed_offset: 1, + credit_limit: DEFAULT_BULK_WINDOW + 1, + }) + .await; + if raw { + result.unwrap(); + let failed = tokio::time::timeout(Duration::from_secs(1), recv_message(&mut rx)) + .await + .unwrap(); + assert_eq!(failed.t, MessageType::TcpFailed); + assert_eq!(failed.flags, FLAG_TERMINAL); + assert!( + failed + .payload::() + .unwrap() + .error + .contains("not admitted") + ); + wait_finished(&session).await; + } else { + assert!(result.unwrap_err().contains("generation-6")); + session.close(); + wait_finished(&session).await; + } + } + } + + #[tokio::test] + async fn both_half_close_orders_preserve_data_and_emit_one_terminal() { + for raw in [false, true] { + for peer_first in [false, true] { + tokio::time::timeout(Duration::from_secs(5), async { + let listener = TcpListener::bind(("127.0.0.1", 0)).await.unwrap(); + let (tx, mut rx) = SessionOutputSender::channel(); + let session = TcpSession::open( + 31, + TcpConnect { + host: "127.0.0.1".into(), + port: listener.local_addr().unwrap().port(), + bulk: raw.then(BulkOffer::tcp), + }, + &tx, + ); + let (mut peer, _) = listener.accept().await.unwrap(); + assert_eq!(recv_message(&mut rx).await.t, MessageType::TcpConnected); + if raw { + assert_eq!(recv_message(&mut rx).await.t, MessageType::BulkAccepted); + } + let host_data = b"host data survives the peer's first EOF"; + let peer_data = b"peer data survives the host's first EOF"; + if peer_first { + peer.write_all(peer_data).await.unwrap(); + peer.shutdown().await.unwrap(); + assert_tcp_output_through_eof(&mut rx, raw, peer_data).await; + assert!(!session.is_finished(), "one EOF must preserve host writes"); + send_test_input_and_eof(&session, raw, host_data).await; + } else { + send_test_input_and_eof(&session, raw, host_data).await; + } + + let mut received = Vec::new(); + peer.read_to_end(&mut received).await.unwrap(); + assert_eq!(received, host_data); + if !peer_first { + assert!(!session.is_finished(), "one EOF must preserve peer output"); + peer.write_all(peer_data).await.unwrap(); + peer.shutdown().await.unwrap(); + assert_tcp_output_through_eof(&mut rx, raw, peer_data).await; + } + assert_one_normal_terminal(&session, &mut rx).await; + }) + .await + .unwrap_or_else(|_| { + panic!("TCP completion timed out: raw={raw}, peer_first={peer_first}") + }); + } + } + } + + #[tokio::test] + async fn raw_finish_waits_for_delayed_record_and_pending_socket_write_before_terminal() { + use std::os::fd::AsRawFd; + + tokio::time::timeout(Duration::from_secs(5), async { + let listener = TcpListener::bind(("127.0.0.1", 0)).await.unwrap(); + let (stream, accepted) = tokio::join!( + TcpStream::connect(listener.local_addr().unwrap()), + listener.accept(), + ); + let stream = stream.unwrap(); + let (mut peer, _) = accepted.unwrap(); + // Make the last record larger than both fixed socket buffers. The test observes a + // delivered prefix before draining the rest, so EOF cannot be credited at enqueue. + for (fd, option, bytes) in [ + (stream.as_raw_fd(), libc::SO_SNDBUF, 4096 as libc::c_int), + (peer.as_raw_fd(), libc::SO_RCVBUF, 65536 as libc::c_int), + ] { + assert_eq!( + unsafe { + libc::setsockopt( + fd, + libc::SOL_SOCKET, + option, + (&bytes as *const libc::c_int).cast(), + std::mem::size_of_val(&bytes) as libc::socklen_t, + ) + }, + 0 + ); + } + let (tx, mut rx) = SessionOutputSender::channel(); + let (commands, commands_rx) = mpsc::channel(TCP_COMMAND_CAPACITY); + let (credit, credit_rx) = watch::channel(None); + let (finish, finish_rx) = mpsc::channel(1); + let task = tokio::spawn(relay_tcp_session( + 37, + stream, + commands_rx, + Some(TcpBulkControlReceivers { + credit: credit_rx, + finish: finish_rx, + }), + tx, + Some(TcpBulkState { + send: BulkSendState::new( + BulkKind::Tcp, + BulkFlow::GuestToHost, + DEFAULT_BULK_RECORD_PAYLOAD, + DEFAULT_BULK_WINDOW, + ) + .unwrap(), + receive: BulkReceiveState::new( + BulkKind::Tcp, + BulkFlow::HostToGuest, + DEFAULT_BULK_RECORD_PAYLOAD, + DEFAULT_BULK_WINDOW, + DEFAULT_BULK_WINDOW, + ) + .unwrap(), + }), + )); + let session = TcpSession { + owner_id: 37, + commands, + bulk_control: Some(TcpBulkControlSenders { credit, finish }), + task, + bulk: true, + }; + peer.shutdown().await.unwrap(); + assert_tcp_output_through_eof(&mut rx, true, b"").await; + let payload = Bytes::from(vec![0x6a; DEFAULT_BULK_RECORD_PAYLOAD as usize]); + session + .finish_bulk(BulkFinish { + kind: BulkKind::Tcp, + flow: BulkFlow::HostToGuest, + final_offset: payload.len() as u64, + }) + .await + .unwrap(); + while session.bulk_control.as_ref().unwrap().finish.capacity() == 0 { + tokio::task::yield_now().await; + } + assert!( + !session.is_finished(), + "finish cannot skip its missing final record" + ); + assert!(matches!( + rx.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + )); + session + .write_bulk(AdmittedBulkRecord::for_test(BulkRecord { + id: 37, + kind: BulkKind::Tcp, + flow: BulkFlow::HostToGuest, + offset: 0, + payload: payload.clone(), + })) + .await + .unwrap(); + let mut received = vec![0]; + peer.read_exact(&mut received).await.unwrap(); + assert!( + !session.is_finished(), + "finish cannot skip a partial socket write" + ); + assert!(matches!( + rx.try_recv(), + Err(mpsc::error::TryRecvError::Empty) + )); + peer.read_to_end(&mut received).await.unwrap(); + assert_eq!(received, payload); + assert_one_normal_terminal(&session, &mut rx).await; + }) + .await + .expect("delayed raw record did not finish normally"); + } + #[tokio::test] async fn raw_bulk_tcp_relays_both_directions_and_exact_half_closes() { let listener = TcpListener::bind(("127.0.0.1", 0)).await.unwrap(); @@ -1331,6 +1559,88 @@ mod tests { session.close(); } + async fn send_test_input_and_eof(session: &TcpSession, raw: bool, data: &[u8]) { + if raw { + session + .write_bulk(AdmittedBulkRecord::for_test(BulkRecord { + id: session.owner_id(), + kind: BulkKind::Tcp, + flow: BulkFlow::HostToGuest, + offset: 0, + payload: Bytes::copy_from_slice(data), + })) + .await + .unwrap(); + session + .finish_bulk(BulkFinish { + kind: BulkKind::Tcp, + flow: BulkFlow::HostToGuest, + final_offset: data.len() as u64, + }) + .await + .unwrap(); + } else { + session.write_data(data.to_vec()).await.unwrap(); + session.close_write().await.unwrap(); + } + } + + async fn assert_tcp_output_through_eof( + rx: &mut mpsc::Receiver, + raw: bool, + expected: &[u8], + ) { + let mut received = Vec::new(); + loop { + let envelope = rx.recv().await.expect("TCP output ended before EOF"); + match envelope.output { + SessionOutput::Bulk(output) => { + assert!(raw); + assert_eq!(output.record.offset, received.len() as u64); + received.extend_from_slice(&output.record.payload); + } + SessionOutput::Raw(mut output) => { + let message = decode_one_message(&mut output.frame); + assert_eq!(message.flags & FLAG_TERMINAL, 0, "terminal preceded EOF"); + match message.t { + MessageType::TcpData => { + assert!(!raw); + received.extend(message.payload::().unwrap().data); + } + MessageType::TcpEof => { + assert!(!raw); + break; + } + MessageType::BulkFinish => { + assert!(raw); + let finish = message.payload::().unwrap(); + assert_eq!(finish.final_offset, received.len() as u64); + break; + } + _ => panic!("unexpected TCP output: {:?}", message.t), + } + } + _ => panic!("unexpected non-TCP output"), + } + } + assert_eq!(received, expected); + } + + async fn assert_one_normal_terminal( + session: &TcpSession, + rx: &mut mpsc::Receiver, + ) { + let closed = recv_message(rx).await; + assert_eq!(closed.t, MessageType::TcpClosed); + assert_eq!(closed.flags, FLAG_TERMINAL); + closed.payload::().unwrap(); + wait_finished(session).await; + assert!(matches!( + rx.try_recv(), + Err(mpsc::error::TryRecvError::Empty | mpsc::error::TryRecvError::Disconnected) + )); + } + async fn wait_finished(session: &TcpSession) { tokio::time::timeout(Duration::from_secs(1), async { while !session.is_finished() { diff --git a/crates/protocol/VERSIONING.md b/crates/protocol/VERSIONING.md index 3ec33acc9..764c31f11 100644 --- a/crates/protocol/VERSIONING.md +++ b/crates/protocol/VERSIONING.md @@ -196,9 +196,9 @@ body = CBOR { v, t, p } <- ordinary control envelope Generation 9 adds attempt-scoped workload freeze/thaw for full checkpoint capture and activation. Hosts reject those operations against generation-8 agents before sending. Generation 8's released bulk-transfer contract remains unchanged; the discarded, unreleased freeze/thaw assignment to generation 8 has no compatibility shim. -The unreleased generation-9 handshake includes complete-frame transport boundaries and `core.workload.transport.credit`. The bundled host and guest use the optional Ready capability `workload_transport_barrier_version: 1`; an absent or unsupported value refuses full capture/pause before mutation. This internal contract does not change SDK framing or add a socket. Older SDK requests still use their negotiated generation; their payloads are not reinterpreted to implement the barrier. +The unreleased generation-9 handshake includes complete-frame transport boundaries and `core.workload.transport.credit`. The bundled host and guest use the optional Ready capability `workload_transport_barrier_version: 2`; an absent or unsupported value refuses full capture/pause before mutation. The superseded development contract `1` charged stdin against command capacity and is refused on full restore, not translated. This internal contract does not change SDK framing or add a socket. Older SDK requests still use their negotiated generation; their payloads are not reinterpreted to implement the barrier. -The host gates ordinary input, finishes any admitted frame, and sends its cumulative control/bulk wire-byte and frame positions through a bounded private lifecycle queue. The guest retains accepted input independently of blocked consumers, freezes workloads, and parks output at complete frames. Frozen reports the dedicated bulk output cut and absolute input grants; combined transport orders its output on the primary stream instead. Raw bulk uses the bulk input counters in either topology. Continue releases source-owned queued input only after Thawed. Restore carries the existing cumulative counters and retained input debt forward instead of granting a fresh window. Incomplete boundaries time out without authorizing capture. These required fields finalize unreleased generation 9 in place; superseded development full snapshots are refused, not translated. +The host gates ordinary input, finishes any admitted frame, and sends its cumulative control/data wire-byte and frame positions through a bounded private lifecycle queue. The existing `bulk_*` fields count logical data: raw bulk, stdin, inline filesystem/TCP payloads, and ordered EOF, regardless of physical port. Command metadata uses separate `control_*` capacity. The guest retains accepted input independently of blocked consumers, freezes workloads, and parks output at complete frames. Frozen reports the dedicated bulk output cut and absolute input grants; combined transport orders its output on the primary stream instead. Continue releases source-owned queued input only after Thawed. Restore carries the existing cumulative counters and retained input debt forward instead of granting a fresh window. Unrelated admitted metadata may bypass credit-blocked data, but per-correlation and client-disconnect ordering are retained. Incomplete boundaries time out without authorizing capture. These required fields finalize unreleased generation 9 in place; superseded development full snapshots are refused, not translated. Generation 8 adds one negotiated data-body alternative without changing the header: diff --git a/crates/protocol/lib/core.rs b/crates/protocol/lib/core.rs index b51f2c4ae..46e86bdfd 100644 --- a/crates/protocol/lib/core.rs +++ b/crates/protocol/lib/core.rs @@ -8,17 +8,20 @@ use crate::transport::{BulkTransportReady, LocalTransportReady, RelayLeaseReady} // Constants //-------------------------------------------------------------------------------------------------- -/// Complete-frame workload barrier and aggregate input-credit contract. -pub const WORKLOAD_TRANSPORT_BARRIER_VERSION: u8 = 1; -/// Maximum outstanding ordinary primary wire bytes, including frame headers. +/// Complete-frame workload barrier with logical control/data admission classes. +/// +/// Version 1 was an unreleased development contract that charged stdin to control. Its captured +/// debt cannot be reinterpreted by this contract; full restore rejects that development state. +pub const WORKLOAD_TRANSPORT_BARRIER_VERSION: u8 = 2; +/// Maximum outstanding command/control wire bytes, including frame headers. pub const WORKLOAD_TRANSPORT_CONTROL_BYTES: u64 = 8 * 1024 * 1024; -/// Maximum outstanding ordinary primary frames, including empty payloads. +/// Maximum outstanding command/control frames, excluding retained workload payloads. pub const WORKLOAD_TRANSPORT_CONTROL_FRAMES: u64 = 256; -/// Maximum outstanding bulk wire bytes, including record headers. +/// Maximum outstanding data wire bytes, including raw bulk, stdin and inline FS/TCP payloads. pub const WORKLOAD_TRANSPORT_BULK_BYTES: u64 = 32 * 1024 * 1024; -/// Maximum outstanding bulk records, including empty payloads. +/// Maximum outstanding data records/messages, including ordered empty EOF messages. /// -/// Together with primary frames, this fits the existing 512-entry guest input +/// Together with control frames, this fits the existing 512-entry guest input /// queues even when all admitted traffic targets one stalled consumer. pub const WORKLOAD_TRANSPORT_BULK_FRAMES: u64 = 256; @@ -148,14 +151,14 @@ pub struct WorkloadFrozen { /// host-queued input that has not been admitted remains source-owned. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct WorkloadTransportPosition { - /// Ordinary control wire bytes admitted, including length/header bytes. - /// Combined-port raw bulk records use `bulk_bytes`, not this counter. + /// Command/control wire bytes admitted, including length/header bytes. + /// Payload-bearing messages and raw bulk use `bulk_bytes` on either physical port. pub control_bytes: u64, - /// Ordinary control frames admitted, excluding raw bulk records. + /// Command/control frames admitted, excluding payload messages and raw bulk. pub control_frames: u64, - /// Bulk wire bytes admitted, including record headers and any incarnation prefix. + /// Data wire bytes admitted, including stdin, inline payloads, and complete raw bulk headers. pub bulk_bytes: u64, - /// Bulk records admitted. + /// Data records/messages admitted, including ordered EOF. pub bulk_frames: u64, } @@ -167,13 +170,13 @@ pub struct WorkloadTransportPosition { /// a workload consuming stdin or a network socket becoming writable. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct WorkloadTransportCredit { - /// Cumulative ordinary primary wire-byte limit. + /// Cumulative command/control wire-byte limit. pub control_bytes: u64, - /// Cumulative ordinary primary frame limit. + /// Cumulative command/control frame limit. pub control_frames: u64, - /// Cumulative bulk wire-byte limit. + /// Cumulative data wire-byte limit across both physical ports. pub bulk_bytes: u64, - /// Cumulative bulk record limit. + /// Cumulative data record/message limit across both physical ports. pub bulk_frames: u64, } diff --git a/crates/protocol/lib/message.rs b/crates/protocol/lib/message.rs index 22fa5cc24..d4ade57cb 100644 --- a/crates/protocol/lib/message.rs +++ b/crates/protocol/lib/message.rs @@ -321,6 +321,18 @@ impl Message { } impl MessageType { + /// Whether host-to-guest delivery can retain payload credit behind a workload consumer. + /// + /// The bundled workload barrier uses logical classes, not physical console ports. Payload + /// messages (including ordered EOF) share data credit with raw bulk records, leaving control + /// capacity available for fresh commands when restored stdin has not yet been consumed. + pub fn uses_workload_data_credit(self) -> bool { + matches!( + self, + Self::ExecStdin | Self::FsData | Self::TcpData | Self::TcpEof + ) + } + /// Computes the frame flags byte for this message type. pub fn flags(&self) -> u8 { match self { @@ -459,6 +471,35 @@ impl<'de> Deserialize<'de> for MessageType { mod tests { use super::*; + #[test] + fn retained_payload_and_eof_use_data_credit_without_changing_frame_flags() { + for message in [ + MessageType::ExecStdin, + MessageType::FsData, + MessageType::TcpData, + MessageType::TcpEof, + ] { + assert!(message.uses_workload_data_credit()); + assert_eq!( + message.flags(), + 0, + "logical admission must not change the wire header" + ); + } + for message in [ + MessageType::ExecRequest, + MessageType::Ping, + MessageType::FsRequest, + MessageType::TcpConnect, + MessageType::ExecSignal, + MessageType::BulkFinish, + MessageType::BulkCancel, + MessageType::RelayClientDisconnected, + ] { + assert!(!message.uses_workload_data_credit()); + } + } + #[test] fn test_message_type_roundtrip() { let types = [ diff --git a/crates/runtime/lib/checkpoint/coordinator.rs b/crates/runtime/lib/checkpoint/coordinator.rs index 1efe658da..8674d1fbe 100644 --- a/crates/runtime/lib/checkpoint/coordinator.rs +++ b/crates/runtime/lib/checkpoint/coordinator.rs @@ -2299,7 +2299,9 @@ mod tests { attempt_id: "checkpoint-42".into(), protocol_generation: 9, ready: Ready { - workload_transport_barrier_version: Some(1), + workload_transport_barrier_version: Some( + microsandbox_protocol::core::WORKLOAD_TRANSPORT_BARRIER_VERSION, + ), ..Ready::default() }, host_input: WorkloadTransportPosition { diff --git a/crates/runtime/lib/checkpoint/restore.rs b/crates/runtime/lib/checkpoint/restore.rs index 631f1404f..b56c824b0 100644 --- a/crates/runtime/lib/checkpoint/restore.rs +++ b/crates/runtime/lib/checkpoint/restore.rs @@ -8,7 +8,9 @@ use std::time::Instant; use microsandbox_image::checkpoint::{ CheckpointClosure, MemoryExtentContent, ObjectId, ResourceDescriptor, ResourceTreatment, }; -use microsandbox_protocol::core::{Ready, WorkloadTransportCredit, WorkloadTransportPosition}; +use microsandbox_protocol::core::{ + Ready, WORKLOAD_TRANSPORT_BARRIER_VERSION, WorkloadTransportCredit, WorkloadTransportPosition, +}; use microsandbox_protocol::message::{MessageType, PROTOCOL_VERSION}; use super::coordinator::TYPE_FS; @@ -422,8 +424,8 @@ fn parse_restored_agent_resource( let ready: Ready = serde_json::from_str(value("ready")?) .map_err(|error| format!("checkpoint guest readiness is invalid: {error}"))?; - if ready.workload_transport_barrier_version != Some(1) { - return Err("checkpoint guest lacks a complete-frame transport barrier".into()); + if ready.workload_transport_barrier_version != Some(WORKLOAD_TRANSPORT_BARRIER_VERSION) { + return Err("checkpoint guest has an unsupported development transport-credit contract; recreate the full snapshot with a matching build".into()); } let host_input: WorkloadTransportPosition = serde_json::from_str(value("transport_host_input")?) @@ -496,7 +498,9 @@ mod tests { boot_time_ns: 10, init_time_ns: 20, ready_time_ns: 30, - workload_transport_barrier_version: Some(1), + workload_transport_barrier_version: Some( + WORKLOAD_TRANSPORT_BARRIER_VERSION, + ), ..Default::default() }) .unwrap(), @@ -528,6 +532,20 @@ mod tests { assert!(error.contains("protocol generation 8 is unsupported")); } + #[test] + fn rejects_development_snapshot_with_stdin_charged_to_control() { + let mut resource = agent_resource(PROTOCOL_VERSION); + let mut ready: Ready = serde_json::from_str(&resource.binding["ready"]).unwrap(); + ready.workload_transport_barrier_version = Some(1); + resource + .binding + .insert("ready".into(), serde_json::to_string(&ready).unwrap()); + let error = parse_restored_agent_resource(&resource, "old-development-cut") + .err() + .unwrap(); + assert!(error.contains("unsupported development transport-credit contract")); + } + #[test] fn rejects_reconstructed_agent_resource() { let mut resource = agent_resource(PROTOCOL_VERSION); diff --git a/crates/runtime/lib/runner/clock.rs b/crates/runtime/lib/runner/clock.rs index cd695657e..cf86bba29 100644 --- a/crates/runtime/lib/runner/clock.rs +++ b/crates/runtime/lib/runner/clock.rs @@ -6,10 +6,9 @@ use bytes::Bytes; use microsandbox_protocol::codec; use microsandbox_protocol::core::ClockSync; use microsandbox_protocol::message::{Message, MessageType}; -use tokio::sync::mpsc; use tokio::task::JoinHandle; -use crate::relay::ControlWrite; +use crate::relay::{ControlWrite, ControlWriter}; use crate::{RuntimeError, RuntimeResult}; //-------------------------------------------------------------------------------------------------- @@ -31,13 +30,13 @@ const CLOCK_SYNC_WAKE_THRESHOLD: Duration = Duration::from_secs(6); /// Spawns a background task that keeps the guest wall clock aligned with the host. pub(crate) fn spawn_clock_sync_task( - agent_tx: mpsc::Sender, + agent_tx: ControlWriter, already_synchronized: bool, ) -> JoinHandle<()> { tokio::spawn(clock_sync_task(agent_tx, already_synchronized)) } -async fn clock_sync_task(agent_tx: mpsc::Sender, already_synchronized: bool) { +async fn clock_sync_task(agent_tx: ControlWriter, already_synchronized: bool) { let mut last_wall = SystemTime::now(); // Full restore completed the kernel clock barrier before workload thaw. Do not immediately // overwrite it with a queued userspace timestamp. Ordinary boot keeps its existing sync. @@ -79,14 +78,28 @@ async fn clock_sync_task(agent_tx: mpsc::Sender, already_synchroni } } -async fn send_clock_sync(agent_tx: &mpsc::Sender) -> RuntimeResult { +async fn send_clock_sync(agent_tx: &ControlWriter) -> RuntimeResult { let now = SystemTime::now(); - let elapsed = now + agent_tx + .send(ControlWrite::clock_sync()?) + .await + .map_err(|_| RuntimeError::Custom("agent relay ring writer channel closed".into()))?; + Ok(now) +} + +/// Sample only when the ordinary writer can admit the maintenance frame to the console queue. +pub(crate) fn current_clock_sync_frame() -> RuntimeResult { + let elapsed = SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .map_err(|e| RuntimeError::Custom(format!("clock sync before Unix epoch: {e}")))?; let unix_time_nanos = u64::try_from(elapsed.as_nanos()).map_err(|_| { RuntimeError::Custom("clock sync timestamp does not fit in u64 nanoseconds".into()) })?; + encode_clock_sync_frame(unix_time_nanos) +} + +/// The queue initially reserves the maximum encoded timestamp size, then sends the actual value. +pub(crate) fn encode_clock_sync_frame(unix_time_nanos: u64) -> RuntimeResult { let sync = ClockSync { unix_time_nanos }; let msg = Message::with_payload(MessageType::ClockSync, 0, &sync) .map_err(|e| RuntimeError::Custom(format!("encode clock sync: {e}")))?; @@ -94,10 +107,5 @@ async fn send_clock_sync(agent_tx: &mpsc::Sender) -> RuntimeResult let mut buf = Vec::new(); codec::encode_to_buf(&msg, &mut buf) .map_err(|e| RuntimeError::Custom(format!("encode clock sync frame: {e}")))?; - agent_tx - .send(Bytes::from(buf).into()) - .await - .map_err(|_| RuntimeError::Custom("agent relay ring writer channel closed".into()))?; - - Ok(now) + Ok(Bytes::from(buf)) } diff --git a/crates/runtime/lib/runner/relay.rs b/crates/runtime/lib/runner/relay.rs index a6910a9f1..45dd92178 100644 --- a/crates/runtime/lib/runner/relay.rs +++ b/crates/runtime/lib/runner/relay.rs @@ -32,9 +32,11 @@ use microsandbox_protocol::AGENT_RELAY_MAX_CLIENTS; use microsandbox_protocol::bulk::BulkRecord; use microsandbox_protocol::bulk::{ BULK_FLOW_MASK_GUEST_TO_HOST, BULK_HEADER_SIZE, BulkAccepted, BulkCancel, BulkCancelReason, - BulkFinish, BulkFlow, BulkKind, MAX_BULK_RECORD_PAYLOAD, + BulkCredit, BulkFinish, BulkFlow, BulkKind, MAX_BULK_RECORD_PAYLOAD, MAX_BULK_WINDOW, }; use microsandbox_protocol::codec::{self, MAX_FRAME_SIZE, MAX_WIRE_FRAME}; +#[cfg(test)] +use microsandbox_protocol::core::WORKLOAD_TRANSPORT_BARRIER_VERSION; use microsandbox_protocol::core::{ CoreError, InitAck, InitResolved, Ready, RelayClientDisconnected, WorkloadThaw, WorkloadThawed, }; @@ -133,9 +135,12 @@ const CLIENT_WRITE_BATCH_BYTES: usize = 256 * 1024; /// Maximum frame slices opportunistically coalesced in one client socket batch. const CLIENT_WRITE_BATCH_FRAMES: usize = 64; -/// At most eight generation-6 frames may wait between clients and the console. -/// Since a frame is capped at 4 MiB, this bounds the channel at 32 MiB. -const AGENT_WRITE_CHANNEL_CAPACITY: usize = 8; +/// Separate admission reserves share one FIFO. Permits survive dequeue and physical writes, so +/// credit-starved payload cannot consume the space needed by another client's metadata. +const AGENT_WRITE_CLASS_FRAMES: usize = 8; +const AGENT_WRITE_CHANNEL_CAPACITY: usize = 2 * AGENT_WRITE_CLASS_FRAMES; +const AGENT_WRITE_DATA_BYTES: usize = 32 * 1024 * 1024; +const AGENT_WRITE_CONTROL_BYTES: usize = 8 * 1024 * 1024; /// Aggregate client-to-bulk-lane bytes waiting outside the console backend. const BULK_WRITE_BYTE_CAPACITY: usize = 32 * 1024 * 1024; @@ -188,10 +193,42 @@ struct ClientState { local_outbound: Option, } -/// One ordered control-lane write, optionally acknowledged after physical ring admission. +/// One primary-lane write, optionally acknowledged after physical ring admission. pub(crate) struct ControlWrite { data: Bytes, completion: Option>, + uses_data_credit: bool, + order: ControlOrder, + admission: Option, +} + +/// A client lease transition fences its range; shutdown and unattributed internal traffic fence +/// the whole FIFO. Ordinary correlations may bypass only unrelated blocked payload. +#[derive(Clone, Copy)] +enum ControlOrder { + Correlation(u32), + MaintenanceClock, + TcpInputData(u32), + TcpInputFinish(u32), + TcpOutputCredit(u32), + ClientFence { start: u32, end: u32 }, + GlobalFence, +} + +struct ControlAdmission { + _bytes: tokio::sync::OwnedSemaphorePermit, + _frame: tokio::sync::OwnedSemaphorePermit, +} + +/// Bounded, class-reserved admission into the existing ordinary transport queue. No payload is +/// copied, and cancellation releases reservations without dropping any already accepted frame. +#[derive(Clone)] +pub(crate) struct ControlWriter { + tx: mpsc::Sender, + data_bytes: Arc, + data_frames: Arc, + control_bytes: Arc, + control_frames: Arc, } /// Every exit, including task abortion while waiting for ring capacity, wakes lifecycle waiters. @@ -464,6 +501,211 @@ struct RestoreActivationRecord<'a> { // Methods //-------------------------------------------------------------------------------------------------- +impl ControlWrite { + pub(crate) fn clock_sync() -> RuntimeResult { + // Reserve the largest CBOR integer representation. The scheduler replaces this sentinel + // before charging transport bytes, so queued time is never replayed after a long pause. + Ok(Self { + order: ControlOrder::MaintenanceClock, + ..crate::clock::encode_clock_sync_frame(u64::MAX)?.into() + }) + } + + fn ordinary(data: Bytes, id: u32, uses_data_credit: bool) -> Self { + let order = if id == 0 + || data + .get(LEN_PREFIX_SIZE + 4) + .is_some_and(|flags| flags & FLAG_SHUTDOWN != 0) + { + ControlOrder::GlobalFence + } else { + ControlOrder::Correlation(id) + }; + Self { + data, + completion: None, + uses_data_credit, + order, + admission: None, + } + } + + fn client_fence(data: Bytes, start: u32, end: u32) -> Self { + Self { + order: ControlOrder::ClientFence { start, end }, + ..data.into() + } + } + + /// Only the independent TCP return-credit flow may cross ordered input. Raw metadata was + /// already validated by the client reader; parse only the two small control payloads here. + fn classify_tcp_order( + &mut self, + id: u32, + raw: Option<(BulkKind, BulkFlow, u64, usize)>, + message: Option<&Message>, + ) { + if matches!(self.order, ControlOrder::GlobalFence) { + return; + } + if matches!(raw, Some((BulkKind::Tcp, BulkFlow::HostToGuest, _, _))) { + self.order = ControlOrder::TcpInputData(id); + return; + } + let Some(message) = message else { + return; + }; + match message.t { + MessageType::BulkFinish + if message.payload::().is_ok_and(|finish| { + finish.kind == BulkKind::Tcp && finish.flow == BulkFlow::HostToGuest + }) => + { + self.order = ControlOrder::TcpInputFinish(id); + } + MessageType::BulkCredit + if message.payload::().is_ok_and(|credit| { + credit.kind == BulkKind::Tcp + && credit.flow == BulkFlow::GuestToHost + && credit.credit_limit >= credit.consumed_offset + && credit.credit_limit - credit.consumed_offset <= MAX_BULK_WINDOW + }) => + { + self.order = ControlOrder::TcpOutputCredit(id); + } + _ => {} + } + } +} + +impl ControlOrder { + fn conflicts(self, other: Self) -> bool { + match (self, other) { + (Self::GlobalFence, _) | (_, Self::GlobalFence) => true, + (Self::MaintenanceClock, Self::MaintenanceClock) => true, + (Self::MaintenanceClock, _) | (_, Self::MaintenanceClock) => false, + (Self::ClientFence { start: a, end: b }, Self::ClientFence { start: c, end: d }) => { + a < d && c < b + } + (Self::ClientFence { start, end }, correlation) + | (correlation, Self::ClientFence { start, end }) => { + (start..end).contains(&correlation.id()) + } + // Credit advances the opposite (guest-to-host) producer, not these input bytes or + // their end marker. Reordering only the credit breaks a full-duplex credit cycle; + // input data and input finish still conflict with each other in their original FIFO. + (Self::TcpInputData(_) | Self::TcpInputFinish(_), Self::TcpOutputCredit(_)) => false, + (a, b) => a.id() == b.id(), + } + } + + fn id(self) -> u32 { + match self { + Self::Correlation(id) + | Self::TcpInputData(id) + | Self::TcpInputFinish(id) + | Self::TcpOutputCredit(id) => id, + Self::ClientFence { .. } | Self::GlobalFence | Self::MaintenanceClock => { + unreachable!("fence or maintenance order handled first") + } + } + } +} + +impl ControlWriter { + fn new() -> (Self, mpsc::Receiver) { + let (tx, rx) = mpsc::channel(AGENT_WRITE_CHANNEL_CAPACITY); + (Self::from_sender(tx), rx) + } + + fn from_sender(tx: mpsc::Sender) -> Self { + Self { + tx, + data_bytes: Arc::new(Semaphore::new(AGENT_WRITE_DATA_BYTES)), + data_frames: Arc::new(Semaphore::new(AGENT_WRITE_CLASS_FRAMES)), + control_bytes: Arc::new(Semaphore::new(AGENT_WRITE_CONTROL_BYTES)), + control_frames: Arc::new(Semaphore::new(AGENT_WRITE_CLASS_FRAMES)), + } + } + + fn budgets(&self, write: &ControlWrite) -> (&Arc, &Arc) { + if write.uses_data_credit { + (&self.data_bytes, &self.data_frames) + } else { + (&self.control_bytes, &self.control_frames) + } + } + + pub(crate) async fn send( + &self, + mut write: ControlWrite, + ) -> Result<(), mpsc::error::SendError> { + let Ok(bytes) = u32::try_from(write.data.len()) else { + return Err(mpsc::error::SendError(write)); + }; + let (byte_budget, frame_budget) = self.budgets(&write); + // Closing the receiver must also wake senders waiting for a class reservation. A canceled + // send drops its partial permits; frames already in the canonical queue retain theirs. + let reservation = async { + let frame = Arc::clone(frame_budget).acquire_owned().await.ok()?; + let bytes = Arc::clone(byte_budget) + .acquire_many_owned(bytes) + .await + .ok()?; + Some(ControlAdmission { + _bytes: bytes, + _frame: frame, + }) + }; + let admission = tokio::select! { + biased; + _ = self.tx.closed() => None, + admission = reservation => admission, + }; + let Some(admission) = admission else { + return Err(mpsc::error::SendError(write)); + }; + write.admission = Some(admission); + self.tx.send(write).await.map_err(|mut error| { + error.0.admission.take(); + error + }) + } + + fn try_send( + &self, + mut write: ControlWrite, + ) -> Result<(), mpsc::error::TrySendError> { + if self.tx.is_closed() { + return Err(mpsc::error::TrySendError::Closed(write)); + } + let Ok(bytes) = u32::try_from(write.data.len()) else { + return Err(mpsc::error::TrySendError::Full(write)); + }; + let (byte_budget, frame_budget) = self.budgets(&write); + let Ok(frame) = Arc::clone(frame_budget).try_acquire_owned() else { + return Err(mpsc::error::TrySendError::Full(write)); + }; + let Ok(bytes) = Arc::clone(byte_budget).try_acquire_many_owned(bytes) else { + return Err(mpsc::error::TrySendError::Full(write)); + }; + write.admission = Some(ControlAdmission { + _bytes: bytes, + _frame: frame, + }); + self.tx.try_send(write).map_err(|error| match error { + mpsc::error::TrySendError::Full(mut write) => { + write.admission.take(); + mpsc::error::TrySendError::Full(write) + } + mpsc::error::TrySendError::Closed(mut write) => { + write.admission.take(); + mpsc::error::TrySendError::Closed(write) + } + }) + } +} + impl GuestFrameMerger { /// Register an opening operation before its request can reach agentd. fn register(&mut self, incarnation: ClientIncarnation, id: u32) -> RuntimeResult<()> { @@ -1507,7 +1749,7 @@ impl AgentRelay { // Bounded channel for client reader tasks to send frames to the ring writer. // Backpressure prevents unbounded memory growth from client floods. - let (agent_tx, agent_rx) = mpsc::channel::(AGENT_WRITE_CHANNEL_CAPACITY); + let (agent_tx, agent_rx) = ControlWriter::new(); self.shared .workload_control .register_ordinary_writer(agent_tx.clone()); @@ -1922,9 +2164,13 @@ impl Drop for AgentRelay { impl From for ControlWrite { fn from(data: Bytes) -> Self { + let uses_data_credit = data.get(LEN_PREFIX_SIZE + 4) == Some(&FLAG_BULK); Self { data, completion: None, + uses_data_credit, + order: ControlOrder::GlobalFence, + admission: None, } } } @@ -2038,15 +2284,15 @@ pub(crate) fn push_guest_frame_until( /// without blocking Tokio's channel APIs or bypassing a frozen/credit-starved FIFO head. fn push_ordered_guest_frame_until( shared: &ConsoleSharedState, - writer: &mpsc::Sender, + writer: &ControlWriter, data: Bytes, timeout: std::time::Duration, ) -> RuntimeResult<()> { let deadline = Instant::now() + timeout; let (completion, mut completed) = oneshot::channel(); let mut pending = Some(ControlWrite { - data, completion: Some(completion), + ..data.into() }); loop { if let Some(write) = pending.take() { @@ -2440,7 +2686,8 @@ async fn ring_writer_task( let workload = &shared.workload_control; let mut private = workload.start(); - let mut pending: Option = None; + let mut pending = VecDeque::with_capacity(AGENT_WRITE_CHANNEL_CAPACITY); + let mut ordinary_closed = false; loop { let changed = workload.changed.notified(); tokio::pin!(changed); @@ -2451,29 +2698,35 @@ async fn ring_writer_task( if workload.gated() { workload.park(false); } - // Keep the exact FIFO head (including its completion and Bytes owner) when admission is - // gated or credit-starved. Only the trusted, bounded lifecycle mailbox bypasses it. + // Moving frames into the bounded scheduler does not release their class admission. A full + // data class therefore cannot hide another client's metadata in the canonical mailbox. + while pending.len() < AGENT_WRITE_CHANNEL_CAPACITY && !ordinary_closed { + match rx.try_recv() { + Ok(write) => pending.push_back(write), + Err(mpsc::error::TryRecvError::Empty) => break, + Err(mpsc::error::TryRecvError::Disconnected) => ordinary_closed = true, + } + } + let mut wait_clock_capacity = false; let write = if let Ok(write) = private.try_recv() { Some(ControlWrite::from(write.0)) - } else if let Some(write) = pending.as_ref() { - if workload - .admit( - write.data.get(LEN_PREFIX_SIZE + 4) == Some(&FLAG_BULK), - write.data.len(), - ) - .map_err(RuntimeError::Custom)? - { - pending.take() - } else { - None - } } else { - None + let (write, wait_capacity) = + select_control_write(&mut pending, workload, Some(&shared)) + .map_err(RuntimeError::Custom)?; + wait_clock_capacity = wait_capacity; + write }; if let Some(write) = write { + let ControlWrite { + data, + completion, + admission, + .. + } = write; if !push_bulk_fragment( &shared, - write.data, + data, #[cfg(unix)] &capacity_fd, ) @@ -2482,12 +2735,16 @@ async fn ring_writer_task( workload.close(); return Err(RuntimeError::Custom("agent console writer closed".into())); } - if let Some(completion) = write.completion { + if let Some(completion) = completion { let _ = completion.send(()); shared.rx_capacity_wake.wake(); } + drop(admission); continue; } + if ordinary_closed && pending.is_empty() { + break; + } tokio::select! { biased; write = private.recv() => { @@ -2498,9 +2755,17 @@ async fn ring_writer_task( } } _ = &mut changed => {} - write = rx.recv(), if pending.is_none() && !workload.gated() => { - let Some(write) = write else { break; }; - pending = Some(write); + available = wait_console_capacity(&shared, #[cfg(unix)] &capacity_fd), if wait_clock_capacity => { + if !available { + return Err(RuntimeError::Custom("agent console capacity watcher closed".into())); + } + } + write = rx.recv(), if pending.len() < AGENT_WRITE_CHANNEL_CAPACITY && !ordinary_closed => { + if let Some(write) = write { + pending.push_back(write); + } else { + ordinary_closed = true; + } } } } @@ -2509,6 +2774,101 @@ async fn ring_writer_task( Ok(()) } +/// Keep the common FIFO path constant-time. Only a credit-blocked payload head enables a bounded +/// scan for unrelated metadata or independent TCP return credit. Input/finish order and all +/// cancellation, opening, lease and global fences remain intact. +fn select_control_write( + pending: &mut VecDeque, + workload: &WorkloadControl, + shared: Option<&ConsoleSharedState>, +) -> Result<(Option, bool), String> { + let mut wait_capacity = false; + let Some(head) = pending.front_mut() else { + return Ok((None, false)); + }; + if admit_control_write( + head, + workload, + shared, + &mut wait_capacity, + crate::clock::current_clock_sync_frame, + )? { + return Ok((pending.pop_front(), wait_capacity)); + } + if !head.uses_data_credit || workload.gated() { + return Ok((None, wait_capacity)); + } + for index in 1..pending.len() { + let candidate = &pending[index]; + if candidate.uses_data_credit + || pending + .iter() + .take(index) + .any(|earlier| earlier.order.conflicts(candidate.order)) + { + continue; + } + if admit_control_write( + &mut pending[index], + workload, + shared, + &mut wait_capacity, + crate::clock::current_clock_sync_frame, + )? { + return Ok((pending.remove(index), wait_capacity)); + } + } + Ok((None, wait_capacity)) +} + +fn admit_control_write( + write: &mut ControlWrite, + workload: &WorkloadControl, + shared: Option<&ConsoleSharedState>, + wait_capacity: &mut bool, + clock_frame: impl FnOnce() -> RuntimeResult, +) -> Result { + if matches!(write.order, ControlOrder::MaintenanceClock) { + // No transport credit has been charged yet. Refresh before each capacity attempt, including + // retries after pause, and charge the actual CBOR length rather than the reserved maximum. + write.data = clock_frame().map_err(|error| error.to_string())?; + if let Some(shared) = shared { + shared.rx_capacity_wake.drain(); + if !shared.rx_ring.can_fit(write.data.len()) { + *wait_capacity = !workload.gated(); + return Ok(false); + } + // This is the sole post-Ready producer. No await separates this successful capacity + // check, transport admission and the atomic whole-frame queue push, so a clock cannot + // acquire a timestamp and then sleep waiting for physical queue capacity. + } + } + workload.admit(write.uses_data_credit, write.data.len()) +} + +async fn wait_console_capacity( + shared: &Arc, + #[cfg(unix)] capacity_fd: &AsyncFd, +) -> bool { + #[cfg(unix)] + { + let _ = shared; + let Ok(mut ready) = capacity_fd.readable().await else { + return false; + }; + ready.clear_ready(); + true + } + #[cfg(windows)] + { + // This select branch is cancelable. A blocking wake waiter would survive cancellation + // and accumulate across other traffic; only a pending, ring-blocked maintenance clock + // needs this bounded retry on platforms without the Unix readiness adapter. + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + !shared.is_closed() + } +} + /// Apply deficit round robin before admitting client raw records to the bulk console ring. async fn bulk_ring_writer_task( shared: Arc, @@ -3649,7 +4009,7 @@ fn queue_client_rejection( async fn client_reader_task( slot: u32, mut reader: impl AsyncRead + Unpin + Send + 'static, - agent_tx: mpsc::Sender, + agent_tx: ControlWriter, clients: Arc>>, used_slots: Arc>>, drain_tx: mpsc::Sender<()>, @@ -4146,9 +4506,37 @@ async fn client_reader_task( tracing::error!("agent relay: bulk ring writer channel closed"); break; } - } else if agent_tx.send(frame.data.into()).await.is_err() { - tracing::error!("agent relay: control ring writer channel closed"); - break; + } else { + #[cfg(unix)] + let data = if let Some(record) = shared_bulk.take() { + // Combined transport has one owned, ordered frame queue rather than the dual + // port's split header/payload writer. Materialize a bounded standard raw frame + // before releasing its arena slot; the queue then retains this copy through the + // complete physical write. The dual-port zero-copy path above is unchanged. + let mut encoded = Vec::with_capacity( + LEN_PREFIX_SIZE + FRAME_HEADER_SIZE + BULK_HEADER_SIZE + record.payload.len(), + ); + if let Err(error) = codec::encode_bulk_to_buf(&record, &mut encoded) { + tracing::error!(%error, "agent relay: encode combined shared bulk frame failed"); + break; + } + Bytes::from(encoded) + } else { + frame.data + }; + #[cfg(not(unix))] + let data = frame.data; + let mut write = ControlWrite::ordinary( + data, + frame.id, + frame.flags == FLAG_BULK + || message_type.is_some_and(MessageType::uses_workload_data_credit), + ); + write.classify_tcp_order(frame.id, bulk_metadata, decoded_message.as_ref()); + if agent_tx.send(write).await.is_err() { + tracing::error!("agent relay: control ring writer channel closed"); + break; + } } } @@ -4234,7 +4622,11 @@ async fn client_reader_task( continue; } - if agent_tx.send(Bytes::from(buf).into()).await.is_err() { + if agent_tx + .send(ControlWrite::ordinary(Bytes::from(buf), session_id, false)) + .await + .is_err() + { tracing::error!("agent relay: ring writer channel closed during cleanup"); break; } @@ -4304,7 +4696,7 @@ async fn random_unused_client_incarnation( /// Establish one dual-port range owner before its SDK connection becomes usable. async fn send_relay_client_connected( - agent_tx: &mpsc::Sender, + agent_tx: &ControlWriter, id_start: u32, id_end_exclusive: u32, incarnation: ClientIncarnation, @@ -4315,14 +4707,18 @@ async fn send_relay_client_connected( incarnation, }); agent_tx - .send(Bytes::copy_from_slice(&frame).into()) + .send(ControlWrite::client_fence( + Bytes::copy_from_slice(&frame), + id_start, + id_end_exclusive, + )) .await .map_err(|_| RuntimeError::Custom("agent control writer stopped".into())) } /// Send cleanup and, in dual-port mode, register the reverse-lane drain acknowledgement first. async fn begin_relay_client_disconnect( - agent_tx: &mpsc::Sender, + agent_tx: &ControlWriter, pending_disconnects: &Arc>>, id_start: u32, id_end_exclusive: u32, @@ -4384,42 +4780,37 @@ async fn complete_relay_client_disconnect( /// Remove exactly the range owner that disconnected, preserving combined-mode compatibility. async fn send_relay_client_disconnected( - agent_tx: &mpsc::Sender, + agent_tx: &ControlWriter, id_start: u32, id_end_exclusive: u32, incarnation: Option, ) -> RuntimeResult<()> { - send_relay_lifecycle( - agent_tx, + let message = Message::with_payload( MessageType::RelayClientDisconnected, + 0, &RelayClientDisconnected { id_start, id_end_exclusive, incarnation, }, ) - .await -} - -async fn send_relay_lifecycle( - agent_tx: &mpsc::Sender, - message_type: MessageType, - payload: &T, -) -> RuntimeResult<()> { - let message = Message::with_payload(message_type, 0, payload) - .map_err(|error| RuntimeError::Custom(format!("encode relay lifecycle: {error}")))?; + .map_err(|error| RuntimeError::Custom(format!("encode relay lifecycle: {error}")))?; let mut frame = Vec::new(); codec::encode_to_buf(&message, &mut frame) .map_err(|error| RuntimeError::Custom(format!("encode relay lifecycle frame: {error}")))?; agent_tx - .send(Bytes::from(frame).into()) + .send(ControlWrite::client_fence( + Bytes::from(frame), + id_start, + id_end_exclusive, + )) .await .map_err(|_| RuntimeError::Custom("agent control writer stopped".into())) } /// Publish typed cancellation for every active raw-bulk operation while control is still usable. async fn handle_relay_transport_failure( - agent_tx: &mpsc::Sender, + agent_tx: &ControlWriter, merge_command_tx: &mpsc::Sender, clients: &Arc>>, wait_for_terminals: bool, @@ -4477,8 +4868,8 @@ async fn handle_relay_transport_failure( let (completion, completed) = oneshot::channel(); agent_tx .send(ControlWrite { - data: Bytes::from(frame), completion: Some(completion), + ..ControlWrite::ordinary(Bytes::from(frame), id, false) }) .await .map_err(|_| RuntimeError::Custom("agent control writer stopped".into()))?; @@ -4611,6 +5002,13 @@ mod tests { use super::*; + fn next_control_write( + pending: &mut VecDeque, + workload: &WorkloadControl, + ) -> Result, String> { + select_control_write(pending, workload, None).map(|(write, _)| write) + } + #[cfg(unix)] use microsandbox_agent_client::local_shm::{ LocalShmClient, LocalShmUpgrade, local_upgrade_request_frame, receive_local_shm_upgrade, @@ -4822,6 +5220,17 @@ mod tests { #[cfg(unix)] #[tokio::test] async fn client_shared_descriptor_reaches_bulk_scheduler_without_socket_payload() { + exercise_shared_descriptor_input(true).await; + } + + #[cfg(unix)] + #[tokio::test] + async fn client_shared_descriptor_combined_preserves_frame_credit_and_arena_release() { + exercise_shared_descriptor_input(false).await; + } + + #[cfg(unix)] + async fn exercise_shared_descriptor_input(dual_port: bool) { let (mut client_socket, server_socket) = tokio::net::UnixStream::pair().unwrap(); let ancillary_fd = server_socket.as_fd().try_clone_to_owned().unwrap(); let (server_reader, server_writer) = tokio::io::split(server_socket); @@ -4853,7 +5262,23 @@ mod tests { local_write_rx, ancillary_fd, )); - let (agent_tx, _agent_rx) = mpsc::channel(1); + let (agent_tx, agent_rx) = ControlWriter::new(); + let queue_budget = agent_tx.clone(); + let expected_len = LEN_PREFIX_SIZE + + FRAME_HEADER_SIZE + + BULK_HEADER_SIZE + + MAX_BULK_RECORD_PAYLOAD as usize; + let shared = workload_test_shared(expected_len, false); + if !dual_port { + // Queue entries are whole owned frames. Occupy some capacity so the next full-size + // frame must wait, without configuring a queue too small to ever admit that frame. + shared + .rx_ring + .push(Bytes::from_static(b"occupied")) + .unwrap(); + } + let ring_writer = + (!dual_port).then(|| tokio::spawn(ring_writer_task(Arc::clone(&shared), agent_rx))); let used_slots = Arc::new(Mutex::new(HashSet::from([0]))); let (drain_tx, _drain_rx) = mpsc::channel(1); let (bulk_tx, mut bulk_rx) = mpsc::channel(1); @@ -4868,8 +5293,8 @@ mod tests { drain_tx, Arc::new(std::sync::Mutex::new(HashMap::new())), Arc::new(AtomicU64::new(1)), - Some(bulk_tx), - Some(Arc::new(Semaphore::new(BULK_WRITE_BYTE_CAPACITY))), + dual_port.then_some(bulk_tx), + dual_port.then(|| Arc::new(Semaphore::new(BULK_WRITE_BYTE_CAPACITY))), merge_tx, pending_disconnects, 1, @@ -4897,27 +5322,117 @@ mod tests { kind: BulkKind::Filesystem, flow: BulkFlow::HostToGuest, offset: 17, - payload: Bytes::from_static(b"arena payload"), + payload: if dual_port { + Bytes::from_static(b"arena payload") + } else { + Bytes::from(vec![0x53; MAX_BULK_RECORD_PAYLOAD as usize]) + }, }; let mut prepared = local.outbound.try_prepare(&record).unwrap(); - let wire = encode_local_bulk_ref(prepared.descriptor()).unwrap(); + let descriptor = prepared.descriptor(); + let wire = encode_local_bulk_ref(descriptor).unwrap(); client_socket.write_all(&wire).await.unwrap(); prepared.commit(); - let command = tokio::time::timeout(Duration::from_secs(1), bulk_rx.recv()) + if dual_port { + let command = tokio::time::timeout(Duration::from_secs(1), bulk_rx.recv()) + .await + .unwrap() + .unwrap(); + let BulkWriterCommand::Write(write) = command else { + panic!("shared record did not enter the bulk scheduler"); + }; + let BulkWriteData::Shared { payload, .. } = write.data else { + panic!("dual-port runtime rebuilt shared input as an in-band socket frame"); + }; + assert_eq!(payload, record.payload); + } else { + // The arena can be released once the fallback owns its copy, even while the console + // queue cannot admit that frame. Reusing the slot must not change the owned copy. + let release = tokio::time::timeout( + Duration::from_secs(1), + codec::read_raw_frame(&mut client_socket), + ) .await .unwrap() .unwrap(); - let BulkWriterCommand::Write(write) = command else { - panic!("shared record did not enter the bulk scheduler"); - }; - let BulkWriteData::Shared { payload, .. } = write.data else { - panic!("runtime rebuilt shared input as an in-band socket frame"); - }; - assert_eq!(payload, record.payload); + let LocalShmFrame::BulkRelease(release) = decode_local_body(&release.body).unwrap() + else { + panic!("copied combined input did not release its arena slot"); + }; + assert_eq!(release.slot, descriptor.slot); + assert_eq!(release.generation, descriptor.generation); + local.outbound.release(release).unwrap(); + let replacement = BulkRecord { + payload: Bytes::from(vec![0xa7; record.payload.len()]), + ..record.clone() + }; + let _replacement = local.outbound.try_prepare(&replacement).unwrap(); + + tokio::time::timeout(Duration::from_secs(1), async { + while shared.rx_ring.snapshot().full_events == 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!( + queue_budget.data_bytes.available_permits(), + AGENT_WRITE_DATA_BYTES - expected_len + ); + assert_eq!( + queue_budget.data_frames.available_permits(), + AGENT_WRITE_CLASS_FRAMES - 1 + ); + assert_eq!( + queue_budget.control_bytes.available_permits(), + AGENT_WRITE_CONTROL_BYTES + ); + assert_eq!( + queue_budget.control_frames.available_permits(), + AGENT_WRITE_CLASS_FRAMES + ); + let gate = shared.workload_control.gate(); + assert_eq!(next_host_fragment(&shared).await.as_ref(), b"occupied"); + let mut wire = BytesMut::from(next_host_fragment(&shared).await.as_ref()); + assert_eq!(wire.len(), expected_len); + let position = tokio::time::timeout( + Duration::from_secs(1), + shared.workload_control.parked_position(), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(position.bulk_bytes, expected_len as u64); + assert_eq!(position.bulk_frames, 1); + assert_eq!(position.control_frames, 0); + assert_eq!( + queue_budget.data_bytes.available_permits(), + AGENT_WRITE_DATA_BYTES + ); + assert_eq!( + queue_budget.data_frames.available_permits(), + AGENT_WRITE_CLASS_FRAMES + ); + let Some(codec::DecodedFrame::Bulk(received)) = + codec::try_decode_frame_from_bytes(&mut wire).unwrap() + else { + panic!("combined descriptor did not produce one complete raw frame"); + }; + assert_eq!(received, record); + assert!(wire.is_empty()); + assert!(shared.rx_ring.pop().is_none()); + gate.release(); + } reader.abort(); writer.abort(); + if let Some(ring_writer) = ring_writer { + ring_writer.abort(); + let _ = ring_writer.await; + } + let _ = reader.await; + let _ = writer.await; } fn lane_frame(bytes: Vec, budget: &Arc) -> LaneFrame { @@ -5005,8 +5520,8 @@ mod tests { let task = tokio::spawn(ring_writer_task(Arc::clone(&shared), rx)); let (completion, completed) = oneshot::channel(); tx.send(ControlWrite { - data: Bytes::from_static(b"control frame"), completion: Some(completion), + ..Bytes::from_static(b"control frame").into() }) .await .unwrap(); @@ -5023,6 +5538,7 @@ mod tests { let id_start = 1; let id_end_exclusive = AGENT_RELAY_ID_RANGE_STEP; let (agent_tx, mut agent_rx) = mpsc::channel(1); + let agent_tx = ControlWriter::from_sender(agent_tx); let pending = Arc::new(Mutex::new(HashMap::new())); let mut completion = begin_relay_client_disconnect( @@ -5067,6 +5583,7 @@ mod tests { let (id_start, id_end_exclusive) = relay_client_id_range(slot).unwrap(); let (reader, mut peer) = tokio::io::duplex(4096); let (agent_tx, mut agent_rx) = mpsc::channel(4); + let agent_tx = ControlWriter::from_sender(agent_tx); let (write_tx, mut write_rx) = mpsc::unbounded_channel(); #[cfg(unix)] let (local_write_tx, _local_write_rx) = mpsc::unbounded_channel(); @@ -5140,6 +5657,39 @@ mod tests { drop(rejected); assert_eq!(write_budget.available_permits(), initial_budget); } + // Existing streams still enter bounded source-owned admission while paused. Classification + // reuses this reader's already decoded envelope, including empty stdin/TCP EOF payloads. + for (kind, uses_data_credit) in [ + (MessageType::ExecStdin, true), + (MessageType::FsData, true), + (MessageType::TcpData, true), + (MessageType::TcpEof, true), + (MessageType::BulkFinish, false), + (MessageType::BulkCredit, false), + (MessageType::Ping, false), + ] { + let wire = encoded_message_id(kind, id_start, &serde_json::json!({})); + peer.write_all(&wire).await.unwrap(); + let admitted = tokio::time::timeout(Duration::from_secs(1), agent_rx.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(admitted.data.as_ref(), wire); + assert_eq!(admitted.uses_data_credit, uses_data_credit, "{kind:?}"); + assert!(matches!(admitted.order, ControlOrder::Correlation(id) if id == id_start)); + } + active_bulk + .lock() + .unwrap() + .insert(id_start, BulkKind::Filesystem); + let raw = encoded_host_raw(id_start, 0, b"raw input"); + peer.write_all(&raw).await.unwrap(); + let admitted = tokio::time::timeout(Duration::from_secs(1), agent_rx.recv()) + .await + .unwrap() + .unwrap(); + assert!(admitted.uses_data_credit); + assert_eq!(admitted.data.as_ref(), raw); task.abort(); let _ = task.await; } @@ -5152,6 +5702,7 @@ mod tests { let (reader, peer) = tokio::io::duplex(64); drop(peer); let (agent_tx, mut agent_rx) = mpsc::channel(4); + let agent_tx = ControlWriter::from_sender(agent_tx); let (write_tx, _write_rx) = mpsc::unbounded_channel(); #[cfg(unix)] let (local_write_tx, _local_write_rx) = mpsc::unbounded_channel(); @@ -6020,7 +6571,7 @@ mod tests { boot_time_ns: 0, init_time_ns: 0, ready_time_ns: 0, - workload_transport_barrier_version: Some(1), + workload_transport_barrier_version: Some(WORKLOAD_TRANSPORT_BARRIER_VERSION), ..Default::default() }, ); @@ -6233,7 +6784,7 @@ mod tests { init_time_ns: 22, ready_time_ns: 33, agent_version: "test-agent".into(), - workload_transport_barrier_version: Some(1), + workload_transport_barrier_version: Some(WORKLOAD_TRANSPORT_BARRIER_VERSION), ..Default::default() }, attempt_id: attempt_id.into(), @@ -6580,7 +7131,7 @@ mod tests { shared.workload_control.install_ready( 9, Ready { - workload_transport_barrier_version: Some(1), + workload_transport_barrier_version: Some(WORKLOAD_TRANSPORT_BARRIER_VERSION), ..Default::default() }, dual_port, @@ -6604,39 +7155,770 @@ mod tests { } #[tokio::test] - async fn workload_gate_keeps_source_fifo_until_confirmed_continue() { + async fn workload_restored_payload_debt_allows_fresh_lease_and_exec() { use microsandbox_protocol::core::{ - WorkloadFreeze, WorkloadFrozen, WorkloadTransportCredit, WorkloadTransportPosition, + WORKLOAD_TRANSPORT_BULK_BYTES, WORKLOAD_TRANSPORT_BULK_FRAMES, + WORKLOAD_TRANSPORT_CONTROL_BYTES, WORKLOAD_TRANSPORT_CONTROL_FRAMES, + WorkloadTransportCredit, WorkloadTransportPosition, }; let shared = workload_test_shared(4096, false); let control = Arc::clone(&shared.workload_control); - // Model a live guest whose previously admitted input still occupies the whole window. - control - .restore( - WorkloadTransportPosition::default(), - WorkloadTransportCredit::default(), - 0, - ) - .unwrap(); - let (tx, rx) = mpsc::channel(2); - let writer = tokio::spawn(ring_writer_task(Arc::clone(&shared), rx)); - let first = Bytes::from(encoded_message_id( - MessageType::Ping, + // The inherited stdin remains unconsumed in guest RAM. Its cumulative debt must not be + // forgiven just to make a new lease/exec usable on the restored host's empty queue. + let position = WorkloadTransportPosition { + bulk_bytes: WORKLOAD_TRANSPORT_BULK_BYTES, + bulk_frames: WORKLOAD_TRANSPORT_BULK_FRAMES, + ..Default::default() + }; + let mut credit = WorkloadTransportCredit { + control_bytes: WORKLOAD_TRANSPORT_CONTROL_BYTES, + control_frames: WORKLOAD_TRANSPORT_CONTROL_FRAMES, + bulk_bytes: position.bulk_bytes, + bulk_frames: position.bulk_frames, + }; + control.restore(position, credit, 0).unwrap(); + let (tx, rx) = ControlWriter::new(); + let stdin = Bytes::from(encoded_message_id( + MessageType::ExecStdin, 1, - µsandbox_protocol::core::Ping {}, + µsandbox_protocol::exec::ExecStdin { + data: b"retained".to_vec(), + }, )); - let second = Bytes::from(encoded_message_id( - MessageType::Ping, - 2, - µsandbox_protocol::core::Ping {}, + let eof = Bytes::from(encoded_message_id( + MessageType::ExecStdin, + 1, + µsandbox_protocol::exec::ExecStdin { data: Vec::new() }, )); - let (done, mut completed) = oneshot::channel(); - tx.send(ControlWrite { - data: first.clone(), - completion: Some(done), - }) - .await - .unwrap(); + tx.send(ControlWrite::ordinary(stdin.clone(), 1, true)) + .await + .unwrap(); + tx.send(ControlWrite::ordinary(eof.clone(), 1, true)) + .await + .unwrap(); + let finish = Bytes::from(encoded_message_id(MessageType::BulkFinish, 1, &())); + tx.send(ControlWrite::ordinary(finish.clone(), 1, false)) + .await + .unwrap(); + send_relay_client_connected(&tx, 100, 200, TEST_INCARNATION) + .await + .unwrap(); + let exec = Bytes::from(encoded_message_id(MessageType::ExecRequest, 100, &())); + tx.send(ControlWrite::ordinary(exec.clone(), 100, false)) + .await + .unwrap(); + let writer = tokio::spawn(ring_writer_task(Arc::clone(&shared), rx)); + assert_eq!( + next_host_fragment(&shared).await.as_ref(), + encode_relay_client_connected(RelayClientConnected { + id_start: 100, + id_end_exclusive: 200, + incarnation: TEST_INCARNATION, + }) + ); + assert_eq!(next_host_fragment(&shared).await, exec); + assert!(shared.rx_ring.pop().is_none()); + let gate = control.gate(); + let parked = control.parked_position().await.unwrap(); + assert_eq!(parked.bulk_bytes, position.bulk_bytes); + assert_eq!(parked.bulk_frames, position.bulk_frames); + credit.bulk_bytes += (stdin.len() + eof.len()) as u64; + credit.bulk_frames += 2; + control.update_credit(credit).unwrap(); + assert!(shared.rx_ring.pop().is_none()); + gate.release(); + // Same-correlation metadata remains behind both bytes and EOF despite spare control credit. + assert_eq!(next_host_fragment(&shared).await, stdin); + assert_eq!(next_host_fragment(&shared).await, eof); + assert_eq!(next_host_fragment(&shared).await, finish); + drop(tx); + writer.await.unwrap().unwrap(); + } + + #[test] + fn workload_metadata_scheduler_preserves_client_and_global_fences() { + use microsandbox_protocol::core::{WorkloadTransportCredit, WorkloadTransportPosition}; + let shared = workload_test_shared(4096, false); + let control = &shared.workload_control; + control + .restore( + WorkloadTransportPosition::default(), + WorkloadTransportCredit { + control_bytes: 4096, + control_frames: 16, + ..Default::default() + }, + 0, + ) + .unwrap(); + let frame = Bytes::from_static(b"fence"); + let mut pending = VecDeque::from([ + ControlWrite::ordinary(frame.clone(), 101, true), + ControlWrite::client_fence(frame.clone(), 100, 200), + ControlWrite::ordinary(frame.clone(), 102, false), + ControlWrite::ordinary(frame.clone(), 201, false), + ControlWrite::from(frame.clone()), + ControlWrite::ordinary(frame, 301, false), + ]); + assert!(matches!( + next_control_write(&mut pending, control) + .unwrap() + .unwrap() + .order, + ControlOrder::Correlation(201) + )); + assert!(next_control_write(&mut pending, control).unwrap().is_none()); + assert_eq!(pending.len(), 5); + } + + #[tokio::test] + async fn workload_maintenance_clock_and_cleanup_do_not_fence_unrelated_clients() { + use microsandbox_protocol::core::{ + ClockSync, WorkloadTransportCredit, WorkloadTransportPosition, + }; + let shared = workload_test_shared(4096, false); + let control = &shared.workload_control; + let mut credit = WorkloadTransportCredit { + control_bytes: 4096, + control_frames: 16, + ..Default::default() + }; + control + .restore(WorkloadTransportPosition::default(), credit, 0) + .unwrap(); + let (tx, rx) = ControlWriter::new(); + let stdin = Bytes::from(encoded_message_id(MessageType::ExecStdin, 1, &())); + let kill = Bytes::from(encoded_message_id( + MessageType::ExecSignal, + 101, + &ExecSignal { signal: 9 }, + )); + let same_flow_kill = Bytes::from(encoded_message_id( + MessageType::ExecSignal, + 1, + &ExecSignal { signal: 9 }, + )); + let exec = Bytes::from(encoded_message_id(MessageType::ExecRequest, 201, &())); + tx.send(ControlWrite::ordinary(stdin.clone(), 1, true)) + .await + .unwrap(); + tx.send(ControlWrite::clock_sync().unwrap()).await.unwrap(); + tx.send(ControlWrite::ordinary(kill.clone(), 101, false)) + .await + .unwrap(); + tx.send(ControlWrite::ordinary(same_flow_kill.clone(), 1, false)) + .await + .unwrap(); + send_relay_client_connected(&tx, 200, 300, TEST_INCARNATION) + .await + .unwrap(); + tx.send(ControlWrite::ordinary(exec.clone(), 201, false)) + .await + .unwrap(); + tx.send(ControlWrite::clock_sync().unwrap()).await.unwrap(); + let writer = tokio::spawn(ring_writer_task(Arc::clone(&shared), rx)); + + let first_clock = decode_frame(&next_host_fragment(&shared).await).unwrap(); + assert_eq!(first_clock.t, MessageType::ClockSync); + assert_eq!(next_host_fragment(&shared).await, kill); + assert_eq!( + next_host_fragment(&shared).await.as_ref(), + encode_relay_client_connected(RelayClientConnected { + id_start: 200, + id_end_exclusive: 300, + incarnation: TEST_INCARNATION, + }) + ); + assert_eq!(next_host_fragment(&shared).await, exec); + let second_clock = decode_frame(&next_host_fragment(&shared).await).unwrap(); + assert_eq!(second_clock.t, MessageType::ClockSync); + assert!( + second_clock.payload::().unwrap().unix_time_nanos + >= first_clock.payload::().unwrap().unix_time_nanos + ); + let gate = control.gate(); + let position = control.parked_position().await.unwrap(); + assert_eq!( + position.bulk_frames, 0, + "no blocked data was discarded or admitted" + ); + assert!(shared.rx_ring.pop().is_none()); + credit.bulk_bytes = stdin.len() as u64; + credit.bulk_frames = 1; + control.update_credit(credit).unwrap(); + gate.release(); + assert_eq!(next_host_fragment(&shared).await, stdin); + assert_eq!(next_host_fragment(&shared).await, same_flow_kill); + drop(tx); + writer.await.unwrap().unwrap(); + } + + #[tokio::test] + async fn workload_maintenance_clock_stays_behind_global_lifecycle_fence() { + use microsandbox_protocol::core::{WorkloadTransportCredit, WorkloadTransportPosition}; + let shared = workload_test_shared(4096, false); + let control = &shared.workload_control; + let mut credit = WorkloadTransportCredit { + control_bytes: 4096, + control_frames: 16, + ..Default::default() + }; + control + .restore(WorkloadTransportPosition::default(), credit, 0) + .unwrap(); + let (tx, rx) = ControlWriter::new(); + let stdin = Bytes::from(encoded_message_id(MessageType::ExecStdin, 1, &())); + let shutdown = Bytes::from(encoded_message_id(MessageType::Shutdown, 0, &())); + tx.send(ControlWrite::ordinary(stdin.clone(), 1, true)) + .await + .unwrap(); + tx.send(shutdown.clone().into()).await.unwrap(); + tx.send(ControlWrite::clock_sync().unwrap()).await.unwrap(); + let writer = tokio::spawn(ring_writer_task(Arc::clone(&shared), rx)); + tokio::time::sleep(Duration::from_millis(10)).await; + assert!(shared.rx_ring.pop().is_none()); + let gate = control.gate(); + assert_eq!( + control.parked_position().await.unwrap(), + WorkloadTransportPosition::default() + ); + credit.bulk_bytes = stdin.len() as u64; + credit.bulk_frames = 1; + control.update_credit(credit).unwrap(); + assert!( + shared.rx_ring.pop().is_none(), + "pause still gates maintenance" + ); + gate.release(); + assert_eq!(next_host_fragment(&shared).await, stdin); + assert_eq!(next_host_fragment(&shared).await, shutdown); + assert_eq!( + decode_frame(&next_host_fragment(&shared).await).unwrap().t, + MessageType::ClockSync + ); + drop(tx); + writer.await.unwrap().unwrap(); + } + + #[tokio::test] + async fn workload_clock_refreshes_after_full_ring_and_pause_without_charging_stale_bytes() { + use microsandbox_protocol::core::{ClockSync, WorkloadTransportPosition}; + let shared = workload_test_shared(4096, false); + shared.rx_ring.push(Bytes::from(vec![0; 4096])).unwrap(); + let (tx, rx) = ControlWriter::new(); + tx.send(ControlWrite::clock_sync().unwrap()).await.unwrap(); + let writer = tokio::spawn(ring_writer_task(Arc::clone(&shared), rx)); + tokio::time::sleep(Duration::from_millis(20)).await; + let gate = shared.workload_control.gate(); + assert_eq!( + shared.workload_control.parked_position().await.unwrap(), + WorkloadTransportPosition::default() + ); + assert_eq!(next_host_fragment(&shared).await.len(), 4096); + tokio::time::sleep(Duration::from_millis(20)).await; + assert!(shared.rx_ring.pop().is_none()); + let not_before = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() as u64; + gate.release(); + let wire = next_host_fragment(&shared).await; + let clock = decode_frame(&wire).unwrap().payload::().unwrap(); + assert!( + clock.unix_time_nanos >= not_before, + "queued clock age must not cross pause" + ); + let gate = shared.workload_control.gate(); + let position = shared.workload_control.parked_position().await.unwrap(); + assert_eq!(position.control_bytes, wire.len() as u64); + assert_eq!(position.control_frames, 1); + assert_eq!( + tx.control_bytes.available_permits(), + AGENT_WRITE_CONTROL_BYTES + ); + gate.release(); + drop(tx); + writer.await.unwrap().unwrap(); + } + + #[tokio::test] + async fn workload_clock_credit_uses_actual_cbor_width_not_reserved_maximum() { + use microsandbox_protocol::core::{ + ClockSync, WorkloadTransportCredit, WorkloadTransportPosition, + }; + for timestamp in [0, u32::MAX as u64, u64::MAX] { + let shared = workload_test_shared(4096, false); + let frame = crate::clock::encode_clock_sync_frame(timestamp).unwrap(); + shared + .workload_control + .restore( + WorkloadTransportPosition::default(), + WorkloadTransportCredit { + control_bytes: frame.len() as u64, + control_frames: 1, + ..Default::default() + }, + 0, + ) + .unwrap(); + let (tx, mut rx) = ControlWriter::new(); + let queued = ControlWrite::clock_sync().unwrap(); + let reservation = queued.data.len(); + assert!(reservation >= frame.len()); + tx.send(queued).await.unwrap(); + let mut write = rx.recv().await.unwrap(); + assert_eq!( + tx.control_bytes.available_permits(), + AGENT_WRITE_CONTROL_BYTES - reservation + ); + assert!( + admit_control_write( + &mut write, + &shared.workload_control, + Some(&shared), + &mut false, + || crate::clock::encode_clock_sync_frame(timestamp) + ) + .unwrap() + ); + assert_eq!(write.data, frame); + assert_eq!( + decode_frame(&write.data) + .unwrap() + .payload::() + .unwrap() + .unix_time_nanos, + timestamp + ); + assert!(!shared.workload_control.admit(false, 1).unwrap()); + drop(write); + assert_eq!( + tx.control_bytes.available_permits(), + AGENT_WRITE_CONTROL_BYTES + ); + } + } + + fn ordered_tcp_metadata(kind: MessageType, payload: &T) -> ControlWrite { + let message = Message::with_payload(kind, 17, payload).unwrap(); + let mut encoded = Vec::new(); + codec::encode_to_buf(&message, &mut encoded).unwrap(); + let mut write = ControlWrite::ordinary(Bytes::from(encoded), 17, false); + write.classify_tcp_order(17, None, Some(&message)); + write + } + + fn ordered_tcp_input() -> ControlWrite { + let record = microsandbox_protocol::bulk::BulkRecord { + id: 17, + kind: BulkKind::Tcp, + flow: BulkFlow::HostToGuest, + offset: 0, + payload: Bytes::from_static(b"request tail"), + }; + let mut encoded = Vec::new(); + codec::encode_bulk_to_buf(&record, &mut encoded).unwrap(); + let mut write = ControlWrite::ordinary(Bytes::from(encoded), 17, true); + write.classify_tcp_order(17, Some(bulk_wire_metadata(&write.data).unwrap()), None); + write + } + + fn tcp_input_finish() -> ControlWrite { + ordered_tcp_metadata( + MessageType::BulkFinish, + &BulkFinish { + kind: BulkKind::Tcp, + flow: BulkFlow::HostToGuest, + final_offset: b"request tail".len() as u64, + }, + ) + } + + fn tcp_output_credit() -> ControlWrite { + ordered_tcp_metadata( + MessageType::BulkCredit, + &BulkCredit { + kind: BulkKind::Tcp, + flow: BulkFlow::GuestToHost, + consumed_offset: 0, + credit_limit: microsandbox_protocol::bulk::DEFAULT_BULK_WINDOW, + }, + ) + } + + #[test] + fn workload_combined_tcp_credit_passes_blocked_input_and_finish_only() { + use microsandbox_protocol::core::{WorkloadTransportCredit, WorkloadTransportPosition}; + let shared = workload_test_shared(4096, false); + let control = &shared.workload_control; + let mut credit = WorkloadTransportCredit { + control_bytes: 4096, + control_frames: 16, + ..Default::default() + }; + control + .restore(WorkloadTransportPosition::default(), credit, 0) + .unwrap(); + let input = ordered_tcp_input(); + let input_bytes = input.data.clone(); + let finish = tcp_input_finish(); + let finish_bytes = finish.data.clone(); + let mut pending = VecDeque::from([input, finish, tcp_output_credit()]); + let gate = control.gate(); + assert!(next_control_write(&mut pending, control).unwrap().is_none()); + gate.release(); + let returned = next_control_write(&mut pending, control).unwrap().unwrap(); + assert!(matches!(returned.order, ControlOrder::TcpOutputCredit(17))); + assert_eq!(pending.len(), 2); + assert!(next_control_write(&mut pending, control).unwrap().is_none()); + credit.bulk_bytes = input_bytes.len() as u64; + credit.bulk_frames = 1; + control.update_credit(credit).unwrap(); + assert_eq!( + next_control_write(&mut pending, control) + .unwrap() + .unwrap() + .data, + input_bytes + ); + assert_eq!( + next_control_write(&mut pending, control) + .unwrap() + .unwrap() + .data, + finish_bytes + ); + assert!(pending.is_empty()); + + // With input capacity available, the new exception does not turn credit into priority. + let shared = workload_test_shared(4096, false); + let mut pending = + VecDeque::from([ordered_tcp_input(), tcp_input_finish(), tcp_output_credit()]); + for expected in [ + ControlOrder::TcpInputData(17), + ControlOrder::TcpInputFinish(17), + ControlOrder::TcpOutputCredit(17), + ] { + let actual = next_control_write(&mut pending, &shared.workload_control) + .unwrap() + .unwrap() + .order; + assert_eq!( + std::mem::discriminant(&actual), + std::mem::discriminant(&expected) + ); + } + } + + #[test] + fn workload_combined_tcp_credit_never_crosses_control_or_owner_fences() { + use microsandbox_protocol::core::{WorkloadTransportCredit, WorkloadTransportPosition}; + let shared = workload_test_shared(4096, false); + let control = &shared.workload_control; + control + .restore( + WorkloadTransportPosition::default(), + WorkloadTransportCredit { + control_bytes: 4096, + control_frames: 16, + ..Default::default() + }, + 0, + ) + .unwrap(); + for fence in [ + ordered_tcp_metadata(MessageType::BulkCancel, &()), + ordered_tcp_metadata(MessageType::TcpConnect, &()), + ordered_tcp_metadata(MessageType::Ping, &()), + ControlWrite::client_fence(Bytes::new(), 1, 100), + ControlWrite::from(Bytes::new()), + ] { + let mut pending = VecDeque::from([ordered_tcp_input(), fence, tcp_output_credit()]); + assert!(next_control_write(&mut pending, control).unwrap().is_none()); + assert_eq!(pending.len(), 3); + } + let valid = BulkCredit { + kind: BulkKind::Tcp, + flow: BulkFlow::GuestToHost, + consumed_offset: 0, + credit_limit: microsandbox_protocol::bulk::DEFAULT_BULK_WINDOW, + }; + for payload in [ + BulkCredit { + kind: BulkKind::Filesystem, + ..valid + }, + BulkCredit { + flow: BulkFlow::HostToGuest, + ..valid + }, + BulkCredit { + consumed_offset: 2, + credit_limit: 1, + ..valid + }, + BulkCredit { + credit_limit: MAX_BULK_WINDOW + 1, + ..valid + }, + ] { + let mut pending = VecDeque::from([ + ordered_tcp_input(), + ordered_tcp_metadata(MessageType::BulkCredit, &payload), + ]); + assert!(next_control_write(&mut pending, control).unwrap().is_none()); + } + let mut pending = VecDeque::from([ + ordered_tcp_input(), + ordered_tcp_metadata(MessageType::BulkCredit, &()), + ]); + assert!(next_control_write(&mut pending, control).unwrap().is_none()); + // The exclusive raw flag is not sufficient: only previously validated TCP input gets + // the exception. Filesystem or reverse-direction raw metadata retains strict ordering. + for (kind, flow) in [ + (BulkKind::Filesystem, BulkFlow::HostToGuest), + (BulkKind::Tcp, BulkFlow::GuestToHost), + ] { + let mut input = ControlWrite::ordinary(ordered_tcp_input().data, 17, true); + input.classify_tcp_order(17, Some((kind, flow, 0, 12)), None); + let mut pending = VecDeque::from([input, tcp_output_credit()]); + assert!(next_control_write(&mut pending, control).unwrap().is_none()); + } + // Only the correctly directed TCP finish commutes; unknown/malformed metadata remains a fence. + for finish in [ + BulkFinish { + kind: BulkKind::Tcp, + flow: BulkFlow::GuestToHost, + final_offset: 0, + }, + BulkFinish { + kind: BulkKind::Filesystem, + flow: BulkFlow::HostToGuest, + final_offset: 0, + }, + ] { + let mut pending = VecDeque::from([ + ordered_tcp_input(), + ordered_tcp_metadata(MessageType::BulkFinish, &finish), + tcp_output_credit(), + ]); + assert!(next_control_write(&mut pending, control).unwrap().is_none()); + } + } + + #[tokio::test] + async fn workload_class_reservations_bound_pending_bytes_and_frames() { + let (tx, mut rx) = ControlWriter::new(); + let data = Bytes::from(vec![0; AGENT_WRITE_DATA_BYTES / AGENT_WRITE_CLASS_FRAMES]); + let metadata = Bytes::from(vec![ + 0; + AGENT_WRITE_CONTROL_BYTES / AGENT_WRITE_CLASS_FRAMES + ]); + let mut pending = Vec::new(); + for id in 1..=AGENT_WRITE_CLASS_FRAMES as u32 { + tx.try_send(ControlWrite::ordinary(data.clone(), id, true)) + .unwrap(); + pending.push(rx.recv().await.unwrap()); + } + assert_eq!(tx.data_bytes.available_permits(), 0); + assert_eq!(tx.data_frames.available_permits(), 0); + assert!(matches!( + tx.try_send(ControlWrite::ordinary(Bytes::new(), 99, true)), + Err(mpsc::error::TrySendError::Full(_)) + )); + for id in 100..100 + AGENT_WRITE_CLASS_FRAMES as u32 { + tx.try_send(ControlWrite::ordinary(metadata.clone(), id, false)) + .unwrap(); + pending.push(rx.recv().await.unwrap()); + } + assert_eq!(pending.len(), AGENT_WRITE_CHANNEL_CAPACITY); + assert_eq!(tx.control_bytes.available_permits(), 0); + assert_eq!(tx.control_frames.available_permits(), 0); + assert!(matches!( + tx.try_send(ControlWrite::ordinary(Bytes::new(), 999, false)), + Err(mpsc::error::TrySendError::Full(_)) + )); + drop(pending); + assert_eq!(tx.data_bytes.available_permits(), AGENT_WRITE_DATA_BYTES); + assert_eq!( + tx.control_bytes.available_permits(), + AGENT_WRITE_CONTROL_BYTES + ); + assert_eq!(tx.data_frames.available_permits(), AGENT_WRITE_CLASS_FRAMES); + assert_eq!( + tx.control_frames.available_permits(), + AGENT_WRITE_CLASS_FRAMES + ); + } + + #[tokio::test] + async fn workload_reservation_waiters_cancel_and_wake_on_receiver_close() { + let (tx, mut rx) = ControlWriter::new(); + let mut retained = Vec::new(); + for id in 1..=AGENT_WRITE_CLASS_FRAMES as u32 { + tx.send(ControlWrite::ordinary(Bytes::new(), id, true)) + .await + .unwrap(); + retained.push(rx.recv().await.unwrap()); + } + assert!( + tokio::time::timeout( + Duration::from_millis(10), + tx.send(ControlWrite::ordinary(Bytes::new(), 99, true)) + ) + .await + .is_err() + ); + assert_eq!(tx.data_frames.available_permits(), 0); + let sender = tx.clone(); + let waiting = tokio::spawn(async move { + sender + .send(ControlWrite::ordinary(Bytes::new(), 100, true)) + .await + }); + drop(rx); + assert!( + tokio::time::timeout(Duration::from_secs(1), waiting) + .await + .unwrap() + .unwrap() + .is_err() + ); + drop(retained); + assert_eq!(tx.data_frames.available_permits(), AGENT_WRITE_CLASS_FRAMES); + } + + #[tokio::test] + async fn workload_private_freeze_bypasses_both_full_admission_classes() { + use microsandbox_protocol::core::{ + WorkloadFreeze, WorkloadFrozen, WorkloadTransportCredit, WorkloadTransportPosition, + }; + let shared = workload_test_shared(4096, false); + let control = Arc::clone(&shared.workload_control); + control + .restore( + WorkloadTransportPosition::default(), + WorkloadTransportCredit::default(), + 0, + ) + .unwrap(); + let gate = control.gate(); + let (tx, rx) = ControlWriter::new(); + let mut expected = Vec::new(); + for uses_data_credit in [true, false] { + for id in 1..=AGENT_WRITE_CLASS_FRAMES as u32 { + let kind = if uses_data_credit { + MessageType::ExecStdin + } else { + MessageType::Ping + }; + let bytes = Bytes::from(encoded_message_id(kind, id, &())); + tx.send(ControlWrite::ordinary(bytes.clone(), id, uses_data_credit)) + .await + .unwrap(); + expected.push(bytes); + } + } + assert_eq!(tx.data_frames.available_permits(), 0); + assert_eq!(tx.control_frames.available_permits(), 0); + let writer = tokio::spawn(ring_writer_task(Arc::clone(&shared), rx)); + let position = tokio::time::timeout(Duration::from_secs(1), control.parked_position()) + .await + .unwrap() + .unwrap(); + assert_eq!(position, WorkloadTransportPosition::default()); + let requester = Arc::clone(&control); + let request = tokio::spawn(async move { + requester + .request( + Message::with_payload( + MessageType::WorkloadFreeze, + 0, + &WorkloadFreeze { + attempt_id: "full-classes".into(), + host_input: position, + }, + ) + .unwrap(), + "full-classes", + ) + .await + }); + assert_eq!( + decode_frame(&next_host_fragment(&shared).await).unwrap().t, + MessageType::WorkloadFreeze + ); + assert!(shared.rx_ring.pop().is_none()); + control + .reply( + Message::with_payload( + MessageType::WorkloadFrozen, + WORKLOAD_CONTROL_ID, + &WorkloadFrozen { + attempt_id: "full-classes".into(), + guest_bulk_bytes_target: 0, + input_credit: WorkloadTransportCredit::default(), + }, + ) + .unwrap(), + ) + .unwrap(); + request.await.unwrap().unwrap(); + control + .update_credit(WorkloadTransportCredit { + control_bytes: 4096, + control_frames: AGENT_WRITE_CLASS_FRAMES as u64, + bulk_bytes: 4096, + bulk_frames: AGENT_WRITE_CLASS_FRAMES as u64, + }) + .unwrap(); + gate.release(); + for frame in expected { + assert_eq!(next_host_fragment(&shared).await, frame); + } + assert_eq!(tx.data_frames.available_permits(), AGENT_WRITE_CLASS_FRAMES); + assert_eq!( + tx.control_frames.available_permits(), + AGENT_WRITE_CLASS_FRAMES + ); + drop(tx); + writer.await.unwrap().unwrap(); + } + + #[tokio::test] + async fn workload_gate_keeps_source_fifo_until_confirmed_continue() { + use microsandbox_protocol::core::{ + WorkloadFreeze, WorkloadFrozen, WorkloadTransportCredit, WorkloadTransportPosition, + }; + let shared = workload_test_shared(4096, false); + let control = Arc::clone(&shared.workload_control); + // Model a live guest whose previously admitted input still occupies the whole window. + control + .restore( + WorkloadTransportPosition::default(), + WorkloadTransportCredit::default(), + 0, + ) + .unwrap(); + let (tx, rx) = mpsc::channel(2); + let writer = tokio::spawn(ring_writer_task(Arc::clone(&shared), rx)); + let first = Bytes::from(encoded_message_id( + MessageType::Ping, + 1, + µsandbox_protocol::core::Ping {}, + )); + let second = Bytes::from(encoded_message_id( + MessageType::Ping, + 2, + µsandbox_protocol::core::Ping {}, + )); + let (done, mut completed) = oneshot::channel(); + tx.send(ControlWrite { + completion: Some(done), + ..first.clone().into() + }) + .await + .unwrap(); tx.send(second.clone().into()).await.unwrap(); let gate = control.gate(); let position = @@ -6913,7 +8195,7 @@ mod tests { async fn workload_post_ready_shutdown_uses_counted_fifo() { let shared = workload_test_shared(4096, false); let control = Arc::clone(&shared.workload_control); - let (tx, rx) = mpsc::channel(2); + let (tx, rx) = ControlWriter::new(); control.register_ordinary_writer(tx.clone()); let first = Bytes::from(encoded_message_id( MessageType::Ping, @@ -6953,7 +8235,7 @@ mod tests { async fn workload_post_ready_shutdown_respects_gate_and_deadline() { let shared = workload_test_shared(4096, false); let control = Arc::clone(&shared.workload_control); - let (tx, rx) = mpsc::channel(2); + let (tx, rx) = ControlWriter::new(); control.register_ordinary_writer(tx); let writer = tokio::spawn(ring_writer_task(Arc::clone(&shared), rx)); let gate = control.gate(); diff --git a/crates/runtime/lib/runner/workload_control.rs b/crates/runtime/lib/runner/workload_control.rs index ca5d6b863..e210b02a5 100644 --- a/crates/runtime/lib/runner/workload_control.rs +++ b/crates/runtime/lib/runner/workload_control.rs @@ -6,9 +6,10 @@ use std::sync::{Arc, Mutex}; use bytes::Bytes; use microsandbox_protocol::codec; use microsandbox_protocol::core::{ - CoreError, Ready, WORKLOAD_TRANSPORT_BULK_BYTES, WORKLOAD_TRANSPORT_BULK_FRAMES, - WORKLOAD_TRANSPORT_CONTROL_BYTES, WORKLOAD_TRANSPORT_CONTROL_FRAMES, WorkloadFrozen, - WorkloadThawed, WorkloadTransportCredit, WorkloadTransportPosition, + CoreError, Ready, WORKLOAD_TRANSPORT_BARRIER_VERSION, WORKLOAD_TRANSPORT_BULK_BYTES, + WORKLOAD_TRANSPORT_BULK_FRAMES, WORKLOAD_TRANSPORT_CONTROL_BYTES, + WORKLOAD_TRANSPORT_CONTROL_FRAMES, WorkloadFrozen, WorkloadThawed, WorkloadTransportCredit, + WorkloadTransportPosition, }; use microsandbox_protocol::message::{Message, MessageType}; use tokio::sync::{Notify, mpsc, oneshot}; @@ -48,7 +49,7 @@ struct State { guest_bulk_bytes: u64, bulk_tail: usize, pending: Vec, - ordinary_writer: Option>, + ordinary_writer: Option, } /// Shared by the trusted coordinator and relay, never exposed through the SDK socket. @@ -97,7 +98,9 @@ impl WorkloadControl { pub(crate) fn install_ready(&self, version: u8, ready: Ready, dual_port: bool) { let mut state = self.state.lock().unwrap(); - if ready.workload_transport_barrier_version == Some(1) && state.ready.is_none() { + if ready.workload_transport_barrier_version == Some(WORKLOAD_TRANSPORT_BARRIER_VERSION) + && state.ready.is_none() + { state.credit = WorkloadTransportCredit { control_bytes: WORKLOAD_TRANSPORT_CONTROL_BYTES, control_frames: WORKLOAD_TRANSPORT_CONTROL_FRAMES, @@ -119,17 +122,12 @@ impl WorkloadControl { .expect("one lifecycle writer") } - pub(crate) fn register_ordinary_writer( - &self, - writer: mpsc::Sender, - ) { + pub(crate) fn register_ordinary_writer(&self, writer: super::relay::ControlWriter) { self.state.lock().unwrap().ordinary_writer = Some(writer); } /// Bootstrap may write directly before Ready. Every later ordinary write joins the same FIFO. - pub(crate) fn ordinary_writer( - &self, - ) -> Result>, String> { + pub(crate) fn ordinary_writer(&self) -> Result, String> { let state = self.state.lock().unwrap(); if state.closed { return Err("workload transport closed".into()); @@ -153,8 +151,10 @@ impl WorkloadControl { return Err("workload control transport is not running".into()); } let (version, ready) = state.ready.clone().ok_or("guest readiness unavailable")?; - if ready.workload_transport_barrier_version != Some(1) { - return Err("guest lacks workload transport barrier version 1".into()); + if ready.workload_transport_barrier_version != Some(WORKLOAD_TRANSPORT_BARRIER_VERSION) { + return Err(format!( + "guest lacks workload transport barrier version {WORKLOAD_TRANSPORT_BARRIER_VERSION}" + )); } Ok((version, ready)) } @@ -229,12 +229,18 @@ impl WorkloadControl { if state.gates != 0 || state.fenced { return Ok(false); } - if !state + match state .ready .as_ref() - .is_some_and(|(_, ready)| ready.workload_transport_barrier_version == Some(1)) + .and_then(|(_, ready)| ready.workload_transport_barrier_version) { - return Ok(true); + None => return Ok(true), + Some(WORKLOAD_TRANSPORT_BARRIER_VERSION) => {} + Some(version) => { + return Err(format!( + "unsupported workload transport barrier version {version}" + )); + } } let (sent_bytes, sent_frames, byte_limit, frame_limit) = if bulk { ( @@ -508,7 +514,7 @@ mod tests { control.install_ready( 9, Ready { - workload_transport_barrier_version: Some(1), + workload_transport_barrier_version: Some(WORKLOAD_TRANSPORT_BARRIER_VERSION), ..Ready::default() }, false, @@ -529,6 +535,23 @@ mod tests { assert!(!control.admit(false, 1).unwrap()); } + #[test] + fn unsupported_private_contract_fails_instead_of_disabling_admission() { + let control = WorkloadControl::new(); + control.install_ready(8, Ready::default(), false); + assert!(control.admit(false, usize::MAX).unwrap()); + control.install_ready( + 9, + Ready { + workload_transport_barrier_version: Some(WORKLOAD_TRANSPORT_BARRIER_VERSION - 1), + ..Default::default() + }, + false, + ); + assert!(control.admit(false, 1).unwrap_err().contains("unsupported")); + assert!(control.admit(true, 1).unwrap_err().contains("unsupported")); + } + #[tokio::test] async fn stable_park_does_not_wake_its_own_waiter() { let control = transport(); diff --git a/scripts/smoke/cli/test_transport_checkpoint.py b/scripts/smoke/cli/test_transport_checkpoint.py index 226255216..d0e4a7fad 100644 --- a/scripts/smoke/cli/test_transport_checkpoint.py +++ b/scripts/smoke/cli/test_transport_checkpoint.py @@ -127,6 +127,66 @@ def test_gate_timing_uses_conservative_clock_bounds(self): with self.assertRaisesRegex(RuntimeError, "overlap unproven"): HARNESS.verify_gate_timing(dict(unix_ns=4600), before, after, operations) + def test_fresh_source_and_child_exec_must_finish_before_autonomous_gate(self): + before = dict(host_before_ns=1000, guest_ns=1500, host_after_ns=1200) + after = dict(host_before_ns=9000, guest_ns=9600, host_after_ns=9300) + operations = [dict(kind="restore_ready", wall_end_ns=4000)] + opened = dict(unix_ns=5000) + HARNESS.verify_gate_timing(opened, before, after, operations) + # A fast restore followed by control credit starvation is not a passing restore: + # the first independent exec must complete while inherited input is still blocked. + for kind in ("child-ready-exec", "source-ready-exec"): + with self.subTest(kind=kind), self.assertRaisesRegex(RuntimeError, "overlap unproven"): + HARNESS.verify_gate_timing(opened, before, after, + [*operations, dict(kind=kind, wall_end_ns=4500)]) + + def test_child_ready_records_first_exec_in_gate_deadline_operations(self): + smoke = self.smoke() + row = dict(case="full-pipe-0", operations=[dict(kind="restore_ready")]) + with mock.patch.object(smoke, "guest", return_value="CHILD_FRAME_OK") as guest, \ + mock.patch.object(HARNESS.time, "monotonic", side_effect=[1.25, 1.5]), \ + mock.patch.object(HARNESS.time, "time_ns", side_effect=[1000, 1250]): + smoke.child_ready(row, "child") + self.assertEqual(guest.call_args.args[:2], ("full-pipe-0-child-ready-exec", "child")) + self.assertNotIn("touch ", guest.call_args.args[2]) + self.assertTrue(row["child_exec_independent"]) + self.assertEqual(row["child_ready_exec_ms"], 250) + self.assertEqual(row["operations"], [dict(kind="restore_ready"), + dict(kind="child-ready-exec", start=1.25, end=1.5, + wall_start_ns=1000, wall_end_ns=1250)]) + + def test_source_ready_records_fresh_exec_without_opening_consumer_gate(self): + smoke = self.smoke() + row = dict(case="full-pipe-0", operations=[dict(kind="capture")]) + with mock.patch.object(smoke, "guest", return_value="SOURCE_FRAME_OK") as guest, \ + mock.patch.object(HARNESS.time, "monotonic", side_effect=[2.0, 2.125]), \ + mock.patch.object(HARNESS.time, "time_ns", side_effect=[2000, 2125]): + smoke.source_ready(row) + guest.assert_called_once_with("full-pipe-0-source-ready-exec", "source", + "printf 'SOURCE_FRAME_OK\\n'") + self.assertTrue(row["source_exec_independent"]) + self.assertEqual(row["source_ready_exec_ms"], 125) + self.assertEqual(row["operations"], [dict(kind="capture"), + dict(kind="source-ready-exec", start=2.0, end=2.125, + wall_start_ns=2000, wall_end_ns=2125)]) + + def test_source_ready_rejects_corrupt_marker_and_keeps_timing_evidence(self): + smoke = self.smoke() + row = dict(case="paused-pipe-0") + with mock.patch.object(smoke, "guest", return_value="SOURCE_FRAME_OK extra"): + with self.assertRaisesRegex(RuntimeError, "source framing mismatch"): + smoke.source_ready(row) + self.assertNotIn("source_exec_independent", row) + self.assertEqual(row["operations"][0]["kind"], "source-ready-exec") + self.assertGreaterEqual(row["source_ready_exec_ms"], 0) + + def test_source_ready_latency_is_summarized_only_for_passing_samples(self): + samples = [dict(kind="full", tty=False, status="passed", source_ready_exec_ms=12), + dict(kind="full", tty=False, status="passed", source_ready_exec_ms=18), + dict(kind="full", tty=False, status="failed", source_ready_exec_ms=900)] + self.assertEqual(HARNESS.summarize(samples)["full/pipe/source_ready_exec_ms"], + dict(n=2, p50=12, p95=18)) + def test_regression_timer_opens_without_an_ordinary_control_exec(self): stream = self.stream(arguments=[str(self.root / "absent-gate"), str(self.root / "receipt"), "0", str(HARNESS.PREFIX_BYTES), ".35"]) @@ -368,6 +428,87 @@ def guest(_label, _name, command): self.assertEqual(sample["bulk"], {}) self.assertNotIn("control_stdout", sample) + def test_source_exec_precedes_post_cut_release_and_source_stdin_finish(self): + smoke = self.smoke() + size = smoke.args.stdin_mib * HARNESS.MIB + receipt = dict(bytes=size, sha256=HARNESS.input_digest(size, size - HARNESS.LATE_BYTES), + eof=True, eof_kind="pipe-close") + stream = mock.Mock(size=size, expected_digest=receipt["sha256"]) + stream.after_cut = threading.Event() + stream.await_prefix.return_value = dict(bytes=HARNESS.PREFIX_BYTES) + stream.await_pressure.return_value = dict(eagain_count=1) + events = [] + + def guest(_label, _name, command): + if command == "printf 'SOURCE_FRAME_OK\\n'": + self.assertFalse(stream.after_cut.is_set()) + self.assertEqual(events, ["pause", "resume"]) + events.append("source-ready-exec") + return "SOURCE_FRAME_OK" + return json.dumps(receipt) if command.startswith("cat ") else "POST_FRAME_OK" + + def finish(): + self.assertTrue(stream.after_cut.is_set()) + self.assertEqual(events[-1], "source-ready-exec") + events.append("stdin-finish") + return dict(bytes=size, seconds=1, receipt=receipt) + + def gate_proof(row, _receipt): + self.assertEqual(row["operations"][-1]["kind"], "source-ready-exec") + self.assertTrue(row["source_exec_independent"]) + + stream.finish.side_effect = finish + with mock.patch.object(HARNESS, "Stream", return_value=stream), \ + mock.patch.object(smoke, "guest", side_effect=guest), \ + mock.patch.object(smoke, "run", side_effect=lambda _label, kind, *args: events.append(kind)), \ + mock.patch.object(smoke, "clock_probe", return_value={}), \ + mock.patch.object(smoke, "check_status"), \ + mock.patch.object(smoke, "gate_proof", side_effect=gate_proof), \ + contextlib.redirect_stdout(io.StringIO()): + smoke.scenario("paused", False, 0) + self.assertEqual(events, ["pause", "resume", "source-ready-exec", "stdin-finish"]) + self.assertEqual(smoke.report["samples"][0]["status"], "passed") + + def test_tcp_source_exec_follows_child_probe_and_precedes_input_finish(self): + smoke = self.smoke() + smoke.args.tcp_mib = 1 + receipt = dict(bytes=HARNESS.MIB, sha256=HARNESS.input_digest(HARNESS.MIB, HARNESS.MIB), + eof=True, eof_kind="pipe-close") + server, client = mock.Mock(), mock.Mock() + server.finish.return_value = dict(receipt=receipt) + client.await_pressure.return_value = dict(eagain_count=1) + events = [] + + def guest(_label, _name, command): + if command == "printf 'SOURCE_FRAME_OK\\n'": + self.assertEqual(events, ["child-ready-exec"]) + events.append("source-ready-exec") + return "SOURCE_FRAME_OK" + return "TCP_POST_FRAME_OK" + + def finish(): + self.assertEqual(events, ["child-ready-exec", "source-ready-exec"]) + events.append("tcp-finish") + return receipt + + def gate_proof(row, _receipt): + self.assertEqual(row["operations"][-1]["kind"], "source-ready-exec") + self.assertTrue(row["source_exec_independent"]) + + client.finish.side_effect = finish + with mock.patch.object(HARNESS, "Stream", return_value=server), \ + mock.patch.object(HARNESS.TCP, "InlineTcp", return_value=client), \ + mock.patch.object(smoke, "guest", side_effect=guest), \ + mock.patch.object(smoke, "run"), \ + mock.patch.object(smoke, "clock_probe", return_value={}), \ + mock.patch.object(smoke, "child_ready", side_effect=lambda *args: events.append("child-ready-exec")), \ + mock.patch.object(smoke, "gate_proof", side_effect=gate_proof), \ + mock.patch.object(smoke, "stop"), \ + contextlib.redirect_stdout(io.StringIO()): + smoke.tcp_scenario("tcp-branch", 0) + self.assertEqual(events, ["child-ready-exec", "source-ready-exec", "tcp-finish"]) + self.assertEqual(smoke.report["samples"][0]["status"], "passed") + def test_failed_samples_and_failed_baselines_do_not_generate_performance_claims(self): sample = dict(kind="full", tty=False, status="failed", capture_ms=12) self.assertEqual(HARNESS.summarize([sample]), {}) @@ -381,7 +522,15 @@ def test_failed_samples_and_failed_baselines_do_not_generate_performance_claims( for report in reports.values(): report["image_manifest_digest"] = "sha256:fixture" report["parameters"] = {"samples": 1} + report["firmware_sha256"] = "same-firmware" + report["host"] = "same-host" + report["host_node"] = "same-machine" self.assertEqual(HARNESS.compare(reports)["full/pipe/capture_ms"]["candidate_over_baseline_p50"], 1) + for field in ("firmware_sha256", "host", "host_node"): + previous = reports["candidate"][field] + reports["candidate"][field] = "different" + self.assertEqual(HARNESS.compare(reports), {}) + reports["candidate"][field] = previous reports["candidate"]["image_manifest_digest"] = "sha256:different-image" self.assertEqual(HARNESS.compare(reports), {}) diff --git a/scripts/smoke/cli/transport-checkpoint.py b/scripts/smoke/cli/transport-checkpoint.py index 03bd467f0..07e21f25c 100644 --- a/scripts/smoke/cli/transport-checkpoint.py +++ b/scripts/smoke/cli/transport-checkpoint.py @@ -56,7 +56,7 @@ CONTROL_SUFFIX = b"0123456789abcdef" * 14 + b"abcdef\n" # Regression consumers acknowledge a prefix, then wait behind an autonomous timer gate. -# This fills forwarding queues without requiring a fresh exec to escape the saturated FIFO. +# The independent gate timestamp proves fresh metadata exec completed without draining input. # Throughput and VM-free fixture modes can still open a file gate explicitly. INPUT_PROGRAM = r''' import hashlib, json, os, select, sys, termios, time @@ -564,10 +564,10 @@ def __init__(self, args): firmware_sha256=file_digest(Path(self.env["MSB_LIBKRUNFW_PATH"])), agentd_sha256=(file_digest(Path(self.env["MSB_AGENTD_PATH"])) if "MSB_AGENTD_PATH" in self.env else None), - host=platform.platform(), harness_python=sys.version, + host=platform.platform(), host_node=platform.node(), harness_python=sys.version, parameters={key: getattr(args, key) for key in ("samples", "stdin_mib", "bulk_mib", "image", "cases", "input_modes")}, - latency_scope="CLI completion; independent child exec verifies readiness afterward", + latency_scope="CLI completion; independent source/child exec verifies readiness before input drains", throughput_scope="end-to-end CLI streams including startup/gating, not raw device throughput", cpu_scope="source runtime ps TIME; CLI children rusage; harness process_time; not total guest CPU", bulk_overlap_scope="checksum-verified CLI operation lifetimes, not guest-frame instrumentation") @@ -672,15 +672,38 @@ def gate_proof(self, row, receipt): raise def child_ready(self, row, name): - actual = self.guest(row["case"] + "-child-private", name, + started, wall_start = time.monotonic(), time.time_ns() + actual = self.guest(row["case"] + "-child-ready-exec", name, "test \"$(cat /transport-marker)\" = source; " "test \"$(cat /dev/shm/transport-marker)\" = source; " "echo child > /transport-marker; echo child > /dev/shm/transport-marker; " "printf 'CHILD_FRAME_OK\\n'") + ended = time.monotonic() + row["child_ready_exec_ms"] = (ended - started) * 1000 + # Restore completion alone cannot prove fresh control traffic escapes inherited + # input debt. This command must also finish before the autonomous read gate opens; + # child_input deliberately opens the child's gate only after this check returns. + row.setdefault("operations", []).append(dict(kind="child-ready-exec", start=started, + end=ended, wall_start_ns=wall_start, wall_end_ns=time.time_ns())) if actual != "CHILD_FRAME_OK": raise RuntimeError(f"child framing mismatch: {actual!r}") row["child_exec_independent"] = True + def source_ready(self, row): + started, wall_start = time.monotonic(), time.time_ns() + actual = self.guest(row["case"] + "-source-ready-exec", "source", + "printf 'SOURCE_FRAME_OK\\n'") + ended = time.monotonic() + row["source_ready_exec_ms"] = (ended - started) * 1000 + # Unlike the restored child's empty host queue, this source still owns queued input. + # Do not open its gate: completion before the autonomous timestamp must prove that + # fresh metadata can pass credit-blocked data while the input consumer stays blocked. + row.setdefault("operations", []).append(dict(kind="source-ready-exec", start=started, + end=ended, wall_start_ns=wall_start, wall_end_ns=time.time_ns())) + if actual != "SOURCE_FRAME_OK": + raise RuntimeError(f"source framing mismatch: {actual!r}") + row["source_exec_independent"] = True + def child_input(self, row, name, gate, receipt, stream): # This read happens before the host releases AFTER! bytes. The source's closed gate # is an independent copy, so inspecting the child cannot drain the source stream. @@ -782,9 +805,10 @@ def scenario(self, kind, is_pty, index): self.child_ready(row, child) self.child_input(row, child, gate, receipt, stream) if regression: + self.source_ready(row) stream.after_cut.set() - # Ordinary exec uses the same saturated FIFO. The prearranged guest timer must - # open this gate autonomously; sending `exec touch` here would deadlock the test. + # The timer remains independent evidence, not a metadata-progress workaround. + # Neither readiness exec opens the source gate before its queued input drains. row["stdin"] = stream.finish() stream.close() verify_receipt(json.loads(self.guest(label + "-source-receipt", "source", f"cat {receipt}")), @@ -802,6 +826,8 @@ def scenario(self, kind, is_pty, index): # Require a verified operation interval covering the *start* of every checkpoint # action. A transfer that starts only after thaw is not counted as overlap. for operation in row["operations"]: + if operation["kind"] in ("child-ready-exec", "source-ready-exec"): + continue # This probes control readiness, not checkpoint/bulk overlap. for direction, samples in row["bulk"].items(): overlaps = [s for s in samples if s["start"] <= operation["start"] < s["end"]] if not overlaps: @@ -867,6 +893,7 @@ def tcp_scenario(self, kind, index): if child: self.child_ready(row, child) row["child_tcp_expectation"] = "old connection detached; only fresh child exec asserted" + self.source_ready(row) row["tcp_receipt"] = client.finish() verify_receipt(row["tcp_receipt"], size, server.expected_digest, False) row["server"] = server.finish() @@ -961,6 +988,7 @@ def summarize(samples): continue prefix = row["kind"] + ("/pty" if row.get("tty") else "/pipe") for key in ("pause_ms", "resume_ms", "capture_ms", "branch_ready_ms", "restore_ready_ms", + "child_ready_exec_ms", "source_ready_exec_ms", "elapsed_ms", "harness_cpu_seconds", "cli_cpu_seconds", "source_runtime_cpu_seconds"): if row.get(key) is not None: groups.setdefault(prefix + "/" + key, []).append(row[key]) @@ -982,8 +1010,12 @@ def compare(reports): baseline, candidate = reports["baseline"], reports["candidate"] if (not baseline.get("image_manifest_digest") or baseline["image_manifest_digest"] != candidate.get("image_manifest_digest") - or baseline.get("parameters") != candidate.get("parameters")): - return {} # A moving image tag or different workload is not a binary performance delta. + or baseline.get("parameters") != candidate.get("parameters") + or not baseline.get("firmware_sha256") + or baseline["firmware_sha256"] != candidate.get("firmware_sha256") + or not baseline.get("host") or baseline["host"] != candidate.get("host") + or not baseline.get("host_node") or baseline["host_node"] != candidate.get("host_node")): + return {} # Changed image, workload, firmware, or host is not an isolated binary delta. before = reports.get("baseline", {}).get("statistics", {}) after = reports.get("candidate", {}).get("statistics", {}) return {key: {"baseline": before[key], "candidate": after[key], @@ -1044,7 +1076,7 @@ def interrupt(_signum, _frame): break # Do not benchmark the next binary alongside an unverified surviving runtime. result = dict(reports={key: str(root / key / "report.json") for key in reports}, comparison=compare(reports), status="failed" if failed else "passed", - comparison_requirement="two completely passing runs with identical pinned image and workload parameters") + comparison_requirement="two completely passing runs with identical host, firmware, pinned image, and workload parameters") (root / "comparison.json").write_text(json.dumps(result, indent=2) + "\n") print(f"Comparison: {root / 'comparison.json'}", flush=True) return int(failed) From 5f46d63eade290410e342bd779f8396205f805c1 Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Thu, 10 Sep 2026 17:17:04 +0100 Subject: [PATCH 25/29] ci(windows): exercise both runtime handoff and stdio guards Run all Windows-specific spawn regressions in the existing quality lane so the lifecycle-handoff cases preserved during stack reconciliation are executed alongside stdio inheritance cleanup. No runtime behavior changes. --- .github/workflows/check.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index ea090f68a..49ad364da 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -576,13 +576,13 @@ jobs: Set-MsvcEnvironment -Architecture ${{ matrix.vs_arch }} -HostArchitecture ${{ matrix.vs_host_arch }} cargo +stable test --no-default-features --features local,net -p microsandbox --lib --target ${{ matrix.rust_target }} sandbox::patch::tests::bind_patch_ - - name: Test Windows stdio inheritance cleanup + - name: Test Windows lifecycle handoff and stdio cleanup shell: pwsh run: | $ErrorActionPreference = "Stop" . "$env:GITHUB_WORKSPACE\vendor\libkrunfw\scripts\msvc-env.ps1" Set-MsvcEnvironment -Architecture ${{ matrix.vs_arch }} -HostArchitecture ${{ matrix.vs_host_arch }} - cargo +stable test --no-default-features --features local,net -p microsandbox --lib --target ${{ matrix.rust_target }} runtime::spawn::tests::windows_stdio_guard_ + cargo +stable test --no-default-features --features local,net -p microsandbox --lib --target ${{ matrix.rust_target }} runtime::spawn::tests::windows_ - name: Test Windows DNS resolver shell: pwsh From c46accc3a9efa1d6187d215c62ae6933891e3a9d Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Thu, 10 Sep 2026 18:39:17 +0100 Subject: [PATCH 26/29] fix(stack): repair grouped restore resolution and test portability Use the shared snapshot resolver from Python, retaining async not-found errors and covering group, member, identity, directory and archive inputs. Avoid the deprecated musl time_t alias without rounding freezer waits. Use mirrored test images so registry authentication cannot hide regressions. --- crates/agentd/lib/workload.rs | 8 +++- scripts/smoke/cli/cow-memory-lifecycle.py | 6 ++- scripts/smoke/cli/direct-branch.py | 4 +- scripts/smoke/cli/snapshot-groups.py | 4 +- sdk/python/integration/test_snapshots.py | 24 +++++++++++- sdk/python/src/error.rs | 6 +++ sdk/python/src/helpers.rs | 48 +---------------------- sdk/python/tests/test_types_enums.py | 16 +++++++- sdk/rust/lib/snapshot/group.rs | 8 +++- sdk/rust/tests/plain_http_secret.rs | 3 +- 10 files changed, 71 insertions(+), 56 deletions(-) diff --git a/crates/agentd/lib/workload.rs b/crates/agentd/lib/workload.rs index 7d805c958..33e21e12d 100644 --- a/crates/agentd/lib/workload.rs +++ b/crates/agentd/lib/workload.rs @@ -392,7 +392,13 @@ fn wait_for_cgroup_event(fd: RawFd, remaining: Duration) -> io::Result<()> { #[cfg(target_os = "linux")] let result = { let timeout = libc::timespec { - tv_sec: remaining.as_secs().min(libc::time_t::MAX as u64) as libc::time_t, + // Infer the platform's field type: naming libc::time_t is deprecated on musl. + tv_sec: remaining.as_secs().try_into().map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidInput, + "cgroup wait duration is too large", + ) + })?, tv_nsec: remaining.subsec_nanos().into(), }; unsafe { libc::ppoll(&mut event, 1, &timeout, std::ptr::null()) } diff --git a/scripts/smoke/cli/cow-memory-lifecycle.py b/scripts/smoke/cli/cow-memory-lifecycle.py index d1e800de3..a596b91f5 100644 --- a/scripts/smoke/cli/cow-memory-lifecycle.py +++ b/scripts/smoke/cli/cow-memory-lifecycle.py @@ -7,6 +7,8 @@ import time binary = os.environ["MSB_PATH"] +# Match CI's public mirror; callers may select an explicit local fixture instead. +image = os.environ.get("MSB_TEST_IMAGE", "mirror.gcr.io/library/alpine:latest") root = Path(os.environ["STACK8_OUT"]) root.mkdir(parents=True, exist_ok=True) prefix = os.environ.get("STACK8_PREFIX", "cow8") @@ -47,14 +49,14 @@ def run(label, *args, expected=0, timeout=120): try: refused = prefix + "-forked-boot" - result = run("forked-boot-rejected", "create", "alpine", "-n", refused, + result = run("forked-boot-rejected", "create", image, "-n", refused, "--forked", expected=None) assert result.returncode != 0, "forked must require captured RAM" source = prefix + "-source" names.append(source) run("fresh-" + mode, "run", "-d", "-n", source, "--root-disk", layout, "--memory", "256M", "--cpus", "2", - *(["--max-memory", "512M"] if resize else []), "alpine", + *(["--max-memory", "512M"] if resize else []), image, "--", "sh", "-c", "mkdir -p /dev/shm; echo captured > /dev/shm/cow-marker; i=0; while :; do echo $i > /tmp/cow-counter; i=$((i+1)); sleep 0.05; done") # Detached launch acknowledges the runtime, not the application's first write. for attempt in range(30): diff --git a/scripts/smoke/cli/direct-branch.py b/scripts/smoke/cli/direct-branch.py index 44fafce1d..7b7ed7204 100644 --- a/scripts/smoke/cli/direct-branch.py +++ b/scripts/smoke/cli/direct-branch.py @@ -8,6 +8,8 @@ import time binary = os.environ["MSB_PATH"] +# Match CI's public mirror; callers may select an explicit local fixture instead. +image = os.environ.get("MSB_TEST_IMAGE", "mirror.gcr.io/library/alpine:latest") root = Path(os.environ["STACK8_OUT"]) root.mkdir(parents=True, exist_ok=True) prefix = os.environ.get("STACK8_PREFIX", f"branch8-{os.getpid()}") @@ -84,7 +86,7 @@ def benchmark(source): try: source = prefix + "-source" names.append(source) - run("boot", "create", "alpine", "--name", source, "--root-disk", layout, + run("boot", "create", image, "--name", source, "--root-disk", layout, "--memory", "256M", "--cpus", "2") exec_guest(source, "echo source > /dev/shm/branch-marker; echo source > /disk-marker; sh -c 'i=0; while :; do i=$((i+1)); echo $i > /dev/shm/branch-counter; sleep 0.02; done' >/tmp/branch-counter.log 2>&1 PyErr { pub fn to_py_err(err: microsandbox::MicrosandboxError) -> PyErr { use microsandbox::MicrosandboxError::*; + // Missing snapshot selectors are resolved when the create future is awaited, like explicit + // artifact paths. Retain Python's useful missing-file exception without duplicating resolution. + if let SnapshotNotFound(_) = &err { + return pyo3::exceptions::PyFileNotFoundError::new_err(err.to_string()); + } + Python::with_gil(|py| { let errors_mod = match py.import("microsandbox.errors") { Ok(m) => m, diff --git a/sdk/python/src/helpers.rs b/sdk/python/src/helpers.rs index 849ec3892..b94c686d4 100644 --- a/sdk/python/src/helpers.rs +++ b/sdk/python/src/helpers.rs @@ -227,18 +227,8 @@ pub fn sandbox_builder_from_args( "from_snapshot must be str or os.PathLike", )); }; - // Preserve Python's established immediate missing-artifact error before constructing an - // awaitable. Descriptor parsing and disk/full admission remain deferred to the shared Rust - // resolver so installed directories and direct archives follow exactly the same path. - let snapshot_path = resolve_snapshot_path(&snap_str); - if !snapshot_path.exists() { - return Err(pyo3::exceptions::PyFileNotFoundError::new_err(format!( - "snapshot artifact not found: {}", - snapshot_path.display() - ))); - } - // Resolution stays deferred until the async build so installed and direct-archive sources - // share the same disk/full admission path. + // Resolve through the shared async builder: a group/member or snapshot identity is not + // a directory name. A Python-only existence check would reject these valid selectors. builder = builder.from_snapshot(snap_str); if let Some(base) = kwargs.get_item("snapshot_base")? { if !base.is_none() { @@ -2020,37 +2010,3 @@ fn extract_required<'py, T: FromPyObject<'py>>( .ok_or_else(|| pyo3::exceptions::PyValueError::new_err(format!("{key} is required")))? .extract() } - -/// Resolve a snapshot reference only far enough to preserve synchronous Python path validation. -fn resolve_snapshot_path(reference: &str) -> std::path::PathBuf { - if snapshot_ref_looks_like_path(reference) { - std::path::PathBuf::from(reference) - } else { - microsandbox::backend::default_backend() - .as_local() - .map(|local| local.snapshots_dir().join(reference)) - .unwrap_or_else(|| std::path::PathBuf::from(reference)) - } -} - -/// Match the Rust snapshot resolver's bare-name versus filesystem-path boundary. -fn snapshot_ref_looks_like_path(reference: &str) -> bool { - if reference.contains('/') || reference.starts_with('.') || reference.starts_with('~') { - return true; - } - - #[cfg(windows)] - { - use typed_path::{Utf8WindowsComponent, Utf8WindowsPath}; - - reference.contains('\\') - || matches!( - Utf8WindowsPath::new(reference).components().next(), - Some(Utf8WindowsComponent::Prefix(_)) - ) - } - #[cfg(not(windows))] - { - false - } -} diff --git a/sdk/python/tests/test_types_enums.py b/sdk/python/tests/test_types_enums.py index 4342a0838..e7ede0707 100644 --- a/sdk/python/tests/test_types_enums.py +++ b/sdk/python/tests/test_types_enums.py @@ -292,7 +292,8 @@ def test_sandbox_create_treats_explicit_none_as_omitted() -> None: with pytest.raises(ValueError, match="image= or from_snapshot= is required"): Sandbox.create("explicit-none-image", image=None) - with pytest.raises(FileNotFoundError, match="snapshot artifact not found"): + # Selector lookup is async; type/options validation still happens before making the future. + with pytest.raises(type(baseline.value)): Sandbox.create( "explicit-none-image-with-snapshot", image=None, @@ -300,6 +301,19 @@ def test_sandbox_create_treats_explicit_none_as_omitted() -> None: ) +@pytest.mark.asyncio +@pytest.mark.parametrize("selector", ["missing-group", "missing-group:missing-member"]) +async def test_missing_snapshot_selector_is_reported_when_awaited(selector: str) -> None: + with pytest.raises(FileNotFoundError): + await Sandbox.create("missing-snapshot-source", image=None, from_snapshot=selector) + + +@pytest.mark.asyncio +async def test_missing_snapshot_pathlike_is_reported_when_awaited(tmp_path) -> None: + with pytest.raises(FileNotFoundError): + await Sandbox.create("missing-snapshot-path", from_snapshot=tmp_path / "missing") + + def test_inactive_mount_enum_fields_are_still_validated() -> None: config = MountConfig( kind=MountKind.BIND, diff --git a/sdk/rust/lib/snapshot/group.rs b/sdk/rust/lib/snapshot/group.rs index 9723f5fb4..9998f81e6 100644 --- a/sdk/rust/lib/snapshot/group.rs +++ b/sdk/rust/lib/snapshot/group.rs @@ -103,7 +103,13 @@ pub(super) async fn resolve(root: &Path, selector: &str) -> MicrosandboxResult

{ + MicrosandboxError::SnapshotNotFound(selector.clone()) + } + other => other, + })?; let state = read_group(&directory)?; let (id, _) = resolve_selected(&directory, &state, member)?; Ok(directory.join(id)) diff --git a/sdk/rust/tests/plain_http_secret.rs b/sdk/rust/tests/plain_http_secret.rs index 4e17b25ef..83f2a5d84 100644 --- a/sdk/rust/tests/plain_http_secret.rs +++ b/sdk/rust/tests/plain_http_secret.rs @@ -11,7 +11,8 @@ use tokio::task::JoinHandle; // Constants -const ALPINE_IMAGE: &str = "alpine"; +// Match the mirrored fixture used by the other SDK integration tests. +const ALPINE_IMAGE: &str = "mirror.gcr.io/library/alpine:latest"; const REAL_SECRET: &str = "real-secret-plain-http"; /// Placeholder the guest sees for the `API_KEY` secret: the env var name with /// the `MSB_` prefix the runtime injects. From 30d09adc2c1c8b0dd9f5b0163b52fe50f2d9063a Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Thu, 10 Sep 2026 19:07:04 +0100 Subject: [PATCH 27/29] chore(docs): restructure readme --- README.md | 266 ++++++++++++++++++++++++++---------------------------- 1 file changed, 129 insertions(+), 137 deletions(-) diff --git a/README.md b/README.md index 3ce42a2d9..e206c8147 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ ## - **Hardware Isolation**: Hardware-level isolation with microVM technology. -- **Branch & Snapshot**: Save state. Fork live sandboxes. +- **Branch & Snapshot**: Save sandbox running state and restore later. Fork live sandboxes. - **Cross Platform**: Runs on Linux, macOS, and Windows. - **OCI Compatible**: Runs standard container images from Docker Hub, GHCR, or any OCI registry. - **Docker-Like Workflows**: Familiar image, command, shell, and volume workflows. @@ -44,34 +44,7 @@ ## rocket-darkrocket  Getting Started -####   Install the SDK -> ```sh -> npm i microsandbox # 🟦 TypeScript -> ``` -> -> ```sh -> cargo add microsandbox # 🦀 Rust -> ``` -> -> ```sh -> uv add microsandbox # 🐍 Python -> ``` -> -> ```sh -> go get github.com/superradcompany/microsandbox/sdk/go # 🐹 Go -> ``` ####   Install the CLI - -> Boot a microVM in a single command: -> -> ```sh -> npx microsandbox run debian -> ``` -> -> ## -> -> Or install the `msb` command globally: -> > ```sh > curl -fsSL https://install.microsandbox.dev | sh # 🍎 macOS / 🐧 Linux > ``` @@ -105,12 +78,30 @@ > > ## > -> Then you can run `msb` directly: +> Start creating sandboxes once installed: +> +> ```sh +> msb run ubuntu +> ``` + +####   Install the SDK +> ```sh +> npm i microsandbox # 🟦 TypeScript +> ``` +> +> ```sh +> cargo add microsandbox # 🦀 Rust +> ``` +> +> ```sh +> uv add microsandbox # 🐍 Python +> ``` > > ```sh -> msb run debian +> go get github.com/superradcompany/microsandbox/sdk/go # 🐹 Go > ``` + ## > **Requirements**: @@ -123,6 +114,114 @@
+## cli-darkcli  CLI + +The `msb` CLI provides a complete interface for managing sandboxes, snapshots, images, and volumes. + +####   Run a Command + +> ```sh +> msb run python -- python3 -c "print('Hello from a microVM!')" +> ``` + +####   Named Sandboxes + +> ```sh +> # Create and start a named sandbox +> msb create --name app python +> ``` +> +> ```sh +> # Execute commands +> msb exec app -- python -c "import this" +> msb exec app -- curl https://example.com +> ``` +> +> ```sh +> # Fork a running sandbox. +> msb branch app --name experiment +> msb exec experiment -- python -c "print('An independent copy!')" +> msb branch experiment --name another-experiment +> ``` +> +> ```sh +> # Save now, resume later +> msb snapshot create saved --from-sandbox app --full +> msb create --name restored --from-snapshot app:saved +> ``` +> +> ```sh +> # Lifecycle +> msb stop app +> msb start app +> msb rm app +> ``` + +####   Image Management + +> ```sh +> msb pull python # Pull an image +> msb image ls # List cached images +> msb image rm python # Remove an image +> ``` + +####   Configuration File + +> ```sh +> msb run --conf sandbox.yaml -- octocat +> ``` +> +> ```yaml +> # sandbox.yaml +> image: python:3.12 +> memory: 64M +> network: +> allow: +> - api.github.com +> scripts: +> octocat: | +> python - <<'PY' +> import urllib.request +> +> request = urllib.request.Request( +> "https://api.github.com/octocat", +> headers={"User-Agent": "microsandbox-example"}, +> ) +> with urllib.request.urlopen(request) as response: +> print(response.read().decode()) +> PY +> ``` + +####   Install & Uninstall Sandboxes + +> ```sh +> msb install ubuntu # Install ubuntu sandbox as 'ubuntu' command +> ubuntu # Opens Ubuntu in a microVM +> msb uninstall ubuntu # Uninstall the ubuntu sandbox +> ``` + +####   Status & Inspection + +> ```sh +> msb ls # List all sandboxes +> msb ps app # Show sandbox status +> msb inspect app # Detailed sandbox info +> msb metrics app # Live CPU/memory/network stats +> ``` + +> [!TIP] +> +> Run:
+> · `msb --help` for quick help menu.
+> · `msb --tree` for complete command hierarchy and descriptions.
+> · `msb --tree` for a specific command tree. + +
+ +CLI Docs + +
+ ## sdk-darksdk  SDK The SDK lets you create and control sandboxes directly from your application. `Sandbox.builder("...").create()` boots a microVM as a child process. No infrastructure required. @@ -283,113 +382,6 @@ The SDK lets you create and control sandboxes directly from your application. `S
-## cli-darkcli  CLI - -The `msb` CLI provides a complete interface for managing sandboxes, snapshots, images, and volumes. - -####   Run a Command - -> ```sh -> msb run python -- python3 -c "print('Hello from a microVM!')" -> ``` - -####   Named Sandboxes - -> ```sh -> # Create and start a named sandbox -> msb create --name app python -> ``` -> -> ```sh -> # Execute commands -> msb exec app -- python -c "import this" -> msb exec app -- curl https://example.com -> ``` -> -> ```sh -> # Fork a running sandbox. -> msb branch app --name experiment -> msb exec experiment -- python -c "print('An independent copy!')" -> msb branch experiment --name another-experiment -> ``` -> -> ```sh -> # Save now, resume later -> msb snapshot create saved --from-sandbox app --full -> msb create --name restored --from-snapshot app:saved -> ``` -> -> ```sh -> # Lifecycle -> msb stop app -> msb start app -> msb rm app -> ``` - -####   Image Management - -> ```sh -> msb pull python # Pull an image -> msb image ls # List cached images -> msb image rm python # Remove an image -> ``` - -####   Configuration File - -> ```sh -> msb run --conf sandbox.yaml -- octocat -> ``` -> -> ```yaml -> # sandbox.yaml -> image: python:3.12 -> network: -> allow: -> - api.github.com -> scripts: -> octocat: | -> python - <<'PY' -> import urllib.request -> -> request = urllib.request.Request( -> "https://api.github.com/octocat", -> headers={"User-Agent": "microsandbox-example"}, -> ) -> with urllib.request.urlopen(request) as response: -> print(response.read().decode()) -> PY -> ``` - -####   Install & Uninstall Sandboxes - -> ```sh -> msb install ubuntu # Install ubuntu sandbox as 'ubuntu' command -> ubuntu # Opens Ubuntu in a microVM -> msb uninstall ubuntu # Uninstall the ubuntu sandbox -> ``` - -####   Status & Inspection - -> ```sh -> msb ls # List all sandboxes -> msb ps app # Show sandbox status -> msb inspect app # Detailed sandbox info -> msb metrics app # Live CPU/memory/network stats -> ``` - -> [!TIP] -> -> Run:
-> · `msb --help` for quick help menu.
-> · `msb --tree` for complete command hierarchy and descriptions.
-> · `msb --tree` for a specific command tree. - -
- -CLI Docs - -
- ## beaker-darkbeaker  Examples Practical ways to put microsandbox to work: From 08fc9700093b80931029f32aceef45925fe47121 Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Thu, 10 Sep 2026 19:12:44 +0100 Subject: [PATCH 28/29] test(snapshot): allow disk-backed transport test homes Keep the short /tmp default while allowing a separate short home parent on hosts with quota-limited tmpfs. Each test still creates its own isolated home and never borrows existing sandbox state. Add a fixture regression for directory ownership and preserve the concurrent README update. All 34 transport harness unit tests pass. --- scripts/smoke/cli/test_transport_checkpoint.py | 16 ++++++++++++++-- scripts/smoke/cli/transport-checkpoint.py | 8 +++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/scripts/smoke/cli/test_transport_checkpoint.py b/scripts/smoke/cli/test_transport_checkpoint.py index d0e4a7fad..27abc61da 100644 --- a/scripts/smoke/cli/test_transport_checkpoint.py +++ b/scripts/smoke/cli/test_transport_checkpoint.py @@ -37,18 +37,30 @@ def setUp(self): self.output = self.root / "logs" self.output.mkdir() - def smoke(self): + def smoke(self, home_parent=Path("/tmp")): smoke = HARNESS.TransportSmoke(argparse.Namespace( binary=Path(sys.executable), output=self.root / "run", label="candidate", firmware=self.firmware, agentd=None, image="never-pull-this-unit-fixture", layout="managed", timeout=3, suite_timeout=20, samples=1, stdin_mib=2, bulk_mib=1, - gate_delay=20, tcp_mib=64, + gate_delay=20, tcp_mib=64, home_parent=home_parent, cases=["idle", "throughput", "paused", "full", "branch"], input_modes=["pipe", "pty"], )) self.addCleanup(shutil.rmtree, smoke.home, True) return smoke + def test_custom_home_parent_allocates_fresh_owned_directory(self): + parent = self.root / "homes" + parent.mkdir() + existing = parent / "keep" + existing.write_text("not owned by the harness") + smoke = self.smoke(home_parent=parent) + self.assertEqual(smoke.home.parent, parent) + self.assertTrue(smoke.home.name.startswith("msb-t-")) + self.assertTrue(smoke.home.is_dir()) + self.assertEqual(smoke.env["MSB_HOME"], str(smoke.home)) + self.assertEqual(existing.read_text(), "not owned by the harness") + def stream(self, program=None, arguments=None, is_pty=False, control=False): # Only the small guest Python fixture runs, as a local child. The real CLI invocation # is translated here before Popen: these tests cannot accidentally launch an msb VM. diff --git a/scripts/smoke/cli/transport-checkpoint.py b/scripts/smoke/cli/transport-checkpoint.py index 07e21f25c..bda5f98a2 100644 --- a/scripts/smoke/cli/transport-checkpoint.py +++ b/scripts/smoke/cli/transport-checkpoint.py @@ -16,6 +16,8 @@ means canonical VEOF, not a nonexistent PTY half-close. Bulk overlap is measured at the CLI operation boundary, not claimed as an instrumented guest-frame boundary. This is a POSIX harness. It never reads or stops sandboxes from the caller's MSB_HOME. +Use --home-parent /short/disk/path when /tmp has a RAM or per-user quota; every run still +creates and owns a fresh home underneath that directory. """ import argparse @@ -547,7 +549,9 @@ def __init__(self, args): # Reports may live under a long build directory; Unix sockets cannot. Allocate the # home independently, never borrow an existing path. Base cleanup still owns exactly # this freshly allocated directory and only removes it after catalog/PID verification. - self.home = Path(tempfile.mkdtemp(prefix="msb-t-", dir="/tmp")) + # Some Linux hosts mount /tmp as quota-limited tmpfs. Keep the short-path default, + # but permit an explicit disk-backed parent for archive and CoW memory fixtures. + self.home = Path(tempfile.mkdtemp(prefix="msb-t-", dir=args.home_parent)) self.env["MSB_HOME"] = str(self.home) self.report["home"] = str(self.home) # Make the CLI and its launched runtime an explicit matching pair. Do not inherit a @@ -1030,6 +1034,8 @@ def main(): parser.add_argument("--" + label + "-firmware", type=Path) parser.add_argument("--" + label + "-agentd", type=Path, help="Otherwise use embedded agentd") parser.add_argument("--output", type=Path, help="Exclusive new report directory") + parser.add_argument("--home-parent", type=Path, default=Path("/tmp"), + help="Existing short directory for a fresh isolated home (default: /tmp)") parser.add_argument("--samples", type=bounded_int, default=3) parser.add_argument("--input-modes", nargs="+", choices=("pipe", "pty"), default=["pipe", "pty"]) parser.add_argument("--cases", nargs="+", choices=("idle", "throughput", "stdin-throughput", "paused", "full", "branch", From df385fc6e4b4a7c11ba4907fa548a7056ee2b36d Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Thu, 10 Sep 2026 19:39:52 +0100 Subject: [PATCH 29/29] chore(docs): minor fix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e206c8147..607740d6b 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ ## - **Hardware Isolation**: Hardware-level isolation with microVM technology. -- **Branch & Snapshot**: Save sandbox running state and restore later. Fork live sandboxes. +- **Branch & Snapshot**: Save running sandbox state and restore later. Fork live sandboxes. - **Cross Platform**: Runs on Linux, macOS, and Windows. - **OCI Compatible**: Runs standard container images from Docker Hub, GHCR, or any OCI registry. - **Docker-Like Workflows**: Familiar image, command, shell, and volume workflows.