From e2562e5a31925aaa3ada1a869655278f641a905f Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Sun, 6 Sep 2026 01:53:42 +0100 Subject: [PATCH 1/6] feat(storage): grow owned root disks without restarting Extend the existing modify path to grow managed and flat ext4 roots, including checkpoint-backed qcow2 heads, without changing sealed layers. Persist pending growth for forward recovery and support staged stopped growth over the same logical disk chain. Add guest protocol generation 9 preflight and resize acknowledgments, per-layer snapshot capacities, user documentation, and a macOS live qualification harness and report. Pin the exact libkrun companion commit until the capacity API is released. Keep this change stacked directly on checkpoint-restore-clone (#6). Linux and Windows qualification and source publication remain pending. --- Cargo.lock | 40 ++- Cargo.toml | 14 + crates/agentd/lib/agent.rs | 31 +++ crates/agentd/lib/init.rs | 2 + crates/agentd/lib/lib.rs | 1 + crates/agentd/lib/root_disk.rs | 185 +++++++++++++ crates/image/lib/checkpoint/compact.rs | 46 +++- crates/image/lib/checkpoint/mod.rs | 4 +- crates/image/lib/ext4/chain.rs | 161 +++++++++++ crates/image/lib/ext4/format.rs | 2 + crates/image/lib/ext4/jbd2.rs | 34 ++- crates/image/lib/ext4/mod.rs | 3 + crates/image/lib/ext4/resize_inode.rs | 3 +- crates/image/lib/ext4/resizer.rs | 136 ++++++++-- crates/image/lib/ext4/storage.rs | 46 ++++ crates/protocol/lib/core.rs | 16 ++ crates/protocol/lib/message.rs | 16 +- crates/protocol/schema/gen-9.json | 168 ++++++++++++ crates/runtime/lib/checkpoint/coordinator.rs | 110 ++++++++ crates/runtime/lib/checkpoint/disk.rs | 256 +++++++++++++++++- crates/runtime/lib/checkpoint/mod.rs | 2 +- crates/runtime/lib/control.rs | 27 ++ crates/runtime/lib/control/executor.rs | 21 +- docs/sandboxes/tuning.mdx | 15 +- docs/sdk/go/sandbox.mdx | 7 +- docs/sdk/python/sandbox.mdx | 8 +- docs/sdk/typescript/sandbox.mdx | 8 +- scripts/smoke/cli/root-disk-growth.py | 118 ++++++++ .../reports/root-disk-growth-2026-09-06.md | 42 +++ sdk/rust/lib/runtime/spawn.rs | 33 ++- sdk/rust/lib/sandbox/modify.rs | 153 +++++++++-- sdk/rust/lib/snapshot/create.rs | 14 +- 32 files changed, 1586 insertions(+), 136 deletions(-) create mode 100644 crates/agentd/lib/root_disk.rs create mode 100644 crates/image/lib/ext4/chain.rs create mode 100644 crates/image/lib/ext4/storage.rs create mode 100644 crates/protocol/schema/gen-9.json create mode 100644 scripts/smoke/cli/root-disk-growth.py create mode 100644 scripts/smoke/reports/root-disk-growth-2026-09-06.md diff --git a/Cargo.lock b/Cargo.lock index b0a33d59b..92ac978cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3530,7 +3530,9 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b3c06ff73c7ce03e780887ec2389d62d2a2a9ddf471ab05c2ff69207cd3f3b4" dependencies = [ + "serde", "vmm-sys-util", + "zerocopy", ] [[package]] @@ -4351,8 +4353,7 @@ dependencies = [ [[package]] name = "msb_krun" version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4c0de669aeda774c3da60692af7e693fdf000f4813922b15389dd1209b042ac" +source = "git+https://github.com/superradcompany/libkrun?rev=ec9f119dcc144f1c011fe62ebd224d660c2191a3#ec9f119dcc144f1c011fe62ebd224d660c2191a3" dependencies = [ "crossbeam-channel", "kvm-bindings", @@ -4371,8 +4372,7 @@ dependencies = [ [[package]] name = "msb_krun_arch" version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e9631350df54bde880a7cf3fda67ff03132b115abae95f7b668599611286c87" +source = "git+https://github.com/superradcompany/libkrun?rev=ec9f119dcc144f1c011fe62ebd224d660c2191a3#ec9f119dcc144f1c011fe62ebd224d660c2191a3" dependencies = [ "kvm-bindings", "kvm-ioctls", @@ -4386,14 +4386,12 @@ dependencies = [ [[package]] name = "msb_krun_arch_gen" version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb11ff7f19adb1ee23d25ca65c9a06a75069e3b949e70922ba52a133dc9e43bb" +source = "git+https://github.com/superradcompany/libkrun?rev=ec9f119dcc144f1c011fe62ebd224d660c2191a3#ec9f119dcc144f1c011fe62ebd224d660c2191a3" [[package]] name = "msb_krun_cpuid" version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a315fabad3ffbf3cd6b4922dfa8e95b25e11624f548850c8029f46039c332a" +source = "git+https://github.com/superradcompany/libkrun?rev=ec9f119dcc144f1c011fe62ebd224d660c2191a3#ec9f119dcc144f1c011fe62ebd224d660c2191a3" dependencies = [ "kvm-bindings", "kvm-ioctls", @@ -4403,9 +4401,9 @@ dependencies = [ [[package]] name = "msb_krun_devices" version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b54befdee0428ae481ecd894306f2ceef19ad951275908bd4d9182a4216a4933" +source = "git+https://github.com/superradcompany/libkrun?rev=ec9f119dcc144f1c011fe62ebd224d660c2191a3#ec9f119dcc144f1c011fe62ebd224d660c2191a3" dependencies = [ + "bincode", "bitflags 1.3.2", "capng", "caps", @@ -4424,6 +4422,7 @@ dependencies = [ "msb_krun_utils", "nix 0.30.1", "rand 0.9.4", + "serde", "tokio", "virtio-bindings", "vm-fdt", @@ -4433,20 +4432,19 @@ dependencies = [ [[package]] name = "msb_krun_hvf" version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbde98ef6b71b920af6e4a8ee89110ffac6204942620799772dad12dcf2273ac" +source = "git+https://github.com/superradcompany/libkrun?rev=ec9f119dcc144f1c011fe62ebd224d660c2191a3#ec9f119dcc144f1c011fe62ebd224d660c2191a3" dependencies = [ "crossbeam-channel", "libloading 0.8.9", "log", "msb_krun_arch", + "serde", ] [[package]] name = "msb_krun_kernel" version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "080d35863dad4a81a5eb6c5ff9292caab0219e50e7c8b3d0da4ebe74e7797322" +source = "git+https://github.com/superradcompany/libkrun?rev=ec9f119dcc144f1c011fe62ebd224d660c2191a3#ec9f119dcc144f1c011fe62ebd224d660c2191a3" dependencies = [ "msb-vm-memory", "msb_krun_utils", @@ -4455,8 +4453,7 @@ dependencies = [ [[package]] name = "msb_krun_polly" version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af85cc51c6f8f196224793975aad7da7e51ef8247b42b5cd8f7635561a9d5fdb" +source = "git+https://github.com/superradcompany/libkrun?rev=ec9f119dcc144f1c011fe62ebd224d660c2191a3#ec9f119dcc144f1c011fe62ebd224d660c2191a3" dependencies = [ "libc", "msb_krun_utils", @@ -4465,8 +4462,7 @@ dependencies = [ [[package]] name = "msb_krun_smbios" version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f36a5c9866ccf755b4675338d4e696ee50e7425a153ac71db89e3bb95b1cffce" +source = "git+https://github.com/superradcompany/libkrun?rev=ec9f119dcc144f1c011fe62ebd224d660c2191a3#ec9f119dcc144f1c011fe62ebd224d660c2191a3" dependencies = [ "msb-vm-memory", ] @@ -4474,8 +4470,7 @@ dependencies = [ [[package]] name = "msb_krun_utils" version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112be8806546f721718a88ba01f9e86d4189aff39f79d5816a90465e9eb4dbe0" +source = "git+https://github.com/superradcompany/libkrun?rev=ec9f119dcc144f1c011fe62ebd224d660c2191a3#ec9f119dcc144f1c011fe62ebd224d660c2191a3" dependencies = [ "bitflags 1.3.2", "crossbeam-channel", @@ -4490,9 +4485,9 @@ dependencies = [ [[package]] name = "msb_krun_vmm" version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9865e5331c61d4dea48fae764f5c6a8248f83cd8d0f961dd03598b79885ace4" +source = "git+https://github.com/superradcompany/libkrun?rev=ec9f119dcc144f1c011fe62ebd224d660c2191a3#ec9f119dcc144f1c011fe62ebd224d660c2191a3" dependencies = [ + "bincode", "bzip2", "crossbeam-channel", "flate2", @@ -4511,6 +4506,7 @@ dependencies = [ "msb_krun_polly", "msb_krun_utils", "nix 0.30.1", + "serde", "windows-sys 0.61.2", "zstd", ] diff --git a/Cargo.toml b/Cargo.toml index ce06e3229..bcaae55ea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -205,3 +205,17 @@ parking_lot = "0.12" rpassword = "7" russh = "0.62.4" russh-sftp = "2.3.0" + +# Keep this stack reproducible until the block-capacity API is released. +[patch.crates-io] +msb_krun = { git = "https://github.com/superradcompany/libkrun", rev = "ec9f119dcc144f1c011fe62ebd224d660c2191a3" } +msb_krun_arch = { git = "https://github.com/superradcompany/libkrun", rev = "ec9f119dcc144f1c011fe62ebd224d660c2191a3" } +msb_krun_arch_gen = { git = "https://github.com/superradcompany/libkrun", rev = "ec9f119dcc144f1c011fe62ebd224d660c2191a3" } +msb_krun_cpuid = { git = "https://github.com/superradcompany/libkrun", rev = "ec9f119dcc144f1c011fe62ebd224d660c2191a3" } +msb_krun_devices = { git = "https://github.com/superradcompany/libkrun", rev = "ec9f119dcc144f1c011fe62ebd224d660c2191a3" } +msb_krun_hvf = { git = "https://github.com/superradcompany/libkrun", rev = "ec9f119dcc144f1c011fe62ebd224d660c2191a3" } +msb_krun_kernel = { git = "https://github.com/superradcompany/libkrun", rev = "ec9f119dcc144f1c011fe62ebd224d660c2191a3" } +msb_krun_polly = { git = "https://github.com/superradcompany/libkrun", rev = "ec9f119dcc144f1c011fe62ebd224d660c2191a3" } +msb_krun_smbios = { git = "https://github.com/superradcompany/libkrun", rev = "ec9f119dcc144f1c011fe62ebd224d660c2191a3" } +msb_krun_utils = { git = "https://github.com/superradcompany/libkrun", rev = "ec9f119dcc144f1c011fe62ebd224d660c2191a3" } +msb_krun_vmm = { git = "https://github.com/superradcompany/libkrun", rev = "ec9f119dcc144f1c011fe62ebd224d660c2191a3" } diff --git a/crates/agentd/lib/agent.rs b/crates/agentd/lib/agent.rs index 4b1391868..38ad96a25 100644 --- a/crates/agentd/lib/agent.rs +++ b/crates/agentd/lib/agent.rs @@ -490,6 +490,37 @@ async fn handle_message( heartbeat_control: &heartbeat::HeartbeatControl, ) -> AgentdResult<()> { match msg.t { + MessageType::RootDiskPrepare | MessageType::RootDiskGrow => { + let Some(request) = decode_payload_or_core_error::< + microsandbox_protocol::core::RootDiskGrow, + >(&msg, out_buf)? + else { + return Ok(()); + }; + let apply = msg.t == MessageType::RootDiskGrow; + let result = tokio::task::spawn_blocking(move || { + crate::root_disk::resize(request.size_bytes, apply) + }) + .await + .map_err(|e| AgentdError::ExecSession(format!("root grow worker: {e}")))?; + let reply = match result { + Ok(state) => Message::with_payload(MessageType::RootDiskState, msg.id, &state), + Err(message) => Message::with_payload( + MessageType::CoreError, + msg.id, + &CoreError { + kind: CoreErrorKind::CapabilityUnavailable, + message, + offending_type: Some(msg.t.as_str().into()), + }, + ), + } + .map_err(|e| AgentdError::ExecSession(format!("encode root capacity: {e}")))?; + codec::encode_to_buf(&reply, out_buf).map_err(|e| { + AgentdError::ExecSession(format!("encode root capacity frame: {e}")) + })?; + } + MessageType::Ping => { let Some(_) = decode_payload_or_core_error::(&msg, out_buf)? else { return Ok(()); diff --git a/crates/agentd/lib/init.rs b/crates/agentd/lib/init.rs index dd7f3882a..497869eaa 100644 --- a/crates/agentd/lib/init.rs +++ b/crates/agentd/lib/init.rs @@ -266,6 +266,7 @@ mod linux { match spec { BlockRootSpec::DiskImage { device, fstype } => { mount_disk_image(device, fstype.as_deref())?; + crate::root_disk::register("/newroot", device); } BlockRootSpec::OciErofs { lower, upper } => { mount_oci_erofs(lower, upper)?; @@ -328,6 +329,7 @@ mod linux { None::<&str>, ) .map_err(|e| AgentdError::Init(format!("mount {device} at {upperfs_dir}: {e}")))?; + crate::root_disk::register(upperfs_dir, device); } BlockRootUpper::Tmpfs { size_mib } => { let data = size_mib diff --git a/crates/agentd/lib/lib.rs b/crates/agentd/lib/lib.rs index 817bb4dc2..50185740a 100644 --- a/crates/agentd/lib/lib.rs +++ b/crates/agentd/lib/lib.rs @@ -9,6 +9,7 @@ mod config; mod error; mod rlimit; +mod root_disk; mod workload; //-------------------------------------------------------------------------------------------------- diff --git a/crates/agentd/lib/root_disk.rs b/crates/agentd/lib/root_disk.rs new file mode 100644 index 000000000..ba16449cd --- /dev/null +++ b/crates/agentd/lib/root_disk.rs @@ -0,0 +1,185 @@ +//! Online ext4 growth using pinned init-time descriptors, never a caller-supplied path. + +use std::fs::File; +use std::io; +use std::os::fd::AsRawFd; +use std::os::unix::fs::FileExt; +use std::sync::OnceLock; +use std::time::{Duration, Instant}; + +use microsandbox_protocol::core::RootDiskState; + +//-------------------------------------------------------------------------------------------------- +// Constants +//-------------------------------------------------------------------------------------------------- + +const EXT4_IOC_RESIZE_FS: libc::c_ulong = 0x4008_6610; +const BLKGETSIZE64: libc::c_ulong = 0x8008_1272; +static ROOT: OnceLock> = OnceLock::new(); + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +struct RootFiles { + mount: File, + device: File, +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +pub(crate) fn register(mount: &str, device: &str) { + // The managed ext4 mount becomes unreachable by path after pivot. Pin it now, just as + // teardown does. An unavailable capability must not prevent ordinary sandbox startup. + let _ = ROOT.set((|| { + Ok(RootFiles { + mount: File::open(mount)?, + device: File::open(device)?, + }) + })()); +} + +pub(crate) fn resize(target: u64, apply: bool) -> Result { + let files = ROOT + .get() + .ok_or("root has no pinned block filesystem")? + .as_ref() + .map_err(|e| format!("root disk unavailable: {e}"))?; + let caps = std::fs::read_to_string("/proc/self/status").map_err(|e| e.to_string())?; + let admin = caps + .lines() + .find_map(|line| line.strip_prefix("CapEff:\t")) + .and_then(|value| u64::from_str_radix(value.trim(), 16).ok()) + .is_some_and(|value| value & (1 << 24) != 0); + if !admin { + return Err("online ext4 growth requires agent CAP_SYS_RESOURCE".into()); + } + let mut sb = [0u8; 1024]; + files + .device + .read_exact_at(&mut sb, 1024) + .map_err(|e| e.to_string())?; + let current = validate_superblock(&sb, target)?; + let mut device_bytes = device_size(&files.device)?; + if apply && target > current { + // Virtio config change is asynchronous in the guest. Never issue the filesystem ioctl + // until the block driver's own capacity reflects the host target. + let deadline = Instant::now() + Duration::from_secs(10); + while device_bytes < target { + if Instant::now() >= deadline { + return Err("guest block capacity did not converge".into()); + } + std::thread::sleep(Duration::from_millis(2)); + device_bytes = device_size(&files.device)?; + } + let blocks = target / 4096; + // SAFETY: ioctl reads one u64 from a valid reference; mount is a pinned open directory. + if unsafe { libc::ioctl(files.mount.as_raw_fd(), EXT4_IOC_RESIZE_FS as _, &blocks) } < 0 { + return Err(format!( + "online ext4 expansion: {}", + io::Error::last_os_error() + )); + } + // Flush the committed superblock before observing it through the block descriptor. + if unsafe { libc::syncfs(files.mount.as_raw_fd()) } < 0 { + return Err(format!("sync grown ext4: {}", io::Error::last_os_error())); + } + files + .device + .read_exact_at(&mut sb, 1024) + .map_err(|e| e.to_string())?; + } + let actual = validate_superblock(&sb, target)?; + if apply && actual != target { + return Err(format!( + "filesystem capacity is {actual}, expected {target}" + )); + } + Ok(RootDiskState { + filesystem_bytes: actual, + device_bytes, + }) +} + +fn device_size(file: &File) -> Result { + let mut size = 0u64; + // SAFETY: BLKGETSIZE64 writes exactly one u64 into the supplied live storage. + if unsafe { libc::ioctl(file.as_raw_fd(), BLKGETSIZE64 as _, &mut size) } < 0 { + return Err(format!( + "read guest block capacity: {}", + io::Error::last_os_error() + )); + } + Ok(size) +} + +fn validate_superblock(sb: &[u8; 1024], target: u64) -> Result { + let u16_at = |i| u16::from_le_bytes([sb[i], sb[i + 1]]); + let u32_at = |i| u32::from_le_bytes(sb[i..i + 4].try_into().unwrap()); + if u16_at(56) != 0xef53 || u32_at(24) != 2 || u32_at(32) != 32768 { + return Err("online growth requires a supported 4 KiB ext4 root".into()); + } + let current_blocks = u64::from(u32_at(4)) | (u64::from(u32_at(0x150)) << 32); + let current = current_blocks + .checked_mul(4096) + .ok_or("ext4 size overflow")?; + if target < current || !target.is_multiple_of(4096) || target == 0 { + return Err("root growth requires a nondecreasing size aligned to 4 KiB".into()); + } + if target == current { + return Ok(current); + } + if u32_at(92) & 0x10 == 0 || u32_at(96) & 0x10 != 0 { + return Err( + "root ext4 lacks supported resize-inode metadata; offline preparation is required" + .into(), + ); + } + let descriptor_size = u64::from(u16_at(254)).max(32); + let groups = current_blocks.div_ceil(32768); + let gdt_blocks = (groups * descriptor_size).div_ceil(4096); + let max_groups = (gdt_blocks + u64::from(u16_at(206))) * 4096 / descriptor_size; + if target / 4096 > max_groups * 32768 { + return Err("target exceeds ext4 reserved group-descriptor capacity".into()); + } + let target_blocks = target / 4096; + let last_group = (target_blocks - 1) / 32768; + if last_group >= groups && !target_blocks.is_multiple_of(32768) { + let sparse = last_group <= 1 + || [3u64, 5, 7].iter().any(|base| { + let mut group = last_group; + while group > 1 && group.is_multiple_of(*base) { + group /= base; + } + group == 1 + }); + let inode_blocks = (u64::from(u32_at(40)) * u64::from(u16_at(88))).div_ceil(4096); + let metadata = inode_blocks + + 2 + + if sparse { + 1 + gdt_blocks + u64::from(u16_at(206)) + } else { + 0 + }; + if target_blocks % 32768 < metadata { + return Err("target leaves a final block group too small for ext4 metadata".into()); + } + } + Ok(current) +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_unknown_filesystems_before_mutation() { + assert!(validate_superblock(&[0; 1024], 1024 * 1024 * 1024).is_err()); + } +} diff --git a/crates/image/lib/checkpoint/compact.rs b/crates/image/lib/checkpoint/compact.rs index 8b84ab916..cba828c2d 100644 --- a/crates/image/lib/checkpoint/compact.rs +++ b/crates/image/lib/checkpoint/compact.rs @@ -42,6 +42,18 @@ pub struct CompactMaterialization { /// Open a complete, explicitly supplied immutable chain read-only. async fn open_chain(layers: &[CompactLayer]) -> io::Result { + open_chain_access(layers, false).await +} + +/// Resolve an owned chain with writes confined to its caller-private staging head. +pub(crate) async fn open_writable_chain(layers: &[CompactLayer]) -> io::Result { + open_chain_access(layers, true).await +} + +async fn open_chain_access( + layers: &[CompactLayer], + writable_head: bool, +) -> io::Result { if layers.is_empty() || layers.iter().skip(1).any(|layer| !layer.qcow2) { return Err(io::Error::new( io::ErrorKind::InvalidInput, @@ -49,12 +61,17 @@ async fn open_chain(layers: &[CompactLayer]) -> io::Result { )); } let mut backing: Option = None; - for layer in layers { - let storage: Box = - Box::new(ImagoFile::try_from(std::fs::File::open(&layer.path)?)?); + for (index, layer) in layers.iter().enumerate() { + let writable = writable_head && index + 1 == layers.len(); + let storage: Box = Box::new(ImagoFile::try_from( + std::fs::OpenOptions::new() + .read(true) + .write(writable) + .open(&layer.path)?, + )?); backing = Some(if layer.qcow2 { let image = Qcow2::, SharedImage>::builder(storage) - .write(false) + .write(writable) .backing(backing) .data_file(None) .open(DenyImplicitOpenGate::default()) @@ -69,7 +86,7 @@ async fn open_chain(layers: &[CompactLayer]) -> io::Result { } else { Arc::new(FormatAccess::new( Raw::>::builder(storage) - .write(false) + .write(writable) .open(DenyImplicitOpenGate::default()) .await?, )) @@ -155,6 +172,25 @@ pub async fn compact_layer_capacity(layer: CompactLayer) -> io::Result { Ok(open_chain(&[layer]).await?.size()) } +/// Read the capacities of a pinned closure from synchronous descriptor-building code. +/// A separate current-thread executor also permits use by callers already inside Tokio. +pub fn layer_capacities(layers: Vec) -> io::Result> { + std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()? + .block_on(async { + let mut capacities = Vec::with_capacity(layers.len()); + for layer in layers { + capacities.push(compact_layer_capacity(layer).await?); + } + Ok(capacities) + }) + }) + .join() + .map_err(|_| io::Error::other("layer-capacity worker panicked"))? +} + //-------------------------------------------------------------------------------------------------- // Tests //-------------------------------------------------------------------------------------------------- diff --git a/crates/image/lib/checkpoint/mod.rs b/crates/image/lib/checkpoint/mod.rs index ddf675a96..69ded76d9 100644 --- a/crates/image/lib/checkpoint/mod.rs +++ b/crates/image/lib/checkpoint/mod.rs @@ -14,8 +14,10 @@ mod store; // Re-Exports //-------------------------------------------------------------------------------------------------- +pub(crate) use compact::open_writable_chain; pub use compact::{ - CompactLayer, CompactMaterialization, compact_layer_capacity, materialize_compact_prefix, + CompactLayer, CompactMaterialization, compact_layer_capacity, layer_capacities, + materialize_compact_prefix, }; pub use layer_selection::{DiskCompactionPlan, DiskLayerExportPlan, LayerSelectionError}; pub use manifest::{ diff --git a/crates/image/lib/ext4/chain.rs b/crates/image/lib/ext4/chain.rs new file mode 100644 index 000000000..2d59a2d79 --- /dev/null +++ b/crates/image/lib/ext4/chain.rs @@ -0,0 +1,161 @@ +//! Offline ext4 growth over a caller-resolved chain with a private staging head. + +use std::io::{self, Read, Seek, SeekFrom, Write}; +use std::sync::Arc; + +use imago::{DynStorage, FormatAccess}; + +use super::formatter::Ext4Error; +use super::resizer::{GrowOutcome, grow_storage}; +use super::storage::Ext4Storage; +use crate::checkpoint::{CompactLayer, open_writable_chain}; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +struct LogicalDisk { + image: Arc>>, + runtime: tokio::runtime::Runtime, + position: u64, +} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +impl Read for LogicalDisk { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + let count = + (self.image.size().saturating_sub(self.position)).min(buffer.len() as u64) as usize; + self.runtime + .block_on(self.image.read(&mut buffer[..count], self.position))?; + self.position += count as u64; + Ok(count) + } +} + +impl Write for LogicalDisk { + fn write(&mut self, buffer: &[u8]) -> io::Result { + self.runtime + .block_on(self.image.write(buffer, self.position))?; + self.position += buffer.len() as u64; + Ok(buffer.len()) + } + + fn flush(&mut self) -> io::Result<()> { + self.runtime.block_on(self.image.flush()) + } +} + +impl Seek for LogicalDisk { + fn seek(&mut self, position: SeekFrom) -> io::Result { + let target = match position { + SeekFrom::Start(value) => i128::from(value), + SeekFrom::Current(delta) => i128::from(self.position) + i128::from(delta), + SeekFrom::End(delta) => i128::from(self.image.size()) + i128::from(delta), + }; + self.position = u64::try_from(target).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "invalid logical disk seek") + })?; + Ok(self.position) + } +} + +impl Ext4Storage for LogicalDisk { + fn length(&self) -> io::Result { + Ok(self.image.size()) + } + + fn grow(&mut self, size: u64) -> io::Result<()> { + self.runtime.block_on( + self.image + .resize_grow(size, imago::format::PreallocateMode::None), + ) + } + + fn sync_all(&self) -> io::Result<()> { + self.runtime.block_on(async { + self.image.flush().await?; + self.image.sync().await + }) + } +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +/// Grow ext4 in a private staging head without flattening or modifying its sealed ancestors. +/// +/// Call outside an async executor with the complete oldest-to-newest chain. The last file must +/// be an unpublished private copy: the offline filesystem rewrite is not rollback-safe on error. +/// The caller publishes the head only after success and excludes all concurrent writers. +pub fn grow_chain(layers: &[CompactLayer], size: u64) -> Result { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + let image = runtime.block_on(open_writable_chain(layers))?; + let mut disk = LogicalDisk { + image, + runtime, + position: 0, + }; + grow_storage(&mut disk, size, true) +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use crate::checkpoint::{create_qcow2_overlay, layer_capacities, sparse_file_integrity}; + use crate::ext4::{Ext4FormatOptions, format_ext4}; + + #[test] + fn qcow2_growth_keeps_ancestors_and_reconciles_completed_target() { + let dir = tempfile::tempdir().unwrap(); + let base = dir.path().join("base.raw"); + let head = dir.path().join("head.qcow2"); + let mib = 1024 * 1024; + format_ext4( + &base, + &Ext4FormatOptions { + size_bytes: 256 * mib, + journal_blocks: 4096, + }, + ) + .unwrap(); + let before = sparse_file_integrity(&base).unwrap().root; + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + rt.block_on(create_qcow2_overlay(&head, 256 * mib, &base, "raw")) + .unwrap(); + let layers = vec![ + CompactLayer { + path: base.clone(), + qcow2: false, + }, + CompactLayer { + path: head, + qcow2: true, + }, + ]; + let grown = grow_chain(&layers, 512 * mib).unwrap(); + assert_eq!(grown.old_blocks * 4096, 256 * mib); + assert_eq!(grown.new_blocks * 4096, 512 * mib); + assert_eq!( + layer_capacities(layers.clone()).unwrap(), + vec![256 * mib, 512 * mib] + ); + assert_eq!(sparse_file_integrity(&base).unwrap().root, before); + let replay = grow_chain(&layers, 512 * mib).unwrap(); + assert_eq!(replay.old_blocks, replay.new_blocks); + assert!(grow_chain(&layers, 128 * mib).is_err()); + assert_eq!(sparse_file_integrity(&base).unwrap().root, before); + } +} diff --git a/crates/image/lib/ext4/format.rs b/crates/image/lib/ext4/format.rs index cbdc9a4b9..7bef58d26 100644 --- a/crates/image/lib/ext4/format.rs +++ b/crates/image/lib/ext4/format.rs @@ -46,6 +46,8 @@ pub const EXT4_FEATURE_RO_COMPAT_METADATA_CSUM: u32 = 0x400; // Group descriptor flags pub const EXT4_BG_INODE_ZEROED: u16 = 0x04; +pub const EXT4_BG_INODE_UNINIT: u16 = 0x01; +pub const EXT4_BG_BLOCK_UNINIT: u16 = 0x02; // jbd2 constants (big-endian on disk) pub const JBD2_MAGIC: u32 = 0xC03B3998; diff --git a/crates/image/lib/ext4/jbd2.rs b/crates/image/lib/ext4/jbd2.rs index 9c9b21562..446b90a65 100644 --- a/crates/image/lib/ext4/jbd2.rs +++ b/crates/image/lib/ext4/jbd2.rs @@ -7,8 +7,9 @@ //! so a journal this module rejects leaves the image untouched. use std::collections::HashMap; -use std::fs::File; -use std::io::{Read, Seek, SeekFrom, Write}; +use std::io::SeekFrom; + +use super::storage::Ext4Storage; use super::format::{ EXT4_BLOCK_SIZE, EXT4_EH_MAGIC, EXT4_EXTENTS_FL, EXT4_INODE_SIZE, EXT4_JOURNAL_INO, JBD2_MAGIC, @@ -101,7 +102,7 @@ struct DescriptorTag { /// Locate the journal via inode 8, trusting nothing: the inode checksum, file type, extent flag, and the formatter's single depth-0 extent shape are all verified. The journal /// inode is never itself journaled, so the on-disk copy is authoritative even on a dirty image. pub(super) fn locate_journal( - file: &mut File, + file: &mut impl Ext4Storage, inode_table_block: u64, csum_seed: u32, ) -> Result { @@ -159,7 +160,7 @@ pub(super) fn locate_journal( /// [`Ext4Error::Unsupported`] with the image untouched. A journal with `s_start == 0` needs no recovery and is not written at all. After replay the data is fsynced, then the /// jbd2 superblock is rewritten with `s_start = 0` and `s_sequence` advanced past every replayed transaction so stale commit blocks can never match again. pub(super) fn recover_journal( - file: &mut File, + file: &mut impl Ext4Storage, loc: &JournalLocation, fs_uuid: &[u8; 16], fs_num_blocks: u64, @@ -219,7 +220,7 @@ pub(super) fn recover_journal( /// tag, commit) simply terminates the walk — the partially written transaction was never committed, so ignoring it is the correct crash semantics. Only impossible states /// (targets out of bounds, malformed tags or revoke counts) are hard errors. fn scan_log( - file: &mut File, + file: &mut impl Ext4Storage, loc: &JournalLocation, jsb: &JournalSuperblock, jseed: u32, @@ -315,7 +316,7 @@ fn scan_log( /// Read and strictly validate the jbd2 superblock: magic, v2 block type, checksum, feature masks (exactly the formatter's), crc32c checksum type, matching filesystem UUID, no /// recorded error, 4 KiB block size, and geometry fields consistent with the journal extent. fn read_journal_superblock( - file: &mut File, + file: &mut impl Ext4Storage, loc: &JournalLocation, fs_uuid: &[u8; 16], ) -> Result { @@ -471,7 +472,7 @@ fn tag_checksum_ok(data: &[u8], jseed: u32, seq: u32, expected: u32) -> bool { //-------------------------------------------------------------------------------------------------- fn read_log_block( - file: &mut File, + file: &mut impl Ext4Storage, loc: &JournalLocation, index: u32, ) -> Result, Ext4Error> { @@ -508,7 +509,7 @@ pub(super) struct TestTransaction { /// journal superblock) so replay fixtures exercise the same on-disk format the recovery code parses. Data blocks beginning with the jbd2 magic are escaped automatically. #[cfg(test)] pub(super) fn write_test_log( - file: &mut File, + file: &mut impl Ext4Storage, loc: &JournalLocation, fs_uuid: &[u8; 16], start_seq: u32, @@ -516,7 +517,12 @@ pub(super) fn write_test_log( ) -> Result<(), Ext4Error> { let jseed = crc32c::crc32c_raw(0xFFFF_FFFF, fs_uuid); let mut cursor = 1u32; - let emit = |file: &mut File, cursor: &mut u32, block: &[u8]| -> Result<(), Ext4Error> { + fn emit( + file: &mut impl Ext4Storage, + cursor: &mut u32, + block: &[u8], + loc: &JournalLocation, + ) -> Result<(), Ext4Error> { assert!( *cursor < loc.len_blocks, "test log fixture overflows the journal" @@ -527,7 +533,7 @@ pub(super) fn write_test_log( file.write_all(block)?; *cursor += 1; Ok(()) - }; + } for (index, txn) in transactions.iter().enumerate() { let seq = start_seq + index as u32; @@ -545,7 +551,7 @@ pub(super) fn write_test_log( off += 8; } set_tail_checksum(&mut block, jseed); - emit(file, &mut cursor, &block)?; + emit(file, &mut cursor, &block, loc)?; } if !txn.writes.is_empty() { @@ -583,9 +589,9 @@ pub(super) fn write_test_log( stored.push(data); } set_tail_checksum(&mut desc, jseed); - emit(file, &mut cursor, &desc)?; + emit(file, &mut cursor, &desc, loc)?; for data in &stored { - emit(file, &mut cursor, data)?; + emit(file, &mut cursor, data, loc)?; } } @@ -603,7 +609,7 @@ pub(super) fn write_test_log( checksum }, ); - emit(file, &mut cursor, &commit)?; + emit(file, &mut cursor, &commit, loc)?; } let mut raw = vec![0u8; JBD2_SB_SIZE]; diff --git a/crates/image/lib/ext4/mod.rs b/crates/image/lib/ext4/mod.rs index ab98b0ee1..878ee26cc 100644 --- a/crates/image/lib/ext4/mod.rs +++ b/crates/image/lib/ext4/mod.rs @@ -1,3 +1,4 @@ +mod chain; mod format; mod formatter; mod jbd2; @@ -5,11 +6,13 @@ mod layout; mod resize_inode; mod resizer; mod rootfs; +mod storage; //-------------------------------------------------------------------------------------------------- // Re-Exports //-------------------------------------------------------------------------------------------------- +pub use chain::grow_chain; pub use formatter::{Ext4Error, Ext4FormatOptions, format_ext4, format_ext4_with_tree}; pub use resizer::{GrowOutcome, grow_image}; pub use rootfs::{ diff --git a/crates/image/lib/ext4/resize_inode.rs b/crates/image/lib/ext4/resize_inode.rs index 1a631359c..00b4f4ecf 100644 --- a/crates/image/lib/ext4/resize_inode.rs +++ b/crates/image/lib/ext4/resize_inode.rs @@ -4,7 +4,6 @@ //! kernel and e2fsprogs. It owns the primary reserved-GDT blocks and their sparse-super backup //! copies; these blocks are filesystem metadata even though ext4 accounts for them through inode 7. -use std::fs::File; use std::io::{Read, Seek, SeekFrom, Write}; use super::format::{ @@ -68,7 +67,7 @@ pub(super) fn write_resize_inode( /// Validate inode 7 and every reserved-GDT pointer against the current filesystem geometry. pub(super) fn validate_resize_inode( - file: &mut File, + file: &mut (impl Read + Seek), geometry: &GroupGeometry, inode_table_block: u64, csum_seed: u32, diff --git a/crates/image/lib/ext4/resizer.rs b/crates/image/lib/ext4/resizer.rs index 352dfa10a..50cb42287 100644 --- a/crates/image/lib/ext4/resizer.rs +++ b/crates/image/lib/ext4/resizer.rs @@ -9,9 +9,12 @@ //! module) and then grown as clean images. use std::fs::{File, OpenOptions}; -use std::io::{Read, Seek, SeekFrom, Write}; +use std::io::SeekFrom; +#[cfg(test)] +use std::io::{Read, Seek, Write}; use std::path::Path; +use super::format::{EXT4_BG_BLOCK_UNINIT, EXT4_BG_INODE_UNINIT}; use super::format::{ EXT4_BG_INODE_ZEROED, EXT4_BLOCK_SIZE, EXT4_BLOCKS_PER_GROUP, EXT4_DESC_SIZE, EXT4_EH_MAGIC, EXT4_EXTENTS_FL, EXT4_FEATURE_COMPAT_DIR_INDEX, EXT4_FEATURE_COMPAT_EXT_ATTR, @@ -24,7 +27,7 @@ use super::format::{ EXT4_ROOT_INO, EXT4_SB_ERROR_COUNT_OFFSET, EXT4_SB_OVERHEAD_BLOCKS_OFFSET, EXT4_SUPER_MAGIC, S_IFDIR, sparse_super_group, }; -use super::formatter::{Ext4Error, mark_sparse}; +use super::formatter::Ext4Error; use super::jbd2; use super::layout::{ GroupDescStats, GroupGeometry, MAX_BLOCKS, bitmap_checksum, build_block_bitmap_base, @@ -33,6 +36,7 @@ use super::layout::{ write_backup_superblock_at, write_gdt_at, }; use super::resize_inode::{validate_resize_inode, write_resize_inode}; +use super::storage::Ext4Storage; use crate::crc32c; //-------------------------------------------------------------------------------------------------- @@ -176,6 +180,15 @@ impl ParsedImage { /// process is interrupted, callers must discard and recreate the artifact rather than use it. pub fn grow_image(path: &Path, new_size_bytes: u64) -> Result { let mut file = OpenOptions::new().read(true).write(true).open(path)?; + grow_storage(&mut file, new_size_bytes, false) +} + +/// Grow a caller-owned staging disk; errors must never publish the staging artifact. +pub(crate) fn grow_storage( + mut file: &mut impl Ext4Storage, + new_size_bytes: u64, + allow_completed_target: bool, +) -> Result { let mut img = parse_and_validate(&mut file)?; // Replay the journal before anything else: growing with a pending log would let the next kernel mount replay stale transactions over the appended GDT entries. After a @@ -201,6 +214,16 @@ pub fn grow_image(path: &Path, new_size_bytes: u64) -> Result Result Result> = None; if new_last_blocks > old_last_blocks { - let mut bitmap = read_block_at(&mut file, old_geo.group_block_bitmap_block(old_last))?; + let off = old_last as usize * EXT4_DESC_SIZE as usize; + let flags = get_le16(&gdt[off..], 0x12); + // Online ext4 growth may leave a lazy block bitmap. Its bytes are not meaningful + // until initialized, so derive the metadata-only bitmap rather than reading stale data. + let mut bitmap = if flags & EXT4_BG_BLOCK_UNINIT != 0 { + let expected_free = old_last_blocks - old_geo.group_metadata_blocks(old_last); + let recorded_free = u32::from(get_le16(&gdt[off..], 0x0C)) + | (u32::from(get_le16(&gdt[off..], 0x2C)) << 16); + if recorded_free != expected_free { + return Err(unsupported("uninitialized group has allocated data")); + } + build_block_bitmap_base(&old_geo, old_last) + } else { + read_block_at(&mut file, old_geo.group_block_bitmap_block(old_last))? + }; for bit in old_last_blocks..new_last_blocks { bitmap[(bit / 8) as usize] &= !(1 << (bit % 8)); } @@ -267,6 +303,7 @@ pub fn grow_image(path: &Path, new_size_bytes: u64) -> Result Result<(), Ext4Error> { /// Parse the primary superblock and GDT, refusing anything that does not match exactly what this /// crate's formatter writes (geometry, feature masks, per-group layout, checksums). -fn parse_and_validate(file: &mut File) -> Result { - let file_len = file.metadata()?.len(); +fn parse_and_validate(file: &mut impl Ext4Storage) -> Result { + let file_len = file.length()?; if file_len < SB_OFFSET + SB_SIZE as u64 { return Err(unsupported("file too small to contain an ext4 superblock")); } @@ -661,7 +698,12 @@ fn parse_and_validate(file: &mut File) -> Result { "group {group} metadata is not at the expected location" ))); } - if get_le16(desc, 0x12) != EXT4_BG_INODE_ZEROED { + let flags = get_le16(desc, 0x12); + let known_flags = EXT4_BG_INODE_ZEROED | EXT4_BG_INODE_UNINIT | EXT4_BG_BLOCK_UNINIT; + if flags & !known_flags != 0 + || (matches!(img.resize_metadata, ResizeMetadata::Legacy) + && flags != EXT4_BG_INODE_ZEROED) + { return Err(unsupported(format!("group {group} has unexpected flags"))); } let mut desc_copy = desc.to_vec(); @@ -705,7 +747,10 @@ fn parse_and_validate(file: &mut File) -> Result { /// zero inode 7, place the root directory immediately after the inode table, /// and leave every still-reserved primary GDT block sparse-zeroed. These /// invariants remain true after any number of grows by the legacy resizer. -fn validate_legacy_resize_metadata(file: &mut File, img: &ParsedImage) -> Result<(), Ext4Error> { +fn validate_legacy_resize_metadata( + file: &mut impl Ext4Storage, + img: &ParsedImage, +) -> Result<(), Ext4Error> { let geometry = img.geometry(); let resize_inode = read_inode(file, &geometry, EXT4_RESIZE_INO)?; if resize_inode.iter().any(|byte| *byte != 0) { @@ -772,7 +817,7 @@ fn validate_legacy_resize_metadata(file: &mut File, img: &ParsedImage) -> Result /// index path preserves that stable fingerprint while accepting extent fanout created by normal /// guest writes. fn first_extent_physical_block( - file: &mut File, + file: &mut impl Ext4Storage, img: &ParsedImage, inode_number: u32, inode: &[u8], @@ -854,7 +899,7 @@ fn first_extent_physical_block( } fn read_inode( - file: &mut File, + file: &mut impl Ext4Storage, geometry: &GroupGeometry, inode_number: u32, ) -> Result, Ext4Error> { @@ -873,7 +918,10 @@ fn read_inode( /// The journal is fully validated before its first write (see [`jbd2::recover_journal`]) and the backup superblocks are validated up front too, so an inconsistent image is /// refused untouched. The write ordering is crash-safe: replayed blocks are fsynced, then the jbd2 superblock is reset to empty, then RECOVER is cleared — a tear at any point /// leaves an image that the next attempt recovers to the same end state (replaying an already-emptied journal is a no-op). -fn replay_journal_and_clear_recover(file: &mut File, img: &ParsedImage) -> Result<(), Ext4Error> { +fn replay_journal_and_clear_recover( + file: &mut impl Ext4Storage, + img: &ParsedImage, +) -> Result<(), Ext4Error> { let geo = img.geometry(); let journal = jbd2::locate_journal(file, geo.group_inode_table_block(0), img.csum_seed)?; if journal.start_block + journal.len_blocks as u64 > img.num_blocks { @@ -918,7 +966,11 @@ fn replay_journal_and_clear_recover(file: &mut File, img: &ParsedImage) -> Resul } /// Read a 1024-byte superblock at `offset`, refusing bad magic or checksum. -fn read_superblock_at(file: &mut File, offset: u64, label: &str) -> Result, Ext4Error> { +fn read_superblock_at( + file: &mut impl Ext4Storage, + offset: u64, + label: &str, +) -> Result, Ext4Error> { let mut sb = vec![0u8; SB_SIZE]; file.seek(SeekFrom::Start(offset))?; file.read_exact(&mut sb)?; @@ -1007,7 +1059,7 @@ fn validate_group_bitmaps( } fn validate_allocated_inode( - file: &mut File, + file: &mut impl Ext4Storage, img: &ParsedImage, group: u32, local_inode: u32, @@ -1048,7 +1100,7 @@ fn validate_allocated_inode( } fn validate_inode_extent_tree( - file: &mut File, + file: &mut impl Ext4Storage, img: &ParsedImage, inode_number: u32, inode: &[u8], @@ -1144,7 +1196,7 @@ fn validate_extent_entries( } fn validate_external_xattrs( - file: &mut File, + file: &mut impl Ext4Storage, img: &ParsedImage, block_number: u64, inode_number: u32, @@ -1208,7 +1260,10 @@ fn validate_xattr_entries( Ok(()) } -fn validate_backup_metadata(file: &mut File, img: &ParsedImage) -> Result<(), Ext4Error> { +fn validate_backup_metadata( + file: &mut impl Ext4Storage, + img: &ParsedImage, +) -> Result<(), Ext4Error> { let geometry = img.geometry(); for group in 1..img.num_groups { if !sparse_super_group(group) { @@ -1231,14 +1286,14 @@ fn validate_backup_metadata(file: &mut File, img: &ParsedImage) -> Result<(), Ex Ok(()) } -fn read_block_at(file: &mut File, block: u64) -> Result, Ext4Error> { +fn read_block_at(file: &mut impl Ext4Storage, block: u64) -> Result, Ext4Error> { let mut buf = vec![0u8; EXT4_BLOCK_SIZE as usize]; file.seek(SeekFrom::Start(block * EXT4_BLOCK_SIZE as u64))?; file.read_exact(&mut buf)?; Ok(buf) } -fn write_block_at(file: &mut File, block: u64, data: &[u8]) -> Result<(), Ext4Error> { +fn write_block_at(file: &mut impl Ext4Storage, block: u64, data: &[u8]) -> Result<(), Ext4Error> { file.seek(SeekFrom::Start(block * EXT4_BLOCK_SIZE as u64))?; file.write_all(data)?; Ok(()) @@ -1278,6 +1333,49 @@ mod tests { format_ext4_legacy_for_test(path, &opts).unwrap(); } + #[test] + fn grow_initializes_a_lazy_partial_group_bitmap_before_extending_it() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("lazy.ext4"); + format_image(&path, 300 * MIB); + let mut file = OpenOptions::new() + .read(true) + .write(true) + .open(&path) + .unwrap(); + let image = parse_and_validate(&mut file).unwrap(); + let last = image.num_groups - 1; + let offset = last as usize * EXT4_DESC_SIZE as usize; + let mut descriptor = image.gdt[offset..offset + EXT4_DESC_SIZE as usize].to_vec(); + put_le16( + &mut descriptor, + 0x12, + EXT4_BG_INODE_ZEROED | EXT4_BG_BLOCK_UNINIT, + ); + put_le16(&mut descriptor, 0x1E, 0); + let checksum = gdt_checksum(image.csum_seed, last, &descriptor); + put_le16(&mut descriptor, 0x1E, checksum); + file.seek(SeekFrom::Start(4096 + offset as u64)).unwrap(); + file.write_all(&descriptor).unwrap(); + // Bytes of an uninitialized bitmap are deliberately meaningless. + write_block_at( + &mut file, + image.geometry().group_block_bitmap_block(last), + &[0xa5; 4096], + ) + .unwrap(); + file.sync_all().unwrap(); + drop(file); + grow_image(&path, 384 * MIB).unwrap(); + let mut file = File::open(&path).unwrap(); + let grown = parse_and_validate(&mut file).unwrap(); + let desc = &grown.gdt[offset..offset + EXT4_DESC_SIZE as usize]; + assert_eq!(get_le16(desc, 0x12), EXT4_BG_INODE_ZEROED); + let bitmap = + read_block_at(&mut file, grown.geometry().group_block_bitmap_block(last)).unwrap(); + assert_eq!(bitmap, build_block_bitmap_base(&grown.geometry(), last)); + } + /// Reproduce a root directory that has outgrown the inode's inline extent leaf. /// /// The kernel normally creates this shape after enough directory churn. Building the same diff --git a/crates/image/lib/ext4/storage.rs b/crates/image/lib/ext4/storage.rs new file mode 100644 index 000000000..7328ca432 --- /dev/null +++ b/crates/image/lib/ext4/storage.rs @@ -0,0 +1,46 @@ +//! Logical disk I/O shared by raw-file and explicitly resolved qcow2 offline growth. + +use std::fs::File; +use std::io::{self, Read, Seek, Write}; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// Offline filesystem storage. Callers must exclude all other writers. +pub(crate) trait Ext4Storage: Read + Write + Seek { + fn length(&self) -> io::Result; + fn grow(&mut self, size: u64) -> io::Result<()>; + fn sync_all(&self) -> io::Result<()>; +} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +impl Ext4Storage for File { + fn length(&self) -> io::Result { + Ok(self.metadata()?.len()) + } + + fn grow(&mut self, size: u64) -> io::Result<()> { + super::formatter::mark_sparse(self).map_err(io::Error::other)?; + self.set_len(size) + } + + fn sync_all(&self) -> io::Result<()> { + File::sync_all(self) + } +} + +impl Ext4Storage for &mut T { + fn length(&self) -> io::Result { + (**self).length() + } + fn grow(&mut self, size: u64) -> io::Result<()> { + (**self).grow(size) + } + fn sync_all(&self) -> io::Result<()> { + (**self).sync_all() + } +} diff --git a/crates/protocol/lib/core.rs b/crates/protocol/lib/core.rs index 02d9df746..644781809 100644 --- a/crates/protocol/lib/core.rs +++ b/crates/protocol/lib/core.rs @@ -106,6 +106,22 @@ pub struct WorkloadThawed { pub attempt_id: String, } +/// Root disk growth target, in bytes, used for preflight and apply. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RootDiskGrow { + /// Desired ext4 size; must be an aligned, nondecreasing target. + pub size_bytes: u64, +} + +/// Observed capacities of the guest root filesystem and its block device. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RootDiskState { + /// ext4 superblock size, including filesystem metadata. + pub filesystem_bytes: u64, + /// Capacity observed by the guest block driver. + pub device_bytes: u64, +} + /// Payload for `core.error` messages. /// /// Sent when a peer can identify a recoverable protocol error for a specific diff --git a/crates/protocol/lib/message.rs b/crates/protocol/lib/message.rs index ae9c41e49..ac92ba6b3 100644 --- a/crates/protocol/lib/message.rs +++ b/crates/protocol/lib/message.rs @@ -9,7 +9,7 @@ use crate::error::ProtocolResult; //-------------------------------------------------------------------------------------------------- /// Current protocol version. -pub const PROTOCOL_VERSION: u8 = 8; +pub const PROTOCOL_VERSION: u8 = 9; /// Frame flag: this is the last message for the given correlation ID. /// @@ -150,6 +150,18 @@ pub enum MessageType { #[strum(serialize = "core.workload.thawed")] WorkloadThawed, + /// Host checks mounted root-filesystem growth before changing block capacity. + #[strum(serialize = "core.root_disk.prepare")] + RootDiskPrepare, + + /// Host requests mounted root-filesystem expansion after block capacity changed. + #[strum(serialize = "core.root_disk.grow")] + RootDiskGrow, + + /// Guest reports the root filesystem and device capacities. + #[strum(serialize = "core.root_disk.state")] + RootDiskState, + /// Peer reports a recoverable protocol-level error. #[strum(serialize = "core.error")] CoreError, @@ -293,6 +305,7 @@ impl MessageType { | Self::Touched | Self::WorkloadFrozen | Self::WorkloadThawed + | Self::RootDiskState | Self::CoreError | Self::ExecExited | Self::ExecFailed @@ -351,6 +364,7 @@ impl MessageType { | Self::WorkloadFrozen | Self::WorkloadThaw | Self::WorkloadThawed => 8, + Self::RootDiskPrepare | Self::RootDiskGrow | Self::RootDiskState => 9, Self::TcpConnect | Self::TcpConnected | Self::TcpData diff --git a/crates/protocol/schema/gen-9.json b/crates/protocol/schema/gen-9.json new file mode 100644 index 000000000..bf9baf54a --- /dev/null +++ b/crates/protocol/schema/gen-9.json @@ -0,0 +1,168 @@ +{ + "frame": { + "flag_session_start": 2, + "flag_shutdown": 4, + "flag_terminal": 1, + "header_size": 5, + "max_frame_size": 4194304 + }, + "message_types": [ + { + "introduced_in": 1, + "wire": "core.ready" + }, + { + "introduced_in": 1, + "wire": "core.init.resolved" + }, + { + "introduced_in": 1, + "wire": "core.init.ack" + }, + { + "introduced_in": 1, + "wire": "core.shutdown" + }, + { + "introduced_in": 1, + "wire": "core.relay.client.disconnected" + }, + { + "introduced_in": 1, + "wire": "core.clock.sync" + }, + { + "introduced_in": 6, + "wire": "core.ping" + }, + { + "introduced_in": 6, + "wire": "core.pong" + }, + { + "introduced_in": 6, + "wire": "core.touch" + }, + { + "introduced_in": 6, + "wire": "core.touched" + }, + { + "introduced_in": 8, + "wire": "core.workload.freeze" + }, + { + "introduced_in": 8, + "wire": "core.workload.frozen" + }, + { + "introduced_in": 8, + "wire": "core.workload.thaw" + }, + { + "introduced_in": 8, + "wire": "core.workload.thawed" + }, + { + "introduced_in": 9, + "wire": "core.root_disk.prepare" + }, + { + "introduced_in": 9, + "wire": "core.root_disk.grow" + }, + { + "introduced_in": 9, + "wire": "core.root_disk.state" + }, + { + "introduced_in": 5, + "wire": "core.error" + }, + { + "introduced_in": 1, + "wire": "core.exec.request" + }, + { + "introduced_in": 1, + "wire": "core.exec.started" + }, + { + "introduced_in": 1, + "wire": "core.exec.stdin" + }, + { + "introduced_in": 1, + "wire": "core.exec.stdin.error" + }, + { + "introduced_in": 1, + "wire": "core.exec.stdout" + }, + { + "introduced_in": 1, + "wire": "core.exec.stderr" + }, + { + "introduced_in": 1, + "wire": "core.exec.exited" + }, + { + "introduced_in": 1, + "wire": "core.exec.failed" + }, + { + "introduced_in": 1, + "wire": "core.exec.resize" + }, + { + "introduced_in": 1, + "wire": "core.exec.signal" + }, + { + "introduced_in": 2, + "wire": "core.fs.request" + }, + { + "introduced_in": 2, + "wire": "core.fs.response" + }, + { + "introduced_in": 2, + "wire": "core.fs.data" + }, + { + "introduced_in": 4, + "wire": "core.tcp.connect" + }, + { + "introduced_in": 4, + "wire": "core.tcp.connected" + }, + { + "introduced_in": 4, + "wire": "core.tcp.data" + }, + { + "introduced_in": 4, + "wire": "core.tcp.eof" + }, + { + "introduced_in": 4, + "wire": "core.tcp.close" + }, + { + "introduced_in": 4, + "wire": "core.tcp.closed" + }, + { + "introduced_in": 4, + "wire": "core.tcp.failed" + }, + { + "introduced_in": 7, + "wire": "core.bootstrap" + } + ], + "protocol_version": 9 +} diff --git a/crates/runtime/lib/checkpoint/coordinator.rs b/crates/runtime/lib/checkpoint/coordinator.rs index a4fe7cc4c..06534b70b 100644 --- a/crates/runtime/lib/checkpoint/coordinator.rs +++ b/crates/runtime/lib/checkpoint/coordinator.rs @@ -137,6 +137,15 @@ impl CheckpointCoordinator { layers: Option, dry_run: bool, ) -> Result { + if self + .root_disk + .as_ref() + .is_some_and(|disk| disk.growth_pending()) + { + return Err(super::disk::RootDiskRolloverError::pre_rebind( + "complete pending root-disk growth before compaction", + )); + } match self.root_disk.as_mut() { Some(disk) => disk.compact(Some(vm), &self.runtime, layers, dry_run), None => Err(super::disk::RootDiskRolloverError::pre_rebind( @@ -145,6 +154,74 @@ impl CheckpointCoordinator { } } + pub(crate) fn grow_root( + &mut self, + vm: &msb_krun::VmControl, + size_bytes: u64, + ) -> Result { + use super::disk::RootDiskRolloverError as Failure; + let started = Instant::now(); + let disk = self + .root_disk + .as_mut() + .ok_or_else(|| Failure::pre_rebind("root is not runtime-owned"))?; + let client = self + .runtime + .block_on(AgentClient::connect_with_timeout( + &self.agent_sock, + WORKLOAD_CONTROL_TIMEOUT, + )) + .map_err(Failure::pre_rebind)?; + // Gate the protocol and validate ext4 before mutating either disk or recovery state. + let request = microsandbox_protocol::core::RootDiskGrow { size_bytes }; + self.runtime + .block_on(root_growth_request( + &client, + MessageType::RootDiskPrepare, + &request, + )) + .map_err(Failure::pre_rebind)?; + disk.begin_growth(size_bytes).map_err(Failure::pre_rebind)?; + let paused_at = Instant::now(); + let pause = vm.pause().map_err(Failure::pre_rebind)?; + if let Err(error) = vm.grow_block_capacity(disk.device_id(), size_bytes) { + // The image may already be larger. Fence execution until restart reopens actual + // capacity; retrying this target then completes the filesystem phase. + return Err(Failure::post_journal(format!( + "root block growth requires forward recovery: {error}" + ))); + } + vm.resume(pause).map_err(Failure::post_journal)?; + let pause_us = paused_at.elapsed().as_micros() as u64; + let guest_started = Instant::now(); + let state = self + .runtime + .block_on(root_growth_request( + &client, + MessageType::RootDiskGrow, + &request, + )) + .map_err(|e| { + Failure::pre_rebind(format!( + "block device grew; filesystem completion is pending, retry this target: {e}" + )) + })?; + if state.filesystem_bytes != size_bytes || state.device_bytes < size_bytes { + return Err(Failure::pre_rebind( + "guest did not acknowledge the requested root capacity; retry to finish growth", + )); + } + let guest_us = guest_started.elapsed().as_micros() as u64; + disk.finish_growth().map_err(Failure::pre_rebind)?; + Ok(crate::control::RootDiskGrowthResult { + filesystem_bytes: state.filesystem_bytes, + device_bytes: state.device_bytes, + total_us: started.elapsed().as_micros() as u64, + pause_us, + guest_us, + }) + } + /// Open the per-runtime object store and managed root-disk state. pub(crate) fn open( runtime_dir: &Path, @@ -186,6 +263,15 @@ impl CheckpointCoordinator { checkpoint_id: &str, intent: CaptureIntent, ) -> Result { + if self + .root_disk + .as_ref() + .is_some_and(|disk| disk.growth_pending()) + { + return Err(CheckpointFailure::before_pause( + "complete pending root-disk growth before checkpointing", + )); + } let total_started = Instant::now(); validate_checkpoint_id(checkpoint_id).map_err(CheckpointFailure::before_pause)?; validate_vm_generation_state(vm.vm_generation_state()) @@ -847,6 +933,30 @@ impl MemoryCaptureSink for MemoryObjectSink<'_> { // Functions //-------------------------------------------------------------------------------------------------- +async fn root_growth_request( + client: &AgentClient, + message_type: MessageType, + request: µsandbox_protocol::core::RootDiskGrow, +) -> Result { + let reply = tokio::time::timeout( + Duration::from_secs(120), + client.request(message_type, request), + ) + .await + .map_err(|_| "guest root growth timed out".to_string())? + .map_err(|e| e.to_string())?; + if reply.t == MessageType::CoreError { + return Err(reply + .payload::() + .map_err(|e| e.to_string())? + .message); + } + if reply.t != MessageType::RootDiskState { + return Err("unexpected root growth response".into()); + } + reply.payload().map_err(|e| e.to_string()) +} + fn validate_workload_reply( reply: Message, expected_type: MessageType, diff --git a/crates/runtime/lib/checkpoint/disk.rs b/crates/runtime/lib/checkpoint/disk.rs index 8fc4b0514..b951763d6 100644 --- a/crates/runtime/lib/checkpoint/disk.rs +++ b/crates/runtime/lib/checkpoint/disk.rs @@ -43,7 +43,7 @@ pub(crate) struct RuntimeOwnedRootDisk { pub struct RuntimeOwnedRootChain { /// Guest-visible block device backed by this chain. pub device_id: String, - /// Guest-visible capacity shared by every layer in the chain. + /// Guest-visible capacity of the writable head; sealed ancestors may be smaller. pub virtual_size: u64, /// Complete oldest-to-head physical closure. pub layers: Vec, @@ -102,6 +102,10 @@ struct RootDiskState { /// Original launch configuration binding, retained across representation-only compaction. #[serde(default, skip_serializing_if = "Option::is_none")] launch_base: Option, + /// Unfinished forward-only growth. Older runtime readers refuse this field rather than + /// accepting a chain whose filesystem expansion has not been acknowledged. + #[serde(default, skip_serializing_if = "Option::is_none")] + growth_target: Option, layers: Vec, } @@ -134,6 +138,60 @@ struct RootDiskLayer { //-------------------------------------------------------------------------------------------------- impl RuntimeOwnedRootDisk { + pub(crate) fn growth_pending(&self) -> bool { + self.state.growth_target.is_some() + } + + pub(crate) fn begin_growth(&mut self, target: u64) -> Result<(), String> { + let capacities = microsandbox_image::checkpoint::layer_capacities( + self.state + .layers + .iter() + .map(|layer| CompactLayer { + path: layer.path.clone(), + qcow2: layer.format == RootDiskFormat::Qcow2, + }) + .collect(), + ) + .map_err(|e| e.to_string())?; + let current = *capacities.last().expect("validated nonempty chain"); + if target < current + || target == 0 + || !target.is_multiple_of(4096) + || capacities.iter().any(|size| *size > current) + { + return Err("root growth requires an aligned nondecreasing capacity with no larger backing ancestor".into()); + } + if self + .state + .growth_target + .is_some_and(|pending| pending != target) + { + return Err( + "complete the pending root-disk growth target before requesting another size" + .into(), + ); + } + let mut next = self.state.clone(); + next.growth_target = Some(target); + // A failed earlier capture may have cached a head hash. Growth changes its bytes; + // no future snapshot may reuse that cached identity after mutation starts. + if let Some(head) = next.layers.last_mut() { + head.integrity_root = None; + } + write_state(&self.state_path, &next)?; + self.state = next; + Ok(()) + } + + pub(crate) fn finish_growth(&mut self) -> Result<(), String> { + let mut next = self.state.clone(); + next.growth_target = None; + write_state(&self.state_path, &next)?; + self.state = next; + Ok(()) + } + /// Open the authoritative chain journal or initialize it from a sandbox-owned root disk. pub(crate) fn open(runtime_dir: &Path, vm: &VmConfig) -> Result, String> { let Some(layout) = configured_layout(vm) else { @@ -154,6 +212,7 @@ impl RuntimeOwnedRootDisk { layout, published_generation: 0, launch_base: None, + growth_target: None, layers: layers .into_iter() .map(|layer| RootDiskLayer { @@ -192,6 +251,11 @@ impl RuntimeOwnedRootDisk { layers: Option, dry_run: bool, ) -> Result { + if self.growth_pending() { + return Err(RootDiskRolloverError::pre_rebind( + "complete pending root-disk growth before compaction", + )); + } let started = Instant::now(); let plan = DiskCompactionPlan::new(self.state.layers.len(), layers) .map_err(RootDiskRolloverError::pre_rebind)?; @@ -354,6 +418,17 @@ impl RuntimeOwnedRootDisk { .published_generation .checked_add(1) .ok_or_else(|| RootDiskRolloverError::pre_rebind("disk generation is exhausted"))?; + let capacities = microsandbox_image::checkpoint::layer_capacities( + self.state + .layers + .iter() + .map(|layer| microsandbox_image::checkpoint::CompactLayer { + path: layer.path.clone(), + qcow2: layer.format == RootDiskFormat::Qcow2, + }) + .collect(), + ) + .map_err(RootDiskRolloverError::pre_rebind)?; let sealed_layers = self .state .layers @@ -362,7 +437,7 @@ impl RuntimeOwnedRootDisk { .map(|(index, layer)| DiskLayerRef { layer_id: layer.layer_id.clone(), format: layer.format.as_str().into(), - virtual_size, + virtual_size: capacities[index], predecessor: index .checked_sub(1) .map(|previous| self.state.layers[previous].layer_id.clone()), @@ -433,6 +508,9 @@ impl RootDiskState { || self.device_id != self.layout.device_id() || self.layers.is_empty() || self.layers.len() > 256 + || self + .growth_target + .is_some_and(|target| target == 0 || !target.is_multiple_of(4096)) { return Err("runtime-owned root-disk state has invalid identity or bounds".into()); } @@ -525,7 +603,7 @@ impl RootDiskRolloverError { } } - fn post_journal(error: impl fmt::Display) -> Self { + pub(crate) fn post_journal(error: impl fmt::Display) -> Self { Self { message: error.to_string(), keep_paused: true, @@ -593,6 +671,9 @@ pub fn load_runtime_owned_root_chain( return Ok(None); } let state = read_state(&state_path)?; + if state.growth_target.is_some() { + return Err("complete pending root-disk growth before snapshotting".into()); + } let head = state .layers .last() @@ -629,6 +710,94 @@ pub fn load_runtime_owned_root_chain( })) } +/// Grow a stopped journal-backed root using a private staging head. Returns false without a journal. +/// The caller must hold the sandbox lifecycle lock and prove all VM writers have stopped. +pub fn grow_stopped_root(runtime_dir: &Path, target: u64) -> Result { + let state_path = runtime_dir.join(ROOT_DISK_STATE_FILE); + if !state_path.exists() { + return Ok(false); + } + let state = read_state(&state_path)?; + if state.growth_target.is_some_and(|pending| pending != target) { + return Err(format!( + "complete pending root-disk growth to {} bytes first", + state.growth_target.unwrap() + )); + } + let capacities = microsandbox_image::checkpoint::layer_capacities( + state + .layers + .iter() + .map(|layer| CompactLayer { + path: layer.path.clone(), + qcow2: layer.format == RootDiskFormat::Qcow2, + }) + .collect(), + ) + .map_err(|e| e.to_string())?; + let current = *capacities.last().expect("validated nonempty chain"); + if target < current && state.growth_target.is_none() { + return Ok(true); + } + if target == current && state.growth_target.is_none() { + return Ok(true); + } + if capacities.iter().any(|size| *size > current) { + return Err("cannot grow a chain with a backing layer larger than its head".into()); + } + let stage = tempfile::Builder::new() + .prefix(".grow-") + .tempdir_in(runtime_dir) + .map_err(|e| e.to_string())?; + let mut next = state.clone(); + if next.launch_base.is_none() { + next.launch_base = state.layers.first().map(|layer| layer.path.clone()); + } + let last = state.layers.len() - 1; + for (index, layer) in next.layers.iter_mut().enumerate() { + let path = stage + .path() + .join(layer.path.file_name().ok_or("invalid root layer name")?); + if index == last { + microsandbox_utils::copy::fast_copy(&layer.path, &path).map_err(|e| e.to_string())?; + layer.layer_id = new_id("layer"); + layer.integrity_root = None; + } else { + // Preserve qcow2's relative backing bindings for independent inspection as well as + // the runtime's explicit closure. Ancestor inodes are never opened writable. + std::fs::hard_link(&layer.path, &path).map_err(|e| e.to_string())?; + } + layer.path = path; + } + let closure = next + .layers + .iter() + .map(|layer| CompactLayer { + path: layer.path.clone(), + qcow2: layer.format == RootDiskFormat::Qcow2, + }) + .collect::>(); + microsandbox_image::ext4::grow_chain(&closure, target).map_err(|e| e.to_string())?; + next.growth_target = None; + sync_directory(stage.path()).map_err(|e| e.to_string())?; + // Preserve staging across an uncertain journal rename/fsync. Recovery can then follow + // whichever durable journal won, without ever opening a partially rewritten filesystem. + let _published = stage.keep(); + write_state(&state_path, &next)?; + Ok(true) +} + +/// Finish a previously recorded online growth before cold boot, under the stopped lifecycle lock. +pub fn recover_stopped_root_growth(runtime_dir: &Path) -> Result<(), String> { + let path = runtime_dir.join(ROOT_DISK_STATE_FILE); + if path.exists() + && let Some(target) = read_state(&path)?.growth_target + { + grow_stopped_root(runtime_dir, target)?; + } + Ok(()) +} + /// Compact a stopped runtime-owned root after the caller acquires the sandbox lifecycle lock. /// The caller must prove no live process can write this disk until the operation finishes. pub fn compact_stopped_root( @@ -846,6 +1015,83 @@ fn sync_directory(path: &Path) -> std::io::Result<()> { #[cfg(test)] mod tests { + #[test] + fn stopped_growth_preserves_ancestors_and_recovers_pending_target() { + use super::*; + use microsandbox_image::ext4::{Ext4FormatOptions, format_ext4}; + for layout in [RootDiskLayout::ManagedUpper, RootDiskLayout::FlatRoot] { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("runtime"); + std::fs::create_dir(&root).unwrap(); + let base = dir.path().join("base.raw"); + let mib = 1024 * 1024; + format_ext4( + &base, + &Ext4FormatOptions { + size_bytes: 256 * mib, + journal_blocks: 4096, + }, + ) + .unwrap(); + let original = sparse_file_integrity(&base).unwrap().root; + let head = root.join("head.qcow2"); + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + rt.block_on(microsandbox_image::checkpoint::create_qcow2_overlay( + &head, + 256 * mib, + &base, + "raw", + )) + .unwrap(); + let mut state = RootDiskState { + schema: ROOT_DISK_STATE_SCHEMA.into(), + volume_id: new_id("vol"), + device_id: layout.device_id().into(), + layout, + published_generation: 1, + launch_base: None, + growth_target: Some(512 * mib), + layers: vec![ + RootDiskLayer { + layer_id: new_id("layer"), + path: base.clone(), + format: RootDiskFormat::Raw, + integrity_root: Some(original.clone()), + }, + RootDiskLayer { + layer_id: new_id("layer"), + path: head, + format: RootDiskFormat::Qcow2, + integrity_root: None, + }, + ], + }; + let journal = root.join(ROOT_DISK_STATE_FILE); + write_state(&journal, &state).unwrap(); + assert!(load_runtime_owned_root_chain(&root).is_err()); + assert!(compact_stopped_root(&root, None, false).is_err()); + assert!(grow_stopped_root(&root, 768 * mib).is_err()); + recover_stopped_root_growth(&root).unwrap(); + let chain = load_runtime_owned_root_chain(&root).unwrap().unwrap(); + assert_eq!(chain.virtual_size, 512 * mib); + assert_eq!(chain.layers.len(), 2); + assert_eq!(sparse_file_integrity(&base).unwrap().root, original); + // Simulate loss of the final acknowledgment after the filesystem already grew. + state = read_state(&journal).unwrap(); + state.growth_target = Some(512 * mib); + write_state(&journal, &state).unwrap(); + recover_stopped_root_growth(&root).unwrap(); + assert!(read_state(&journal).unwrap().growth_target.is_none()); + let before = std::fs::read(&journal).unwrap(); + assert!(grow_stopped_root(&root, 512 * mib + 1).is_err()); + assert_eq!(std::fs::read(&journal).unwrap(), before); + assert_eq!(sparse_file_integrity(&base).unwrap().root, original); + } + } + #[test] fn stopped_compaction_preserves_head_and_recovers_both_layouts() { use super::*; @@ -902,6 +1148,7 @@ mod tests { layout, published_generation: 3, launch_base: None, + growth_target: None, layers, }, ) @@ -979,6 +1226,7 @@ mod tests { layout: RootDiskLayout::ManagedUpper, published_generation: 1, launch_base: None, + growth_target: None, layers: vec![ RootDiskLayer { layer_id: new_id("layer"), @@ -1025,6 +1273,7 @@ mod tests { layout: RootDiskLayout::FlatRoot, published_generation: 0, launch_base: None, + growth_target: None, layers: vec![RootDiskLayer { layer_id: new_id("layer"), path: base.clone(), @@ -1065,6 +1314,7 @@ mod tests { layout: RootDiskLayout::ManagedUpper, published_generation: 0, launch_base: None, + growth_target: None, layers: vec![RootDiskLayer { layer_id: new_id("layer"), path: "upper.ext4".into(), diff --git a/crates/runtime/lib/checkpoint/mod.rs b/crates/runtime/lib/checkpoint/mod.rs index cb74614e1..a7b947bce 100644 --- a/crates/runtime/lib/checkpoint/mod.rs +++ b/crates/runtime/lib/checkpoint/mod.rs @@ -12,7 +12,7 @@ pub(crate) use coordinator::{CheckpointCoordinator, CheckpointResult}; pub(crate) use disk::recover_runtime_owned_root; pub use disk::{ DiskCompactionResult, RuntimeOwnedRootChain, RuntimeOwnedRootLayer, compact_stopped_root, - load_runtime_owned_root_chain, + grow_stopped_root, load_runtime_owned_root_chain, recover_stopped_root_growth, }; pub(crate) use restore::{PreparedCheckpointRestore, RestoredAgentState}; diff --git a/crates/runtime/lib/control.rs b/crates/runtime/lib/control.rs index 4f2b9b0f4..78dfda8ad 100644 --- a/crates/runtime/lib/control.rs +++ b/crates/runtime/lib/control.rs @@ -42,6 +42,11 @@ pub const CONTROL_SOCKET_EXTENSION: &str = "control.sock"; #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(tag = "op", rename_all = "snake_case")] pub enum ControlRequest { + /// Grow the owned root disk and mounted ext4 filesystem without rebooting. + RootDiskGrow { + /// Target capacity in bytes. + size_bytes: u64, + }, /// Explicitly consolidate the oldest sealed root-disk layers. DiskCompact { /// Oldest layer count including the base; omitted selects all sealed layers. @@ -140,6 +145,9 @@ pub struct SecretValue(pub String); /// The reply to any control request. #[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct ControlResponse { + /// Guest-observed root capacities after successful filesystem expansion. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub root_disk: Option, /// Explicit disk-compaction result or dry-run projection. #[serde(default, skip_serializing_if = "Option::is_none")] pub compaction: Option, @@ -189,12 +197,30 @@ pub struct CheckpointControlState { pub memory_emitted_bytes: u64, } +/// Verified capacity and measured phases of a completed online root growth. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RootDiskGrowthResult { + /// Committed ext4 capacity in bytes. + pub filesystem_bytes: u64, + /// Guest-observed virtio-block capacity in bytes. + pub device_bytes: u64, + /// Total runtime operation time, including preflight and persistence. + pub total_us: u64, + /// VM pause through resume; excludes online filesystem expansion. + pub pause_us: u64, + /// Guest expansion and verification time after VM resume. + pub guest_us: u64, +} + /// Live-control operations supported by this sandbox process, carried in /// [`ControlResponse`]. Runtimes that predate this op only served the socket /// when they could resize, so the SDK treats a missing reply as /// resize-capable and secrets-incapable. #[derive(Debug, Default, Clone, Copy, Serialize, Deserialize)] pub struct ControlCapabilities { + /// Host control supports root growth; guest capability is checked before mutation. + #[serde(default)] + pub root_disk_grow: bool, /// Explicit root-disk prefix compaction is supported. #[serde(default)] pub disk_compact: bool, @@ -556,6 +582,7 @@ mod tests { let response = ControlResponse { ok: true, capabilities: Some(ControlCapabilities { + root_disk_grow: true, disk_compact: true, cpu_resize: true, memory_resize: false, diff --git a/crates/runtime/lib/control/executor.rs b/crates/runtime/lib/control/executor.rs index 7045c58fd..33f8ae06b 100644 --- a/crates/runtime/lib/control/executor.rs +++ b/crates/runtime/lib/control/executor.rs @@ -245,6 +245,7 @@ impl RuntimeControlExecutor { let mutation = matches!( request, ControlRequest::MemoryTarget { .. } + | ControlRequest::RootDiskGrow { .. } | ControlRequest::CpuTarget { .. } | ControlRequest::SecretsUpdate { .. } | ControlRequest::CheckpointCreate { .. } @@ -258,6 +259,21 @@ impl RuntimeControlExecutor { } let response = match request { + ControlRequest::RootDiskGrow { size_bytes } => { + match state.checkpoint.grow_root(&self.vm, size_bytes) { + Ok(root_disk) => ControlResponse { + ok: true, + root_disk: Some(root_disk), + ..Default::default() + }, + Err(error) => { + if error.keep_paused { + state.lifecycle = RuntimeLifecycle::Quiesced; + } + control_error("root_disk_growth_incomplete", error.to_string()) + } + } + } ControlRequest::DiskCompact { layers, dry_run } => { match state.checkpoint.compact(&self.vm, layers, dry_run) { Ok(result) => ControlResponse { @@ -379,6 +395,7 @@ impl RuntimeControlExecutor { secrets_update: self.secrets_update_supported(), checkpoint_create: true, disk_compact: true, + root_disk_grow: true, }), ..Default::default() }, @@ -397,7 +414,9 @@ impl RuntimeControlExecutor { } ControlRequest::CpuState => cpu(self.vm.cpu_state()), ControlRequest::SecretsUpdate { changes } => self.handle_secrets_update(changes), - ControlRequest::CheckpointCreate { .. } | ControlRequest::DiskCompact { .. } => { + ControlRequest::CheckpointCreate { .. } + | ControlRequest::DiskCompact { .. } + | ControlRequest::RootDiskGrow { .. } => { unreachable!("checkpoint requests are handled by the executor lifecycle path") } } diff --git a/docs/sandboxes/tuning.mdx b/docs/sandboxes/tuning.mdx index 08f4086fd..d8382c997 100644 --- a/docs/sandboxes/tuning.mdx +++ b/docs/sandboxes/tuning.mdx @@ -342,46 +342,47 @@ Resize an OCI sandbox's root disk through `modify`: ```rust Rust sb.modify() .root_disk_size_mib(8192) - .restart() .apply() .await?; ``` ```typescript TypeScript -await sandbox.modify({ rootDiskSize: 8192, policy: "restart" }); +await sandbox.modify({ rootDiskSize: 8192 }); ``` ```python Python await sb.modify( root_disk_size=8192, - policy=ModificationPolicy.RESTART, ) ``` ```go Go _, err := sb.Modify(ctx, m.ModifyOptions{ RootDiskSizeMiB: 8192, - Policy: m.ModificationPolicyRestart, }) ``` ```bash CLI -msb modify worker --root-disk 8G --restart +msb modify worker --root-disk 8G ``` -Root disk changes are offline, not live. For a running sandbox, use the restart policy to make the new size active immediately, or the next-start policy to save it without touching the running VM. A stopped sandbox grows before the new size is persisted. +Managed and flat ext4 roots can grow while the sandbox is running, including roots backed by checkpoint layers. Microsandbox briefly pauses the VM to extend the writable disk, resumes it, then asks ext4 to make the space usable. Existing snapshots and sealed backing layers are unchanged. A stopped sandbox grows before the new size is persisted. + +Use `--next-start` to defer a running sandbox's growth until its next boot, or `--restart` to explicitly use stopped growth. Older runtimes and guest filesystems without online-resize support return an error instead of silently restarting. Cloud and user-owned disk behavior is unchanged. The backing determines which changes are valid: - Managed and flat OCI root disks are grow-only. Shrinking is rejected because it risks filesystem data loss. -- After a full snapshot has rolled a root disk onto a qcow2 chain, root-disk growth is refused for now. Microsandbox will not resize the chain's sealed raw ancestor; chain-aware online growth is tracked separately. +- Checkpoint-backed roots grow only their writable head. New snapshots capture the new capacity; old snapshots retain their original capacity. - A tmpfs OCI root disk can grow or shrink on the next boot, but its size cannot exceed sandbox memory. - User-supplied root disk images are not resized or deleted by microsandbox; resize the image file yourself while it is detached. - Existing disk-backed named volumes do not have a resize operation. Their capacity is fixed when the volume is created. The ext4 grower uses the image's reserved metadata headroom. If an older image cannot grow to the requested size in place, recreate the sandbox with a larger root disk. +If the disk grows but guest filesystem expansion fails, the error reports incomplete growth. Retry the same target, or stop and restart to finish recovery. Microsandbox never truncates the disk to roll back; snapshots and compaction are blocked until recovery completes. + Other storage capacity is still chosen where the storage is defined: - named volume size or quota on the volume diff --git a/docs/sdk/go/sandbox.mdx b/docs/sdk/go/sandbox.mdx index e88dfc568..c6d105231 100644 --- a/docs/sdk/go/sandbox.mdx +++ b/docs/sdk/go/sandbox.mdx @@ -561,10 +561,9 @@ _, err = sb.Modify(ctx, m.ModifyOptions{ Env: map[string]string{"MODE": "prod"}, Policy: m.ModificationPolicyRestart, }) -// Grow the managed OCI root disk offline and restart +// Grow the owned OCI root disk while it is running _, err = sb.Modify(ctx, m.ModifyOptions{ RootDiskSizeMiB: 8192, - Policy: m.ModificationPolicyRestart, }) // Add or rotate a host-environment secret; restart if it is newly added @@ -586,9 +585,9 @@ _, err = sb.Modify(ctx, m.ModifyOptions{ -Plan or apply a configuration change. The returned plan labels each change `"live"`, `"next start"`, `"requires restart"`, or `"unsupported"`, and apply is all-or-nothing. +Plan or apply a configuration change. The returned plan labels each change `"live"`, `"next start"`, `"requires restart"`, or `"unsupported"`. -`CPUs` and `MemoryMiB` resize live within the [`WithMaxCPUs`](#withmaxcpus) / [`WithMaxMemory`](#withmaxmemory) ceilings; raising a ceiling requires a restart. `RootDiskSizeMiB` changes are offline: managed and flat OCI root disks grow only, tmpfs root disks can change in either direction on the next boot, and user-supplied disk images are rejected. Env and workdir changes affect future execs only. On a stopped sandbox, changes are saved for the next boot. +`CPUs` and `MemoryMiB` resize live within the [`WithMaxCPUs`](#withmaxcpus) / [`WithMaxMemory`](#withmaxmemory) ceilings; raising a ceiling requires a restart. `RootDiskSizeMiB` grows owned managed and flat ext4 roots live, including checkpoint-backed roots, tmpfs root disks can change in either direction on the next boot, and user-supplied disk images are rejected. Env and workdir changes affect future execs only. Stopped roots grow before the new size is saved. Explicit restart and next-start policies remain available. Secret specs are keyed by stable secret name. Each `SecretModifySpec` selects at most one source—`Env`, `Value`, or `Store`—and may also set `Placeholder` and `AllowedHosts`; omitting a source updates only the other supplied fields. Plans expose only safe references and metadata; raw secret values never appear in a plan. Removal is explicit through `SecretsRemove`. diff --git a/docs/sdk/python/sandbox.mdx b/docs/sdk/python/sandbox.mdx index 8955723b8..90aa9c02d 100644 --- a/docs/sdk/python/sandbox.mdx +++ b/docs/sdk/python/sandbox.mdx @@ -553,8 +553,8 @@ for change in plan["changes"]: # Make an env change active now by restarting await sb.modify(env={"MODE": "prod"}, policy=ModificationPolicy.RESTART) -# Grow the managed OCI root disk offline and restart -await sb.modify(root_disk_size=8192, policy=ModificationPolicy.RESTART) +# Grow the owned OCI root disk while it is running +await sb.modify(root_disk_size=8192) # Add or rotate a host-environment secret; restart if it is newly added await sb.modify( @@ -573,9 +573,9 @@ await sb.modify(secrets_rm=["OLD_API_KEY"]) -Plan or apply a configuration change. The returned plan uses [`ModificationDisposition`](#modificationdisposition) to classify when each change takes effect, and apply is all-or-nothing. +Plan or apply a configuration change. The returned plan uses [`ModificationDisposition`](#modificationdisposition) to classify when each change takes effect. -`cpus` and `memory` resize live within the `max_cpus` / `max_memory` ceilings; raising a ceiling requires a restart. `root_disk_size` changes are offline: managed and flat OCI root disks grow only, tmpfs root disks can change in either direction on the next boot, and user-supplied disk images are rejected. Env and workdir changes affect future execs only. On a stopped sandbox, changes are saved for the next boot. +`cpus` and `memory` resize live within the `max_cpus` / `max_memory` ceilings; raising a ceiling requires a restart. `root_disk_size` grows owned managed and flat ext4 roots live, including checkpoint-backed roots, tmpfs root disks can change in either direction on the next boot, and user-supplied disk images are rejected. Env and workdir changes affect future execs only. Stopped roots grow before the new size is saved. Explicit restart and next-start policies remain available. Secret specs are keyed by stable secret name. Each `SecretModifySpec` selects at most one source—`env`, `value`, or `store`—and may also set `placeholder` and `allowed_hosts`; omitting a source updates only the other supplied fields. Plans expose only safe references and metadata; raw secret values never appear in a plan. Removal is explicit through `secrets_rm`. diff --git a/docs/sdk/typescript/sandbox.mdx b/docs/sdk/typescript/sandbox.mdx index a5f972299..12525ac83 100644 --- a/docs/sdk/typescript/sandbox.mdx +++ b/docs/sdk/typescript/sandbox.mdx @@ -553,8 +553,8 @@ for (const c of preview.changes) { // Make an env change active now by restarting await sandbox.modify({ env: { MODE: "prod" }, policy: "restart" }); -// Grow the managed OCI root disk offline and restart -await sandbox.modify({ rootDiskSize: 8192, policy: "restart" }); +// Grow the owned OCI root disk while it is running +await sandbox.modify({ rootDiskSize: 8192 }); // Add or rotate a host-environment secret; restart if it is newly added await sandbox.modify({ @@ -570,9 +570,9 @@ await sandbox.modify({ secretsRemove: ["OLD_API_KEY"] }); -Plan or apply a configuration change. The returned plan labels each change `"live"`, `"next start"`, `"requires restart"`, or `"unsupported"`, and apply is all-or-nothing. +Plan or apply a configuration change. The returned plan labels each change `"live"`, `"next start"`, `"requires restart"`, or `"unsupported"`. -`cpus` and `memory` resize live within the [`maxCpus`](#maxcpus) / [`maxMemory`](#maxmemory) ceilings; raising a ceiling requires a restart. `rootDiskSize` changes are offline: managed and flat OCI root disks grow only, tmpfs root disks can change in either direction on the next boot, and user-supplied disk images are rejected. Env and workdir changes affect future execs only. On a stopped sandbox, changes are saved for the next boot. +`cpus` and `memory` resize live within the [`maxCpus`](#maxcpus) / [`maxMemory`](#maxmemory) ceilings; raising a ceiling requires a restart. `rootDiskSize` grows owned managed and flat ext4 roots live, including checkpoint-backed roots, tmpfs root disks can change in either direction on the next boot, and user-supplied disk images are rejected. Env and workdir changes affect future execs only. Stopped roots grow before the new size is saved. Explicit restart and next-start policies remain available. Secret specs are keyed by stable secret name. Each `SecretModifySpec` selects at most one source—`env`, `value`, or `store`—and may also set `placeholder` and `allowedHosts`; omitting a source updates only the other supplied fields. Plans expose only safe references and metadata; raw secret values never appear in a plan. Removal is explicit through `secretsRemove`. diff --git a/scripts/smoke/cli/root-disk-growth.py b/scripts/smoke/cli/root-disk-growth.py new file mode 100644 index 000000000..60d997c15 --- /dev/null +++ b/scripts/smoke/cli/root-disk-growth.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Live root-growth matrix. Requires an isolated MSB_HOME and matching MSB_BIN/firmware.""" + +import hashlib +import json +import os +from pathlib import Path +import socket +import subprocess +import time + + +binary = os.environ["MSB_BIN"] +home = Path(os.environ["MSB_HOME"]) +report = Path(os.environ["QUAL_ROOT"]) +report.mkdir(parents=True, exist_ok=True) +names = [] +results = [] + + +def run(label, *args, refuse=False): + started = time.perf_counter() + result = subprocess.run([binary, *args], text=True, capture_output=True, timeout=180) + elapsed = (time.perf_counter() - started) * 1000 + (report / f"{label}.stdout").write_text(result.stdout) + (report / f"{label}.stderr").write_text(result.stderr) + passed = result.returncode != 0 if refuse else result.returncode == 0 + results.append({"case": label, "passed": passed, "elapsed_ms": elapsed}) + (report / "results.json").write_text(json.dumps(results, indent=2)) + print(f"{label}: {'PASS' if passed else 'FAIL'} {elapsed:.2f} ms", flush=True) + if not passed: + raise AssertionError(result.stdout + result.stderr) + return result.stdout + + +def guest(label, name, script): + return run(label, "exec", name, "--", "sh", "-c", script) + + +def phase_grow(label, name, mib): + # The runtime response separates the short VM pause from online ext4 expansion. + # Follow with the SDK-backed CLI call to reconcile/persist its desired configuration. + if os.name != "nt": + digest = hashlib.sha256(name.encode()).hexdigest()[:24] + path = home / "run" / "sandboxes" / digest / "control.sock" + with socket.socket(socket.AF_UNIX) as connection: + connection.settimeout(180) + connection.connect(str(path)) + connection.sendall(json.dumps({"op": "root_disk_grow", "size_bytes": mib * 1048576}).encode() + b"\n") + with connection.makefile("rb") as reader: + reply = json.loads(reader.readline()) + (report / f"{label}.phases.json").write_text(json.dumps(reply, indent=2)) + assert reply["ok"], reply + assert reply["root_disk"]["filesystem_bytes"] == mib * 1048576 + run(label, "modify", name, "--root-disk", f"{mib}M", "--format", "json") + + +def journal(name): + return json.loads((home / "sandboxes" / name / "runtime" / "root-disk.json").read_text()) + + +def file_hash(path): + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +try: + for layout in ("managed", "flat"): + name = f"{os.environ.get('QUAL_PREFIX', 'grow')}-{layout}" + names.append(name) + disk = "512M" if layout == "managed" else "flat:512M" + run(f"{layout}-create", "create", "-n", name, "--root-disk", disk, "-m", "256M", "--max-duration", "20m", "alpine") + guest(f"{layout}-seed", name, "dd if=/dev/urandom of=/payload bs=1048576 count=8 2>/dev/null; sha256sum /payload >/expected; echo ram >/dev/shm/grow-marker; sync") + phase_grow(f"{layout}-raw-live", name, 768) + guest(f"{layout}-new-space", name, "dd if=/dev/zero of=/space bs=1048576 count=600 conv=fsync && test $(stat -c %s /space) = 629145600 && rm /space && sha256sum -c /expected") + run(f"{layout}-old-snapshot", "snapshot", "create", f"{name}-old", "--from", name, "--full") + before = journal(name) + ancestor = Path(before["layers"][0]["path"]) + ancestor_before = file_hash(ancestor) + phase_grow(f"{layout}-qcow-live", name, 1024) + phase_grow(f"{layout}-qcow-repeat", name, 1280) + assert len(journal(name)["layers"]) == len(before["layers"]) + assert file_hash(ancestor) == ancestor_before + run(f"{layout}-shrink-refused", "modify", name, "--root-disk", "512M", refuse=True) + run(f"{layout}-new-snapshot", "snapshot", "create", f"{name}-new", "--from", name, "--full") + run(f"{layout}-compact", "modify", name, "--compact", "--format", "json") + phase_grow(f"{layout}-compacted-live", name, 1536) + phase_grow(f"{layout}-partial-group-live", name, 1700) + run(f"{layout}-stop", "stop", name) + run(f"{layout}-stopped-grow", "modify", name, "--root-disk", "1792M", "--format", "json") + run(f"{layout}-start", "start", name) + guest(f"{layout}-after-stopped-grow", name, "sha256sum -c /expected; df -k /") + run(f"{layout}-defer", "modify", name, "--root-disk", "2G", "--next-start", "--format", "json") + run(f"{layout}-defer-stop", "stop", name) + run(f"{layout}-defer-start", "start", name) + guest(f"{layout}-defer-space", name, "dd if=/dev/zero of=/space bs=1048576 count=1800 conv=fsync && rm /space && sha256sum -c /expected") + run(f"{layout}-final-stop", "stop", name) + run(f"{layout}-stopped-snapshot", "snapshot", "create", f"{name}-stopped", "--from", name, "--integrity") + run(f"{layout}-verify", "snapshot", "verify", f"{name}-stopped") + for suffix, capacity in (("old", 768), ("new", 1280)): + child = f"{name}-{suffix}-child" + names.append(child) + run(f"{layout}-{suffix}-restore", "create", "-n", child, "--from-snapshot", f"{name}-{suffix}") + guest(f"{layout}-{suffix}-restored-data", child, "sha256sum -c /expected && test $(cat /dev/shm/grow-marker) = ram") + # Full restore retains captured block capacity, not the source's later size. + layers = journal(child)["layers"] + head = Path(layers[-1]["path"]) + with head.open("rb") as file: + file.seek(24) + assert int.from_bytes(file.read(8), "big") == capacity * 1048576 + run(f"{layout}-{suffix}-child-stop", "stop", child) + print(f"Root-growth matrix passed: {report / 'results.json'}", flush=True) +finally: + for name in names: + subprocess.run([binary, "stop", name], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=30) diff --git a/scripts/smoke/reports/root-disk-growth-2026-09-06.md b/scripts/smoke/reports/root-disk-growth-2026-09-06.md new file mode 100644 index 000000000..9cf059ca2 --- /dev/null +++ b/scripts/smoke/reports/root-disk-growth-2026-09-06.md @@ -0,0 +1,42 @@ +# Root-disk growth qualification — 2026-09-06 + +Stack: Microsandbox `appcypher/live-root-disk-growth`, directly based on #6 (`ed4167f7`), with libkrun `ec9f119dcc144f1c011fe62ebd224d660c2191a3`. No newer main changes were imported. Publication and Linux/Windows source transfers await explicit approval. + +## Coverage + +The reproducible matrix is `scripts/smoke/cli/root-disk-growth.py`. Set `MSB_BIN`, an isolated `MSB_HOME`, matching `MSB_LIBKRUNFW_PATH`, `QUAL_ROOT` and a fresh `QUAL_PREFIX`. It always stops its own sandboxes, including on failure. + +macOS/HVF passed 58 CLI checks and ten direct control-phase measurements in both debug and release builds. Hardware: Apple M5 Max, 36 GiB RAM. Guest: Alpine, 256 MiB RAM, initially 512 MiB ext4 root. The release matrix covers both managed and flat roots: raw growth to 768 MiB; a 600 MiB file written and synced beyond the original capacity; full checkpoint rollover; qcow2 growth to 1 GiB and then 1280 MiB; unchanged ancestor SHA-256 and chain length; shrink rejection; a second full snapshot; compaction followed by growth; a partial final block group at 1700 MiB; stopped growth to 1792 MiB; next-start growth to 2 GiB; a synced 1800 MiB file; stopped snapshot creation and verification; old/new full restores at 768/1280 MiB with payload checksums and tmpfs memory markers intact. All test sandboxes were confirmed stopped afterward. + +The initial live tests found that Linux online growth leaves valid lazy group flags which the older offline resizer rejected. The resizer now preserves those flags and initializes a lazy partial block bitmap before extending it. The failed attempt modified only staging; the original head and journal remained usable. + +## Release phase measurements + +These are individual observations, not percentiles or latency guarantees. Runtime total includes preflight, persistence, pause/resume and guest acknowledgment. Pause includes draining the block worker. Guest time includes online ext4 expansion, sync and verification, after VM resume. + +| Operation | Managed total / pause / guest (ms) | Flat total / pause / guest (ms) | +| --- | ---: | ---: | +| Raw 512 → 768 MiB | 69.14 / 7.80 / 45.38 | 62.49 / 7.91 / 38.18 | +| Qcow2 768 → 1024 MiB | 69.44 / 7.11 / 45.79 | 67.76 / 8.03 / 43.03 | +| Qcow2 1024 → 1280 MiB | 50.49 / 4.15 / 30.32 | 55.39 / 4.18 / 35.12 | +| Compacted 1280 → 1536 MiB | 54.50 / 7.06 / 30.97 | 52.09 / 3.95 / 32.09 | +| Partial group 1536 → 1700 MiB | 50.10 / 4.02 / 29.98 | 52.24 / 3.80 / 32.07 | + +Stopped CLI growth from 1700 to 1792 MiB took 45.65 ms managed and 43.66 ms flat. The matrix's online CLI rows are same-target configuration reconciliation after a separately measured control request; they must not be quoted as end-to-end first-growth latency. Raw outputs and phase JSON are in `/private/tmp/msb-grow-qual.6VfI8S/release`; debug results are in its `matrix2` sibling. + +## Other checks + +- libkrun device suite: 135 passed, including preserved data, zero-filled added capacity, repeated targets, and read-only/shrink rejection. +- Runtime disk tests: five passed, including both layouts, immutable ancestors, pending-target snapshot/compaction refusal, recovery before boot, lost-completion acknowledgment, and failed staging without journal publication. +- Rust SDK modification tests: 51 passed, including explicit restart/next-start policies, new root capability and old control sockets. +- Full Rust SDK library suite: 666 passed and three ignored with `cargo test --offline -p microsandbox --lib -- --test-threads=1`. A parallel run had one failure in the unchanged stale Unix-socket detection test; that test passed in isolation and in the serial suite. The initial sandboxed run also denied four socket bindings; validation was repeated with local socket access. +- Image ext4 tests and four external `e2fsck` tests passed; dedicated tests cover qcow2 growth and lazy partial block groups. +- Protocol generation-9 schema and append-only checks: four passed. +- Pinned CLI build/check succeeds offline after seeding Cargo's cache from the exact signed local libkrun commit. The companion commit must be pushed before other machines can fetch it. +- Strict Clippy initially found pre-existing `derivable_impls` and `too_many_arguments` warnings in #6. Focused linting allows those two baseline classes; new lint findings were fixed. + +## Remaining qualification and limitations + +Linux/KVM and Windows/WHP have not been tested for this item. Both machines were reachable, but transferring unreleased source was blocked pending explicit approval. SDK bindings already route the existing modification options through Rust; fresh Python/TypeScript/Go native live runs have not been performed for #7. Direct/dependent archives after growth, sustained concurrent-I/O latency, request cancellation, process/power-loss injection, and every admission-failure variant remain additional qualification work. Do not describe this report as exhaustive fault testing. + +Stopped growth currently retains previous private file bindings after publishing the replacement head. It does not add chain depth or alter snapshots, but retained files can consume host space; automatic reclamation was not implemented because its deletion scope needs explicit approval/stronger ownership proof. Pending online growth is forward-only. Older runtimes refuse the pending journal field; finish recovery before switching back to an older runtime. A failed operation may leave saved desired configuration behind physical capacity until the caller retries the same requested size. diff --git a/sdk/rust/lib/runtime/spawn.rs b/sdk/rust/lib/runtime/spawn.rs index 50e7859f2..25099c21b 100644 --- a/sdk/rust/lib/runtime/spawn.rs +++ b/sdk/rust/lib/runtime/spawn.rs @@ -953,22 +953,21 @@ async fn prepare_oci_upper(config: &SandboxConfig, sandbox_dir: &Path) -> Micros | Some(microsandbox_types::RootDisk::Managed { size_mib }) => *size_mib, _ => None, }; - if let Some(chain) = microsandbox_runtime::checkpoint::load_runtime_owned_root_chain( - &sandbox_dir.join("runtime"), - ) - .map_err(|error| { - MicrosandboxError::Runtime(format!( - "cannot inspect the root-disk chain before startup: {error}" - )) - })? && chain.layers.len() > 1 - { - if desired_mib.is_some_and(|desired| u64::from(desired) * 1024 * 1024 > chain.virtual_size) - { - return Err(MicrosandboxError::Custom( - "cannot grow a checkpoint-backed root disk yet; its sealed raw ancestor must not be resized" - .into(), - )); + let runtime_dir = sandbox_dir.join("runtime"); + let handled = tokio::task::spawn_blocking(move || { + microsandbox_runtime::checkpoint::recover_stopped_root_growth(&runtime_dir)?; + match desired_mib { + Some(desired) => microsandbox_runtime::checkpoint::grow_stopped_root( + &runtime_dir, + u64::from(desired) * 1024 * 1024, + ), + None => Ok(runtime_dir.join("root-disk.json").exists()), } + }) + .await + .map_err(|e| MicrosandboxError::Runtime(e.to_string()))? + .map_err(MicrosandboxError::Runtime)?; + if handled { return Ok(()); } match &oci.root_disk { @@ -4043,7 +4042,7 @@ mod tests { } #[tokio::test] - async fn test_persisted_checkpoint_chain_refuses_deferred_root_grow() { + async fn test_invalid_checkpoint_chain_cannot_mutate_the_sealed_base_during_deferred_grow() { let temp = tempdir().unwrap(); let runtime = temp.path().join("runtime"); std::fs::create_dir(&runtime).unwrap(); @@ -4091,7 +4090,7 @@ mod tests { let error = super::prepare_oci_upper(&config, temp.path()) .await .unwrap_err(); - assert!(error.to_string().contains("checkpoint-backed root disk")); + assert!(!error.to_string().is_empty()); assert_eq!(std::fs::metadata(base).unwrap().len(), 4096); } diff --git a/sdk/rust/lib/sandbox/modify.rs b/sdk/rust/lib/sandbox/modify.rs index 37f25cc16..c63c5e0fd 100644 --- a/sdk/rust/lib/sandbox/modify.rs +++ b/sdk/rust/lib/sandbox/modify.rs @@ -91,6 +91,8 @@ struct ExistingSecret { /// discovered through the control socket's `capabilities` op. #[derive(Debug, Clone, Copy, Default)] struct LiveControl { + /// Host understands root growth; the runtime separately preflights its guest. + root_disk_grow: bool, /// CPU and memory resize targets are served. resize: bool, @@ -375,6 +377,33 @@ impl SandboxModificationBuilder { } } } + if running_status(status) + && !restart_required + && self.policy == ModificationPolicy::NoRestart + && let Some(target_mib) = root_disk_grow_target(&plan, &self.patch, &config) + { + let size_bytes = u64::from(target_mib) * 1024 * 1024; + let request = serde_json::to_string( + µsandbox_runtime::control::ControlRequest::RootDiskGrow { size_bytes }, + )? + "\n"; + let response = control_request(&self.name, request).await?; + let observed = response.root_disk.ok_or_else(|| { + crate::MicrosandboxError::Runtime("root growth reply missing capacity".into()) + })?; + if observed.filesystem_bytes != size_bytes || observed.device_bytes < size_bytes { + return Err(crate::MicrosandboxError::Runtime( + "root growth did not confirm usable capacity".into(), + )); + } + if let Some(active) = active.as_mut() { + let disk_patch = SandboxModificationPatch { + root_disk_size_mib: Some(target_mib), + ..Default::default() + }; + apply_patch_to_config(active, &disk_patch); + persist_active_config(&self.backend, &handle, active).await?; + } + } // Grow the real upper.ext4 before persisting the new desired size: // the persisted value may only ever claim capacity the file actually // has. A running sandbox under `--next-start` keeps its mounted upper @@ -481,6 +510,23 @@ fn build_plan( &mut warnings, ); push_root_disk_size_change(status, config, &patch, policy, &mut changes); + if live.root_disk_grow + && running_status(status) + && policy == ModificationPolicy::NoRestart + && matches!( + root_disk_size_state(config), + Some(RootDiskSizeState::Managed { .. }) + ) + { + for change in &mut changes { + if let PlannedChange::Config(change) = change + && change.field == ROOT_DISK_FIELD + { + change.disposition = ModificationDisposition::Live; + change.reason = None; + } + } + } push_spec_changes(status, config, &patch, policy, &mut changes, &mut warnings); push_secret_changes( status, @@ -578,7 +624,25 @@ async fn grow_root_disk_now( ) })?; let sandbox_dir = local_backend.sandboxes_dir().join(name); - refuse_checkpoint_backed_root_grow(&sandbox_dir)?; + let runtime_dir = sandbox_dir.join("runtime"); + let handled = tokio::task::spawn_blocking(move || { + microsandbox_runtime::checkpoint::grow_stopped_root( + &runtime_dir, + u64::from(target_mib) * 1024 * 1024, + ) + }) + .await + .map_err(|e| crate::MicrosandboxError::Runtime(e.to_string()))? + .map_err(crate::MicrosandboxError::Runtime)?; + if handled { + return Ok(()); + } + if !config.snapshot_upper_layers.is_empty() { + return Err(crate::MicrosandboxError::Runtime( + "start this restored sandbox once to initialize its owned root chain before resizing" + .into(), + )); + } if matches!( &config.spec.image, RootfsSource::Oci(oci) if matches!(&oci.root_disk, Some(RootDisk::Flat { .. })) @@ -592,29 +656,6 @@ async fn grow_root_disk_now( super::upper::grow_upper_to_mib(sandbox_dir.join("upper.ext4"), target_mib).await } -/// Refuse the legacy raw-file grow path once checkpoint rollover has installed a qcow2 head. -/// -/// A one-layer journal still names the ordinary mutable raw disk and is safe to grow in place. -/// With two or more layers, however, the raw file is a sealed ancestor and only a future -/// chain-aware resize may extend the active qcow2 head and filesystem. -fn refuse_checkpoint_backed_root_grow(sandbox_dir: &std::path::Path) -> MicrosandboxResult<()> { - let chain = microsandbox_runtime::checkpoint::load_runtime_owned_root_chain( - &sandbox_dir.join("runtime"), - ) - .map_err(|error| { - crate::MicrosandboxError::Runtime(format!( - "cannot inspect the root-disk chain before resize: {error}" - )) - })?; - if chain.is_some_and(|chain| chain.layers.len() > 1) { - return Err(crate::MicrosandboxError::Custom( - "cannot grow a checkpoint-backed root disk yet; its sealed raw ancestor must not be resized" - .into(), - )); - } - Ok(()) -} - /// Path of the sandbox's host-side runtime control socket. #[cfg(windows)] fn control_socket_path(name: &str) -> MicrosandboxResult { @@ -658,12 +699,14 @@ async fn live_control(name: &str, status: SandboxStatus) -> LiveControl { } match control_capabilities(name).await { Ok(caps) => LiveControl { + root_disk_grow: caps.root_disk_grow, resize: caps.cpu_resize || caps.memory_resize, secrets: caps.secrets_update, }, // Runtimes that predate the capabilities op served the socket only // when they could resize; live secret ops did not exist yet. Err(_) => LiveControl { + root_disk_grow: false, resize: true, secrets: false, }, @@ -2638,6 +2681,7 @@ mod tests { &desired, Some(&active), LiveControl { + root_disk_grow: false, resize: true, secrets: false, }, @@ -2675,6 +2719,7 @@ mod tests { &desired, Some(&active), LiveControl { + root_disk_grow: false, resize: live_memory_supported, secrets: false, }, @@ -2709,6 +2754,7 @@ mod tests { &config(2, 1024), Some(&active), LiveControl { + root_disk_grow: false, resize: true, secrets: false, }, @@ -2938,7 +2984,7 @@ mod tests { } #[test] - fn checkpoint_backed_root_grow_refuses_to_mutate_the_sealed_base() { + fn invalid_checkpoint_backed_root_grow_does_not_mutate_the_sealed_base() { let sandbox = tempdir().unwrap(); let runtime = sandbox.path().join("runtime"); std::fs::create_dir(&runtime).unwrap(); @@ -2973,8 +3019,7 @@ mod tests { ) .unwrap(); - let error = refuse_checkpoint_backed_root_grow(sandbox.path()).unwrap_err(); - assert!(error.to_string().contains("checkpoint-backed root disk")); + assert!(microsandbox_runtime::checkpoint::grow_stopped_root(&runtime, 8192).is_err()); assert_eq!( std::fs::metadata(sandbox.path().join("rootfs.raw")) .unwrap() @@ -2984,19 +3029,20 @@ mod tests { } #[test] - fn running_upper_grow_is_restart_backed_never_live() { + fn old_runtime_upper_grow_requires_explicit_restart() { let patch = SandboxModificationPatch { root_disk_size_mib: Some(8192), ..SandboxModificationPatch::default() }; - // Even a resize-capable runtime cannot grow the mounted upper live. + // CPU/memory resize capability alone does not advertise root growth. let plan = build_plan( "api".to_string(), SandboxStatus::Running, &oci_config_with_upper(4096), None, LiveControl { + root_disk_grow: false, resize: true, secrets: true, }, @@ -3029,6 +3075,47 @@ mod tests { assert!(plan_requires_restart(&restart_plan)); } + #[test] + fn owned_root_growth_uses_live_capability_but_respects_explicit_policies() { + for root in [RootDisk::managed(512), RootDisk::flat(512)] { + let config = oci_config_with_root_disk(root); + for (policy, expected) in [ + (ModificationPolicy::NoRestart, ModificationDisposition::Live), + ( + ModificationPolicy::NextStart, + ModificationDisposition::NextStart, + ), + ( + ModificationPolicy::Restart, + ModificationDisposition::RequiresRestart, + ), + ] { + let plan = build_plan( + "grow".into(), + SandboxStatus::Running, + &config, + None, + LiveControl { + root_disk_grow: true, + resize: false, + secrets: false, + }, + SandboxModificationPatch { + root_disk_size_mib: Some(1024), + ..Default::default() + }, + policy, + ); + assert!(plan.conflicts.is_empty()); + let PlannedChange::Config(change) = &plan.changes[0] else { + panic!("expected disk change") + }; + assert_eq!(change.disposition, expected); + assert!(validate_apply_supported(&plan).is_ok()); + } + } + } + #[test] fn running_upper_grow_under_next_start_persists_desired_only() { let patch = SandboxModificationPatch { @@ -3663,6 +3750,7 @@ mod tests { &config, None, LiveControl { + root_disk_grow: false, resize: false, secrets: true, }, @@ -3688,6 +3776,7 @@ mod tests { &config, None, LiveControl { + root_disk_grow: false, resize: false, secrets: true, }, @@ -3719,6 +3808,7 @@ mod tests { &config, None, LiveControl { + root_disk_grow: false, resize: false, secrets: true, }, @@ -3754,6 +3844,7 @@ mod tests { &config, None, LiveControl { + root_disk_grow: false, resize: false, secrets: true, }, @@ -4131,6 +4222,7 @@ mod tests { &config, None, LiveControl { + root_disk_grow: false, resize: false, secrets: true, }, @@ -4199,6 +4291,7 @@ mod tests { &config, None, LiveControl { + root_disk_grow: false, resize: false, secrets: true, }, @@ -4249,6 +4342,7 @@ mod tests { &config, None, LiveControl { + root_disk_grow: false, resize: false, secrets: true, }, @@ -4269,6 +4363,7 @@ mod tests { &config, None, LiveControl { + root_disk_grow: false, resize: false, secrets: true, }, diff --git a/sdk/rust/lib/snapshot/create.rs b/sdk/rust/lib/snapshot/create.rs index caa144007..d1bc8cacf 100644 --- a/sdk/rust/lib/snapshot/create.rs +++ b/sdk/rust/lib/snapshot/create.rs @@ -779,15 +779,25 @@ fn new_file_manifest_with_id( .last() .expect("checked non-empty layer inputs") .1; + let capacities = microsandbox_image::checkpoint::layer_capacities( + disk.sources + .iter() + .map(|source| microsandbox_image::checkpoint::CompactLayer { + path: source.path.clone(), + qcow2: source.format == SnapshotFormat::Qcow2, + }) + .collect(), + )?; let mut predecessor = None; let layers = layer_inputs .into_iter() - .map(|(layer_id, format, integrity)| { + .zip(capacities) + .map(|((layer_id, format, integrity), virtual_size)| { let backing = predecessor.replace(layer_id.clone()); DiskLayer { layer_id, format, - virtual_size: disk.virtual_size, + virtual_size, backing, payload: LayerPayload { file_kind: LayerFileKind::Regular, From 4582174a533e3dbd228d9a10d095e0fe4d16a717 Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Sun, 6 Sep 2026 02:42:01 +0100 Subject: [PATCH 2/6] test(storage): qualify one-shot multi-gib root growth Exercise 512 MiB to 4 GiB and 8320 MiB across managed and flat roots, raw and qcow2 heads, and live and stopped modification paths. Verify allocated data, snapshot capacities, immutable ancestors and cold boots. Add independent qemu-img and read-only e2fsck qualification on disposable copies and record the macOS release timings and remaining coverage gaps. No runtime behavior changes are included. --- scripts/smoke/cli/root-disk-large-growth.py | 213 ++++++++++++++++++ .../reports/root-disk-growth-2026-09-06.md | 28 +++ 2 files changed, 241 insertions(+) create mode 100644 scripts/smoke/cli/root-disk-large-growth.py diff --git a/scripts/smoke/cli/root-disk-large-growth.py b/scripts/smoke/cli/root-disk-large-growth.py new file mode 100644 index 000000000..7a50a85f6 --- /dev/null +++ b/scripts/smoke/cli/root-disk-large-growth.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +"""One-shot root growth across GiB/metadata boundaries in an isolated MSB_HOME. + +Uses the existing release binary, not a build step. QUAL_PREFIX must be fresh. Each +case stops its own VMs, keeps artifacts for inspection, and records failed checks. +""" + +import hashlib +import itertools +import json +import os +from pathlib import Path +import subprocess +import tempfile +import time + + +BINARY = os.environ["MSB_BIN"] +TEST_HOME = Path(os.environ["MSB_HOME"]) +REPORT = Path(os.environ["QUAL_ROOT"]) +PREFIX = os.environ["QUAL_PREFIX"] +MIB = 1048576 +RESULTS = [] +REPORT.mkdir(parents=True, exist_ok=True) + + +def record(label, passed, **details): + RESULTS.append(dict(case=label, passed=passed, **details)) + (REPORT / "results.json").write_text(json.dumps(RESULTS, indent=2)) + print(f"{label}: {'PASS' if passed else 'FAIL'} {details}", flush=True) + + +def check(label, condition, **details): + record(label, bool(condition), **details) + if not condition: + raise AssertionError(label) + + +def run(label, *args, refuse=False): + started = time.perf_counter() + result = subprocess.run([BINARY, *args], capture_output=True, text=True, timeout=300) + elapsed = (time.perf_counter() - started) * 1000 + (REPORT / f"{label}.stdout").write_text(result.stdout) + (REPORT / f"{label}.stderr").write_text(result.stderr) + passed = result.returncode != 0 if refuse else result.returncode == 0 + record(label, passed, elapsed_ms=round(elapsed, 3), returncode=result.returncode) + if not passed: + raise AssertionError(result.stdout + result.stderr) + return result.stdout + + +def guest(label, name, script): + # A failed checksum must not be hidden by a later successful df/sync command. + return run(label, "exec", name, "--", "sh", "-ec", script) + + +def state(name): + return json.loads((TEST_HOME / "sandboxes" / name / "runtime" / "root-disk.json").read_text()) + + +def capacity(layer): + path = Path(layer["path"]) + with path.open("rb") as stream: + magic = stream.read(4) + if magic == b"QFI\xfb": + stream.seek(24) + return int.from_bytes(stream.read(8), "big") + return path.stat().st_size + + +def digest(path): + result = hashlib.sha256() + with Path(path).open("rb") as stream: + for block in iter(lambda: stream.read(MIB), b""): + result.update(block) + return result.hexdigest() + + +def case(layout, backing, mode, target): + label = f"{layout}-{backing}-{mode}-{target}" + name = f"{PREFIX}-{label}" + names = [name] + try: + disk = "512M" if layout == "managed" else "flat:512M" + run(label + "-create", "create", "-n", name, "--root-disk", disk, + "-m", "256M", "--max-duration", "30m", "alpine") + guest(label + "-seed", name, + "dd if=/dev/urandom of=/payload bs=1048576 count=8; " + "sha256sum /payload >/expected; echo retained >/dev/shm/grow-marker; " + "cat /proc/sys/kernel/random/boot_id >/boot-before; sync") + if backing == "qcow2": + run(label + "-old-snapshot", "snapshot", "create", name + "-old", + "--from", name, "--full") + before = state(name) + check(label + "-initial-capacity", capacity(before["layers"][-1]) == 512 * MIB) + check(label + "-initial-format", + before["layers"][-1]["format"] == ("qcow2" if backing == "qcow2" else "raw")) + ancestors = [(layer["path"], digest(layer["path"])) for layer in before["layers"][:-1]] + if mode == "stopped": + run(label + "-stop-before-grow", "stop", name) + # Time the first real CLI growth, including planning, control and configuration persistence. + size = "4G" if target == 4096 else f"{target}M" + run(label + "-grow", "modify", name, "--root-disk", size, "--format", "json") + after = state(name) + check(label + "-exact-capacity", capacity(after["layers"][-1]) == target * MIB) + check(label + "-depth-and-completion", len(after["layers"]) == len(before["layers"]) + and after.get("growth_target") is None) + check(label + "-immutable-ancestors", all(digest(path) == old for path, old in ancestors)) + # The public CLI deliberately rejects an already configured size. Runtime forward + # recovery is repeatable, but that is distinct from a redundant completed CLI request. + repeated = run(label + "-same-target", "modify", name, "--root-disk", size, + "--format", "json", refuse=True) + check(label + "-same-target-reason", "only grow is supported" in repeated) + run(label + "-shrink-rejected", "modify", name, "--root-disk", "512M", refuse=True) + check(label + "-refusal-keeps-capacity", capacity(state(name)["layers"][-1]) == target * MIB) + if mode == "stopped": + run(label + "-start-after-grow", "start", name) + guest(label + "-ram-seed", name, "echo retained >/dev/shm/grow-marker") + else: + guest(label + "-no-reboot", name, + "test $(cat /proc/sys/kernel/random/boot_id) = $(cat /boot-before); " + "test $(cat /dev/shm/grow-marker) = retained") + # Allocate (not merely truncate/seek) 3 GiB at the main target, using repeated random + # bytes. Smaller allocation in metadata-boundary cases keeps the matrix practical. + count = 384 if target == 4096 else 96 + allocated = count * 8 * MIB + guest(label + "-write-added-space", name, + f"i=0; while test $i -lt {count}; do cat /payload >>/large; i=$((i+1)); done; " + f"test $(stat -c %s /large) = {allocated}; " + f"test $(stat -c %b /large) -ge {allocated // 512}; " + "sync; sha256sum /large >/large.expected; sha256sum -c /expected; " + f"dd if=/payload of=/far bs=1048576 seek={target - 256} conv=fsync; " + f"dd if=/far bs=1048576 skip={target - 256} count=8 2>/dev/null | cmp - /payload; " + "df -k /; du -k /large") + # Snapshot the allocated file, not just metadata or an empty resized filesystem. + run(label + "-new-snapshot", "snapshot", "create", name + "-new", "--from", name, "--full") + run(label + "-source-stop", "stop", name) + for suffix, expected in [("new", target)] + ([("old", 512)] if backing == "qcow2" else []): + child = name + "-" + suffix + "-child" + names.append(child) + run(label + "-" + suffix + "-restore", "create", "-n", child, "--from-snapshot", name + "-" + suffix) + check(label + "-" + suffix + "-restored-capacity", capacity(state(child)["layers"][-1]) == expected * MIB) + script = "sha256sum -c /expected; test $(cat /dev/shm/grow-marker) = retained; " + if suffix == "new": + script += ("sha256sum -c /large.expected; " + f"dd if=/far bs=1048576 skip={target - 256} count=8 2>/dev/null | cmp - /payload") + else: + script += "test ! -e /large; test ! -e /far" + guest(label + "-" + suffix + "-restored-data", child, script) + run(label + "-" + suffix + "-child-stop", "stop", child) + # Verify a cold boot after full capture as well as restoring captured memory. + run(label + "-cold-start", "start", name) + guest(label + "-cold-data", name, "sha256sum -c /expected; sha256sum -c /large.expected") + run(label + "-cold-stop", "stop", name) + finally: + for owned in names: + result = subprocess.run([BINARY, "stop", owned], capture_output=True, text=True, timeout=40) + if result.returncode: + print(f"Cleanup needs inspection: {owned}: {result.stderr}", flush=True) + + +def offline_check(layout, backing, mode, target): + """Independently check a disposable flattened copy, never repair the source chain.""" + label = f"{layout}-{backing}-{mode}-{target}" + name = f"{PREFIX}-{label}" + statuses = json.loads(run(label + "-list-for-fsck", "list", "--format", "json")) + check(label + "-stopped-for-fsck", any(item["name"] == name and item["status"] == "Stopped" for item in statuses)) + head = state(name)["layers"][-1]["path"] + qemu = os.environ.get("QEMU_IMG", "qemu-img") + fsck = os.environ["E2FSCK"] + + def external(suffix, args): + started = time.perf_counter() + result = subprocess.run(args, capture_output=True, text=True, timeout=300) + (REPORT / f"{label}-{suffix}.stdout").write_text(result.stdout) + (REPORT / f"{label}-{suffix}.stderr").write_text(result.stderr) + check(label + "-" + suffix, result.returncode == 0, + elapsed_ms=round((time.perf_counter() - started) * 1000, 3), + returncode=result.returncode) + + external("qemu-chain", [qemu, "info", "--backing-chain", "--output=json", head]) + # The only automatically removed files belong to this newly created scratch directory. + with tempfile.TemporaryDirectory(prefix="growth-fsck-") as scratch: + disk = str(Path(scratch) / "check.raw") + external("flatten-for-check", [qemu, "convert", "-O", "raw", head, disk]) + # A stopped guest can leave a replayable journal. Replay only on the disposable copy, + # then require a read-only full filesystem check with no corrective repairs. + external("journal-replay", [fsck, "-p", "-E", "journal_only", disk]) + external("fsck", [fsck, "-f", "-n", disk]) + + +def main(): + # 8 GiB occupies 64 groups; 8320 MiB needs a second 64-byte-descriptor GDT block. + targets = [int(value) for value in os.environ.get("QUAL_TARGETS", "4096,8320").split(",")] + failures = [] + for target, layout, backing, mode in itertools.product( + targets, ("managed", "flat"), ("raw", "qcow2"), ("live", "stopped")): + try: + if os.environ.get("QUAL_OFFLINE_ONLY") == "1": + offline_check(layout, backing, mode, target) + else: + case(layout, backing, mode, target) + except Exception as error: + label = f"{layout}-{backing}-{mode}-{target}" + record(label + "-exception", False, error=str(error)) + failures.append(label) + if failures: + raise SystemExit("Failed cases: " + ", ".join(failures)) + print(f"All {len(targets) * 8} large-jump scenarios passed", flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/smoke/reports/root-disk-growth-2026-09-06.md b/scripts/smoke/reports/root-disk-growth-2026-09-06.md index 9cf059ca2..c71ac2910 100644 --- a/scripts/smoke/reports/root-disk-growth-2026-09-06.md +++ b/scripts/smoke/reports/root-disk-growth-2026-09-06.md @@ -35,6 +35,34 @@ Stopped CLI growth from 1700 to 1792 MiB took 45.65 ms managed and 43.66 ms flat - Pinned CLI build/check succeeds offline after seeding Cargo's cache from the exact signed local libkrun commit. The companion commit must be pushed before other machines can fetch it. - Strict Clippy initially found pre-existing `derivable_impls` and `too_many_arguments` warnings in #6. Focused linting allows those two baseline classes; new lint findings were fixed. +## One-shot MiB-to-GiB growth + +The original matrix did not establish large one-shot growth. Follow-up release qualification on the same Mac uses `scripts/smoke/cli/root-disk-large-growth.py`: 16 scenarios and 424 recorded checks passed, plus 96 independent offline validation checks. No runtime implementation changes were required. Each scenario starts afresh at 512 MiB, rather than reaching its target through intermediate grows. + +These measurements time the first SDK-backed CLI `modify` invocation from process launch through successful exit, including planning, control communication and configuration persistence. They are single observations, not percentiles. Stopped timings exclude the separately tested stop and start operations. Unlike the earlier phase table, these are not same-target reconciliation measurements and do not isolate VM pause time. + +| One-shot target | Initial backing | Managed live (ms) | Managed stopped (ms) | Flat live (ms) | Flat stopped (ms) | +| --- | --- | ---: | ---: | ---: | ---: | +| 512 MiB → 4 GiB | Raw | 74.37 | 40.95 | 75.83 | 38.12 | +| 512 MiB → 4 GiB | Qcow2 | 84.16 | 34.24 | 78.39 | 35.20 | +| 512 MiB → 8320 MiB (8.125 GiB) | Raw | 76.99 | 45.30 | 78.05 | 48.11 | +| 512 MiB → 8320 MiB (8.125 GiB) | Qcow2 | 85.97 | 35.93 | 84.62 | 40.32 | + +The 8320 MiB target crosses from 64 to 65 ext4 groups, requiring another group-descriptor block in the current 4 KiB-block/64-byte-descriptor layout. Every scenario verifies: + +- Exact head capacity, no pending-growth marker, unchanged chain depth and sealed-ancestor SHA-256 values. +- Same-target CLI rejection with the expected reason and shrink refusal without reducing capacity. Completed CLI requests deliberately reject an already configured size; this differs from retrying unfinished runtime growth. The first harness run incorrectly expected redundant CLI requests to succeed and was corrected without changing runtime behavior. +- For live growth, unchanged guest boot ID and a retained tmpfs marker. +- Real allocation and fsync of a 3 GiB file at the 4 GiB target, with allocated-block counts checked to exclude a sparse-only test. Boundary cases allocate 768 MiB. Both also write and read an 8 MiB random marker at a file offset 256 MiB below the new capacity; that is a file-offset check, not proof of allocation in the last physical block group. +- A full snapshot with that file present, full restore at the enlarged capacity, original/random-payload and large-file checksum checks, and retained tmpfs contents. +- For qcow2 cases, restoration of the pre-grow full snapshot at exactly 512 MiB with its original data and without files created after growth. +- A separate cold boot and checksum verification after the grown snapshot is captured. +- Independent `qemu-img info --backing-chain`, flattening to a disposable raw copy, journal-only replay on that copy, then `e2fsck -f -n` with exit zero. All 16 full read-only filesystem checks passed; source disks were not repaired or modified by these tools. + +The 4 GiB full snapshots containing the allocated 3 GiB file took 5.34–7.18 seconds and their full restores took 8.05–8.56 seconds. Those timings describe this populated-disk workload, not growth latency or empty snapshot performance. All VMs created by both large-jump runs were confirmed stopped. Test sandbox and snapshot artifacts remain available for inspection; disposable filesystem-check copies were removed automatically. + +Reproduce with an isolated `MSB_HOME`, matching `MSB_BIN`/`MSB_LIBKRUNFW_PATH`, a fresh `QUAL_PREFIX` and output `QUAL_ROOT`. The default targets are `4096,8320` MiB; `QUAL_TARGETS` can override them. Run the script normally first, then set `QUAL_OFFLINE_ONLY=1`, `E2FSCK` and optionally `QEMU_IMG` to independently validate those same stopped cases using a separate output directory. This run's outputs are `/private/tmp/msb-grow-qual.6VfI8S/large-jumps2` and `/private/tmp/msb-grow-qual.6VfI8S/large-jumps2-fsck`. + ## Remaining qualification and limitations Linux/KVM and Windows/WHP have not been tested for this item. Both machines were reachable, but transferring unreleased source was blocked pending explicit approval. SDK bindings already route the existing modification options through Rust; fresh Python/TypeScript/Go native live runs have not been performed for #7. Direct/dependent archives after growth, sustained concurrent-I/O latency, request cancellation, process/power-loss injection, and every admission-failure variant remain additional qualification work. Do not describe this report as exhaustive fault testing. From ce04099b660adcd75e4b63a985bc51fb30d30504 Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Sun, 6 Sep 2026 04:00:21 +0100 Subject: [PATCH 3/6] refactor(sdk)!: remove redundant modification size setters Use memory, max_memory and root_disk_size for both bare MiB integers and typed sizes. Update CLI callers and documentation and verify both input forms produce the same serialized patch values. BREAKING CHANGE: remove memory_mib, max_memory_mib, root_disk_size_mib, oci_upper_size and oci_upper_size_mib from SandboxModificationBuilder. Use the unsuffixed setters; no deprecation aliases are retained. Other SDK interfaces and wire fields are unchanged. --- crates/cli/lib/commands/modify.rs | 7 +- docs/sandboxes/tuning.mdx | 8 +-- .../reports/root-disk-growth-2026-09-06.md | 2 + sdk/rust/lib/sandbox/modify.rs | 66 +++++++++---------- 4 files changed, 40 insertions(+), 43 deletions(-) diff --git a/crates/cli/lib/commands/modify.rs b/crates/cli/lib/commands/modify.rs index 3f2048a1d..06b129878 100644 --- a/crates/cli/lib/commands/modify.rs +++ b/crates/cli/lib/commands/modify.rs @@ -187,14 +187,13 @@ fn apply_resource_args( builder = builder.max_cpus(max_cpus); } if let Some(memory) = &args.memory { - builder = builder.memory_mib(ui::parse_size_mib(memory).map_err(anyhow::Error::msg)?); + builder = builder.memory(ui::parse_size_mib(memory).map_err(anyhow::Error::msg)?); } if let Some(max_memory) = &args.max_memory { - builder = - builder.max_memory_mib(ui::parse_size_mib(max_memory).map_err(anyhow::Error::msg)?); + builder = builder.max_memory(ui::parse_size_mib(max_memory).map_err(anyhow::Error::msg)?); } if let Some(size) = args.root_disk.as_ref().or(args.oci_upper_size.as_ref()) { - builder = builder.root_disk_size_mib(ui::parse_size_mib(size).map_err(anyhow::Error::msg)?); + builder = builder.root_disk_size(ui::parse_size_mib(size).map_err(anyhow::Error::msg)?); } Ok(builder) } diff --git a/docs/sandboxes/tuning.mdx b/docs/sandboxes/tuning.mdx index d8382c997..f48daa64d 100644 --- a/docs/sandboxes/tuning.mdx +++ b/docs/sandboxes/tuning.mdx @@ -116,7 +116,7 @@ The plan labels each change as `live`, `next start`, `requires restart`, or `uns | `labels` | Live | Host-side metadata; no guest process changes | | `env`, `workdir` | Future execs | Running processes keep what they already have | | `secrets` | Live for rotation | Placeholder changes need a restart | -| Root disk size | Restart or next start | Managed and flat OCI disks grow only; tmpfs changes on next boot | +| Root disk size | Live, restart or next start | Managed and flat ext4 roots grow only; tmpfs changes on next boot | | Other storage | Create or mount time | Named volumes, mount tmpfs, and user disk images are sized outside `modify` | ## Dry runs @@ -126,7 +126,7 @@ Use a dry run to see what would happen before applying the patch: ```rust Rust let plan = sb.modify() - .max_memory_mib(16 * 1024) + .max_memory(16 * 1024) .dry_run() .await?; ``` @@ -158,7 +158,7 @@ By default, `modify` only applies changes that do not require a restart. Use `-- ```rust Rust sb.modify() - .max_memory_mib(16 * 1024) + .max_memory(16 * 1024) .next_start() .apply() .await?; @@ -341,7 +341,7 @@ Resize an OCI sandbox's root disk through `modify`: ```rust Rust sb.modify() - .root_disk_size_mib(8192) + .root_disk_size(8192) .apply() .await?; ``` diff --git a/scripts/smoke/reports/root-disk-growth-2026-09-06.md b/scripts/smoke/reports/root-disk-growth-2026-09-06.md index c71ac2910..907669122 100644 --- a/scripts/smoke/reports/root-disk-growth-2026-09-06.md +++ b/scripts/smoke/reports/root-disk-growth-2026-09-06.md @@ -65,6 +65,8 @@ Reproduce with an isolated `MSB_HOME`, matching `MSB_BIN`/`MSB_LIBKRUNFW_PATH`, ## Remaining qualification and limitations +Follow-up Rust API cleanup removes the redundant `modify()` methods `memory_mib`, `max_memory_mib`, `root_disk_size_mib`, `oci_upper_size` and `oci_upper_size_mib` without deprecation. Use `memory`, `max_memory` and `root_disk_size`; each accepts bare MiB integers and typed sizes. Serialized fields and Python/TypeScript/Go surfaces are unchanged. After cleanup, the serial Rust SDK suite passed 667 tests with three ignored, and all ten CLI modification tests passed. The live measurements above preceded this surface-only cleanup; the new test verifies that both supported input forms produce the same patch values. + Linux/KVM and Windows/WHP have not been tested for this item. Both machines were reachable, but transferring unreleased source was blocked pending explicit approval. SDK bindings already route the existing modification options through Rust; fresh Python/TypeScript/Go native live runs have not been performed for #7. Direct/dependent archives after growth, sustained concurrent-I/O latency, request cancellation, process/power-loss injection, and every admission-failure variant remain additional qualification work. Do not describe this report as exhaustive fault testing. Stopped growth currently retains previous private file bindings after publishing the replacement head. It does not add chain depth or alter snapshots, but retained files can consume host space; automatic reclamation was not implemented because its deletion scope needs explicit approval/stronger ownership proof. Pending online growth is forward-only. Older runtimes refuse the pending journal field; finish recovery before switching back to an older runtime. A failed operation may leave saved desired configuration behind physical capacity until the caller retries the same requested size. diff --git a/sdk/rust/lib/sandbox/modify.rs b/sdk/rust/lib/sandbox/modify.rs index c63c5e0fd..16da7b806 100644 --- a/sdk/rust/lib/sandbox/modify.rs +++ b/sdk/rust/lib/sandbox/modify.rs @@ -126,56 +126,26 @@ impl SandboxModificationBuilder { self } - /// Set the desired effective guest memory. + /// Set the desired effective guest memory. Accepts a bare `u32` in MiB or a typed size. pub fn memory(mut self, size: impl Into) -> Self { self.patch.memory_mib = Some(size.into().as_u32()); self } - /// Set the desired effective guest memory in MiB. - pub fn memory_mib(mut self, memory_mib: u32) -> Self { - self.patch.memory_mib = Some(memory_mib); - self - } - - /// Set the desired boot-time maximum hotpluggable memory. + /// Set the boot-time maximum hotpluggable memory. Accepts a bare `u32` in MiB or a typed size. pub fn max_memory(mut self, size: impl Into) -> Self { self.patch.max_memory_mib = Some(size.into().as_u32()); self } - /// Set the desired boot-time maximum hotpluggable memory in MiB. - pub fn max_memory_mib(mut self, max_memory_mib: u32) -> Self { - self.patch.max_memory_mib = Some(max_memory_mib); - self - } - - /// Set the desired root disk size. Managed kind: grow-only (shrinking an - /// existing upper risks data loss and is rejected). Tmpfs kind: any - /// direction, effective next boot. Disk-image kind: rejected (user-owned). + /// Set the desired total root disk size, accepting a bare `u32` in MiB or a typed size. + /// Managed and flat roots are grow-only. Tmpfs changes take effect on the next boot; + /// user-owned disk images cannot be resized through this API. pub fn root_disk_size(mut self, size: impl Into) -> Self { self.patch.root_disk_size_mib = Some(size.into().as_u32()); self } - /// Set the desired root disk size in MiB. See [`root_disk_size`](Self::root_disk_size). - pub fn root_disk_size_mib(mut self, size_mib: u32) -> Self { - self.patch.root_disk_size_mib = Some(size_mib); - self - } - - /// Set the desired OCI writable overlay upper size. - #[deprecated(since = "0.6.0", note = "use `root_disk_size` instead")] - pub fn oci_upper_size(self, size: impl Into) -> Self { - self.root_disk_size(size) - } - - /// Set the desired OCI writable overlay upper size in MiB. - #[deprecated(since = "0.6.0", note = "use `root_disk_size_mib` instead")] - pub fn oci_upper_size_mib(self, size_mib: u32) -> Self { - self.root_disk_size_mib(size_mib) - } - /// Set an environment variable for future execs. pub fn env(mut self, key: impl Into, value: impl Into) -> Self { self.patch.env.push(EnvVar::new(key, value)); @@ -2473,6 +2443,32 @@ mod tests { use super::*; use crate::backend::LocalBackend; + use crate::size::SizeExt; + + #[tokio::test] + async fn size_setters_accept_bare_mib_and_typed_sizes() { + let temp = tempdir().unwrap(); + let backend: Arc = Arc::new( + LocalBackend::builder() + .home(temp.path()) + .build() + .await + .unwrap(), + ); + let plain = SandboxModificationBuilder::new(backend.clone(), "size-api") + .memory(1024) + .max_memory(8192) + .root_disk_size(4096); + let typed = SandboxModificationBuilder::new(backend, "size-api") + .memory(1.gib()) + .max_memory(8.gib()) + .root_disk_size(4.gib()); + for patch in [&plain.patch, &typed.patch] { + assert_eq!(patch.memory_mib, Some(1024)); + assert_eq!(patch.max_memory_mib, Some(8192)); + assert_eq!(patch.root_disk_size_mib, Some(4096)); + } + } #[test] #[cfg(unix)] From 99f18d78fb1fd2bb6c5e58f0fb7675e06ac73116 Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Thu, 10 Sep 2026 04:59:38 +0100 Subject: [PATCH 4/6] refactor(cli)!: name the snapshot source flag explicitly Rename snapshot create --from to --from-sandbox to match the existing SDK source-sandbox naming. Update documentation and smoke scripts, including the root-disk growth coverage. Test the required source, full capture, direct archives, and rejection of the old flag. Snapshot parser tests pass (7 tests). BREAKING CHANGE: msb snapshot create requires --from-sandbox instead of --from. No compatibility alias is provided. --- crates/cli/lib/commands/snapshot.rs | 39 ++++++++++++++----- docs/changelog/2026-05-15.mdx | 6 +-- docs/changelog/2026-07-17.mdx | 2 +- docs/examples/data/migration-rehearsal.mdx | 4 +- docs/examples/file-processing/ffmpeg.mdx | 2 +- .../file-processing/libreoffice-pdf.mdx | 2 +- docs/examples/plugins/terraform.mdx | 4 +- docs/examples/sandboxing/warm-workers.mdx | 6 +-- docs/examples/web-automation/scrapy.mdx | 4 +- docs/sandboxes/snapshots.mdx | 16 ++++---- scripts/smoke/cli/checkpoint-clock.sh | 4 +- scripts/smoke/cli/disk-compaction-depth.ps1 | 2 +- scripts/smoke/cli/disk-compaction-depth.sh | 2 +- scripts/smoke/cli/disk-compaction-export.ps1 | 6 +-- scripts/smoke/cli/disk-compaction-export.sh | 6 +-- scripts/smoke/cli/disk-compaction-negative.sh | 2 +- scripts/smoke/cli/root-disk-growth.py | 6 +-- scripts/smoke/cli/root-disk-large-growth.py | 4 +- 18 files changed, 69 insertions(+), 48 deletions(-) diff --git a/crates/cli/lib/commands/snapshot.rs b/crates/cli/lib/commands/snapshot.rs index 8c0d1fd77..a96e4cbc0 100644 --- a/crates/cli/lib/commands/snapshot.rs +++ b/crates/cli/lib/commands/snapshot.rs @@ -54,9 +54,9 @@ pub struct SnapshotCreateArgs { /// (or under `--dest-dir` when given). pub name: String, - /// Source sandbox name. Must be stopped (or crashed). + /// Source sandbox name. #[arg(long, value_name = "SANDBOX")] - pub from: String, + pub from_sandbox: String, /// Parent directory to create the artifact in, instead of the /// default snapshots directory. The artifact lands at `DIR/`. @@ -207,7 +207,7 @@ pub async fn run(args: SnapshotArgs) -> anyhow::Result<()> { } async fn create(args: SnapshotCreateArgs) -> anyhow::Result<()> { - let mut builder = Snapshot::builder(&args.name).from_sandbox(&args.from); + let mut builder = Snapshot::builder(&args.name).from_sandbox(&args.from_sandbox); if let Some(ref dest_dir) = args.dest_dir { builder = builder.dest_dir(dest_dir); } @@ -230,7 +230,7 @@ async fn create(args: SnapshotCreateArgs) -> anyhow::Result<()> { let spinner = if args.quiet { ui::Spinner::quiet() } else { - ui::Spinner::start("Snapshotting", &args.from) + ui::Spinner::start("Snapshotting", &args.from_sandbox) }; if let Some(archive_path) = args.archive.as_ref() { @@ -564,21 +564,42 @@ mod tests { TestCli::parse_from(std::iter::once("msb").chain(args.iter().copied())).args } + #[test] + fn create_requires_explicit_source_sandbox_flag() { + let error = TestCli::try_parse_from(["msb", "create", "clean"]).unwrap_err(); + assert_eq!( + error.kind(), + clap::error::ErrorKind::MissingRequiredArgument + ); + assert!(error.to_string().contains("--from-sandbox ")); + + // This is a clean rename, not an alias: reject the old ambiguous spelling. + let error = + TestCli::try_parse_from(["msb", "create", "clean", "--from", "box"]).unwrap_err(); + assert_eq!(error.kind(), clap::error::ErrorKind::UnknownArgument); + } + #[test] fn create_parses_full_capture_flag() { - let args = parse_snapshot_args(&["create", "clean", "--from", "box", "--full"]); + let args = parse_snapshot_args(&["create", "clean", "--from-sandbox", "box", "--full"]); let SnapshotCommands::Create(args) = args.command else { panic!("expected create command"); }; assert_eq!(args.name, "clean"); - assert_eq!(args.from, "box"); + assert_eq!(args.from_sandbox, "box"); assert!(args.full); } #[test] fn create_parses_dest_dir() { - let args = - parse_snapshot_args(&["create", "clean", "--from", "box", "--dest-dir", "/mnt/big"]); + let args = parse_snapshot_args(&[ + "create", + "clean", + "--from-sandbox", + "box", + "--dest-dir", + "/mnt/big", + ]); let SnapshotCommands::Create(args) = args.command else { panic!("expected create command"); }; @@ -593,7 +614,7 @@ mod tests { let args = parse_snapshot_args(&[ "create", "clean", - "--from", + "--from-sandbox", "box", "--archive", "/tmp/clean.tar", diff --git a/docs/changelog/2026-05-15.mdx b/docs/changelog/2026-05-15.mdx index 2a0b0ba5a..70c0b1c90 100644 --- a/docs/changelog/2026-05-15.mdx +++ b/docs/changelog/2026-05-15.mdx @@ -31,11 +31,11 @@ See the [Go SDK reference](/sdk/go/sandbox). **File-first disk snapshots** -Stopped sandboxes can be snapshotted to a content-addressed directory and used to boot fresh sandboxes on any compatible host. Snapshots stay sparse via reflink and `SEEK_DATA`/`SEEK_HOLE` copying, so create and inspect stay fast. The CLI ships `msb snapshot create | open | list | list-dir | remove | reindex | export | import | verify`, and `msb run --from ` boots a fork. The same surface is available in all four SDKs. +Stopped sandboxes can be snapshotted to a content-addressed directory and used to boot fresh sandboxes on any compatible host. Snapshots stay sparse via reflink and `SEEK_DATA`/`SEEK_HOLE` copying, so create and inspect stay fast. The CLI ships `msb snapshot create | open | list | list-dir | remove | reindex | export | import | verify`, and `msb run --from-snapshot ` boots a fork. The same surface is available in all four SDKs. ```bash -msb snapshot create my-sandbox --name baseline -msb run alpine --from baseline --name fork +msb snapshot create baseline --from-sandbox my-sandbox +msb run --from-snapshot baseline --name fork ``` See [Sandbox snapshots](/sandboxes/snapshots). diff --git a/docs/changelog/2026-07-17.mdx b/docs/changelog/2026-07-17.mdx index ea7186dd7..941c9c69d 100644 --- a/docs/changelog/2026-07-17.mdx +++ b/docs/changelog/2026-07-17.mdx @@ -44,7 +44,7 @@ The snapshot surface is now locked in across the CLI and every SDK ahead of 1.0. Booting from a snapshot is spelled the same way everywhere: `--from-snapshot`, `from_snapshot=`, `fromSnapshot`, and Go `WithFromSnapshot`. ```bash -msb snapshot create clean --from box --dest-dir /mnt/big +msb snapshot create clean --from-sandbox box --dest-dir /mnt/big msb snapshot save clean /tmp/clean.tar.zst --with-image msb snapshot load /tmp/clean.tar.zst msb run --name worker --from-snapshot clean -- python -V diff --git a/docs/examples/data/migration-rehearsal.mdx b/docs/examples/data/migration-rehearsal.mdx index d54905ac7..48cf9e6c1 100644 --- a/docs/examples/data/migration-rehearsal.mdx +++ b/docs/examples/data/migration-rehearsal.mdx @@ -77,13 +77,13 @@ msb stop migration-base ```sh macOS & Linux msb snapshot create postgres-before-migration \ - --from migration-base \ + --from-sandbox migration-base \ --integrity ``` ```powershell Windows msb snapshot create postgres-before-migration ` - --from migration-base ` + --from-sandbox migration-base ` --integrity ``` diff --git a/docs/examples/file-processing/ffmpeg.mdx b/docs/examples/file-processing/ffmpeg.mdx index 08e45649b..c311c2d8d 100644 --- a/docs/examples/file-processing/ffmpeg.mdx +++ b/docs/examples/file-processing/ffmpeg.mdx @@ -87,7 +87,7 @@ msb run --name ffmpeg-base --replace ` Capture the prepared toolchain: ```sh -msb snapshot create ffmpeg-tools --from ffmpeg-base --integrity --force +msb snapshot create ffmpeg-tools --from-sandbox ffmpeg-base --integrity --force ``` Verify the snapshot before using it: diff --git a/docs/examples/file-processing/libreoffice-pdf.mdx b/docs/examples/file-processing/libreoffice-pdf.mdx index 60b500938..ffcea2bfa 100644 --- a/docs/examples/file-processing/libreoffice-pdf.mdx +++ b/docs/examples/file-processing/libreoffice-pdf.mdx @@ -85,7 +85,7 @@ msb run --name office-base --replace ` Capture the prepared toolchain: ```sh -msb snapshot create office-tools --from office-base --integrity --force +msb snapshot create office-tools --from-sandbox office-base --integrity --force ``` Verify the snapshot before using it: diff --git a/docs/examples/plugins/terraform.mdx b/docs/examples/plugins/terraform.mdx index 275e0dee5..880a84e9f 100644 --- a/docs/examples/plugins/terraform.mdx +++ b/docs/examples/plugins/terraform.mdx @@ -69,12 +69,12 @@ Capture the downloaded provider: ```sh macOS & Linux msb snapshot create terraform-runtime \ - --from terraform-base --integrity --force + --from-sandbox terraform-base --integrity --force ``` ```powershell Windows msb snapshot create terraform-runtime ` - --from terraform-base --integrity --force + --from-sandbox terraform-base --integrity --force ``` diff --git a/docs/examples/sandboxing/warm-workers.mdx b/docs/examples/sandboxing/warm-workers.mdx index 8c8d6dd97..26bda833a 100644 --- a/docs/examples/sandboxing/warm-workers.mdx +++ b/docs/examples/sandboxing/warm-workers.mdx @@ -72,14 +72,14 @@ When the command exits, `agent-base` is stopped and ready to snapshot. ```sh macOS & Linux msb snapshot create coding-agent-base \ - --from agent-base \ + --from-sandbox agent-base \ --integrity \ --force ``` ```powershell Windows msb snapshot create coding-agent-base ` - --from agent-base ` + --from-sandbox agent-base ` --integrity ` --force ``` @@ -238,7 +238,7 @@ msb snapshot rm coding-agent-base Snapshots are immutable. To update packages, recreate `agent-base`, then overwrite the named snapshot intentionally: ```sh -msb snapshot create coding-agent-base --from agent-base --integrity --force +msb snapshot create coding-agent-base --from-sandbox agent-base --integrity --force ``` See [Snapshots](/sandboxes/snapshots) for archive, integrity, and portability details. diff --git a/docs/examples/web-automation/scrapy.mdx b/docs/examples/web-automation/scrapy.mdx index 2c8928899..bacd9912f 100644 --- a/docs/examples/web-automation/scrapy.mdx +++ b/docs/examples/web-automation/scrapy.mdx @@ -69,12 +69,12 @@ Capture the prepared environment: ```sh macOS & Linux msb snapshot create scrapy-runtime \ - --from scrapy-base --integrity --force + --from-sandbox scrapy-base --integrity --force ``` ```powershell Windows msb snapshot create scrapy-runtime ` - --from scrapy-base --integrity --force + --from-sandbox scrapy-base --integrity --force ``` diff --git a/docs/sandboxes/snapshots.mdx b/docs/sandboxes/snapshots.mdx index 26af4df6b..52a56c567 100644 --- a/docs/sandboxes/snapshots.mdx +++ b/docs/sandboxes/snapshots.mdx @@ -32,7 +32,7 @@ msb exec baseline -- pip install requests msb stop baseline # 2. Snapshot the stopped sandbox -msb snapshot create after-pip-install --from baseline +msb snapshot create after-pip-install --from-sandbox baseline # 3. Boot a fresh sandbox from the snapshot msb run --name worker --from-snapshot after-pip-install \ @@ -94,11 +94,11 @@ fmt.Println(snap.Digest()) // sha256:... ``` ```bash CLI -msb snapshot create after-pip-install --from baseline -msb snapshot create after-pip-install --from baseline --label stage=ready +msb snapshot create after-pip-install --from-sandbox baseline +msb snapshot create after-pip-install --from-sandbox baseline --label stage=ready # Create the artifact on another volume: lands at /mnt/big/after-pip-install -msb snapshot create after-pip-install --from baseline --dest-dir /mnt/big +msb snapshot create after-pip-install --from-sandbox baseline --dest-dir /mnt/big ``` @@ -142,7 +142,7 @@ snap, err := m.Snapshot.Create(ctx, m.SnapshotCreateOptions{ ``` ```bash CLI -msb snapshot create worker-checkpoint --from worker --full +msb snapshot create worker-checkpoint --from-sandbox worker --full ``` @@ -155,7 +155,7 @@ Creating a sandbox from the resulting snapshot restores into a child-owned disk Repeated full checkpoints add disk layers. You choose when to export changes and when to compact; neither happens automatically. ```bash -msb snapshot create checkpoint-b --from worker --full +msb snapshot create checkpoint-b --from-sandbox worker --full msb snapshot save checkpoint-b changes.tar.zst --since checkpoint-a msb modify worker --compact --layers 3 --dry-run msb modify worker --compact --layers 3 @@ -216,7 +216,7 @@ archive, err := m.Snapshot.CreateArchive(ctx, m.SnapshotArchiveOptions{ ```bash CLI msb snapshot create after-pip-install \ - --from baseline \ + --from-sandbox baseline \ --archive /tmp/after-pip-install.tar.zst ``` @@ -484,7 +484,7 @@ report, err := snap.Verify(ctx) ```bash CLI # Compute and record an integrity hash at create time -msb snapshot create after-pip-install --from baseline --integrity +msb snapshot create after-pip-install --from-sandbox baseline --integrity # Verify a snapshot's recorded integrity on demand msb snapshot verify after-pip-install diff --git a/scripts/smoke/cli/checkpoint-clock.sh b/scripts/smoke/cli/checkpoint-clock.sh index 2285461f7..bf4253aaa 100644 --- a/scripts/smoke/cli/checkpoint-clock.sh +++ b/scripts/smoke/cli/checkpoint-clock.sh @@ -17,10 +17,10 @@ for _ in $(seq 1 30); do if msb exec "$source_name" -- test -s /tmp/clock-records.csv; then break; fi sleep 0.05 done -msb snapshot create "$CLOCK_PREFIX-full" --from "$source_name" --full --info >"$CLOCK_OUT/capture.out" 2>"$CLOCK_OUT/capture.err" +msb snapshot create "$CLOCK_PREFIX-full" --from-sandbox "$source_name" --full --info >"$CLOCK_OUT/capture.out" 2>"$CLOCK_OUT/capture.err" snapshot="$CLOCK_PREFIX-full" if [ "${CLOCK_INCREMENTAL:-0}" = 1 ]; then - msb snapshot create "$CLOCK_PREFIX-next" --from "$source_name" --full --info >"$CLOCK_OUT/capture-next.out" 2>"$CLOCK_OUT/capture-next.err" + msb snapshot create "$CLOCK_PREFIX-next" --from-sandbox "$source_name" --full --info >"$CLOCK_OUT/capture-next.out" 2>"$CLOCK_OUT/capture-next.err" snapshot="$CLOCK_PREFIX-next" fi if [ "${CLOCK_ARCHIVE:-0}" = 1 ]; then diff --git a/scripts/smoke/cli/disk-compaction-depth.ps1 b/scripts/smoke/cli/disk-compaction-depth.ps1 index 1145bf66f..f87a6f70d 100644 --- a/scripts/smoke/cli/disk-compaction-depth.ps1 +++ b/scripts/smoke/cli/disk-compaction-depth.ps1 @@ -27,7 +27,7 @@ try { M @('exec',$name,'--','sh','-c','dd if=/dev/urandom of=/payload bs=1048576 count=4 2>/dev/null; sha256sum /payload >/expected; sync') foreach ($generation in 1..64) { M @('exec',$name,'--','sh','-c',"echo $generation >/version; sync") - M @('snapshot','create',"$name-$generation",'--from',$name,'--full') *> "$output\$layout-$generation.log" + M @('snapshot','create',"$name-$generation",'--from-sandbox',$name,'--full') *> "$output\$layout-$generation.log" if ($generation -in @(1,4,16,64)) { M @('modify',$name,'--compact','--dry-run','--format','json') > "$output\$layout-depth-$generation.json" } } } diff --git a/scripts/smoke/cli/disk-compaction-depth.sh b/scripts/smoke/cli/disk-compaction-depth.sh index 9fefbe5a5..f9ef6a46e 100644 --- a/scripts/smoke/cli/disk-compaction-depth.sh +++ b/scripts/smoke/cli/disk-compaction-depth.sh @@ -13,7 +13,7 @@ for layout in managed flat; do msb exec "$name" -- sh -c 'dd if=/dev/urandom of=/payload bs=1048576 count=4 2>/dev/null; sha256sum /payload >/expected; sync' for generation in $(seq 1 64); do msb exec "$name" -- sh -c "echo $generation >/version; sync" >/dev/null - msb snapshot create "$name-$generation" --from "$name" --full >"$QUAL_ROOT/$layout-$generation.out" 2>&1 + msb snapshot create "$name-$generation" --from-sandbox "$name" --full >"$QUAL_ROOT/$layout-$generation.out" 2>&1 case $generation in 1|4|16|64) msb modify "$name" --compact --dry-run --format json >"$QUAL_ROOT/$layout-depth-$generation.json" msb exec "$name" -- sh -c 'sha256sum -c /expected' >/dev/null;; diff --git a/scripts/smoke/cli/disk-compaction-export.ps1 b/scripts/smoke/cli/disk-compaction-export.ps1 index 4b5eb5ecd..08ecfc686 100644 --- a/scripts/smoke/cli/disk-compaction-export.ps1 +++ b/scripts/smoke/cli/disk-compaction-export.ps1 @@ -34,7 +34,7 @@ try { Measure-Msb "$layout-seed" @('exec',$name,'--','sh','-c','dd if=/dev/urandom of=/payload bs=1048576 count=8 2>/dev/null; sha256sum /payload >/expected; mkdir -p /dev/shm; echo volatile >/dev/shm/ram-marker; sync') foreach ($generation in 1..4) { Measure-Msb "$layout-write-$generation" @('exec',$name,'--','sh','-c',"echo $generation >/version; sync") - Measure-Msb "$layout-checkpoint-$generation" @('snapshot','create',"$name-$generation",'--from',$name,'--full') + Measure-Msb "$layout-checkpoint-$generation" @('snapshot','create',"$name-$generation",'--from-sandbox',$name,'--full') } Measure-Msb "$layout-dry-run" @('modify',$name,'--compact','--layers','3','--dry-run','--format','json') $plan = Get-Content "$output\logs\$layout-dry-run.out" -Raw | ConvertFrom-Json @@ -53,11 +53,11 @@ try { Measure-Msb "$layout-data-after" @('exec',$name,'--','sh','-c','sha256sum -c /expected && test $(cat /version) = 4 && echo after >/after && sync') Measure-Msb "$layout-stop" @('stop',$name) Measure-Msb "$layout-offline-compact" @('modify',$name,'--compact','--format','json') - Measure-Msb "$layout-stopped-snapshot" @('snapshot','create',"$name-stopped",'--from',$name,'--integrity') + Measure-Msb "$layout-stopped-snapshot" @('snapshot','create',"$name-stopped",'--from-sandbox',$name,'--integrity') Measure-Msb "$layout-stopped-verify" @('snapshot','verify',"$name-stopped") Measure-Msb "$layout-restart" @('start',$name) Measure-Msb "$layout-restarted-data" @('exec',$name,'--','sh','-c','sha256sum -c /expected && test $(cat /version) = 4 && test $(cat /after) = after') - Measure-Msb "$layout-post-compact-checkpoint" @('snapshot','create',"$name-new",'--from',$name,'--full') + Measure-Msb "$layout-post-compact-checkpoint" @('snapshot','create',"$name-new",'--from-sandbox',$name,'--full') Measure-Msb "$layout-old-prefix-rejected" @('snapshot','save',"$name-new","$output\invalid.tar",'--since',"$name-4") $true Measure-Msb "$layout-stop-source" @('stop',$name) foreach ($variant in @('old','full','disk','stopped')) { diff --git a/scripts/smoke/cli/disk-compaction-export.sh b/scripts/smoke/cli/disk-compaction-export.sh index 950e2ba22..0381f11d4 100644 --- a/scripts/smoke/cli/disk-compaction-export.sh +++ b/scripts/smoke/cli/disk-compaction-export.sh @@ -36,7 +36,7 @@ for layout in managed flat; do measure "$layout-seed" guest "$name" 'dd if=/dev/urandom of=/payload bs=1048576 count=8 2>/dev/null; sha256sum /payload >/expected; printf 1 >/version; mkdir -p /dev/shm; echo volatile >/dev/shm/ram-marker; sync' for generation in 1 2 3 4; do measure "$layout-write-$generation" guest "$name" "printf $generation >/version; sync" - measure "$layout-checkpoint-$generation" msb snapshot create "$name-$generation" --from "$name" --full + measure "$layout-checkpoint-$generation" msb snapshot create "$name-$generation" --from-sandbox "$name" --full done measure "$layout-dry-run" msb modify "$name" --compact --layers 3 --dry-run --format json measure "$layout-dry-run-counts" jq -e '.dry_run and .input_layers == 5 and .selected_layers == 3 and .output_layers == 3' "$QUAL_ROOT/logs/$layout-dry-run.out" @@ -61,11 +61,11 @@ for layout in managed flat; do measure "$layout-online-counts" jq -e '.input_layers == 5 and .output_layers == 3 and .selected_layers == 3' "$QUAL_ROOT/logs/$layout-online-compact.out" measure "$layout-stop" msb stop "$name" measure "$layout-offline-compact" msb modify "$name" --compact --format json - measure "$layout-stopped-snapshot" msb snapshot create "$name-stopped" --from "$name" --integrity + measure "$layout-stopped-snapshot" msb snapshot create "$name-stopped" --from-sandbox "$name" --integrity measure "$layout-stopped-verify" msb snapshot verify "$name-stopped" measure "$layout-restart" msb start "$name" measure "$layout-restarted-data" guest "$name" 'sha256sum -c /expected && test "$(cat /version)" = 4' - measure "$layout-post-compact-checkpoint" msb snapshot create "$name-new" --from "$name" --full + measure "$layout-post-compact-checkpoint" msb snapshot create "$name-new" --from-sandbox "$name" --full measure "$layout-old-prefix-rejected" refuse msb snapshot save "$name-new" "$QUAL_ROOT/invalid.tar" --since "$name-4" measure "$layout-stop-source" msb stop "$name" diff --git a/scripts/smoke/cli/disk-compaction-negative.sh b/scripts/smoke/cli/disk-compaction-negative.sh index 42bb2c0d0..2619e0e80 100644 --- a/scripts/smoke/cli/disk-compaction-negative.sh +++ b/scripts/smoke/cli/disk-compaction-negative.sh @@ -8,7 +8,7 @@ trap cleanup EXIT refuse() { if "$@"; then echo 'unexpected success' >&2; exit 1; fi; } msb create -n compact-neg-tmpfs --root-disk tmpfs:128M -m 256M --max-duration 5m alpine refuse msb modify compact-neg-tmpfs --compact -msb snapshot create compact-neg-tmpfs-snap --from compact-neg-tmpfs --full +msb snapshot create compact-neg-tmpfs-snap --from-sandbox compact-neg-tmpfs --full refuse msb snapshot save compact-neg-tmpfs-snap "$QUAL_ROOT/tmpfs.tar" --last-layers 1 msb stop compact-neg-tmpfs cp "$MSB_HOME/sandboxes/$QUAL_SOURCE/upper.ext4" "$QUAL_ROOT/owned.ext4" diff --git a/scripts/smoke/cli/root-disk-growth.py b/scripts/smoke/cli/root-disk-growth.py index 60d997c15..9adc913b3 100644 --- a/scripts/smoke/cli/root-disk-growth.py +++ b/scripts/smoke/cli/root-disk-growth.py @@ -76,7 +76,7 @@ def file_hash(path): guest(f"{layout}-seed", name, "dd if=/dev/urandom of=/payload bs=1048576 count=8 2>/dev/null; sha256sum /payload >/expected; echo ram >/dev/shm/grow-marker; sync") phase_grow(f"{layout}-raw-live", name, 768) guest(f"{layout}-new-space", name, "dd if=/dev/zero of=/space bs=1048576 count=600 conv=fsync && test $(stat -c %s /space) = 629145600 && rm /space && sha256sum -c /expected") - run(f"{layout}-old-snapshot", "snapshot", "create", f"{name}-old", "--from", name, "--full") + run(f"{layout}-old-snapshot", "snapshot", "create", f"{name}-old", "--from-sandbox", name, "--full") before = journal(name) ancestor = Path(before["layers"][0]["path"]) ancestor_before = file_hash(ancestor) @@ -85,7 +85,7 @@ def file_hash(path): assert len(journal(name)["layers"]) == len(before["layers"]) assert file_hash(ancestor) == ancestor_before run(f"{layout}-shrink-refused", "modify", name, "--root-disk", "512M", refuse=True) - run(f"{layout}-new-snapshot", "snapshot", "create", f"{name}-new", "--from", name, "--full") + run(f"{layout}-new-snapshot", "snapshot", "create", f"{name}-new", "--from-sandbox", name, "--full") run(f"{layout}-compact", "modify", name, "--compact", "--format", "json") phase_grow(f"{layout}-compacted-live", name, 1536) phase_grow(f"{layout}-partial-group-live", name, 1700) @@ -98,7 +98,7 @@ def file_hash(path): run(f"{layout}-defer-start", "start", name) guest(f"{layout}-defer-space", name, "dd if=/dev/zero of=/space bs=1048576 count=1800 conv=fsync && rm /space && sha256sum -c /expected") run(f"{layout}-final-stop", "stop", name) - run(f"{layout}-stopped-snapshot", "snapshot", "create", f"{name}-stopped", "--from", name, "--integrity") + run(f"{layout}-stopped-snapshot", "snapshot", "create", f"{name}-stopped", "--from-sandbox", name, "--integrity") run(f"{layout}-verify", "snapshot", "verify", f"{name}-stopped") for suffix, capacity in (("old", 768), ("new", 1280)): child = f"{name}-{suffix}-child" diff --git a/scripts/smoke/cli/root-disk-large-growth.py b/scripts/smoke/cli/root-disk-large-growth.py index 7a50a85f6..fadcc0b6d 100644 --- a/scripts/smoke/cli/root-disk-large-growth.py +++ b/scripts/smoke/cli/root-disk-large-growth.py @@ -90,7 +90,7 @@ def case(layout, backing, mode, target): "cat /proc/sys/kernel/random/boot_id >/boot-before; sync") if backing == "qcow2": run(label + "-old-snapshot", "snapshot", "create", name + "-old", - "--from", name, "--full") + "--from-sandbox", name, "--full") before = state(name) check(label + "-initial-capacity", capacity(before["layers"][-1]) == 512 * MIB) check(label + "-initial-format", @@ -133,7 +133,7 @@ def case(layout, backing, mode, target): f"dd if=/far bs=1048576 skip={target - 256} count=8 2>/dev/null | cmp - /payload; " "df -k /; du -k /large") # Snapshot the allocated file, not just metadata or an empty resized filesystem. - run(label + "-new-snapshot", "snapshot", "create", name + "-new", "--from", name, "--full") + run(label + "-new-snapshot", "snapshot", "create", name + "-new", "--from-sandbox", name, "--full") run(label + "-source-stop", "stop", name) for suffix, expected in [("new", target)] + ([("old", 512)] if backing == "qcow2" else []): child = name + "-" + suffix + "-child" From 29a3ecf5f2d9d7584ced6712ef782cbf12d80a25 Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Thu, 10 Sep 2026 13:02:57 +0100 Subject: [PATCH 5/6] build(runtime): use msb_krun 0.1.35 Pin msb_krun and msb_krun_utils to the published 0.1.35 release and refresh only their matching lockfile entries. Pick up runtime resolution of optional macOS GIC APIs without changing the snapshot format, guest protocol, or stack ancestry. --- Cargo.lock | 44 ++++++++++++++++++++++---------------------- Cargo.toml | 4 ++-- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 18d8a1f55..ba1b70887 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4379,9 +4379,9 @@ dependencies = [ [[package]] name = "msb_krun" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6a401c1ec192a3dfbc382901d8c0275f42c36097991a6426f5502a30f017ff0" +checksum = "1d82b78e6694529738582317c2db84b775f6b2f6e76e8429336d40a9ada9dc7e" dependencies = [ "crossbeam-channel", "kvm-bindings", @@ -4399,9 +4399,9 @@ dependencies = [ [[package]] name = "msb_krun_arch" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7362283e6bc924947c2a2415d7cd9025616fa45ea61bc4ec3ea00bddf8df36df" +checksum = "234c8c322b99007c2ad7e60a705ae4b89c16c0b69906259425e32671dca1a32e" dependencies = [ "kvm-bindings", "kvm-ioctls", @@ -4414,15 +4414,15 @@ dependencies = [ [[package]] name = "msb_krun_arch_gen" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4101f8a221a95e8e747684c65d001b6d49899f88e81b41122f9de0fba3eefc3" +checksum = "a8d6558b6564ad5fceadd7291b895fd4ba8d317c8d51a5a89a27117c3bd19837" [[package]] name = "msb_krun_cpuid" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dacf8d69bc3a568970bfc98918f2f60786a5438f8b5701b16dc0ac9f40af6f58" +checksum = "5fae1941e85a4ba925807c45956c1014ccb39e7364dd4e416ab189bf06d3f950" dependencies = [ "kvm-bindings", "kvm-ioctls", @@ -4431,9 +4431,9 @@ dependencies = [ [[package]] name = "msb_krun_devices" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe443bd0f03c68eaf33d63da12e8b3084ec080a6c105d8768d711cab062f2c60" +checksum = "69b2fdcb2c1573272ca936d009796b77072e38e2812a574a1334c9c21897d3b8" dependencies = [ "bincode", "bitflags 1.3.2", @@ -4463,9 +4463,9 @@ dependencies = [ [[package]] name = "msb_krun_hvf" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f5833d1d7e6404ae1e3cc3be50a5ad3eceb1c61c2bdc3bb296f223ac0615a2a" +checksum = "454e8f0088ae171a55dfbcfe69542d1d0c12f7f3b43d3281d9d0124016a3d8ec" dependencies = [ "crossbeam-channel", "libloading 0.8.9", @@ -4476,9 +4476,9 @@ dependencies = [ [[package]] name = "msb_krun_kernel" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "189508d48404ed0768d882f7e93e8ca1a093a77621462c4aa96f0637a02792be" +checksum = "5ecbe5ba13631934f84b57b231c1e7bef371e1d42d2ec940242f9629d28fb196" dependencies = [ "msb-vm-memory", "msb_krun_utils", @@ -4486,9 +4486,9 @@ dependencies = [ [[package]] name = "msb_krun_polly" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e849910d2531272df93806a1e2f5830145d76e64c2d0161aa75a69dcc24ead8" +checksum = "50d0d1bf51ad1fd68cf61d41f3c17d51abb8287e72e1d9a8da0419579a81ba92" dependencies = [ "libc", "msb_krun_utils", @@ -4496,18 +4496,18 @@ dependencies = [ [[package]] name = "msb_krun_smbios" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cf963ea3fcf2e564dd8982ae663339629eaf19a0ec1a4bd161d42c43e23a4ad" +checksum = "f7e5b6e6bd33e9097e882d359abeaa325553edfcf44cf51baa7a8cc827f6dcc7" dependencies = [ "msb-vm-memory", ] [[package]] name = "msb_krun_utils" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "078445461df117b827681679390565e6f1ecf06a8d1f309d641dc6a1e390531c" +checksum = "d9e68f68e5d3c953e5d71e89f6c2cf4a2a1c69fcafa6b856521eee4bb6485646" dependencies = [ "bitflags 1.3.2", "crossbeam-channel", @@ -4521,9 +4521,9 @@ dependencies = [ [[package]] name = "msb_krun_vmm" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "223013c35221736169599da5a71fb5a801346c7a5e4439f21461bf5e503f16bb" +checksum = "099b488e7e337eee46a77a67ba3a99a280ceada4b621639b81a6ab8f7b9c81ba" dependencies = [ "bincode", "bzip2", diff --git a/Cargo.toml b/Cargo.toml index 7d4bd3563..9c02b323d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -91,8 +91,8 @@ microsandbox-protocol = { version = "=0.6.18", path = "crates/protocol" } microsandbox-runtime = { version = "=0.6.18", path = "crates/runtime", default-features = false } microsandbox-utils = { version = "=0.6.18", path = "crates/utils" } microsandbox-vsock = { version = "=0.6.18", path = "crates/vsock" } -msb_krun = "=0.1.34" -msb_krun_utils = "=0.1.34" +msb_krun = "=0.1.35" +msb_krun_utils = "=0.1.35" test-macros = { path = "crates/testing/macros" } test-utils = { path = "crates/testing/utils" } From b4b972b6677d5bb42a1e02ecd6477d6cb82fe344 Mon Sep 17 00:00:00 2001 From: Stephen Akinyemi Date: Thu, 10 Sep 2026 14:01:28 +0100 Subject: [PATCH 6/6] fix(runtime): restore Windows stdio inheritance on failure Keep stdio guard construction in the I/O error domain so the runtime spawn helper compiles on Windows. Own successful flag changes immediately and roll them back if a later handle update fails. Add Windows regression tests for partial failure, duplicate handles and unchanged non-inheritable handles, and run them in both Windows CI jobs. --- .github/workflows/check.yml | 8 ++ sdk/rust/lib/runtime/spawn.rs | 135 +++++++++++++++++++++++++++++++--- 2 files changed, 133 insertions(+), 10 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index d4743b324..943d396ff 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -569,6 +569,14 @@ jobs: Set-MsvcEnvironment -Architecture ${{ matrix.vs_arch }} -HostArchitecture ${{ matrix.vs_host_arch }} cargo +stable test --no-default-features --features local,net -p microsandbox --lib --target ${{ matrix.rust_target }} sandbox::patch::tests::bind_patch_ + - name: Test Windows stdio inheritance cleanup + shell: pwsh + run: | + $ErrorActionPreference = "Stop" + . "$env:GITHUB_WORKSPACE\vendor\libkrunfw\scripts\msvc-env.ps1" + Set-MsvcEnvironment -Architecture ${{ matrix.vs_arch }} -HostArchitecture ${{ matrix.vs_host_arch }} + cargo +stable test --no-default-features --features local,net -p microsandbox --lib --target ${{ matrix.rust_target }} runtime::spawn::tests::windows_stdio_guard_ + - name: Test Windows DNS resolver shell: pwsh run: | diff --git a/sdk/rust/lib/runtime/spawn.rs b/sdk/rust/lib/runtime/spawn.rs index 2da21f87a..8b48fe47e 100644 --- a/sdk/rust/lib/runtime/spawn.rs +++ b/sdk/rust/lib/runtime/spawn.rs @@ -207,15 +207,32 @@ impl EnsuredNamedVolumes { #[cfg(windows)] impl StdioInheritGuard { - fn new() -> MicrosandboxResult { - let mut states = Vec::new(); + fn new() -> std::io::Result { + let handles = [STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, STD_ERROR_HANDLE] + .map(|std_handle| unsafe { GetStdHandle(std_handle) }); + Self::from_handles(handles, |handle| { + if unsafe { SetHandleInformation(handle, HANDLE_FLAG_INHERIT, 0) } == 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } + }) + } + + fn from_handles( + handles: impl IntoIterator, + mut clear_inherit: impl FnMut(HANDLE) -> std::io::Result<()>, + ) -> std::io::Result { + // Own each successful change immediately so an error on a later handle + // restores the earlier handles through Drop before returning. + let mut guard = Self { states: Vec::new() }; - for std_handle in [STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, STD_ERROR_HANDLE] { - let handle = unsafe { GetStdHandle(std_handle) }; + for handle in handles { if handle.is_null() || handle == INVALID_HANDLE_VALUE { continue; } - if states + if guard + .states .iter() .any(|state: &HandleInheritState| state.handle == handle) { @@ -233,13 +250,11 @@ impl StdioInheritGuard { // A redirected `msb create` can receive inheritable stdout/stderr // pipe handles from its own parent. Detached sandbox children must // not keep those pipes alive after the launcher exits. - if unsafe { SetHandleInformation(handle, HANDLE_FLAG_INHERIT, 0) } == 0 { - return Err(std::io::Error::last_os_error().into()); - } - states.push(HandleInheritState { handle, flags }); + clear_inherit(handle)?; + guard.states.push(HandleInheritState { handle, flags }); } - Ok(Self { states }) + Ok(guard) } } @@ -2922,6 +2937,106 @@ mod tests { volume::VolumeKind, }; + #[cfg(windows)] + fn windows_handle_flags(handle: super::HANDLE) -> u32 { + let mut flags = 0; + assert_ne!( + unsafe { super::GetHandleInformation(handle, &mut flags) }, + 0 + ); + flags + } + + #[cfg(windows)] + fn windows_set_handle_inherit(handle: super::HANDLE, inherit: bool) { + assert_ne!( + unsafe { + super::SetHandleInformation( + handle, + super::HANDLE_FLAG_INHERIT, + if inherit { + super::HANDLE_FLAG_INHERIT + } else { + 0 + }, + ) + }, + 0 + ); + } + + #[cfg(windows)] + #[test] + fn windows_stdio_guard_deduplicates_and_restores_handles() { + use std::os::windows::io::AsRawHandle; + + // Use private file handles; never change the test runner's process-wide stdio. + let file = tempfile::tempfile().unwrap(); + let untouched = tempfile::tempfile().unwrap(); + let handle = file.as_raw_handle(); + let untouched_handle = untouched.as_raw_handle(); + windows_set_handle_inherit(handle, true); + windows_set_handle_inherit(untouched_handle, false); + let original = windows_handle_flags(handle); + let mut calls = 0; + let guard = super::StdioInheritGuard::from_handles( + [ + std::ptr::null_mut(), + super::INVALID_HANDLE_VALUE, + handle, + handle, + untouched_handle, + ], + |handle| { + calls += 1; + windows_set_handle_inherit(handle, false); + Ok(()) + }, + ) + .unwrap(); + assert_eq!(calls, 1); + assert_eq!(windows_handle_flags(handle) & super::HANDLE_FLAG_INHERIT, 0); + assert_eq!( + windows_handle_flags(untouched_handle) & super::HANDLE_FLAG_INHERIT, + 0 + ); + drop(guard); + assert_eq!(windows_handle_flags(handle), original); + assert_eq!( + windows_handle_flags(untouched_handle) & super::HANDLE_FLAG_INHERIT, + 0 + ); + } + + #[cfg(windows)] + #[test] + fn windows_stdio_guard_rolls_back_partial_failure() { + use std::os::windows::io::AsRawHandle; + + let first = tempfile::tempfile().unwrap(); + let second = tempfile::tempfile().unwrap(); + let handles = [first.as_raw_handle(), second.as_raw_handle()]; + for handle in handles { + windows_set_handle_inherit(handle, true); + } + let original = handles.map(windows_handle_flags); + let mut calls = 0; + let result = super::StdioInheritGuard::from_handles(handles, |handle| { + calls += 1; + if calls == 2 { + return Err(std::io::Error::from_raw_os_error(5)); + } + windows_set_handle_inherit(handle, false); + Ok(()) + }); + let error = match result { + Ok(_) => panic!("expected the second handle update to fail"), + Err(error) => error, + }; + assert_eq!(error.raw_os_error(), Some(5)); + assert_eq!(handles.map(windows_handle_flags), original); + } + #[test] #[cfg(unix)] fn test_inherited_fd_source_needs_spare_for_cross_reserved_fd() {