diff --git a/crates/agentd/lib/agent.rs b/crates/agentd/lib/agent.rs index 9b794d6e0..ec4b9d05d 100644 --- a/crates/agentd/lib/agent.rs +++ b/crates/agentd/lib/agent.rs @@ -2161,6 +2161,37 @@ async fn handle_message( // queued output after a disconnect instead of relabelling it with a recycled correlation ID. let session_tx = root_session_tx.with_incarnation(client_incarnation_for_id(state, msg.id)); match msg.t { + MessageType::RootDiskPrepare | MessageType::RootDiskGrow => { + let Some(request) = decode_payload_or_core_error::< + microsandbox_protocol::core::RootDiskGrow, + >(&msg, out_buf)? + else { + return Ok(()); + }; + let apply = msg.t == MessageType::RootDiskGrow; + let result = tokio::task::spawn_blocking(move || { + crate::root_disk::resize(request.size_bytes, apply) + }) + .await + .map_err(|e| AgentdError::ExecSession(format!("root grow worker: {e}")))?; + let reply = match result { + Ok(state) => Message::with_payload(MessageType::RootDiskState, msg.id, &state), + Err(message) => Message::with_payload( + MessageType::CoreError, + msg.id, + &CoreError { + kind: CoreErrorKind::CapabilityUnavailable, + message, + offending_type: Some(msg.t.as_str().into()), + }, + ), + } + .map_err(|e| AgentdError::ExecSession(format!("encode root capacity: {e}")))?; + codec::encode_to_buf(&reply, out_buf).map_err(|e| { + AgentdError::ExecSession(format!("encode root capacity frame: {e}")) + })?; + } + MessageType::Ping => { let Some(_) = decode_payload_or_core_error::(&msg, out_buf)? else { return Ok(()); diff --git a/crates/agentd/lib/init.rs b/crates/agentd/lib/init.rs index dd7f3882a..497869eaa 100644 --- a/crates/agentd/lib/init.rs +++ b/crates/agentd/lib/init.rs @@ -266,6 +266,7 @@ mod linux { match spec { BlockRootSpec::DiskImage { device, fstype } => { mount_disk_image(device, fstype.as_deref())?; + crate::root_disk::register("/newroot", device); } BlockRootSpec::OciErofs { lower, upper } => { mount_oci_erofs(lower, upper)?; @@ -328,6 +329,7 @@ mod linux { None::<&str>, ) .map_err(|e| AgentdError::Init(format!("mount {device} at {upperfs_dir}: {e}")))?; + crate::root_disk::register(upperfs_dir, device); } BlockRootUpper::Tmpfs { size_mib } => { let data = size_mib diff --git a/crates/agentd/lib/lib.rs b/crates/agentd/lib/lib.rs index 817bb4dc2..50185740a 100644 --- a/crates/agentd/lib/lib.rs +++ b/crates/agentd/lib/lib.rs @@ -9,6 +9,7 @@ mod config; mod error; mod rlimit; +mod root_disk; mod workload; //-------------------------------------------------------------------------------------------------- diff --git a/crates/agentd/lib/root_disk.rs b/crates/agentd/lib/root_disk.rs new file mode 100644 index 000000000..ba16449cd --- /dev/null +++ b/crates/agentd/lib/root_disk.rs @@ -0,0 +1,185 @@ +//! Online ext4 growth using pinned init-time descriptors, never a caller-supplied path. + +use std::fs::File; +use std::io; +use std::os::fd::AsRawFd; +use std::os::unix::fs::FileExt; +use std::sync::OnceLock; +use std::time::{Duration, Instant}; + +use microsandbox_protocol::core::RootDiskState; + +//-------------------------------------------------------------------------------------------------- +// Constants +//-------------------------------------------------------------------------------------------------- + +const EXT4_IOC_RESIZE_FS: libc::c_ulong = 0x4008_6610; +const BLKGETSIZE64: libc::c_ulong = 0x8008_1272; +static ROOT: OnceLock> = OnceLock::new(); + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +struct RootFiles { + mount: File, + device: File, +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +pub(crate) fn register(mount: &str, device: &str) { + // The managed ext4 mount becomes unreachable by path after pivot. Pin it now, just as + // teardown does. An unavailable capability must not prevent ordinary sandbox startup. + let _ = ROOT.set((|| { + Ok(RootFiles { + mount: File::open(mount)?, + device: File::open(device)?, + }) + })()); +} + +pub(crate) fn resize(target: u64, apply: bool) -> Result { + let files = ROOT + .get() + .ok_or("root has no pinned block filesystem")? + .as_ref() + .map_err(|e| format!("root disk unavailable: {e}"))?; + let caps = std::fs::read_to_string("/proc/self/status").map_err(|e| e.to_string())?; + let admin = caps + .lines() + .find_map(|line| line.strip_prefix("CapEff:\t")) + .and_then(|value| u64::from_str_radix(value.trim(), 16).ok()) + .is_some_and(|value| value & (1 << 24) != 0); + if !admin { + return Err("online ext4 growth requires agent CAP_SYS_RESOURCE".into()); + } + let mut sb = [0u8; 1024]; + files + .device + .read_exact_at(&mut sb, 1024) + .map_err(|e| e.to_string())?; + let current = validate_superblock(&sb, target)?; + let mut device_bytes = device_size(&files.device)?; + if apply && target > current { + // Virtio config change is asynchronous in the guest. Never issue the filesystem ioctl + // until the block driver's own capacity reflects the host target. + let deadline = Instant::now() + Duration::from_secs(10); + while device_bytes < target { + if Instant::now() >= deadline { + return Err("guest block capacity did not converge".into()); + } + std::thread::sleep(Duration::from_millis(2)); + device_bytes = device_size(&files.device)?; + } + let blocks = target / 4096; + // SAFETY: ioctl reads one u64 from a valid reference; mount is a pinned open directory. + if unsafe { libc::ioctl(files.mount.as_raw_fd(), EXT4_IOC_RESIZE_FS as _, &blocks) } < 0 { + return Err(format!( + "online ext4 expansion: {}", + io::Error::last_os_error() + )); + } + // Flush the committed superblock before observing it through the block descriptor. + if unsafe { libc::syncfs(files.mount.as_raw_fd()) } < 0 { + return Err(format!("sync grown ext4: {}", io::Error::last_os_error())); + } + files + .device + .read_exact_at(&mut sb, 1024) + .map_err(|e| e.to_string())?; + } + let actual = validate_superblock(&sb, target)?; + if apply && actual != target { + return Err(format!( + "filesystem capacity is {actual}, expected {target}" + )); + } + Ok(RootDiskState { + filesystem_bytes: actual, + device_bytes, + }) +} + +fn device_size(file: &File) -> Result { + let mut size = 0u64; + // SAFETY: BLKGETSIZE64 writes exactly one u64 into the supplied live storage. + if unsafe { libc::ioctl(file.as_raw_fd(), BLKGETSIZE64 as _, &mut size) } < 0 { + return Err(format!( + "read guest block capacity: {}", + io::Error::last_os_error() + )); + } + Ok(size) +} + +fn validate_superblock(sb: &[u8; 1024], target: u64) -> Result { + let u16_at = |i| u16::from_le_bytes([sb[i], sb[i + 1]]); + let u32_at = |i| u32::from_le_bytes(sb[i..i + 4].try_into().unwrap()); + if u16_at(56) != 0xef53 || u32_at(24) != 2 || u32_at(32) != 32768 { + return Err("online growth requires a supported 4 KiB ext4 root".into()); + } + let current_blocks = u64::from(u32_at(4)) | (u64::from(u32_at(0x150)) << 32); + let current = current_blocks + .checked_mul(4096) + .ok_or("ext4 size overflow")?; + if target < current || !target.is_multiple_of(4096) || target == 0 { + return Err("root growth requires a nondecreasing size aligned to 4 KiB".into()); + } + if target == current { + return Ok(current); + } + if u32_at(92) & 0x10 == 0 || u32_at(96) & 0x10 != 0 { + return Err( + "root ext4 lacks supported resize-inode metadata; offline preparation is required" + .into(), + ); + } + let descriptor_size = u64::from(u16_at(254)).max(32); + let groups = current_blocks.div_ceil(32768); + let gdt_blocks = (groups * descriptor_size).div_ceil(4096); + let max_groups = (gdt_blocks + u64::from(u16_at(206))) * 4096 / descriptor_size; + if target / 4096 > max_groups * 32768 { + return Err("target exceeds ext4 reserved group-descriptor capacity".into()); + } + let target_blocks = target / 4096; + let last_group = (target_blocks - 1) / 32768; + if last_group >= groups && !target_blocks.is_multiple_of(32768) { + let sparse = last_group <= 1 + || [3u64, 5, 7].iter().any(|base| { + let mut group = last_group; + while group > 1 && group.is_multiple_of(*base) { + group /= base; + } + group == 1 + }); + let inode_blocks = (u64::from(u32_at(40)) * u64::from(u16_at(88))).div_ceil(4096); + let metadata = inode_blocks + + 2 + + if sparse { + 1 + gdt_blocks + u64::from(u16_at(206)) + } else { + 0 + }; + if target_blocks % 32768 < metadata { + return Err("target leaves a final block group too small for ext4 metadata".into()); + } + } + Ok(current) +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_unknown_filesystems_before_mutation() { + assert!(validate_superblock(&[0; 1024], 1024 * 1024 * 1024).is_err()); + } +} diff --git a/crates/cli/lib/commands/modify.rs b/crates/cli/lib/commands/modify.rs index e225ac1f2..ceeed993a 100644 --- a/crates/cli/lib/commands/modify.rs +++ b/crates/cli/lib/commands/modify.rs @@ -187,14 +187,13 @@ fn apply_resource_args( builder = builder.max_cpus(max_cpus); } if let Some(memory) = &args.memory { - builder = builder.memory_mib(ui::parse_size_mib(memory).map_err(anyhow::Error::msg)?); + builder = builder.memory(ui::parse_size_mib(memory).map_err(anyhow::Error::msg)?); } if let Some(max_memory) = &args.max_memory { - builder = - builder.max_memory_mib(ui::parse_size_mib(max_memory).map_err(anyhow::Error::msg)?); + builder = builder.max_memory(ui::parse_size_mib(max_memory).map_err(anyhow::Error::msg)?); } if let Some(size) = args.root_disk.as_ref().or(args.oci_upper_size.as_ref()) { - builder = builder.root_disk_size_mib(ui::parse_size_mib(size).map_err(anyhow::Error::msg)?); + builder = builder.root_disk_size(ui::parse_size_mib(size).map_err(anyhow::Error::msg)?); } Ok(builder) } diff --git a/crates/image/lib/checkpoint/compact.rs b/crates/image/lib/checkpoint/compact.rs index 105b4e817..f181ca504 100644 --- a/crates/image/lib/checkpoint/compact.rs +++ b/crates/image/lib/checkpoint/compact.rs @@ -42,6 +42,18 @@ pub struct CompactMaterialization { /// Open a complete, explicitly supplied immutable chain read-only. async fn open_chain(layers: &[CompactLayer]) -> io::Result { + open_chain_access(layers, false).await +} + +/// Resolve an owned chain with writes confined to its caller-private staging head. +pub(crate) async fn open_writable_chain(layers: &[CompactLayer]) -> io::Result { + open_chain_access(layers, true).await +} + +async fn open_chain_access( + layers: &[CompactLayer], + writable_head: bool, +) -> io::Result { if layers.is_empty() || layers.iter().skip(1).any(|layer| !layer.qcow2) { return Err(io::Error::new( io::ErrorKind::InvalidInput, @@ -49,12 +61,17 @@ async fn open_chain(layers: &[CompactLayer]) -> io::Result { )); } let mut backing: Option = None; - for layer in layers { - let storage: Box = - Box::new(ImagoFile::try_from(std::fs::File::open(&layer.path)?)?); + for (index, layer) in layers.iter().enumerate() { + let writable = writable_head && index + 1 == layers.len(); + let storage: Box = Box::new(ImagoFile::try_from( + std::fs::OpenOptions::new() + .read(true) + .write(writable) + .open(&layer.path)?, + )?); backing = Some(if layer.qcow2 { let image = Qcow2::, SharedImage>::builder(storage) - .write(false) + .write(writable) .backing(backing) .data_file(None) .open(DenyImplicitOpenGate::default()) @@ -69,7 +86,7 @@ async fn open_chain(layers: &[CompactLayer]) -> io::Result { } else { Arc::new(FormatAccess::new( Raw::>::builder(storage) - .write(false) + .write(writable) .open(DenyImplicitOpenGate::default()) .await?, )) @@ -155,6 +172,25 @@ pub async fn compact_layer_capacity(layer: CompactLayer) -> io::Result { Ok(open_chain(&[layer]).await?.size()) } +/// Read the capacities of a pinned closure from synchronous descriptor-building code. +/// A separate current-thread executor also permits use by callers already inside Tokio. +pub fn layer_capacities(layers: Vec) -> io::Result> { + std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()? + .block_on(async { + let mut capacities = Vec::with_capacity(layers.len()); + for layer in layers { + capacities.push(compact_layer_capacity(layer).await?); + } + Ok(capacities) + }) + }) + .join() + .map_err(|_| io::Error::other("layer-capacity worker panicked"))? +} + /// Validate a complete explicit chain without linking a VM runner or following header paths. /// /// The caller must prevent concurrent mutation until publication completes. diff --git a/crates/image/lib/checkpoint/mod.rs b/crates/image/lib/checkpoint/mod.rs index ef604b455..af2b79e47 100644 --- a/crates/image/lib/checkpoint/mod.rs +++ b/crates/image/lib/checkpoint/mod.rs @@ -14,9 +14,10 @@ mod store; // Re-Exports //-------------------------------------------------------------------------------------------------- +pub(crate) use compact::open_writable_chain; pub use compact::{ - CompactLayer, CompactMaterialization, compact_layer_capacity, materialize_compact_prefix, - validate_compact_chain, + CompactLayer, CompactMaterialization, compact_layer_capacity, layer_capacities, + materialize_compact_prefix, validate_compact_chain, }; pub use layer_selection::{DiskCompactionPlan, DiskLayerExportPlan, LayerSelectionError}; pub use manifest::{ diff --git a/crates/image/lib/ext4/chain.rs b/crates/image/lib/ext4/chain.rs new file mode 100644 index 000000000..2d59a2d79 --- /dev/null +++ b/crates/image/lib/ext4/chain.rs @@ -0,0 +1,161 @@ +//! Offline ext4 growth over a caller-resolved chain with a private staging head. + +use std::io::{self, Read, Seek, SeekFrom, Write}; +use std::sync::Arc; + +use imago::{DynStorage, FormatAccess}; + +use super::formatter::Ext4Error; +use super::resizer::{GrowOutcome, grow_storage}; +use super::storage::Ext4Storage; +use crate::checkpoint::{CompactLayer, open_writable_chain}; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +struct LogicalDisk { + image: Arc>>, + runtime: tokio::runtime::Runtime, + position: u64, +} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +impl Read for LogicalDisk { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + let count = + (self.image.size().saturating_sub(self.position)).min(buffer.len() as u64) as usize; + self.runtime + .block_on(self.image.read(&mut buffer[..count], self.position))?; + self.position += count as u64; + Ok(count) + } +} + +impl Write for LogicalDisk { + fn write(&mut self, buffer: &[u8]) -> io::Result { + self.runtime + .block_on(self.image.write(buffer, self.position))?; + self.position += buffer.len() as u64; + Ok(buffer.len()) + } + + fn flush(&mut self) -> io::Result<()> { + self.runtime.block_on(self.image.flush()) + } +} + +impl Seek for LogicalDisk { + fn seek(&mut self, position: SeekFrom) -> io::Result { + let target = match position { + SeekFrom::Start(value) => i128::from(value), + SeekFrom::Current(delta) => i128::from(self.position) + i128::from(delta), + SeekFrom::End(delta) => i128::from(self.image.size()) + i128::from(delta), + }; + self.position = u64::try_from(target).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "invalid logical disk seek") + })?; + Ok(self.position) + } +} + +impl Ext4Storage for LogicalDisk { + fn length(&self) -> io::Result { + Ok(self.image.size()) + } + + fn grow(&mut self, size: u64) -> io::Result<()> { + self.runtime.block_on( + self.image + .resize_grow(size, imago::format::PreallocateMode::None), + ) + } + + fn sync_all(&self) -> io::Result<()> { + self.runtime.block_on(async { + self.image.flush().await?; + self.image.sync().await + }) + } +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +/// Grow ext4 in a private staging head without flattening or modifying its sealed ancestors. +/// +/// Call outside an async executor with the complete oldest-to-newest chain. The last file must +/// be an unpublished private copy: the offline filesystem rewrite is not rollback-safe on error. +/// The caller publishes the head only after success and excludes all concurrent writers. +pub fn grow_chain(layers: &[CompactLayer], size: u64) -> Result { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + let image = runtime.block_on(open_writable_chain(layers))?; + let mut disk = LogicalDisk { + image, + runtime, + position: 0, + }; + grow_storage(&mut disk, size, true) +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use crate::checkpoint::{create_qcow2_overlay, layer_capacities, sparse_file_integrity}; + use crate::ext4::{Ext4FormatOptions, format_ext4}; + + #[test] + fn qcow2_growth_keeps_ancestors_and_reconciles_completed_target() { + let dir = tempfile::tempdir().unwrap(); + let base = dir.path().join("base.raw"); + let head = dir.path().join("head.qcow2"); + let mib = 1024 * 1024; + format_ext4( + &base, + &Ext4FormatOptions { + size_bytes: 256 * mib, + journal_blocks: 4096, + }, + ) + .unwrap(); + let before = sparse_file_integrity(&base).unwrap().root; + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + rt.block_on(create_qcow2_overlay(&head, 256 * mib, &base, "raw")) + .unwrap(); + let layers = vec![ + CompactLayer { + path: base.clone(), + qcow2: false, + }, + CompactLayer { + path: head, + qcow2: true, + }, + ]; + let grown = grow_chain(&layers, 512 * mib).unwrap(); + assert_eq!(grown.old_blocks * 4096, 256 * mib); + assert_eq!(grown.new_blocks * 4096, 512 * mib); + assert_eq!( + layer_capacities(layers.clone()).unwrap(), + vec![256 * mib, 512 * mib] + ); + assert_eq!(sparse_file_integrity(&base).unwrap().root, before); + let replay = grow_chain(&layers, 512 * mib).unwrap(); + assert_eq!(replay.old_blocks, replay.new_blocks); + assert!(grow_chain(&layers, 128 * mib).is_err()); + assert_eq!(sparse_file_integrity(&base).unwrap().root, before); + } +} diff --git a/crates/image/lib/ext4/format.rs b/crates/image/lib/ext4/format.rs index cbdc9a4b9..7bef58d26 100644 --- a/crates/image/lib/ext4/format.rs +++ b/crates/image/lib/ext4/format.rs @@ -46,6 +46,8 @@ pub const EXT4_FEATURE_RO_COMPAT_METADATA_CSUM: u32 = 0x400; // Group descriptor flags pub const EXT4_BG_INODE_ZEROED: u16 = 0x04; +pub const EXT4_BG_INODE_UNINIT: u16 = 0x01; +pub const EXT4_BG_BLOCK_UNINIT: u16 = 0x02; // jbd2 constants (big-endian on disk) pub const JBD2_MAGIC: u32 = 0xC03B3998; diff --git a/crates/image/lib/ext4/jbd2.rs b/crates/image/lib/ext4/jbd2.rs index 9c9b21562..446b90a65 100644 --- a/crates/image/lib/ext4/jbd2.rs +++ b/crates/image/lib/ext4/jbd2.rs @@ -7,8 +7,9 @@ //! so a journal this module rejects leaves the image untouched. use std::collections::HashMap; -use std::fs::File; -use std::io::{Read, Seek, SeekFrom, Write}; +use std::io::SeekFrom; + +use super::storage::Ext4Storage; use super::format::{ EXT4_BLOCK_SIZE, EXT4_EH_MAGIC, EXT4_EXTENTS_FL, EXT4_INODE_SIZE, EXT4_JOURNAL_INO, JBD2_MAGIC, @@ -101,7 +102,7 @@ struct DescriptorTag { /// Locate the journal via inode 8, trusting nothing: the inode checksum, file type, extent flag, and the formatter's single depth-0 extent shape are all verified. The journal /// inode is never itself journaled, so the on-disk copy is authoritative even on a dirty image. pub(super) fn locate_journal( - file: &mut File, + file: &mut impl Ext4Storage, inode_table_block: u64, csum_seed: u32, ) -> Result { @@ -159,7 +160,7 @@ pub(super) fn locate_journal( /// [`Ext4Error::Unsupported`] with the image untouched. A journal with `s_start == 0` needs no recovery and is not written at all. After replay the data is fsynced, then the /// jbd2 superblock is rewritten with `s_start = 0` and `s_sequence` advanced past every replayed transaction so stale commit blocks can never match again. pub(super) fn recover_journal( - file: &mut File, + file: &mut impl Ext4Storage, loc: &JournalLocation, fs_uuid: &[u8; 16], fs_num_blocks: u64, @@ -219,7 +220,7 @@ pub(super) fn recover_journal( /// tag, commit) simply terminates the walk — the partially written transaction was never committed, so ignoring it is the correct crash semantics. Only impossible states /// (targets out of bounds, malformed tags or revoke counts) are hard errors. fn scan_log( - file: &mut File, + file: &mut impl Ext4Storage, loc: &JournalLocation, jsb: &JournalSuperblock, jseed: u32, @@ -315,7 +316,7 @@ fn scan_log( /// Read and strictly validate the jbd2 superblock: magic, v2 block type, checksum, feature masks (exactly the formatter's), crc32c checksum type, matching filesystem UUID, no /// recorded error, 4 KiB block size, and geometry fields consistent with the journal extent. fn read_journal_superblock( - file: &mut File, + file: &mut impl Ext4Storage, loc: &JournalLocation, fs_uuid: &[u8; 16], ) -> Result { @@ -471,7 +472,7 @@ fn tag_checksum_ok(data: &[u8], jseed: u32, seq: u32, expected: u32) -> bool { //-------------------------------------------------------------------------------------------------- fn read_log_block( - file: &mut File, + file: &mut impl Ext4Storage, loc: &JournalLocation, index: u32, ) -> Result, Ext4Error> { @@ -508,7 +509,7 @@ pub(super) struct TestTransaction { /// journal superblock) so replay fixtures exercise the same on-disk format the recovery code parses. Data blocks beginning with the jbd2 magic are escaped automatically. #[cfg(test)] pub(super) fn write_test_log( - file: &mut File, + file: &mut impl Ext4Storage, loc: &JournalLocation, fs_uuid: &[u8; 16], start_seq: u32, @@ -516,7 +517,12 @@ pub(super) fn write_test_log( ) -> Result<(), Ext4Error> { let jseed = crc32c::crc32c_raw(0xFFFF_FFFF, fs_uuid); let mut cursor = 1u32; - let emit = |file: &mut File, cursor: &mut u32, block: &[u8]| -> Result<(), Ext4Error> { + fn emit( + file: &mut impl Ext4Storage, + cursor: &mut u32, + block: &[u8], + loc: &JournalLocation, + ) -> Result<(), Ext4Error> { assert!( *cursor < loc.len_blocks, "test log fixture overflows the journal" @@ -527,7 +533,7 @@ pub(super) fn write_test_log( file.write_all(block)?; *cursor += 1; Ok(()) - }; + } for (index, txn) in transactions.iter().enumerate() { let seq = start_seq + index as u32; @@ -545,7 +551,7 @@ pub(super) fn write_test_log( off += 8; } set_tail_checksum(&mut block, jseed); - emit(file, &mut cursor, &block)?; + emit(file, &mut cursor, &block, loc)?; } if !txn.writes.is_empty() { @@ -583,9 +589,9 @@ pub(super) fn write_test_log( stored.push(data); } set_tail_checksum(&mut desc, jseed); - emit(file, &mut cursor, &desc)?; + emit(file, &mut cursor, &desc, loc)?; for data in &stored { - emit(file, &mut cursor, data)?; + emit(file, &mut cursor, data, loc)?; } } @@ -603,7 +609,7 @@ pub(super) fn write_test_log( checksum }, ); - emit(file, &mut cursor, &commit)?; + emit(file, &mut cursor, &commit, loc)?; } let mut raw = vec![0u8; JBD2_SB_SIZE]; diff --git a/crates/image/lib/ext4/mod.rs b/crates/image/lib/ext4/mod.rs index ab98b0ee1..878ee26cc 100644 --- a/crates/image/lib/ext4/mod.rs +++ b/crates/image/lib/ext4/mod.rs @@ -1,3 +1,4 @@ +mod chain; mod format; mod formatter; mod jbd2; @@ -5,11 +6,13 @@ mod layout; mod resize_inode; mod resizer; mod rootfs; +mod storage; //-------------------------------------------------------------------------------------------------- // Re-Exports //-------------------------------------------------------------------------------------------------- +pub use chain::grow_chain; pub use formatter::{Ext4Error, Ext4FormatOptions, format_ext4, format_ext4_with_tree}; pub use resizer::{GrowOutcome, grow_image}; pub use rootfs::{ diff --git a/crates/image/lib/ext4/resize_inode.rs b/crates/image/lib/ext4/resize_inode.rs index 1a631359c..00b4f4ecf 100644 --- a/crates/image/lib/ext4/resize_inode.rs +++ b/crates/image/lib/ext4/resize_inode.rs @@ -4,7 +4,6 @@ //! kernel and e2fsprogs. It owns the primary reserved-GDT blocks and their sparse-super backup //! copies; these blocks are filesystem metadata even though ext4 accounts for them through inode 7. -use std::fs::File; use std::io::{Read, Seek, SeekFrom, Write}; use super::format::{ @@ -68,7 +67,7 @@ pub(super) fn write_resize_inode( /// Validate inode 7 and every reserved-GDT pointer against the current filesystem geometry. pub(super) fn validate_resize_inode( - file: &mut File, + file: &mut (impl Read + Seek), geometry: &GroupGeometry, inode_table_block: u64, csum_seed: u32, diff --git a/crates/image/lib/ext4/resizer.rs b/crates/image/lib/ext4/resizer.rs index 352dfa10a..50cb42287 100644 --- a/crates/image/lib/ext4/resizer.rs +++ b/crates/image/lib/ext4/resizer.rs @@ -9,9 +9,12 @@ //! module) and then grown as clean images. use std::fs::{File, OpenOptions}; -use std::io::{Read, Seek, SeekFrom, Write}; +use std::io::SeekFrom; +#[cfg(test)] +use std::io::{Read, Seek, Write}; use std::path::Path; +use super::format::{EXT4_BG_BLOCK_UNINIT, EXT4_BG_INODE_UNINIT}; use super::format::{ EXT4_BG_INODE_ZEROED, EXT4_BLOCK_SIZE, EXT4_BLOCKS_PER_GROUP, EXT4_DESC_SIZE, EXT4_EH_MAGIC, EXT4_EXTENTS_FL, EXT4_FEATURE_COMPAT_DIR_INDEX, EXT4_FEATURE_COMPAT_EXT_ATTR, @@ -24,7 +27,7 @@ use super::format::{ EXT4_ROOT_INO, EXT4_SB_ERROR_COUNT_OFFSET, EXT4_SB_OVERHEAD_BLOCKS_OFFSET, EXT4_SUPER_MAGIC, S_IFDIR, sparse_super_group, }; -use super::formatter::{Ext4Error, mark_sparse}; +use super::formatter::Ext4Error; use super::jbd2; use super::layout::{ GroupDescStats, GroupGeometry, MAX_BLOCKS, bitmap_checksum, build_block_bitmap_base, @@ -33,6 +36,7 @@ use super::layout::{ write_backup_superblock_at, write_gdt_at, }; use super::resize_inode::{validate_resize_inode, write_resize_inode}; +use super::storage::Ext4Storage; use crate::crc32c; //-------------------------------------------------------------------------------------------------- @@ -176,6 +180,15 @@ impl ParsedImage { /// process is interrupted, callers must discard and recreate the artifact rather than use it. pub fn grow_image(path: &Path, new_size_bytes: u64) -> Result { let mut file = OpenOptions::new().read(true).write(true).open(path)?; + grow_storage(&mut file, new_size_bytes, false) +} + +/// Grow a caller-owned staging disk; errors must never publish the staging artifact. +pub(crate) fn grow_storage( + mut file: &mut impl Ext4Storage, + new_size_bytes: u64, + allow_completed_target: bool, +) -> Result { let mut img = parse_and_validate(&mut file)?; // Replay the journal before anything else: growing with a pending log would let the next kernel mount replay stale transactions over the appended GDT entries. After a @@ -201,6 +214,16 @@ pub fn grow_image(path: &Path, new_size_bytes: u64) -> Result Result Result> = None; if new_last_blocks > old_last_blocks { - let mut bitmap = read_block_at(&mut file, old_geo.group_block_bitmap_block(old_last))?; + let off = old_last as usize * EXT4_DESC_SIZE as usize; + let flags = get_le16(&gdt[off..], 0x12); + // Online ext4 growth may leave a lazy block bitmap. Its bytes are not meaningful + // until initialized, so derive the metadata-only bitmap rather than reading stale data. + let mut bitmap = if flags & EXT4_BG_BLOCK_UNINIT != 0 { + let expected_free = old_last_blocks - old_geo.group_metadata_blocks(old_last); + let recorded_free = u32::from(get_le16(&gdt[off..], 0x0C)) + | (u32::from(get_le16(&gdt[off..], 0x2C)) << 16); + if recorded_free != expected_free { + return Err(unsupported("uninitialized group has allocated data")); + } + build_block_bitmap_base(&old_geo, old_last) + } else { + read_block_at(&mut file, old_geo.group_block_bitmap_block(old_last))? + }; for bit in old_last_blocks..new_last_blocks { bitmap[(bit / 8) as usize] &= !(1 << (bit % 8)); } @@ -267,6 +303,7 @@ pub fn grow_image(path: &Path, new_size_bytes: u64) -> Result Result<(), Ext4Error> { /// Parse the primary superblock and GDT, refusing anything that does not match exactly what this /// crate's formatter writes (geometry, feature masks, per-group layout, checksums). -fn parse_and_validate(file: &mut File) -> Result { - let file_len = file.metadata()?.len(); +fn parse_and_validate(file: &mut impl Ext4Storage) -> Result { + let file_len = file.length()?; if file_len < SB_OFFSET + SB_SIZE as u64 { return Err(unsupported("file too small to contain an ext4 superblock")); } @@ -661,7 +698,12 @@ fn parse_and_validate(file: &mut File) -> Result { "group {group} metadata is not at the expected location" ))); } - if get_le16(desc, 0x12) != EXT4_BG_INODE_ZEROED { + let flags = get_le16(desc, 0x12); + let known_flags = EXT4_BG_INODE_ZEROED | EXT4_BG_INODE_UNINIT | EXT4_BG_BLOCK_UNINIT; + if flags & !known_flags != 0 + || (matches!(img.resize_metadata, ResizeMetadata::Legacy) + && flags != EXT4_BG_INODE_ZEROED) + { return Err(unsupported(format!("group {group} has unexpected flags"))); } let mut desc_copy = desc.to_vec(); @@ -705,7 +747,10 @@ fn parse_and_validate(file: &mut File) -> Result { /// zero inode 7, place the root directory immediately after the inode table, /// and leave every still-reserved primary GDT block sparse-zeroed. These /// invariants remain true after any number of grows by the legacy resizer. -fn validate_legacy_resize_metadata(file: &mut File, img: &ParsedImage) -> Result<(), Ext4Error> { +fn validate_legacy_resize_metadata( + file: &mut impl Ext4Storage, + img: &ParsedImage, +) -> Result<(), Ext4Error> { let geometry = img.geometry(); let resize_inode = read_inode(file, &geometry, EXT4_RESIZE_INO)?; if resize_inode.iter().any(|byte| *byte != 0) { @@ -772,7 +817,7 @@ fn validate_legacy_resize_metadata(file: &mut File, img: &ParsedImage) -> Result /// index path preserves that stable fingerprint while accepting extent fanout created by normal /// guest writes. fn first_extent_physical_block( - file: &mut File, + file: &mut impl Ext4Storage, img: &ParsedImage, inode_number: u32, inode: &[u8], @@ -854,7 +899,7 @@ fn first_extent_physical_block( } fn read_inode( - file: &mut File, + file: &mut impl Ext4Storage, geometry: &GroupGeometry, inode_number: u32, ) -> Result, Ext4Error> { @@ -873,7 +918,10 @@ fn read_inode( /// The journal is fully validated before its first write (see [`jbd2::recover_journal`]) and the backup superblocks are validated up front too, so an inconsistent image is /// refused untouched. The write ordering is crash-safe: replayed blocks are fsynced, then the jbd2 superblock is reset to empty, then RECOVER is cleared — a tear at any point /// leaves an image that the next attempt recovers to the same end state (replaying an already-emptied journal is a no-op). -fn replay_journal_and_clear_recover(file: &mut File, img: &ParsedImage) -> Result<(), Ext4Error> { +fn replay_journal_and_clear_recover( + file: &mut impl Ext4Storage, + img: &ParsedImage, +) -> Result<(), Ext4Error> { let geo = img.geometry(); let journal = jbd2::locate_journal(file, geo.group_inode_table_block(0), img.csum_seed)?; if journal.start_block + journal.len_blocks as u64 > img.num_blocks { @@ -918,7 +966,11 @@ fn replay_journal_and_clear_recover(file: &mut File, img: &ParsedImage) -> Resul } /// Read a 1024-byte superblock at `offset`, refusing bad magic or checksum. -fn read_superblock_at(file: &mut File, offset: u64, label: &str) -> Result, Ext4Error> { +fn read_superblock_at( + file: &mut impl Ext4Storage, + offset: u64, + label: &str, +) -> Result, Ext4Error> { let mut sb = vec![0u8; SB_SIZE]; file.seek(SeekFrom::Start(offset))?; file.read_exact(&mut sb)?; @@ -1007,7 +1059,7 @@ fn validate_group_bitmaps( } fn validate_allocated_inode( - file: &mut File, + file: &mut impl Ext4Storage, img: &ParsedImage, group: u32, local_inode: u32, @@ -1048,7 +1100,7 @@ fn validate_allocated_inode( } fn validate_inode_extent_tree( - file: &mut File, + file: &mut impl Ext4Storage, img: &ParsedImage, inode_number: u32, inode: &[u8], @@ -1144,7 +1196,7 @@ fn validate_extent_entries( } fn validate_external_xattrs( - file: &mut File, + file: &mut impl Ext4Storage, img: &ParsedImage, block_number: u64, inode_number: u32, @@ -1208,7 +1260,10 @@ fn validate_xattr_entries( Ok(()) } -fn validate_backup_metadata(file: &mut File, img: &ParsedImage) -> Result<(), Ext4Error> { +fn validate_backup_metadata( + file: &mut impl Ext4Storage, + img: &ParsedImage, +) -> Result<(), Ext4Error> { let geometry = img.geometry(); for group in 1..img.num_groups { if !sparse_super_group(group) { @@ -1231,14 +1286,14 @@ fn validate_backup_metadata(file: &mut File, img: &ParsedImage) -> Result<(), Ex Ok(()) } -fn read_block_at(file: &mut File, block: u64) -> Result, Ext4Error> { +fn read_block_at(file: &mut impl Ext4Storage, block: u64) -> Result, Ext4Error> { let mut buf = vec![0u8; EXT4_BLOCK_SIZE as usize]; file.seek(SeekFrom::Start(block * EXT4_BLOCK_SIZE as u64))?; file.read_exact(&mut buf)?; Ok(buf) } -fn write_block_at(file: &mut File, block: u64, data: &[u8]) -> Result<(), Ext4Error> { +fn write_block_at(file: &mut impl Ext4Storage, block: u64, data: &[u8]) -> Result<(), Ext4Error> { file.seek(SeekFrom::Start(block * EXT4_BLOCK_SIZE as u64))?; file.write_all(data)?; Ok(()) @@ -1278,6 +1333,49 @@ mod tests { format_ext4_legacy_for_test(path, &opts).unwrap(); } + #[test] + fn grow_initializes_a_lazy_partial_group_bitmap_before_extending_it() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("lazy.ext4"); + format_image(&path, 300 * MIB); + let mut file = OpenOptions::new() + .read(true) + .write(true) + .open(&path) + .unwrap(); + let image = parse_and_validate(&mut file).unwrap(); + let last = image.num_groups - 1; + let offset = last as usize * EXT4_DESC_SIZE as usize; + let mut descriptor = image.gdt[offset..offset + EXT4_DESC_SIZE as usize].to_vec(); + put_le16( + &mut descriptor, + 0x12, + EXT4_BG_INODE_ZEROED | EXT4_BG_BLOCK_UNINIT, + ); + put_le16(&mut descriptor, 0x1E, 0); + let checksum = gdt_checksum(image.csum_seed, last, &descriptor); + put_le16(&mut descriptor, 0x1E, checksum); + file.seek(SeekFrom::Start(4096 + offset as u64)).unwrap(); + file.write_all(&descriptor).unwrap(); + // Bytes of an uninitialized bitmap are deliberately meaningless. + write_block_at( + &mut file, + image.geometry().group_block_bitmap_block(last), + &[0xa5; 4096], + ) + .unwrap(); + file.sync_all().unwrap(); + drop(file); + grow_image(&path, 384 * MIB).unwrap(); + let mut file = File::open(&path).unwrap(); + let grown = parse_and_validate(&mut file).unwrap(); + let desc = &grown.gdt[offset..offset + EXT4_DESC_SIZE as usize]; + assert_eq!(get_le16(desc, 0x12), EXT4_BG_INODE_ZEROED); + let bitmap = + read_block_at(&mut file, grown.geometry().group_block_bitmap_block(last)).unwrap(); + assert_eq!(bitmap, build_block_bitmap_base(&grown.geometry(), last)); + } + /// Reproduce a root directory that has outgrown the inode's inline extent leaf. /// /// The kernel normally creates this shape after enough directory churn. Building the same diff --git a/crates/image/lib/ext4/storage.rs b/crates/image/lib/ext4/storage.rs new file mode 100644 index 000000000..7328ca432 --- /dev/null +++ b/crates/image/lib/ext4/storage.rs @@ -0,0 +1,46 @@ +//! Logical disk I/O shared by raw-file and explicitly resolved qcow2 offline growth. + +use std::fs::File; +use std::io::{self, Read, Seek, Write}; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +/// Offline filesystem storage. Callers must exclude all other writers. +pub(crate) trait Ext4Storage: Read + Write + Seek { + fn length(&self) -> io::Result; + fn grow(&mut self, size: u64) -> io::Result<()>; + fn sync_all(&self) -> io::Result<()>; +} + +//-------------------------------------------------------------------------------------------------- +// Trait Implementations +//-------------------------------------------------------------------------------------------------- + +impl Ext4Storage for File { + fn length(&self) -> io::Result { + Ok(self.metadata()?.len()) + } + + fn grow(&mut self, size: u64) -> io::Result<()> { + super::formatter::mark_sparse(self).map_err(io::Error::other)?; + self.set_len(size) + } + + fn sync_all(&self) -> io::Result<()> { + File::sync_all(self) + } +} + +impl Ext4Storage for &mut T { + fn length(&self) -> io::Result { + (**self).length() + } + fn grow(&mut self, size: u64) -> io::Result<()> { + (**self).grow(size) + } + fn sync_all(&self) -> io::Result<()> { + (**self).sync_all() + } +} diff --git a/crates/protocol/lib/core.rs b/crates/protocol/lib/core.rs index 3ca1e5014..3dce49f7b 100644 --- a/crates/protocol/lib/core.rs +++ b/crates/protocol/lib/core.rs @@ -134,6 +134,22 @@ pub struct WorkloadThawed { pub attempt_id: String, } +/// Root disk growth target, in bytes, used for preflight and apply. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RootDiskGrow { + /// Desired ext4 size; must be an aligned, nondecreasing target. + pub size_bytes: u64, +} + +/// Observed capacities of the guest root filesystem and its block device. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RootDiskState { + /// ext4 superblock size, including filesystem metadata. + pub filesystem_bytes: u64, + /// Capacity observed by the guest block driver. + pub device_bytes: u64, +} + /// Payload for `core.error` messages. /// /// Sent when a peer can identify a recoverable protocol error for a specific diff --git a/crates/protocol/lib/message.rs b/crates/protocol/lib/message.rs index 999a991ea..f2b867bc1 100644 --- a/crates/protocol/lib/message.rs +++ b/crates/protocol/lib/message.rs @@ -153,6 +153,18 @@ pub enum MessageType { #[strum(serialize = "core.workload.thawed")] WorkloadThawed, + /// Host checks mounted root-filesystem growth before changing block capacity. + #[strum(serialize = "core.root_disk.prepare")] + RootDiskPrepare, + + /// Host requests mounted root-filesystem expansion after block capacity changed. + #[strum(serialize = "core.root_disk.grow")] + RootDiskGrow, + + /// Guest reports the root filesystem and device capacities. + #[strum(serialize = "core.root_disk.state")] + RootDiskState, + /// Peer reports a recoverable protocol-level error. #[strum(serialize = "core.error")] CoreError, @@ -312,6 +324,7 @@ impl MessageType { | Self::Touched | Self::WorkloadFrozen | Self::WorkloadThawed + | Self::RootDiskState | Self::CoreError | Self::ExecExited | Self::ExecFailed @@ -370,6 +383,7 @@ impl MessageType { | Self::WorkloadFrozen | Self::WorkloadThaw | Self::WorkloadThawed => 9, + Self::RootDiskPrepare | Self::RootDiskGrow | Self::RootDiskState => 9, Self::BulkAccepted | Self::BulkCredit | Self::BulkFinish | Self::BulkCancel => 8, Self::TcpConnect | Self::TcpConnected diff --git a/crates/protocol/schema/gen-9.json b/crates/protocol/schema/gen-9.json index 93dea7094..ff254753d 100644 --- a/crates/protocol/schema/gen-9.json +++ b/crates/protocol/schema/gen-9.json @@ -74,6 +74,18 @@ "introduced_in": 9, "wire": "core.workload.thawed" }, + { + "introduced_in": 9, + "wire": "core.root_disk.prepare" + }, + { + "introduced_in": 9, + "wire": "core.root_disk.grow" + }, + { + "introduced_in": 9, + "wire": "core.root_disk.state" + }, { "introduced_in": 5, "wire": "core.error" diff --git a/crates/runtime/lib/checkpoint/coordinator.rs b/crates/runtime/lib/checkpoint/coordinator.rs index 94e82d161..ea58e49bb 100644 --- a/crates/runtime/lib/checkpoint/coordinator.rs +++ b/crates/runtime/lib/checkpoint/coordinator.rs @@ -137,6 +137,15 @@ impl CheckpointCoordinator { layers: Option, dry_run: bool, ) -> Result { + if self + .root_disk + .as_ref() + .is_some_and(|disk| disk.growth_pending()) + { + return Err(super::disk::RootDiskRolloverError::pre_rebind( + "complete pending root-disk growth before compaction", + )); + } match self.root_disk.as_mut() { Some(disk) => disk.compact(Some(vm), &self.runtime, layers, dry_run), None => Err(super::disk::RootDiskRolloverError::pre_rebind( @@ -145,6 +154,74 @@ impl CheckpointCoordinator { } } + pub(crate) fn grow_root( + &mut self, + vm: &msb_krun::VmControl, + size_bytes: u64, + ) -> Result { + use super::disk::RootDiskRolloverError as Failure; + let started = Instant::now(); + let disk = self + .root_disk + .as_mut() + .ok_or_else(|| Failure::pre_rebind("root is not runtime-owned"))?; + let client = self + .runtime + .block_on(AgentClient::connect_with_timeout( + &self.agent_sock, + WORKLOAD_CONTROL_TIMEOUT, + )) + .map_err(Failure::pre_rebind)?; + // Gate the protocol and validate ext4 before mutating either disk or recovery state. + let request = microsandbox_protocol::core::RootDiskGrow { size_bytes }; + self.runtime + .block_on(root_growth_request( + &client, + MessageType::RootDiskPrepare, + &request, + )) + .map_err(Failure::pre_rebind)?; + disk.begin_growth(size_bytes).map_err(Failure::pre_rebind)?; + let paused_at = Instant::now(); + let pause = vm.pause().map_err(Failure::pre_rebind)?; + if let Err(error) = vm.grow_block_capacity(disk.device_id(), size_bytes) { + // The image may already be larger. Fence execution until restart reopens actual + // capacity; retrying this target then completes the filesystem phase. + return Err(Failure::post_journal(format!( + "root block growth requires forward recovery: {error}" + ))); + } + vm.resume(pause).map_err(Failure::post_journal)?; + let pause_us = paused_at.elapsed().as_micros() as u64; + let guest_started = Instant::now(); + let state = self + .runtime + .block_on(root_growth_request( + &client, + MessageType::RootDiskGrow, + &request, + )) + .map_err(|e| { + Failure::pre_rebind(format!( + "block device grew; filesystem completion is pending, retry this target: {e}" + )) + })?; + if state.filesystem_bytes != size_bytes || state.device_bytes < size_bytes { + return Err(Failure::pre_rebind( + "guest did not acknowledge the requested root capacity; retry to finish growth", + )); + } + let guest_us = guest_started.elapsed().as_micros() as u64; + disk.finish_growth().map_err(Failure::pre_rebind)?; + Ok(crate::control::RootDiskGrowthResult { + filesystem_bytes: state.filesystem_bytes, + device_bytes: state.device_bytes, + total_us: started.elapsed().as_micros() as u64, + pause_us, + guest_us, + }) + } + /// Open the per-runtime object store and managed root-disk state. pub(crate) fn open( runtime_dir: &Path, @@ -186,6 +263,15 @@ impl CheckpointCoordinator { checkpoint_id: &str, intent: CaptureIntent, ) -> Result { + if self + .root_disk + .as_ref() + .is_some_and(|disk| disk.growth_pending()) + { + return Err(CheckpointFailure::before_pause( + "complete pending root-disk growth before checkpointing", + )); + } let total_started = Instant::now(); validate_checkpoint_id(checkpoint_id).map_err(CheckpointFailure::before_pause)?; validate_vm_generation_state(vm.vm_generation_state()) @@ -853,6 +939,30 @@ impl MemoryCaptureSink for MemoryObjectSink<'_> { // Functions //-------------------------------------------------------------------------------------------------- +async fn root_growth_request( + client: &AgentClient, + message_type: MessageType, + request: µsandbox_protocol::core::RootDiskGrow, +) -> Result { + let reply = tokio::time::timeout( + Duration::from_secs(120), + client.request(message_type, request), + ) + .await + .map_err(|_| "guest root growth timed out".to_string())? + .map_err(|e| e.to_string())?; + if reply.t == MessageType::CoreError { + return Err(reply + .payload::() + .map_err(|e| e.to_string())? + .message); + } + if reply.t != MessageType::RootDiskState { + return Err("unexpected root growth response".into()); + } + reply.payload().map_err(|e| e.to_string()) +} + fn validate_workload_reply( reply: Message, expected_type: MessageType, diff --git a/crates/runtime/lib/checkpoint/disk.rs b/crates/runtime/lib/checkpoint/disk.rs index 542a98b60..5df994e5b 100644 --- a/crates/runtime/lib/checkpoint/disk.rs +++ b/crates/runtime/lib/checkpoint/disk.rs @@ -47,7 +47,7 @@ pub(crate) struct RuntimeOwnedRootDisk { pub struct RuntimeOwnedRootChain { /// Guest-visible block device backed by this chain. pub device_id: String, - /// Guest-visible capacity shared by every layer in the chain. + /// Guest-visible capacity of the writable head; sealed ancestors may be smaller. pub virtual_size: u64, /// Complete oldest-to-head physical closure. pub layers: Vec, @@ -89,6 +89,10 @@ struct RootDiskState { /// Original launch configuration binding, retained across representation-only compaction. #[serde(default, skip_serializing_if = "Option::is_none")] launch_base: Option, + /// Unfinished forward-only growth. Older runtime readers refuse this field rather than + /// accepting a chain whose filesystem expansion has not been acknowledged. + #[serde(default, skip_serializing_if = "Option::is_none")] + growth_target: Option, layers: Vec, } @@ -121,6 +125,60 @@ struct RootDiskLayer { //-------------------------------------------------------------------------------------------------- impl RuntimeOwnedRootDisk { + pub(crate) fn growth_pending(&self) -> bool { + self.state.growth_target.is_some() + } + + pub(crate) fn begin_growth(&mut self, target: u64) -> Result<(), String> { + let capacities = microsandbox_image::checkpoint::layer_capacities( + self.state + .layers + .iter() + .map(|layer| CompactLayer { + path: layer.path.clone(), + qcow2: layer.format == RootDiskFormat::Qcow2, + }) + .collect(), + ) + .map_err(|e| e.to_string())?; + let current = *capacities.last().expect("validated nonempty chain"); + if target < current + || target == 0 + || !target.is_multiple_of(4096) + || capacities.iter().any(|size| *size > current) + { + return Err("root growth requires an aligned nondecreasing capacity with no larger backing ancestor".into()); + } + if self + .state + .growth_target + .is_some_and(|pending| pending != target) + { + return Err( + "complete the pending root-disk growth target before requesting another size" + .into(), + ); + } + let mut next = self.state.clone(); + next.growth_target = Some(target); + // A failed earlier capture may have cached a head hash. Growth changes its bytes; + // no future snapshot may reuse that cached identity after mutation starts. + if let Some(head) = next.layers.last_mut() { + head.integrity_root = None; + } + write_state(&self.state_path, &next)?; + self.state = next; + Ok(()) + } + + pub(crate) fn finish_growth(&mut self) -> Result<(), String> { + let mut next = self.state.clone(); + next.growth_target = None; + write_state(&self.state_path, &next)?; + self.state = next; + Ok(()) + } + /// Open the authoritative chain journal or initialize it from a sandbox-owned root disk. #[cfg(feature = "runner")] pub(crate) fn open(runtime_dir: &Path, vm: &VmConfig) -> Result, String> { @@ -142,6 +200,7 @@ impl RuntimeOwnedRootDisk { layout, published_generation: 0, launch_base: None, + growth_target: None, layers: layers .into_iter() .map(|layer| RootDiskLayer { @@ -181,6 +240,11 @@ impl RuntimeOwnedRootDisk { layers: Option, dry_run: bool, ) -> Result { + if self.growth_pending() { + return Err(RootDiskRolloverError::pre_rebind( + "complete pending root-disk growth before compaction", + )); + } let started = Instant::now(); let plan = DiskCompactionPlan::new(self.state.layers.len(), layers) .map_err(RootDiskRolloverError::pre_rebind)?; @@ -370,6 +434,17 @@ impl RuntimeOwnedRootDisk { .published_generation .checked_add(1) .ok_or_else(|| RootDiskRolloverError::pre_rebind("disk generation is exhausted"))?; + let capacities = microsandbox_image::checkpoint::layer_capacities( + self.state + .layers + .iter() + .map(|layer| microsandbox_image::checkpoint::CompactLayer { + path: layer.path.clone(), + qcow2: layer.format == RootDiskFormat::Qcow2, + }) + .collect(), + ) + .map_err(RootDiskRolloverError::pre_rebind)?; let sealed_layers = self .state .layers @@ -378,7 +453,7 @@ impl RuntimeOwnedRootDisk { .map(|(index, layer)| DiskLayerRef { layer_id: layer.layer_id.clone(), format: layer.format.as_str().into(), - virtual_size, + virtual_size: capacities[index], predecessor: index .checked_sub(1) .map(|previous| self.state.layers[previous].layer_id.clone()), @@ -449,6 +524,9 @@ impl RootDiskState { || self.device_id != self.layout.device_id() || self.layers.is_empty() || self.layers.len() > 256 + || self + .growth_target + .is_some_and(|target| target == 0 || !target.is_multiple_of(4096)) { return Err("runtime-owned root-disk state has invalid identity or bounds".into()); } @@ -546,7 +624,7 @@ impl RootDiskRolloverError { } } - fn post_journal(error: impl fmt::Display) -> Self { + pub(crate) fn post_journal(error: impl fmt::Display) -> Self { Self { message: error.to_string(), #[cfg(feature = "runner")] @@ -616,6 +694,9 @@ pub fn load_runtime_owned_root_chain( return Ok(None); } let state = read_state(&state_path)?; + if state.growth_target.is_some() { + return Err("complete pending root-disk growth before snapshotting".into()); + } let head = state .layers .last() @@ -652,6 +733,94 @@ pub fn load_runtime_owned_root_chain( })) } +/// Grow a stopped journal-backed root using a private staging head. Returns false without a journal. +/// The caller must hold the sandbox lifecycle lock and prove all VM writers have stopped. +pub fn grow_stopped_root(runtime_dir: &Path, target: u64) -> Result { + let state_path = runtime_dir.join(ROOT_DISK_STATE_FILE); + if !state_path.exists() { + return Ok(false); + } + let state = read_state(&state_path)?; + if state.growth_target.is_some_and(|pending| pending != target) { + return Err(format!( + "complete pending root-disk growth to {} bytes first", + state.growth_target.unwrap() + )); + } + let capacities = microsandbox_image::checkpoint::layer_capacities( + state + .layers + .iter() + .map(|layer| CompactLayer { + path: layer.path.clone(), + qcow2: layer.format == RootDiskFormat::Qcow2, + }) + .collect(), + ) + .map_err(|e| e.to_string())?; + let current = *capacities.last().expect("validated nonempty chain"); + if target < current && state.growth_target.is_none() { + return Ok(true); + } + if target == current && state.growth_target.is_none() { + return Ok(true); + } + if capacities.iter().any(|size| *size > current) { + return Err("cannot grow a chain with a backing layer larger than its head".into()); + } + let stage = tempfile::Builder::new() + .prefix(".grow-") + .tempdir_in(runtime_dir) + .map_err(|e| e.to_string())?; + let mut next = state.clone(); + if next.launch_base.is_none() { + next.launch_base = state.layers.first().map(|layer| layer.path.clone()); + } + let last = state.layers.len() - 1; + for (index, layer) in next.layers.iter_mut().enumerate() { + let path = stage + .path() + .join(layer.path.file_name().ok_or("invalid root layer name")?); + if index == last { + microsandbox_utils::copy::fast_copy(&layer.path, &path).map_err(|e| e.to_string())?; + layer.layer_id = new_id("layer"); + layer.integrity_root = None; + } else { + // Preserve qcow2's relative backing bindings for independent inspection as well as + // the runtime's explicit closure. Ancestor inodes are never opened writable. + std::fs::hard_link(&layer.path, &path).map_err(|e| e.to_string())?; + } + layer.path = path; + } + let closure = next + .layers + .iter() + .map(|layer| CompactLayer { + path: layer.path.clone(), + qcow2: layer.format == RootDiskFormat::Qcow2, + }) + .collect::>(); + microsandbox_image::ext4::grow_chain(&closure, target).map_err(|e| e.to_string())?; + next.growth_target = None; + sync_directory(stage.path()).map_err(|e| e.to_string())?; + // Preserve staging across an uncertain journal rename/fsync. Recovery can then follow + // whichever durable journal won, without ever opening a partially rewritten filesystem. + let _published = stage.keep(); + write_state(&state_path, &next)?; + Ok(true) +} + +/// Finish a previously recorded online growth before cold boot, under the stopped lifecycle lock. +pub fn recover_stopped_root_growth(runtime_dir: &Path) -> Result<(), String> { + let path = runtime_dir.join(ROOT_DISK_STATE_FILE); + if path.exists() + && let Some(target) = read_state(&path)?.growth_target + { + grow_stopped_root(runtime_dir, target)?; + } + Ok(()) +} + /// Compact a stopped runtime-owned root after the caller acquires the sandbox lifecycle lock. /// The caller must prove no live process can write this disk until the operation finishes. pub fn compact_stopped_root( @@ -881,6 +1050,83 @@ fn sync_directory(path: &Path) -> std::io::Result<()> { #[cfg(test)] mod tests { + #[test] + fn stopped_growth_preserves_ancestors_and_recovers_pending_target() { + use super::*; + use microsandbox_image::ext4::{Ext4FormatOptions, format_ext4}; + for layout in [RootDiskLayout::ManagedUpper, RootDiskLayout::FlatRoot] { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("runtime"); + std::fs::create_dir(&root).unwrap(); + let base = dir.path().join("base.raw"); + let mib = 1024 * 1024; + format_ext4( + &base, + &Ext4FormatOptions { + size_bytes: 256 * mib, + journal_blocks: 4096, + }, + ) + .unwrap(); + let original = sparse_file_integrity(&base).unwrap().root; + let head = root.join("head.qcow2"); + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + rt.block_on(microsandbox_image::checkpoint::create_qcow2_overlay( + &head, + 256 * mib, + &base, + "raw", + )) + .unwrap(); + let mut state = RootDiskState { + schema: ROOT_DISK_STATE_SCHEMA.into(), + volume_id: new_id("vol"), + device_id: layout.device_id().into(), + layout, + published_generation: 1, + launch_base: None, + growth_target: Some(512 * mib), + layers: vec![ + RootDiskLayer { + layer_id: new_id("layer"), + path: base.clone(), + format: RootDiskFormat::Raw, + integrity_root: Some(original.clone()), + }, + RootDiskLayer { + layer_id: new_id("layer"), + path: head, + format: RootDiskFormat::Qcow2, + integrity_root: None, + }, + ], + }; + let journal = root.join(ROOT_DISK_STATE_FILE); + write_state(&journal, &state).unwrap(); + assert!(load_runtime_owned_root_chain(&root).is_err()); + assert!(compact_stopped_root(&root, None, false).is_err()); + assert!(grow_stopped_root(&root, 768 * mib).is_err()); + recover_stopped_root_growth(&root).unwrap(); + let chain = load_runtime_owned_root_chain(&root).unwrap().unwrap(); + assert_eq!(chain.virtual_size, 512 * mib); + assert_eq!(chain.layers.len(), 2); + assert_eq!(sparse_file_integrity(&base).unwrap().root, original); + // Simulate loss of the final acknowledgment after the filesystem already grew. + state = read_state(&journal).unwrap(); + state.growth_target = Some(512 * mib); + write_state(&journal, &state).unwrap(); + recover_stopped_root_growth(&root).unwrap(); + assert!(read_state(&journal).unwrap().growth_target.is_none()); + let before = std::fs::read(&journal).unwrap(); + assert!(grow_stopped_root(&root, 512 * mib + 1).is_err()); + assert_eq!(std::fs::read(&journal).unwrap(), before); + assert_eq!(sparse_file_integrity(&base).unwrap().root, original); + } + } + #[test] fn stopped_compaction_preserves_head_and_recovers_both_layouts() { use super::*; @@ -937,6 +1183,7 @@ mod tests { layout, published_generation: 3, launch_base: None, + growth_target: None, layers, }, ) @@ -1012,6 +1259,7 @@ mod tests { layout: RootDiskLayout::ManagedUpper, published_generation: 1, launch_base: None, + growth_target: None, layers: vec![ RootDiskLayer { layer_id: new_id("layer"), @@ -1059,6 +1307,7 @@ mod tests { layout: RootDiskLayout::FlatRoot, published_generation: 0, launch_base: None, + growth_target: None, layers: vec![RootDiskLayer { layer_id: new_id("layer"), path: base.clone(), @@ -1099,6 +1348,7 @@ mod tests { layout: RootDiskLayout::ManagedUpper, published_generation: 0, launch_base: None, + growth_target: None, layers: vec![RootDiskLayer { layer_id: new_id("layer"), path: "upper.ext4".into(), diff --git a/crates/runtime/lib/checkpoint/mod.rs b/crates/runtime/lib/checkpoint/mod.rs index 73d3e6e78..e6a61f2b2 100644 --- a/crates/runtime/lib/checkpoint/mod.rs +++ b/crates/runtime/lib/checkpoint/mod.rs @@ -16,7 +16,7 @@ pub(crate) use coordinator::{CheckpointCoordinator, CheckpointResult}; pub(crate) use disk::recover_runtime_owned_root; pub use disk::{ DiskCompactionResult, RuntimeOwnedRootChain, RuntimeOwnedRootLayer, compact_stopped_root, - load_runtime_owned_root_chain, + grow_stopped_root, load_runtime_owned_root_chain, recover_stopped_root_growth, }; #[cfg(feature = "runner")] pub(crate) use restore::{PreparedCheckpointRestore, RestoredAgentState}; diff --git a/crates/runtime/lib/client/control.rs b/crates/runtime/lib/client/control.rs index f35e1d59d..adddb48b6 100644 --- a/crates/runtime/lib/client/control.rs +++ b/crates/runtime/lib/client/control.rs @@ -35,6 +35,11 @@ pub const CONTROL_PROTOCOL_VERSION: u16 = 1; #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(tag = "op", rename_all = "snake_case")] pub enum ControlRequest { + /// Grow the owned root disk and mounted ext4 filesystem without rebooting. + RootDiskGrow { + /// Target capacity in bytes. + size_bytes: u64, + }, /// Explicitly consolidate the oldest sealed root-disk layers. DiskCompact { /// Oldest layer count including the base; omitted selects all sealed layers. @@ -133,6 +138,9 @@ pub struct SecretValue(pub String); /// The reply to any control request. #[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct ControlResponse { + /// Guest-observed root capacities after successful filesystem expansion. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub root_disk: Option, /// Explicit disk-compaction result or dry-run projection. #[serde(default, skip_serializing_if = "Option::is_none")] pub compaction: Option, @@ -182,12 +190,30 @@ pub struct CheckpointControlState { pub memory_emitted_bytes: u64, } +/// Verified capacity and measured phases of a completed online root growth. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RootDiskGrowthResult { + /// Committed ext4 capacity in bytes. + pub filesystem_bytes: u64, + /// Guest-observed virtio-block capacity in bytes. + pub device_bytes: u64, + /// Total runtime operation time, including preflight and persistence. + pub total_us: u64, + /// VM pause through resume; excludes online filesystem expansion. + pub pause_us: u64, + /// Guest expansion and verification time after VM resume. + pub guest_us: u64, +} + /// Live-control operations supported by this sandbox process, carried in /// [`ControlResponse`]. Runtimes that predate this op only served the socket /// when they could resize, so the SDK treats a missing reply as /// resize-capable and secrets-incapable. #[derive(Debug, Default, Clone, Copy, Serialize, Deserialize)] pub struct ControlCapabilities { + /// Host control supports root growth; guest capability is checked before mutation. + #[serde(default)] + pub root_disk_grow: bool, /// Explicit root-disk prefix compaction is supported. #[serde(default)] pub disk_compact: bool, @@ -371,6 +397,7 @@ mod tests { let response = ControlResponse { ok: true, capabilities: Some(ControlCapabilities { + root_disk_grow: true, cpu_resize: true, memory_resize: false, secrets_update: true, diff --git a/crates/runtime/lib/runner/control.rs b/crates/runtime/lib/runner/control.rs index 0c82b6a33..5015b88db 100644 --- a/crates/runtime/lib/runner/control.rs +++ b/crates/runtime/lib/runner/control.rs @@ -323,6 +323,7 @@ mod tests { let response = ControlResponse { ok: true, capabilities: Some(ControlCapabilities { + root_disk_grow: true, disk_compact: true, cpu_resize: true, memory_resize: false, diff --git a/crates/runtime/lib/runner/control/executor.rs b/crates/runtime/lib/runner/control/executor.rs index cae458ec9..b1fb71df9 100644 --- a/crates/runtime/lib/runner/control/executor.rs +++ b/crates/runtime/lib/runner/control/executor.rs @@ -183,6 +183,7 @@ impl RuntimeControlExecutor { let mutation = matches!( request, ControlRequest::MemoryTarget { .. } + | ControlRequest::RootDiskGrow { .. } | ControlRequest::CpuTarget { .. } | ControlRequest::SecretsUpdate { .. } | ControlRequest::CheckpointCreate { .. } @@ -196,6 +197,21 @@ impl RuntimeControlExecutor { } let response = match request { + ControlRequest::RootDiskGrow { size_bytes } => { + match state.checkpoint.grow_root(&self.vm, size_bytes) { + Ok(root_disk) => ControlResponse { + ok: true, + root_disk: Some(root_disk), + ..Default::default() + }, + Err(error) => { + if error.keep_paused { + state.lifecycle = RuntimeLifecycle::Quiesced; + } + control_error("root_disk_growth_incomplete", error.to_string()) + } + } + } ControlRequest::DiskCompact { layers, dry_run } => { match state.checkpoint.compact(&self.vm, layers, dry_run) { Ok(result) => ControlResponse { @@ -317,6 +333,7 @@ impl RuntimeControlExecutor { secrets_update: self.secrets_update_supported(), checkpoint_create: true, disk_compact: true, + root_disk_grow: true, }), ..Default::default() }, @@ -335,7 +352,9 @@ impl RuntimeControlExecutor { } ControlRequest::CpuState => cpu(self.vm.cpu_state()), ControlRequest::SecretsUpdate { changes } => self.handle_secrets_update(changes), - ControlRequest::CheckpointCreate { .. } | ControlRequest::DiskCompact { .. } => { + ControlRequest::CheckpointCreate { .. } + | ControlRequest::DiskCompact { .. } + | ControlRequest::RootDiskGrow { .. } => { unreachable!("checkpoint requests are handled by the executor lifecycle path") } } diff --git a/docs/sandboxes/tuning.mdx b/docs/sandboxes/tuning.mdx index 6cffce3da..745dd95c5 100644 --- a/docs/sandboxes/tuning.mdx +++ b/docs/sandboxes/tuning.mdx @@ -23,7 +23,7 @@ The live paths come first, but `modify` also plans changes that need a restart o | Existing secret material or removal | Live | Adding a secret or changing its placeholder needs a restart | | Environment and workdir | Future execs | Processes that are already running keep their current values | | `max_cpus` and `max_memory` | Restart or next start | These ceilings are fixed when the VM boots | -| Root disk size | Restart or next start | Managed and flat OCI disks grow only; tmpfs changes on boot | +| Root disk size | Live, restart or next start | Managed and flat ext4 roots grow only; tmpfs changes on boot | | Named volumes, mount tmpfs, and user disk images | Outside `modify` | Capacity is managed where the storage is defined | The default policy applies only changes that can complete without restarting. If a patch contains one restart-required change, microsandbox rejects the whole patch and the old configuration stays intact. @@ -356,6 +356,38 @@ plan, err := sb.Modify(ctx, m.ModifyOptions{ Adding a new secret or changing its guest-visible placeholder requires a restart because existing processes cannot receive a new placeholder. Removing a secret does not require recreating the sandbox. See [Secrets](/sandboxes/secrets) for sources, host allow lists, and storage behavior. +### Root disk growth + +Grow an owned managed or flat ext4 root without restarting, including roots backed by checkpoint layers: + + +```bash CLI +msb modify worker --root-disk 8G +``` + +```typescript TypeScript +await sandbox.modify({ rootDiskSize: 8192 }); +``` + +```rust Rust +sb.modify().root_disk_size(8192).apply().await?; +``` + +```python Python +await sb.modify(root_disk_size=8192) +``` + +```go Go +_, err := sb.Modify(ctx, m.ModifyOptions{RootDiskSizeMiB: 8192}) +``` + + +Microsandbox briefly pauses the VM to extend its writable disk, resumes it, then asks ext4 to make the space usable. Existing snapshots and sealed backing layers stay unchanged. A stopped sandbox grows before the new size is persisted. + +Growth is explicit and grow-only. The ext4 image must have enough reserved metadata headroom; older runtimes and filesystems without online-resize support return an error instead of silently restarting. User-supplied disks and cloud sandboxes remain unsupported. Use `--next-start` to defer growth or `--restart` to use stopped growth. + +If the disk grows but guest filesystem expansion fails, the error reports incomplete growth. Retry the same target, or stop and restart to finish recovery. Microsandbox never truncates the disk to roll back; snapshots and compaction stay blocked until recovery completes. + ## Changes that need a boot boundary Some settings define VM capacity or disk layout and cannot change in place: @@ -363,7 +395,6 @@ Some settings define VM capacity or disk layout and cannot change in place: | Change | Apply now | Defer safely | | --- | --- | --- | | Raise `max_cpus` or `max_memory` | `--restart` | `--next-start` | -| Grow a managed or flat OCI root disk | `--restart` | `--next-start` | | Resize a tmpfs root disk | On restart | `--next-start` | | Add a new secret or change its placeholder | `--restart` | `--next-start` | diff --git a/docs/sdk/go/sandbox.mdx b/docs/sdk/go/sandbox.mdx index b425a8895..08c6dbd48 100644 --- a/docs/sdk/go/sandbox.mdx +++ b/docs/sdk/go/sandbox.mdx @@ -604,10 +604,9 @@ _, err = sb.Modify(ctx, m.ModifyOptions{ Env: map[string]string{"MODE": "prod"}, Policy: m.ModificationPolicyRestart, }) -// Grow the managed OCI root disk offline and restart +// Grow the owned OCI root disk while it is running _, err = sb.Modify(ctx, m.ModifyOptions{ RootDiskSizeMiB: 8192, - Policy: m.ModificationPolicyRestart, }) // Add or rotate a host-environment secret; restart if it is newly added @@ -629,9 +628,9 @@ _, err = sb.Modify(ctx, m.ModifyOptions{ -Plan or apply a configuration change. The returned plan labels each change `"live"`, `"next start"`, `"requires restart"`, or `"unsupported"`, and apply is all-or-nothing. +Plan or apply a configuration change. The returned plan labels each change `"live"`, `"next start"`, `"requires restart"`, or `"unsupported"`. -`CPUs` and `MemoryMiB` resize live within the [`WithMaxCPUs`](#withmaxcpus) / [`WithMaxMemory`](#withmaxmemory) ceilings; raising a ceiling requires a restart. `RootDiskSizeMiB` changes are offline: managed and flat OCI root disks grow only, tmpfs root disks can change in either direction on the next boot, and user-supplied disk images are rejected. Env and workdir changes affect future execs only. On a stopped sandbox, changes are saved for the next boot. +`CPUs` and `MemoryMiB` resize live within the [`WithMaxCPUs`](#withmaxcpus) / [`WithMaxMemory`](#withmaxmemory) ceilings; raising a ceiling requires a restart. `RootDiskSizeMiB` grows owned managed and flat ext4 roots live, including checkpoint-backed roots, tmpfs root disks can change in either direction on the next boot, and user-supplied disk images are rejected. Env and workdir changes affect future execs only. Stopped roots grow before the new size is saved. Explicit restart and next-start policies remain available. Secret specs are keyed by stable secret name. Each `SecretModifySpec` selects at most one source (`Env`, `Value`, or `Store`) and may also set `Placeholder` and `AllowedHosts`; omitting a source updates only the other supplied fields. Plans expose only safe references and metadata; raw secret values never appear in a plan. Removal is explicit through `SecretsRemove`. diff --git a/docs/sdk/python/sandbox.mdx b/docs/sdk/python/sandbox.mdx index eccadace1..3bc8327da 100644 --- a/docs/sdk/python/sandbox.mdx +++ b/docs/sdk/python/sandbox.mdx @@ -596,8 +596,8 @@ for change in plan["changes"]: # Make an env change active now by restarting await sb.modify(env={"MODE": "prod"}, policy=ModificationPolicy.RESTART) -# Grow the managed OCI root disk offline and restart -await sb.modify(root_disk_size=8192, policy=ModificationPolicy.RESTART) +# Grow the owned OCI root disk while it is running +await sb.modify(root_disk_size=8192) # Add or rotate a host-environment secret; restart if it is newly added await sb.modify( @@ -616,9 +616,9 @@ await sb.modify(secrets_rm=["OLD_API_KEY"]) -Plan or apply a configuration change. The returned plan uses [`ModificationDisposition`](#modificationdisposition) to classify when each change takes effect, and apply is all-or-nothing. +Plan or apply a configuration change. The returned plan uses [`ModificationDisposition`](#modificationdisposition) to classify when each change takes effect. -`cpus` and `memory` resize live within the `max_cpus` / `max_memory` ceilings; raising a ceiling requires a restart. `root_disk_size` changes are offline: managed and flat OCI root disks grow only, tmpfs root disks can change in either direction on the next boot, and user-supplied disk images are rejected. Env and workdir changes affect future execs only. On a stopped sandbox, changes are saved for the next boot. +`cpus` and `memory` resize live within the `max_cpus` / `max_memory` ceilings; raising a ceiling requires a restart. `root_disk_size` grows owned managed and flat ext4 roots live, including checkpoint-backed roots, tmpfs root disks can change in either direction on the next boot, and user-supplied disk images are rejected. Env and workdir changes affect future execs only. Stopped roots grow before the new size is saved. Explicit restart and next-start policies remain available. Secret specs are keyed by stable secret name. Each `SecretModifySpec` selects at most one source (`env`, `value`, or `store`) and may also set `placeholder` and `allowed_hosts`; omitting a source updates only the other supplied fields. Plans expose only safe references and metadata; raw secret values never appear in a plan. Removal is explicit through `secrets_rm`. diff --git a/docs/sdk/typescript/sandbox.mdx b/docs/sdk/typescript/sandbox.mdx index 0b059c32f..8cc23a776 100644 --- a/docs/sdk/typescript/sandbox.mdx +++ b/docs/sdk/typescript/sandbox.mdx @@ -570,8 +570,8 @@ for (const c of preview.changes) { // Make an env change active now by restarting await sandbox.modify({ env: { MODE: "prod" }, policy: "restart" }); -// Grow the managed OCI root disk offline and restart -await sandbox.modify({ rootDiskSize: 8192, policy: "restart" }); +// Grow the owned OCI root disk while it is running +await sandbox.modify({ rootDiskSize: 8192 }); // Add or rotate a host-environment secret; restart if it is newly added await sandbox.modify({ @@ -587,9 +587,9 @@ await sandbox.modify({ secretsRemove: ["OLD_API_KEY"] }); -Plan or apply a configuration change. The returned plan labels each change `"live"`, `"next start"`, `"requires restart"`, or `"unsupported"`, and apply is all-or-nothing. +Plan or apply a configuration change. The returned plan labels each change `"live"`, `"next start"`, `"requires restart"`, or `"unsupported"`. -`cpus` and `memory` resize live within the [`maxCpus`](#sandboxbuilder-maxcpus) / [`maxMemory`](#sandboxbuilder-maxmemory) ceilings; raising a ceiling requires a restart. `rootDiskSize` changes are offline: managed and flat OCI root disks grow only, tmpfs root disks can change in either direction on the next boot, and user-supplied disk images are rejected. Env and workdir changes affect future execs only. On a stopped sandbox, changes are saved for the next boot. +`cpus` and `memory` resize live within the [`maxCpus`](#sandboxbuilder-maxcpus) / [`maxMemory`](#sandboxbuilder-maxmemory) ceilings; raising a ceiling requires a restart. `rootDiskSize` grows owned managed and flat ext4 roots live, including checkpoint-backed roots, tmpfs root disks can change in either direction on the next boot, and user-supplied disk images are rejected. Env and workdir changes affect future execs only. Stopped roots grow before the new size is saved. Explicit restart and next-start policies remain available. Secret specs are keyed by stable secret name. Each `SecretModifySpec` selects at most one source (`env`, `value`, or `store`) and may also set `placeholder` and `allowedHosts`; omitting a source updates only the other supplied fields. Plans expose only safe references and metadata; raw secret values never appear in a plan. Removal is explicit through `secretsRemove`. diff --git a/scripts/smoke/cli/root-disk-growth.py b/scripts/smoke/cli/root-disk-growth.py new file mode 100644 index 000000000..9adc913b3 --- /dev/null +++ b/scripts/smoke/cli/root-disk-growth.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Live root-growth matrix. Requires an isolated MSB_HOME and matching MSB_BIN/firmware.""" + +import hashlib +import json +import os +from pathlib import Path +import socket +import subprocess +import time + + +binary = os.environ["MSB_BIN"] +home = Path(os.environ["MSB_HOME"]) +report = Path(os.environ["QUAL_ROOT"]) +report.mkdir(parents=True, exist_ok=True) +names = [] +results = [] + + +def run(label, *args, refuse=False): + started = time.perf_counter() + result = subprocess.run([binary, *args], text=True, capture_output=True, timeout=180) + elapsed = (time.perf_counter() - started) * 1000 + (report / f"{label}.stdout").write_text(result.stdout) + (report / f"{label}.stderr").write_text(result.stderr) + passed = result.returncode != 0 if refuse else result.returncode == 0 + results.append({"case": label, "passed": passed, "elapsed_ms": elapsed}) + (report / "results.json").write_text(json.dumps(results, indent=2)) + print(f"{label}: {'PASS' if passed else 'FAIL'} {elapsed:.2f} ms", flush=True) + if not passed: + raise AssertionError(result.stdout + result.stderr) + return result.stdout + + +def guest(label, name, script): + return run(label, "exec", name, "--", "sh", "-c", script) + + +def phase_grow(label, name, mib): + # The runtime response separates the short VM pause from online ext4 expansion. + # Follow with the SDK-backed CLI call to reconcile/persist its desired configuration. + if os.name != "nt": + digest = hashlib.sha256(name.encode()).hexdigest()[:24] + path = home / "run" / "sandboxes" / digest / "control.sock" + with socket.socket(socket.AF_UNIX) as connection: + connection.settimeout(180) + connection.connect(str(path)) + connection.sendall(json.dumps({"op": "root_disk_grow", "size_bytes": mib * 1048576}).encode() + b"\n") + with connection.makefile("rb") as reader: + reply = json.loads(reader.readline()) + (report / f"{label}.phases.json").write_text(json.dumps(reply, indent=2)) + assert reply["ok"], reply + assert reply["root_disk"]["filesystem_bytes"] == mib * 1048576 + run(label, "modify", name, "--root-disk", f"{mib}M", "--format", "json") + + +def journal(name): + return json.loads((home / "sandboxes" / name / "runtime" / "root-disk.json").read_text()) + + +def file_hash(path): + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +try: + for layout in ("managed", "flat"): + name = f"{os.environ.get('QUAL_PREFIX', 'grow')}-{layout}" + names.append(name) + disk = "512M" if layout == "managed" else "flat:512M" + run(f"{layout}-create", "create", "-n", name, "--root-disk", disk, "-m", "256M", "--max-duration", "20m", "alpine") + guest(f"{layout}-seed", name, "dd if=/dev/urandom of=/payload bs=1048576 count=8 2>/dev/null; sha256sum /payload >/expected; echo ram >/dev/shm/grow-marker; sync") + phase_grow(f"{layout}-raw-live", name, 768) + guest(f"{layout}-new-space", name, "dd if=/dev/zero of=/space bs=1048576 count=600 conv=fsync && test $(stat -c %s /space) = 629145600 && rm /space && sha256sum -c /expected") + run(f"{layout}-old-snapshot", "snapshot", "create", f"{name}-old", "--from-sandbox", name, "--full") + before = journal(name) + ancestor = Path(before["layers"][0]["path"]) + ancestor_before = file_hash(ancestor) + phase_grow(f"{layout}-qcow-live", name, 1024) + phase_grow(f"{layout}-qcow-repeat", name, 1280) + assert len(journal(name)["layers"]) == len(before["layers"]) + assert file_hash(ancestor) == ancestor_before + run(f"{layout}-shrink-refused", "modify", name, "--root-disk", "512M", refuse=True) + run(f"{layout}-new-snapshot", "snapshot", "create", f"{name}-new", "--from-sandbox", name, "--full") + run(f"{layout}-compact", "modify", name, "--compact", "--format", "json") + phase_grow(f"{layout}-compacted-live", name, 1536) + phase_grow(f"{layout}-partial-group-live", name, 1700) + run(f"{layout}-stop", "stop", name) + run(f"{layout}-stopped-grow", "modify", name, "--root-disk", "1792M", "--format", "json") + run(f"{layout}-start", "start", name) + guest(f"{layout}-after-stopped-grow", name, "sha256sum -c /expected; df -k /") + run(f"{layout}-defer", "modify", name, "--root-disk", "2G", "--next-start", "--format", "json") + run(f"{layout}-defer-stop", "stop", name) + run(f"{layout}-defer-start", "start", name) + guest(f"{layout}-defer-space", name, "dd if=/dev/zero of=/space bs=1048576 count=1800 conv=fsync && rm /space && sha256sum -c /expected") + run(f"{layout}-final-stop", "stop", name) + run(f"{layout}-stopped-snapshot", "snapshot", "create", f"{name}-stopped", "--from-sandbox", name, "--integrity") + run(f"{layout}-verify", "snapshot", "verify", f"{name}-stopped") + for suffix, capacity in (("old", 768), ("new", 1280)): + child = f"{name}-{suffix}-child" + names.append(child) + run(f"{layout}-{suffix}-restore", "create", "-n", child, "--from-snapshot", f"{name}-{suffix}") + guest(f"{layout}-{suffix}-restored-data", child, "sha256sum -c /expected && test $(cat /dev/shm/grow-marker) = ram") + # Full restore retains captured block capacity, not the source's later size. + layers = journal(child)["layers"] + head = Path(layers[-1]["path"]) + with head.open("rb") as file: + file.seek(24) + assert int.from_bytes(file.read(8), "big") == capacity * 1048576 + run(f"{layout}-{suffix}-child-stop", "stop", child) + print(f"Root-growth matrix passed: {report / 'results.json'}", flush=True) +finally: + for name in names: + subprocess.run([binary, "stop", name], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=30) diff --git a/scripts/smoke/cli/root-disk-large-growth.py b/scripts/smoke/cli/root-disk-large-growth.py new file mode 100644 index 000000000..fadcc0b6d --- /dev/null +++ b/scripts/smoke/cli/root-disk-large-growth.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +"""One-shot root growth across GiB/metadata boundaries in an isolated MSB_HOME. + +Uses the existing release binary, not a build step. QUAL_PREFIX must be fresh. Each +case stops its own VMs, keeps artifacts for inspection, and records failed checks. +""" + +import hashlib +import itertools +import json +import os +from pathlib import Path +import subprocess +import tempfile +import time + + +BINARY = os.environ["MSB_BIN"] +TEST_HOME = Path(os.environ["MSB_HOME"]) +REPORT = Path(os.environ["QUAL_ROOT"]) +PREFIX = os.environ["QUAL_PREFIX"] +MIB = 1048576 +RESULTS = [] +REPORT.mkdir(parents=True, exist_ok=True) + + +def record(label, passed, **details): + RESULTS.append(dict(case=label, passed=passed, **details)) + (REPORT / "results.json").write_text(json.dumps(RESULTS, indent=2)) + print(f"{label}: {'PASS' if passed else 'FAIL'} {details}", flush=True) + + +def check(label, condition, **details): + record(label, bool(condition), **details) + if not condition: + raise AssertionError(label) + + +def run(label, *args, refuse=False): + started = time.perf_counter() + result = subprocess.run([BINARY, *args], capture_output=True, text=True, timeout=300) + elapsed = (time.perf_counter() - started) * 1000 + (REPORT / f"{label}.stdout").write_text(result.stdout) + (REPORT / f"{label}.stderr").write_text(result.stderr) + passed = result.returncode != 0 if refuse else result.returncode == 0 + record(label, passed, elapsed_ms=round(elapsed, 3), returncode=result.returncode) + if not passed: + raise AssertionError(result.stdout + result.stderr) + return result.stdout + + +def guest(label, name, script): + # A failed checksum must not be hidden by a later successful df/sync command. + return run(label, "exec", name, "--", "sh", "-ec", script) + + +def state(name): + return json.loads((TEST_HOME / "sandboxes" / name / "runtime" / "root-disk.json").read_text()) + + +def capacity(layer): + path = Path(layer["path"]) + with path.open("rb") as stream: + magic = stream.read(4) + if magic == b"QFI\xfb": + stream.seek(24) + return int.from_bytes(stream.read(8), "big") + return path.stat().st_size + + +def digest(path): + result = hashlib.sha256() + with Path(path).open("rb") as stream: + for block in iter(lambda: stream.read(MIB), b""): + result.update(block) + return result.hexdigest() + + +def case(layout, backing, mode, target): + label = f"{layout}-{backing}-{mode}-{target}" + name = f"{PREFIX}-{label}" + names = [name] + try: + disk = "512M" if layout == "managed" else "flat:512M" + run(label + "-create", "create", "-n", name, "--root-disk", disk, + "-m", "256M", "--max-duration", "30m", "alpine") + guest(label + "-seed", name, + "dd if=/dev/urandom of=/payload bs=1048576 count=8; " + "sha256sum /payload >/expected; echo retained >/dev/shm/grow-marker; " + "cat /proc/sys/kernel/random/boot_id >/boot-before; sync") + if backing == "qcow2": + run(label + "-old-snapshot", "snapshot", "create", name + "-old", + "--from-sandbox", name, "--full") + before = state(name) + check(label + "-initial-capacity", capacity(before["layers"][-1]) == 512 * MIB) + check(label + "-initial-format", + before["layers"][-1]["format"] == ("qcow2" if backing == "qcow2" else "raw")) + ancestors = [(layer["path"], digest(layer["path"])) for layer in before["layers"][:-1]] + if mode == "stopped": + run(label + "-stop-before-grow", "stop", name) + # Time the first real CLI growth, including planning, control and configuration persistence. + size = "4G" if target == 4096 else f"{target}M" + run(label + "-grow", "modify", name, "--root-disk", size, "--format", "json") + after = state(name) + check(label + "-exact-capacity", capacity(after["layers"][-1]) == target * MIB) + check(label + "-depth-and-completion", len(after["layers"]) == len(before["layers"]) + and after.get("growth_target") is None) + check(label + "-immutable-ancestors", all(digest(path) == old for path, old in ancestors)) + # The public CLI deliberately rejects an already configured size. Runtime forward + # recovery is repeatable, but that is distinct from a redundant completed CLI request. + repeated = run(label + "-same-target", "modify", name, "--root-disk", size, + "--format", "json", refuse=True) + check(label + "-same-target-reason", "only grow is supported" in repeated) + run(label + "-shrink-rejected", "modify", name, "--root-disk", "512M", refuse=True) + check(label + "-refusal-keeps-capacity", capacity(state(name)["layers"][-1]) == target * MIB) + if mode == "stopped": + run(label + "-start-after-grow", "start", name) + guest(label + "-ram-seed", name, "echo retained >/dev/shm/grow-marker") + else: + guest(label + "-no-reboot", name, + "test $(cat /proc/sys/kernel/random/boot_id) = $(cat /boot-before); " + "test $(cat /dev/shm/grow-marker) = retained") + # Allocate (not merely truncate/seek) 3 GiB at the main target, using repeated random + # bytes. Smaller allocation in metadata-boundary cases keeps the matrix practical. + count = 384 if target == 4096 else 96 + allocated = count * 8 * MIB + guest(label + "-write-added-space", name, + f"i=0; while test $i -lt {count}; do cat /payload >>/large; i=$((i+1)); done; " + f"test $(stat -c %s /large) = {allocated}; " + f"test $(stat -c %b /large) -ge {allocated // 512}; " + "sync; sha256sum /large >/large.expected; sha256sum -c /expected; " + f"dd if=/payload of=/far bs=1048576 seek={target - 256} conv=fsync; " + f"dd if=/far bs=1048576 skip={target - 256} count=8 2>/dev/null | cmp - /payload; " + "df -k /; du -k /large") + # Snapshot the allocated file, not just metadata or an empty resized filesystem. + run(label + "-new-snapshot", "snapshot", "create", name + "-new", "--from-sandbox", name, "--full") + run(label + "-source-stop", "stop", name) + for suffix, expected in [("new", target)] + ([("old", 512)] if backing == "qcow2" else []): + child = name + "-" + suffix + "-child" + names.append(child) + run(label + "-" + suffix + "-restore", "create", "-n", child, "--from-snapshot", name + "-" + suffix) + check(label + "-" + suffix + "-restored-capacity", capacity(state(child)["layers"][-1]) == expected * MIB) + script = "sha256sum -c /expected; test $(cat /dev/shm/grow-marker) = retained; " + if suffix == "new": + script += ("sha256sum -c /large.expected; " + f"dd if=/far bs=1048576 skip={target - 256} count=8 2>/dev/null | cmp - /payload") + else: + script += "test ! -e /large; test ! -e /far" + guest(label + "-" + suffix + "-restored-data", child, script) + run(label + "-" + suffix + "-child-stop", "stop", child) + # Verify a cold boot after full capture as well as restoring captured memory. + run(label + "-cold-start", "start", name) + guest(label + "-cold-data", name, "sha256sum -c /expected; sha256sum -c /large.expected") + run(label + "-cold-stop", "stop", name) + finally: + for owned in names: + result = subprocess.run([BINARY, "stop", owned], capture_output=True, text=True, timeout=40) + if result.returncode: + print(f"Cleanup needs inspection: {owned}: {result.stderr}", flush=True) + + +def offline_check(layout, backing, mode, target): + """Independently check a disposable flattened copy, never repair the source chain.""" + label = f"{layout}-{backing}-{mode}-{target}" + name = f"{PREFIX}-{label}" + statuses = json.loads(run(label + "-list-for-fsck", "list", "--format", "json")) + check(label + "-stopped-for-fsck", any(item["name"] == name and item["status"] == "Stopped" for item in statuses)) + head = state(name)["layers"][-1]["path"] + qemu = os.environ.get("QEMU_IMG", "qemu-img") + fsck = os.environ["E2FSCK"] + + def external(suffix, args): + started = time.perf_counter() + result = subprocess.run(args, capture_output=True, text=True, timeout=300) + (REPORT / f"{label}-{suffix}.stdout").write_text(result.stdout) + (REPORT / f"{label}-{suffix}.stderr").write_text(result.stderr) + check(label + "-" + suffix, result.returncode == 0, + elapsed_ms=round((time.perf_counter() - started) * 1000, 3), + returncode=result.returncode) + + external("qemu-chain", [qemu, "info", "--backing-chain", "--output=json", head]) + # The only automatically removed files belong to this newly created scratch directory. + with tempfile.TemporaryDirectory(prefix="growth-fsck-") as scratch: + disk = str(Path(scratch) / "check.raw") + external("flatten-for-check", [qemu, "convert", "-O", "raw", head, disk]) + # A stopped guest can leave a replayable journal. Replay only on the disposable copy, + # then require a read-only full filesystem check with no corrective repairs. + external("journal-replay", [fsck, "-p", "-E", "journal_only", disk]) + external("fsck", [fsck, "-f", "-n", disk]) + + +def main(): + # 8 GiB occupies 64 groups; 8320 MiB needs a second 64-byte-descriptor GDT block. + targets = [int(value) for value in os.environ.get("QUAL_TARGETS", "4096,8320").split(",")] + failures = [] + for target, layout, backing, mode in itertools.product( + targets, ("managed", "flat"), ("raw", "qcow2"), ("live", "stopped")): + try: + if os.environ.get("QUAL_OFFLINE_ONLY") == "1": + offline_check(layout, backing, mode, target) + else: + case(layout, backing, mode, target) + except Exception as error: + label = f"{layout}-{backing}-{mode}-{target}" + record(label + "-exception", False, error=str(error)) + failures.append(label) + if failures: + raise SystemExit("Failed cases: " + ", ".join(failures)) + print(f"All {len(targets) * 8} large-jump scenarios passed", flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/smoke/reports/root-disk-growth-2026-09-06.md b/scripts/smoke/reports/root-disk-growth-2026-09-06.md new file mode 100644 index 000000000..907669122 --- /dev/null +++ b/scripts/smoke/reports/root-disk-growth-2026-09-06.md @@ -0,0 +1,72 @@ +# Root-disk growth qualification — 2026-09-06 + +Stack: Microsandbox `appcypher/live-root-disk-growth`, directly based on #6 (`ed4167f7`), with libkrun `ec9f119dcc144f1c011fe62ebd224d660c2191a3`. No newer main changes were imported. Publication and Linux/Windows source transfers await explicit approval. + +## Coverage + +The reproducible matrix is `scripts/smoke/cli/root-disk-growth.py`. Set `MSB_BIN`, an isolated `MSB_HOME`, matching `MSB_LIBKRUNFW_PATH`, `QUAL_ROOT` and a fresh `QUAL_PREFIX`. It always stops its own sandboxes, including on failure. + +macOS/HVF passed 58 CLI checks and ten direct control-phase measurements in both debug and release builds. Hardware: Apple M5 Max, 36 GiB RAM. Guest: Alpine, 256 MiB RAM, initially 512 MiB ext4 root. The release matrix covers both managed and flat roots: raw growth to 768 MiB; a 600 MiB file written and synced beyond the original capacity; full checkpoint rollover; qcow2 growth to 1 GiB and then 1280 MiB; unchanged ancestor SHA-256 and chain length; shrink rejection; a second full snapshot; compaction followed by growth; a partial final block group at 1700 MiB; stopped growth to 1792 MiB; next-start growth to 2 GiB; a synced 1800 MiB file; stopped snapshot creation and verification; old/new full restores at 768/1280 MiB with payload checksums and tmpfs memory markers intact. All test sandboxes were confirmed stopped afterward. + +The initial live tests found that Linux online growth leaves valid lazy group flags which the older offline resizer rejected. The resizer now preserves those flags and initializes a lazy partial block bitmap before extending it. The failed attempt modified only staging; the original head and journal remained usable. + +## Release phase measurements + +These are individual observations, not percentiles or latency guarantees. Runtime total includes preflight, persistence, pause/resume and guest acknowledgment. Pause includes draining the block worker. Guest time includes online ext4 expansion, sync and verification, after VM resume. + +| Operation | Managed total / pause / guest (ms) | Flat total / pause / guest (ms) | +| --- | ---: | ---: | +| Raw 512 → 768 MiB | 69.14 / 7.80 / 45.38 | 62.49 / 7.91 / 38.18 | +| Qcow2 768 → 1024 MiB | 69.44 / 7.11 / 45.79 | 67.76 / 8.03 / 43.03 | +| Qcow2 1024 → 1280 MiB | 50.49 / 4.15 / 30.32 | 55.39 / 4.18 / 35.12 | +| Compacted 1280 → 1536 MiB | 54.50 / 7.06 / 30.97 | 52.09 / 3.95 / 32.09 | +| Partial group 1536 → 1700 MiB | 50.10 / 4.02 / 29.98 | 52.24 / 3.80 / 32.07 | + +Stopped CLI growth from 1700 to 1792 MiB took 45.65 ms managed and 43.66 ms flat. The matrix's online CLI rows are same-target configuration reconciliation after a separately measured control request; they must not be quoted as end-to-end first-growth latency. Raw outputs and phase JSON are in `/private/tmp/msb-grow-qual.6VfI8S/release`; debug results are in its `matrix2` sibling. + +## Other checks + +- libkrun device suite: 135 passed, including preserved data, zero-filled added capacity, repeated targets, and read-only/shrink rejection. +- Runtime disk tests: five passed, including both layouts, immutable ancestors, pending-target snapshot/compaction refusal, recovery before boot, lost-completion acknowledgment, and failed staging without journal publication. +- Rust SDK modification tests: 51 passed, including explicit restart/next-start policies, new root capability and old control sockets. +- Full Rust SDK library suite: 666 passed and three ignored with `cargo test --offline -p microsandbox --lib -- --test-threads=1`. A parallel run had one failure in the unchanged stale Unix-socket detection test; that test passed in isolation and in the serial suite. The initial sandboxed run also denied four socket bindings; validation was repeated with local socket access. +- Image ext4 tests and four external `e2fsck` tests passed; dedicated tests cover qcow2 growth and lazy partial block groups. +- Protocol generation-9 schema and append-only checks: four passed. +- Pinned CLI build/check succeeds offline after seeding Cargo's cache from the exact signed local libkrun commit. The companion commit must be pushed before other machines can fetch it. +- Strict Clippy initially found pre-existing `derivable_impls` and `too_many_arguments` warnings in #6. Focused linting allows those two baseline classes; new lint findings were fixed. + +## One-shot MiB-to-GiB growth + +The original matrix did not establish large one-shot growth. Follow-up release qualification on the same Mac uses `scripts/smoke/cli/root-disk-large-growth.py`: 16 scenarios and 424 recorded checks passed, plus 96 independent offline validation checks. No runtime implementation changes were required. Each scenario starts afresh at 512 MiB, rather than reaching its target through intermediate grows. + +These measurements time the first SDK-backed CLI `modify` invocation from process launch through successful exit, including planning, control communication and configuration persistence. They are single observations, not percentiles. Stopped timings exclude the separately tested stop and start operations. Unlike the earlier phase table, these are not same-target reconciliation measurements and do not isolate VM pause time. + +| One-shot target | Initial backing | Managed live (ms) | Managed stopped (ms) | Flat live (ms) | Flat stopped (ms) | +| --- | --- | ---: | ---: | ---: | ---: | +| 512 MiB → 4 GiB | Raw | 74.37 | 40.95 | 75.83 | 38.12 | +| 512 MiB → 4 GiB | Qcow2 | 84.16 | 34.24 | 78.39 | 35.20 | +| 512 MiB → 8320 MiB (8.125 GiB) | Raw | 76.99 | 45.30 | 78.05 | 48.11 | +| 512 MiB → 8320 MiB (8.125 GiB) | Qcow2 | 85.97 | 35.93 | 84.62 | 40.32 | + +The 8320 MiB target crosses from 64 to 65 ext4 groups, requiring another group-descriptor block in the current 4 KiB-block/64-byte-descriptor layout. Every scenario verifies: + +- Exact head capacity, no pending-growth marker, unchanged chain depth and sealed-ancestor SHA-256 values. +- Same-target CLI rejection with the expected reason and shrink refusal without reducing capacity. Completed CLI requests deliberately reject an already configured size; this differs from retrying unfinished runtime growth. The first harness run incorrectly expected redundant CLI requests to succeed and was corrected without changing runtime behavior. +- For live growth, unchanged guest boot ID and a retained tmpfs marker. +- Real allocation and fsync of a 3 GiB file at the 4 GiB target, with allocated-block counts checked to exclude a sparse-only test. Boundary cases allocate 768 MiB. Both also write and read an 8 MiB random marker at a file offset 256 MiB below the new capacity; that is a file-offset check, not proof of allocation in the last physical block group. +- A full snapshot with that file present, full restore at the enlarged capacity, original/random-payload and large-file checksum checks, and retained tmpfs contents. +- For qcow2 cases, restoration of the pre-grow full snapshot at exactly 512 MiB with its original data and without files created after growth. +- A separate cold boot and checksum verification after the grown snapshot is captured. +- Independent `qemu-img info --backing-chain`, flattening to a disposable raw copy, journal-only replay on that copy, then `e2fsck -f -n` with exit zero. All 16 full read-only filesystem checks passed; source disks were not repaired or modified by these tools. + +The 4 GiB full snapshots containing the allocated 3 GiB file took 5.34–7.18 seconds and their full restores took 8.05–8.56 seconds. Those timings describe this populated-disk workload, not growth latency or empty snapshot performance. All VMs created by both large-jump runs were confirmed stopped. Test sandbox and snapshot artifacts remain available for inspection; disposable filesystem-check copies were removed automatically. + +Reproduce with an isolated `MSB_HOME`, matching `MSB_BIN`/`MSB_LIBKRUNFW_PATH`, a fresh `QUAL_PREFIX` and output `QUAL_ROOT`. The default targets are `4096,8320` MiB; `QUAL_TARGETS` can override them. Run the script normally first, then set `QUAL_OFFLINE_ONLY=1`, `E2FSCK` and optionally `QEMU_IMG` to independently validate those same stopped cases using a separate output directory. This run's outputs are `/private/tmp/msb-grow-qual.6VfI8S/large-jumps2` and `/private/tmp/msb-grow-qual.6VfI8S/large-jumps2-fsck`. + +## Remaining qualification and limitations + +Follow-up Rust API cleanup removes the redundant `modify()` methods `memory_mib`, `max_memory_mib`, `root_disk_size_mib`, `oci_upper_size` and `oci_upper_size_mib` without deprecation. Use `memory`, `max_memory` and `root_disk_size`; each accepts bare MiB integers and typed sizes. Serialized fields and Python/TypeScript/Go surfaces are unchanged. After cleanup, the serial Rust SDK suite passed 667 tests with three ignored, and all ten CLI modification tests passed. The live measurements above preceded this surface-only cleanup; the new test verifies that both supported input forms produce the same patch values. + +Linux/KVM and Windows/WHP have not been tested for this item. Both machines were reachable, but transferring unreleased source was blocked pending explicit approval. SDK bindings already route the existing modification options through Rust; fresh Python/TypeScript/Go native live runs have not been performed for #7. Direct/dependent archives after growth, sustained concurrent-I/O latency, request cancellation, process/power-loss injection, and every admission-failure variant remain additional qualification work. Do not describe this report as exhaustive fault testing. + +Stopped growth currently retains previous private file bindings after publishing the replacement head. It does not add chain depth or alter snapshots, but retained files can consume host space; automatic reclamation was not implemented because its deletion scope needs explicit approval/stronger ownership proof. Pending online growth is forward-only. Older runtimes refuse the pending journal field; finish recovery before switching back to an older runtime. A failed operation may leave saved desired configuration behind physical capacity until the caller retries the same requested size. diff --git a/sdk/rust/lib/runtime/spawn.rs b/sdk/rust/lib/runtime/spawn.rs index 3946ced6b..982ffd8ae 100644 --- a/sdk/rust/lib/runtime/spawn.rs +++ b/sdk/rust/lib/runtime/spawn.rs @@ -1010,22 +1010,21 @@ async fn prepare_oci_upper(config: &SandboxConfig, sandbox_dir: &Path) -> Micros | Some(microsandbox_types::RootDisk::Managed { size_mib }) => *size_mib, _ => None, }; - if let Some(chain) = microsandbox_runtime::checkpoint::load_runtime_owned_root_chain( - &sandbox_dir.join("runtime"), - ) - .map_err(|error| { - MicrosandboxError::Runtime(format!( - "cannot inspect the root-disk chain before startup: {error}" - )) - })? && chain.layers.len() > 1 - { - if desired_mib.is_some_and(|desired| u64::from(desired) * 1024 * 1024 > chain.virtual_size) - { - return Err(MicrosandboxError::Custom( - "cannot grow a checkpoint-backed root disk yet; its sealed raw ancestor must not be resized" - .into(), - )); + let runtime_dir = sandbox_dir.join("runtime"); + let handled = tokio::task::spawn_blocking(move || { + microsandbox_runtime::checkpoint::recover_stopped_root_growth(&runtime_dir)?; + match desired_mib { + Some(desired) => microsandbox_runtime::checkpoint::grow_stopped_root( + &runtime_dir, + u64::from(desired) * 1024 * 1024, + ), + None => Ok(runtime_dir.join("root-disk.json").exists()), } + }) + .await + .map_err(|e| MicrosandboxError::Runtime(e.to_string()))? + .map_err(MicrosandboxError::Runtime)?; + if handled { return Ok(()); } match &oci.root_disk { @@ -4200,17 +4199,15 @@ mod tests { } #[tokio::test] - async fn test_persisted_checkpoint_chain_refuses_deferred_root_grow() { + async fn test_invalid_checkpoint_chain_cannot_mutate_the_sealed_base_during_deferred_grow() { let temp = tempdir().unwrap(); let runtime = temp.path().join("runtime"); std::fs::create_dir(&runtime).unwrap(); let base = temp.path().join("rootfs.raw"); let head = temp.path().join("root-active.qcow2"); std::fs::write(&base, vec![0; 4096]).unwrap(); - // Capacity inspection must succeed before the sealed-chain grow guard. - microsandbox_image::checkpoint::create_qcow2_overlay(&head, 4096, &base, "raw") - .await - .unwrap(); + // Valid checkpoint chains support growth; a corrupt head must fail before any write. + std::fs::write(&head, b"qcow").unwrap(); let head_before = std::fs::read(&head).unwrap(); let state = serde_json::json!({ "schema": "microsandbox.runtime-root-disk/1", @@ -4252,30 +4249,20 @@ mod tests { let error = super::prepare_oci_upper(&config, temp.path()) .await .unwrap_err(); - assert!(error.to_string().contains("checkpoint-backed root disk")); + let expected = microsandbox_image::checkpoint::layer_capacities(vec![ + microsandbox_image::checkpoint::CompactLayer { + path: head.clone(), + qcow2: true, + }, + ]) + .unwrap_err(); + assert!(error.to_string().contains(&expected.to_string())); assert_eq!(std::fs::read(&base).unwrap(), vec![0; 4096]); assert_eq!(std::fs::read(&head).unwrap(), head_before); assert_eq!( std::fs::read(runtime.join("root-disk.json")).unwrap(), serde_json::to_vec(&state).unwrap() ); - - // A corrupt head must also fail closed, not fall back to growing its base. - std::fs::write(&head, b"qcow").unwrap(); - let error = super::prepare_oci_upper(&config, temp.path()) - .await - .unwrap_err(); - assert!( - error - .to_string() - .contains("cannot inspect the root-disk chain") - ); - assert_eq!(std::fs::read(&base).unwrap(), vec![0; 4096]); - assert_eq!(std::fs::read(&head).unwrap(), b"qcow"); - assert_eq!( - std::fs::read(runtime.join("root-disk.json")).unwrap(), - serde_json::to_vec(&state).unwrap() - ); } #[tokio::test] diff --git a/sdk/rust/lib/sandbox/modify.rs b/sdk/rust/lib/sandbox/modify.rs index 97323abbd..0281d005a 100644 --- a/sdk/rust/lib/sandbox/modify.rs +++ b/sdk/rust/lib/sandbox/modify.rs @@ -93,6 +93,8 @@ struct ExistingSecret { /// discovered through the control socket's `capabilities` op. #[derive(Debug, Clone, Copy, Default)] struct LiveControl { + /// Host understands root growth; the runtime separately preflights its guest. + root_disk_grow: bool, /// CPU and memory resize targets are served. resize: bool, @@ -132,56 +134,26 @@ impl SandboxModificationBuilder { self } - /// Set the desired effective guest memory. + /// Set the desired effective guest memory. Accepts a bare `u32` in MiB or a typed size. pub fn memory(mut self, size: impl Into) -> Self { self.patch.memory_mib = Some(size.into().as_u32()); self } - /// Set the desired effective guest memory in MiB. - pub fn memory_mib(mut self, memory_mib: u32) -> Self { - self.patch.memory_mib = Some(memory_mib); - self - } - - /// Set the desired boot-time maximum hotpluggable memory. + /// Set the boot-time maximum hotpluggable memory. Accepts a bare `u32` in MiB or a typed size. pub fn max_memory(mut self, size: impl Into) -> Self { self.patch.max_memory_mib = Some(size.into().as_u32()); self } - /// Set the desired boot-time maximum hotpluggable memory in MiB. - pub fn max_memory_mib(mut self, max_memory_mib: u32) -> Self { - self.patch.max_memory_mib = Some(max_memory_mib); - self - } - - /// Set the desired root disk size. Managed kind: grow-only (shrinking an - /// existing upper risks data loss and is rejected). Tmpfs kind: any - /// direction, effective next boot. Disk-image kind: rejected (user-owned). + /// Set the desired total root disk size, accepting a bare `u32` in MiB or a typed size. + /// Managed and flat roots are grow-only. Tmpfs changes take effect on the next boot; + /// user-owned disk images cannot be resized through this API. pub fn root_disk_size(mut self, size: impl Into) -> Self { self.patch.root_disk_size_mib = Some(size.into().as_u32()); self } - /// Set the desired root disk size in MiB. See [`root_disk_size`](Self::root_disk_size). - pub fn root_disk_size_mib(mut self, size_mib: u32) -> Self { - self.patch.root_disk_size_mib = Some(size_mib); - self - } - - /// Set the desired OCI writable overlay upper size. - #[deprecated(since = "0.6.0", note = "use `root_disk_size` instead")] - pub fn oci_upper_size(self, size: impl Into) -> Self { - self.root_disk_size(size) - } - - /// Set the desired OCI writable overlay upper size in MiB. - #[deprecated(since = "0.6.0", note = "use `root_disk_size_mib` instead")] - pub fn oci_upper_size_mib(self, size_mib: u32) -> Self { - self.root_disk_size_mib(size_mib) - } - /// Set an environment variable for future execs. pub fn env(mut self, key: impl Into, value: impl Into) -> Self { self.patch.env.push(EnvVar::new(key, value)); @@ -383,6 +355,33 @@ impl SandboxModificationBuilder { } } } + if running_status(status) + && !restart_required + && self.policy == ModificationPolicy::NoRestart + && let Some(target_mib) = root_disk_grow_target(&plan, &self.patch, &config) + { + let size_bytes = u64::from(target_mib) * 1024 * 1024; + let request = serde_json::to_string( + µsandbox_runtime::control::ControlRequest::RootDiskGrow { size_bytes }, + )? + "\n"; + let response = control_request(&self.name, request).await?; + let observed = response.root_disk.ok_or_else(|| { + crate::MicrosandboxError::Runtime("root growth reply missing capacity".into()) + })?; + if observed.filesystem_bytes != size_bytes || observed.device_bytes < size_bytes { + return Err(crate::MicrosandboxError::Runtime( + "root growth did not confirm usable capacity".into(), + )); + } + if let Some(active) = active.as_mut() { + let disk_patch = SandboxModificationPatch { + root_disk_size_mib: Some(target_mib), + ..Default::default() + }; + apply_patch_to_config(active, &disk_patch); + persist_active_config(&self.backend, &handle, active).await?; + } + } // Grow the real upper.ext4 before persisting the new desired size: // the persisted value may only ever claim capacity the file actually // has. A running sandbox under `--next-start` keeps its mounted upper @@ -513,6 +512,23 @@ fn build_plan( &mut warnings, ); push_root_disk_size_change(status, config, &patch, policy, &mut changes); + if live.root_disk_grow + && running_status(status) + && policy == ModificationPolicy::NoRestart + && matches!( + root_disk_size_state(config), + Some(RootDiskSizeState::Managed { .. }) + ) + { + for change in &mut changes { + if let PlannedChange::Config(change) = change + && change.field == ROOT_DISK_FIELD + { + change.disposition = ModificationDisposition::Live; + change.reason = None; + } + } + } push_spec_changes(status, config, &patch, policy, &mut changes, &mut warnings); push_secret_changes( status, @@ -610,7 +626,25 @@ async fn grow_root_disk_now( ) })?; let sandbox_dir = local_backend.sandboxes_dir().join(name); - refuse_checkpoint_backed_root_grow(&sandbox_dir)?; + let runtime_dir = sandbox_dir.join("runtime"); + let handled = tokio::task::spawn_blocking(move || { + microsandbox_runtime::checkpoint::grow_stopped_root( + &runtime_dir, + u64::from(target_mib) * 1024 * 1024, + ) + }) + .await + .map_err(|e| crate::MicrosandboxError::Runtime(e.to_string()))? + .map_err(crate::MicrosandboxError::Runtime)?; + if handled { + return Ok(()); + } + if !config.snapshot_upper_layers.is_empty() { + return Err(crate::MicrosandboxError::Runtime( + "start this restored sandbox once to initialize its owned root chain before resizing" + .into(), + )); + } if matches!( &config.spec.image, RootfsSource::Oci(oci) if matches!(&oci.root_disk, Some(RootDisk::Flat { .. })) @@ -624,29 +658,6 @@ async fn grow_root_disk_now( super::upper::grow_upper_to_mib(sandbox_dir.join("upper.ext4"), target_mib).await } -/// Refuse the legacy raw-file grow path once checkpoint rollover has installed a qcow2 head. -/// -/// A one-layer journal still names the ordinary mutable raw disk and is safe to grow in place. -/// With two or more layers, however, the raw file is a sealed ancestor and only a future -/// chain-aware resize may extend the active qcow2 head and filesystem. -fn refuse_checkpoint_backed_root_grow(sandbox_dir: &std::path::Path) -> MicrosandboxResult<()> { - let chain = microsandbox_runtime::checkpoint::load_runtime_owned_root_chain( - &sandbox_dir.join("runtime"), - ) - .map_err(|error| { - crate::MicrosandboxError::Runtime(format!( - "cannot inspect the root-disk chain before resize: {error}" - )) - })?; - if chain.is_some_and(|chain| chain.layers.len() > 1) { - return Err(crate::MicrosandboxError::Custom( - "cannot grow a checkpoint-backed root disk yet; its sealed raw ancestor must not be resized" - .into(), - )); - } - Ok(()) -} - /// Path of the sandbox's host-side runtime control socket. #[cfg(windows)] fn control_socket_path(name: &str) -> MicrosandboxResult { @@ -690,12 +701,14 @@ async fn live_control(name: &str, status: SandboxStatus) -> LiveControl { } match control_capabilities(name).await { Ok(caps) => LiveControl { + root_disk_grow: caps.root_disk_grow, resize: caps.cpu_resize || caps.memory_resize, secrets: caps.secrets_update, }, // Runtimes that predate the capabilities op served the socket only // when they could resize; live secret ops did not exist yet. Err(_) => LiveControl { + root_disk_grow: false, resize: true, secrets: false, }, @@ -2480,6 +2493,32 @@ mod tests { use super::*; use crate::backend::LocalBackend; + use crate::size::SizeExt; + + #[tokio::test] + async fn size_setters_accept_bare_mib_and_typed_sizes() { + let temp = tempdir().unwrap(); + let backend: Arc = Arc::new( + LocalBackend::builder() + .home(temp.path()) + .build() + .await + .unwrap(), + ); + let plain = SandboxModificationBuilder::new(backend.clone(), "size-api") + .memory(1024) + .max_memory(8192) + .root_disk_size(4096); + let typed = SandboxModificationBuilder::new(backend, "size-api") + .memory(1.gib()) + .max_memory(8.gib()) + .root_disk_size(4.gib()); + for patch in [&plain.patch, &typed.patch] { + assert_eq!(patch.memory_mib, Some(1024)); + assert_eq!(patch.max_memory_mib, Some(8192)); + assert_eq!(patch.root_disk_size_mib, Some(4096)); + } + } fn checkpoint_reply(ok: bool) -> microsandbox_runtime::control::ControlResponse { microsandbox_runtime::control::ControlResponse { @@ -2739,6 +2778,7 @@ mod tests { &desired, Some(&active), LiveControl { + root_disk_grow: false, resize: true, secrets: false, }, @@ -2776,6 +2816,7 @@ mod tests { &desired, Some(&active), LiveControl { + root_disk_grow: false, resize: live_memory_supported, secrets: false, }, @@ -2810,6 +2851,7 @@ mod tests { &config(2, 1024), Some(&active), LiveControl { + root_disk_grow: false, resize: true, secrets: false, }, @@ -3038,18 +3080,16 @@ mod tests { ); } - #[tokio::test] - async fn checkpoint_backed_root_grow_refuses_to_mutate_the_sealed_base() { + #[test] + fn invalid_checkpoint_backed_root_grow_does_not_mutate_the_sealed_base() { let sandbox = tempdir().unwrap(); let runtime = sandbox.path().join("runtime"); std::fs::create_dir(&runtime).unwrap(); let base = sandbox.path().join("rootfs.raw"); let head = sandbox.path().join("root-active.qcow2"); std::fs::write(&base, vec![0; 4096]).unwrap(); - // Exercise the grow guard with a valid head, not a truncated-header error. - microsandbox_image::checkpoint::create_qcow2_overlay(&head, 4096, &base, "raw") - .await - .unwrap(); + // Checkpoint chains can grow here; malformed heads must still fail before any write. + std::fs::write(&head, b"qcow").unwrap(); let head_before = std::fs::read(&head).unwrap(); let state = serde_json::json!({ "schema": "microsandbox.runtime-root-disk/1", @@ -3078,44 +3118,39 @@ mod tests { ) .unwrap(); - let error = refuse_checkpoint_backed_root_grow(sandbox.path()).unwrap_err(); - assert!(error.to_string().contains("checkpoint-backed root disk")); + let error = + microsandbox_runtime::checkpoint::grow_stopped_root(&runtime, 8192).unwrap_err(); + let expected = microsandbox_image::checkpoint::layer_capacities(vec![ + microsandbox_image::checkpoint::CompactLayer { + path: head.clone(), + qcow2: true, + }, + ]) + .unwrap_err(); + assert_eq!(error, expected.to_string()); assert_eq!(std::fs::read(&base).unwrap(), vec![0; 4096]); assert_eq!(std::fs::read(&head).unwrap(), head_before); assert_eq!( std::fs::read(runtime.join("root-disk.json")).unwrap(), serde_json::to_vec(&state).unwrap() ); - - std::fs::write(&head, b"qcow").unwrap(); - let error = refuse_checkpoint_backed_root_grow(sandbox.path()).unwrap_err(); - assert!( - error - .to_string() - .contains("cannot inspect the root-disk chain") - ); - assert_eq!(std::fs::read(&base).unwrap(), vec![0; 4096]); - assert_eq!(std::fs::read(&head).unwrap(), b"qcow"); - assert_eq!( - std::fs::read(runtime.join("root-disk.json")).unwrap(), - serde_json::to_vec(&state).unwrap() - ); } #[test] - fn running_upper_grow_is_restart_backed_never_live() { + fn old_runtime_upper_grow_requires_explicit_restart() { let patch = SandboxModificationPatch { root_disk_size_mib: Some(8192), ..SandboxModificationPatch::default() }; - // Even a resize-capable runtime cannot grow the mounted upper live. + // CPU/memory resize capability alone does not advertise root growth. let plan = build_plan( "api".to_string(), SandboxStatus::Running, &oci_config_with_upper(4096), None, LiveControl { + root_disk_grow: false, resize: true, secrets: true, }, @@ -3148,6 +3183,47 @@ mod tests { assert!(plan_requires_restart(&restart_plan)); } + #[test] + fn owned_root_growth_uses_live_capability_but_respects_explicit_policies() { + for root in [RootDisk::managed(512), RootDisk::flat(512)] { + let config = oci_config_with_root_disk(root); + for (policy, expected) in [ + (ModificationPolicy::NoRestart, ModificationDisposition::Live), + ( + ModificationPolicy::NextStart, + ModificationDisposition::NextStart, + ), + ( + ModificationPolicy::Restart, + ModificationDisposition::RequiresRestart, + ), + ] { + let plan = build_plan( + "grow".into(), + SandboxStatus::Running, + &config, + None, + LiveControl { + root_disk_grow: true, + resize: false, + secrets: false, + }, + SandboxModificationPatch { + root_disk_size_mib: Some(1024), + ..Default::default() + }, + policy, + ); + assert!(plan.conflicts.is_empty()); + let PlannedChange::Config(change) = &plan.changes[0] else { + panic!("expected disk change") + }; + assert_eq!(change.disposition, expected); + assert!(validate_apply_supported(&plan).is_ok()); + } + } + } + #[test] fn running_upper_grow_under_next_start_persists_desired_only() { let patch = SandboxModificationPatch { @@ -3788,6 +3864,7 @@ mod tests { &config, None, LiveControl { + root_disk_grow: false, resize: false, secrets: true, }, @@ -3813,6 +3890,7 @@ mod tests { &config, None, LiveControl { + root_disk_grow: false, resize: false, secrets: true, }, @@ -3844,6 +3922,7 @@ mod tests { &config, None, LiveControl { + root_disk_grow: false, resize: false, secrets: true, }, @@ -3879,6 +3958,7 @@ mod tests { &config, None, LiveControl { + root_disk_grow: false, resize: false, secrets: true, }, @@ -4256,6 +4336,7 @@ mod tests { &config, None, LiveControl { + root_disk_grow: false, resize: false, secrets: true, }, @@ -4324,6 +4405,7 @@ mod tests { &config, None, LiveControl { + root_disk_grow: false, resize: false, secrets: true, }, @@ -4374,6 +4456,7 @@ mod tests { &config, None, LiveControl { + root_disk_grow: false, resize: false, secrets: true, }, @@ -4394,6 +4477,7 @@ mod tests { &config, None, LiveControl { + root_disk_grow: false, resize: false, secrets: true, }, diff --git a/sdk/rust/lib/snapshot/create.rs b/sdk/rust/lib/snapshot/create.rs index 65ed984b4..81d488309 100644 --- a/sdk/rust/lib/snapshot/create.rs +++ b/sdk/rust/lib/snapshot/create.rs @@ -930,15 +930,25 @@ fn new_file_manifest_with_id( .last() .expect("checked non-empty layer inputs") .1; + let capacities = microsandbox_image::checkpoint::layer_capacities( + disk.sources + .iter() + .map(|source| microsandbox_image::checkpoint::CompactLayer { + path: source.path.clone(), + qcow2: source.format == SnapshotFormat::Qcow2, + }) + .collect(), + )?; let mut predecessor = None; let layers = layer_inputs .into_iter() - .map(|(layer_id, format, integrity)| { + .zip(capacities) + .map(|((layer_id, format, integrity), virtual_size)| { let backing = predecessor.replace(layer_id.clone()); DiskLayer { layer_id, format, - virtual_size: disk.virtual_size, + virtual_size, backing, payload: LayerPayload { file_kind: LayerFileKind::Regular,