diff --git a/rs/moq-video/src/capture/mod.rs b/rs/moq-video/src/capture/mod.rs index a594d2d5b..6b0dc8169 100644 --- a/rs/moq-video/src/capture/mod.rs +++ b/rs/moq-video/src/capture/mod.rs @@ -286,6 +286,11 @@ impl FrameStream { self.framerate } + /// The first frame's declared color space, when its capture backend knows it. + pub(crate) fn color(&self) -> Option { + self.pending.as_ref().and_then(Surface::color) + } + pub(crate) fn device(&self) -> &str { &self.device } diff --git a/rs/moq-video/src/capture/pipewire.rs b/rs/moq-video/src/capture/pipewire.rs index f10caa71c..72eeaf192 100644 --- a/rs/moq-video/src/capture/pipewire.rs +++ b/rs/moq-video/src/capture/pipewire.rs @@ -3,8 +3,9 @@ //! The ScreenCast portal owns source selection: [`open`] pops the compositor's //! picker dialog, the user chooses a monitor, and the portal hands us a PipeWire //! fd + node id. A dedicated thread then runs the PipeWire main loop, converting -//! each RGB frame to CPU [`I420`] and pushing it into the shared [`FrameChannel`] -//! (callback-driven like the macOS delegate, not a pull-style pump). +//! each packed RGB or NV12 frame to CPU [`I420`] and pushing it into the shared +//! [`FrameChannel`] (callback-driven like the macOS delegate, not a pull-style +//! pump). //! //! Two quirks worth knowing: //! - `publish_capture` releases the capture while unwatched and reopens it on @@ -18,6 +19,7 @@ //! the encoder. A loop timer re-emits the last frame whenever a frame interval //! passes without a fresh one, mirroring the Windows Desktop Duplication pacing. +use std::borrow::Cow; use std::cell::RefCell; use std::os::fd::OwnedFd; use std::rc::Rc; @@ -34,8 +36,8 @@ use spa::param::video::{VideoFormat, VideoInfoRaw}; use super::channel::FrameChannel; use super::pump::Geometry; use super::{Config, FrameStream}; -use crate::Error; use crate::frame::{I420, Surface}; +use crate::{Color, Error, Size}; const DEFAULT_FRAMERATE: u32 = 30; /// The compositor sends the negotiated format right after the stream connects; @@ -75,6 +77,7 @@ pub(super) async fn open(config: &Config, device: Option<&str>) -> Result, + /// The NV12 color space used to configure the encoder. + color: Option, geo_tx: Option>>, /// Most recent converted frame, re-emitted while the screen is static. last: Option, @@ -322,7 +326,7 @@ fn run_loop( } let mut state = state.borrow_mut(); - if let Err(e) = state.format.parse(param) { + if let Err(e) = replace_video_format(&mut state.format, |format| format.parse(param)) { tracing::warn!(error = %e, "failed to parse pipewire video format"); return; } @@ -335,9 +339,25 @@ fn run_loop( tracing::warn!(width = size.width, height = size.height, "unusable capture size"); return; } + let color = match pipewire_color(state.format, width, height) { + Ok(color) => color, + Err(e) => { + match state.geo_tx.take() { + Some(tx) => drop(tx.send(Err(e))), + None => { + tracing::warn!(error = %e, "unsupported pipewire video color space"); + if let Some(mainloop) = mainloop.upgrade() { + mainloop.quit(); + } + } + } + return; + } + }; if let Some(tx) = state.geo_tx.take() { state.geometry = Some((width, height)); + state.color = color; // The compositor reports 0/1 for a variable rate; only a real // rate is worth forwarding to the encoder. let fr = state.format.framerate(); @@ -348,11 +368,10 @@ fn run_loop( framerate, device: format!("pipewire:{node_id}"), })); - } else if state.geometry != Some((width, height)) { - // Renegotiated to a new size (e.g. the monitor changed mode). - // End the stream; the encode loop reopens at the new geometry - // and the restore token skips the picker. - tracing::info!(width, height, "capture size changed; restarting the stream"); + } else if format_requires_restart(state.geometry, state.color, width, height, color) { + // The encoder's geometry and VUI are fixed when it opens. End the + // stream so the encode loop reopens it with the new format. + tracing::info!(width, height, ?color, "capture format changed; restarting the stream"); if let Some(mainloop) = mainloop.upgrade() { mainloop.quit(); } @@ -366,20 +385,66 @@ fn run_loop( move |stream, _| { let mut state = state.borrow_mut(); let Some((width, height)) = state.geometry else { return }; + let source_height = state.format.size().height; + let color = state.color; let Some(mut buffer) = stream.dequeue_buffer() else { return; }; let datas = buffer.datas_mut(); let Some(data) = datas.first_mut() else { return }; - let offset = data.chunk().offset() as usize; - let size = data.chunk().size() as usize; - let stride = data.chunk().stride(); - // Compositors mark skipped frames as empty or corrupted; drop those - // rather than treating them as a fatal conversion failure below. - if size == 0 || data.chunk().flags().contains(spa::buffer::ChunkFlags::CORRUPTED) { + let maxsize = data.as_raw().maxsize; + let size = clamp_chunk_size(data.chunk().size(), maxsize); + match chunk_kind(size, data.chunk().flags()) { + ChunkKind::Data => {} + ChunkKind::Empty => { + let Ok(frame) = neutral_frame(width, height, color) else { + return; + }; + chan.push(Surface::I420(frame.clone())); + state.last = Some(frame); + state.fresh = true; + return; + } + ChunkKind::Invalid => return, + } + let Some(offset) = normalize_chunk_offset(data.chunk().offset(), maxsize) else { + tracing::warn!("pipewire buffer has zero maximum size"); + return; + }; + // Fall back to the unclamped source width: for an odd-width source + // the real row is one pixel wider than the clamped `width`. The + // packing differs per format, so NV12 cannot assume 4 bytes/pixel. + let stride = match u32::try_from(data.chunk().stride()) { + Ok(stride) if stride > 0 => stride, + _ => match state.format.format() { + VideoFormat::NV12 => state.format.size().width, + _ => state.format.size().width.saturating_mul(4), + }, + }; + let layout = FrameLayout { + stride, + width, + height, + source_height, + }; + + // The chunk only says how many bytes the producer wrote. Check it + // actually spans every row `convert` will sample, so a short or + // mislabeled buffer is dropped here rather than read past. + let Some(required) = frame_data_size(state.format.format(), layout) else { + tracing::warn!("pipewire frame layout overflows its buffer"); + return; + }; + if required > size || required > maxsize as usize { + tracing::warn!( + required, + available = size, + "pipewire chunk does not contain a complete frame" + ); return; } + // Without dmabuf modifiers in our format offer the compositor uses // shared memory, which MAP_BUFFERS mmaps for us; `None` here means // it forced something we can't read, so give up cleanly. @@ -390,18 +455,14 @@ fn run_loop( } return; }; - let Some(bytes) = bytes.get(offset..offset + size) else { + let Some(allocation) = bytes.get(..maxsize as usize) else { return; }; - // Fall back to the unclamped source width: for an odd-width source - // the real row is one pixel wider than the clamped `width`. - let stride = if stride > 0 { - stride as u32 - } else { - state.format.size().width * 4 + let Some(bytes) = chunk_bytes(allocation, offset, required) else { + return; }; - match convert(state.format.format(), bytes, stride, width, height) { + match convert(state.format.format(), bytes.as_ref(), layout, color) { Ok(i420) => { chan.push(Surface::I420(i420.clone())); state.last = Some(i420); @@ -470,19 +531,262 @@ fn run_loop( Ok(()) } -/// Convert one strided RGB screen frame to tightly-packed I420. -fn convert(format: VideoFormat, bytes: &[u8], stride: u32, width: u32, height: u32) -> Result { +/// The geometry `convert` needs: the producer's row stride and source height, +/// plus the even-clamped output size. +#[derive(Clone, Copy)] +struct FrameLayout { + stride: u32, + width: u32, + height: u32, + /// Unclamped source height. NV12's chroma plane starts after this many luma + /// rows, which is one more than `height` for an odd-height source. + source_height: u32, +} + +const CHUNK_FLAG_EMPTY: i32 = 1 << 1; + +#[derive(Debug, PartialEq, Eq)] +enum ChunkKind { + Data, + Empty, + Invalid, +} + +fn chunk_kind(size: usize, flags: spa::buffer::ChunkFlags) -> ChunkKind { + if flags.contains(spa::buffer::ChunkFlags::CORRUPTED) { + ChunkKind::Invalid + } else if flags.bits() & CHUNK_FLAG_EMPTY != 0 { + ChunkKind::Empty + } else if size == 0 { + ChunkKind::Invalid + } else { + ChunkKind::Data + } +} + +fn neutral_frame(width: u32, height: u32, color: Option) -> Result { + let color = color.unwrap_or_else(|| Color::infer(Size::new(width, height))); + let luma = width as usize * height as usize; + let mut data = vec![128; I420::len(width, height)]; + data[..luma].fill(if color.limited() { 16 } else { 0 }); + Ok(I420::new(width, height, data)?.with_color(color)) +} + +/// Parse into a zeroed value before replacing the current format. libspa leaves +/// omitted optional properties untouched, so parsing into the reused value +/// would retain stale color metadata across renegotiation. +fn replace_video_format( + current: &mut VideoInfoRaw, + parse: impl FnOnce(&mut VideoInfoRaw) -> Result, +) -> Result { + let mut next = VideoInfoRaw::default(); + let result = parse(&mut next)?; + *current = next; + Ok(result) +} + +/// Preserve the color description that names NV12 samples. Unknown fields use +/// the same size-based, limited-range fallback as the encoder. Reject matrices +/// the crate cannot represent rather than writing a false 601/709 VUI. +fn pipewire_color(format: VideoInfoRaw, width: u32, height: u32) -> Result, Error> { + if format.format() != VideoFormat::NV12 { + return Ok(None); + } + let size = Size::new(width, height); + let color = color_from_pipewire(format.color_range(), format.color_matrix(), size)?; + validate_pipewire_description( + color.unwrap_or_else(|| Color::infer(size)), + format.color_primaries(), + format.transfer_function(), + )?; + Ok(color) +} + +fn color_from_pipewire(range: u32, matrix: u32, size: Size) -> Result, Error> { + if range == spa::sys::SPA_VIDEO_COLOR_RANGE_UNKNOWN && matrix == spa::sys::SPA_VIDEO_COLOR_MATRIX_UNKNOWN { + return Ok(None); + } + + let limited = match range { + spa::sys::SPA_VIDEO_COLOR_RANGE_UNKNOWN | spa::sys::SPA_VIDEO_COLOR_RANGE_16_235 => true, + spa::sys::SPA_VIDEO_COLOR_RANGE_0_255 => false, + _ => { + return Err(Error::Codec(anyhow::anyhow!( + "unsupported PipeWire NV12 color range {range}" + ))); + } + }; + let bt709 = match matrix { + spa::sys::SPA_VIDEO_COLOR_MATRIX_UNKNOWN => { + matches!(Color::infer(size), Color::Bt709Limited | Color::Bt709Full) + } + spa::sys::SPA_VIDEO_COLOR_MATRIX_BT709 => true, + spa::sys::SPA_VIDEO_COLOR_MATRIX_BT601 => false, + _ => { + return Err(Error::Codec(anyhow::anyhow!( + "unsupported PipeWire NV12 color matrix {matrix}" + ))); + } + }; + + Ok(Some(match (bt709, limited) { + (false, true) => Color::Bt601Limited, + (false, false) => Color::Bt601Full, + (true, true) => Color::Bt709Limited, + (true, false) => Color::Bt709Full, + })) +} + +fn validate_pipewire_description(color: Color, primaries: u32, transfer: u32) -> Result<(), Error> { + let expected_primaries = match color { + Color::Bt601Limited | Color::Bt601Full => spa::sys::SPA_VIDEO_COLOR_PRIMARIES_SMPTE170M, + Color::Bt709Limited | Color::Bt709Full => spa::sys::SPA_VIDEO_COLOR_PRIMARIES_BT709, + }; + if primaries != spa::sys::SPA_VIDEO_COLOR_PRIMARIES_UNKNOWN && primaries != expected_primaries { + return Err(Error::Codec(anyhow::anyhow!( + "PipeWire NV12 primaries {primaries} do not match the negotiated matrix" + ))); + } + if !matches!( + transfer, + spa::sys::SPA_VIDEO_TRANSFER_UNKNOWN + | spa::sys::SPA_VIDEO_TRANSFER_BT709 + | spa::sys::SPA_VIDEO_TRANSFER_BT601 + | spa::sys::SPA_VIDEO_TRANSFER_BT2020_10 + ) { + return Err(Error::Codec(anyhow::anyhow!( + "unsupported PipeWire NV12 transfer function {transfer}" + ))); + } + Ok(()) +} + +fn format_requires_restart( + geometry: Option<(u32, u32)>, + color: Option, + width: u32, + height: u32, + next_color: Option, +) -> bool { + geometry.is_some_and(|geometry| geometry != (width, height) || color != next_color) +} + +/// A chunk offset is a ring position within the allocation, so wrap it rather +/// than trusting it to be in range. `None` means the allocation is unusable. +fn normalize_chunk_offset(offset: u32, maxsize: u32) -> Option { + (maxsize != 0).then(|| (offset % maxsize) as usize) +} + +fn clamp_chunk_size(size: u32, maxsize: u32) -> usize { + size.min(maxsize) as usize +} + +fn chunk_bytes(data: &[u8], offset: usize, size: usize) -> Option> { + if offset >= data.len() || size > data.len() { + return None; + } + let end = offset.checked_add(size)?; + if end <= data.len() { + return Some(Cow::Borrowed(&data[offset..end])); + } + + let mut wrapped = Vec::with_capacity(size); + wrapped.extend_from_slice(&data[offset..]); + let head = size - wrapped.len(); + wrapped.extend_from_slice(&data[..head]); + Some(Cow::Owned(wrapped)) +} + +/// Bytes from the chunk start through the span required by `convert`. NV12 stops +/// at the visible width of its final row; packed RGB requires every full stride. +fn frame_data_size(format: VideoFormat, layout: FrameLayout) -> Option { + let stride = layout.stride as usize; + let width = layout.width as usize; + let height = layout.height as usize; + let row_size = match format { + VideoFormat::NV12 => width, + VideoFormat::BGRx | VideoFormat::BGRA | VideoFormat::RGBx | VideoFormat::RGBA => width.checked_mul(4)?, + _ => return None, + }; + if stride < row_size { + return None; + } + + match format { + // Chroma follows every source luma row, then runs at half height. + VideoFormat::NV12 => (layout.source_height as usize) + .checked_add(height / 2)? + .checked_sub(1)? + .checked_mul(stride)? + .checked_add(row_size), + _ => stride.checked_mul(height), + } +} + +/// Deinterleave strided NV12 into the crate's tightly packed I420 layout. +fn nv12_to_i420(data: &[u8], layout: FrameLayout) -> Result { + let (stride, width, height, source_height) = ( + layout.stride as usize, + layout.width as usize, + layout.height as usize, + layout.source_height as usize, + ); + if source_height < height { + return Err(Error::Codec(anyhow::anyhow!( + "NV12 source is shorter than the cropped output" + ))); + } + let y_len = stride + .checked_mul(source_height) + .ok_or_else(|| Error::Codec(anyhow::anyhow!("NV12 luma size overflow")))?; + let uv_rows = height / 2; + let uv_len = uv_rows + .checked_sub(1) + .and_then(|rows| rows.checked_mul(stride)) + .and_then(|offset| offset.checked_add(width)) + .ok_or_else(|| Error::Codec(anyhow::anyhow!("NV12 chroma size overflow")))?; + let frame_len = y_len + .checked_add(uv_len) + .ok_or_else(|| Error::Codec(anyhow::anyhow!("NV12 frame size overflow")))?; + if stride < width || data.len() < frame_len { + return Err(Error::Codec(anyhow::anyhow!( + "NV12 frame is shorter than its declared rows" + ))); + } + + let mut packed = vec![0; I420::len(width as u32, height as u32)]; + for row in 0..height { + packed[row * width..(row + 1) * width].copy_from_slice(&data[row * stride..row * stride + width]); + } + let uv = &data[y_len..frame_len]; + let packed_uv = width * height; + for row in 0..uv_rows { + packed[packed_uv + row * width..packed_uv + (row + 1) * width] + .copy_from_slice(&uv[row * stride..row * stride + width]); + } + I420::from_nv12(&packed, width as u32, height as u32) +} + +/// Convert one strided screen frame to tightly-packed I420. +fn convert(format: VideoFormat, bytes: &[u8], layout: FrameLayout, color: Option) -> Result { match format { - VideoFormat::BGRx | VideoFormat::BGRA => I420::from_bgra(bytes, stride, width, height), - VideoFormat::RGBx | VideoFormat::RGBA => I420::from_rgba(bytes, stride, width, height), + VideoFormat::NV12 => { + let frame = nv12_to_i420(bytes, layout)?; + Ok(match color { + Some(color) => frame.with_color(color), + None => frame, + }) + } + VideoFormat::BGRx | VideoFormat::BGRA => I420::from_bgra(bytes, layout.stride, layout.width, layout.height), + VideoFormat::RGBx | VideoFormat::RGBA => I420::from_rgba(bytes, layout.stride, layout.width, layout.height), other => Err(Error::Codec(anyhow::anyhow!( "pipewire negotiated an unsupported video format {other:?}" ))), } } -/// Serialize the `EnumFormat` pod offering the RGB layouts we can convert, -/// any size, and a framerate range preferring `framerate`. +/// Serialize the `EnumFormat` pod offering the layouts we can convert (packed +/// RGB first, then NV12), any size, and a framerate range preferring `framerate`. fn format_offer(framerate: u32) -> Vec { let obj = spa::pod::object!( spa::utils::SpaTypes::ObjectParamFormat, @@ -507,6 +811,7 @@ fn format_offer(framerate: u32) -> Vec { VideoFormat::BGRA, VideoFormat::RGBx, VideoFormat::RGBA, + VideoFormat::NV12, ), spa::pod::property!( spa::param::format::FormatProperties::VideoSize, @@ -547,11 +852,267 @@ mod tests { use super::*; use crate::capture::Config; - /// The serialized format offer must parse back as a valid pod. + /// The serialized format offer must parse back as a valid pod, carrying every + /// layout `convert` knows how to handle. #[test] fn format_offer_is_valid_pod() { let bytes = format_offer(30); - assert!(spa::pod::Pod::from_bytes(&bytes).is_some(), "offer did not round-trip"); + let (remaining, value) = spa::pod::deserialize::PodDeserializer::deserialize_any_from(&bytes) + .expect("format offer did not round-trip"); + assert!(remaining.is_empty()); + let spa::pod::Value::Object(object) = value else { + panic!("format offer is not an object"); + }; + let property = object + .properties + .iter() + .find(|property| property.key == spa::param::format::FormatProperties::VideoFormat.as_raw()) + .expect("missing video format property"); + assert_eq!( + property.value, + spa::pod::Value::Choice(spa::pod::ChoiceValue::Id(spa::utils::Choice( + spa::utils::ChoiceFlags::empty(), + spa::utils::ChoiceEnum::Enum { + default: spa::utils::Id(VideoFormat::BGRx.as_raw()), + alternatives: vec![ + spa::utils::Id(VideoFormat::BGRx.as_raw()), + spa::utils::Id(VideoFormat::BGRA.as_raw()), + spa::utils::Id(VideoFormat::RGBx.as_raw()), + spa::utils::Id(VideoFormat::RGBA.as_raw()), + spa::utils::Id(VideoFormat::NV12.as_raw()), + ], + }, + ))) + ); + } + + #[test] + fn chunk_offset_wraps_to_the_allocation() { + assert_eq!(normalize_chunk_offset(18, 16), Some(2)); + assert_eq!(normalize_chunk_offset(0, 0), None); + } + + #[test] + fn chunk_size_is_clamped_to_the_allocation() { + assert_eq!(clamp_chunk_size(18, 16), 16); + assert_eq!(clamp_chunk_size(8, 16), 8); + } + + #[test] + fn chunk_flags_distinguish_neutral_and_invalid_frames() { + let empty = spa::buffer::ChunkFlags::from_bits_retain(CHUNK_FLAG_EMPTY); + assert_eq!(chunk_kind(0, spa::buffer::ChunkFlags::empty()), ChunkKind::Invalid); + assert_eq!(chunk_kind(1, spa::buffer::ChunkFlags::CORRUPTED), ChunkKind::Invalid); + assert_eq!(chunk_kind(1, empty), ChunkKind::Empty); + assert_eq!(chunk_kind(0, empty), ChunkKind::Empty); + assert_eq!(chunk_kind(1, spa::buffer::ChunkFlags::empty()), ChunkKind::Data); + } + + #[test] + fn empty_chunk_is_limited_range_black() { + let frame = neutral_frame(4, 2, None).unwrap(); + assert_eq!(frame.y(), &[16; 8]); + assert_eq!(frame.u(), &[128; 2]); + assert_eq!(frame.v(), &[128; 2]); + } + + #[test] + fn negotiated_nv12_color_overrides_size_inference() { + let size = Size::new(1920, 1080); + assert_eq!( + color_from_pipewire( + spa::sys::SPA_VIDEO_COLOR_RANGE_0_255, + spa::sys::SPA_VIDEO_COLOR_MATRIX_BT601, + size, + ) + .unwrap(), + Some(Color::Bt601Full) + ); + assert_eq!( + color_from_pipewire( + spa::sys::SPA_VIDEO_COLOR_RANGE_UNKNOWN, + spa::sys::SPA_VIDEO_COLOR_MATRIX_UNKNOWN, + size, + ) + .unwrap(), + None + ); + assert!( + color_from_pipewire( + spa::sys::SPA_VIDEO_COLOR_RANGE_16_235, + spa::sys::SPA_VIDEO_COLOR_MATRIX_BT2020, + size, + ) + .is_err() + ); + let mut format = VideoInfoRaw::default(); + format.set_format(VideoFormat::NV12); + format.set_color_range(spa::sys::SPA_VIDEO_COLOR_RANGE_16_235); + format.set_color_matrix(spa::sys::SPA_VIDEO_COLOR_MATRIX_BT709); + format.set_color_primaries(spa::sys::SPA_VIDEO_COLOR_PRIMARIES_BT2020); + format.set_transfer_function(spa::sys::SPA_VIDEO_TRANSFER_BT709); + assert!(pipewire_color(format, 1920, 1080).is_err()); + format.set_color_range(spa::sys::SPA_VIDEO_COLOR_RANGE_UNKNOWN); + format.set_color_matrix(spa::sys::SPA_VIDEO_COLOR_MATRIX_UNKNOWN); + format.set_transfer_function(spa::sys::SPA_VIDEO_TRANSFER_SMPTE2084); + assert!(pipewire_color(format, 1920, 1080).is_err()); + + let layout = FrameLayout { + stride: 4, + width: 4, + height: 2, + source_height: 2, + }; + let frame = convert( + VideoFormat::NV12, + &[16, 16, 16, 16, 16, 16, 16, 16, 128, 128, 128, 128], + layout, + Some(Color::Bt601Full), + ) + .unwrap(); + assert_eq!(frame.color(), Some(Color::Bt601Full)); + } + + #[test] + fn color_renegotiation_restarts_the_capture() { + let geometry = Some((1920, 1080)); + assert!(!format_requires_restart( + geometry, + Some(Color::Bt709Limited), + 1920, + 1080, + Some(Color::Bt709Limited), + )); + assert!(format_requires_restart( + geometry, + Some(Color::Bt709Limited), + 1920, + 1080, + Some(Color::Bt709Full), + )); + } + + #[test] + fn omitted_color_fields_reset_on_renegotiation() { + let mut format = VideoInfoRaw::default(); + format.set_color_range(spa::sys::SPA_VIDEO_COLOR_RANGE_0_255); + format.set_color_matrix(spa::sys::SPA_VIDEO_COLOR_MATRIX_BT709); + replace_video_format(&mut format, |next| { + next.set_format(VideoFormat::NV12); + Ok::<_, std::convert::Infallible>(()) + }) + .unwrap(); + assert_eq!(format.color_range(), spa::sys::SPA_VIDEO_COLOR_RANGE_UNKNOWN); + assert_eq!(format.color_matrix(), spa::sys::SPA_VIDEO_COLOR_MATRIX_UNKNOWN); + } + + #[test] + fn empty_chunk_uses_full_range_black() { + let frame = neutral_frame(4, 2, Some(Color::Bt709Full)).unwrap(); + assert_eq!(frame.y(), &[0; 8]); + assert_eq!(frame.u(), &[128; 2]); + assert_eq!(frame.v(), &[128; 2]); + assert_eq!(frame.color(), Some(Color::Bt709Full)); + } + + /// The required size must reach the last sampled row, but not the padding + /// past it, or a tightly-sized final row would be rejected. + #[test] + fn frame_data_size_covers_every_sampled_row() { + let packed = FrameLayout { + stride: 20, + width: 4, + height: 2, + source_height: 2, + }; + assert_eq!(frame_data_size(VideoFormat::BGRx, packed), Some(40)); + + let nv12 = FrameLayout { + stride: 6, + width: 4, + height: 2, + source_height: 3, + }; + assert_eq!(frame_data_size(VideoFormat::NV12, nv12), Some(22)); + + // A stride narrower than one row means the layout is nonsense. + assert_eq!( + frame_data_size(VideoFormat::BGRx, FrameLayout { stride: 15, ..packed }), + None + ); + } + + #[test] + fn wrapped_chunk_is_reassembled() { + let data = [0, 1, 2, 3, 4, 5]; + assert_eq!(chunk_bytes(&data, 1, 3).as_deref(), Some([1, 2, 3].as_slice())); + assert_eq!(chunk_bytes(&data, 4, 4).as_deref(), Some([4, 5, 0, 1].as_slice())); + } + + #[test] + fn nv12_stride_is_removed_and_chroma_is_deinterleaved() { + let data = [ + 1, 2, 3, 4, 99, 99, // Y row 0 plus padding + 5, 6, 7, 8, 99, 99, // Y row 1 plus padding + 9, 10, 11, 12, 99, 99, // UV row plus padding + ]; + let frame = nv12_to_i420( + &data, + FrameLayout { + stride: 6, + width: 4, + height: 2, + source_height: 2, + }, + ) + .unwrap(); + assert_eq!(frame.y(), &[1, 2, 3, 4, 5, 6, 7, 8]); + assert_eq!(frame.u(), &[9, 11]); + assert_eq!(frame.v(), &[10, 12]); + } + + #[test] + fn nv12_accepts_a_width_precise_final_row() { + let layout = FrameLayout { + stride: 6, + width: 4, + height: 2, + source_height: 2, + }; + let mut data = vec![99; frame_data_size(VideoFormat::NV12, layout).unwrap()]; + data[..4].copy_from_slice(&[1, 2, 3, 4]); + data[6..10].copy_from_slice(&[5, 6, 7, 8]); + data[12..16].copy_from_slice(&[9, 10, 11, 12]); + + let frame = nv12_to_i420(&data, layout).unwrap(); + assert_eq!(frame.y(), &[1, 2, 3, 4, 5, 6, 7, 8]); + assert_eq!(frame.u(), &[9, 11]); + assert_eq!(frame.v(), &[10, 12]); + } + + /// An odd-height source has one more luma row than the clamped output, and + /// chroma starts after all of them. + #[test] + fn nv12_crop_uses_the_source_height_for_chroma() { + let data = [ + 1, 2, 3, 4, // Y row 0 + 5, 6, 7, 8, // Y row 1 + 99, 99, 99, 99, // cropped Y row 2 + 9, 10, 11, 12, // UV row 0 + ]; + let frame = nv12_to_i420( + &data, + FrameLayout { + stride: 4, + width: 4, + height: 2, + source_height: 3, + }, + ) + .unwrap(); + assert_eq!(frame.y(), &[1, 2, 3, 4, 5, 6, 7, 8]); + assert_eq!(frame.u(), &[9, 11]); + assert_eq!(frame.v(), &[10, 12]); } /// Open the portal, grab a few frames, and check geometry. Ignored because it diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index 493d530d1..251662c7e 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -285,6 +285,7 @@ pub async fn publish_capture( probe_config.bitrate = encode.bitrate; probe_config.codec = encode.codec; probe_config.kind = encode.kind.clone(); + probe_config.color = camera.color(); probe_config.probe().await? }; @@ -431,6 +432,7 @@ async fn capture_loop( encoder_config.bitrate = encode.bitrate; encoder_config.codec = encode.codec; encoder_config.kind = encode.kind.clone(); + encoder_config.color = camera.color(); // Off macOS this opens the encoder on a dedicated thread; see `sink`. let mut encoder = Sink::open(&encoder_config).await?; // Force an IDR on the first frame of each (re)open so a viewer subscribing diff --git a/rs/moq-video/src/frame.rs b/rs/moq-video/src/frame.rs index 06f3a5e15..6627a1873 100644 --- a/rs/moq-video/src/frame.rs +++ b/rs/moq-video/src/frame.rs @@ -489,9 +489,9 @@ impl I420 { /// Split tightly-packed NV12 (Y plane `width * height`, then interleaved UV /// `width/2 * height/2` pairs) into planar I420. A chroma deinterleave, no - /// color-space conversion. Used for the Windows Media Foundation capture path, - /// whose source reader hands us NV12. - #[cfg(target_os = "windows")] + /// color-space conversion. Used by the Windows Media Foundation and Linux + /// PipeWire capture paths. + #[cfg(any(target_os = "windows", all(target_os = "linux", feature = "pipewire")))] pub(crate) fn from_nv12(nv12: &[u8], width: u32, height: u32) -> Result { let (w, h) = (width as usize, height as usize); let luma = w * h; @@ -631,7 +631,7 @@ pub(crate) fn interleave_uv(u: &[u8], v: &[u8], uv: &mut [u8]) { /// Split a packed NV12 chroma plane into separate U and V planes, the inverse of /// [`interleave_uv`]. -#[cfg(target_os = "windows")] +#[cfg(any(target_os = "windows", all(target_os = "linux", feature = "pipewire")))] pub(crate) fn deinterleave_uv(uv: &[u8], u: &mut [u8], v: &mut [u8]) { for (pair, (u, v)) in uv.chunks_exact(2).zip(u.iter_mut().zip(v)) { *u = pair[0];