diff --git a/CHANGELOG.md b/CHANGELOG.md index 630879c7b..8b12e9d76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,10 +11,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/). updates automatically** setting. Turning it off keeps notification and DMG verification active while deferring local package builds until an explicit **Check for updates**. -- The embedded Computer Use backend is synchronized to standalone v0.4.7 as - `0.4.7-linux-alpha1`, including generic X11/EWMH window control, X11 - `xdotool` keyboard, text, and coordinate-click input, KDE portal scroll - polarity, and portal key chords, with generic X11 registered last. +- The embedded Computer Use backend is synchronized to standalone v0.4.9 as + `0.4.9-linux-alpha1`, including generic X11/EWMH window control, deep GTK4 + accessibility traversal, bounded queue and child-read work, X11 `xdotool` + keyboard, text, and coordinate-click input, KDE portal scroll polarity, and + portal key chords, with generic X11 registered last. - A shared upstream DMG acceptance profile now produces the same structured decision for local installs, updater rebuilds, and scheduled CI. Scheduled rejections create one fingerprinted drift issue and supersede issues for diff --git a/Cargo.lock b/Cargo.lock index 72a7b5120..e3936a160 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -490,7 +490,7 @@ dependencies = [ [[package]] name = "codex-computer-use-linux" -version = "0.4.7-linux-alpha1" +version = "0.4.9-linux-alpha1" dependencies = [ "anyhow", "atspi", diff --git a/computer-use-linux/Cargo.toml b/computer-use-linux/Cargo.toml index 175c4635e..5f8b7bdcb 100644 --- a/computer-use-linux/Cargo.toml +++ b/computer-use-linux/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codex-computer-use-linux" -version = "0.4.7-linux-alpha1" +version = "0.4.9-linux-alpha1" edition = "2021" [[bin]] diff --git a/computer-use-linux/src/atspi_tree.rs b/computer-use-linux/src/atspi_tree.rs index b6277d7c2..c835cdd3b 100644 --- a/computer-use-linux/src/atspi_tree.rs +++ b/computer-use-linux/src/atspi_tree.rs @@ -10,9 +10,11 @@ use atspi::{ // Direct dependency (p2p feature off) — see Cargo.toml for why we bypass // atspi's "connection" re-export. use atspi_connection::AccessibilityConnection; +use futures_util::{stream, StreamExt}; use schemars::JsonSchema; use serde::Serialize; -use std::collections::VecDeque; +use std::{collections::VecDeque, future::Future, time::Duration}; +use tokio::time::timeout; use zbus::{ fdo::DBusProxy, names::{BusName, UniqueName}, @@ -102,10 +104,136 @@ pub enum ValueSetInvocation { const MAX_TEXT_READBACK_CHARS: i32 = 4096; const MAX_TEXT_SELECTIONS: i32 = 8; +const DEFAULT_SNAPSHOT_MAX_NODES: usize = 1_000; +const HARD_SNAPSHOT_MAX_NODES: usize = 2_000; +const DEFAULT_SNAPSHOT_MAX_DEPTH: u32 = 32; +const HARD_SNAPSHOT_MAX_DEPTH: u32 = 64; +const CHILD_READ_CONCURRENCY: usize = 16; +const SNAPSHOT_TIMEOUT: Duration = Duration::from_secs(10); +const MAX_DISCOVERY_ROOTS: usize = 256; +const ROOT_MATCH_CHILD_LIMIT: usize = 8; +const MAX_DISCOVERY_CHILD_READS: usize = MAX_DISCOVERY_ROOTS * ROOT_MATCH_CHILD_LIMIT; + +fn snapshot_child_read_budgets(max_nodes: usize) -> (usize, usize, usize) { + (MAX_DISCOVERY_ROOTS, MAX_DISCOVERY_CHILD_READS, max_nodes) +} + +pub(crate) fn snapshot_limits( + requested_max_nodes: Option, + requested_max_depth: Option, +) -> (usize, u32) { + ( + requested_max_nodes + .unwrap_or(DEFAULT_SNAPSHOT_MAX_NODES) + .clamp(1, HARD_SNAPSHOT_MAX_NODES), + requested_max_depth + .unwrap_or(DEFAULT_SNAPSHOT_MAX_DEPTH) + .min(HARD_SNAPSHOT_MAX_DEPTH), + ) +} + +struct BoundedTraversal { + queue: VecDeque, + attempted: usize, + max_items: usize, +} + +impl BoundedTraversal { + fn new(max_items: usize) -> Self { + Self { + queue: VecDeque::new(), + attempted: 0, + max_items, + } + } + + fn enqueue(&mut self, items: impl IntoIterator) { + self.queue + .extend(items.into_iter().take(self.remaining_capacity())); + } + + fn pop(&mut self) -> Option { + if self.attempted >= self.max_items { + return None; + } + let item = self.queue.pop_front()?; + self.attempted += 1; + Some(item) + } + + fn remaining_capacity(&self) -> usize { + self.max_items + .saturating_sub(self.attempted.saturating_add(self.queue.len())) + } +} + +fn bounded_child_count(reported: i32, limit: usize) -> usize { + usize::try_from(reported).unwrap_or_default().min(limit) +} + +struct IndexedReadBatch { + items: Vec, + attempted: usize, +} + +impl IndexedReadBatch { + fn all_failed(&self) -> bool { + self.attempted > 0 && self.items.is_empty() + } +} + +async fn fetch_indexed_up_to( + reported: i32, + limit: usize, + remaining_attempts: &mut usize, + fetch: F, +) -> IndexedReadBatch +where + F: Fn(i32) -> Fut, + Fut: Future>, +{ + let attempt_count = bounded_child_count(reported, limit).min(*remaining_attempts); + *remaining_attempts = (*remaining_attempts).saturating_sub(attempt_count); + let end_index = i32::try_from(attempt_count).unwrap_or(i32::MAX); + + let items = stream::iter(0..end_index) + .map(fetch) + .buffered(CHILD_READ_CONCURRENCY) + .filter_map(|result| async move { result.ok() }) + .collect() + .await; + + IndexedReadBatch { + items, + attempted: attempt_count, + } +} + +async fn children_up_to( + proxy: &AccessibleProxy<'_>, + limit: usize, + remaining_attempts: &mut usize, +) -> zbus::Result> { + if limit == 0 || *remaining_attempts == 0 { + return Ok(IndexedReadBatch { + items: Vec::new(), + attempted: 0, + }); + } + + let child_count = proxy.child_count().await?; + Ok( + fetch_indexed_up_to(child_count, limit, remaining_attempts, |index| { + proxy.get_child_at_index(index) + }) + .await, + ) +} pub async fn list_accessible_apps(limit: usize) -> Result> { let conn = connect().await?; - let roots = registry_children(&conn).await?; + let mut remaining_child_reads = limit; + let roots = registry_children(&conn, limit, &mut remaining_child_reads).await?; let dbus = DBusProxy::new(conn.connection()).await.ok(); let mut apps = Vec::new(); @@ -123,38 +251,73 @@ pub async fn snapshot_tree( target_pid: Option, max_nodes: usize, max_depth: u32, +) -> Result> { + let (max_nodes, max_depth) = snapshot_limits(Some(max_nodes), Some(max_depth)); + timeout( + SNAPSHOT_TIMEOUT, + snapshot_tree_inner( + app_name_or_bundle_identifier, + target_pid, + max_nodes, + max_depth, + ), + ) + .await + .context("AT-SPI snapshot exceeded its 10-second deadline")? +} + +async fn snapshot_tree_inner( + app_name_or_bundle_identifier: Option<&str>, + target_pid: Option, + max_nodes: usize, + max_depth: u32, ) -> Result> { let conn = connect().await?; - let roots = registry_children(&conn).await?; - let selected_roots = - select_roots(&conn, roots, app_name_or_bundle_identifier, target_pid).await; + // App discovery is bounded independently so a tiny requested tree still + // finds a target registered after the first accessibility root. + let (mut remaining_registry_reads, mut remaining_filter_reads, mut remaining_traversal_reads) = + snapshot_child_read_budgets(max_nodes); + let roots = + registry_children(&conn, MAX_DISCOVERY_ROOTS, &mut remaining_registry_reads).await?; + let selected_roots = select_roots( + &conn, + roots, + app_name_or_bundle_identifier, + target_pid, + &mut remaining_filter_reads, + ) + .await; let mut nodes = Vec::new(); - let mut queue = VecDeque::new(); + let mut traversal = BoundedTraversal::new(max_nodes); - for object_ref in selected_roots { - queue.push_back((object_ref, 0_u32, None)); - } - - while let Some((object_ref, depth, parent_index)) = queue.pop_front() { - if nodes.len() >= max_nodes { - break; - } + traversal.enqueue( + selected_roots + .into_iter() + .map(|object_ref| (object_ref, 0_u32, None)), + ); + while let Some((object_ref, depth, parent_index)) = traversal.pop() { let Ok(proxy) = open_accessible(&conn, &object_ref).await else { continue; }; let index = nodes.len() as u32; - let child_refs = if depth < max_depth { - proxy.get_children().await.unwrap_or_default() + let remaining = traversal.remaining_capacity(); + let child_refs = if depth < max_depth && remaining > 0 { + children_up_to(&proxy, remaining, &mut remaining_traversal_reads) + .await + .map(|batch| batch.items) + .unwrap_or_default() } else { Vec::new() }; nodes.push(read_node(&proxy, &object_ref, index, parent_index, depth).await); - for child in child_refs { - queue.push_back((child, depth + 1, Some(index))); - } + traversal.enqueue( + child_refs + .into_iter() + .map(|child| (child, depth + 1, Some(index))), + ); } Ok(nodes) @@ -181,21 +344,22 @@ pub async fn focused_element_summary( target_pid: Option, ) -> Result> { let conn = connect().await?; - let roots = registry_children(&conn).await?; - let selected_roots = select_roots(&conn, roots, None, target_pid).await; - let mut visited = 0_usize; - let mut queue = VecDeque::new(); - - for object_ref in selected_roots { - queue.push_back((object_ref, 0_u32)); - } + let mut remaining_registry_reads = MAX_DISCOVERY_ROOTS; + let roots = + registry_children(&conn, MAX_DISCOVERY_ROOTS, &mut remaining_registry_reads).await?; + let mut remaining_filter_reads = MAX_DISCOVERY_CHILD_READS; + let selected_roots = + select_roots(&conn, roots, None, target_pid, &mut remaining_filter_reads).await; + let mut traversal = BoundedTraversal::new(FOCUS_PROBE_MAX_NODES); + let mut remaining_traversal_reads = FOCUS_PROBE_MAX_NODES; - while let Some((object_ref, depth)) = queue.pop_front() { - if visited >= FOCUS_PROBE_MAX_NODES { - break; - } - visited += 1; + traversal.enqueue( + selected_roots + .into_iter() + .map(|object_ref| (object_ref, 0_u32)), + ); + while let Some((object_ref, depth)) = traversal.pop() { let Ok(proxy) = open_accessible(&conn, &object_ref).await else { continue; }; @@ -212,9 +376,12 @@ pub async fn focused_element_summary( })); } if depth < FOCUS_PROBE_MAX_DEPTH { - for child in proxy.get_children().await.unwrap_or_default() { - queue.push_back((child, depth + 1)); - } + let remaining = traversal.remaining_capacity(); + let children = children_up_to(&proxy, remaining, &mut remaining_traversal_reads) + .await + .map(|batch| batch.items) + .unwrap_or_default(); + traversal.enqueue(children.into_iter().map(|child| (child, depth + 1))); } } @@ -323,14 +490,24 @@ async fn open_accessible<'r>( object_ref.as_accessible_proxy(conn.connection()).await } -async fn registry_children(conn: &AccessibilityConnection) -> Result> { +async fn registry_children( + conn: &AccessibilityConnection, + limit: usize, + remaining_child_reads: &mut usize, +) -> Result> { let root = conn .root_accessible_on_registry() .await .context("failed to open AT-SPI registry root")?; - root.get_children() + let batch = children_up_to(&root, limit, remaining_child_reads) .await - .context("failed to read AT-SPI registry children") + .context("failed to read AT-SPI registry children")?; + if batch.all_failed() { + return Err(anyhow!( + "AT-SPI registry reported children, but every indexed child read failed" + )); + } + Ok(batch.items) } async fn select_roots( @@ -338,6 +515,7 @@ async fn select_roots( roots: Vec, app_name_or_bundle_identifier: Option<&str>, target_pid: Option, + remaining_child_reads: &mut usize, ) -> Vec { let needle = app_name_or_bundle_identifier .map(str::trim) @@ -354,7 +532,7 @@ async fn select_roots( for object_ref in remaining { if object_ref_pid(dbus.as_ref(), &object_ref).await == Some(target_pid) { if let Some(needle) = needle.as_deref() { - if root_matches(conn, &object_ref, needle).await { + if root_matches(conn, &object_ref, needle, remaining_child_reads).await { pid_and_filter_matches.push(object_ref); } else { pid_matches.push(object_ref); @@ -383,7 +561,7 @@ async fn select_roots( let mut selected = Vec::new(); for object_ref in remaining { - if root_matches(conn, &object_ref, needle).await { + if root_matches(conn, &object_ref, needle, remaining_child_reads).await { selected.push(object_ref); } } @@ -395,6 +573,7 @@ async fn root_matches( conn: &AccessibilityConnection, object_ref: &ObjectRefOwned, needle: &str, + remaining_child_reads: &mut usize, ) -> bool { let Ok(proxy) = open_accessible(conn, object_ref).await else { return object_ref_id(object_ref) @@ -406,8 +585,11 @@ async fn root_matches( return true; } - let children = proxy.get_children().await.unwrap_or_default(); - for child_ref in children.into_iter().take(8) { + for child_ref in children_up_to(&proxy, ROOT_MATCH_CHILD_LIMIT, remaining_child_reads) + .await + .map(|batch| batch.items) + .unwrap_or_default() + { let Ok(child_proxy) = open_accessible(conn, &child_ref).await else { continue; }; @@ -748,4 +930,87 @@ mod tests { assert_eq!(labels, vec!["checked".to_string(), "focused".to_string()]); } + + #[test] + fn default_snapshot_limits_cover_deep_gtk4_trees() { + // Nautilus 50 places file-list cells below depth 20 and can expose + // more than 850 raw nodes. Keep the defaults above that known shape. + assert_eq!(snapshot_limits(None, None), (1_000, 32)); + } + + #[test] + fn requested_snapshot_limits_remain_bounded() { + assert_eq!(snapshot_limits(Some(0), Some(0)), (1, 0)); + assert_eq!(snapshot_limits(Some(10_000), Some(128)), (2_000, 64)); + } + + #[test] + fn app_discovery_budget_is_independent_of_requested_tree_size() { + assert_eq!(snapshot_child_read_budgets(1), (256, 2_048, 1)); + } + + #[test] + fn traversal_attempts_and_queue_share_one_work_budget() { + let mut traversal = BoundedTraversal::new(4); + traversal.enqueue([1]); + + assert_eq!(traversal.pop(), Some(1)); + traversal.enqueue(2..=10_000); + assert_eq!(traversal.queue, VecDeque::from([2, 3, 4])); + + assert_eq!(traversal.pop(), Some(2)); + traversal.enqueue(5..=10_000); + assert_eq!(traversal.queue, VecDeque::from([3, 4])); + assert_eq!(traversal.pop(), Some(3)); + assert_eq!(traversal.pop(), Some(4)); + assert_eq!(traversal.pop(), None); + assert_eq!(traversal.attempted, 4); + } + + #[test] + fn child_count_is_clamped_before_indexed_reads() { + assert_eq!(bounded_child_count(-1, 4), 0); + assert_eq!(bounded_child_count(3, 4), 3); + assert_eq!(bounded_child_count(i32::MAX, 4), 4); + } + + #[tokio::test] + async fn indexed_child_reads_consume_attempts_even_when_one_fails() { + let calls = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let mut remaining_attempts = 3; + let batch = fetch_indexed_up_to(100, 10, &mut remaining_attempts, { + let calls = calls.clone(); + move |index| { + let calls = calls.clone(); + async move { + calls.lock().unwrap().push(index); + if index == 1 { + Err(()) + } else { + Ok(index) + } + } + } + }) + .await; + + assert_eq!(batch.items, vec![0, 2]); + assert_eq!(batch.attempted, 3); + assert!(!batch.all_failed()); + assert_eq!(*calls.lock().unwrap(), vec![0, 1, 2]); + assert_eq!(remaining_attempts, 0); + + let no_children = fetch_indexed_up_to(100, 10, &mut remaining_attempts, |_| async { + Ok::<_, ()>(99) + }) + .await; + assert!(no_children.items.is_empty()); + assert_eq!(no_children.attempted, 0); + assert!(!no_children.all_failed()); + + let mut failed_attempts = 2; + let all_failed = + fetch_indexed_up_to(2, 2, &mut failed_attempts, |_| async { Err::(()) }).await; + assert!(all_failed.all_failed()); + } } diff --git a/computer-use-linux/src/main.rs b/computer-use-linux/src/main.rs index 78917bed9..a3e7c6a18 100644 --- a/computer-use-linux/src/main.rs +++ b/computer-use-linux/src/main.rs @@ -57,9 +57,14 @@ async fn main() -> Result<()> { } Some("state") => { let app_name_or_bundle_identifier = std::env::args().nth(2); - let nodes = - atspi_tree::snapshot_tree(app_name_or_bundle_identifier.as_deref(), None, 120, 12) - .await?; + let (max_nodes, max_depth) = atspi_tree::snapshot_limits(None, None); + let nodes = atspi_tree::snapshot_tree( + app_name_or_bundle_identifier.as_deref(), + None, + max_nodes, + max_depth, + ) + .await?; println!( "{}", serde_json::to_string_pretty(&nodes) diff --git a/computer-use-linux/src/server.rs b/computer-use-linux/src/server.rs index 77500c1f0..ed3456683 100644 --- a/computer-use-linux/src/server.rs +++ b/computer-use-linux/src/server.rs @@ -311,8 +311,8 @@ impl ComputerUseLinux { .expect("diagnostics task panicked"); let (window_context, window_error, window_permissions_hint) = self.resolve_window_context(¶ms).await; - let max_nodes = params.max_nodes.unwrap_or(120).clamp(1, 500); - let max_depth = params.max_depth.unwrap_or(12).min(12); + let (max_nodes, max_depth) = + crate::atspi_tree::snapshot_limits(params.max_nodes, params.max_depth); let include_screenshot = params.include_screenshot.unwrap_or(true); let screenshot_options = params.screenshot_options(); let screenshot_target_requested = params.window_target().has_target(); @@ -1928,7 +1928,7 @@ impl ComputerUseLinux { // The rmcp tool_handler macro only accepts a string literal here, so this // can't be env!("CARGO_PKG_VERSION"); the MCP safety check (CI) fails the // build if it drifts from the Cargo version. - version = "0.4.7-linux-alpha1", + version = "0.4.9-linux-alpha1", instructions = "Begin every turn that uses Computer Use by calling get_app_state. If diagnostics report disabled GNOME accessibility, call setup_accessibility before asking the user to retry. Use list_windows/focused_window before targeted keyboard input. If diagnostics report windowing.can_list_windows=false on GNOME, call setup_window_targeting to install the optional GNOME Shell extension backend, then ask the user to log out and back in if the setup report says a shell reload is required. This Linux backend can capture size-bounded screenshots through GNOME Shell, the Codex GNOME Shell extension, or XDG Desktop Portal, read AT-SPI trees with action/value metadata, invoke native AT-SPI actions, set AT-SPI values or editable text, list/focus compositor windows through registered Linux window backends when the session permits it, attach best-effort terminal tty/process metadata to terminal windows, send coordinate or element-targeted click/scroll/drag input through the Wayland remote desktop portal when available, and send layout-safe literal type_text through KDE clipboard integration on Plasma Wayland or through portal keysyms on other Wayland sessions before falling back to ydotool. Screenshot results include width/height for the returned image plus coordinate_width/coordinate_height and scale for desktop coordinate conversion; request more detail with max_width, max_height, max_bytes, format=jpeg, quality, or a smaller target/crop instead of relying on unbounded screenshots. Tools with readOnlyHint=false may mutate local desktop or application state; hosts should require approval for actions that can submit, delete, send, purchase, or overwrite data. For element-targeted actions, prefer element_index from the latest get_app_state result; click, perform_action, and set_value can also use semantic role/name/text/states selectors when the target is unique. type_text and press_key accept optional window_id, pid, app_id, wm_class, title, tty, terminal_pid, terminal_command, or terminal_cwd selectors and refuse targeted input if focus cannot be verified. After targeted keyboard input, results append focused-element feedback from AT-SPI (role, name, editable) and warn when no editable element holds focus — treat that warning as the input not landing. Screenshot, click, and input results warn when the target window or coordinate is partially or fully off-screen; use move_window/resize_window (GNOME Shell extension backend) to bring a window fully on-screen before retrying. scroll accepts the same window targeting and relative coordinates as click. get_app_state returns a compact readiness block by default; pass verbose=true for the full diagnostics dump. Electron apps expose no AT-SPI tree unless launched with --force-renderer-accessibility." )] impl ServerHandler for ComputerUseLinux {} @@ -2084,8 +2084,10 @@ struct GetAppStateParams { wm_class: Option, #[serde(default)] title: Option, + /// Maximum raw AT-SPI nodes to inspect before compaction (default 1000, hard max 2000). #[serde(default)] max_nodes: Option, + /// Maximum AT-SPI traversal depth (default 32, hard max 64). #[serde(default)] max_depth: Option, #[serde(default)]