From 44d18953a2cdae836c817c31a47d4f47f0e9d44b Mon Sep 17 00:00:00 2001 From: "Yifeng \"Evan\" Wang" <7312949+doodlewind@users.noreply.github.com> Date: Thu, 30 Jul 2026 21:51:45 +0900 Subject: [PATCH 1/2] feat(persona): add Pocket-native parity POC --- .gitignore | 1 + Cargo.lock | 46 +- Cargo.toml | 19 +- README.md | 24 + crates/pocket-persona/Cargo.toml | 21 + crates/pocket-persona/guest/main.ts | 23 + crates/pocket-persona/guest/sdk.ts | 38 + crates/pocket-persona/src/bridge.rs | 868 ++++++++++ crates/pocket-persona/src/catalog.rs | 327 ++++ crates/pocket-persona/src/guest.rs | 109 ++ crates/pocket-persona/src/main.rs | 188 +++ crates/pocket-persona/src/sim.rs | 215 +++ crates/pocket-persona/src/widget.rs | 758 +++++++++ docs/PERSONA.md | 740 +++++++++ fixtures/persona/library.json | 74 + package.json | 6 +- scripts/accept-persona.ts | 1012 ++++++++++++ scripts/bench-persona.ts | 2188 ++++++++++++++++++++++++++ tests/persona-bench.test.ts | 102 ++ vendor/pocketjs | 2 +- 20 files changed, 6751 insertions(+), 10 deletions(-) create mode 100644 crates/pocket-persona/Cargo.toml create mode 100644 crates/pocket-persona/guest/main.ts create mode 100644 crates/pocket-persona/guest/sdk.ts create mode 100644 crates/pocket-persona/src/bridge.rs create mode 100644 crates/pocket-persona/src/catalog.rs create mode 100644 crates/pocket-persona/src/guest.rs create mode 100644 crates/pocket-persona/src/main.rs create mode 100644 crates/pocket-persona/src/sim.rs create mode 100644 crates/pocket-persona/src/widget.rs create mode 100644 docs/PERSONA.md create mode 100644 fixtures/persona/library.json create mode 100644 scripts/accept-persona.ts create mode 100644 scripts/bench-persona.ts create mode 100644 tests/persona-bench.test.ts diff --git a/.gitignore b/.gitignore index 7fd47af..e8f4c5c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ target/ node_modules/ dist/ +# Tool-owned reference checkouts, benchmark reports, and temporary receipts. out/ assets/*.vrm assets/*.vrma diff --git a/Cargo.lock b/Cargo.lock index 46ad1b6..ea2c796 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1611,13 +1611,40 @@ dependencies = [ ] [[package]] -name = "pocket-ui-wgpu" +name = "pocket-persona" version = "0.1.0" dependencies = [ "anyhow", - "bytemuck", + "env_logger", + "glam", "log", "pocket-mod", + "pocket-vrm", + "pocket-widget", + "pocket3d", + "serde", + "serde_json", + "wgpu", + "winit", +] + +[[package]] +name = "pocket-ui-surface" +version = "0.1.0" +dependencies = [ + "anyhow", + "log", + "pocket-mod", + "pocketjs-core", +] + +[[package]] +name = "pocket-ui-wgpu" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytemuck", + "pocket-ui-surface", "pocket3d", "pocketjs-core", "wgpu", @@ -1635,6 +1662,20 @@ dependencies = [ "serde_json", ] +[[package]] +name = "pocket-widget" +version = "0.1.0" +dependencies = [ + "anyhow", + "glam", + "log", + "pocket-ui-wgpu", + "pocket3d", + "pocketjs-core", + "wgpu", + "winit", +] + [[package]] name = "pocket3d" version = "0.1.0" @@ -1648,6 +1689,7 @@ dependencies = [ "png", "pocket3d-bsp", "pollster", + "serde_json", "wgpu", "winit", ] diff --git a/Cargo.toml b/Cargo.toml index 3ce49e8..4b2f280 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,10 @@ [workspace] resolver = "2" -members = ["crates/pocket-character", "crates/pocket-character-core"] +members = [ + "crates/pocket-character", + "crates/pocket-character-core", + "crates/pocket-persona", +] # The vendored engine keeps its own workspaces (Cargo would otherwise try to # adopt the path-dependency crates into this one and break their inheritance). exclude = ["vendor"] @@ -14,11 +18,12 @@ repository = "https://github.com/pocket-stack/pocket-character" [workspace.dependencies] # The engine family is vendored as a git submodule (vendor/pocketjs) so the # Rust crates and the JS framework/build pipeline stay pinned to ONE commit. -pocket3d = { path = "vendor/pocketjs/pocket3d/crates/pocket3d" } -pocket-vrm = { path = "vendor/pocketjs/pocket3d/crates/pocket-vrm" } -pocket-mod = { path = "vendor/pocketjs/pocket3d/crates/pocket-mod" } -pocket-ui-wgpu = { path = "vendor/pocketjs/pocket3d/crates/pocket-ui-wgpu" } -pocketjs-core = { path = "vendor/pocketjs/core", features = ["std"] } +pocket3d = { path = "vendor/pocketjs/engine/pocket3d/crates/pocket3d" } +pocket-vrm = { path = "vendor/pocketjs/engine/crates/pocket-vrm" } +pocket-mod = { path = "vendor/pocketjs/engine/crates/pocket-mod" } +pocket-ui-wgpu = { path = "vendor/pocketjs/engine/crates/pocket-ui-wgpu" } +pocket-widget = { path = "vendor/pocketjs/engine/crates/pocket-widget" } +pocketjs-core = { path = "vendor/pocketjs/engine/core", features = ["std"] } pocket-character-core = { path = "crates/pocket-character-core" } wgpu = "25" @@ -27,6 +32,8 @@ glam = "0.33" anyhow = "1" log = "0.4" env_logger = "0.11" +serde = { version = "1", features = ["derive"] } +serde_json = "1" [profile.dev] opt-level = 1 diff --git a/README.md b/README.md index 775cb9e..bfebb74 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,30 @@ cost on the Pocket architecture instead of Electron?* See [DESIGN.md](DESIGN.md) for the architecture and the parity contract, and the measurement section below for the answer. +## Persona parity POC + +This repository also owns the Pocket-native vertical slice of +[xikhar/persona](https://github.com/xikhar/persona). The two visual acceptance +commands prepare a pinned reference checkout, validate and stage the same local +VRM/VRMA inputs, build the selected target, launch it, and drive the same +idle/speaking/lip-sync/action sequence: + +```sh +bun run accept:persona +bun run accept:pocket +``` + +Press Ctrl-C to terminate the complete target process tree. For the sequential +resource comparison: + +```sh +bun run bench:persona +bun run bench:persona:controlled +``` + +See [docs/PERSONA.md](docs/PERSONA.md) for the parity boundary, benchmark +methodology, measurements, and asset-license constraints. + ## What it does - **AvatarSample_A** (VRoid official sample) with airi's `idle_loop.vrma` diff --git a/crates/pocket-persona/Cargo.toml b/crates/pocket-persona/Cargo.toml new file mode 100644 index 0000000..888aaf8 --- /dev/null +++ b/crates/pocket-persona/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "pocket-persona" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "Persona-compatible native VRM desktop character proof of concept" + +[dependencies] +pocket3d = { workspace = true } +pocket-mod = { workspace = true } +pocket-vrm = { workspace = true } +pocket-widget = { workspace = true } +wgpu = { workspace = true } +winit = { workspace = true } +glam = { workspace = true, features = ["std"] } +anyhow = { workspace = true } +log = { workspace = true } +env_logger = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } diff --git a/crates/pocket-persona/guest/main.ts b/crates/pocket-persona/guest/main.ts new file mode 100644 index 0000000..e145976 --- /dev/null +++ b/crates/pocket-persona/guest/main.ts @@ -0,0 +1,23 @@ +import { onPersonaTick, persona } from "./sdk"; + +console.log( + `pocket-persona: model=${persona.boot.model}`, + `actions=[${persona.boot.actions.join(", ")}]`, +); + +// The native core owns continuous animation, facial motion, and physics. +// This deliberately small Pocket guest is the hot-swappable personality +// seam: product-specific reactions can be added without rebuilding Rust. +let lastHeartbeat = 0; +onPersonaTick((state, events) => { + for (const event of events) { + console.log(`pocket-persona: ${event.type}=${event.value}`); + } + if (state.t - lastHeartbeat >= 60) { + lastHeartbeat = state.t; + console.log( + `pocket-persona: t=${state.t.toFixed(0)}s animation=${state.animation}`, + `level=${state.audioLevel.toFixed(2)} fps=${state.renderFps.toFixed(1)}`, + ); + } +}); diff --git a/crates/pocket-persona/guest/sdk.ts b/crates/pocket-persona/guest/sdk.ts new file mode 100644 index 0000000..8a39457 --- /dev/null +++ b/crates/pocket-persona/guest/sdk.ts @@ -0,0 +1,38 @@ +export interface PersonaTick { + t: number; + activity: "idle" | "listening" | "speaking"; + audioLevel: number; + animation: string; + blink: number; + renderFps: number; +} + +export interface PersonaEvent { + type: "animationChanged" | "voiceChanged"; + value: string; +} + +interface PersonaSurface { + readonly boot: { + readonly model: string; + readonly actions: readonly string[]; + }; + playAnimation(name: string): void; + setExpression(name: string, weight: number): void; + quit(): void; + __dispatch?: (state: PersonaTick, events: PersonaEvent[]) => void; +} + +declare global { + // Mounted by the native `pocket-mod` host before this bundle is evaluated. + // eslint-disable-next-line no-var + var persona: PersonaSurface; +} + +export const persona = globalThis.persona; + +export function onPersonaTick( + callback: (state: PersonaTick, events: PersonaEvent[]) => void, +): void { + globalThis.persona.__dispatch = callback; +} diff --git a/crates/pocket-persona/src/bridge.rs b/crates/pocket-persona/src/bridge.rs new file mode 100644 index 0000000..2ab8d1d --- /dev/null +++ b/crates/pocket-persona/src/bridge.rs @@ -0,0 +1,868 @@ +//! Persona-compatible loopback bridge and minimal Streamable HTTP MCP server. +//! +//! The control plane intentionally stays off the render thread. Requests are +//! validated on a loopback-only worker and reduced to bounded commands that +//! the fixed-step core drains once per tick. + +use std::collections::HashMap; +use std::io::{Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, mpsc}; +use std::thread::JoinHandle; +use std::time::Duration; + +use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +use crate::catalog::Catalog; + +const MAX_REQUEST_BYTES: usize = 64 * 1024; +const COMMAND_QUEUE_CAPACITY: usize = 128; +const MCP_SESSION: &str = "pocket-persona-v1"; +const MCP_PROTOCOL_VERSION: &str = "2025-06-18"; + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +pub struct VoiceState { + pub phase: String, + pub activity: String, + #[serde(rename = "microphoneMuted")] + pub microphone_muted: bool, + #[serde(rename = "outputMuted")] + pub output_muted: bool, +} + +impl Default for VoiceState { + fn default() -> Self { + Self { + phase: "inactive".into(), + activity: "idle".into(), + microphone_muted: false, + output_muted: false, + } + } +} + +impl VoiceState { + pub fn speaking(&self) -> bool { + self.phase == "active" && self.activity == "speaking" && !self.output_muted + } + + fn valid(&self) -> bool { + matches!( + self.phase.as_str(), + "inactive" | "starting" | "active" | "stopping" + ) && matches!(self.activity.as_str(), "idle" | "listening" | "speaking") + } +} + +#[derive(Clone, Debug)] +pub enum BridgeCommand { + Voice(VoiceState), + AudioLevel(f32), + PlayAnimation(String), + Window { visible: bool }, +} + +#[derive(Clone, Debug, Serialize)] +pub struct StatusSnapshot { + #[serde(rename = "modelConfigured")] + pub model_configured: bool, + #[serde(rename = "windowVisible")] + pub window_visible: bool, + #[serde(rename = "voiceState")] + pub voice_state: VoiceState, + #[serde(rename = "audioLevel")] + pub audio_level: f32, + #[serde(rename = "activeAnimation")] + pub active_animation: String, + #[serde(rename = "renderFps")] + pub render_fps: f32, + #[serde(rename = "frameTimeP95Ms")] + pub frame_time_p95_ms: f32, + #[serde(rename = "frameTimeP99Ms")] + pub frame_time_p99_ms: f32, + #[serde(rename = "frameTimeMaxMs")] + pub frame_time_max_ms: f32, +} + +impl StatusSnapshot { + pub fn new() -> Self { + Self { + model_configured: true, + window_visible: true, + voice_state: VoiceState::default(), + audio_level: 0.0, + active_animation: "idle".into(), + render_fps: 0.0, + frame_time_p95_ms: 0.0, + frame_time_p99_ms: 0.0, + frame_time_max_ms: 0.0, + } + } +} + +pub struct Bridge { + receiver: mpsc::Receiver, + status: Arc>, + shutdown: Arc, + worker: Option>, +} + +impl Bridge { + pub fn start(port: u16, catalog: Arc) -> Result { + let listener = TcpListener::bind(("127.0.0.1", port)) + .with_context(|| format!("binding Pocket Persona bridge port {port}"))?; + let address = listener.local_addr()?; + let (sender, receiver) = mpsc::sync_channel(COMMAND_QUEUE_CAPACITY); + let status = Arc::new(Mutex::new(StatusSnapshot::new())); + let thread_status = status.clone(); + let shutdown = Arc::new(AtomicBool::new(false)); + let thread_shutdown = shutdown.clone(); + listener.set_nonblocking(true)?; + let worker = std::thread::Builder::new() + .name("pocket-persona-bridge".into()) + .spawn(move || { + while !thread_shutdown.load(Ordering::Relaxed) { + match listener.accept() { + Ok((stream, _address)) => { + if let Err(error) = + handle_connection(stream, &catalog, &thread_status, &sender) + { + log::warn!("Pocket Persona bridge request: {error:#}"); + } + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + std::thread::sleep(Duration::from_millis(10)); + } + Err(error) => { + log::warn!("Pocket Persona bridge accept: {error}"); + std::thread::sleep(Duration::from_millis(50)); + } + } + } + })?; + log::info!("Pocket Persona bridge: http://{address}"); + Ok(Self { + receiver, + status, + shutdown, + worker: Some(worker), + }) + } + + pub fn drain(&self) -> impl Iterator + '_ { + self.receiver.try_iter() + } + + pub fn update_status(&self, update: impl FnOnce(&mut StatusSnapshot)) { + if let Ok(mut status) = self.status.lock() { + update(&mut status); + } + } +} + +impl Drop for Bridge { + fn drop(&mut self) { + self.shutdown.store(true, Ordering::Relaxed); + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + } +} + +struct Request { + method: String, + path: String, + headers: HashMap, + body: Vec, +} + +struct Response { + status: u16, + headers: Vec<(&'static str, String)>, + body: Vec, +} + +impl Response { + fn empty(status: u16) -> Self { + Self { + status, + headers: Vec::new(), + body: Vec::new(), + } + } + + fn json(status: u16, value: Value) -> Self { + Self { + status, + headers: vec![("Content-Type", "application/json".into())], + body: serde_json::to_vec(&value).expect("JSON value serializes"), + } + } +} + +fn handle_connection( + mut stream: TcpStream, + catalog: &Catalog, + status: &Arc>, + sender: &mpsc::SyncSender, +) -> Result<()> { + stream.set_read_timeout(Some(Duration::from_secs(2)))?; + stream.set_write_timeout(Some(Duration::from_secs(2)))?; + let request = read_request(&mut stream)?; + let response = route(&request, catalog, status, sender); + write_response(&mut stream, response)?; + Ok(()) +} + +fn read_request(stream: &mut TcpStream) -> Result { + let mut bytes = Vec::with_capacity(4096); + let mut scratch = [0u8; 4096]; + let (header_end, content_length) = loop { + let read = stream.read(&mut scratch)?; + if read == 0 { + anyhow::bail!("request ended before its headers"); + } + bytes.extend_from_slice(&scratch[..read]); + if bytes.len() > MAX_REQUEST_BYTES { + anyhow::bail!("request exceeds {MAX_REQUEST_BYTES} bytes"); + } + if let Some(end) = find_subslice(&bytes, b"\r\n\r\n") { + let headers = std::str::from_utf8(&bytes[..end])?; + let content_length = headers + .lines() + .skip(1) + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().ok()) + .flatten() + }) + .unwrap_or(0); + if end + 4 + content_length > MAX_REQUEST_BYTES { + anyhow::bail!("request body exceeds the bridge limit"); + } + break (end, content_length); + } + }; + let body_end = header_end + 4 + content_length; + while bytes.len() < body_end { + let read = stream.read(&mut scratch)?; + if read == 0 { + anyhow::bail!("request body is truncated"); + } + bytes.extend_from_slice(&scratch[..read]); + } + + let head = std::str::from_utf8(&bytes[..header_end])?; + let mut lines = head.split("\r\n"); + let request_line = lines.next().context("request line is missing")?; + let mut request_parts = request_line.split_whitespace(); + let method = request_parts.next().context("request method is missing")?; + let path = request_parts.next().context("request path is missing")?; + if request_parts.next() != Some("HTTP/1.1") { + anyhow::bail!("only HTTP/1.1 is supported"); + } + let mut headers = HashMap::new(); + for line in lines { + let (name, value) = line.split_once(':').context("malformed request header")?; + headers.insert(name.trim().to_ascii_lowercase(), value.trim().to_string()); + } + Ok(Request { + method: method.into(), + path: path.into(), + headers, + body: bytes[header_end + 4..body_end].to_vec(), + }) +} + +fn route( + request: &Request, + catalog: &Catalog, + status: &Arc>, + sender: &mpsc::SyncSender, +) -> Response { + if request + .headers + .get("host") + .is_none_or(|host| !allowed_host(host)) + { + return Response::empty(403); + } + let origin = request.headers.get("origin").map(String::as_str); + if origin.is_some_and(|origin| !allowed_origin(origin)) { + return Response::empty(403); + } + + if request.method == "GET" && request.path == "/health" { + let snapshot = status + .lock() + .map(|status| status.clone()) + .unwrap_or_else(|_| StatusSnapshot::new()); + return Response::json(200, json!({ "ok": true, "status": snapshot })); + } + + if request.method == "OPTIONS" && request.path == "/events" { + let mut response = Response::empty(204); + add_cors(&mut response, origin); + response + .headers + .push(("Access-Control-Allow-Methods", "POST, OPTIONS".into())); + response + .headers + .push(("Access-Control-Allow-Headers", "content-type".into())); + return response; + } + + if request.method == "POST" && request.path == "/events" { + let Ok(value) = serde_json::from_slice::(&request.body) else { + return Response::empty(400); + }; + let Some(command) = normalize_event(&value, catalog) else { + return Response::empty(422); + }; + if sender.try_send(command.clone()).is_err() { + return Response::empty(503); + } + apply_status_for_command(status, &command); + let mut response = Response::json(202, json!({ "accepted": true })); + add_cors(&mut response, origin); + return response; + } + + if request.path == "/mcp" { + if request.method == "GET" { + let mut response = Response::empty(405); + response.headers.push(("Allow", "POST, DELETE".into())); + return response; + } + if request.method == "DELETE" { + return Response::empty(200); + } + if request.method != "POST" { + return Response::empty(405); + } + let Ok(value) = serde_json::from_slice::(&request.body) else { + return json_rpc_error(Value::Null, -32700, "Parse error"); + }; + return route_mcp(value, catalog, status, sender); + } + + Response::empty(404) +} + +fn normalize_event(value: &Value, catalog: &Catalog) -> Option { + match value.get("type")?.as_str()? { + "state" => { + let state: VoiceState = serde_json::from_value(value.get("state")?.clone()).ok()?; + state.valid().then_some(BridgeCommand::Voice(state)) + } + "audio-level" => { + let level = value.get("level")?.as_f64()?; + level + .is_finite() + .then_some(BridgeCommand::AudioLevel(level.clamp(0.0, 1.0) as f32)) + } + "animation" => { + let name = value.get("animation_name")?.as_str()?; + (valid_action_name(name) + && catalog + .action(name) + .is_some_and(|action| !action.clips.is_empty())) + .then(|| BridgeCommand::PlayAnimation(name.into())) + } + _ => None, + } +} + +fn route_mcp( + value: Value, + catalog: &Catalog, + status: &Arc>, + sender: &mpsc::SyncSender, +) -> Response { + if value.get("jsonrpc").and_then(Value::as_str) != Some("2.0") { + return json_rpc_error(Value::Null, -32600, "Invalid Request"); + } + let id = value.get("id").cloned().unwrap_or(Value::Null); + let Some(method) = value.get("method").and_then(Value::as_str) else { + return json_rpc_error(id, -32600, "Invalid Request"); + }; + if method.starts_with("notifications/") { + return Response::empty(202); + } + let result = match method { + "initialize" => json!({ + "protocolVersion": MCP_PROTOCOL_VERSION, + "capabilities": { "tools": { "listChanged": false } }, + "serverInfo": { "name": "Pocket Persona", "version": "0.1.0" }, + "instructions": "Pocket Persona controls the local native desktop character. It never speaks, records, transcribes, or sends audio." + }), + "ping" => json!({}), + "tools/list" => json!({ "tools": tools(catalog) }), + "tools/call" => { + let Some(name) = value.pointer("/params/name").and_then(Value::as_str) else { + return json_rpc_error(id, -32602, "Tool name is required"); + }; + return mcp_tool_call( + id, + name, + value.pointer("/params/arguments"), + catalog, + status, + sender, + ); + } + _ => return json_rpc_error(id, -32601, "Method not found"), + }; + let mut response = Response::json(200, json!({ "jsonrpc": "2.0", "id": id, "result": result })); + response + .headers + .push(("Mcp-Session-Id", MCP_SESSION.into())); + response +} + +fn tools(catalog: &Catalog) -> Vec { + let actions = describe_actions(catalog); + vec![ + json!({ + "name": "play_animation", + "title": "Play Pocket Persona animation", + "description": format!("Play an installed character action once.\n{actions}"), + "inputSchema": { + "type": "object", + "properties": { + "animation": { + "type": "string", + "description": format!("Installed action name.\n{actions}") + } + }, + "required": ["animation"], + "additionalProperties": false + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "idempotentHint": false, + "openWorldHint": false + } + }), + json!({ + "name": "list_animations", + "title": "List Pocket Persona animations", + "description": "Read installed action names, descriptions, and trigger scenarios.", + "inputSchema": { "type": "object", "additionalProperties": false }, + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false + } + }), + json!({ + "name": "control_window", + "title": "Control Pocket Persona window", + "description": "Show, hide, or toggle the native character window.", + "inputSchema": { + "type": "object", + "properties": { + "action": { "type": "string", "enum": ["show", "hide", "toggle"] } + }, + "required": ["action"], + "additionalProperties": false + }, + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "idempotentHint": false, + "openWorldHint": false + } + }), + json!({ + "name": "get_status", + "title": "Get Pocket Persona status", + "description": "Read model, window, voice, animation, and render state.", + "inputSchema": { "type": "object", "additionalProperties": false }, + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false + } + }), + ] +} + +fn mcp_tool_call( + id: Value, + name: &str, + arguments: Option<&Value>, + catalog: &Catalog, + status: &Arc>, + sender: &mpsc::SyncSender, +) -> Response { + let (text, is_error) = match name { + "play_animation" => { + let animation = arguments + .and_then(|arguments| arguments.get("animation")) + .and_then(Value::as_str); + match animation.and_then(|name| catalog.action(name)) { + Some(action) if !action.clips.is_empty() => { + let command = BridgeCommand::PlayAnimation(action.name.clone()); + if sender.try_send(command.clone()).is_err() { + ("Pocket Persona is busy or shutting down.".into(), true) + } else { + apply_status_for_command(status, &command); + (format!("Pocket Persona is playing the {} action.", action.name), false) + } + } + _ => ( + "That action is not currently playable. Call list_animations for the current catalog.".into(), + true, + ), + } + } + "list_animations" => (describe_actions(catalog), false), + "control_window" => { + let action = arguments + .and_then(|arguments| arguments.get("action")) + .and_then(Value::as_str); + let current = status + .lock() + .map(|status| status.window_visible) + .unwrap_or(true); + let visible = match action { + Some("show") => true, + Some("hide") => false, + Some("toggle") => !current, + _ => { + return json_rpc_error( + id, + -32602, + "Window action must be show, hide, or toggle", + ); + } + }; + let command = BridgeCommand::Window { visible }; + if sender.try_send(command.clone()).is_err() { + ("Pocket Persona is busy or shutting down.".into(), true) + } else { + apply_status_for_command(status, &command); + ( + format!( + "Pocket Persona's window is now {}.", + if visible { "visible" } else { "hidden" } + ), + false, + ) + } + } + "get_status" => { + let snapshot = status + .lock() + .map(|status| status.clone()) + .unwrap_or_else(|_| StatusSnapshot::new()); + ( + serde_json::to_string(&snapshot).expect("status serializes"), + false, + ) + } + _ => return json_rpc_error(id, -32602, "Unknown tool"), + }; + let mut result = json!({ + "content": [{ "type": "text", "text": text }] + }); + if is_error { + result["isError"] = Value::Bool(true); + } + let mut response = Response::json(200, json!({ "jsonrpc": "2.0", "id": id, "result": result })); + response + .headers + .push(("Mcp-Session-Id", MCP_SESSION.into())); + response +} + +fn apply_status_for_command(status: &Arc>, command: &BridgeCommand) { + let Ok(mut status) = status.lock() else { + return; + }; + match command { + BridgeCommand::Voice(voice) => status.voice_state = voice.clone(), + BridgeCommand::AudioLevel(level) => status.audio_level = *level, + BridgeCommand::PlayAnimation(name) => status.active_animation = name.clone(), + BridgeCommand::Window { visible } => status.window_visible = *visible, + } +} + +fn describe_actions(catalog: &Catalog) -> String { + let rows: Vec = catalog + .playable_actions() + .map(|action| { + format!( + "- {}: {} Trigger scenario: {}", + action.name, action.description, action.trigger_scenario + ) + }) + .collect(); + if rows.is_empty() { + "- No animation actions currently have playable clips.".into() + } else { + rows.join("\n") + } +} + +fn json_rpc_error(id: Value, code: i32, message: &str) -> Response { + Response::json( + 200, + json!({ + "jsonrpc": "2.0", + "id": id, + "error": { "code": code, "message": message } + }), + ) +} + +fn add_cors(response: &mut Response, origin: Option<&str>) { + if let Some(origin) = origin { + response + .headers + .push(("Access-Control-Allow-Origin", origin.into())); + response.headers.push(("Vary", "Origin".into())); + } +} + +fn allowed_host(host: &str) -> bool { + allowed_loopback_authority(host) +} + +fn allowed_origin(origin: &str) -> bool { + if let Some(authority) = origin.strip_prefix("codex-app://") { + return authority + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'~' | b'-')); + } + ["http://", "https://"].into_iter().any(|scheme| { + origin + .strip_prefix(scheme) + .is_some_and(allowed_loopback_authority) + }) +} + +fn allowed_loopback_authority(authority: &str) -> bool { + ["localhost", "127.0.0.1", "[::1]"].into_iter().any(|host| { + authority == host + || authority + .strip_prefix(&format!("{host}:")) + .is_some_and(|port| !port.is_empty() && port.parse::().is_ok()) + }) +} + +fn valid_action_name(value: &str) -> bool { + let bytes = value.as_bytes(); + !bytes.is_empty() + && bytes[0].is_ascii_lowercase() + && !value.ends_with('-') + && !value.contains("--") + && bytes + .iter() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || *byte == b'-') +} + +fn find_subslice(haystack: &[u8], needle: &[u8]) -> Option { + haystack + .windows(needle.len()) + .position(|window| window == needle) +} + +fn write_response(stream: &mut TcpStream, response: Response) -> Result<()> { + let reason = match response.status { + 200 => "OK", + 202 => "Accepted", + 204 => "No Content", + 400 => "Bad Request", + 403 => "Forbidden", + 404 => "Not Found", + 405 => "Method Not Allowed", + 422 => "Unprocessable Entity", + 503 => "Service Unavailable", + _ => "Error", + }; + write!(stream, "HTTP/1.1 {} {reason}\r\n", response.status)?; + for (name, value) in response.headers { + write!(stream, "{name}: {value}\r\n")?; + } + write!( + stream, + "Content-Length: {}\r\nConnection: close\r\n\r\n", + response.body.len() + )?; + stream.write_all(&response.body)?; + stream.flush()?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use crate::catalog::{ActionRole, ActionSpec}; + + use super::*; + + fn catalog() -> Catalog { + Catalog { + model_name: "Test".into(), + model_path: PathBuf::from("model.vrm"), + actions: vec![ActionSpec { + name: "wave-hello".into(), + description: "A friendly wave.".into(), + trigger_scenario: "When greeting.".into(), + role: ActionRole::Custom, + clips: vec![PathBuf::from("wave.vrma")], + }], + } + } + + fn request(method: &str, path: &str, body: Value) -> Request { + Request { + method: method.into(), + path: path.into(), + headers: HashMap::from([("host".into(), "127.0.0.1:47831".into())]), + body: serde_json::to_vec(&body).unwrap(), + } + } + + #[test] + fn persona_events_are_normalized_and_clamped() { + assert!(matches!( + normalize_event(&json!({ "type": "audio-level", "level": 4.0 }), &catalog()), + Some(BridgeCommand::AudioLevel(1.0)) + )); + assert!( + normalize_event( + &json!({ + "type": "state", + "state": { + "phase": "active", + "activity": "singing", + "microphoneMuted": false, + "outputMuted": false + } + }), + &catalog() + ) + .is_none() + ); + assert!( + normalize_event( + &json!({ "type": "animation", "animation_name": "not-installed" }), + &catalog() + ) + .is_none() + ); + } + + #[test] + fn bridge_rejects_loopback_prefix_spoofing() { + assert!(allowed_host("localhost:47831")); + assert!(allowed_origin("https://127.0.0.1:47831")); + assert!(!allowed_host("localhost.evil.example")); + assert!(!allowed_host("localhost:not-a-port")); + assert!(!allowed_origin("https://localhost.evil.example")); + assert!(!allowed_origin("http://127.0.0.1:47831.evil.example")); + assert!(!allowed_origin("codex-app://trusted/path")); + } + + #[test] + fn mcp_lists_the_persona_compatible_tools() { + let (sender, _receiver) = mpsc::sync_channel(COMMAND_QUEUE_CAPACITY); + let status = Arc::new(Mutex::new(StatusSnapshot::new())); + let response = route( + &request( + "POST", + "/mcp", + json!({ "jsonrpc": "2.0", "id": 1, "method": "tools/list" }), + ), + &catalog(), + &status, + &sender, + ); + let body: Value = serde_json::from_slice(&response.body).unwrap(); + let names: Vec<&str> = body + .pointer("/result/tools") + .unwrap() + .as_array() + .unwrap() + .iter() + .filter_map(|tool| tool.get("name")?.as_str()) + .collect(); + assert_eq!( + names, + [ + "play_animation", + "list_animations", + "control_window", + "get_status" + ] + ); + } + + #[test] + fn mcp_negotiates_the_supported_protocol_version() { + let (sender, _receiver) = mpsc::sync_channel(COMMAND_QUEUE_CAPACITY); + let status = Arc::new(Mutex::new(StatusSnapshot::new())); + let response = route( + &request( + "POST", + "/mcp", + json!({ + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { "protocolVersion": "2099-01-01" } + }), + ), + &catalog(), + &status, + &sender, + ); + let body: Value = serde_json::from_slice(&response.body).unwrap(); + assert_eq!( + body.pointer("/result/protocolVersion") + .and_then(Value::as_str), + Some(MCP_PROTOCOL_VERSION) + ); + } + + #[test] + fn unknown_animation_is_rejected_without_a_command() { + let (sender, receiver) = mpsc::sync_channel(COMMAND_QUEUE_CAPACITY); + let status = Arc::new(Mutex::new(StatusSnapshot::new())); + let response = route( + &request( + "POST", + "/mcp", + json!({ + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": { + "name": "play_animation", + "arguments": { "animation": "not-installed" } + } + }), + ), + &catalog(), + &status, + &sender, + ); + let body: Value = serde_json::from_slice(&response.body).unwrap(); + assert_eq!( + body.pointer("/result/isError").and_then(Value::as_bool), + Some(true) + ); + assert!(receiver.try_recv().is_err()); + } +} diff --git a/crates/pocket-persona/src/catalog.rs b/crates/pocket-persona/src/catalog.rs new file mode 100644 index 0000000..8837bf0 --- /dev/null +++ b/crates/pocket-persona/src/catalog.rs @@ -0,0 +1,327 @@ +//! Read the immutable subset of Persona's `library.json` format. +//! +//! Keeping the same data contract lets the native renderer consume an +//! existing Persona asset library without copying paths into a second config. + +use std::path::{Component, Path, PathBuf}; + +use anyhow::{Context, Result, bail}; +use serde::Deserialize; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ActionRole { + Idle, + Speaking, + Custom, +} + +#[derive(Clone, Debug)] +pub struct ActionSpec { + pub name: String, + pub description: String, + pub trigger_scenario: String, + pub role: ActionRole, + pub clips: Vec, +} + +#[derive(Clone, Debug)] +pub struct Catalog { + pub model_name: String, + pub model_path: PathBuf, + pub actions: Vec, +} + +#[derive(Deserialize)] +struct LibraryFile { + schema_version: u32, + default_model_id: Option, + models: Vec, + animations: Vec, +} + +#[derive(Deserialize)] +struct ModelRecord { + id: String, + model_name: String, + asset_path: String, +} + +#[derive(Deserialize)] +struct AnimationRecord { + id: String, + animation_name: String, + animation_description: String, + animation_trigger_scenario: String, + animation_type: Option, + asset_paths: Vec, +} + +impl Catalog { + pub fn from_path(path: &Path) -> Result { + let bytes = std::fs::read(path) + .with_context(|| format!("reading Persona library {}", path.display()))?; + let base = path.parent().unwrap_or_else(|| Path::new(".")); + Self::from_slice(&bytes, base) + } + + fn from_slice(bytes: &[u8], base: &Path) -> Result { + let file: LibraryFile = + serde_json::from_slice(bytes).context("parsing Persona library.json")?; + if file.schema_version != 1 { + bail!( + "unsupported Persona library schema {}; expected 1", + file.schema_version + ); + } + if file.models.is_empty() { + bail!("Persona library has no configured model"); + } + let mut model_ids = std::collections::HashSet::new(); + for model in &file.models { + validate_id(&model.id, "model id")?; + if !model_ids.insert(model.id.as_str()) { + bail!("duplicate Persona model id '{}'", model.id); + } + if model.model_name.trim().is_empty() { + bail!("Persona model name cannot be empty"); + } + } + let model = match file.default_model_id { + Some(ref id) => file + .models + .iter() + .find(|model| model.id == *id) + .with_context(|| format!("default Persona model '{id}' does not exist"))?, + None => &file.models[0], + }; + let model_path = resolve_media(base, &model.asset_path, "vrm")?; + + let mut ids = std::collections::HashSet::new(); + let mut names = std::collections::HashSet::new(); + let mut configured = Vec::with_capacity(file.animations.len()); + for animation in file.animations { + validate_id(&animation.id, "animation id")?; + if !ids.insert(animation.id.clone()) { + bail!("duplicate Persona animation id '{}'", animation.id); + } + let name = animation.animation_name.trim().to_ascii_lowercase(); + validate_action_name(&name)?; + if !names.insert(name.clone()) { + bail!("duplicate Persona action '{name}'"); + } + if animation.animation_description.trim().is_empty() + || animation.animation_trigger_scenario.trim().is_empty() + { + bail!("Persona action '{name}' needs description and trigger scenario"); + } + let role = match animation.animation_type.as_deref() { + Some("IDLE") => ActionRole::Idle, + Some("TALK") => ActionRole::Speaking, + None | Some("GREETING" | "HAPPY" | "FINGER_GUN" | "DANCE") => ActionRole::Custom, + Some(kind) => bail!("invalid Persona animation type '{kind}'"), + }; + match animation.id.as_str() { + "system-idle" if name != "idle" || role != ActionRole::Idle => { + bail!("system-idle must retain the idle name and IDLE type") + } + "system-speaking" if name != "speaking" || role != ActionRole::Speaking => { + bail!("system-speaking must retain the speaking name and TALK type") + } + "system-idle" | "system-speaking" => {} + _ if matches!(role, ActionRole::Idle | ActionRole::Speaking) => { + bail!("idle and speaking roles belong to their permanent system slots") + } + _ => {} + } + let clips = animation + .asset_paths + .iter() + .map(|asset| resolve_media(base, asset, "vrma")) + .collect::>>()?; + configured.push(( + animation.id, + ActionSpec { + name, + description: animation.animation_description.trim().into(), + trigger_scenario: animation.animation_trigger_scenario.trim().into(), + role, + clips, + }, + )); + } + + let mut take_system = |id: &str, fallback: ActionSpec| -> Result { + if let Some(index) = configured.iter().position(|(candidate, _)| candidate == id) { + return Ok(configured.remove(index).1); + } + if names.contains(&fallback.name) { + bail!( + "Persona action '{}' conflicts with the permanent {id} slot", + fallback.name + ); + } + Ok(fallback) + }; + let idle = take_system( + "system-idle", + system_action("idle", ActionRole::Idle, "A calm resting motion."), + )?; + let speaking = take_system( + "system-speaking", + system_action( + "speaking", + ActionRole::Speaking, + "Conversational body motion.", + ), + )?; + let mut actions = Vec::with_capacity(configured.len() + 2); + actions.push(idle); + actions.push(speaking); + actions.extend(configured.into_iter().map(|(_, action)| action)); + + Ok(Self { + model_name: model.model_name.clone(), + model_path, + actions, + }) + } + + pub fn action(&self, name: &str) -> Option<&ActionSpec> { + self.actions.iter().find(|action| action.name == name) + } + + #[cfg(test)] + pub fn action_for_role(&self, role: ActionRole) -> Option<&ActionSpec> { + self.actions.iter().find(|action| action.role == role) + } + + pub fn playable_actions(&self) -> impl Iterator { + self.actions + .iter() + .filter(|action| !action.clips.is_empty()) + } +} + +fn resolve_media(base: &Path, raw: &str, extension: &str) -> Result { + let normalized = raw.replace('\\', "/"); + let relative = Path::new(&normalized); + if raw.trim().is_empty() + || relative.is_absolute() + || relative + .components() + .any(|part| matches!(part, Component::ParentDir | Component::RootDir)) + { + bail!("Persona asset path must stay relative to library.json: {raw}"); + } + if relative + .extension() + .and_then(|value| value.to_str()) + .is_none_or(|value| !value.eq_ignore_ascii_case(extension)) + { + bail!("Persona asset path must end in .{extension}: {raw}"); + } + Ok(base.join(relative)) +} + +fn system_action(name: &str, role: ActionRole, description: &str) -> ActionSpec { + ActionSpec { + name: name.into(), + description: description.into(), + trigger_scenario: "Used automatically by the voice state machine.".into(), + role, + clips: Vec::new(), + } +} + +fn validate_id(value: &str, label: &str) -> Result<()> { + if value.is_empty() + || !value.bytes().enumerate().all(|(index, byte)| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || (index > 0 && byte == b'-') + }) + { + bail!("invalid Persona {label}: {value}"); + } + Ok(()) +} + +fn validate_action_name(value: &str) -> Result<()> { + validate_id(value, "action name")?; + if !value.as_bytes()[0].is_ascii_lowercase() || value.contains("--") || value.ends_with('-') { + bail!("invalid Persona action name: {value}"); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + const LIBRARY: &str = r#"{ + "schema_version": 1, + "default_model_id": "model-a", + "models": [{ + "id": "model-a", + "model_name": "Model A", + "asset_path": "models/model.vrm" + }], + "animations": [{ + "id": "system-idle", + "animation_name": "idle", + "animation_description": "A calm resting motion.", + "animation_trigger_scenario": "While waiting.", + "animation_type": "IDLE", + "asset_paths": ["animations/idle.vrma"] + }, { + "id": "wave", + "animation_name": "wave-hello", + "animation_description": "A friendly wave.", + "animation_trigger_scenario": "When greeting.", + "animation_type": "GREETING", + "asset_paths": [] + }] + }"#; + + #[test] + fn reads_persona_library_without_translation() { + let catalog = Catalog::from_slice(LIBRARY.as_bytes(), Path::new("/tmp/library")).unwrap(); + assert_eq!(catalog.model_name, "Model A"); + assert_eq!( + catalog.model_path, + Path::new("/tmp/library/models/model.vrm") + ); + assert_eq!( + catalog.action_for_role(ActionRole::Idle).unwrap().name, + "idle" + ); + assert_eq!(catalog.action("wave-hello").unwrap().clips.len(), 0); + assert_eq!(catalog.playable_actions().count(), 1); + } + + #[test] + fn asset_paths_cannot_escape_the_library() { + let bad = LIBRARY.replace("models/model.vrm", "../secret.vrm"); + let error = Catalog::from_slice(bad.as_bytes(), Path::new("/tmp")).unwrap_err(); + assert!(error.to_string().contains("stay relative")); + } + + #[test] + fn action_names_keep_persona_mcp_shape() { + let bad = LIBRARY.replace("wave-hello", "Wave Hello"); + let error = Catalog::from_slice(bad.as_bytes(), Path::new("/tmp")).unwrap_err(); + assert!(error.to_string().contains("action name")); + } + + #[test] + fn packaged_ids_and_system_roles_follow_persona_invariants() { + let duplicate = LIBRARY.replace(r#""id": "wave""#, r#""id": "system-idle""#); + let error = Catalog::from_slice(duplicate.as_bytes(), Path::new("/tmp")).unwrap_err(); + assert!(error.to_string().contains("duplicate Persona animation id")); + + let extra_talk = LIBRARY.replace( + r#""animation_type": "GREETING""#, + r#""animation_type": "TALK""#, + ); + let error = Catalog::from_slice(extra_talk.as_bytes(), Path::new("/tmp")).unwrap_err(); + assert!(error.to_string().contains("permanent system slots")); + } +} diff --git a/crates/pocket-persona/src/guest.rs b/crates/pocket-persona/src/guest.rs new file mode 100644 index 0000000..b518cd8 --- /dev/null +++ b/crates/pocket-persona/src/guest.rs @@ -0,0 +1,109 @@ +//! Pocket `persona` surface: bounded facts into QuickJS, queued intents out. + +use std::cell::RefCell; +use std::rc::Rc; + +use anyhow::{Result, anyhow}; +use pocket_mod::Guest; +use pocket_mod::qjs::{Array, Function, Object}; + +#[derive(Clone, Debug)] +pub enum GuestCommand { + PlayAnimation(String), + SetExpression(String, f32), + Quit, +} + +pub struct GuestState<'a> { + pub t: f64, + pub activity: &'a str, + pub audio_level: f32, + pub animation: &'a str, + pub blink: f32, + pub render_fps: f32, +} + +pub struct GuestEvent<'a> { + pub kind: &'a str, + pub value: &'a str, +} + +pub struct PersonaGuest { + guest: Guest, + commands: Rc>>, +} + +impl PersonaGuest { + pub fn boot(bundle: &str, model_name: &str, action_names: &[String]) -> Result { + let guest = Guest::new()?; + let commands: Rc>> = Rc::default(); + let queued = commands.clone(); + let model_name = model_name.to_string(); + let action_names = action_names.to_vec(); + guest.mount("persona", move |ctx, namespace| { + let boot = Object::new(ctx.clone())?; + boot.set("model", model_name.as_str())?; + boot.set("actions", action_names.clone())?; + namespace.set("boot", boot)?; + + let queue = queued.clone(); + namespace.set( + "playAnimation", + Function::new(ctx.clone(), move |name: String| { + queue.borrow_mut().push(GuestCommand::PlayAnimation(name)); + })?, + )?; + let queue = queued.clone(); + namespace.set( + "setExpression", + Function::new(ctx.clone(), move |name: String, weight: f64| { + queue + .borrow_mut() + .push(GuestCommand::SetExpression(name, weight as f32)); + })?, + )?; + let queue = queued.clone(); + namespace.set( + "quit", + Function::new(ctx.clone(), move || { + queue.borrow_mut().push(GuestCommand::Quit); + })?, + )?; + Ok(()) + })?; + guest.eval("pocket-persona", bundle)?; + Ok(Self { guest, commands }) + } + + pub fn turn( + &self, + state: &GuestState<'_>, + events: &[GuestEvent<'_>], + ) -> Result> { + self.guest.with(|ctx| -> Result<()> { + let namespace: Object = ctx.globals().get("persona")?; + let Ok(dispatch) = namespace.get::<_, Function>("__dispatch") else { + return Ok(()); + }; + let js_state = Object::new(ctx.clone())?; + js_state.set("t", state.t)?; + js_state.set("activity", state.activity)?; + js_state.set("audioLevel", state.audio_level as f64)?; + js_state.set("animation", state.animation)?; + js_state.set("blink", state.blink as f64)?; + js_state.set("renderFps", state.render_fps as f64)?; + let js_events = Array::new(ctx.clone())?; + for (index, event) in events.iter().enumerate() { + let js_event = Object::new(ctx.clone())?; + js_event.set("type", event.kind)?; + js_event.set("value", event.value)?; + js_events.set(index, js_event)?; + } + dispatch + .call::<_, ()>((js_state, js_events)) + .map_err(|error| anyhow!("persona.__dispatch threw: {error}")) + })?; + self.guest.frame(0)?; + Ok(self.commands.borrow_mut().drain(..).collect()) + } +} diff --git a/crates/pocket-persona/src/main.rs b/crates/pocket-persona/src/main.rs new file mode 100644 index 0000000..2fe274a --- /dev/null +++ b/crates/pocket-persona/src/main.rs @@ -0,0 +1,188 @@ +//! Pocket Persona: a native renderer vertical slice compatible with Persona's +//! asset catalog, local event bridge, and MCP tools. + +mod bridge; +mod catalog; +mod guest; +mod sim; +mod widget; + +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use anyhow::{Context, Result, bail}; +use pocket_widget::{WidgetConfig, WidgetGame}; +use pocket3d::gpu::{Gpu, OffscreenTarget}; +use pocket3d::input::Input; +use pocket3d::renderer::Renderer; + +use catalog::Catalog; +use widget::{PersonaConfig, PersonaWidget}; + +const DEFAULT_SIZE: (u32, u32) = (430, 680); + +struct Args { + library: PathBuf, + bundle: PathBuf, + bridge_port: Option, + fps: f32, + size: (u32, u32), + max_texture_dim: u32, + headless_shot: Option, + ticks: u32, +} + +fn main() -> Result<()> { + env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init(); + let args = parse_args(std::env::args().skip(1).collect())?; + let catalog = Arc::new(Catalog::from_path(&args.library)?); + let widget = PersonaWidget::new(PersonaConfig { + catalog, + bundle_path: args.bundle, + bridge_port: if args.headless_shot.is_some() { + None + } else { + args.bridge_port + }, + size: args.size, + max_texture_dim: args.max_texture_dim, + }); + if let Some(output) = args.headless_shot { + return headless_shot(widget, args.size, args.fps, args.ticks, &output); + } + pocket_widget::run( + WidgetConfig { + title: "Pocket Persona".into(), + size: args.size, + tick_hz: args.fps, + max_fps: args.fps, + transparent: true, + decorations: false, + always_on_top: true, + resizable: true, + min_size: (320, 480), + ime: false, + }, + widget, + ) +} + +fn parse_args(values: Vec) -> Result { + let mut library = None; + let mut bundle = default_repo_root().join("dist/pocket-persona/guest.js"); + let mut bridge_port = Some(47_831); + let mut fps = 60.0; + let mut size = DEFAULT_SIZE; + let mut max_texture_dim = 2048; + let mut headless_shot = None; + let mut ticks = 90; + let mut index = 0; + while index < values.len() { + let flag = &values[index]; + let next = |index: &mut usize| -> Result<&str> { + *index += 1; + values + .get(*index) + .map(String::as_str) + .with_context(|| format!("{flag} needs a value")) + }; + match flag.as_str() { + "--library" => library = Some(PathBuf::from(next(&mut index)?)), + "--bundle" => bundle = PathBuf::from(next(&mut index)?), + "--bridge-port" => bridge_port = Some(next(&mut index)?.parse()?), + "--no-bridge" => bridge_port = None, + "--fps" | "--max-fps" => fps = next(&mut index)?.parse()?, + "--size" => size = parse_size(next(&mut index)?)?, + "--max-texture-dim" => max_texture_dim = next(&mut index)?.parse()?, + "--headless-shot" => headless_shot = Some(PathBuf::from(next(&mut index)?)), + "--ticks" => ticks = next(&mut index)?.parse()?, + "--help" | "-h" => { + println!( + "Pocket Persona\n\ + \n\ + Usage: pocket-persona --library [options]\n\ + \n\ + Options:\n\ + \t--bundle Pocket policy bundle\n\ + \t--bridge-port Persona HTTP/MCP port (default 47831; 0 = any)\n\ + \t--no-bridge Disable HTTP/MCP\n\ + \t--fps Fixed update/render cap (default 60)\n\ + \t--size x Logical window size (default 430x680)\n\ + \t--max-texture-dim Texture cap (default 2048)\n\ + \t--headless-shot Render one offscreen verification frame\n\ + \t--ticks Headless fixed steps (default 90)" + ); + std::process::exit(0); + } + _ => bail!("unknown Pocket Persona argument: {flag}"), + } + index += 1; + } + if !(1.0..=240.0).contains(&fps) { + bail!("--fps must be between 1 and 240"); + } + if !(256..=4096).contains(&max_texture_dim) { + bail!("--max-texture-dim must be between 256 and 4096"); + } + let library = library.context("--library is required")?; + Ok(Args { + library, + bundle, + bridge_port, + fps, + size, + max_texture_dim, + headless_shot, + ticks, + }) +} + +fn parse_size(value: &str) -> Result<(u32, u32)> { + let (width, height) = value + .split_once(['x', 'X']) + .context("--size must look like 430x680")?; + let size = (width.parse()?, height.parse()?); + if size.0 < 64 || size.1 < 64 || size.0 > 4096 || size.1 > 4096 { + bail!("--size dimensions must be between 64 and 4096"); + } + Ok(size) +} + +fn default_repo_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../..") +} + +fn headless_shot( + mut widget: PersonaWidget, + size: (u32, u32), + fps: f32, + ticks: u32, + output: &Path, +) -> Result<()> { + let gpu = Gpu::new_headless()?; + let mut renderer = Renderer::new(&gpu, pocket3d::gpu::OFFSCREEN_FORMAT)?; + widget.init(&gpu, &mut renderer)?; + let input = Input::default(); + for _ in 0..ticks { + widget.tick(1.0 / fps, &input, size)?; + } + widget.prepare(&gpu)?; + let (scene, camera, hud) = widget.compose(ticks as f32 / fps, size); + let target = OffscreenTarget::new(&gpu, size.0, size.1); + renderer.render(&gpu, &target.view, size, scene, camera, hud); + target.save_png(&gpu, output)?; + println!("wrote {}", output.display()); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_window_size() { + assert_eq!(parse_size("430x680").unwrap(), (430, 680)); + assert!(parse_size("430").is_err()); + assert!(parse_size("20x20").is_err()); + } +} diff --git a/crates/pocket-persona/src/sim.rs b/crates/pocket-persona/src/sim.rs new file mode 100644 index 0000000..31a7825 --- /dev/null +++ b/crates/pocket-persona/src/sim.rs @@ -0,0 +1,215 @@ +//! Deterministic facial behavior for the Pocket Persona renderer. +//! +//! The timings and amplitude-only viseme driver mirror Persona's React hooks, +//! but live in the native fixed-step core so the QuickJS guest only decides +//! policy. + +const BLINK_MIN_INTERVAL: f32 = 2.0; +const BLINK_MAX_INTERVAL: f32 = 6.0; +const BLINK_DURATION: f32 = 0.24; +const LIP_AUDIBLE_THRESHOLD: f32 = 0.008; +const VISEME_COUNT: usize = 5; + +#[derive(Clone)] +pub struct Pcg32 { + state: u64, +} + +impl Pcg32 { + pub fn new(seed: u64) -> Self { + let mut rng = Self { + state: seed.wrapping_add(0x853c_49e6_748f_ea9b), + }; + rng.next_u32(); + rng + } + + fn next_u32(&mut self) -> u32 { + let old = self.state; + self.state = old + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + let xorshifted = (((old >> 18) ^ old) >> 27) as u32; + xorshifted.rotate_right((old >> 59) as u32) + } + + pub fn next_f32(&mut self) -> f32 { + (self.next_u32() >> 8) as f32 / (1u32 << 24) as f32 + } + + pub fn range(&mut self, lo: f32, hi: f32) -> f32 { + lo + self.next_f32() * (hi - lo) + } + + pub fn index(&mut self, len: usize) -> usize { + if len <= 1 { + 0 + } else { + ((self.next_f32() * len as f32) as usize).min(len - 1) + } + } +} + +#[derive(Clone, Copy, Debug, Default)] +pub struct FaceOutputs { + pub blink: f32, + pub blink_changed: bool, + pub visemes: [f32; VISEME_COUNT], + pub visemes_changed: bool, +} + +pub struct FaceSim { + rng: Pcg32, + blink_wait: f32, + blink_progress: Option, + last_blink: f32, + lip_smoothed: f32, + lip_phase: f32, + last_visemes: [f32; VISEME_COUNT], +} + +impl FaceSim { + pub fn new(seed: u64) -> Self { + let mut rng = Pcg32::new(seed); + let blink_wait = rng.range(BLINK_MIN_INTERVAL, BLINK_MAX_INTERVAL); + Self { + rng, + blink_wait, + blink_progress: None, + last_blink: 0.0, + lip_smoothed: 0.0, + lip_phase: 0.0, + last_visemes: [0.0; VISEME_COUNT], + } + } + + pub fn tick(&mut self, dt: f32, audio_level: f32, speaking: bool) -> FaceOutputs { + let dt = dt.max(0.0); + let blink = if let Some(progress) = self.blink_progress.as_mut() { + *progress += dt / BLINK_DURATION; + if *progress >= 1.0 { + self.blink_progress = None; + self.blink_wait = self.rng.range(BLINK_MIN_INTERVAL, BLINK_MAX_INTERVAL); + 0.0 + } else { + (core::f32::consts::PI * *progress).sin() + } + } else { + self.blink_wait -= dt; + if self.blink_wait <= 0.0 { + self.blink_progress = Some(f32::EPSILON); + } + 0.0 + }; + + let audible = speaking && audio_level > LIP_AUDIBLE_THRESHOLD; + let normalized = if audible { + (audio_level.clamp(0.0, 1.0) * 2.8).min(1.0) + } else { + 0.0 + }; + let tau = if normalized > self.lip_smoothed { + 0.055 + } else { + 0.1 + }; + let smoothing = 1.0 - (-dt / tau).exp(); + self.lip_smoothed += (normalized - self.lip_smoothed) * smoothing; + self.lip_phase += dt * (8.0 + self.lip_smoothed * 9.0); + let active = self.lip_phase.floor() as usize % VISEME_COUNT; + let mut visemes = [0.0; VISEME_COUNT]; + for (index, weight) in visemes.iter_mut().enumerate() { + let shape = + (1.0 - (index as isize - active as isize).unsigned_abs() as f32 * 0.72).max(0.0); + let flutter = 0.74 + (self.lip_phase * 5.7 + index as f32).sin() * 0.18; + *weight = (self.lip_smoothed * shape * flutter).min(0.62); + } + + let blink_changed = blink.to_bits() != self.last_blink.to_bits(); + let visemes_changed = visemes + .iter() + .zip(self.last_visemes) + .any(|(a, b)| a.to_bits() != b.to_bits()); + self.last_blink = blink; + self.last_visemes = visemes; + FaceOutputs { + blink, + blink_changed, + visemes, + visemes_changed, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fixed_seed_replays_exactly() { + let mut a = FaceSim::new(7); + let mut b = FaceSim::new(7); + for frame in 0..3_600 { + let level = if frame % 240 < 120 { 0.2 } else { 0.0 }; + let (a, b) = ( + a.tick(1.0 / 60.0, level, true), + b.tick(1.0 / 60.0, level, true), + ); + assert_eq!(a.blink.to_bits(), b.blink.to_bits()); + assert_eq!(a.visemes.map(f32::to_bits), b.visemes.map(f32::to_bits)); + } + } + + #[test] + fn blink_intervals_match_persona_bounds() { + let mut sim = FaceSim::new(42); + let mut starts = Vec::new(); + let mut previous = 0.0; + for frame in 0..60 * 120 { + let output = sim.tick(1.0 / 60.0, 0.0, false); + if previous == 0.0 && output.blink > 0.0 { + starts.push(frame as f32 / 60.0); + } + previous = output.blink; + } + assert!(starts.len() > 15); + for pair in starts.windows(2) { + let interval = pair[1] - pair[0] - BLINK_DURATION; + assert!( + (BLINK_MIN_INTERVAL - 1.0 / 60.0..=BLINK_MAX_INTERVAL + 1.0 / 60.0) + .contains(&interval), + "blink wait {interval}" + ); + } + } + + #[test] + fn lip_sync_rises_and_releases_smoothly() { + let mut sim = FaceSim::new(1); + let mut peak = 0.0f32; + for _ in 0..60 { + peak = peak.max( + sim.tick(1.0 / 60.0, 0.3, true) + .visemes + .into_iter() + .fold(0.0, f32::max), + ); + } + assert!(peak > 0.4); + let first_release = sim + .tick(1.0 / 60.0, 0.0, false) + .visemes + .into_iter() + .fold(0.0, f32::max); + assert!(first_release > 0.0, "release should not snap"); + let mut settled = first_release; + for _ in 0..120 { + settled = sim + .tick(1.0 / 60.0, 0.0, false) + .visemes + .into_iter() + .fold(0.0, f32::max); + } + assert!(settled < 1e-4); + } +} diff --git a/crates/pocket-persona/src/widget.rs b/crates/pocket-persona/src/widget.rs new file mode 100644 index 0000000..b9e8cfd --- /dev/null +++ b/crates/pocket-persona/src/widget.rs @@ -0,0 +1,758 @@ +//! Native Persona renderer built from the Pocket character substrate. + +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Instant; + +use anyhow::{Context, Result}; +use glam::{Mat4, Vec2, Vec3}; +use pocket_vrm::{SpringSolver, VrmDoc}; +use pocket_widget::{WidgetGame, WindowCommand}; +use pocket3d::anim::{Clip, NodeTrs}; +use pocket3d::camera::Camera; +use pocket3d::gpu::Gpu; +use pocket3d::hud::Hud; +use pocket3d::input::Input; +use pocket3d::model::{ModelAsset, ModelInstance, ModelLoadOptions}; +use pocket3d::renderer::Renderer; +use pocket3d::scene::Scene; +use winit::event::MouseButton; + +use crate::bridge::{Bridge, BridgeCommand, VoiceState}; +use crate::catalog::{ActionRole, Catalog}; +use crate::guest::{GuestCommand, GuestEvent, GuestState, PersonaGuest}; +use crate::sim::{FaceSim, Pcg32}; + +const VOICE_IDLE_DELAY: f32 = 0.65; +const ORBIT_RADIANS_PER_PIXEL: f32 = 0.006; +const PAN_UNITS_PER_PIXEL: f32 = 0.0015; + +pub struct PersonaConfig { + pub catalog: Arc, + pub bundle_path: PathBuf, + pub bridge_port: Option, + pub size: (u32, u32), + pub max_texture_dim: u32, +} + +struct Action { + name: String, + role: ActionRole, + clips: Vec, +} + +#[derive(Clone, Copy)] +struct Playback { + action: usize, + clip: Option, + time: f32, + one_shot: bool, +} + +struct RenderRate { + frames: u32, + window_start: Instant, + last_frame: Option, + intervals_ms: Vec, + fps: f32, + p95_ms: f32, + p99_ms: f32, + max_ms: f32, +} + +impl RenderRate { + fn new() -> Self { + Self { + frames: 0, + window_start: Instant::now(), + last_frame: None, + intervals_ms: Vec::with_capacity(256), + fps: 0.0, + p95_ms: 0.0, + p99_ms: 0.0, + max_ms: 0.0, + } + } + + fn rendered(&mut self) { + let now = Instant::now(); + if let Some(previous) = self.last_frame { + self.intervals_ms + .push((now - previous).as_secs_f32() * 1000.0); + } + self.last_frame = Some(now); + self.frames += 1; + let elapsed = self.window_start.elapsed().as_secs_f32(); + if elapsed >= 1.0 { + self.fps = self.frames as f32 / elapsed; + self.intervals_ms.sort_by(f32::total_cmp); + self.p95_ms = percentile(&self.intervals_ms, 0.95); + self.p99_ms = percentile(&self.intervals_ms, 0.99); + self.max_ms = self.intervals_ms.last().copied().unwrap_or(0.0); + self.frames = 0; + self.intervals_ms.clear(); + self.window_start = now; + } + } +} + +pub struct PersonaWidget { + config: PersonaConfig, + bridge: Option, + guest: Option, + model: Option>, + vrm: Option, + actions: Vec, + last_clip: Vec>, + playback: Option, + fade_from: Option>, + fade_elapsed: f32, + fade_duration: f32, + rng: Pcg32, + face: FaceSim, + springs: Option, + locals: Vec, + sampled_locals: Vec, + globals: Vec, + scene: Scene, + camera: Camera, + hud: Hud, + camera_target: Vec3, + camera_distance: f32, + camera_yaw: f32, + camera_pitch: f32, + last_cursor: Option, + last_window_size: (u32, u32), + voice: VoiceState, + audio_level: f32, + voice_idle_delay: Option, + pending_events: Vec<(String, String)>, + window_command: Option, + dirty: bool, + exit: bool, + tick_count: u64, + render_rate: RenderRate, +} + +impl PersonaWidget { + pub fn new(config: PersonaConfig) -> Self { + let last_window_size = config.size; + Self { + config, + bridge: None, + guest: None, + model: None, + vrm: None, + actions: Vec::new(), + last_clip: Vec::new(), + playback: None, + fade_from: None, + fade_elapsed: 0.0, + fade_duration: 0.0, + rng: Pcg32::new(0x0070_6572_736f_6e61), + face: FaceSim::new(0xface_cafe), + springs: None, + locals: Vec::new(), + sampled_locals: Vec::new(), + globals: Vec::new(), + scene: Scene::default(), + camera: Camera::default(), + hud: Hud::default(), + camera_target: Vec3::ZERO, + camera_distance: 1.0, + camera_yaw: 0.0, + camera_pitch: 0.0, + last_cursor: None, + last_window_size, + voice: VoiceState::default(), + audio_level: 0.0, + voice_idle_delay: None, + pending_events: Vec::new(), + window_command: None, + dirty: true, + exit: false, + tick_count: 0, + render_rate: RenderRate::new(), + } + } + + fn active_action_name(&self) -> &str { + self.playback + .and_then(|playback| self.actions.get(playback.action)) + .map(|action| action.name.as_str()) + .unwrap_or("idle") + } + + fn action_index_for_role(&self, role: ActionRole) -> Option { + self.actions.iter().position(|action| action.role == role) + } + + fn action_index(&self, name: &str) -> Option { + self.actions.iter().position(|action| action.name == name) + } + + fn voice_action_index(&self) -> Option { + self.action_index_for_role(if self.voice.speaking() { + ActionRole::Speaking + } else { + ActionRole::Idle + }) + } + + fn choose_clip(&mut self, action_index: usize) -> Option { + let count = self.actions.get(action_index)?.clips.len(); + if count == 0 { + return None; + } + let previous = self.last_clip[action_index]; + let mut index = self.rng.index(count); + if count > 1 && Some(index) == previous { + index = (index + 1 + self.rng.index(count - 1)) % count; + } + self.last_clip[action_index] = Some(index); + Some(index) + } + + fn transition_duration(&self, next: usize) -> f32 { + let Some(previous) = self.playback else { + return 0.0; + }; + let previous_role = self.actions[previous.action].role; + let next_role = self.actions[next].role; + if previous_role == ActionRole::Speaking && next_role == ActionRole::Idle { + 1.15 + } else if next_role == ActionRole::Speaking { + 0.85 + } else { + 0.7 + } + } + + fn start_action_index(&mut self, action_index: usize, one_shot: bool) { + if action_index >= self.actions.len() { + return; + } + let old_name = self.active_action_name().to_string(); + let duration = self.transition_duration(action_index); + let clip = self.choose_clip(action_index); + self.fade_from = (duration > 0.0 && !self.locals.is_empty()).then(|| self.locals.clone()); + self.fade_elapsed = 0.0; + self.fade_duration = duration; + self.playback = Some(Playback { + action: action_index, + clip, + time: 0.0, + one_shot, + }); + let new_name = self.actions[action_index].name.clone(); + if old_name != new_name { + self.pending_events + .push(("animationChanged".into(), new_name.clone())); + } + if let Some(bridge) = &self.bridge { + bridge.update_status(|status| status.active_animation = new_name); + } + self.dirty = true; + } + + fn start_action(&mut self, name: &str, one_shot: bool) -> bool { + let Some(index) = self.action_index(name) else { + return false; + }; + if one_shot && self.actions[index].clips.is_empty() { + return false; + } + self.start_action_index(index, one_shot); + true + } + + fn resume_voice_action(&mut self) { + if let Some(index) = self.voice_action_index() { + self.start_action_index(index, false); + } + } + + fn apply_bridge_commands(&mut self) { + let commands: Vec<_> = self + .bridge + .as_ref() + .map(|bridge| bridge.drain().collect()) + .unwrap_or_default(); + for command in commands { + match command { + BridgeCommand::Voice(voice) => { + let old_activity = self.voice.activity.clone(); + self.voice = voice; + if old_activity != self.voice.activity { + self.pending_events + .push(("voiceChanged".into(), self.voice.activity.clone())); + } + if self.voice.speaking() { + self.voice_idle_delay = None; + if !self.playback.is_some_and(|playback| playback.one_shot) + && let Some(index) = self.action_index_for_role(ActionRole::Speaking) + && self + .playback + .is_none_or(|playback| playback.action != index) + { + self.start_action_index(index, false); + } + } else if self.voice.phase == "active" && self.voice.activity == "listening" { + self.voice_idle_delay = Some(VOICE_IDLE_DELAY); + } else { + self.voice_idle_delay = None; + if !self.playback.is_some_and(|playback| playback.one_shot) { + self.resume_voice_action(); + } + } + } + BridgeCommand::AudioLevel(level) => self.audio_level = level, + BridgeCommand::PlayAnimation(name) => { + if !self.start_action(&name, true) { + log::warn!("Pocket Persona action is not playable: {name}"); + } + } + BridgeCommand::Window { visible } => { + self.window_command = Some(if visible { + WindowCommand::Show + } else { + WindowCommand::Hide + }); + } + } + } + } + + fn apply_guest_commands(&mut self, commands: Vec) { + for command in commands { + match command { + GuestCommand::PlayAnimation(name) => { + if !self.start_action(&name, true) { + log::warn!("persona.playAnimation: unknown or empty action '{name}'"); + } + } + GuestCommand::SetExpression(name, weight) => { + self.apply_expression(&name, weight.clamp(0.0, 1.0)); + self.dirty = true; + } + GuestCommand::Quit => self.exit = true, + } + } + } + + fn update_voice_delay(&mut self, dt: f32) { + let Some(delay) = self.voice_idle_delay.as_mut() else { + return; + }; + *delay -= dt; + if *delay <= 0.0 { + self.voice_idle_delay = None; + if !self.playback.is_some_and(|playback| playback.one_shot) + && let Some(index) = self.action_index_for_role(ActionRole::Idle) + && self + .playback + .is_none_or(|playback| playback.action != index) + { + self.start_action_index(index, false); + } + } + } + + /// Advance body animation and report whether this tick produced a pose + /// that must be presented. Remember the pre-step fade state so the final + /// crossfade sample is not lost when `fade_from` is cleared. + fn sample_animation(&mut self, dt: f32, model: &ModelAsset) -> bool { + let Some(mut playback) = self.playback else { + model + .skeleton + .sample_locals(None, 0.0, false, &mut self.locals); + return false; + }; + playback.time += dt; + let action = &self.actions[playback.action]; + let clip = playback.clip.and_then(|index| action.clips.get(index)); + let clip_advanced = clip.is_some(); + let fade_advanced = self.fade_from.is_some(); + model.skeleton.sample_locals( + clip, + playback.time, + !playback.one_shot, + &mut self.sampled_locals, + ); + + if let Some(from) = &self.fade_from { + self.fade_elapsed += dt; + let amount = if self.fade_duration <= 0.0 { + 1.0 + } else { + smoothstep01(self.fade_elapsed / self.fade_duration) + }; + self.locals.clear(); + self.locals.extend( + from.iter() + .zip(&self.sampled_locals) + .map(|(from, to)| blend_trs(*from, *to, amount)), + ); + if amount >= 1.0 { + self.fade_from = None; + } + } else { + std::mem::swap(&mut self.locals, &mut self.sampled_locals); + } + self.playback = Some(playback); + + if playback.one_shot && clip.is_none_or(|clip| playback.time >= clip.duration) { + self.playback = None; + self.resume_voice_action(); + } + clip_advanced || fade_advanced + } + + fn apply_expression(&mut self, name: &str, weight: f32) -> bool { + let (Some(vrm), Some(model), Some(instance)) = ( + self.vrm.as_ref(), + self.model.as_ref(), + self.scene.models.first_mut(), + ) else { + return false; + }; + let Some(morph) = instance.morph.as_mut() else { + return false; + }; + let Some(expression) = vrm + .expressions + .iter() + .find(|expression| expression.name == name) + else { + return false; + }; + for binding in &expression.binds { + if let Some(slot) = model.morph_mesh_slot(binding.mesh) { + morph.set_weight(slot, binding.target, weight * binding.weight); + } + } + true + } + + fn apply_face(&mut self, dt: f32) -> f32 { + let outputs = self.face.tick(dt, self.audio_level, self.voice.speaking()); + if outputs.blink_changed { + self.apply_expression("blink", outputs.blink); + self.dirty = true; + } + if outputs.visemes_changed { + const VISEMES: [[&str; 2]; 5] = [ + ["aa", "a"], + ["ee", "e"], + ["ih", "i"], + ["oh", "o"], + ["ou", "u"], + ]; + for (candidates, weight) in VISEMES.iter().zip(outputs.visemes) { + if !self.apply_expression(candidates[0], weight) { + self.apply_expression(candidates[1], weight); + } + } + self.dirty = true; + } + outputs.blink + } + + fn update_camera(&mut self, input: &Input, window_size: (u32, u32)) { + let cursor = input.cursor(); + let previous = self.last_cursor; + self.last_cursor = cursor; + let mut changed = false; + if let (Some(cursor), Some(previous)) = (cursor, previous) { + let delta = cursor - previous; + if input.mouse_button_down(MouseButton::Left) { + self.camera_yaw = (self.camera_yaw - delta.x * ORBIT_RADIANS_PER_PIXEL) + .rem_euclid(core::f32::consts::TAU); + self.camera_pitch = + (self.camera_pitch - delta.y * ORBIT_RADIANS_PER_PIXEL).clamp(-1.2, 1.2); + changed = delta != Vec2::ZERO; + } else if input.mouse_button_down(MouseButton::Right) { + let scale = self.camera_distance * PAN_UNITS_PER_PIXEL; + self.camera_target += Vec3::new(-delta.x * scale, delta.y * scale, 0.0); + changed = delta != Vec2::ZERO; + } + } + let scroll = input.scroll().y; + if scroll != 0.0 { + self.camera_distance = + (self.camera_distance * (-scroll * 0.0015).exp()).clamp(0.4, 20.0); + changed = true; + } + if window_size != self.last_window_size { + self.last_window_size = window_size; + changed = true; + } + if changed { + self.rebuild_camera(window_size); + self.dirty = true; + } + } + + fn rebuild_camera(&mut self, window_size: (u32, u32)) { + let _ = window_size; + let horizontal = Vec3::new(self.camera_yaw.sin(), 0.0, -self.camera_yaw.cos()); + let direction = Vec3::new( + horizontal.x * self.camera_pitch.cos(), + self.camera_pitch.sin(), + horizontal.z * self.camera_pitch.cos(), + ); + self.camera.pos = self.camera_target + direction * self.camera_distance; + self.camera.look_at(self.camera_target); + } +} + +impl WidgetGame for PersonaWidget { + fn init(&mut self, gpu: &Gpu, renderer: &mut Renderer) -> Result<()> { + let started = Instant::now(); + let model = ModelAsset::load_glb_opts( + gpu, + &renderer.model_material_layout, + &renderer.samplers, + &self.config.catalog.model_path, + &ModelLoadOptions { + max_texture_dim: Some(self.config.max_texture_dim), + }, + ) + .context("loading Persona VRM model")?; + let vrm = VrmDoc::from_path(&self.config.catalog.model_path) + .context("parsing Persona VRM extension")?; + + self.actions.clear(); + for action in &self.config.catalog.actions { + let mut clips = Vec::with_capacity(action.clips.len()); + for path in &action.clips { + let bytes = std::fs::read(path) + .with_context(|| format!("reading Persona action {}", path.display()))?; + let animation = pocket_vrm::load_vrma_bytes(&bytes) + .with_context(|| format!("parsing Persona action {}", path.display()))?; + clips.push( + pocket_vrm::retarget(&animation, &vrm.humanoid, &model.skeleton).with_context( + || format!("retargeting Persona action {}", path.display()), + )?, + ); + } + self.actions.push(Action { + name: action.name.clone(), + role: action.role, + clips, + }); + } + self.last_clip = vec![None; self.actions.len()]; + + model + .skeleton + .sample_locals(None, 0.0, false, &mut self.locals); + self.springs = Some(SpringSolver::new( + &vrm.springs, + &model.skeleton, + &self.locals, + )); + + let mut instance = ModelInstance::new(model.clone()); + instance.morph = model.create_morph_state(gpu); + instance.cutout = 0.5; + instance.lit = 0.25; + self.scene.transparent_clear = true; + self.scene.models.push(instance); + + let aabb = model.aabb; + let center = (aabb.0 + aabb.1) * 0.5; + let size = aabb.1 - aabb.0; + self.camera.fov_y = 20f32.to_radians(); + self.camera.znear = 0.01; + self.camera.zfar = 100.0; + self.camera_target = center; + let aspect = self.config.size.0 as f32 / self.config.size.1 as f32; + let horizontal_fov = 2.0 * ((self.camera.fov_y * 0.5).tan() * aspect).atan(); + let vertical_distance = size.y * 0.5 / (self.camera.fov_y * 0.5).tan(); + let horizontal_distance = size.x * 0.5 / (horizontal_fov * 0.5).tan(); + // Persona's default character_size=1 is passed to its framing helper + // as a 1.5x zoom, with half the model depth retained as padding. + self.camera_distance = + vertical_distance.max(horizontal_distance).max(0.5) * 1.12 / 1.5 + size.z * 0.5; + self.rebuild_camera(self.config.size); + + let bundle = std::fs::read_to_string(&self.config.bundle_path).with_context(|| { + format!("reading guest bundle {}", self.config.bundle_path.display()) + })?; + let action_names: Vec = self + .actions + .iter() + .map(|action| action.name.clone()) + .collect(); + self.guest = Some(PersonaGuest::boot( + &bundle, + &self.config.catalog.model_name, + &action_names, + )?); + self.model = Some(model); + self.vrm = Some(vrm); + + if let Some(index) = self.action_index_for_role(ActionRole::Idle) { + self.start_action_index(index, false); + } + if let Some(port) = self.config.bridge_port { + self.bridge = Some(Bridge::start(port, self.config.catalog.clone())?); + } + log::info!( + "Pocket Persona initialized in {:.0} ms", + started.elapsed().as_secs_f32() * 1000.0 + ); + Ok(()) + } + + fn tick(&mut self, dt: f32, input: &Input, window_px: (u32, u32)) -> Result<()> { + self.tick_count += 1; + self.apply_bridge_commands(); + self.update_voice_delay(dt); + self.update_camera(input, window_px); + let Some(model) = self.model.clone() else { + return Ok(()); + }; + let pose_advanced = self.sample_animation(dt, &model); + let spring_advanced = if let Some(springs) = self.springs.as_mut() { + let advanced = springs.joint_count() > 0; + springs.step(dt, &model.skeleton, &mut self.locals, Mat4::IDENTITY); + advanced + } else { + false + }; + self.globals = self.scene.models[0].pose.take().unwrap_or_default(); + model + .skeleton + .globals_from_locals(&self.locals, &mut self.globals); + self.scene.models[0].pose = Some(std::mem::take(&mut self.globals)); + let blink = self.apply_face(dt); + + let event_storage = std::mem::take(&mut self.pending_events); + let events: Vec> = event_storage + .iter() + .map(|(kind, value)| GuestEvent { kind, value }) + .collect(); + let guest_state = GuestState { + t: self.tick_count as f64 * dt as f64, + activity: &self.voice.activity, + audio_level: self.audio_level, + animation: self.active_action_name(), + blink, + render_fps: self.render_rate.fps, + }; + let commands = if let Some(guest) = &self.guest { + guest.turn(&guest_state, &events)? + } else { + Vec::new() + }; + drop(events); + drop(event_storage); + self.apply_guest_commands(commands); + + if pose_advanced || spring_advanced { + self.dirty = true; + } + if let Some(bridge) = &self.bridge { + let active_animation = self.active_action_name().to_string(); + bridge.update_status(|status| { + status.voice_state = self.voice.clone(); + status.audio_level = self.audio_level; + status.active_animation = active_animation; + status.render_fps = self.render_rate.fps; + status.frame_time_p95_ms = self.render_rate.p95_ms; + status.frame_time_p99_ms = self.render_rate.p99_ms; + status.frame_time_max_ms = self.render_rate.max_ms; + }); + } + Ok(()) + } + + fn take_dirty(&mut self) -> bool { + std::mem::take(&mut self.dirty) + } + + fn prepare(&mut self, _gpu: &Gpu) -> Result<()> { + Ok(()) + } + + fn compose(&mut self, time: f32, _size: (u32, u32)) -> (&Scene, &Camera, &Hud) { + self.scene.time = time; + self.render_rate.rendered(); + (&self.scene, &self.camera, &self.hud) + } + + fn drag_at(&mut self, _cursor: Vec2) -> bool { + false + } + + fn take_window_command(&mut self) -> Option { + self.window_command.take() + } + + fn wants_exit(&self) -> bool { + self.exit + } +} + +fn blend_trs(from: NodeTrs, to: NodeTrs, amount: f32) -> NodeTrs { + NodeTrs { + translation: from.translation.lerp(to.translation, amount), + rotation: from.rotation.slerp(to.rotation, amount), + scale: from.scale.lerp(to.scale, amount), + } +} + +fn smoothstep01(value: f32) -> f32 { + let value = value.clamp(0.0, 1.0); + value * value * (3.0 - 2.0 * value) +} + +fn percentile(sorted: &[f32], quantile: f32) -> f32 { + if sorted.is_empty() { + return 0.0; + } + let index = ((sorted.len() - 1) as f32 * quantile) + .round() + .clamp(0.0, (sorted.len() - 1) as f32) as usize; + sorted[index] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn animation_blend_preserves_endpoints() { + let from = NodeTrs { + translation: Vec3::new(1.0, 2.0, 3.0), + rotation: glam::Quat::IDENTITY, + scale: Vec3::ONE, + }; + let to = NodeTrs { + translation: Vec3::new(4.0, 5.0, 6.0), + rotation: glam::Quat::from_rotation_y(1.0), + scale: Vec3::splat(2.0), + }; + assert_eq!(blend_trs(from, to, 0.0).translation, from.translation); + assert_eq!(blend_trs(from, to, 1.0).translation, to.translation); + assert_eq!(blend_trs(from, to, 0.5).scale, Vec3::splat(1.5)); + } + + #[test] + fn transition_curve_is_bounded() { + assert_eq!(smoothstep01(-1.0), 0.0); + assert_eq!(smoothstep01(0.0), 0.0); + assert_eq!(smoothstep01(1.0), 1.0); + assert_eq!(smoothstep01(2.0), 1.0); + assert!((smoothstep01(0.5) - 0.5).abs() < f32::EPSILON); + } + + #[test] + fn frame_percentiles_are_selected_from_sorted_intervals() { + assert_eq!(percentile(&[], 0.95), 0.0); + assert_eq!(percentile(&[1.0, 2.0, 3.0, 4.0], 0.5), 3.0); + assert_eq!(percentile(&[1.0, 2.0, 3.0, 4.0], 0.95), 4.0); + } +} diff --git a/docs/PERSONA.md b/docs/PERSONA.md new file mode 100644 index 0000000..a1d7c6a --- /dev/null +++ b/docs/PERSONA.md @@ -0,0 +1,740 @@ +# Pocket Persona — a controlled native VRM vertical slice + +*A feature-parity proof of concept for +[xikhar/persona](https://github.com/xikhar/persona), implemented as one +Pocket-native process so the architecture can be measured without carrying +Electron, React, React Three Fiber, or Three.js into the result.* + +Pocket Persona is deliberately a clean vertical slice, not a source fork. It +keeps Persona's observable character contract — the library schema, VRM/VRMA +content, voice-state transitions, facial behavior, animation actions, local +event bridge, and MCP tools — while replacing the implementation beneath that +contract with `pocket3d`, `pocket-vrm`, `pocket-widget`, and a `pocket-mod` +QuickJS guest. + +This is a performance and architecture POC, not yet a drop-in replacement for +the complete Persona desktop product. The exact boundary is recorded below. + +## One-line visual acceptance + +From this repository, each command prepares its own pinned inputs, builds the +target, launches it attached to the terminal, and drives the same visible +idle → speaking/lip-sync → listening → greeting → idle sequence: + +```sh +bun run accept:persona +bun run accept:pocket +``` + +The first command clones Persona commit `4efec3ac…` into the ignored +`out/persona-reference/` directory; it never modifies a user checkout. Both +commands validate the same VRM/VRMA hashes and stage the checked-in +`fixtures/persona/library.json`. Press Ctrl-C to terminate the complete target +process tree. + +Run the sequential production or controlled resource comparison with: + +```sh +bun run bench:persona +bun run bench:persona:controlled +``` + +The benchmark never runs both renderers concurrently. + +## 1. Reference and decision + +The reference was inspected and measured at Persona commit +[`4efec3ac729944d0b36137dd8847cc1b488e0bcb`](https://github.com/xikhar/persona/tree/4efec3ac729944d0b36137dd8847cc1b488e0bcb), +version `0.1.0-beta.0`. + +Primary upstream references: + +- [README and product contract](https://github.com/xikhar/persona/blob/4efec3ac729944d0b36137dd8847cc1b488e0bcb/README.md) +- [Architecture and development](https://github.com/xikhar/persona/blob/4efec3ac729944d0b36137dd8847cc1b488e0bcb/docs/DEVELOPMENT.md) +- [Local bridge and MCP integration](https://github.com/xikhar/persona/blob/4efec3ac729944d0b36137dd8847cc1b488e0bcb/docs/INTEGRATIONS.md) +- [Electron lifecycle and control plane](https://github.com/xikhar/persona/blob/4efec3ac729944d0b36137dd8847cc1b488e0bcb/electron/main.cjs) +- [React/Three scene](https://github.com/xikhar/persona/blob/4efec3ac729944d0b36137dd8847cc1b488e0bcb/src/components/Scene.tsx) +- [VRM animation component](https://github.com/xikhar/persona/blob/4efec3ac729944d0b36137dd8847cc1b488e0bcb/src/components/Avatar.tsx) +- [Source license](https://github.com/xikhar/persona/blob/4efec3ac729944d0b36137dd8847cc1b488e0bcb/LICENSE) +- [Asset-license boundary](https://github.com/xikhar/persona/blob/4efec3ac729944d0b36137dd8847cc1b488e0bcb/ASSET_LICENSES.md) + +The local `~/code/pocket-character` checkout used for the implementation +comparison was at +[`b0932770ba0b4f4a490f607652b0cf09c76ab513`](https://github.com/pocket-stack/pocket-character/tree/b0932770ba0b4f4a490f607652b0cf09c76ab513). +It already proved the relevant Pocket primitives: one native wgpu process, +VRM 0.x parsing, VRMA retargeting, spring bones, morph expressions, a +demand-rendered widget shell, and a small QuickJS policy surface. + +### Why a clean vertical slice + +A fork would preserve the largest variables under test: Electron's process +tree, Chromium's renderer and GPU service, React reconciliation, React Three +Fiber, and Three.js. It could demonstrate UI changes, but it could not answer +how much of Persona's steady-state cost comes from that architecture. + +The clean slice instead treats Persona as a behavioral and data contract: + +1. Consume the same `library.json`, `.vrm`, and `.vrma` inputs. +2. Preserve the same idle/speaking/custom-action state machine and transition + timings. +3. Preserve the same loopback event shapes and four MCP tools. +4. Keep continuous simulation, facial motion, physics, and rendering native. +5. Keep product-specific policy hot-swappable as a bounded QuickJS bundle. +6. Measure the whole launched process tree sequentially on the same machine. + +This makes the controlled benchmark a cap-controlled POC comparison with the +same data contract and source texture ceiling. It exposes the runtime-stack +effect without pretending to isolate architecture completely: the renderer +feature and texture-role differences in section 4 still apply. A second, +production-oriented lane measures Pocket's intentional quality/power choices +separately. + +## 2. Architecture and data flow + +Persona's reference stack has four process and trust layers: + +```text +Core Audio / WASAPI / PipeWire output activity + -> native listener process or PipeWire adapter + -> Electron main process + settings store + window/tray/protocol lifecycle + loopback HTTP events + Streamable HTTP MCP + -> sandboxed preload bridge / IPC + -> React 19 + React Three Fiber + Three.js + VRM/VRMA loading, animation mixer, blink/lip hooks, WebGL rendering +``` + +The Pocket slice collapses the continuous path into one native process: + +```text +Persona library.json + -> immutable catalog validation + -> pocket3d GLB upload + pocket-vrm VRM parse + -> pocket-vrm VRMA retargeting + +loopback HTTP /events or /mcp + -> bridge worker thread + -> bounded BridgeCommand channel + -> fixed-step PersonaWidget core + action selection and crossfade + skeleton pose + spring bones + blink + amplitude visemes + morph upload + pocket3d render + -> bounded state/events into pocket-mod QuickJS + <- guest intents: playAnimation, setExpression, quit +``` + +The native core owns time and all per-frame work. The guest receives only: + +```text +{ t, activity, audioLevel, animation, blink, renderFps } +{ type: "animationChanged" | "voiceChanged", value } +``` + +The guest cannot access the filesystem, network, renderer, or raw model. It +can request `playAnimation(name)`, `setExpression(name, weight)`, or `quit()`. +That is the Pocket runtime-family split: native core, narrow surface, +hot-swappable guest. + +The implementation lives in: + +| Path | Responsibility | +| --- | --- | +| `crates/pocket-persona/` | Product-level native core, catalog, bridge, MCP server, deterministic face simulation, and guest bundle | +| `vendor/pocketjs/engine/crates/pocket-vrm/` | VRM 0.x semantics, expressions, spring bones, VRMA parsing and humanoid retargeting | +| `vendor/pocketjs/engine/pocket3d/crates/pocket3d/` | glTF upload, skinning, morph targets, camera, wgpu renderer, and offscreen output | +| `vendor/pocketjs/engine/crates/pocket-widget/` | Transparent native window, fixed ticks, frame pacing, occlusion handling, demand rendering, and show/hide requests | +| `scripts/accept-persona.ts` | Pinned reference setup, asset staging, target build/run, visible event sequence, and cleanup | +| `scripts/bench-persona.ts` | Sequential, full-process-tree A/B resource harness | +| `fixtures/persona/library.json` | Canonical media-free acceptance catalog shared by both targets | + +The PocketJS submodule currently pins +[pocket-stack/pocketjs#204](https://github.com/pocket-stack/pocketjs/pull/204). +That upstream change contains only generic runtime window Show/Hide support; +the Persona catalog, renderer product, bridge, MCP server, acceptance commands, +benchmark, and report remain owned by this repository. Repin the submodule to +the eventual PocketJS merge commit before merging this product change. + +## 3. Implemented parity + +“Parity” here means the visible/runtime slice listed in this table. It does +not mean that the deferred desktop-management features in the next section +are silently present. + +| Contract | Persona reference | Pocket Persona | Status | +| --- | --- | --- | --- | +| Character input | Packaged or imported VRM | Default model from Persona schema-v1 `library.json` | Implemented for the controlled VRM 0.x asset | +| Animation input | One or more VRMA clips per action | Same relative `.vrma` paths, parsed and retargeted at load | Implemented for the controlled clip; loader limits are below | +| Idle role | Permanent `IDLE` action | Looped native idle action | Implemented | +| Speaking role | Permanent `TALK` action follows voice output | Looped native speaking action follows validated voice state | Implemented; state is externally supplied | +| Custom actions | Named one-shot actions | Named one-shots, then resume the current voice role | Implemented | +| Clip choice | Random clip, avoid immediate repeat | Deterministic seeded choice, avoid immediate repeat | Behavior parity with reproducible tests | +| Crossfades | General `0.7 s`; into speaking `0.85 s`; speaking to idle `1.15 s` | Same durations with smoothstep TRS blending | Implemented | +| Talk-to-idle hold | Listener activity gate plus app-side settling | `0.65 s` app-side delay after listening state | Implemented at the app boundary; native gate deferred | +| Lip sync | Amplitude-only five-viseme driver | Same five VRM expression families, audible threshold, attack/release smoothing, and `0.62` cap | Implemented | +| Blink | Random `2–6 s`, `0.24 s` envelope | Deterministic seeded `2–6 s`, `0.24 s` sine envelope | Implemented | +| Secondary motion | VRM spring bones | Native `pocket-vrm` spring solver every fixed tick | Implemented | +| Voice/action interaction | One-shot body action can override voice body motion while lip sync continues | Same; voice role resumes after the one-shot | Implemented | +| Camera framing | Full-body framing at default character size | Bounds-derived 20-degree perspective framing calibrated to Persona's default | Implemented, visually approximate | +| Camera input | Scroll zoom, left orbit, right pan | Same gestures; pitch and distance are bounded | Implemented | +| Window | `430×680`, transparent, frameless, topmost | Same logical default, resizable with `320×480` minimum | Implemented | +| Render scheduling | Continuous browser animation loop | Fixed tick plus dirty-frame governor; clips, crossfades, spring motion, and face changes re-arm presentation | Implemented | +| Local status | `GET /health` | Loopback-only `GET /health` with model, window, voice, audio, animation, and render state | Implemented | +| Event bridge | `POST /events` | Same `state`, `audio-level`, and `animation` event shapes; bounded and validated | Implemented | +| MCP animations | `play_animation`, `list_animations` | Same names and catalog metadata | Implemented | +| MCP window/status | `control_window`, `get_status` | Same names and show/hide/toggle/status behavior | Implemented | +| Agent policy seam | Electron/MCP control plane | `pocket-mod` QuickJS guest receives facts/events and emits bounded intents | Implemented, intentionally Pocket-shaped | +| Headless acceptance | Renderer tests, no product-level offscreen command | Fixed-step offscreen render to a transparent PNG | Implemented | + +The bridge binds only to `127.0.0.1`, limits requests to 64 KiB, validates the +`Host` and optional `Origin`, clamps audio levels to `[0, 1]`, validates voice +enums and action names, and transfers commands to the render thread through an +MPSC queue. The MCP endpoint implements the JSON-RPC methods needed by the +four tools; it is a deliberately small Streamable HTTP subset with one static +local session, not a vendored copy of the upstream JavaScript MCP SDK. It has +been exercised with the official SDK client, but dynamic session allocation, +strict protocol-version negotiation, session expiry, and tools-changed +notifications remain desktop control-plane work. + +## 4. Deliberately deferred gaps + +The following are out of the POC's parity claim: + +### Settings, catalog mutation, and imports + +Pocket Persona reads an immutable, validated subset of Persona's schema-v1 +`library.json`. It selects `default_model_id` or the first model, and reads +action names, descriptions, trigger scenarios, roles, and relative clip +paths. The catalog rejects absolute paths and lexical `..` traversal, but the +library directory remains a trusted local input: symlink targets are allowed +so local, uncommitted development assets can stay outside either checkout. +This is not a filesystem sandbox for an untrusted catalog. + +The catalog also enforces Persona's unique model/animation IDs and names, +allowed animation types, and reserved `system-idle`/`system-speaking` slots. +Missing system slots are materialized as empty permanent actions, matching the +reference catalog behavior. + +It does not implement Persona's Settings window, previews, model switching, +character-size preference, user-level `settings.json`, copy-on-write packaged +overrides, tombstones, reset flow, or the model/action CRUD UI. It also does +not implement `.vrm`/`.vrma` import, upload limits, or library migration. +Those are desktop product and persistence features, not part of the renderer +performance slice. + +### Native audio detection + +Pocket Persona does not ship Persona's Core Audio, WASAPI, or PipeWire output +listener. It never records, stores, transcribes, or sends audio. The local +bridge accepts normalized voice state and amplitude, so the complete character +behavior can be exercised deterministically by a test client or a future +platform adapter. + +This means automatic detection of a supported voice application's output, +macOS audio-capture permission UX, the listener's activity gate, native helper +lifecycle, and listener health reporting are deferred. Do not describe the +current POC as automatic voice detection. + +### Desktop lifecycle and protocol + +The native window supports close, resize, topmost presentation, occlusion +suspension, and MCP show/hide. It does not yet provide Persona's tray menu, +global shortcut, background-start behavior, login/startup integration, +single-instance handoff, all-Spaces macOS policy, or `persona://` protocol +registration and deep-link actions. + +### Rendering differences + +The visual output is intentionally not a pixel-parity port: + +- Persona uses `@pixiv/three-vrm`/Three.js, its VRM material path, a + `dawn.exr` environment, directional and ambient lights, sRGB output, + `NoToneMapping`, and a device-pixel-ratio cap of `1.5`. Its Settings preview + also adds contact shadows. +- Pocket currently uses glTF base-color materials with pocket3d's simple + hemisphere-plus-sun shader, `lit = 0.25`, alpha cutout `0.5`, and a + transparent clear. `pocket-vrm` parses MToon facts, but this POC does not + render the complete MToon shading model. +- Pocket's production default halves oversized textures until their longest + side is at most `2048`; the reference keeps the model's `4096` authoring + textures. Pocket also uploads only images sampled as base color by a used + material. These are intentional memory levers, not free visual parity. +- Persona's live renderer follows the display refresh rate. Pocket's + production tick/render cap is `60 Hz`; its active idle clip therefore + presents at most 60 frames per second. +- Orbit inertia/damping, exact HDR lighting, exact alpha/material behavior, + preview contact shadows, and pixel-identical framing remain deferred. + +The accepted 3D formats are narrower than Persona's general Three.js path. +This POC supports VRM 0.x, not VRM 1.0. Its VRMA loader consumes the first +animation, retargets humanoid rotation plus hips translation, and supports the +accessor subset exercised by the controlled clip. Sparse accessors, exact +CUBICSPLINE interpolation, non-humanoid translation, and +`VRMC_vrm_animation` expression/look-at tracks are not parity claims. + +For that reason, performance must be reported in two lanes: a controlled +`120 Hz / 4096` run that avoids crediting Pocket for a lower configured +quality target, and the actual `60 Hz / 2048` production default that measures +the user-facing optimization. + +## 5. Asset and license boundary + +The Persona source is MIT, but its own asset policy explicitly says that MIT +does not grant rights to local VRM or VRMA media. Persona's distributable +catalog is empty at the pinned commit, and there is no reference character +asset in the repository that this POC can legally copy by implication. + +The controlled local inputs are: + +| Input | Size | SHA-256 | +| --- | ---: | --- | +| `~/code/pocket-character/assets/AvatarSample_A.vrm` | 26,781,812 bytes | `2a0ccd84880b03d7b65503d8b6287f7a97f3bb4fab70a5fd0a47b433c97827f5` | +| `~/code/pocket-character/assets/idle_loop.vrma` | 157,664 bytes | `ace95ba6dcc0bdf2ed1081c002332b4184441117c8d543b6f642b3d2c5cf99be` | + +They are local development inputs only. `pocket-character` also fetches them +instead of committing them because the VRoid sample terms and animation +provenance are not covered by its source license. Do not commit them here, +attach them to a release, or infer redistribution permission from either +project's source license. + +The VRM contains 40,406 vertices, 29,221 triangles, 95 nodes, three skins, +seven primitives/materials, and thirteen PNG images. The source images include +four `4096²` and five `2048²` textures. Decoded RGBA is approximately +336.50 MiB before mipmaps and 448.67 MiB with a full mip chain, which is why +the source texture ceiling is held constant in the controlled lane and changed +only in the production lane. + +No Persona art, icon, HDR environment, VRM, or VRMA file is added to this +repository by the POC. + +## 6. Reproducible local library + +The acceptance wrapper owns this setup in normal use. The expanded commands +below document what it stages inside its ignored Persona checkout: + +```sh +export POCKET_CHARACTER_ROOT="$PWD" +export PERSONA_ROOT="$POCKET_CHARACTER_ROOT/out/persona-reference" +export PERSONA_ASSETS="$PERSONA_ROOT/public/assets" +export PERSONA_LIBRARY="$PERSONA_ASSETS/library.json" + +cd "$PERSONA_ROOT" +git switch --detach 4efec3ac729944d0b36137dd8847cc1b488e0bcb + +mkdir -p "$PERSONA_ASSETS/models" "$PERSONA_ASSETS/animations" +ln -sfn "$POCKET_CHARACTER_ROOT/assets/AvatarSample_A.vrm" \ + "$PERSONA_ASSETS/models/model.vrm" +for name in idle talk1 talk2 greeting happy finger-gun dance; do + ln -sfn "$POCKET_CHARACTER_ROOT/assets/idle_loop.vrma" \ + "$PERSONA_ASSETS/animations/$name.vrma" +done + +cat >"$PERSONA_LIBRARY" <<'JSON' +{ + "schema_version": 1, + "default_model_id": "packaged-model", + "models": [ + { + "id": "packaged-model", + "model_name": "Packaged model", + "asset_path": "models/model.vrm" + } + ], + "animations": [ + { + "id": "system-idle", + "animation_name": "idle", + "animation_description": "A calm resting motion for the character.", + "animation_trigger_scenario": "Used automatically while Persona is waiting and not speaking.", + "animation_type": "IDLE", + "asset_paths": ["animations/idle.vrma"] + }, + { + "id": "system-speaking", + "animation_name": "speaking", + "animation_description": "Natural conversational body movement while the character speaks.", + "animation_trigger_scenario": "Used automatically while supported voice output is active.", + "animation_type": "TALK", + "asset_paths": [ + "animations/talk1.vrma", + "animations/talk2.vrma" + ] + }, + { + "id": "packaged-greeting", + "animation_name": "greeting", + "animation_description": "A friendly greeting motion.", + "animation_trigger_scenario": "Use when beginning an interaction or welcoming the user.", + "animation_type": "GREETING", + "asset_paths": ["animations/greeting.vrma"] + }, + { + "id": "packaged-happy", + "animation_name": "happy", + "animation_description": "A warm, upbeat reaction.", + "animation_trigger_scenario": "Use for good news, success, gratitude, or a positive response.", + "animation_type": "HAPPY", + "asset_paths": ["animations/happy.vrma"] + }, + { + "id": "packaged-finger-gun", + "animation_name": "finger-gun", + "animation_description": "A playful finger-gun gesture.", + "animation_trigger_scenario": "Use for lighthearted confidence, a clever solution, or playful approval.", + "animation_type": "FINGER_GUN", + "asset_paths": ["animations/finger-gun.vrma"] + }, + { + "id": "packaged-dance", + "animation_name": "dance", + "animation_description": "A celebratory dance.", + "animation_trigger_scenario": "Use for a major success, an exciting milestone, or an explicit request to dance.", + "animation_type": "DANCE", + "asset_paths": ["animations/dance.vrma"] + } + ] +} +JSON +``` + +All seven staged paths intentionally resolve to the one available local clip; +the speaking slot has two aliases so the no-immediate-repeat path is exercised. +This exact catalog was used by both final benchmark reports. It verifies role +switching, one-shot completion, clip choice, crossfades, and control-plane +behavior without inventing or distributing additional media; it is not a claim +that the motions are artistically distinct. + +Build the pinned reference after staging the catalog so Vite copies the same +media into `dist`: + +```sh +cd "$PERSONA_ROOT" +npm ci +npm run native:build +npm run native:test +npm run check +``` + +Return to the Pocket Character checkout before running the remaining commands: + +```sh +cd "$POCKET_CHARACTER_ROOT" +export PERSONA_LIBRARY="$PERSONA_ROOT/public/assets/library.json" +``` + +## 7. Build, run, and headless verification + +The normal one-line command builds the minified IIFE guest and release native +binary before launching: + +```sh +bun run accept:pocket +``` + +Outputs: + +```text +dist/pocket-persona/guest.js +dist/pocket-persona/guest.js.map +target/release/pocket-persona +``` + +Run the production-oriented defaults — `430×680`, fixed `60 Hz`, texture cap +`2048`, loopback bridge on port `47831`: + +```sh +./target/release/pocket-persona \ + --library "$PERSONA_LIBRARY" \ + --bundle "$PWD/dist/pocket-persona/guest.js" \ + --fps 60 \ + --max-texture-dim 2048 +``` + +Disable the control plane for a renderer-only manual run: + +```sh +./target/release/pocket-persona \ + --library "$PERSONA_LIBRARY" \ + --bundle "$PWD/dist/pocket-persona/guest.js" \ + --fps 60 \ + --max-texture-dim 2048 \ + --no-bridge +``` + +Render a deterministic fixed-step acceptance frame without creating a window. +Headless mode disables the bridge automatically: + +```sh +mkdir -p "$PWD/dist/pocket-persona" +./target/release/pocket-persona \ + --library "$PERSONA_LIBRARY" \ + --bundle "$PWD/dist/pocket-persona/guest.js" \ + --fps 60 \ + --max-texture-dim 2048 \ + --ticks 90 \ + --headless-shot "$PWD/dist/pocket-persona/headless.png" +``` + +Useful native tests: + +```sh +cargo test -p pocket-persona +cargo test --manifest-path vendor/pocketjs/engine/Cargo.toml -p pocket-vrm +``` + +## 8. Bridge acceptance + +Keep the normal windowed process running, then use a second terminal. + +Read health and current state: + +```sh +curl --fail-with-body --silent --show-error \ + http://127.0.0.1:47831/health | jq +``` + +Enter speaking state and drive the amplitude-only lip synchronizer: + +```sh +curl --fail-with-body --silent --show-error \ + -H 'Content-Type: application/json' \ + -d '{"type":"state","state":{"phase":"active","activity":"speaking","microphoneMuted":false,"outputMuted":false}}' \ + http://127.0.0.1:47831/events + +curl --fail-with-body --silent --show-error \ + -H 'Content-Type: application/json' \ + -d '{"type":"audio-level","level":0.28}' \ + http://127.0.0.1:47831/events +``` + +Return through listening to idle after the `0.65 s` app-side delay: + +```sh +curl --fail-with-body --silent --show-error \ + -H 'Content-Type: application/json' \ + -d '{"type":"state","state":{"phase":"active","activity":"listening","microphoneMuted":false,"outputMuted":false}}' \ + http://127.0.0.1:47831/events +``` + +Play the configured one-shot action: + +```sh +curl --fail-with-body --silent --show-error \ + -H 'Content-Type: application/json' \ + -d '{"type":"animation","animation_name":"greeting"}' \ + http://127.0.0.1:47831/events +``` + +Expected event responses are HTTP `202` with `{"accepted":true}`. Malformed +JSON returns `400`; a structurally invalid event returns `422`. + +## 9. MCP acceptance + +Initialize the minimal Streamable HTTP endpoint and inspect its session header: + +```sh +curl --silent --show-error --include \ + -H 'Content-Type: application/json' \ + -H 'Accept: application/json, text/event-stream' \ + -d '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{},"clientInfo":{"name":"curl","version":"1"}}}' \ + http://127.0.0.1:47831/mcp +``` + +List tools: + +```sh +curl --fail-with-body --silent --show-error \ + -H 'Content-Type: application/json' \ + -H 'Accept: application/json, text/event-stream' \ + -H 'Mcp-Session-Id: pocket-persona-v1' \ + -d '{"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}' \ + http://127.0.0.1:47831/mcp | jq +``` + +Call an animation and read status: + +```sh +curl --fail-with-body --silent --show-error \ + -H 'Content-Type: application/json' \ + -H 'Accept: application/json, text/event-stream' \ + -H 'Mcp-Session-Id: pocket-persona-v1' \ + -d '{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"play_animation","arguments":{"animation":"greeting"}}}' \ + http://127.0.0.1:47831/mcp | jq + +curl --fail-with-body --silent --show-error \ + -H 'Content-Type: application/json' \ + -H 'Accept: application/json, text/event-stream' \ + -H 'Mcp-Session-Id: pocket-persona-v1' \ + -d '{"jsonrpc":"2.0","id":4,"method":"tools/call","params":{"name":"get_status","arguments":{}}}' \ + http://127.0.0.1:47831/mcp | jq +``` + +To make the running POC available to new Codex sessions: + +```sh +codex mcp add pocket-persona --url http://127.0.0.1:47831/mcp +``` + +## 10. Performance methodology + +Do not launch Persona and Pocket Persona together. Chromium GPU work, shader +compilation, asset decoding, and workspace builds materially contaminate each +other. `scripts/bench-persona.ts` runs the reference first, terminates its full +process group, then runs Pocket. For each target it: + +1. requires `GET /health` to return `200` with `ok: true`, then waits for the + configured settle period; +2. snapshots every descendant process; +3. samples cumulative process CPU time over fixed intervals; +4. sums RSS and records the complete process inventory; +5. reports median and p95 values; and +6. captures a health receipt alongside every resource sample; and +7. writes machine facts, exact launch commands, raw samples, and comparison + formulas to JSON. + +The harness requires `--library` to resolve to the reference checkout's own +`public/assets/library.json`, so both targets cannot silently benchmark +different catalogs. Both always-on-top windows must remain visible and +uncovered. The harness fails closed if Pocket's health receipts stop reporting +a configured, visible model and delivered frames, so compositor suspension +cannot silently become a renderer optimization. + +Persona starts its bridge before the avatar renderer, so its health response +is a control-plane readiness check rather than proof of visible model output. +The benchmark result must therefore be paired with a rendered-model +screenshot and a frame-rate receipt. Pocket health is stronger: the bridge +starts after model/animation/guest initialization and reports `renderFps`, +`modelConfigured`, and `windowVisible` in every sample. + +CPU percentages are percent of one logical core: `100%` means one saturated +core. Summed RSS is useful for a same-machine process-tree comparison, but it +can double-count shared mappings and is not a substitute for platform physical +footprint tools. + +### Cap-controlled POC lane: same asset, 120 Hz, 4096 textures + +Use the same `library.json` and source hashes for both targets. Set the macOS +display to `120 Hz`, keep both windows untouched and visible, and ensure no +other build or benchmark overlaps the run. Persona is display-driven; Pocket +is explicitly fixed/capped at `120`. + +```sh +bun run bench:persona:controlled +``` + +`4096` leaves the controlled model's source textures at authoring resolution. +This is the strongest like-for-like POC lane, but not an architecture-only +headline: Pocket still omits the reference MToon/HDR path and unused texture +roles. Record the actual frame rate and physical framebuffer of each window as +a companion receipt: the reference caps DPR at `1.5`, while the native +swapchain follows the OS backing scale. Equal logical window size does not by +itself prove equal fragment workload. + +### Production lane: Pocket 60 Hz, 2048 textures + +This keeps the reference unchanged and applies Pocket's intended shipping +defaults. It measures the total user-facing saving including the refresh and +texture-cap choices: + +```sh +bun run bench:persona +``` + +Before reporting a result, verify the two JSON reports have `status: "ok"`, +both runs contain all requested samples, the commands contain the intended +caps, the asset hashes still match this document, and no unexpected helper or +build process joined either process tree. + +## 11. Performance results + +Both final sequential reports completed with `status: "ok"` on 2026-07-30. +Each target settled for 30 seconds and then produced nine five-second process +tree samples. No build or second benchmark overlapped either run. + +Reference environment: MacBook Pro `Mac15,8`, Apple M3 Max, 128 GiB RAM, +macOS `26.5.2 (25F84)`, Electron `39.8.10`, Chromium `142.0.7444.265`, +logical viewport `430×680`, and a 120 Hz display. Persona rendered a +`645×1020` WebGL canvas at DPR `1.5`; Pocket's native 2× swapchain was +`860×1360`, or 1.78 times as many backing pixels. Both targets consumed the +same catalog, 26,781,812-byte VRM, and 157,664-byte VRMA recorded above. +Neither run included Persona's optional native audio helper. + +RSS comes from the nine process-tree snapshots and remains a diagnostic +mapping total. The physical-footprint rows come from separate 30-second +sustained-render measurements with macOS `footprint`: all four Electron +process IDs were included for Persona, and Pocket had one process. The +reference footprint was measured once because the reference configuration is +identical in both lanes. These are steady-state snapshots, not startup peaks. +Pocket's auxiliary per-process peak fields were 1,400,440,104 bytes at 4096 +and 1,030,063,256 bytes at 2048, so this POC makes no peak-memory reduction +claim. + +### Controlled result + +This lane used the same authoring-resolution textures and requested 120 Hz +from Pocket: + +| Metric | Persona | Pocket | Pocket delta | +| --- | ---: | ---: | ---: | +| Observed frame rate | 120.017 fps | 102.135 fps | 14.9% fewer frames | +| Frame interval p95 / p99 | 9.700 / 10.200 ms | 10.951 / 11.116 ms | see sampling note | +| Process-tree CPU median / p95 | 10.800% / 11.517% | 15.400% / 16.396% | **42.6% / 42.4% more** | +| CPU percent per delivered fps | 0.0900 | 0.1508 | **67.6% more** | +| Summed process-tree RSS median / p95 | 1,181,392 / 1,182,906 KiB | 115,488 / 116,102 KiB | **90.2% less** | +| Process count | 4 | 1 | **75.0% fewer** | +| Settled macOS physical footprint (30 s) | 1,395,873,216 bytes | 679,920,456 bytes | **51.3% less; 2.05× smaller** | + +The cap-controlled POC therefore has a large memory and process-count win, but +this is not an architecture-only attribution: Pocket omits renderer features +and texture roles listed in section 4. It also does **not** have a controlled +high-refresh CPU win. It delivered fewer frames while using more process CPU. +Pocket's larger native backing surface makes the fragment workload stricter +than exact pixel parity, but it does not turn the CPU-per-frame result into an +optimization claim. + +Persona's frame receipt is one continuous 45-second CDP `requestAnimationFrame` +sample; the Pocket p95/p99 values above are the medians of nine native rolling +one-second receipts. They describe each renderer accurately but are not an +identical long-window percentile estimator. Persona had zero intervals over +20 ms across 5,401 delivered frames. Pocket's worst reported one-second maximum +was 12.592 ms. + +### Production result + +This lane retained the unchanged 120 Hz Persona reference and used Pocket's +shipping defaults, a 60 Hz cap and 2048 texture cap: + +| Metric | Persona | Pocket | Pocket delta | +| --- | ---: | ---: | ---: | +| Observed frame rate | 120.008 fps | 54.372 fps | 54.7% fewer frames | +| Frame interval p95 / p99 | 9.600 / 10.200 ms | 19.448 / 19.639 ms | see sampling note | +| Process-tree CPU median / p95 | 10.600% / 11.638% | 8.800% / 9.000% | **17.0% / 22.7% less** | +| CPU percent per delivered fps | 0.0883 | 0.1619 | **83.3% more** | +| Summed process-tree RSS median / p95 | 1,190,720 / 1,192,163 KiB | 89,232 / 89,344 KiB | **92.5% less** | +| Process count | 4 | 1 | **75.0% fewer** | +| Settled macOS physical footprint (30 s) | 1,395,873,216 bytes | 517,260,128 bytes | **62.9% less; 2.70× smaller** | + +The production configuration saves 17.0% median process CPU in absolute +terms, but only by delivering about half as many frames and downscaling source +textures. Normalized per delivered frame, it is 83.3% more CPU-expensive than +Persona in this run. The defensible performance headline is therefore: + +- 51.3% less settled controlled physical footprint, or 62.9% less settled + footprint at production texture settings; +- 90.2–92.5% less summed RSS and one process instead of four; +- 17.0% less total CPU at the default 60 Hz / 2048 configuration; and +- no CPU-throughput optimization yet—the controlled lane regresses 42.6% in + raw CPU and 67.6% per delivered frame. + +The unsigned upstream arm64 `.app` occupied 302,128 KiB and its distributable +zip was 129,730,393 bytes. The final POC release binary, guest bundle, model, +and seven declared local clip paths total 38,876,395 bytes before application +packaging; the catalog adds 2,363 bytes. All seven clip files are byte-identical +test aliases. This suggests substantial distribution-size headroom, but it is +not reported as a parity optimization because this POC intentionally omits +settings, import, tray/update plumbing, and native audio. + +Startup is also left out of the percentage claim. Persona exposes bridge +health before avatar readiness, while Pocket starts its bridge after model, +animation, and guest initialization, so those timestamps do not mark the same +event. diff --git a/fixtures/persona/library.json b/fixtures/persona/library.json new file mode 100644 index 0000000..0c493f1 --- /dev/null +++ b/fixtures/persona/library.json @@ -0,0 +1,74 @@ +{ + "schema_version": 1, + "default_model_id": "packaged-model", + "models": [ + { + "id": "packaged-model", + "model_name": "Packaged model", + "asset_path": "models/model.vrm" + } + ], + "animations": [ + { + "id": "system-idle", + "animation_name": "idle", + "animation_description": "A calm resting motion for the character.", + "animation_trigger_scenario": "Used automatically while Persona is waiting and not speaking.", + "animation_type": "IDLE", + "asset_paths": [ + "animations/idle.vrma" + ] + }, + { + "id": "system-speaking", + "animation_name": "speaking", + "animation_description": "Natural conversational body movement while the character speaks.", + "animation_trigger_scenario": "Used automatically while supported voice output is active.", + "animation_type": "TALK", + "asset_paths": [ + "animations/talk1.vrma", + "animations/talk2.vrma" + ] + }, + { + "id": "packaged-greeting", + "animation_name": "greeting", + "animation_description": "A friendly greeting motion.", + "animation_trigger_scenario": "Use when beginning an interaction or welcoming the user.", + "animation_type": "GREETING", + "asset_paths": [ + "animations/greeting.vrma" + ] + }, + { + "id": "packaged-happy", + "animation_name": "happy", + "animation_description": "A warm, upbeat reaction.", + "animation_trigger_scenario": "Use for good news, success, gratitude, or a positive response.", + "animation_type": "HAPPY", + "asset_paths": [ + "animations/happy.vrma" + ] + }, + { + "id": "packaged-finger-gun", + "animation_name": "finger-gun", + "animation_description": "A playful finger-gun gesture.", + "animation_trigger_scenario": "Use for lighthearted confidence, a clever solution, or playful approval.", + "animation_type": "FINGER_GUN", + "asset_paths": [ + "animations/finger-gun.vrma" + ] + }, + { + "id": "packaged-dance", + "animation_name": "dance", + "animation_description": "A celebratory dance.", + "animation_trigger_scenario": "Use for a major success, an exciting milestone, or an explicit request to dance.", + "animation_type": "DANCE", + "asset_paths": [ + "animations/dance.vrma" + ] + } + ] +} diff --git a/package.json b/package.json index e041fd6..b9f5032 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,10 @@ "setup": "cd vendor/pocketjs && bun install && cd ../.. && mkdir -p node_modules/@pocketjs && ln -sfn ../../vendor/pocketjs node_modules/@pocketjs/framework && ln -sfn ../vendor/pocketjs/node_modules/solid-js node_modules/solid-js && ln -sfn ../vendor/pocketjs/node_modules/bun-types node_modules/bun-types && bun scripts/fetch-assets.ts", "typecheck": "bun vendor/pocketjs/node_modules/typescript/bin/tsc --noEmit", "build:ui": "bun scripts/build-ui.ts", - "widget": "bun scripts/widget.ts" + "widget": "bun scripts/widget.ts", + "accept:persona": "bun scripts/accept-persona.ts reference", + "accept:pocket": "bun scripts/accept-persona.ts pocket", + "bench:persona": "bun scripts/accept-persona.ts bench --profile production", + "bench:persona:controlled": "bun scripts/accept-persona.ts bench --profile controlled" } } diff --git a/scripts/accept-persona.ts b/scripts/accept-persona.ts new file mode 100644 index 0000000..9dc3723 --- /dev/null +++ b/scripts/accept-persona.ts @@ -0,0 +1,1012 @@ +#!/usr/bin/env bun + +import { createHash } from "node:crypto"; +import { + existsSync, + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + readlinkSync, + renameSync, + rmSync, + statSync, + symlinkSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { platform, tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + allocateLoopbackPorts, + runPersonaBenchmark, + terminateProcessTree, + type TargetName, +} from "./bench-persona"; + +type Mode = "reference" | "pocket" | "bench"; +type BenchmarkProfile = "production" | "controlled"; +type SpawnedProcess = ReturnType; + +interface CliOptions { + mode: Mode; + profile: BenchmarkProfile; + cycles: number; + realAudio: boolean; + settleSeconds: number; + sampleCount: number; + intervalSeconds: number; + outPath: string | null; +} + +interface Asset { + name: string; + url: string; + path: string; + bytes: number; + sha256: string; +} + +const ROOT = fileURLToPath(new URL("..", import.meta.url)); +const OUT = join(ROOT, "out"); +const REFERENCE_ROOT = join(OUT, "persona-reference"); +const REFERENCE_STATE = join(OUT, "persona-reference-state"); +const REFERENCE_REPOSITORY = "https://github.com/xikhar/persona.git"; +const REFERENCE_COMMIT = "4efec3ac729944d0b36137dd8847cc1b488e0bcb"; +const FIXTURE_LIBRARY = join(ROOT, "fixtures", "persona", "library.json"); +const REFERENCE_LIBRARY = join( + REFERENCE_ROOT, + "public", + "assets", + "library.json", +); +const REFERENCE_ELECTRON = join( + REFERENCE_ROOT, + "node_modules", + "electron", + "dist", + "Electron.app", + "Contents", + "MacOS", + "Electron", +); +const REFERENCE_NATIVE_HELPER = join( + REFERENCE_ROOT, + "native", + "bin", + "darwin", + "persona-audio-listener", +); +const POCKET_BINARY = join(ROOT, "target", "release", "pocket-persona"); +const POCKET_GUEST_DIR = join(ROOT, "dist", "pocket-persona"); +const POCKET_GUEST = join(POCKET_GUEST_DIR, "guest.js"); +const READINESS_TIMEOUT_MS = 60_000; +const REQUEST_TIMEOUT_MS = 2_000; + +const ASSETS: Asset[] = [ + { + name: "AvatarSample_A.vrm", + url: "https://dist.ayaka.moe/vrm-models/VRoid-Hub/AvatarSample-A/AvatarSample_A.vrm", + path: join(ROOT, "assets", "AvatarSample_A.vrm"), + bytes: 26_781_812, + sha256: "2a0ccd84880b03d7b65503d8b6287f7a97f3bb4fab70a5fd0a47b433c97827f5", + }, + { + name: "idle_loop.vrma", + url: "https://raw.githubusercontent.com/moeru-ai/airi/main/packages/stage-ui-three/src/assets/vrm/animations/idle_loop.vrma", + path: join(ROOT, "assets", "idle_loop.vrma"), + bytes: 157_664, + sha256: "ace95ba6dcc0bdf2ed1081c002332b4184441117c8d543b6f642b3d2c5cf99be", + }, +]; + +let activeChild: SpawnedProcess | null = null; +let activeTarget: TargetName = "pocket"; +let signalShutdown: Promise | null = null; +let signalExitCode: number | null = null; +let benchmarkOwnsSignals = false; +const temporaryDirectories = new Set(); + +if (import.meta.main) { + process.exitCode = await main(Bun.argv.slice(2)); +} + +async function main(argv: string[]): Promise { + let options: CliOptions; + try { + options = parseArgs(argv); + } catch (error) { + console.error(errorMessage(error)); + console.error(""); + console.error(usage()); + return 2; + } + + if (platform() !== "darwin") { + console.error( + "persona acceptance currently requires macOS because it launches Electron.app directly", + ); + return 2; + } + + installSignalHandlers(); + + try { + await preflight(options); + await ensureAssets(); + await ensureReferenceCheckout(); + stageReferenceLibrary(); + + if (options.mode === "reference") { + await ensureReferenceBuild(options.realAudio); + await runVisibleReference(options); + return 0; + } + + await ensurePocketBuild(); + if (options.mode === "pocket") { + await runVisiblePocket(options); + return 0; + } + + await ensureReferenceBuild(false); + return runBenchmark(options); + } catch (error) { + if (signalExitCode != null) return signalExitCode; + console.error(`persona acceptance failed: ${errorMessage(error)}`); + return 1; + } finally { + if (!benchmarkOwnsSignals) { + await cleanupActiveChild(); + cleanupTemporaryDirectories(); + } + } +} + +function parseArgs(argv: string[]): CliOptions { + const args = argv.filter((argument) => argument !== "--"); + if (args.includes("--help") || args.includes("-h")) { + console.log(usage()); + process.exit(0); + } + + const mode = args.shift(); + if (mode !== "reference" && mode !== "pocket" && mode !== "bench") { + throw new Error("first argument must be reference, pocket, or bench"); + } + + let profile: BenchmarkProfile = "production"; + let cycles = 0; + let realAudio = false; + let settleSeconds = 30; + let sampleCount = 9; + let intervalSeconds = 5; + let outPath: string | null = null; + + const readValue = (name: string): string => { + const value = args.shift(); + if (value == null || value.startsWith("--")) { + throw new Error(`${name} requires a value`); + } + return value; + }; + const readNumber = ( + name: string, + minimum: number, + integer = false, + ): number => { + const value = Number(readValue(name)); + if ( + !Number.isFinite(value) || + value < minimum || + (integer && !Number.isInteger(value)) + ) { + throw new Error( + `${name} must be ${integer ? "an integer" : "a number"} >= ${minimum}`, + ); + } + return value; + }; + + while (args.length > 0) { + const flag = args.shift(); + switch (flag) { + case "--profile": { + const value = readValue(flag); + if (value !== "production" && value !== "controlled") { + throw new Error("--profile must be production or controlled"); + } + profile = value; + break; + } + case "--cycles": + cycles = readNumber(flag, 0, true); + break; + case "--real-audio": + realAudio = true; + break; + case "--settle": + settleSeconds = readNumber(flag, 0); + break; + case "--samples": + sampleCount = readNumber(flag, 1, true); + break; + case "--interval": + intervalSeconds = readNumber(flag, Number.EPSILON); + break; + case "--out": + outPath = resolve(readValue(flag)); + break; + default: + throw new Error(`unknown argument ${flag}`); + } + } + + if (mode !== "bench" && profile !== "production") { + throw new Error("--profile is only valid in bench mode"); + } + if (mode === "bench" && realAudio) { + throw new Error("--real-audio is only valid for accept:persona"); + } + + return { + mode, + profile, + cycles, + realAudio, + settleSeconds, + sampleCount, + intervalSeconds, + outPath, + }; +} + +async function preflight(options: CliOptions): Promise { + await requireCommand("git", ["--version"]); + if (options.mode !== "pocket") { + const nodeVersion = await commandOutput(["node", "--version"], ROOT, "reference"); + const major = Number(nodeVersion.trim().replace(/^v/, "").split(".")[0]); + if (!Number.isInteger(major) || major < 24) { + throw new Error(`Persona requires Node.js >=24; found ${nodeVersion.trim()}`); + } + await requireCommand("npm", ["--version"]); + } + if (options.mode !== "reference") { + await requireCommand("cargo", ["--version"]); + } + if (options.realAudio) { + await requireCommand("xcrun", ["--version"]); + } +} + +async function requireCommand(command: string, args: string[]): Promise { + try { + await commandOutput([command, ...args], ROOT, "pocket"); + } catch (error) { + throw new Error(`${command} is required: ${errorMessage(error)}`); + } +} + +async function ensureAssets(): Promise { + mkdirSync(join(ROOT, "assets"), { recursive: true }); + for (const asset of ASSETS) { + if (assetMatches(asset.path, asset)) { + console.log(`asset ${asset.name} (${asset.sha256.slice(0, 12)}…)`); + continue; + } + + console.log(`fetch ${asset.url}`); + const response = await fetch(asset.url, { + signal: AbortSignal.timeout(120_000), + }); + if (!response.ok) { + throw new Error(`${asset.url}: HTTP ${response.status}`); + } + + const temporary = `${asset.path}.tmp-${process.pid}`; + try { + await Bun.write(temporary, await response.arrayBuffer()); + assertAsset(temporary, asset); + renameSync(temporary, asset.path); + } finally { + rmSync(temporary, { force: true }); + } + console.log(`asset ${asset.name} (${asset.sha256.slice(0, 12)}…)`); + } +} + +function assetMatches(path: string, asset: Asset): boolean { + return ( + existsSync(path) && + statSync(path).isFile() && + statSync(path).size === asset.bytes && + sha256File(path) === asset.sha256 + ); +} + +function assertAsset(path: string, asset: Asset): void { + if (!existsSync(path) || !statSync(path).isFile()) { + throw new Error(`${asset.name} download did not produce a file`); + } + const bytes = statSync(path).size; + const sha256 = sha256File(path); + if (bytes !== asset.bytes || sha256 !== asset.sha256) { + throw new Error( + `${asset.name} failed integrity validation: expected ${asset.bytes} bytes ` + + `${asset.sha256}, got ${bytes} bytes ${sha256}`, + ); + } +} + +async function ensureReferenceCheckout(): Promise { + const gitDirectory = join(REFERENCE_ROOT, ".git"); + if (!existsSync(gitDirectory)) { + if ( + existsSync(REFERENCE_ROOT) && + (!statSync(REFERENCE_ROOT).isDirectory() || + readdirSync(REFERENCE_ROOT).length > 0) + ) { + throw new Error( + `${REFERENCE_ROOT} exists but is not a tool-owned Persona checkout`, + ); + } + mkdirSync(REFERENCE_ROOT, { recursive: true }); + await runCommand(["git", "init"], REFERENCE_ROOT, "reference"); + await runCommand( + ["git", "remote", "add", "origin", REFERENCE_REPOSITORY], + REFERENCE_ROOT, + "reference", + ); + } + + const remote = ( + await commandOutput( + ["git", "remote", "get-url", "origin"], + REFERENCE_ROOT, + "reference", + ) + ).trim(); + if (remote !== REFERENCE_REPOSITORY) { + throw new Error( + `refusing unexpected Persona reference remote ${JSON.stringify(remote)}`, + ); + } + + const head = await optionalCommandOutput( + ["git", "rev-parse", "HEAD"], + REFERENCE_ROOT, + "reference", + ); + if (head.trim() !== REFERENCE_COMMIT) { + const status = ( + await commandOutput( + ["git", "status", "--porcelain", "--untracked-files=no"], + REFERENCE_ROOT, + "reference", + ) + ).trim(); + if (status.length > 0) { + throw new Error( + `Persona reference has tracked edits and cannot switch commits: ${status}`, + ); + } + await runCommand( + ["git", "fetch", "--depth=1", "origin", REFERENCE_COMMIT], + REFERENCE_ROOT, + "reference", + ); + await runCommand( + ["git", "checkout", "--detach", "FETCH_HEAD"], + REFERENCE_ROOT, + "reference", + ); + } + + const verifiedHead = ( + await commandOutput( + ["git", "rev-parse", "HEAD"], + REFERENCE_ROOT, + "reference", + ) + ).trim(); + if (verifiedHead !== REFERENCE_COMMIT) { + throw new Error(`Persona reference resolved to unexpected commit ${verifiedHead}`); + } +} + +function stageReferenceLibrary(): void { + const assetsRoot = join(REFERENCE_ROOT, "public", "assets"); + const modelRoot = join(assetsRoot, "models"); + const animationRoot = join(assetsRoot, "animations"); + mkdirSync(modelRoot, { recursive: true }); + mkdirSync(animationRoot, { recursive: true }); + + ensureSymlink(ASSETS[0].path, join(modelRoot, "model.vrm")); + for (const name of [ + "idle", + "talk1", + "talk2", + "greeting", + "happy", + "finger-gun", + "dance", + ]) { + ensureSymlink(ASSETS[1].path, join(animationRoot, `${name}.vrma`)); + } + + const fixture = readFileSync(FIXTURE_LIBRARY); + if ( + !existsSync(REFERENCE_LIBRARY) || + !readFileSync(REFERENCE_LIBRARY).equals(fixture) + ) { + writeFileSync(REFERENCE_LIBRARY, fixture); + } +} + +function ensureSymlink(source: string, target: string): void { + if (existsSync(target) || isDanglingSymlink(target)) { + const current = lstatSync(target); + if ( + current.isSymbolicLink() && + resolve(dirname(target), readlinkSync(target)) === source + ) { + return; + } + if (current.isDirectory()) { + throw new Error(`refusing to replace directory ${target}`); + } + unlinkSync(target); + } + symlinkSync(source, target, "file"); +} + +function isDanglingSymlink(path: string): boolean { + try { + return lstatSync(path).isSymbolicLink(); + } catch { + return false; + } +} + +async function ensureReferenceBuild(realAudio: boolean): Promise { + const packageLock = join(REFERENCE_ROOT, "package-lock.json"); + const installFingerprint = sha256File(packageLock); + const installMarker = join(REFERENCE_STATE, "install.sha256"); + if ( + readText(installMarker).trim() !== installFingerprint || + !existsSync(REFERENCE_ELECTRON) + ) { + console.log("setup Persona npm dependencies"); + await runCommand(["npm", "ci"], REFERENCE_ROOT, "reference"); + writeText(installMarker, `${installFingerprint}\n`); + } + + const buildFingerprint = sha256Text( + [ + REFERENCE_COMMIT, + installFingerprint, + sha256File(FIXTURE_LIBRARY), + ...ASSETS.map((asset) => asset.sha256), + ].join("\n"), + ); + const buildMarker = join(REFERENCE_STATE, "renderer.sha256"); + const builtModel = join( + REFERENCE_ROOT, + "dist", + "assets", + "models", + "model.vrm", + ); + if ( + readText(buildMarker).trim() !== buildFingerprint || + !existsSync(join(REFERENCE_ROOT, "dist", "index.html")) || + !existsSync(builtModel) + ) { + console.log("build Persona renderer"); + await runCommand(["npm", "run", "build"], REFERENCE_ROOT, "reference"); + writeText(buildMarker, `${buildFingerprint}\n`); + } + + if (realAudio && !existsSync(REFERENCE_NATIVE_HELPER)) { + console.log("build Persona Core Audio listener"); + await runCommand( + ["npm", "run", "native:build"], + REFERENCE_ROOT, + "reference", + ); + } +} + +async function ensurePocketBuild(): Promise { + console.log("build Pocket Persona guest"); + mkdirSync(POCKET_GUEST_DIR, { recursive: true }); + const result = await Bun.build({ + entrypoints: [join(ROOT, "crates", "pocket-persona", "guest", "main.ts")], + outdir: POCKET_GUEST_DIR, + naming: "guest.js", + target: "browser", + format: "iife", + minify: true, + sourcemap: "external", + }); + if (!result.success) { + for (const log of result.logs) console.error(log); + throw new Error("Pocket Persona guest build failed"); + } + + console.log("build Pocket Persona release binary"); + await runCommand( + ["cargo", "build", "--release", "-p", "pocket-persona"], + ROOT, + "pocket", + ); + if (!existsSync(POCKET_BINARY)) { + throw new Error(`Pocket Persona binary is missing after build: ${POCKET_BINARY}`); + } +} + +async function runVisibleReference(options: CliOptions): Promise { + const [port] = await allocateLoopbackPorts(1); + const userData = mkdtempSync(join(tmpdir(), "pocket-character-persona-")); + temporaryDirectories.add(userData); + const env = { + ...process.env, + PERSONA_BRIDGE_PORT: String(port), + ...(options.realAudio + ? {} + : { + // Visual acceptance drives the shared event contract directly. Avoid + // attaching to an unrelated Codex/ChatGPT process or prompting for + // System Audio Recording permission unless explicitly requested. + PERSONA_TARGET_PROCESS_PATTERN: "a^", + }), + }; + + console.log( + options.realAudio + ? "audio real Persona listener enabled; macOS may request System Audio Recording" + : "audio visual event driver enabled; pass --real-audio to test Core Audio", + ); + await runVisibleTarget( + "reference", + [ + REFERENCE_ELECTRON, + REFERENCE_ROOT, + `--user-data-dir=${userData}`, + ], + REFERENCE_ROOT, + env, + port, + options.cycles, + ); +} + +async function runVisiblePocket(options: CliOptions): Promise { + const [port] = await allocateLoopbackPorts(1); + await runVisibleTarget( + "pocket", + [ + POCKET_BINARY, + "--library", + REFERENCE_LIBRARY, + "--bundle", + POCKET_GUEST, + "--bridge-port", + String(port), + "--fps", + "60", + "--max-texture-dim", + "2048", + ], + ROOT, + process.env, + port, + options.cycles, + ); +} + +async function runVisibleTarget( + target: TargetName, + argv: string[], + cwd: string, + env: Record, + port: number, + cycles: number, +): Promise { + console.log(`launch ${target}: ${formatCommand(argv)}`); + const child = Bun.spawn(argv, { + cwd, + env, + stdin: "inherit", + stdout: "inherit", + stderr: "inherit", + detached: true, + }); + activeChild = child; + activeTarget = target; + + try { + await waitForVisibleReady(target, child, port); + printVisualChecklist(target, port, cycles); + await driveVisualSequence(child, port, cycles); + } finally { + await cleanupActiveChild(); + } +} + +async function waitForVisibleReady( + target: TargetName, + child: SpawnedProcess, + port: number, +): Promise { + const deadline = performance.now() + READINESS_TIMEOUT_MS; + let lastFailure = "bridge has not responded"; + while (performance.now() < deadline) { + assertChildRunning(child, `${target} exited before visual readiness`); + try { + const response = await fetch(`http://127.0.0.1:${port}/health`, { + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + const body = (await response.json()) as Record; + if (!response.ok || body.ok !== true) { + lastFailure = `health returned ${response.status} ${JSON.stringify(body)}`; + } else if (target === "pocket") { + const status = body.status; + if ( + status != null && + typeof status === "object" && + (status as Record).modelConfigured === true && + (status as Record).windowVisible === true && + typeof (status as Record).renderFps === "number" && + ((status as Record).renderFps as number) > 1 + ) { + return; + } + lastFailure = `Pocket health is not render-ready: ${JSON.stringify(body)}`; + } else { + // Persona publishes bridge health before its renderer. Allow the + // configured avatar window to finish WebGL/model initialization. + await sleepWhileRunning(child, 3_000, "Persona exited during renderer warmup"); + return; + } + } catch (error) { + lastFailure = errorMessage(error); + } + await sleepWhileRunning(child, 250, `${target} exited during readiness`); + } + throw new Error(`${target} was not ready within 60s: ${lastFailure}`); +} + +function printVisualChecklist( + target: TargetName, + port: number, + cycles: number, +): void { + console.log(""); + console.log(`ready ${target} at http://127.0.0.1:${port}`); + console.log("check full character in a transparent, frameless, topmost 430x680 window"); + console.log("check idle loop, autonomous blink, and spring motion"); + console.log("check speaking body + pulsed lips, then action crossfade and return"); + console.log("input scroll zoom · left-drag orbit · right-drag pan"); + console.log( + cycles === 0 + ? "demo repeats until Ctrl-C" + : `demo ${cycles} cycle${cycles === 1 ? "" : "s"}, then exits`, + ); + console.log(""); +} + +async function driveVisualSequence( + child: SpawnedProcess, + port: number, + cycles: number, +): Promise { + let completed = 0; + while (cycles === 0 || completed < cycles) { + console.log(`demo cycle ${completed + 1}: idle`); + await postEvent(port, { + type: "state", + state: voiceState("inactive", "idle"), + }); + await sleepWhileRunning(child, 2_500, "target exited during idle demo"); + + console.log(`demo cycle ${completed + 1}: speaking + lip sync`); + await postEvent(port, { + type: "state", + state: voiceState("active", "speaking"), + }); + const levels = [0.04, 0.16, 0.34, 0.12, 0.52, 0.24, 0.08, 0]; + for (let repeat = 0; repeat < 4; repeat++) { + for (const level of levels) { + await postEvent(port, { type: "audio-level", level }); + await sleepWhileRunning( + child, + 110, + "target exited during lip-sync demo", + ); + } + } + + console.log(`demo cycle ${completed + 1}: listening`); + await postEvent(port, { + type: "state", + state: voiceState("active", "listening"), + }); + await sleepWhileRunning(child, 1_000, "target exited during listening demo"); + + console.log(`demo cycle ${completed + 1}: greeting action`); + await postEvent(port, { + type: "animation", + animation_name: "greeting", + }); + await sleepWhileRunning(child, 3_500, "target exited during action demo"); + completed++; + } +} + +function voiceState( + phase: "inactive" | "active", + activity: "idle" | "listening" | "speaking", +) { + return { + phase, + activity, + microphoneMuted: false, + outputMuted: false, + }; +} + +async function postEvent(port: number, body: unknown): Promise { + let lastFailure = "event request did not run"; + for (let attempt = 1; attempt <= 5; attempt++) { + try { + const response = await fetch(`http://127.0.0.1:${port}/events`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + const text = await response.text(); + if (response.status === 202) { + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + throw new Error(`event returned invalid JSON: ${text}`); + } + if ( + parsed == null || + typeof parsed !== "object" || + (parsed as Record).accepted !== true + ) { + throw new Error(`event was not accepted: ${text}`); + } + return; + } + lastFailure = `HTTP ${response.status}: ${text}`; + if (response.status !== 502 && response.status !== 503) { + break; + } + } catch (error) { + lastFailure = errorMessage(error); + } + if (attempt < 5) await Bun.sleep(75); + } + throw new Error(`event ${JSON.stringify(body)} failed: ${lastFailure}`); +} + +async function runBenchmark(options: CliOptions): Promise { + const profile = + options.profile === "controlled" + ? { fps: 120, texture: 4096 } + : { fps: 60, texture: 2048 }; + const stamp = new Date().toISOString().replace(/[:.]/g, "-"); + const outPath = + options.outPath ?? + join( + OUT, + "bench", + `persona-${options.profile}-${profile.fps}hz-${profile.texture}-${stamp}.json`, + ); + console.log( + `bench ${options.profile}: ${profile.fps} Hz / ${profile.texture}px textures`, + ); + console.log("bench keep each topmost window visible and leave input untouched"); + + benchmarkOwnsSignals = true; + try { + return await runPersonaBenchmark([ + "--reference-bin", + REFERENCE_ELECTRON, + "--reference-root", + REFERENCE_ROOT, + "--pocket-bin", + POCKET_BINARY, + "--library", + REFERENCE_LIBRARY, + "--bundle", + POCKET_GUEST, + "--max-fps", + String(profile.fps), + "--max-texture-dim", + String(profile.texture), + "--settle", + String(options.settleSeconds), + "--samples", + String(options.sampleCount), + "--interval", + String(options.intervalSeconds), + "--out", + outPath, + ]); + } finally { + benchmarkOwnsSignals = false; + } +} + +async function runCommand( + argv: string[], + cwd: string, + target: TargetName, +): Promise { + console.log(`run ${formatCommand(argv)}`); + const child = Bun.spawn(argv, { + cwd, + stdin: "inherit", + stdout: "inherit", + stderr: "inherit", + detached: true, + }); + activeChild = child; + activeTarget = target; + const exitCode = await child.exited; + if (activeChild?.pid === child.pid) activeChild = null; + if (exitCode !== 0) { + throw new Error(`${argv[0]} exited with status ${exitCode}`); + } +} + +async function commandOutput( + argv: string[], + cwd: string, + target: TargetName, +): Promise { + const child = Bun.spawn(argv, { + cwd, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + detached: true, + }); + activeChild = child; + activeTarget = target; + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]); + if (activeChild?.pid === child.pid) activeChild = null; + if (exitCode !== 0) { + throw new Error( + `${formatCommand(argv)} exited with status ${exitCode}: ${stderr.trim()}`, + ); + } + return stdout; +} + +async function optionalCommandOutput( + argv: string[], + cwd: string, + target: TargetName, +): Promise { + try { + return await commandOutput(argv, cwd, target); + } catch { + return ""; + } +} + +async function sleepWhileRunning( + child: SpawnedProcess, + milliseconds: number, + context: string, +): Promise { + const result = await Promise.race([ + Bun.sleep(milliseconds).then(() => null), + child.exited, + ]); + if (result !== null) { + throw new Error(`${context} (exit ${result})`); + } + assertChildRunning(child, context); +} + +function assertChildRunning(child: SpawnedProcess, context: string): void { + try { + process.kill(child.pid, 0); + } catch { + throw new Error(context); + } +} + +async function cleanupActiveChild(): Promise { + const child = activeChild; + if (child == null) return; + activeChild = null; + await terminateProcessTree(child, activeTarget); +} + +function installSignalHandlers(): void { + const handle = ( + signal: "SIGHUP" | "SIGINT" | "SIGTERM", + exitCode: number, + ) => { + if (benchmarkOwnsSignals || signalShutdown != null) return; + signalExitCode = exitCode; + signalShutdown = (async () => { + console.error(`persona acceptance: received ${signal}; cleaning up`); + await cleanupActiveChild(); + cleanupTemporaryDirectories(); + process.exit(exitCode); + })(); + }; + process.on("SIGHUP", () => handle("SIGHUP", 129)); + process.on("SIGINT", () => handle("SIGINT", 130)); + process.on("SIGTERM", () => handle("SIGTERM", 143)); +} + +function cleanupTemporaryDirectories(): void { + for (const directory of temporaryDirectories) { + rmSync(directory, { recursive: true, force: true }); + temporaryDirectories.delete(directory); + } +} + +function readText(path: string): string { + return existsSync(path) ? readFileSync(path, "utf8") : ""; +} + +function writeText(path: string, value: string): void { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, value); +} + +function sha256File(path: string): string { + return createHash("sha256").update(readFileSync(path)).digest("hex"); +} + +function sha256Text(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} + +function formatCommand(argv: string[]): string { + return argv + .map((argument) => + /^[A-Za-z0-9_./:=+-]+$/.test(argument) + ? argument + : JSON.stringify(argument), + ) + .join(" "); +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function usage(): string { + return `Pocket Persona acceptance + +Usage: + bun run accept:persona [-- --cycles N] [--real-audio] + bun run accept:pocket [-- --cycles N] + bun run bench:persona [-- --settle 30 --samples 9 --interval 5] + bun run bench:persona:controlled [-- --settle 30 --samples 9 --interval 5] + +The two visual commands stage the same pinned Persona catalog and repeat the +same idle, speaking/lip-sync, listening, and greeting sequence. N=0 (default) +repeats until Ctrl-C. Benchmark modes run Persona and Pocket sequentially and +write a timestamped JSON report under out/bench/.`; +} diff --git a/scripts/bench-persona.ts b/scripts/bench-persona.ts new file mode 100644 index 0000000..d699427 --- /dev/null +++ b/scripts/bench-persona.ts @@ -0,0 +1,2188 @@ +#!/usr/bin/env bun + +// Reproducible process-tree A/B harness for Persona's Electron reference and +// the Pocket-native implementation. The two targets run sequentially so they +// never compete for CPU/GPU/memory during sampling. + +import { + existsSync, + mkdirSync, + mkdtempSync, + realpathSync, + renameSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { + arch, + cpus, + homedir, + platform, + release, + tmpdir, + totalmem, +} from "node:os"; +import { createServer, type AddressInfo, type Server } from "node:net"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +export type TargetName = "reference" | "pocket"; +type RunStatus = "running" | "ok" | "failed"; + +interface Options { + referenceBin: string; + referenceRoot: string; + pocketBin: string; + library: string; + bundle: string; + maxFps: number | null; + maxTextureDim: number | null; + settleSeconds: number; + sampleCount: number; + intervalSeconds: number; + outPath: string; +} + +interface LaunchSpec { + target: TargetName; + argv: string[]; + cwd: string; + envOverrides: Record; + bridgePort: number; + healthUrl: string; + cdpPort?: number; + cdpListUrl?: string; +} + +type JsonObject = Record; + +interface HealthCapture { + captured_at: string; + response_time_ms: number; + http_status: number; + body: JsonObject; +} + +interface ReferenceCanvasReceipt { + ready_state: string; + viewport: [number, number, number]; + canvas: { + client: [number, number]; + backing: [number, number]; + }; + webgl: { + version: string; + renderer: string; + vendor: string; + }; +} + +interface ReferenceCdpReceipt { + ready_at: string; + target: { + id: string; + title: string; + url: string; + }; + canvas: ReferenceCanvasReceipt; +} + +interface ReferenceFrameSample { + requested_duration_ms: number; + elapsed_ms: number; + frames: number; + fps: number; + mean_interval_ms: number; + p50_interval_ms: number; + p95_interval_ms: number; + p99_interval_ms: number; + max_interval_ms: number; + over_20ms: number; +} + +interface ReferenceFrameReceipt { + captured_at: string; + frame_sample: ReferenceFrameSample; + task_duration_delta_seconds: number; + main_thread_cpu_percent: number; + performance_deltas_seconds: { + TaskDuration: number; + ScriptDuration: number; + LayoutDuration: number; + RecalcStyleDuration: number; + V8CompileDuration: number; + }; +} + +interface ProcessRow { + pid: number; + ppid: number; + state: string; + cpu_time: string; + cpu_time_seconds: number; + ps_cpu_percent: number; + rss_kib: number; + command: string; +} + +interface TreeSnapshot { + captured_at: string; + root_pid: number; + cumulative_cpu_time_seconds: number; + ps_cpu_percent_sum_diagnostic: number; + rss_kib: number; + process_count: number; + processes: ProcessRow[]; +} + +interface TimedTreeSnapshot { + clock_ms: number; + snapshot: TreeSnapshot; +} + +interface IntervalProcessRow extends ProcessRow { + interval_cpu_time_seconds: number; + interval_cpu_percent: number; + interval_identity: "continued" | "new-or-restarted"; +} + +interface ResourceSample { + index: number; + interval_started_at: string; + captured_at: string; + elapsed_since_baseline_seconds: number; + interval_seconds: number; + root_pid: number; + interval_cpu_time_seconds: number; + interval_cpu_percent: number; + cumulative_cpu_time_seconds: number; + ps_cpu_percent_sum_diagnostic: number; + rss_kib: number; + process_count: number; + processes: IntervalProcessRow[]; + disappeared_processes: ProcessRow[]; + health: HealthCapture; +} + +interface SummaryStats { + n: number; + mean: number; + min: number; + median: number; + p95: number; + max: number; +} + +interface RunSummary { + interval_cpu_percent: SummaryStats; + rss_kib: SummaryStats; + process_count: SummaryStats; + render_fps: SummaryStats | null; +} + +interface RunResult { + status: RunStatus; + root_pid: number; + launched_at: string; + ready_at: string | null; + readiness_health: HealthCapture | null; + reference_cdp: ReferenceCdpReceipt | null; + frame_receipt: ReferenceFrameReceipt | null; + settled_at: string | null; + sampling_completed_at: string | null; + terminated_at: string | null; + baseline: TreeSnapshot | null; + samples: ResourceSample[]; + summary: RunSummary | null; + error?: string; +} + +interface Report { + schema_version: 4; + benchmark: "persona-reference-vs-pocket"; + status: "running" | "ok" | "failed" | "interrupted"; + started_at: string; + completed_at: string | null; + elapsed_seconds: number | null; + configuration: { + settle_seconds: number; + sample_count: number; + interval_seconds: number; + max_fps: number | null; + max_texture_dim: number | null; + readiness_timeout_seconds: number; + cdp_timeout_seconds: number; + output: string; + }; + methodology: { + primary_cpu_metric: "interval_cpu_percent"; + cpu_time_source: "ps cumulative time/cputime per process"; + interval_cpu_formula: "sum(process CPU-time deltas) / wall interval * 100"; + cpu_percent_scale: "100 percent equals one logical core"; + ps_cpu_percent_role: "diagnostic-only"; + occlusion_policy: string; + readiness_policy: string; + sample_health_policy: string; + reference_frame_policy: string; + pocket_frame_policy: string; + exited_process_caveat: string; + }; + inputs: { + reference_root: string; + library: string; + bundle: string; + }; + machine: ReturnType; + commands: Partial>; + runs: Partial>; + comparison: ReturnType | null; + error?: string; +} + +type SpawnedProcess = ReturnType; + +const REPO_ROOT = fileURLToPath(new URL("..", import.meta.url)); +const DEFAULT_SETTLE_SECONDS = 15; +const DEFAULT_SAMPLE_COUNT = 13; +const DEFAULT_INTERVAL_SECONDS = 5; +const READINESS_TIMEOUT_SECONDS = 30; +const CDP_TIMEOUT_SECONDS = 30; +const HEALTH_POLL_INTERVAL_MS = 250; +const HEALTH_REQUEST_TIMEOUT_MS = 1_000; +const CDP_POLL_INTERVAL_MS = 250; +const CDP_COMMAND_TIMEOUT_MS = 10_000; +const REFERENCE_VIEWPORT = { + width: 430, + height: 680, + deviceScaleFactor: 1.5, +} as const; +const REFERENCE_CANVAS_BACKING = [645, 1020] as const; +const TERM_GRACE_MS = 4_000; +const KILL_GRACE_MS = 2_000; +const POLL_MS = 100; + +let activeRun: { target: TargetName; child: SpawnedProcess } | null = null; +const terminationPromises = new Map>(); +let persistInterruptedReport: ((signal: string) => void) | null = null; +let signalShutdown: Promise | null = null; +let signalHandlersInstalled = false; + +if (import.meta.main) { + process.exitCode = await runPersonaBenchmark(Bun.argv.slice(2)); +} + +export async function runPersonaBenchmark(argv: string[]): Promise { + installSignalHandlers(); + return main(argv); +} + +async function main(argv: string[]): Promise { + let options: Options; + try { + options = parseArgs(argv); + validateInputs(options); + prepareOutput(options.outPath); + } catch (error) { + console.error(errorMessage(error)); + console.error(""); + console.error(usage()); + return 2; + } + + const startedAt = new Date(); + const startedClock = performance.now(); + const report: Report = { + schema_version: 4, + benchmark: "persona-reference-vs-pocket", + status: "running", + started_at: startedAt.toISOString(), + completed_at: null, + elapsed_seconds: null, + configuration: { + settle_seconds: options.settleSeconds, + sample_count: options.sampleCount, + interval_seconds: options.intervalSeconds, + max_fps: options.maxFps, + max_texture_dim: options.maxTextureDim, + readiness_timeout_seconds: READINESS_TIMEOUT_SECONDS, + cdp_timeout_seconds: CDP_TIMEOUT_SECONDS, + output: options.outPath, + }, + methodology: { + primary_cpu_metric: "interval_cpu_percent", + cpu_time_source: "ps cumulative time/cputime per process", + interval_cpu_formula: + "sum(process CPU-time deltas) / wall interval * 100", + cpu_percent_scale: "100 percent equals one logical core", + ps_cpu_percent_role: "diagnostic-only", + occlusion_policy: + "both targets use normal compositor visibility; keep each benchmark window visible and uncovered", + readiness_policy: + "each target must return HTTP 200 JSON with ok:true from its isolated loopback /health endpoint before settling", + sample_health_policy: + "every resource sample includes a contemporaneous successful /health response", + reference_frame_policy: + "Electron must expose a non-settings CDP page with a ready 430x680 DPR 1.5 WebGL canvas backed by 645x1020 pixels; one lightweight rAF promise spans the resource sampling window", + pocket_frame_policy: + "every Pocket health receipt must report modelConfigured=true, windowVisible=true, and renderFps>1", + exited_process_caveat: + "CPU accrued after the previous snapshot by a process that exits before the next snapshot is not observable", + }, + inputs: { + reference_root: options.referenceRoot, + library: options.library, + bundle: options.bundle, + }, + machine: machineFacts(), + commands: {}, + runs: {}, + comparison: null, + }; + + let referenceBridgePort: number; + let pocketBridgePort: number; + let referenceCdpPort: number; + try { + [referenceBridgePort, pocketBridgePort, referenceCdpPort] = + await allocateLoopbackPorts(3); + } catch (error) { + report.status = "failed"; + report.error = `unable to allocate loopback ports: ${errorMessage(error)}`; + finishReportClock(report, startedClock); + writeReport(options.outPath, report); + console.error(`bench-persona: ${report.error}`); + return 1; + } + + const userDataDir = mkdtempSync( + join(tmpdir(), "pocket-character-persona-bench-"), + ); + persistInterruptedReport = (signal) => { + report.status = "interrupted"; + report.error = `interrupted by ${signal}`; + finishReportClock(report, startedClock); + try { + writeReport(options.outPath, report); + } finally { + rmSync(userDataDir, { recursive: true, force: true }); + } + }; + + const referenceSpec: LaunchSpec = { + target: "reference", + argv: [ + options.referenceBin, + options.referenceRoot, + `--user-data-dir=${userDataDir}`, + `--remote-debugging-port=${referenceCdpPort}`, + ], + cwd: options.referenceRoot, + envOverrides: { + PERSONA_BRIDGE_PORT: String(referenceBridgePort), + // The benchmark drives the shared event contract directly. Do not let a + // concurrently running voice app add an optional native capture helper + // to only the Electron process tree. + PERSONA_TARGET_PROCESS_PATTERN: "a^", + }, + bridgePort: referenceBridgePort, + healthUrl: `http://127.0.0.1:${referenceBridgePort}/health`, + cdpPort: referenceCdpPort, + cdpListUrl: `http://127.0.0.1:${referenceCdpPort}/json/list`, + }; + const pocketSpec: LaunchSpec = { + target: "pocket", + argv: [ + options.pocketBin, + "--library", + options.library, + "--bundle", + options.bundle, + ...(options.maxFps == null + ? [] + : ["--max-fps", String(options.maxFps)]), + ...(options.maxTextureDim == null + ? [] + : ["--max-texture-dim", String(options.maxTextureDim)]), + "--bridge-port", + String(pocketBridgePort), + ], + cwd: process.cwd(), + envOverrides: {}, + bridgePort: pocketBridgePort, + healthUrl: `http://127.0.0.1:${pocketBridgePort}/health`, + }; + report.commands.reference = referenceSpec; + report.commands.pocket = pocketSpec; + + let exitCode = 0; + try { + await benchmarkTarget(referenceSpec, options, (run) => { + report.runs.reference = run; + }); + + // The reference must be fully gone before the Pocket target starts. + if (options.outPath !== "-") writeReport(options.outPath, report); + + await benchmarkTarget(pocketSpec, options, (run) => { + report.runs.pocket = run; + }); + + const reference = report.runs.reference; + const pocket = report.runs.pocket; + if (!reference?.summary || !pocket?.summary) { + throw new Error("both benchmark runs must have summaries"); + } + if (reference.frame_receipt == null || pocket.summary.render_fps == null) { + throw new Error("frame receipts are incomplete"); + } + report.comparison = compareRuns(reference, pocket); + report.status = "ok"; + } catch (error) { + report.status = "failed"; + report.error = errorMessage(error); + console.error(`bench-persona: ${report.error}`); + exitCode = 1; + } finally { + persistInterruptedReport = null; + rmSync(userDataDir, { recursive: true, force: true }); + finishReportClock(report, startedClock); + writeReport(options.outPath, report); + } + + if (options.outPath === "-") { + console.error("bench-persona: JSON written to stdout"); + } else { + console.error(`bench-persona: report ${options.outPath}`); + } + return exitCode; +} + +export async function allocateLoopbackPorts(count: number): Promise { + if (!Number.isInteger(count) || count < 1) { + throw new Error(`invalid loopback port count ${count}`); + } + const servers: Server[] = Array.from({ length: count }, () => createServer()); + const ports: number[] = []; + try { + for (const server of servers) { + await new Promise((resolveListen, rejectListen) => { + const onError = (error: Error) => rejectListen(error); + server.once("error", onError); + server.listen( + { + host: "127.0.0.1", + port: 0, + exclusive: true, + }, + () => { + server.off("error", onError); + resolveListen(); + }, + ); + }); + const address = server.address(); + if (address == null || typeof address === "string") { + throw new Error("loopback reservation returned no numeric address"); + } + ports.push((address as AddressInfo).port); + } + } finally { + await Promise.all( + servers.map( + (server) => + new Promise((resolveClose, rejectClose) => { + if (!server.listening) { + resolveClose(); + return; + } + server.close((error) => { + if (error) rejectClose(error); + else resolveClose(); + }); + }), + ), + ); + } + if (ports.length !== count || new Set(ports).size !== count) { + throw new Error( + `expected ${count} distinct loopback ports, got ${ports.join(",")}`, + ); + } + return ports; +} + +async function waitForReadyHealth( + spec: LaunchSpec, + child: SpawnedProcess, +): Promise { + const deadline = + performance.now() + READINESS_TIMEOUT_SECONDS * 1_000; + let lastFailure = "health endpoint has not responded"; + + while (performance.now() < deadline) { + const remainingMs = deadline - performance.now(); + const result = await probeHealth( + spec, + child, + Math.max(1, Math.min(HEALTH_REQUEST_TIMEOUT_MS, remainingMs)), + ); + if (result.ok) { + const validationFailure = healthValidationFailure(spec, result.capture); + if (validationFailure == null) return result.capture; + lastFailure = validationFailure; + } else { + lastFailure = result.failure; + } + + const waitMs = Math.min( + HEALTH_POLL_INTERVAL_MS, + Math.max(0, deadline - performance.now()), + ); + if (waitMs > 0) { + await delayWhileRunning( + child, + waitMs, + `${spec.target} exited before readiness`, + ); + } + } + + throw new Error( + `${spec.target} readiness timed out after ${READINESS_TIMEOUT_SECONDS}s: ${lastFailure}`, + ); +} + +async function captureRequiredHealth( + spec: LaunchSpec, + child: SpawnedProcess, + context: string, +): Promise { + const result = await probeHealth( + spec, + child, + HEALTH_REQUEST_TIMEOUT_MS, + ); + if (!result.ok) { + throw new Error( + `${spec.target} ${context} health check failed: ${result.failure}`, + ); + } + const validationFailure = healthValidationFailure(spec, result.capture); + if (validationFailure != null) { + throw new Error( + `${spec.target} ${context} health check failed: ${validationFailure}`, + ); + } + return result.capture; +} + +function healthValidationFailure( + spec: LaunchSpec, + capture: HealthCapture, +): string | null { + if (spec.target !== "pocket") return null; + try { + validatePocketHealthBody(capture.body); + return null; + } catch (error) { + return errorMessage(error); + } +} + +export function validatePocketHealthBody(body: JsonObject): number { + if (!isJsonObject(body.status)) { + throw new Error("Pocket health status must be an object"); + } + const status = body.status; + if (status.modelConfigured !== true) { + throw new Error("Pocket health modelConfigured must be true"); + } + if (status.windowVisible !== true) { + throw new Error("Pocket health windowVisible must be true"); + } + if ( + typeof status.renderFps !== "number" || + !Number.isFinite(status.renderFps) || + status.renderFps <= 1 + ) { + throw new Error("Pocket health renderFps must be a finite number > 1"); + } + return status.renderFps; +} + +async function probeHealth( + spec: LaunchSpec, + child: SpawnedProcess, + timeoutMs: number, +): Promise< + | { ok: true; capture: HealthCapture } + | { ok: false; failure: string } +> { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + const started = performance.now(); + const request = (async () => { + try { + const response = await fetch(spec.healthUrl, { + headers: { accept: "application/json" }, + signal: controller.signal, + }); + const text = await response.text(); + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + return { + ok: false as const, + failure: `HTTP ${response.status} returned invalid JSON: ${text.slice(0, 200)}`, + }; + } + if (!isJsonObject(parsed)) { + return { + ok: false as const, + failure: `HTTP ${response.status} returned a non-object JSON body`, + }; + } + if (response.status !== 200 || parsed.ok !== true) { + return { + ok: false as const, + failure: + `HTTP ${response.status} health ok=${String(parsed.ok)}: ` + + text.slice(0, 200), + }; + } + return { + ok: true as const, + capture: { + captured_at: new Date().toISOString(), + response_time_ms: round(performance.now() - started), + http_status: response.status, + body: parsed, + }, + }; + } catch (error) { + return { + ok: false as const, + failure: + controller.signal.aborted + ? `request exceeded ${round(timeoutMs)}ms` + : errorMessage(error), + }; + } + })(); + + const outcome = await Promise.race([ + request.then((result) => ({ kind: "health" as const, result })), + child.exited.then((exitCode) => ({ + kind: "exit" as const, + exitCode, + })), + ]); + clearTimeout(timeout); + if (outcome.kind === "exit") { + controller.abort(); + throw new Error( + `${spec.target} exited before health readiness (exit ${outcome.exitCode})`, + ); + } + return outcome.result; +} + +function isJsonObject(value: unknown): value is JsonObject { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +interface CdpTarget { + id: string; + title: string; + url: string; + webSocketDebuggerUrl: string; +} + +interface PreparedReferenceCdp { + client: CdpClient; + receipt: ReferenceCdpReceipt; +} + +interface ActiveReferenceFrameMeasurement { + requestedDurationMs: number; + beforeMetrics: Record; + framePromise: Promise; +} + +interface CdpPending { + resolve: (value: unknown) => void; + reject: (error: Error) => void; + timeout: ReturnType; +} + +class CdpClient { + private nextId = 1; + private readonly pending = new Map(); + private closed = false; + + constructor(private readonly socket: WebSocket) { + socket.addEventListener("message", (event) => { + void this.handleMessage(event.data); + }); + socket.addEventListener("close", () => { + this.failPending(new Error("CDP socket closed")); + }); + socket.addEventListener("error", () => { + this.failPending(new Error("CDP socket error")); + }); + } + + send( + method: string, + params: JsonObject = {}, + timeoutMs = CDP_COMMAND_TIMEOUT_MS, + ): Promise { + if (this.closed || this.socket.readyState !== WebSocket.OPEN) { + return Promise.reject(new Error(`CDP socket is not open for ${method}`)); + } + const id = this.nextId++; + return new Promise((resolveSend, rejectSend) => { + const timeout = setTimeout(() => { + this.pending.delete(id); + rejectSend(new Error(`CDP ${method} timed out after ${timeoutMs}ms`)); + }, timeoutMs); + this.pending.set(id, { + resolve: (value) => resolveSend(value as T), + reject: rejectSend, + timeout, + }); + try { + this.socket.send(JSON.stringify({ id, method, params })); + } catch (error) { + clearTimeout(timeout); + this.pending.delete(id); + rejectSend(new Error(`CDP ${method} send failed: ${errorMessage(error)}`)); + } + }); + } + + close(): void { + if (this.closed) return; + this.closed = true; + this.failPending(new Error("CDP client closed")); + this.socket.close(); + } + + private async handleMessage(data: unknown): Promise { + let text: string; + if (typeof data === "string") text = data; + else if (data instanceof ArrayBuffer) text = new TextDecoder().decode(data); + else if (data instanceof Blob) text = await data.text(); + else return; + + let message: unknown; + try { + message = JSON.parse(text); + } catch { + this.failPending(new Error("CDP returned invalid JSON")); + return; + } + if (!isJsonObject(message) || typeof message.id !== "number") return; + const pending = this.pending.get(message.id); + if (!pending) return; + this.pending.delete(message.id); + clearTimeout(pending.timeout); + if (message.error != null) { + pending.reject( + new Error(`CDP command failed: ${JSON.stringify(message.error)}`), + ); + } else { + pending.resolve(message.result ?? {}); + } + } + + private failPending(error: Error): void { + for (const pending of this.pending.values()) { + clearTimeout(pending.timeout); + pending.reject(error); + } + this.pending.clear(); + } +} + +async function prepareReferenceCdp( + spec: LaunchSpec, + child: SpawnedProcess, +): Promise { + if (spec.cdpListUrl == null || spec.cdpPort == null) { + throw new Error("reference launch is missing its CDP endpoint"); + } + const deadline = performance.now() + CDP_TIMEOUT_SECONDS * 1_000; + let lastFailure = "CDP target list has not responded"; + let target: CdpTarget | null = null; + + while (performance.now() < deadline && target == null) { + try { + const raw = await fetchJsonWhileRunning( + spec.cdpListUrl, + spec, + child, + Math.min( + HEALTH_REQUEST_TIMEOUT_MS, + Math.max(1, deadline - performance.now()), + ), + ); + target = selectReferenceCdpTarget(raw); + if (target == null) lastFailure = "no non-settings page target"; + } catch (error) { + lastFailure = errorMessage(error); + } + if (target == null) { + await delayWhileRunning( + child, + Math.min( + CDP_POLL_INTERVAL_MS, + Math.max(0, deadline - performance.now()), + ), + "reference exited before CDP target readiness", + ); + } + } + if (target == null) { + throw new Error( + `reference CDP target timed out after ${CDP_TIMEOUT_SECONDS}s: ${lastFailure}`, + ); + } + + const client = await connectCdp(target.webSocketDebuggerUrl, spec, child); + try { + await client.send("Runtime.enable"); + await client.send("Performance.enable"); + await client.send("Emulation.setDeviceMetricsOverride", { + width: REFERENCE_VIEWPORT.width, + height: REFERENCE_VIEWPORT.height, + deviceScaleFactor: REFERENCE_VIEWPORT.deviceScaleFactor, + mobile: false, + }); + const canvas = await waitForReferenceCanvas(client, child); + return { + client, + receipt: { + ready_at: new Date().toISOString(), + target: { + id: target.id, + title: target.title, + url: target.url, + }, + canvas, + }, + }; + } catch (error) { + client.close(); + throw error; + } +} + +async function fetchJsonWhileRunning( + url: string, + spec: LaunchSpec, + child: SpawnedProcess, + timeoutMs: number, +): Promise { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + const request = (async () => { + const response = await fetch(url, { + headers: { accept: "application/json" }, + signal: controller.signal, + }); + if (!response.ok) throw new Error(`HTTP ${response.status} from ${url}`); + return response.json() as Promise; + })(); + const outcome = await Promise.race([ + request.then( + (value) => ({ kind: "response" as const, value }), + (error) => ({ kind: "error" as const, error }), + ), + child.exited.then((exitCode) => ({ kind: "exit" as const, exitCode })), + ]); + clearTimeout(timeout); + if (outcome.kind === "exit") { + controller.abort(); + throw new Error( + `${spec.target} exited before CDP readiness (exit ${outcome.exitCode})`, + ); + } + if (outcome.kind === "error") throw outcome.error; + return outcome.value; +} + +function selectReferenceCdpTarget(value: unknown): CdpTarget | null { + if (!Array.isArray(value)) return null; + for (const candidate of value) { + if ( + !isJsonObject(candidate) || + candidate.type !== "page" || + typeof candidate.id !== "string" || + typeof candidate.title !== "string" || + typeof candidate.url !== "string" || + typeof candidate.webSocketDebuggerUrl !== "string" + ) { + continue; + } + const identity = `${candidate.title} ${candidate.url}`.toLowerCase(); + if (identity.includes("settings") || candidate.url.startsWith("devtools://")) { + continue; + } + return { + id: candidate.id, + title: candidate.title, + url: candidate.url, + webSocketDebuggerUrl: candidate.webSocketDebuggerUrl, + }; + } + return null; +} + +async function connectCdp( + url: string, + spec: LaunchSpec, + child: SpawnedProcess, +): Promise { + const socket = new WebSocket(url); + const opened = new Promise< + { kind: "open" } | { kind: "error"; error: Error } + >((resolveOpen) => { + socket.addEventListener( + "open", + () => resolveOpen({ kind: "open" }), + { once: true }, + ); + socket.addEventListener( + "error", + () => + resolveOpen({ + kind: "error", + error: new Error(`unable to connect CDP socket ${url}`), + }), + { once: true }, + ); + }); + let timeoutId: ReturnType; + const timeout = new Promise<{ kind: "timeout" }>((resolveTimeout) => { + timeoutId = setTimeout( + () => resolveTimeout({ kind: "timeout" }), + CDP_COMMAND_TIMEOUT_MS, + ); + }); + const outcome = await Promise.race([ + opened, + timeout, + child.exited.then((exitCode) => ({ kind: "exit" as const, exitCode })), + ]); + clearTimeout(timeoutId!); + if (outcome.kind !== "open") { + socket.close(); + if (outcome.kind === "exit") { + throw new Error( + `${spec.target} exited before CDP socket readiness (exit ${outcome.exitCode})`, + ); + } + if (outcome.kind === "error") throw outcome.error; + throw new Error(`CDP socket connection timed out after ${CDP_COMMAND_TIMEOUT_MS}ms`); + } + return new CdpClient(socket); +} + +function referenceCanvasExpression(): string { + return `(() => { + const canvas = document.querySelector("canvas"); + const gl = canvas?.getContext("webgl2") ?? canvas?.getContext("webgl"); + const debug = gl?.getExtension("WEBGL_debug_renderer_info"); + return { + ready_state: document.readyState, + viewport: [innerWidth, innerHeight, devicePixelRatio], + canvas: canvas ? { + client: [canvas.clientWidth, canvas.clientHeight], + backing: [canvas.width, canvas.height], + } : null, + webgl: gl ? { + version: String(gl.getParameter(gl.VERSION)), + renderer: String(debug ? gl.getParameter(debug.UNMASKED_RENDERER_WEBGL) : gl.getParameter(gl.RENDERER)), + vendor: String(debug ? gl.getParameter(debug.UNMASKED_VENDOR_WEBGL) : gl.getParameter(gl.VENDOR)), + } : null, + }; +})()`; +} + +async function waitForReferenceCanvas( + client: CdpClient, + child: SpawnedProcess, +): Promise { + const deadline = performance.now() + CDP_TIMEOUT_SECONDS * 1_000; + let lastFailure = "document and canvas are not ready"; + while (performance.now() < deadline) { + try { + const evaluation = await client.send("Runtime.evaluate", { + expression: referenceCanvasExpression(), + returnByValue: true, + }); + const value = runtimeEvaluationValue(evaluation, "reference canvas"); + return parseReferenceCanvasReceipt(value); + } catch (error) { + lastFailure = errorMessage(error); + } + await delayWhileRunning( + child, + Math.min( + CDP_POLL_INTERVAL_MS, + Math.max(0, deadline - performance.now()), + ), + "reference exited before canvas readiness", + ); + } + throw new Error( + `reference canvas validation timed out after ${CDP_TIMEOUT_SECONDS}s: ${lastFailure}`, + ); +} + +export function parseReferenceCanvasReceipt( + value: unknown, +): ReferenceCanvasReceipt { + if (!isJsonObject(value)) throw new Error("canvas receipt is not an object"); + if (value.ready_state !== "complete") { + throw new Error(`document readyState is ${String(value.ready_state)}`); + } + const viewport = numberTuple(value.viewport, 3, "viewport"); + if ( + viewport[0] !== REFERENCE_VIEWPORT.width || + viewport[1] !== REFERENCE_VIEWPORT.height || + viewport[2] !== REFERENCE_VIEWPORT.deviceScaleFactor + ) { + throw new Error(`unexpected viewport ${viewport.join("x")}`); + } + if (!isJsonObject(value.canvas)) throw new Error("canvas is missing"); + const client = numberTuple(value.canvas.client, 2, "canvas client"); + const backing = numberTuple(value.canvas.backing, 2, "canvas backing"); + if ( + backing[0] !== REFERENCE_CANVAS_BACKING[0] || + backing[1] !== REFERENCE_CANVAS_BACKING[1] + ) { + throw new Error(`unexpected canvas backing ${backing.join("x")}`); + } + if (!isJsonObject(value.webgl)) throw new Error("WebGL context is missing"); + const version = requiredString(value.webgl.version, "WebGL version"); + const renderer = requiredString(value.webgl.renderer, "WebGL renderer"); + const vendor = requiredString(value.webgl.vendor, "WebGL vendor"); + return { + ready_state: "complete", + viewport: [viewport[0], viewport[1], viewport[2]], + canvas: { + client: [client[0], client[1]], + backing: [backing[0], backing[1]], + }, + webgl: { version, renderer, vendor }, + }; +} + +function runtimeEvaluationValue(value: unknown, context: string): unknown { + if (!isJsonObject(value)) throw new Error(`${context} CDP result is invalid`); + if (value.exceptionDetails != null) { + throw new Error(`${context} evaluation failed: ${JSON.stringify(value.exceptionDetails)}`); + } + if (!isJsonObject(value.result) || !("value" in value.result)) { + throw new Error(`${context} evaluation returned no value`); + } + return value.result.value; +} + +function numberTuple( + value: unknown, + length: number, + name: string, +): number[] { + if ( + !Array.isArray(value) || + value.length !== length || + value.some((item) => typeof item !== "number" || !Number.isFinite(item)) + ) { + throw new Error(`${name} must contain ${length} finite numbers`); + } + return value; +} + +function requiredString(value: unknown, name: string): string { + if (typeof value !== "string" || value.length === 0) { + throw new Error(`${name} is missing`); + } + return value; +} + +async function startReferenceFrameMeasurement( + client: CdpClient, + durationMs: number, +): Promise { + if (!Number.isFinite(durationMs) || durationMs <= 0) { + throw new Error(`invalid reference frame duration ${durationMs}`); + } + const beforeMetrics = performanceMetricMap( + await client.send("Performance.getMetrics"), + ); + const expression = `new Promise((resolve) => { + const intervals = []; + const start = performance.now(); + let last = start; + function tick(now) { + intervals.push(now - last); + last = now; + if (now - start < ${JSON.stringify(durationMs)}) { + requestAnimationFrame(tick); + return; + } + const sorted = intervals.slice().sort((a, b) => a - b); + const percentile = (p) => sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * p))] ?? null; + let total = 0; + let over20 = 0; + for (const interval of intervals) { + total += interval; + if (interval > 20) over20++; + } + const elapsed = now - start; + resolve({ + requested_duration_ms: ${JSON.stringify(durationMs)}, + elapsed_ms: elapsed, + frames: intervals.length, + fps: intervals.length / (elapsed / 1000), + mean_interval_ms: total / intervals.length, + p50_interval_ms: percentile(0.50), + p95_interval_ms: percentile(0.95), + p99_interval_ms: percentile(0.99), + max_interval_ms: sorted.at(-1) ?? null, + over_20ms: over20, + }); + } + requestAnimationFrame(tick); + })`; + const framePromise = client + .send( + "Runtime.evaluate", + { + expression, + awaitPromise: true, + returnByValue: true, + }, + durationMs + CDP_COMMAND_TIMEOUT_MS, + ) + .then((result) => + parseReferenceFrameSample( + runtimeEvaluationValue(result, "reference frame sample"), + durationMs, + ), + ); + void framePromise.catch(() => undefined); + return { requestedDurationMs: durationMs, beforeMetrics, framePromise }; +} + +async function finishReferenceFrameMeasurement( + client: CdpClient, + active: ActiveReferenceFrameMeasurement, +): Promise { + const frameSample = await active.framePromise; + const afterMetrics = performanceMetricMap( + await client.send("Performance.getMetrics"), + ); + const names = [ + "TaskDuration", + "ScriptDuration", + "LayoutDuration", + "RecalcStyleDuration", + "V8CompileDuration", + ] as const; + const deltas = Object.fromEntries( + names.map((name) => [ + name, + round( + name === "TaskDuration" + ? requiredMetric(afterMetrics, name) - + requiredMetric(active.beforeMetrics, name) + : (afterMetrics[name] ?? 0) - + (active.beforeMetrics[name] ?? 0), + ), + ]), + ) as ReferenceFrameReceipt["performance_deltas_seconds"]; + if (deltas.TaskDuration < 0) { + throw new Error(`negative CDP TaskDuration delta ${deltas.TaskDuration}`); + } + const mainThreadCpu = round( + (100 * deltas.TaskDuration) / (frameSample.elapsed_ms / 1_000), + ); + if (!Number.isFinite(mainThreadCpu)) { + throw new Error("invalid CDP main-thread CPU result"); + } + return { + captured_at: new Date().toISOString(), + frame_sample: frameSample, + task_duration_delta_seconds: deltas.TaskDuration, + main_thread_cpu_percent: mainThreadCpu, + performance_deltas_seconds: deltas, + }; +} + +function performanceMetricMap(value: unknown): Record { + if (!isJsonObject(value) || !Array.isArray(value.metrics)) { + throw new Error("CDP Performance.getMetrics returned no metrics"); + } + const result: Record = {}; + for (const metric of value.metrics) { + if ( + isJsonObject(metric) && + typeof metric.name === "string" && + typeof metric.value === "number" && + Number.isFinite(metric.value) + ) { + result[metric.name] = metric.value; + } + } + return result; +} + +function requiredMetric(metrics: Record, name: string): number { + const value = metrics[name]; + if (value == null || !Number.isFinite(value)) { + throw new Error(`CDP metric ${name} is missing`); + } + return value; +} + +export function parseReferenceFrameSample( + value: unknown, + requestedDurationMs: number, +): ReferenceFrameSample { + if (!isJsonObject(value)) throw new Error("frame sample is not an object"); + const finite = (name: string): number => { + const item = value[name]; + if (typeof item !== "number" || !Number.isFinite(item)) { + throw new Error(`frame sample ${name} is invalid`); + } + return item; + }; + const sample: ReferenceFrameSample = { + requested_duration_ms: finite("requested_duration_ms"), + elapsed_ms: finite("elapsed_ms"), + frames: finite("frames"), + fps: finite("fps"), + mean_interval_ms: finite("mean_interval_ms"), + p50_interval_ms: finite("p50_interval_ms"), + p95_interval_ms: finite("p95_interval_ms"), + p99_interval_ms: finite("p99_interval_ms"), + max_interval_ms: finite("max_interval_ms"), + over_20ms: finite("over_20ms"), + }; + if ( + sample.requested_duration_ms !== requestedDurationMs || + sample.elapsed_ms < requestedDurationMs * 0.9 || + !Number.isInteger(sample.frames) || + sample.frames < 2 || + sample.fps <= 1 || + sample.mean_interval_ms <= 0 || + sample.p50_interval_ms <= 0 || + sample.p95_interval_ms <= 0 || + sample.p99_interval_ms <= 0 || + sample.max_interval_ms <= 0 || + !Number.isInteger(sample.over_20ms) || + sample.over_20ms < 0 + ) { + throw new Error(`reference FPS receipt failed validation: ${JSON.stringify(sample)}`); + } + return sample; +} + +async function benchmarkTarget( + spec: LaunchSpec, + options: Options, + onStart: (run: RunResult) => void, +): Promise { + console.error(`bench-persona: launching ${spec.target}: ${JSON.stringify(spec.argv)}`); + const child = Bun.spawn(spec.argv, { + cwd: spec.cwd, + env: { + ...process.env, + ...spec.envOverrides, + }, + stdin: "ignore", + // Keep stdout available for a JSON report while still exposing app logs. + stdout: 2, + stderr: 2, + detached: true, + }); + const run: RunResult = { + status: "running", + root_pid: child.pid, + launched_at: new Date().toISOString(), + ready_at: null, + readiness_health: null, + reference_cdp: null, + frame_receipt: null, + settled_at: null, + sampling_completed_at: null, + terminated_at: null, + baseline: null, + samples: [], + summary: null, + }; + onStart(run); + activeRun = { target: spec.target, child }; + let referenceCdp: CdpClient | null = null; + let frameMeasurement: ActiveReferenceFrameMeasurement | null = null; + + try { + console.error( + `bench-persona: ${spec.target} pid=${child.pid}; waiting for ${spec.healthUrl}`, + ); + const readiness = await waitForReadyHealth(spec, child); + run.ready_at = readiness.captured_at; + run.readiness_health = readiness; + if (spec.target === "reference") { + const prepared = await prepareReferenceCdp(spec, child); + referenceCdp = prepared.client; + run.reference_cdp = prepared.receipt; + console.error( + `bench-persona: reference CDP canvas ready ` + + `${prepared.receipt.canvas.canvas.backing.join("x")}`, + ); + } + console.error( + `bench-persona: ${spec.target} ready; settling ${options.settleSeconds}s`, + ); + await delayWhileRunning( + child, + options.settleSeconds * 1_000, + `${spec.target} exited during settle`, + ); + run.settled_at = new Date().toISOString(); + + if (referenceCdp != null) { + frameMeasurement = await startReferenceFrameMeasurement( + referenceCdp, + options.sampleCount * options.intervalSeconds * 1_000, + ); + } + let previous = await captureProcessTree(child.pid); + run.baseline = previous.snapshot; + const samplingStart = previous.clock_ms; + console.error( + `bench-persona: ${spec.target} baseline ` + + `cpu-time=${previous.snapshot.cumulative_cpu_time_seconds.toFixed(2)}s ` + + `rss=${formatMiB(previous.snapshot.rss_kib)} ` + + `processes=${previous.snapshot.process_count}`, + ); + + for (let index = 0; index < options.sampleCount; index++) { + const scheduledAt = + samplingStart + (index + 1) * options.intervalSeconds * 1_000; + await delayWhileRunning( + child, + Math.max(0, scheduledAt - performance.now()), + `${spec.target} exited between samples`, + ); + const current = await captureProcessTree(child.pid); + const health = await captureRequiredHealth( + spec, + child, + `sample ${index + 1}`, + ); + const sample = buildIntervalSample( + index + 1, + samplingStart, + previous, + current, + health, + ); + run.samples.push(sample); + console.error( + `bench-persona: ${spec.target} ${index + 1}/${options.sampleCount} ` + + `interval-cpu=${sample.interval_cpu_percent.toFixed(1)}% ` + + `ps-cpu=${sample.ps_cpu_percent_sum_diagnostic.toFixed(1)}% ` + + `rss=${formatMiB(sample.rss_kib)} ` + + `processes=${sample.process_count}`, + ); + previous = current; + } + + run.sampling_completed_at = new Date().toISOString(); + if (referenceCdp != null && frameMeasurement != null) { + run.frame_receipt = await finishReferenceFrameMeasurement( + referenceCdp, + frameMeasurement, + ); + console.error( + `bench-persona: reference delivered ` + + `${run.frame_receipt.frame_sample.fps.toFixed(1)} fps`, + ); + } + if (spec.target === "reference" && run.frame_receipt == null) { + throw new Error("reference frame receipt is missing"); + } + run.summary = summarizeSamples(run.samples, spec.target); + run.status = "ok"; + } catch (error) { + run.status = "failed"; + run.error = errorMessage(error); + throw error; + } finally { + referenceCdp?.close(); + await terminateProcessTree(child, spec.target); + run.terminated_at = new Date().toISOString(); + if (activeRun?.child.pid === child.pid) activeRun = null; + } +} + +async function captureProcessTree(rootPid: number): Promise { + const clockMs = performance.now(); + const capturedAt = new Date().toISOString(); + const table = await readProcessTable(); + const processes = selectProcessTree(table, rootPid); + if (processes.length === 0) { + throw new Error(`benchmark process ${rootPid} is no longer present`); + } + return { + clock_ms: clockMs, + snapshot: { + captured_at: capturedAt, + root_pid: rootPid, + cumulative_cpu_time_seconds: round( + processes.reduce( + (total, process) => total + process.cpu_time_seconds, + 0, + ), + ), + ps_cpu_percent_sum_diagnostic: round( + processes.reduce( + (total, process) => total + process.ps_cpu_percent, + 0, + ), + ), + rss_kib: processes.reduce( + (total, process) => total + process.rss_kib, + 0, + ), + process_count: processes.length, + processes, + }, + }; +} + +function buildIntervalSample( + index: number, + samplingStart: number, + previous: TimedTreeSnapshot, + current: TimedTreeSnapshot, + health: HealthCapture, +): ResourceSample { + const intervalSeconds = (current.clock_ms - previous.clock_ms) / 1_000; + if (!(intervalSeconds > 0)) { + throw new Error(`sample ${index} has a non-positive interval`); + } + + const previousByPid = new Map( + previous.snapshot.processes.map((process) => [process.pid, process]), + ); + const currentPids = new Set( + current.snapshot.processes.map((process) => process.pid), + ); + const processes: IntervalProcessRow[] = current.snapshot.processes.map( + (process) => { + const before = previousByPid.get(process.pid); + const continued = + before != null && + before.ppid === process.ppid && + before.command === process.command && + process.cpu_time_seconds >= before.cpu_time_seconds; + const cpuTime = continued + ? process.cpu_time_seconds - before.cpu_time_seconds + : process.cpu_time_seconds; + return { + ...process, + interval_cpu_time_seconds: round(cpuTime), + interval_cpu_percent: round((cpuTime / intervalSeconds) * 100), + interval_identity: continued ? "continued" : "new-or-restarted", + }; + }, + ); + const intervalCpuTime = processes.reduce( + (total, process) => total + process.interval_cpu_time_seconds, + 0, + ); + const disappearedProcesses = previous.snapshot.processes.filter( + (process) => !currentPids.has(process.pid), + ); + if (disappearedProcesses.length > 0) { + console.error( + `bench-persona: sample ${index} cannot observe post-snapshot CPU for ` + + `${disappearedProcesses.length} exited process(es)`, + ); + } + + return { + index, + interval_started_at: previous.snapshot.captured_at, + captured_at: current.snapshot.captured_at, + elapsed_since_baseline_seconds: round( + (current.clock_ms - samplingStart) / 1_000, + ), + interval_seconds: round(intervalSeconds), + root_pid: current.snapshot.root_pid, + interval_cpu_time_seconds: round(intervalCpuTime), + interval_cpu_percent: round((intervalCpuTime / intervalSeconds) * 100), + cumulative_cpu_time_seconds: + current.snapshot.cumulative_cpu_time_seconds, + ps_cpu_percent_sum_diagnostic: + current.snapshot.ps_cpu_percent_sum_diagnostic, + rss_kib: current.snapshot.rss_kib, + process_count: current.snapshot.process_count, + processes, + disappeared_processes: disappearedProcesses, + health, + }; +} + +async function readProcessTable(): Promise { + const ps = Bun.spawn( + [ + "ps", + "-axo", + "pid=,ppid=,state=,time=,%cpu=,rss=,command=", + "-ww", + ], + { + env: { + ...process.env, + LC_ALL: "C", + LANG: "C", + }, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }, + ); + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(ps.stdout).text(), + new Response(ps.stderr).text(), + ps.exited, + ]); + if (exitCode !== 0) { + throw new Error(`ps failed (${exitCode}): ${stderr.trim() || "no stderr"}`); + } + + const rows: ProcessRow[] = []; + for (const line of stdout.split("\n")) { + const match = + /^\s*(\d+)\s+(\d+)\s+(\S+)\s+(\S+)\s+([0-9]+(?:\.[0-9]+)?)\s+(\d+)\s*(.*)$/.exec( + line, + ); + if (!match) continue; + const cpuTimeSeconds = parseCpuTimeSeconds(match[4]); + rows.push({ + pid: Number(match[1]), + ppid: Number(match[2]), + state: match[3], + cpu_time: match[4], + cpu_time_seconds: cpuTimeSeconds, + ps_cpu_percent: Number(match[5]), + rss_kib: Number(match[6]), + command: match[7], + }); + } + if (rows.length === 0) throw new Error("ps returned no parseable processes"); + return rows; +} + +export function parseCpuTimeSeconds(value: string): number { + const dayParts = value.split("-"); + if (dayParts.length > 2) throw new Error(`invalid ps CPU time ${value}`); + const days = dayParts.length === 2 ? Number(dayParts[0]) : 0; + const clock = dayParts.at(-1) ?? ""; + const parts = clock.split(":").map(Number); + if ( + !Number.isInteger(days) || + days < 0 || + (parts.length !== 2 && parts.length !== 3) || + parts.some((part) => !Number.isFinite(part) || part < 0) + ) { + throw new Error(`invalid ps CPU time ${value}`); + } + + let hours = 0; + let minutes: number; + let seconds: number; + if (parts.length === 3) { + [hours, minutes, seconds] = parts; + if (minutes >= 60) throw new Error(`invalid ps CPU time ${value}`); + } else { + [minutes, seconds] = parts; + } + if (seconds >= 60) throw new Error(`invalid ps CPU time ${value}`); + return days * 86_400 + hours * 3_600 + minutes * 60 + seconds; +} + +function selectProcessTree(table: ProcessRow[], rootPid: number): ProcessRow[] { + const byPid = new Map(table.map((row) => [row.pid, row])); + const root = byPid.get(rootPid); + if (!root || root.state.startsWith("Z")) return []; + + const children = new Map(); + for (const row of table) { + const siblings = children.get(row.ppid); + if (siblings) siblings.push(row); + else children.set(row.ppid, [row]); + } + for (const siblings of children.values()) { + siblings.sort((left, right) => left.pid - right.pid); + } + + const result: ProcessRow[] = []; + const seen = new Set(); + const visit = (pid: number): void => { + if (seen.has(pid)) return; + seen.add(pid); + const row = byPid.get(pid); + if (!row) return; + if (!row.state.startsWith("Z")) result.push(row); + for (const child of children.get(pid) ?? []) visit(child.pid); + }; + visit(rootPid); + return result; +} + +async function delayWhileRunning( + child: SpawnedProcess, + milliseconds: number, + context: string, +): Promise { + if (milliseconds <= 0) { + assertProcessAlive(child.pid, context); + return; + } + const outcome = await Promise.race([ + Bun.sleep(milliseconds).then(() => ({ kind: "elapsed" as const })), + child.exited.then((exitCode) => ({ + kind: "exit" as const, + exitCode, + })), + ]); + if (outcome.kind === "exit") { + throw new Error(`${context} (exit ${outcome.exitCode})`); + } + assertProcessAlive(child.pid, context); +} + +function assertProcessAlive(pid: number, context: string): void { + if (!isProcessAlive(pid)) throw new Error(context); +} + +export async function terminateProcessTree( + child: SpawnedProcess, + target: TargetName, +): Promise { + const existing = terminationPromises.get(child.pid); + if (existing) return existing; + + const termination = terminateProcessTreeInner(child, target); + terminationPromises.set(child.pid, termination); + try { + await termination; + } finally { + terminationPromises.delete(child.pid); + } +} + +async function terminateProcessTreeInner( + child: SpawnedProcess, + target: TargetName, +): Promise { + const rootPid = child.pid; + if (rootPid <= 1 || rootPid === process.pid) { + throw new Error(`refusing to terminate unsafe pid ${rootPid}`); + } + + let knownTree: ProcessRow[] = []; + try { + knownTree = selectProcessTree(await readProcessTable(), rootPid); + } catch (error) { + console.error( + `bench-persona: unable to snapshot ${target} tree: ${errorMessage(error)}`, + ); + } + const knownPids = new Set([rootPid, ...knownTree.map((row) => row.pid)]); + + signalProcessGroup(rootPid, "SIGTERM"); + signalPids([...knownPids].reverse(), "SIGTERM"); + + if (await waitUntilStopped(rootPid, knownPids, TERM_GRACE_MS)) { + await reapChild(child); + return; + } + + console.error(`bench-persona: ${target} did not exit after SIGTERM; sending SIGKILL`); + try { + for (const row of selectProcessTree(await readProcessTable(), rootPid)) { + knownPids.add(row.pid); + } + } catch { + // The root may already be gone while a previously observed helper remains. + } + signalProcessGroup(rootPid, "SIGKILL"); + signalPids([...knownPids].reverse(), "SIGKILL"); + + const stopped = await waitUntilStopped(rootPid, knownPids, KILL_GRACE_MS); + await reapChild(child); + if (!stopped) { + const survivors = [...knownPids].filter(isProcessAlive); + console.error( + `bench-persona: warning: ${target} survivors after SIGKILL: ` + + (survivors.length ? survivors.join(", ") : `process group ${rootPid}`), + ); + } +} + +function signalProcessGroup( + rootPid: number, + signal: NodeJS.Signals, +): void { + try { + process.kill(-rootPid, signal); + } catch (error) { + if (!isMissingProcess(error)) { + console.error( + `bench-persona: unable to signal process group ${rootPid}: ${errorMessage(error)}`, + ); + } + } +} + +function signalPids(pids: number[], signal: NodeJS.Signals): void { + for (const pid of pids) { + if (pid <= 1 || pid === process.pid) continue; + try { + process.kill(pid, signal); + } catch (error) { + if (!isMissingProcess(error)) { + console.error( + `bench-persona: unable to signal pid ${pid}: ${errorMessage(error)}`, + ); + } + } + } +} + +async function waitUntilStopped( + processGroup: number, + pids: Set, + timeoutMs: number, +): Promise { + const deadline = performance.now() + timeoutMs; + while (performance.now() < deadline) { + if ( + !isProcessGroupAlive(processGroup) && + ![...pids].some(isProcessAlive) + ) { + return true; + } + await Bun.sleep(POLL_MS); + } + return ( + !isProcessGroupAlive(processGroup) && ![...pids].some(isProcessAlive) + ); +} + +async function reapChild(child: SpawnedProcess): Promise { + await Promise.race([ + child.exited.then(() => undefined), + Bun.sleep(POLL_MS * 5), + ]); +} + +function isProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return !isMissingProcess(error); + } +} + +function isProcessGroupAlive(processGroup: number): boolean { + try { + process.kill(-processGroup, 0); + return true; + } catch (error) { + return !isMissingProcess(error); + } +} + +function isMissingProcess(error: unknown): boolean { + return ( + typeof error === "object" && + error !== null && + "code" in error && + error.code === "ESRCH" + ); +} + +function summarizeSamples( + samples: ResourceSample[], + target: TargetName, +): RunSummary { + if (samples.length === 0) throw new Error("cannot summarize zero samples"); + return { + interval_cpu_percent: summarize( + samples.map((sample) => sample.interval_cpu_percent), + ), + rss_kib: summarize(samples.map((sample) => sample.rss_kib)), + process_count: summarize(samples.map((sample) => sample.process_count)), + render_fps: + target === "pocket" + ? summarize( + samples.map((sample) => + validatePocketHealthBody(sample.health.body), + ), + ) + : null, + }; +} + +function summarize(values: number[]): SummaryStats { + const sorted = [...values].sort((left, right) => left - right); + return { + n: sorted.length, + mean: round(values.reduce((sum, value) => sum + value, 0) / values.length), + min: sorted[0], + median: round(quantile(sorted, 0.5)), + p95: round(quantile(sorted, 0.95)), + max: sorted[sorted.length - 1], + }; +} + +function quantile(sorted: number[], percentile: number): number { + if (sorted.length === 1) return sorted[0]; + const position = (sorted.length - 1) * percentile; + const lower = Math.floor(position); + const upper = Math.ceil(position); + const fraction = position - lower; + return sorted[lower] * (1 - fraction) + sorted[upper] * fraction; +} + +function compareRuns(referenceRun: RunResult, pocketRun: RunResult) { + const reference = referenceRun.summary; + const pocket = pocketRun.summary; + const frameReceipt = referenceRun.frame_receipt; + if ( + reference == null || + pocket == null || + frameReceipt == null || + pocket.render_fps == null + ) { + throw new Error("cannot compare incomplete frame-aware benchmark runs"); + } + const referenceFps = frameReceipt.frame_sample.fps; + const pocketFps = pocket.render_fps.median; + const referenceCpuPerFrame = + reference.interval_cpu_percent.median / referenceFps; + const pocketCpuPerFrame = + pocket.interval_cpu_percent.median / pocketFps; + return { + lower_is_better: true, + reduction_percent_formula: + "(reference - pocket) / reference * 100", + ratio_formula: "reference / pocket", + interval_cpu_percent: compareStats( + reference.interval_cpu_percent, + pocket.interval_cpu_percent, + ), + rss_kib: compareStats(reference.rss_kib, pocket.rss_kib), + process_count: compareStats(reference.process_count, pocket.process_count), + observed_fps: { + reference: round(referenceFps), + pocket: round(pocketFps), + reference_source: "CDP requestAnimationFrame", + pocket_source: "median health renderFps", + }, + cpu_per_delivered_frame: { + formula: "median interval CPU percent / observed fps", + ...compareValue(referenceCpuPerFrame, pocketCpuPerFrame), + }, + }; +} + +function compareStats(reference: SummaryStats, pocket: SummaryStats) { + return { + median: compareValue(reference.median, pocket.median), + p95: compareValue(reference.p95, pocket.p95), + }; +} + +function compareValue(reference: number, pocket: number) { + return { + reference, + pocket, + reduction_percent: + reference === 0 ? null : round(((reference - pocket) / reference) * 100), + ratio_reference_over_pocket: + pocket === 0 ? null : round(reference / pocket), + }; +} + +function parseArgs(argv: string[]): Options { + if (argv.includes("--help") || argv.includes("-h")) { + console.log(usage()); + process.exit(0); + } + + const names = new Set([ + "--reference-bin", + "--reference-root", + "--pocket-bin", + "--library", + "--bundle", + "--max-fps", + "--max-texture-dim", + "--settle", + "--samples", + "--interval", + "--out", + ]); + const values = new Map(); + + for (let index = 0; index < argv.length; index++) { + const argument = argv[index]; + const equals = argument.indexOf("="); + const name = equals >= 0 ? argument.slice(0, equals) : argument; + if (!names.has(name)) throw new Error(`unknown argument ${argument}`); + if (values.has(name)) throw new Error(`${name} may only be specified once`); + + let value: string; + if (equals >= 0) { + value = argument.slice(equals + 1); + } else { + const next = argv[index + 1]; + if (next == null) throw new Error(`${name} requires a value`); + value = next; + index++; + } + if (value.length === 0) throw new Error(`${name} requires a value`); + values.set(name, value); + } + + const required = (name: string): string => { + const value = values.get(name); + if (value == null) throw new Error(`missing required ${name}`); + return value; + }; + const numberValue = ( + name: string, + fallback: number, + minimum: number, + ): number => { + const raw = values.get(name); + const value = raw == null ? fallback : Number(raw); + if (!Number.isFinite(value) || value < minimum) { + throw new Error(`${name} must be a number >= ${minimum}`); + } + return value; + }; + + const sampleCount = numberValue("--samples", DEFAULT_SAMPLE_COUNT, 1); + if (!Number.isInteger(sampleCount)) { + throw new Error("--samples must be an integer"); + } + + const maxFpsRaw = values.get("--max-fps"); + const maxFps = maxFpsRaw == null ? null : Number(maxFpsRaw); + if (maxFps != null && (!Number.isFinite(maxFps) || maxFps <= 0)) { + throw new Error("--max-fps must be a number > 0"); + } + + const maxTextureDimRaw = values.get("--max-texture-dim"); + const maxTextureDim = + maxTextureDimRaw == null ? null : Number(maxTextureDimRaw); + if ( + maxTextureDim != null && + (!Number.isInteger(maxTextureDim) || maxTextureDim <= 0) + ) { + throw new Error("--max-texture-dim must be an integer > 0"); + } + + const intervalSeconds = numberValue( + "--interval", + DEFAULT_INTERVAL_SECONDS, + 0, + ); + if (intervalSeconds <= 0) { + throw new Error("--interval must be a number > 0"); + } + + const stamp = new Date().toISOString().replace(/[:.]/g, "-"); + const outRaw = + values.get("--out") ?? + join(REPO_ROOT, "dist", "bench", `persona-ab-${stamp}.json`); + return { + referenceBin: executablePath(required("--reference-bin")), + referenceRoot: absolutePath(required("--reference-root")), + pocketBin: executablePath(required("--pocket-bin")), + library: absolutePath(required("--library")), + bundle: absolutePath(required("--bundle")), + maxFps, + maxTextureDim, + settleSeconds: numberValue( + "--settle", + DEFAULT_SETTLE_SECONDS, + 0, + ), + sampleCount, + intervalSeconds, + outPath: outRaw === "-" ? "-" : absolutePath(outRaw), + }; +} + +function validateInputs(options: Options): void { + if (platform() === "win32") { + throw new Error("bench-persona requires a POSIX ps implementation"); + } + assertDirectory(options.referenceRoot, "--reference-root"); + assertExists(options.library, "--library"); + assertExists(options.bundle, "--bundle"); + assertExecutablePath(options.referenceBin, "--reference-bin"); + assertExecutablePath(options.pocketBin, "--pocket-bin"); + const referenceLibrary = join( + options.referenceRoot, + "public", + "assets", + "library.json", + ); + assertExists(referenceLibrary, "Persona reference library"); + if (realpathSync(options.library) !== realpathSync(referenceLibrary)) { + throw new Error( + "--library must be the reference checkout's public/assets/library.json " + + "so both targets consume the same catalog", + ); + } +} + +function assertDirectory(path: string, name: string): void { + if (!existsSync(path) || !statSync(path).isDirectory()) { + throw new Error(`${name} is not a directory: ${path}`); + } +} + +function assertExists(path: string, name: string): void { + if (!existsSync(path)) throw new Error(`${name} does not exist: ${path}`); +} + +function assertExecutablePath(value: string, name: string): void { + if (!value.includes("/")) return; + if (!existsSync(value) || !statSync(value).isFile()) { + throw new Error(`${name} is not a file: ${value}`); + } +} + +function executablePath(value: string): string { + const expanded = expandHome(value); + return expanded.includes("/") ? resolve(expanded) : expanded; +} + +function absolutePath(value: string): string { + return resolve(expandHome(value)); +} + +function expandHome(value: string): string { + if (value === "~") return homedir(); + if (value.startsWith("~/")) return join(homedir(), value.slice(2)); + return value; +} + +function prepareOutput(outPath: string): void { + if (outPath !== "-") mkdirSync(dirname(outPath), { recursive: true }); +} + +function writeReport(outPath: string, report: Report): void { + const json = `${JSON.stringify(report, null, 2)}\n`; + if (outPath === "-") { + process.stdout.write(json); + return; + } + const temporary = `${outPath}.tmp-${process.pid}`; + writeFileSync(temporary, json); + renameSync(temporary, outPath); +} + +function finishReportClock(report: Report, startedClock: number): void { + report.completed_at = new Date().toISOString(); + report.elapsed_seconds = round((performance.now() - startedClock) / 1_000); +} + +function machineFacts() { + const cpuList = cpus(); + return { + platform: platform(), + release: release(), + arch: arch(), + cpu_model: cpuList[0]?.model ?? "unknown", + logical_cpu_count: cpuList.length, + total_memory_bytes: totalmem(), + bun_version: Bun.version, + time_zone: Intl.DateTimeFormat().resolvedOptions().timeZone, + utc_offset_minutes: -new Date().getTimezoneOffset(), + }; +} + +function formatMiB(kib: number): string { + return `${(kib / 1_024).toFixed(1)}MiB`; +} + +function round(value: number): number { + return Math.round(value * 1_000_000) / 1_000_000; +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function installSignalHandlers(): void { + if (signalHandlersInstalled) return; + signalHandlersInstalled = true; + const handle = (signal: "SIGHUP" | "SIGINT" | "SIGTERM", exitCode: number) => { + if (signalShutdown) return; + signalShutdown = (async () => { + console.error(`bench-persona: received ${signal}; cleaning up`); + if (activeRun) { + await terminateProcessTree(activeRun.child, activeRun.target); + activeRun = null; + } + try { + persistInterruptedReport?.(signal); + } catch (error) { + console.error( + `bench-persona: unable to persist interrupted report: ${errorMessage(error)}`, + ); + } + process.exit(exitCode); + })(); + }; + process.on("SIGHUP", () => handle("SIGHUP", 129)); + process.on("SIGINT", () => handle("SIGINT", 130)); + process.on("SIGTERM", () => handle("SIGTERM", 143)); +} + +function usage(): string { + return `usage: bun scripts/bench-persona.ts \\ + --reference-bin PATH --reference-root PATH \\ + --pocket-bin PATH --library PATH --bundle PATH \\ + [--max-fps FPS] [--max-texture-dim PIXELS] \\ + [--settle 15] [--samples 13] [--interval 5] [--out PATH] + +Runs the Electron reference first, terminates its full process tree, then runs +the Pocket binary. Each target gets an isolated loopback bridge port and must +return {ok:true} from /health within ${READINESS_TIMEOUT_SECONDS}s. Only then +does it settle, capture a cumulative CPU-time baseline, and record resource plus +health samples after each --interval. Reference also requires a non-settings CDP +page, validated 430x680 DPR 1.5 WebGL canvas, and a whole-window rAF receipt; +Pocket health must report a configured, visible model and renderFps > 1. Use +--out - for JSON on stdout.`; +} diff --git a/tests/persona-bench.test.ts b/tests/persona-bench.test.ts new file mode 100644 index 0000000..0fc03cc --- /dev/null +++ b/tests/persona-bench.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, test } from "bun:test"; + +import { + parseCpuTimeSeconds, + parseReferenceCanvasReceipt, + parseReferenceFrameSample, + validatePocketHealthBody, +} from "../scripts/bench-persona"; + +describe("Persona benchmark receipts", () => { + test("accepts Pocket's nested health status", () => { + expect( + validatePocketHealthBody({ + ok: true, + status: { + modelConfigured: true, + windowVisible: true, + renderFps: 59.8, + }, + }), + ).toBe(59.8); + expect(() => + validatePocketHealthBody({ + modelConfigured: true, + windowVisible: true, + renderFps: 59.8, + }), + ).toThrow("status must be an object"); + }); + + test("requires the reference viewport, backing size, and WebGL context", () => { + expect( + parseReferenceCanvasReceipt({ + ready_state: "complete", + viewport: [430, 680, 1.5], + canvas: { + client: [430, 680], + backing: [645, 1020], + }, + webgl: { + version: "WebGL 2.0", + renderer: "ANGLE Metal", + vendor: "Apple", + }, + }), + ).toEqual({ + ready_state: "complete", + viewport: [430, 680, 1.5], + canvas: { + client: [430, 680], + backing: [645, 1020], + }, + webgl: { + version: "WebGL 2.0", + renderer: "ANGLE Metal", + vendor: "Apple", + }, + }); + expect(() => + parseReferenceCanvasReceipt({ + ready_state: "complete", + viewport: [430, 680, 1.5], + canvas: { + client: [430, 680], + backing: [860, 1360], + }, + webgl: { + version: "WebGL 2.0", + renderer: "ANGLE Metal", + vendor: "Apple", + }, + }), + ).toThrow("unexpected canvas backing"); + }); + + test("validates a full-window reference frame receipt", () => { + const receipt = { + requested_duration_ms: 1_000, + elapsed_ms: 1_001, + frames: 120, + fps: 119.88, + mean_interval_ms: 8.34, + p50_interval_ms: 8.3, + p95_interval_ms: 9.6, + p99_interval_ms: 10.2, + max_interval_ms: 10.4, + over_20ms: 0, + }; + expect(parseReferenceFrameSample(receipt, 1_000)).toEqual(receipt); + expect(() => + parseReferenceFrameSample({ ...receipt, elapsed_ms: 100 }, 1_000), + ).toThrow("failed validation"); + }); + + test("parses POSIX cumulative CPU time", () => { + expect(parseCpuTimeSeconds("02:03")).toBe(123); + expect(parseCpuTimeSeconds("1-02:03:04")).toBe(93_784); + expect(() => parseCpuTimeSeconds("not-a-time")).toThrow( + "invalid ps CPU time", + ); + }); +}); diff --git a/vendor/pocketjs b/vendor/pocketjs index 8acf9f4..581aa35 160000 --- a/vendor/pocketjs +++ b/vendor/pocketjs @@ -1 +1 @@ -Subproject commit 8acf9f45830363b02d33dbe8ed8205e040e33b31 +Subproject commit 581aa35c896dc129e1ddf4cb2f0133700d7a5506 From 3f225500650a33758a75c7d63ffdc6f650909485 Mon Sep 17 00:00:00 2001 From: "Yifeng \"Evan\" Wang" <7312949+doodlewind@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:44:57 +0800 Subject: [PATCH 2/2] feat(persona): align latest upstream at 30 fps --- README.md | 14 +- crates/pocket-persona/src/bridge.rs | 9 +- crates/pocket-persona/src/main.rs | 31 ++- crates/pocket-persona/src/widget.rs | 311 ++++++++++++++++++++++---- docs/PERSONA.md | 325 +++++++++++++++++----------- fixtures/persona/library.json | 14 +- package.json | 1 + scripts/accept-persona.ts | 247 ++++++++++++++++----- scripts/bench-persona.ts | 197 ++++++++++++++++- tests/persona-bench.test.ts | 55 +++++ vendor/pocketjs | 2 +- 11 files changed, 959 insertions(+), 247 deletions(-) diff --git a/README.md b/README.md index bfebb74..a5c606f 100644 --- a/README.md +++ b/README.md @@ -24,16 +24,22 @@ bun run accept:persona bun run accept:pocket ``` -Press Ctrl-C to terminate the complete target process tree. For the sequential -resource comparison: +Visual acceptance includes the whole pose, not only a running window: compare +the shoulders, wrists/hands, hips, knees, and ankles/feet through idle and +speaking, and reject any persistent rest-axis twist. Press Ctrl-C to terminate +the complete target process tree. For the sequential resource comparison: ```sh bun run bench:persona +bun run bench:persona:speaking bun run bench:persona:controlled ``` -See [docs/PERSONA.md](docs/PERSONA.md) for the parity boundary, benchmark -methodology, measurements, and asset-license constraints. +The production commands compare stock Persona at its display-driven rate with +Pocket at its intended 30 fps / 2048 texture cap, in idle or sustained-speaking +state. See [docs/PERSONA.md](docs/PERSONA.md) for the latest upstream pin, +parity boundary, benchmark methodology, measurements, and asset-license +constraints. ## What it does diff --git a/crates/pocket-persona/src/bridge.rs b/crates/pocket-persona/src/bridge.rs index 2ab8d1d..70db657 100644 --- a/crates/pocket-persona/src/bridge.rs +++ b/crates/pocket-persona/src/bridge.rs @@ -79,6 +79,8 @@ pub struct StatusSnapshot { pub active_animation: String, #[serde(rename = "renderFps")] pub render_fps: f32, + #[serde(rename = "renderFrameCount")] + pub render_frame_count: u64, #[serde(rename = "frameTimeP95Ms")] pub frame_time_p95_ms: f32, #[serde(rename = "frameTimeP99Ms")] @@ -96,6 +98,7 @@ impl StatusSnapshot { audio_level: 0.0, active_animation: "idle".into(), render_fps: 0.0, + render_frame_count: 0, frame_time_p95_ms: 0.0, frame_time_p99_ms: 0.0, frame_time_max_ms: 0.0, @@ -134,7 +137,7 @@ impl Bridge { } } Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { - std::thread::sleep(Duration::from_millis(10)); + std::thread::sleep(Duration::from_millis(50)); } Err(error) => { log::warn!("Pocket Persona bridge accept: {error}"); @@ -209,6 +212,10 @@ fn handle_connection( status: &Arc>, sender: &mpsc::SyncSender, ) -> Result<()> { + // Accepted sockets can inherit O_NONBLOCK from the listener on macOS. + // The request parser expects a bounded blocking read, so clear it before + // installing timeouts instead of surfacing a transient EAGAIN as HTTP 502. + stream.set_nonblocking(false)?; stream.set_read_timeout(Some(Duration::from_secs(2)))?; stream.set_write_timeout(Some(Duration::from_secs(2)))?; let request = read_request(&mut stream)?; diff --git a/crates/pocket-persona/src/main.rs b/crates/pocket-persona/src/main.rs index 2fe274a..c4faa23 100644 --- a/crates/pocket-persona/src/main.rs +++ b/crates/pocket-persona/src/main.rs @@ -29,6 +29,7 @@ struct Args { size: (u32, u32), max_texture_dim: u32, headless_shot: Option, + headless_speaking: bool, ticks: u32, } @@ -48,7 +49,14 @@ fn main() -> Result<()> { max_texture_dim: args.max_texture_dim, }); if let Some(output) = args.headless_shot { - return headless_shot(widget, args.size, args.fps, args.ticks, &output); + return headless_shot( + widget, + args.size, + args.fps, + args.ticks, + args.headless_speaking, + &output, + ); } pocket_widget::run( WidgetConfig { @@ -71,10 +79,11 @@ fn parse_args(values: Vec) -> Result { let mut library = None; let mut bundle = default_repo_root().join("dist/pocket-persona/guest.js"); let mut bridge_port = Some(47_831); - let mut fps = 60.0; + let mut fps = 30.0; let mut size = DEFAULT_SIZE; let mut max_texture_dim = 2048; let mut headless_shot = None; + let mut headless_speaking = false; let mut ticks = 90; let mut index = 0; while index < values.len() { @@ -95,6 +104,7 @@ fn parse_args(values: Vec) -> Result { "--size" => size = parse_size(next(&mut index)?)?, "--max-texture-dim" => max_texture_dim = next(&mut index)?.parse()?, "--headless-shot" => headless_shot = Some(PathBuf::from(next(&mut index)?)), + "--headless-speaking" => headless_speaking = true, "--ticks" => ticks = next(&mut index)?.parse()?, "--help" | "-h" => { println!( @@ -106,10 +116,11 @@ fn parse_args(values: Vec) -> Result { \t--bundle Pocket policy bundle\n\ \t--bridge-port Persona HTTP/MCP port (default 47831; 0 = any)\n\ \t--no-bridge Disable HTTP/MCP\n\ - \t--fps Fixed update/render cap (default 60)\n\ + \t--fps Fixed update/render cap (default 30)\n\ \t--size x Logical window size (default 430x680)\n\ \t--max-texture-dim Texture cap (default 2048)\n\ \t--headless-shot Render one offscreen verification frame\n\ + \t--headless-speaking Start the offscreen receipt in speaking state\n\ \t--ticks Headless fixed steps (default 90)" ); std::process::exit(0); @@ -124,6 +135,9 @@ fn parse_args(values: Vec) -> Result { if !(256..=4096).contains(&max_texture_dim) { bail!("--max-texture-dim must be between 256 and 4096"); } + if headless_speaking && headless_shot.is_none() { + bail!("--headless-speaking requires --headless-shot"); + } let library = library.context("--library is required")?; Ok(Args { library, @@ -133,6 +147,7 @@ fn parse_args(values: Vec) -> Result { size, max_texture_dim, headless_shot, + headless_speaking, ticks, }) } @@ -157,11 +172,15 @@ fn headless_shot( size: (u32, u32), fps: f32, ticks: u32, + speaking: bool, output: &Path, ) -> Result<()> { let gpu = Gpu::new_headless()?; let mut renderer = Renderer::new(&gpu, pocket3d::gpu::OFFSCREEN_FORMAT)?; widget.init(&gpu, &mut renderer)?; + if speaking { + widget.set_headless_speaking(); + } let input = Input::default(); for _ in 0..ticks { widget.tick(1.0 / fps, &input, size)?; @@ -185,4 +204,10 @@ mod tests { assert!(parse_size("430").is_err()); assert!(parse_size("20x20").is_err()); } + + #[test] + fn defaults_to_thirty_fps() { + let args = parse_args(vec!["--library".into(), "library.json".into()]).unwrap(); + assert_eq!(args.fps, 30.0); + } } diff --git a/crates/pocket-persona/src/widget.rs b/crates/pocket-persona/src/widget.rs index b9e8cfd..68a0d44 100644 --- a/crates/pocket-persona/src/widget.rs +++ b/crates/pocket-persona/src/widget.rs @@ -23,7 +23,12 @@ use crate::catalog::{ActionRole, Catalog}; use crate::guest::{GuestCommand, GuestEvent, GuestState, PersonaGuest}; use crate::sim::{FaceSim, Pcg32}; -const VOICE_IDLE_DELAY: f32 = 0.65; +const VOICE_IDLE_DELAY: f32 = 0.9; +const BODY_TRANSITION_SECONDS: f32 = 0.35; +const SPEAKING_CHUNK_HALF_BASE_SECONDS: f32 = 0.45; +const SPEAKING_CHUNK_FACTOR_MIN: f32 = 1.5; +const SPEAKING_CHUNK_FACTOR_MAX: f32 = 1.8; +const SPEAKING_RESUME_HOLD_SECONDS: f32 = 0.7; const ORBIT_RADIANS_PER_PIXEL: f32 = 0.006; const PAN_UNITS_PER_PIXEL: f32 = 0.0015; @@ -47,10 +52,12 @@ struct Playback { clip: Option, time: f32, one_shot: bool, + next_chunk_at: Option, } struct RenderRate { frames: u32, + total_frames: u64, window_start: Instant, last_frame: Option, intervals_ms: Vec, @@ -64,6 +71,7 @@ impl RenderRate { fn new() -> Self { Self { frames: 0, + total_frames: 0, window_start: Instant::now(), last_frame: None, intervals_ms: Vec::with_capacity(256), @@ -82,6 +90,7 @@ impl RenderRate { } self.last_frame = Some(now); self.frames += 1; + self.total_frames += 1; let elapsed = self.window_start.elapsed().as_secs_f32(); if elapsed >= 1.0 { self.fps = self.frames as f32 / elapsed; @@ -124,7 +133,9 @@ pub struct PersonaWidget { last_cursor: Option, last_window_size: (u32, u32), voice: VoiceState, + automatic_role: ActionRole, audio_level: f32, + has_observed_audio_level: bool, voice_idle_delay: Option, pending_events: Vec<(String, String)>, window_command: Option, @@ -165,7 +176,9 @@ impl PersonaWidget { last_cursor: None, last_window_size, voice: VoiceState::default(), + automatic_role: ActionRole::Idle, audio_level: 0.0, + has_observed_audio_level: false, voice_idle_delay: None, pending_events: Vec::new(), window_command: None, @@ -176,6 +189,24 @@ impl PersonaWidget { } } + /// Deterministic offscreen receipt hook. Windowed/benchmark runs still + /// enter this state through the same Persona-compatible event bridge. + pub(crate) fn set_headless_speaking(&mut self) { + self.voice = VoiceState { + phase: "active".into(), + activity: "speaking".into(), + microphone_muted: false, + output_muted: false, + }; + self.automatic_role = ActionRole::Speaking; + self.audio_level = 0.35; + self.has_observed_audio_level = true; + self.voice_idle_delay = None; + if let Some(index) = self.automatic_action_index() { + self.start_action_index(index, false); + } + } + fn active_action_name(&self) -> &str { self.playback .and_then(|playback| self.actions.get(playback.action)) @@ -191,12 +222,8 @@ impl PersonaWidget { self.actions.iter().position(|action| action.name == name) } - fn voice_action_index(&self) -> Option { - self.action_index_for_role(if self.voice.speaking() { - ActionRole::Speaking - } else { - ActionRole::Idle - }) + fn automatic_action_index(&self) -> Option { + self.action_index_for_role(self.automatic_role) } fn choose_clip(&mut self, action_index: usize) -> Option { @@ -213,19 +240,39 @@ impl PersonaWidget { Some(index) } - fn transition_duration(&self, next: usize) -> f32 { - let Some(previous) = self.playback else { - return 0.0; - }; - let previous_role = self.actions[previous.action].role; - let next_role = self.actions[next].role; - if previous_role == ActionRole::Speaking && next_role == ActionRole::Idle { - 1.15 - } else if next_role == ActionRole::Speaking { - 0.85 + fn transition_duration(&self, _next: usize) -> f32 { + if self.playback.is_some() { + BODY_TRANSITION_SECONDS } else { - 0.7 + 0.0 + } + } + + fn speaking_transition_duration(&mut self) -> f32 { + SPEAKING_CHUNK_HALF_BASE_SECONDS + * (self + .rng + .range(SPEAKING_CHUNK_FACTOR_MIN, SPEAKING_CHUNK_FACTOR_MAX) + + self + .rng + .range(SPEAKING_CHUNK_FACTOR_MIN, SPEAKING_CHUNK_FACTOR_MAX)) + } + + fn speaking_chunk_schedule( + &mut self, + action_index: usize, + clip_index: Option, + one_shot: bool, + ) -> (Option, f32) { + let action = &self.actions[action_index]; + if one_shot || action.role != ActionRole::Speaking || action.clips.len() <= 1 { + return (None, 0.0); } + let clip_duration = clip_index + .and_then(|index| action.clips.get(index)) + .map_or(0.0, |clip| clip.duration); + let fade = self.speaking_transition_duration(); + (Some(speaking_chunk_dwell(clip_duration, fade)), fade) } fn start_action_index(&mut self, action_index: usize, one_shot: bool) { @@ -235,6 +282,7 @@ impl PersonaWidget { let old_name = self.active_action_name().to_string(); let duration = self.transition_duration(action_index); let clip = self.choose_clip(action_index); + let (next_chunk_at, _) = self.speaking_chunk_schedule(action_index, clip, one_shot); self.fade_from = (duration > 0.0 && !self.locals.is_empty()).then(|| self.locals.clone()); self.fade_elapsed = 0.0; self.fade_duration = duration; @@ -243,6 +291,7 @@ impl PersonaWidget { clip, time: 0.0, one_shot, + next_chunk_at, }); let new_name = self.actions[action_index].name.clone(); if old_name != new_name { @@ -266,12 +315,54 @@ impl PersonaWidget { true } - fn resume_voice_action(&mut self) { - if let Some(index) = self.voice_action_index() { + fn resume_automatic_action(&mut self) { + if let Some(index) = self.automatic_action_index() { self.start_action_index(index, false); } } + fn advance_speaking_chunk(&mut self, action_index: usize) { + let Some(clip) = self.choose_clip(action_index) else { + return; + }; + let (next_chunk_at, fade_duration) = + self.speaking_chunk_schedule(action_index, Some(clip), false); + self.fade_from = (!self.locals.is_empty()).then(|| self.locals.clone()); + self.fade_elapsed = 0.0; + self.fade_duration = fade_duration; + self.playback = Some(Playback { + action: action_index, + clip: Some(clip), + time: 0.0, + one_shot: false, + next_chunk_at, + }); + self.dirty = true; + } + + fn hold_speaking_chunk_after_resume(&mut self) { + let Some(playback) = self.playback.as_mut() else { + return; + }; + let modular_speaking = !playback.one_shot + && self.actions.get(playback.action).is_some_and(|action| { + action.role == ActionRole::Speaking && action.clips.len() > 1 + }); + if modular_speaking && let Some(deadline) = playback.next_chunk_at.as_mut() { + *deadline = speaking_resume_deadline(*deadline, playback.time); + } + } + + fn speaking_motion_active(&self) -> bool { + speaking_motion_active(&self.voice, self.has_observed_audio_level, self.audio_level) + } + + fn hold_if_speaking_motion_resumed(&mut self, was_active: bool) { + if !was_active && self.speaking_motion_active() { + self.hold_speaking_chunk_after_resume(); + } + } + fn apply_bridge_commands(&mut self) { let commands: Vec<_> = self .bridge @@ -282,31 +373,45 @@ impl PersonaWidget { match command { BridgeCommand::Voice(voice) => { let old_activity = self.voice.activity.clone(); + let motion_was_active = self.speaking_motion_active(); self.voice = voice; + if self.voice.phase == "inactive" { + self.has_observed_audio_level = false; + } if old_activity != self.voice.activity { self.pending_events .push(("voiceChanged".into(), self.voice.activity.clone())); } - if self.voice.speaking() { - self.voice_idle_delay = None; - if !self.playback.is_some_and(|playback| playback.one_shot) - && let Some(index) = self.action_index_for_role(ActionRole::Speaking) - && self - .playback - .is_none_or(|playback| playback.action != index) - { - self.start_action_index(index, false); + match immediate_automatic_role(&self.voice) { + Some(role) => { + self.automatic_role = role; + self.voice_idle_delay = None; + if role == ActionRole::Idle { + self.audio_level = 0.0; + } + if !self.playback.is_some_and(|playback| playback.one_shot) + && let Some(index) = self.automatic_action_index() + && self + .playback + .is_none_or(|playback| playback.action != index) + { + self.start_action_index(index, false); + } } - } else if self.voice.phase == "active" && self.voice.activity == "listening" { - self.voice_idle_delay = Some(VOICE_IDLE_DELAY); - } else { - self.voice_idle_delay = None; - if !self.playback.is_some_and(|playback| playback.one_shot) { - self.resume_voice_action(); + None => { + // Active idle/listening keeps the current automatic + // body role until the sentence-gap timer expires. + self.voice_idle_delay = Some(VOICE_IDLE_DELAY); } } + self.hold_if_speaking_motion_resumed(motion_was_active); + } + BridgeCommand::AudioLevel(level) => { + let motion_was_active = self.speaking_motion_active(); + self.audio_level = level; + self.has_observed_audio_level = true; + self.hold_if_speaking_motion_resumed(motion_was_active); } - BridgeCommand::AudioLevel(level) => self.audio_level = level, BridgeCommand::PlayAnimation(name) => { if !self.start_action(&name, true) { log::warn!("Pocket Persona action is not playable: {name}"); @@ -347,6 +452,7 @@ impl PersonaWidget { *delay -= dt; if *delay <= 0.0 { self.voice_idle_delay = None; + self.automatic_role = ActionRole::Idle; if !self.playback.is_some_and(|playback| playback.one_shot) && let Some(index) = self.action_index_for_role(ActionRole::Idle) && self @@ -371,12 +477,21 @@ impl PersonaWidget { playback.time += dt; let action = &self.actions[playback.action]; let clip = playback.clip.and_then(|index| action.clips.get(index)); + let modular_speaking = + !playback.one_shot && action.role == ActionRole::Speaking && action.clips.len() > 1; + let sample_time = if modular_speaking { + clip.map_or(playback.time, |clip| { + ping_pong_sample_time(playback.time, clip.duration) + }) + } else { + playback.time + }; let clip_advanced = clip.is_some(); let fade_advanced = self.fade_from.is_some(); model.skeleton.sample_locals( clip, - playback.time, - !playback.one_shot, + sample_time, + !playback.one_shot && !modular_speaking, &mut self.sampled_locals, ); @@ -402,8 +517,13 @@ impl PersonaWidget { self.playback = Some(playback); if playback.one_shot && clip.is_none_or(|clip| playback.time >= clip.duration) { - self.playback = None; - self.resume_voice_action(); + self.resume_automatic_action(); + } else if self.speaking_motion_active() + && playback + .next_chunk_at + .is_some_and(|deadline| playback.time >= deadline) + { + self.advance_speaking_chunk(playback.action); } clip_advanced || fade_advanced } @@ -661,6 +781,7 @@ impl WidgetGame for PersonaWidget { status.audio_level = self.audio_level; status.active_animation = active_animation; status.render_fps = self.render_rate.fps; + status.render_frame_count = self.render_rate.total_frames; status.frame_time_p95_ms = self.render_rate.p95_ms; status.frame_time_p99_ms = self.render_rate.p99_ms; status.frame_time_max_ms = self.render_rate.max_ms; @@ -709,6 +830,45 @@ fn smoothstep01(value: f32) -> f32 { value * value * (3.0 - 2.0 * value) } +fn speaking_chunk_dwell(clip_duration: f32, transition_duration: f32) -> f32 { + clip_duration.max(transition_duration + 0.5) +} + +fn ping_pong_sample_time(elapsed: f32, duration: f32) -> f32 { + if duration <= 0.0 { + return 0.0; + } + let period = duration * 2.0; + let phase = elapsed.rem_euclid(period); + if phase <= duration { + phase + } else { + period - phase + } +} + +fn speaking_resume_deadline(next_transition_at: f32, playback_time: f32) -> f32 { + next_transition_at.max(playback_time + SPEAKING_RESUME_HOLD_SECONDS) +} + +fn speaking_motion_active( + voice: &VoiceState, + has_observed_audio_level: bool, + audio_level: f32, +) -> bool { + voice.speaking() && (!has_observed_audio_level || audio_level > 0.0) +} + +fn immediate_automatic_role(voice: &VoiceState) -> Option { + if voice.phase != "active" || voice.output_muted { + Some(ActionRole::Idle) + } else if voice.activity == "speaking" { + Some(ActionRole::Speaking) + } else { + None + } +} + fn percentile(sorted: &[f32], quantile: f32) -> f32 { if sorted.is_empty() { return 0.0; @@ -755,4 +915,75 @@ mod tests { assert_eq!(percentile(&[1.0, 2.0, 3.0, 4.0], 0.5), 3.0); assert_eq!(percentile(&[1.0, 2.0, 3.0, 4.0], 0.95), 4.0); } + + #[test] + fn speaking_chunk_dwell_preserves_gesture_and_blend() { + assert_eq!(speaking_chunk_dwell(3.0, 1.5), 3.0); + assert_eq!(speaking_chunk_dwell(1.0, 1.5), 2.0); + } + + #[test] + fn speaking_chunks_ping_pong_instead_of_wrapping() { + assert_eq!(ping_pong_sample_time(0.75, 1.0), 0.75); + assert_eq!(ping_pong_sample_time(1.0, 1.0), 1.0); + assert_eq!(ping_pong_sample_time(1.25, 1.0), 0.75); + assert_eq!(ping_pong_sample_time(1.5, 1.0), 0.5); + assert_eq!(ping_pong_sample_time(2.25, 1.0), 0.25); + assert_eq!(ping_pong_sample_time(1.0, 0.0), 0.0); + } + + #[test] + fn speaking_resume_holds_the_current_chunk() { + assert_eq!(speaking_resume_deadline(4.0, 2.0), 4.0); + assert_eq!(speaking_resume_deadline(2.1, 2.0), 2.7); + } + + #[test] + fn body_transitions_match_latest_persona_default() { + assert_eq!(BODY_TRANSITION_SECONDS, 0.35); + } + + #[test] + fn audio_observation_pauses_and_resumes_speaking_motion() { + let voice = VoiceState { + phase: "active".into(), + activity: "speaking".into(), + microphone_muted: false, + output_muted: false, + }; + assert!(speaking_motion_active(&voice, false, 0.0)); + assert!(!speaking_motion_active(&voice, true, 0.0)); + assert!(speaking_motion_active(&voice, true, 0.1)); + + let mut muted = voice.clone(); + muted.output_muted = true; + assert!(!speaking_motion_active(&muted, true, 0.1)); + } + + #[test] + fn automatic_role_holds_during_active_idle_and_listening() { + for activity in ["idle", "listening"] { + let voice = VoiceState { + phase: "active".into(), + activity: activity.into(), + microphone_muted: false, + output_muted: false, + }; + assert_eq!(immediate_automatic_role(&voice), None); + } + assert_eq!(VOICE_IDLE_DELAY, 0.9); + + let mut inactive = VoiceState::default(); + assert_eq!(immediate_automatic_role(&inactive), Some(ActionRole::Idle)); + inactive.phase = "active".into(); + inactive.output_muted = true; + assert_eq!(immediate_automatic_role(&inactive), Some(ActionRole::Idle)); + + inactive.output_muted = false; + inactive.activity = "speaking".into(); + assert_eq!( + immediate_automatic_role(&inactive), + Some(ActionRole::Speaking) + ); + } } diff --git a/docs/PERSONA.md b/docs/PERSONA.md index a1d7c6a..73bba1a 100644 --- a/docs/PERSONA.md +++ b/docs/PERSONA.md @@ -26,16 +26,22 @@ bun run accept:persona bun run accept:pocket ``` -The first command clones Persona commit `4efec3ac…` into the ignored +The first command clones Persona commit `bb7ef245…` into the ignored `out/persona-reference/` directory; it never modifies a user checkout. Both commands validate the same VRM/VRMA hashes and stage the checked-in `fixtures/persona/library.json`. Press Ctrl-C to terminate the complete target -process tree. +process tree. This is the controlled visual fixture; the production benchmark +uses Persona's packaged idle and 17 speaking clips. A visual pass must inspect +the complete pose through idle and speaking, especially shoulders, +wrists/hands, hips, knees, and ankles/feet, and reject any persistent rest-axis +twist rather than treating process or bridge readiness as sufficient. -Run the sequential production or controlled resource comparison with: +Run the sequential production idle, sustained-speaking, or high-refresh +diagnostic comparison with: ```sh bun run bench:persona +bun run bench:persona:speaking bun run bench:persona:controlled ``` @@ -44,19 +50,27 @@ The benchmark never runs both renderers concurrently. ## 1. Reference and decision The reference was inspected and measured at Persona commit -[`4efec3ac729944d0b36137dd8847cc1b488e0bcb`](https://github.com/xikhar/persona/tree/4efec3ac729944d0b36137dd8847cc1b488e0bcb), +[`bb7ef2455b23aee685f68c9e83a185347d257964`](https://github.com/xikhar/persona/tree/bb7ef2455b23aee685f68c9e83a185347d257964), version `0.1.0-beta.0`. +This is eight commits after the original POC pin. The material changes are +per-model lighting, configurable cross-platform voice sources, Core Audio +lifecycle fixes, a packaged default avatar, and a modular 17-clip speaking +sequence. Startup, bridge, MCP, and direct Electron launch contracts remain +compatible. The production benchmarks below consume the new packaged catalog +and assets directly; the media-free controlled fixture is now used only for +repeatable visual/control acceptance. + Primary upstream references: -- [README and product contract](https://github.com/xikhar/persona/blob/4efec3ac729944d0b36137dd8847cc1b488e0bcb/README.md) -- [Architecture and development](https://github.com/xikhar/persona/blob/4efec3ac729944d0b36137dd8847cc1b488e0bcb/docs/DEVELOPMENT.md) -- [Local bridge and MCP integration](https://github.com/xikhar/persona/blob/4efec3ac729944d0b36137dd8847cc1b488e0bcb/docs/INTEGRATIONS.md) -- [Electron lifecycle and control plane](https://github.com/xikhar/persona/blob/4efec3ac729944d0b36137dd8847cc1b488e0bcb/electron/main.cjs) -- [React/Three scene](https://github.com/xikhar/persona/blob/4efec3ac729944d0b36137dd8847cc1b488e0bcb/src/components/Scene.tsx) -- [VRM animation component](https://github.com/xikhar/persona/blob/4efec3ac729944d0b36137dd8847cc1b488e0bcb/src/components/Avatar.tsx) -- [Source license](https://github.com/xikhar/persona/blob/4efec3ac729944d0b36137dd8847cc1b488e0bcb/LICENSE) -- [Asset-license boundary](https://github.com/xikhar/persona/blob/4efec3ac729944d0b36137dd8847cc1b488e0bcb/ASSET_LICENSES.md) +- [README and product contract](https://github.com/xikhar/persona/blob/bb7ef2455b23aee685f68c9e83a185347d257964/README.md) +- [Architecture and development](https://github.com/xikhar/persona/blob/bb7ef2455b23aee685f68c9e83a185347d257964/docs/DEVELOPMENT.md) +- [Local bridge and MCP integration](https://github.com/xikhar/persona/blob/bb7ef2455b23aee685f68c9e83a185347d257964/docs/INTEGRATIONS.md) +- [Electron lifecycle and control plane](https://github.com/xikhar/persona/blob/bb7ef2455b23aee685f68c9e83a185347d257964/electron/main.cjs) +- [React/Three scene](https://github.com/xikhar/persona/blob/bb7ef2455b23aee685f68c9e83a185347d257964/src/components/Scene.tsx) +- [VRM animation component](https://github.com/xikhar/persona/blob/bb7ef2455b23aee685f68c9e83a185347d257964/src/components/Avatar.tsx) +- [Source license](https://github.com/xikhar/persona/blob/bb7ef2455b23aee685f68c9e83a185347d257964/LICENSE) +- [Asset-license boundary](https://github.com/xikhar/persona/blob/bb7ef2455b23aee685f68c9e83a185347d257964/public/assets/LICENSES.md) The local `~/code/pocket-character` checkout used for the implementation comparison was at @@ -116,7 +130,7 @@ loopback HTTP /events or /mcp -> bridge worker thread -> bounded BridgeCommand channel -> fixed-step PersonaWidget core - action selection and crossfade + action selection, modular speaking-chunk rotation, and crossfade skeleton pose + spring bones blink + amplitude visemes morph upload + pocket3d render @@ -148,12 +162,14 @@ The implementation lives in: | `scripts/bench-persona.ts` | Sequential, full-process-tree A/B resource harness | | `fixtures/persona/library.json` | Canonical media-free acceptance catalog shared by both targets | -The PocketJS submodule currently pins -[pocket-stack/pocketjs#204](https://github.com/pocket-stack/pocketjs/pull/204). -That upstream change contains only generic runtime window Show/Hide support; -the Persona catalog, renderer product, bridge, MCP server, acceptance commands, +The PocketJS dependency is split into two generic changes: runtime window +Show/Hide support in +[pocket-stack/pocketjs#204](https://github.com/pocket-stack/pocketjs/pull/204) +and normalized humanoid-space VRMA retargeting in +[pocket-stack/pocketjs#207](https://github.com/pocket-stack/pocketjs/pull/207). +The Persona catalog, renderer product, bridge, MCP server, acceptance commands, benchmark, and report remain owned by this repository. Repin the submodule to -the eventual PocketJS merge commit before merging this product change. +the eventual PocketJS merge commits before merging this product change. ## 3. Implemented parity @@ -164,13 +180,13 @@ are silently present. | Contract | Persona reference | Pocket Persona | Status | | --- | --- | --- | --- | | Character input | Packaged or imported VRM | Default model from Persona schema-v1 `library.json` | Implemented for the controlled VRM 0.x asset | -| Animation input | One or more VRMA clips per action | Same relative `.vrma` paths, parsed and retargeted at load | Implemented for the controlled clip; loader limits are below | +| Animation input | One or more VRMA clips per action | Same relative `.vrma` paths, retargeted at load from the source raw rig through VRM normalized humanoid space into the target VRM 0.x local rest basis | Implemented for the latest packaged idle plus 17 speaking clips; loader limits are below | | Idle role | Permanent `IDLE` action | Looped native idle action | Implemented | -| Speaking role | Permanent `TALK` action follows voice output | Looped native speaking action follows validated voice state | Implemented; state is externally supplied | +| Speaking role | Permanent `TALK` action follows voice output, pauses chunk rotation at an observed zero audio level, ping-pongs short gestures, rotates modular chunks, and holds the current chunk for `0.7 s` after motion resumes | Native speaking action follows validated voice/audio state with the same pause, ping-pong, no-repeat, resume-hold, and deterministic chunk rotation/fade behavior | Implemented for the latest packaged sequence; state is externally supplied | | Custom actions | Named one-shot actions | Named one-shots, then resume the current voice role | Implemented | | Clip choice | Random clip, avoid immediate repeat | Deterministic seeded choice, avoid immediate repeat | Behavior parity with reproducible tests | -| Crossfades | General `0.7 s`; into speaking `0.85 s`; speaking to idle `1.15 s` | Same durations with smoothstep TRS blending | Implemented | -| Talk-to-idle hold | Listener activity gate plus app-side settling | `0.65 s` app-side delay after listening state | Implemented at the app boundary; native gate deferred | +| Crossfades | Default body transitions `0.35 s` plus randomized two-action speaking blends | Matching `0.35 s` body transition and randomized speaking duration/dwell bounds with smoothstep pose blending | Behavior parity; Pocket blends from the outgoing sampled pose rather than evaluating two mixers concurrently | +| Talk-to-idle hold | Active idle/listening activity gate plus app-side settling | Matching `0.9 s` app-side delay before leaving speaking body motion | Implemented at the app boundary; native gate deferred | | Lip sync | Amplitude-only five-viseme driver | Same five VRM expression families, audible threshold, attack/release smoothing, and `0.62` cap | Implemented | | Blink | Random `2–6 s`, `0.24 s` envelope | Deterministic seeded `2–6 s`, `0.24 s` sine envelope | Implemented | | Secondary motion | VRM spring bones | Native `pocket-vrm` spring solver every fixed tick | Implemented | @@ -260,31 +276,34 @@ The visual output is intentionally not a pixel-parity port: textures. Pocket also uploads only images sampled as base color by a used material. These are intentional memory levers, not free visual parity. - Persona's live renderer follows the display refresh rate. Pocket's - production tick/render cap is `60 Hz`; its active idle clip therefore - presents at most 60 frames per second. + production tick/render cap is `30 Hz`; its active character therefore + presents at most 30 frames per second. Thirty hertz is also the spring + solver's natural fixed-step boundary before physics substeps are required. - Orbit inertia/damping, exact HDR lighting, exact alpha/material behavior, preview contact shadows, and pixel-identical framing remain deferred. The accepted 3D formats are narrower than Persona's general Three.js path. This POC supports VRM 0.x, not VRM 1.0. Its VRMA loader consumes the first -animation, retargets humanoid rotation plus hips translation, and supports the -accessor subset exercised by the controlled clip. Sparse accessors, exact +animation, converts humanoid rotations from the source raw rig through VRM's +normalized humanoid space into the target VRM 0.x local rest basis, retargets +hips translation, and supports the accessor subset exercised by both the +controlled fixture and Persona's packaged clips. Sparse accessors, exact CUBICSPLINE interpolation, non-humanoid translation, and `VRMC_vrm_animation` expression/look-at tracks are not parity claims. For that reason, performance must be reported in two lanes: a controlled `120 Hz / 4096` run that avoids crediting Pocket for a lower configured -quality target, and the actual `60 Hz / 2048` production default that measures +quality target, and the actual `30 Hz / 2048` production default that measures the user-facing optimization. ## 5. Asset and license boundary -The Persona source is MIT, but its own asset policy explicitly says that MIT -does not grant rights to local VRM or VRMA media. Persona's distributable -catalog is empty at the pinned commit, and there is no reference character -asset in the repository that this POC can legally copy by implication. +The Persona source is MIT, while its packaged media has separate terms recorded +in `public/assets/LICENSES.md`. At the current pin Persona includes a default +AvatarSample_A model, one idle clip, and 17 speaking clips. The benchmark uses +those upstream bytes in place and does not copy them into this repository. -The controlled local inputs are: +The separate media-free visual-acceptance fixture uses these local inputs: | Input | Size | SHA-256 | | --- | ---: | --- | @@ -297,15 +316,18 @@ provenance are not covered by its source license. Do not commit them here, attach them to a release, or infer redistribution permission from either project's source license. -The VRM contains 40,406 vertices, 29,221 triangles, 95 nodes, three skins, +The packaged and local AvatarSample_A files are byte-identical. The VRM +contains 40,406 vertices, 29,221 triangles, 95 nodes, three skins, seven primitives/materials, and thirteen PNG images. The source images include four `4096²` and five `2048²` textures. Decoded RGBA is approximately 336.50 MiB before mipmaps and 448.67 MiB with a full mip chain, which is why the source texture ceiling is held constant in the controlled lane and changed only in the production lane. -No Persona art, icon, HDR environment, VRM, or VRMA file is added to this -repository by the POC. +Persona's packaged idle is 337,240 bytes with SHA-256 +`033e4eda03faf33118988488ebfc1aecf116b25508f339182785f4c65308d83a`; +the 17 speaking clips total 1,546,316 bytes. No Persona art, icon, HDR +environment, VRM, or VRMA file is added to this repository by the POC. ## 6. Reproducible local library @@ -319,14 +341,14 @@ export PERSONA_ASSETS="$PERSONA_ROOT/public/assets" export PERSONA_LIBRARY="$PERSONA_ASSETS/library.json" cd "$PERSONA_ROOT" -git switch --detach 4efec3ac729944d0b36137dd8847cc1b488e0bcb +git switch --detach bb7ef2455b23aee685f68c9e83a185347d257964 mkdir -p "$PERSONA_ASSETS/models" "$PERSONA_ASSETS/animations" ln -sfn "$POCKET_CHARACTER_ROOT/assets/AvatarSample_A.vrm" \ "$PERSONA_ASSETS/models/model.vrm" for name in idle talk1 talk2 greeting happy finger-gun dance; do ln -sfn "$POCKET_CHARACTER_ROOT/assets/idle_loop.vrma" \ - "$PERSONA_ASSETS/animations/$name.vrma" + "$PERSONA_ASSETS/animations/pocket-controlled-$name.vrma" done cat >"$PERSONA_LIBRARY" <<'JSON' @@ -347,7 +369,7 @@ cat >"$PERSONA_LIBRARY" <<'JSON' "animation_description": "A calm resting motion for the character.", "animation_trigger_scenario": "Used automatically while Persona is waiting and not speaking.", "animation_type": "IDLE", - "asset_paths": ["animations/idle.vrma"] + "asset_paths": ["animations/pocket-controlled-idle.vrma"] }, { "id": "system-speaking", @@ -356,8 +378,8 @@ cat >"$PERSONA_LIBRARY" <<'JSON' "animation_trigger_scenario": "Used automatically while supported voice output is active.", "animation_type": "TALK", "asset_paths": [ - "animations/talk1.vrma", - "animations/talk2.vrma" + "animations/pocket-controlled-talk1.vrma", + "animations/pocket-controlled-talk2.vrma" ] }, { @@ -366,7 +388,7 @@ cat >"$PERSONA_LIBRARY" <<'JSON' "animation_description": "A friendly greeting motion.", "animation_trigger_scenario": "Use when beginning an interaction or welcoming the user.", "animation_type": "GREETING", - "asset_paths": ["animations/greeting.vrma"] + "asset_paths": ["animations/pocket-controlled-greeting.vrma"] }, { "id": "packaged-happy", @@ -374,7 +396,7 @@ cat >"$PERSONA_LIBRARY" <<'JSON' "animation_description": "A warm, upbeat reaction.", "animation_trigger_scenario": "Use for good news, success, gratitude, or a positive response.", "animation_type": "HAPPY", - "asset_paths": ["animations/happy.vrma"] + "asset_paths": ["animations/pocket-controlled-happy.vrma"] }, { "id": "packaged-finger-gun", @@ -382,7 +404,7 @@ cat >"$PERSONA_LIBRARY" <<'JSON' "animation_description": "A playful finger-gun gesture.", "animation_trigger_scenario": "Use for lighthearted confidence, a clever solution, or playful approval.", "animation_type": "FINGER_GUN", - "asset_paths": ["animations/finger-gun.vrma"] + "asset_paths": ["animations/pocket-controlled-finger-gun.vrma"] }, { "id": "packaged-dance", @@ -390,7 +412,7 @@ cat >"$PERSONA_LIBRARY" <<'JSON' "animation_description": "A celebratory dance.", "animation_trigger_scenario": "Use for a major success, an exciting milestone, or an explicit request to dance.", "animation_type": "DANCE", - "asset_paths": ["animations/dance.vrma"] + "asset_paths": ["animations/pocket-controlled-dance.vrma"] } ] } @@ -399,10 +421,12 @@ JSON All seven staged paths intentionally resolve to the one available local clip; the speaking slot has two aliases so the no-immediate-repeat path is exercised. -This exact catalog was used by both final benchmark reports. It verifies role -switching, one-shot completion, clip choice, crossfades, and control-plane +This catalog is used only by the two visual-acceptance commands. It verifies +role switching, one-shot completion, clip choice, crossfades, and control-plane behavior without inventing or distributing additional media; it is not a claim -that the motions are artistically distinct. +that the motions are artistically distinct. Benchmarks restore the clean +upstream checkout and consume Persona's packaged catalog, idle, and speaking +chunks directly. Build the pinned reference after staging the catalog so Vite copies the same media into `dist`: @@ -439,14 +463,14 @@ dist/pocket-persona/guest.js.map target/release/pocket-persona ``` -Run the production-oriented defaults — `430×680`, fixed `60 Hz`, texture cap +Run the production-oriented defaults — `430×680`, fixed `30 Hz`, texture cap `2048`, loopback bridge on port `47831`: ```sh ./target/release/pocket-persona \ --library "$PERSONA_LIBRARY" \ --bundle "$PWD/dist/pocket-persona/guest.js" \ - --fps 60 \ + --fps 30 \ --max-texture-dim 2048 ``` @@ -456,7 +480,7 @@ Disable the control plane for a renderer-only manual run: ./target/release/pocket-persona \ --library "$PERSONA_LIBRARY" \ --bundle "$PWD/dist/pocket-persona/guest.js" \ - --fps 60 \ + --fps 30 \ --max-texture-dim 2048 \ --no-bridge ``` @@ -469,12 +493,21 @@ mkdir -p "$PWD/dist/pocket-persona" ./target/release/pocket-persona \ --library "$PERSONA_LIBRARY" \ --bundle "$PWD/dist/pocket-persona/guest.js" \ - --fps 60 \ + --fps 30 \ --max-texture-dim 2048 \ --ticks 90 \ --headless-shot "$PWD/dist/pocket-persona/headless.png" ``` +Add `--headless-speaking` and choose `--ticks` to capture deterministic +speaking/transition frames through the same offscreen renderer. + +For a packaged-animation receipt, point `PERSONA_LIBRARY` at the restored +upstream catalog and retain idle plus at least two speaking-time screenshots. +The character must remain fully in frame, with natural shoulder-to-hand and +hip-to-foot axes: no persistent twist at the wrists/hands or ankles/feet. A +transparent PNG or healthy bridge alone does not establish pose parity. + Useful native tests: ```sh @@ -507,7 +540,7 @@ curl --fail-with-body --silent --show-error \ http://127.0.0.1:47831/events ``` -Return through listening to idle after the `0.65 s` app-side delay: +Return through listening to idle after the `0.9 s` app-side delay: ```sh curl --fail-with-body --silent --show-error \ @@ -582,36 +615,41 @@ compilation, asset decoding, and workspace builds materially contaminate each other. `scripts/bench-persona.ts` runs the reference first, terminates its full process group, then runs Pocket. For each target it: -1. requires `GET /health` to return `200` with `ok: true`, then waits for the +1. requires `GET /health` to return `200` with `ok: true`; +2. drives the requested sustained idle or speaking state, then waits for the configured settle period; -2. snapshots every descendant process; -3. samples cumulative process CPU time over fixed intervals; -4. sums RSS and records the complete process inventory; -5. reports median and p95 values; and -6. captures a health receipt alongside every resource sample; and -7. writes machine facts, exact launch commands, raw samples, and comparison +3. snapshots every descendant process; +4. samples cumulative process CPU time over fixed intervals; +5. sums RSS and records the complete process inventory; +6. reports median and p95 values; +7. captures a health receipt alongside every resource sample; and +8. writes machine facts, exact launch commands, raw samples, and comparison formulas to JSON. -The harness requires `--library` to resolve to the reference checkout's own -`public/assets/library.json`, so both targets cannot silently benchmark -different catalogs. Both always-on-top windows must remain visible and +The harness restores and requires `--library` to resolve to the reference +checkout's own packaged `public/assets/library.json`, so both targets cannot +silently benchmark different catalogs. Both always-on-top windows must remain visible and uncovered. The harness fails closed if Pocket's health receipts stop reporting -a configured, visible model and delivered frames, so compositor suspension +a configured, visible model or if monotonic `renderFrameCount` stops advancing, +so compositor suspension cannot silently become a renderer optimization. Persona starts its bridge before the avatar renderer, so its health response is a control-plane readiness check rather than proof of visible model output. The benchmark result must therefore be paired with a rendered-model -screenshot and a frame-rate receipt. Pocket health is stronger: the bridge -starts after model/animation/guest initialization and reports `renderFps`, -`modelConfigured`, and `windowVisible` in every sample. +screenshot and a frame-rate receipt. The screenshot receipt must use the +packaged catalog and the same retarget build as the benchmark, show both hands +and feet, and cover idle plus speaking so wrist/ankle basis errors cannot pass +as an optimization. Pocket health is stronger: the bridge starts after +model/animation/guest initialization and reports `renderFps`, +`renderFrameCount`, `modelConfigured`, and `windowVisible` in every sample. CPU percentages are percent of one logical core: `100%` means one saturated core. Summed RSS is useful for a same-machine process-tree comparison, but it can double-count shared mappings and is not a substitute for platform physical footprint tools. -### Cap-controlled POC lane: same asset, 120 Hz, 4096 textures +### High-refresh diagnostic lane: packaged assets, 120 Hz, 4096 textures Use the same `library.json` and source hashes for both targets. Set the macOS display to `120 Hz`, keep both windows untouched and visible, and ensure no @@ -622,7 +660,7 @@ is explicitly fixed/capped at `120`. bun run bench:persona:controlled ``` -`4096` leaves the controlled model's source textures at authoring resolution. +`4096` leaves the packaged model's source textures at authoring resolution. This is the strongest like-for-like POC lane, but not an architecture-only headline: Pocket still omits the reference MToon/HDR path and unused texture roles. Record the actual frame rate and physical framebuffer of each window as @@ -630,7 +668,7 @@ a companion receipt: the reference caps DPR at `1.5`, while the native swapchain follows the OS backing scale. Equal logical window size does not by itself prove equal fragment workload. -### Production lane: Pocket 60 Hz, 2048 textures +### Production lane: stock Persona vs Pocket 30 Hz, 2048 textures This keeps the reference unchanged and applies Pocket's intended shipping defaults. It measures the total user-facing saving including the refresh and @@ -640,99 +678,124 @@ texture-cap choices: bun run bench:persona ``` +Measure the latest modular speaking path instead of idle with: + +```sh +bun run bench:persona:speaking +``` + Before reporting a result, verify the two JSON reports have `status: "ok"`, both runs contain all requested samples, the commands contain the intended caps, the asset hashes still match this document, and no unexpected helper or -build process joined either process tree. +build process joined either process tree. Also inspect the packaged idle and +speaking screenshot receipts for natural wrists/hands and ankles/feet. ## 11. Performance results -Both final sequential reports completed with `status: "ok"` on 2026-07-30. -Each target settled for 30 seconds and then produced nine five-second process -tree samples. No build or second benchmark overlapped either run. +The three refreshed sequential reports completed with `status: "ok"` on +2026-08-03. Each target settled for 30 seconds and then produced nine +five-second process-tree samples. No build or second benchmark overlapped a +run. Reference environment: MacBook Pro `Mac15,8`, Apple M3 Max, 128 GiB RAM, macOS `26.5.2 (25F84)`, Electron `39.8.10`, Chromium `142.0.7444.265`, logical viewport `430×680`, and a 120 Hz display. Persona rendered a `645×1020` WebGL canvas at DPR `1.5`; Pocket's native 2× swapchain was `860×1360`, or 1.78 times as many backing pixels. Both targets consumed the -same catalog, 26,781,812-byte VRM, and 157,664-byte VRMA recorded above. -Neither run included Persona's optional native audio helper. +same clean packaged catalog at `bb7ef245…`: the byte-identical 26,781,812-byte +VRM, 337,240-byte idle, and 17 speaking chunks. No run included Persona's +optional native audio helper. RSS comes from the nine process-tree snapshots and remains a diagnostic -mapping total. The physical-footprint rows come from separate 30-second -sustained-render measurements with macOS `footprint`: all four Electron -process IDs were included for Persona, and Pocket had one process. The -reference footprint was measured once because the reference configuration is -identical in both lanes. These are steady-state snapshots, not startup peaks. -Pocket's auxiliary per-process peak fields were 1,400,440,104 bytes at 4096 -and 1,030,063,256 bytes at 2048, so this POC makes no peak-memory reduction -claim. +mapping total that can double-count shared pages. This refresh did not repeat +the separate macOS `footprint` experiment, so it makes no new physical-footprint +or startup-peak claim. -### Controlled result +### High-refresh idle diagnostic This lane used the same authoring-resolution textures and requested 120 Hz from Pocket: | Metric | Persona | Pocket | Pocket delta | | --- | ---: | ---: | ---: | -| Observed frame rate | 120.017 fps | 102.135 fps | 14.9% fewer frames | -| Frame interval p95 / p99 | 9.700 / 10.200 ms | 10.951 / 11.116 ms | see sampling note | -| Process-tree CPU median / p95 | 10.800% / 11.517% | 15.400% / 16.396% | **42.6% / 42.4% more** | -| CPU percent per delivered fps | 0.0900 | 0.1508 | **67.6% more** | -| Summed process-tree RSS median / p95 | 1,181,392 / 1,182,906 KiB | 115,488 / 116,102 KiB | **90.2% less** | +| Observed frame rate | 120.013 fps | 111.077 fps | 7.4% fewer frames | +| Process-tree CPU median / p95 | 21.196% / 22.559% | 9.002% / 10.041% | **57.5% / 55.5% less** | +| CPU percent per delivered fps | 0.1766 | 0.0810 | **54.1% less** | +| Summed process-tree RSS median / p95 | 1,197,728 / 1,199,219 KiB | 121,920 / 122,086 KiB | **89.8% less** | | Process count | 4 | 1 | **75.0% fewer** | -| Settled macOS physical footprint (30 s) | 1,395,873,216 bytes | 679,920,456 bytes | **51.3% less; 2.05× smaller** | -The cap-controlled POC therefore has a large memory and process-count win, but -this is not an architecture-only attribution: Pocket omits renderer features -and texture roles listed in section 4. It also does **not** have a controlled -high-refresh CPU win. It delivered fewer frames while using more process CPU. -Pocket's larger native backing surface makes the fragment workload stricter -than exact pixel parity, but it does not turn the CPU-per-frame result into an -optimization claim. +The refreshed POC now has a high-refresh CPU win as well as memory/process +wins. This is still not an architecture-only attribution: Pocket omits renderer +features and texture roles listed in section 4, renders a larger native backing +surface, and delivered 7.4% fewer frames. It is a current product-stack +diagnostic, not pixel-identical renderer science. Persona's frame receipt is one continuous 45-second CDP `requestAnimationFrame` -sample; the Pocket p95/p99 values above are the medians of nine native rolling -one-second receipts. They describe each renderer accurately but are not an -identical long-window percentile estimator. Persona had zero intervals over -20 ms across 5,401 delivered frames. Pocket's worst reported one-second maximum -was 12.592 ms. +sample. Pocket FPS is the median of nine native rolling receipts; these are not +identical frame-present instrumentation, so CPU-per-frame remains a diagnostic. -### Production result +### Production idle result -This lane retained the unchanged 120 Hz Persona reference and used Pocket's -shipping defaults, a 60 Hz cap and 2048 texture cap: +This lane retained stock, display-driven Persona and used Pocket's intended +30 Hz / 2048 production configuration: | Metric | Persona | Pocket | Pocket delta | | --- | ---: | ---: | ---: | -| Observed frame rate | 120.008 fps | 54.372 fps | 54.7% fewer frames | -| Frame interval p95 / p99 | 9.600 / 10.200 ms | 19.448 / 19.639 ms | see sampling note | -| Process-tree CPU median / p95 | 10.600% / 11.638% | 8.800% / 9.000% | **17.0% / 22.7% less** | -| CPU percent per delivered fps | 0.0883 | 0.1619 | **83.3% more** | -| Summed process-tree RSS median / p95 | 1,190,720 / 1,192,163 KiB | 89,232 / 89,344 KiB | **92.5% less** | +| Observed frame rate | 120.021 fps | 28.358 fps | 76.4% fewer frames | +| Process-tree CPU median / p95 | 17.603% / 21.996% | 4.600% / 4.800% | **73.9% / 78.2% less** | +| CPU percent per delivered fps | 0.1467 | 0.1622 | **Pocket 10.6% worse** | +| Summed process-tree RSS median / p95 | 1,193,536 / 1,195,155 KiB | 90,784 / 90,784 KiB | **92.4% less** | | Process count | 4 | 1 | **75.0% fewer** | -| Settled macOS physical footprint (30 s) | 1,395,873,216 bytes | 517,260,128 bytes | **62.9% less; 2.70× smaller** | - -The production configuration saves 17.0% median process CPU in absolute -terms, but only by delivering about half as many frames and downscaling source -textures. Normalized per delivered frame, it is 83.3% more CPU-expensive than -Persona in this run. The defensible performance headline is therefore: - -- 51.3% less settled controlled physical footprint, or 62.9% less settled - footprint at production texture settings; -- 90.2–92.5% less summed RSS and one process instead of four; -- 17.0% less total CPU at the default 60 Hz / 2048 configuration; and -- no CPU-throughput optimization yet—the controlled lane regresses 42.6% in - raw CPU and 67.6% per delivered frame. - -The unsigned upstream arm64 `.app` occupied 302,128 KiB and its distributable -zip was 129,730,393 bytes. The final POC release binary, guest bundle, model, -and seven declared local clip paths total 38,876,395 bytes before application -packaging; the catalog adds 2,363 bytes. All seven clip files are byte-identical -test aliases. This suggests substantial distribution-size headroom, but it is -not reported as a parity optimization because this POC intentionally omits -settings, import, tray/update plumbing, and native audio. + +The median CPU ratio in this aligned production-idle run is +Persona/Pocket `3.826×`. + +### Production sustained-speaking result + +This uses the same production configuration while holding both targets in +speaking state with amplitude `0.35`. Persona exercises its latest 17-clip +random sequence; Pocket exercises deterministic no-repeat chunk rotation and +the matching ping-pong, randomized dwell/fade, and resume-hold rules. + +| Metric | Persona | Pocket | Pocket delta | +| --- | ---: | ---: | ---: | +| Observed frame rate | 120.011 fps | 29.062 fps | 75.8% fewer frames | +| Process-tree CPU median / p95 | 17.999% / 18.442% | 3.000% / 3.200% | **83.3% / 82.6% less** | +| CPU percent per delivered fps | 0.1500 | 0.1032 | **31.2% less** | +| Summed process-tree RSS median / p95 | 1,219,344 / 1,220,746 KiB | 139,424 / 139,424 KiB | **88.6% less** | +| Process count | 4 | 1 | **75.0% fewer** | + +The defensible current headline is therefore: + +- **73.9% less settled idle CPU** and **83.3% less sustained-speaking CPU** for + stock Persona versus Pocket's intended 30 Hz production configuration; +- **88.6–92.4% less summed RSS** and one process instead of four in those + production runs; and +- even the high-refresh diagnostic used **57.5% less median CPU** and **54.1% + less CPU per delivered frame** with the current packaged idle. + +The previously reported **89.5% idle CPU reduction is invalid**: that run used +the current packaged animation bytes, but Pocket interpreted raw local VRMA +rotations as normalized humanoid rotations, visibly twisting wrists/hands and +ankles/feet. The tables above are post-alignment reruns using load-time +source-normalized-target basis conversion and screenshot-verified poses. The +older 2026-07-30 result is also not comparable: it pinned Persona before the +packaged avatar/modular-speaking update, overlaid a synthetic AIRI idle fixture, +and used Pocket at 60 Hz for the production lane. + +The aligned idle result also makes the tradeoff explicit: total CPU is 73.9% +lower because Pocket delivers roughly one quarter as many frames, while CPU per +delivered idle frame is 10.6% worse. Sustained speaking still uses 31.2% less +CPU per delivered frame, and the near-refresh-matched diagnostic uses 54.1% +less. The remaining Pocket CPU profile is concentrated in the +`Renderer`/wgpu/Metal submission path, not load-time retargeting, spring solving, +or the QuickJS guest; further steady-state work should target render submission +and presentation rather than those subsystems. + +A strict patched 30-vs-30 WebGL/native experiment would answer a different +question and must count actual draw/present calls; merely throttling the CDP +`requestAnimationFrame` observer would be circular. The unmodified-stock versus +shipping-config production lane above is the user-facing power comparison. Startup is also left out of the percentage claim. Persona exposes bridge health before avatar readiness, while Pocket starts its bridge after model, diff --git a/fixtures/persona/library.json b/fixtures/persona/library.json index 0c493f1..7795065 100644 --- a/fixtures/persona/library.json +++ b/fixtures/persona/library.json @@ -16,7 +16,7 @@ "animation_trigger_scenario": "Used automatically while Persona is waiting and not speaking.", "animation_type": "IDLE", "asset_paths": [ - "animations/idle.vrma" + "animations/pocket-controlled-idle.vrma" ] }, { @@ -26,8 +26,8 @@ "animation_trigger_scenario": "Used automatically while supported voice output is active.", "animation_type": "TALK", "asset_paths": [ - "animations/talk1.vrma", - "animations/talk2.vrma" + "animations/pocket-controlled-talk1.vrma", + "animations/pocket-controlled-talk2.vrma" ] }, { @@ -37,7 +37,7 @@ "animation_trigger_scenario": "Use when beginning an interaction or welcoming the user.", "animation_type": "GREETING", "asset_paths": [ - "animations/greeting.vrma" + "animations/pocket-controlled-greeting.vrma" ] }, { @@ -47,7 +47,7 @@ "animation_trigger_scenario": "Use for good news, success, gratitude, or a positive response.", "animation_type": "HAPPY", "asset_paths": [ - "animations/happy.vrma" + "animations/pocket-controlled-happy.vrma" ] }, { @@ -57,7 +57,7 @@ "animation_trigger_scenario": "Use for lighthearted confidence, a clever solution, or playful approval.", "animation_type": "FINGER_GUN", "asset_paths": [ - "animations/finger-gun.vrma" + "animations/pocket-controlled-finger-gun.vrma" ] }, { @@ -67,7 +67,7 @@ "animation_trigger_scenario": "Use for a major success, an exciting milestone, or an explicit request to dance.", "animation_type": "DANCE", "asset_paths": [ - "animations/dance.vrma" + "animations/pocket-controlled-dance.vrma" ] } ] diff --git a/package.json b/package.json index b9f5032..de0a885 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "accept:persona": "bun scripts/accept-persona.ts reference", "accept:pocket": "bun scripts/accept-persona.ts pocket", "bench:persona": "bun scripts/accept-persona.ts bench --profile production", + "bench:persona:speaking": "bun scripts/accept-persona.ts bench --profile production --activity speaking", "bench:persona:controlled": "bun scripts/accept-persona.ts bench --profile controlled" } } diff --git a/scripts/accept-persona.ts b/scripts/accept-persona.ts index 9dc3723..3ab69e5 100644 --- a/scripts/accept-persona.ts +++ b/scripts/accept-persona.ts @@ -17,7 +17,7 @@ import { writeFileSync, } from "node:fs"; import { platform, tmpdir } from "node:os"; -import { dirname, join, resolve } from "node:path"; +import { dirname, isAbsolute, join, relative, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { @@ -29,11 +29,14 @@ import { type Mode = "reference" | "pocket" | "bench"; type BenchmarkProfile = "production" | "controlled"; +type BenchmarkActivity = "idle" | "speaking"; +type ReferenceCatalogMode = "packaged" | "controlled"; type SpawnedProcess = ReturnType; interface CliOptions { mode: Mode; profile: BenchmarkProfile; + activity: BenchmarkActivity; cycles: number; realAudio: boolean; settleSeconds: number; @@ -55,7 +58,7 @@ const OUT = join(ROOT, "out"); const REFERENCE_ROOT = join(OUT, "persona-reference"); const REFERENCE_STATE = join(OUT, "persona-reference-state"); const REFERENCE_REPOSITORY = "https://github.com/xikhar/persona.git"; -const REFERENCE_COMMIT = "4efec3ac729944d0b36137dd8847cc1b488e0bcb"; +const REFERENCE_COMMIT = "bb7ef2455b23aee685f68c9e83a185347d257964"; const FIXTURE_LIBRARY = join(ROOT, "fixtures", "persona", "library.json"); const REFERENCE_LIBRARY = join( REFERENCE_ROOT, @@ -85,6 +88,26 @@ const POCKET_GUEST_DIR = join(ROOT, "dist", "pocket-persona"); const POCKET_GUEST = join(POCKET_GUEST_DIR, "guest.js"); const READINESS_TIMEOUT_MS = 60_000; const REQUEST_TIMEOUT_MS = 2_000; +const LEGACY_CONTROLLED_LIBRARY_SHA256 = + "4c344b94c35316ac928784a42171ed94db55667e9f52a95db713a7f223401991"; +const CONTROLLED_ANIMATION_NAMES = [ + "pocket-controlled-idle", + "pocket-controlled-talk1", + "pocket-controlled-talk2", + "pocket-controlled-greeting", + "pocket-controlled-happy", + "pocket-controlled-finger-gun", + "pocket-controlled-dance", +] as const; +const LEGACY_CONTROLLED_ANIMATION_NAMES = [ + "idle", + "talk1", + "talk2", + "greeting", + "happy", + "finger-gun", + "dance", +] as const; const ASSETS: Asset[] = [ { @@ -136,12 +159,18 @@ async function main(argv: string[]): Promise { try { await preflight(options); - await ensureAssets(); await ensureReferenceCheckout(); - stageReferenceLibrary(); + const referenceCatalogMode: ReferenceCatalogMode = + options.mode === "bench" ? "packaged" : "controlled"; + if (referenceCatalogMode === "packaged") { + await restoreToolOwnedReferenceOverlay(); + } else { + await ensureAssets(); + stageControlledReferenceLibrary(); + } if (options.mode === "reference") { - await ensureReferenceBuild(options.realAudio); + await ensureReferenceBuild(options.realAudio, referenceCatalogMode); await runVisibleReference(options); return 0; } @@ -152,7 +181,7 @@ async function main(argv: string[]): Promise { return 0; } - await ensureReferenceBuild(false); + await ensureReferenceBuild(false, referenceCatalogMode); return runBenchmark(options); } catch (error) { if (signalExitCode != null) return signalExitCode; @@ -179,6 +208,7 @@ function parseArgs(argv: string[]): CliOptions { } let profile: BenchmarkProfile = "production"; + let activity: BenchmarkActivity = "idle"; let cycles = 0; let realAudio = false; let settleSeconds = 30; @@ -222,6 +252,14 @@ function parseArgs(argv: string[]): CliOptions { profile = value; break; } + case "--activity": { + const value = readValue(flag); + if (value !== "idle" && value !== "speaking") { + throw new Error("--activity must be idle or speaking"); + } + activity = value; + break; + } case "--cycles": cycles = readNumber(flag, 0, true); break; @@ -248,6 +286,9 @@ function parseArgs(argv: string[]): CliOptions { if (mode !== "bench" && profile !== "production") { throw new Error("--profile is only valid in bench mode"); } + if (mode !== "bench" && activity !== "idle") { + throw new Error("--activity is only valid in bench mode"); + } if (mode === "bench" && realAudio) { throw new Error("--real-audio is only valid for accept:persona"); } @@ -255,6 +296,7 @@ function parseArgs(argv: string[]): CliOptions { return { mode, profile, + activity, cycles, realAudio, settleSeconds, @@ -381,18 +423,7 @@ async function ensureReferenceCheckout(): Promise { "reference", ); if (head.trim() !== REFERENCE_COMMIT) { - const status = ( - await commandOutput( - ["git", "status", "--porcelain", "--untracked-files=no"], - REFERENCE_ROOT, - "reference", - ) - ).trim(); - if (status.length > 0) { - throw new Error( - `Persona reference has tracked edits and cannot switch commits: ${status}`, - ); - } + await restoreToolOwnedReferenceOverlay(); await runCommand( ["git", "fetch", "--depth=1", "origin", REFERENCE_COMMIT], REFERENCE_ROOT, @@ -417,7 +448,72 @@ async function ensureReferenceCheckout(): Promise { } } -function stageReferenceLibrary(): void { +async function restoreToolOwnedReferenceOverlay(): Promise { + const status = ( + await commandOutput( + ["git", "status", "--porcelain", "--untracked-files=no"], + REFERENCE_ROOT, + "reference", + ) + ).trim(); + if (status.length > 0 && !isToolOwnedReferenceOverlay(status)) { + throw new Error( + `Persona reference has tracked edits outside the controlled overlay: ${status}`, + ); + } + if (status.length > 0) { + await runCommand( + [ + "git", + "restore", + "--source=HEAD", + "--staged", + "--worktree", + "--", + "public/assets/library.json", + ], + REFERENCE_ROOT, + "reference", + ); + } + removeToolOwnedReferenceSymlinks(); +} + +function isToolOwnedReferenceOverlay(status: string): boolean { + const librarySha256 = existsSync(REFERENCE_LIBRARY) + ? sha256File(REFERENCE_LIBRARY) + : null; + return ( + (librarySha256 === sha256File(FIXTURE_LIBRARY) || + librarySha256 === LEGACY_CONTROLLED_LIBRARY_SHA256) && + status + .split("\n") + .every((line) => line.endsWith(" public/assets/library.json")) + ); +} + +function removeToolOwnedReferenceSymlinks(): void { + const controlledPaths = [ + [join(REFERENCE_ROOT, "public", "assets", "models", "model.vrm"), ASSETS[0].path], + ...[ + ...CONTROLLED_ANIMATION_NAMES, + ...LEGACY_CONTROLLED_ANIMATION_NAMES, + ].map((name) => [ + join(REFERENCE_ROOT, "public", "assets", "animations", `${name}.vrma`), + ASSETS[1].path, + ]), + ]; + for (const [path, expectedSource] of controlledPaths) { + if ( + isDanglingSymlink(path) && + resolve(dirname(path), readlinkSync(path)) === expectedSource + ) { + unlinkSync(path); + } + } +} + +function stageControlledReferenceLibrary(): void { const assetsRoot = join(REFERENCE_ROOT, "public", "assets"); const modelRoot = join(assetsRoot, "models"); const animationRoot = join(assetsRoot, "animations"); @@ -425,15 +521,7 @@ function stageReferenceLibrary(): void { mkdirSync(animationRoot, { recursive: true }); ensureSymlink(ASSETS[0].path, join(modelRoot, "model.vrm")); - for (const name of [ - "idle", - "talk1", - "talk2", - "greeting", - "happy", - "finger-gun", - "dance", - ]) { + for (const name of CONTROLLED_ANIMATION_NAMES) { ensureSymlink(ASSETS[1].path, join(animationRoot, `${name}.vrma`)); } @@ -471,7 +559,10 @@ function isDanglingSymlink(path: string): boolean { } } -async function ensureReferenceBuild(realAudio: boolean): Promise { +async function ensureReferenceBuild( + realAudio: boolean, + catalogMode: ReferenceCatalogMode, +): Promise { const packageLock = join(REFERENCE_ROOT, "package-lock.json"); const installFingerprint = sha256File(packageLock); const installMarker = join(REFERENCE_STATE, "install.sha256"); @@ -488,25 +579,26 @@ async function ensureReferenceBuild(realAudio: boolean): Promise { [ REFERENCE_COMMIT, installFingerprint, - sha256File(FIXTURE_LIBRARY), - ...ASSETS.map((asset) => asset.sha256), + catalogMode, + ...(catalogMode === "controlled" + ? [sha256File(FIXTURE_LIBRARY), ...ASSETS.map((asset) => asset.sha256)] + : [sha256File(REFERENCE_LIBRARY)]), ].join("\n"), ); const buildMarker = join(REFERENCE_STATE, "renderer.sha256"); - const builtModel = join( - REFERENCE_ROOT, - "dist", - "assets", - "models", - "model.vrm", - ); + const catalogAssets = referenceCatalogAssetPaths(); if ( readText(buildMarker).trim() !== buildFingerprint || !existsSync(join(REFERENCE_ROOT, "dist", "index.html")) || - !existsSync(builtModel) + !referenceBuildAssetsMatch(catalogAssets) ) { console.log("build Persona renderer"); await runCommand(["npm", "run", "build"], REFERENCE_ROOT, "reference"); + if (!referenceBuildAssetsMatch(catalogAssets)) { + throw new Error( + "Persona build did not preserve every model/animation asset declared by library.json", + ); + } writeText(buildMarker, `${buildFingerprint}\n`); } @@ -520,6 +612,55 @@ async function ensureReferenceBuild(realAudio: boolean): Promise { } } +function referenceCatalogAssetPaths(): string[] { + const parsed = JSON.parse(readFileSync(REFERENCE_LIBRARY, "utf8")) as { + models?: Array<{ asset_path?: unknown }>; + animations?: Array<{ asset_paths?: unknown }>; + }; + const rawPaths: unknown[] = [ + ...(parsed.models ?? []).map((model) => model.asset_path), + ...(parsed.animations ?? []).flatMap((animation) => + Array.isArray(animation.asset_paths) ? animation.asset_paths : [], + ), + ]; + if (rawPaths.length === 0) { + throw new Error("Persona library.json declares no model or animation assets"); + } + const publicAssets = join(REFERENCE_ROOT, "public", "assets"); + return [...new Set(rawPaths.map((rawPath) => { + if (typeof rawPath !== "string" || rawPath.trim().length === 0) { + throw new Error("Persona library.json contains an invalid asset path"); + } + const normalized = rawPath.replaceAll("\\", "/"); + const source = resolve(publicAssets, normalized); + const withinRoot = relative(publicAssets, source); + if ( + isAbsolute(normalized) || + withinRoot === ".." || + withinRoot.startsWith(`..${process.platform === "win32" ? "\\" : "/"}`) + ) { + throw new Error(`Persona library asset escapes public/assets: ${rawPath}`); + } + return normalized; + }))]; +} + +function referenceBuildAssetsMatch(assetPaths: readonly string[]): boolean { + const publicAssets = join(REFERENCE_ROOT, "public", "assets"); + const builtAssets = join(REFERENCE_ROOT, "dist", "assets"); + return assetPaths.every((assetPath) => { + const source = join(publicAssets, assetPath); + const built = join(builtAssets, assetPath); + return ( + existsSync(source) && + existsSync(built) && + statSync(source).isFile() && + statSync(built).isFile() && + sha256File(source) === sha256File(built) + ); + }); +} + async function ensurePocketBuild(): Promise { console.log("build Pocket Persona guest"); mkdirSync(POCKET_GUEST_DIR, { recursive: true }); @@ -597,7 +738,7 @@ async function runVisiblePocket(options: CliOptions): Promise { "--bridge-port", String(port), "--fps", - "60", + "30", "--max-texture-dim", "2048", ], @@ -689,6 +830,9 @@ function printVisualChecklist( console.log(`ready ${target} at http://127.0.0.1:${port}`); console.log("check full character in a transparent, frameless, topmost 430x680 window"); console.log("check idle loop, autonomous blink, and spring motion"); + console.log( + "check natural shoulder/wrist/hand and hip/knee/ankle/foot axes; no twist", + ); console.log("check speaking body + pulsed lips, then action crossfade and return"); console.log("input scroll zoom · left-drag orbit · right-drag pan"); console.log( @@ -802,17 +946,17 @@ async function runBenchmark(options: CliOptions): Promise { const profile = options.profile === "controlled" ? { fps: 120, texture: 4096 } - : { fps: 60, texture: 2048 }; + : { fps: 30, texture: 2048 }; const stamp = new Date().toISOString().replace(/[:.]/g, "-"); const outPath = options.outPath ?? join( OUT, "bench", - `persona-${options.profile}-${profile.fps}hz-${profile.texture}-${stamp}.json`, + `persona-${options.profile}-${options.activity}-${profile.fps}hz-${profile.texture}-${stamp}.json`, ); console.log( - `bench ${options.profile}: ${profile.fps} Hz / ${profile.texture}px textures`, + `bench ${options.profile}/${options.activity}: Pocket ${profile.fps} Hz / ${profile.texture}px textures`, ); console.log("bench keep each topmost window visible and leave input untouched"); @@ -833,6 +977,8 @@ async function runBenchmark(options: CliOptions): Promise { String(profile.fps), "--max-texture-dim", String(profile.texture), + "--activity", + options.activity, "--settle", String(options.settleSeconds), "--samples", @@ -1002,11 +1148,12 @@ function usage(): string { Usage: bun run accept:persona [-- --cycles N] [--real-audio] bun run accept:pocket [-- --cycles N] - bun run bench:persona [-- --settle 30 --samples 9 --interval 5] - bun run bench:persona:controlled [-- --settle 30 --samples 9 --interval 5] - -The two visual commands stage the same pinned Persona catalog and repeat the -same idle, speaking/lip-sync, listening, and greeting sequence. N=0 (default) -repeats until Ctrl-C. Benchmark modes run Persona and Pocket sequentially and -write a timestamped JSON report under out/bench/.`; + bun run bench:persona [-- --activity idle|speaking --settle 30 --samples 9 --interval 5] + bun run bench:persona:controlled [-- --activity idle|speaking --settle 30 --samples 9 --interval 5] + +The two visual commands stage the same controlled fixture catalog and repeat +the same idle, speaking/lip-sync, listening, and greeting sequence. N=0 +(default) repeats until Ctrl-C. Benchmark modes restore Persona's packaged +catalog, run Persona and Pocket sequentially, and write a timestamped JSON +report under out/bench/.`; } diff --git a/scripts/bench-persona.ts b/scripts/bench-persona.ts index d699427..877c229 100644 --- a/scripts/bench-persona.ts +++ b/scripts/bench-persona.ts @@ -29,6 +29,7 @@ import { fileURLToPath } from "node:url"; export type TargetName = "reference" | "pocket"; type RunStatus = "running" | "ok" | "failed"; +type BenchmarkActivity = "idle" | "speaking"; interface Options { referenceBin: string; @@ -38,6 +39,7 @@ interface Options { bundle: string; maxFps: number | null; maxTextureDim: number | null; + activity: BenchmarkActivity; settleSeconds: number; sampleCount: number; intervalSeconds: number; @@ -187,6 +189,7 @@ interface RunResult { launched_at: string; ready_at: string | null; readiness_health: HealthCapture | null; + settled_activity_health: HealthCapture | null; reference_cdp: ReferenceCdpReceipt | null; frame_receipt: ReferenceFrameReceipt | null; settled_at: string | null; @@ -199,7 +202,7 @@ interface RunResult { } interface Report { - schema_version: 4; + schema_version: 5; benchmark: "persona-reference-vs-pocket"; status: "running" | "ok" | "failed" | "interrupted"; started_at: string; @@ -211,6 +214,7 @@ interface Report { interval_seconds: number; max_fps: number | null; max_texture_dim: number | null; + activity: BenchmarkActivity; readiness_timeout_seconds: number; cdp_timeout_seconds: number; output: string; @@ -250,6 +254,7 @@ const READINESS_TIMEOUT_SECONDS = 30; const CDP_TIMEOUT_SECONDS = 30; const HEALTH_POLL_INTERVAL_MS = 250; const HEALTH_REQUEST_TIMEOUT_MS = 1_000; +const EVENT_REQUEST_TIMEOUT_MS = 2_000; const CDP_POLL_INTERVAL_MS = 250; const CDP_COMMAND_TIMEOUT_MS = 10_000; const REFERENCE_VIEWPORT = { @@ -293,7 +298,7 @@ async function main(argv: string[]): Promise { const startedAt = new Date(); const startedClock = performance.now(); const report: Report = { - schema_version: 4, + schema_version: 5, benchmark: "persona-reference-vs-pocket", status: "running", started_at: startedAt.toISOString(), @@ -305,6 +310,7 @@ async function main(argv: string[]): Promise { interval_seconds: options.intervalSeconds, max_fps: options.maxFps, max_texture_dim: options.maxTextureDim, + activity: options.activity, readiness_timeout_seconds: READINESS_TIMEOUT_SECONDS, cdp_timeout_seconds: CDP_TIMEOUT_SECONDS, output: options.outPath, @@ -321,11 +327,11 @@ async function main(argv: string[]): Promise { readiness_policy: "each target must return HTTP 200 JSON with ok:true from its isolated loopback /health endpoint before settling", sample_health_policy: - "every resource sample includes a contemporaneous successful /health response", + "settle completion and every sample require the requested voice state on both targets; Pocket also requires the matching active animation and amplitude", reference_frame_policy: "Electron must expose a non-settings CDP page with a ready 430x680 DPR 1.5 WebGL canvas backed by 645x1020 pixels; one lightweight rAF promise spans the resource sampling window", pocket_frame_policy: - "every Pocket health receipt must report modelConfigured=true, windowVisible=true, and renderFps>1", + "every Pocket health receipt must report modelConfigured=true, windowVisible=true, renderFps>1, and a positive renderFrameCount that advances between samples", exited_process_caveat: "CPU accrued after the previous snapshot by a process that exits before the next snapshot is not observable", }, @@ -557,6 +563,7 @@ async function captureRequiredHealth( spec: LaunchSpec, child: SpawnedProcess, context: string, + activity?: BenchmarkActivity, ): Promise { const result = await probeHealth( spec, @@ -568,7 +575,7 @@ async function captureRequiredHealth( `${spec.target} ${context} health check failed: ${result.failure}`, ); } - const validationFailure = healthValidationFailure(spec, result.capture); + const validationFailure = healthValidationFailure(spec, result.capture, activity); if (validationFailure != null) { throw new Error( `${spec.target} ${context} health check failed: ${validationFailure}`, @@ -580,16 +587,70 @@ async function captureRequiredHealth( function healthValidationFailure( spec: LaunchSpec, capture: HealthCapture, + activity?: BenchmarkActivity, ): string | null { - if (spec.target !== "pocket") return null; try { - validatePocketHealthBody(capture.body); + if (spec.target === "pocket") validatePocketHealthBody(capture.body); + if (activity != null) { + validateBenchmarkActivityHealth(spec.target, capture.body, activity); + } return null; } catch (error) { return errorMessage(error); } } +export function validateBenchmarkActivityHealth( + target: TargetName, + body: JsonObject, + activity: BenchmarkActivity, +): void { + const expectedState = { + phase: activity === "speaking" ? "active" : "inactive", + activity, + microphoneMuted: false, + outputMuted: false, + }; + const state = + target === "reference" + ? body.lastState + : isJsonObject(body.status) + ? body.status.voiceState + : null; + if (!isJsonObject(state)) { + throw new Error(`${target} activity voice state must be an object`); + } + for (const [name, expected] of Object.entries(expectedState)) { + if (state[name] !== expected) { + throw new Error( + `${target} activity ${name} must be ${JSON.stringify(expected)}; ` + + `found ${JSON.stringify(state[name])}`, + ); + } + } + + if (target === "pocket") { + const status = body.status as JsonObject; + if (status.activeAnimation !== activity) { + throw new Error( + `Pocket activeAnimation must be ${activity}; ` + + `found ${JSON.stringify(status.activeAnimation)}`, + ); + } + const expectedLevel = activity === "speaking" ? 0.35 : 0; + if ( + typeof status.audioLevel !== "number" || + !Number.isFinite(status.audioLevel) || + Math.abs(status.audioLevel - expectedLevel) > 1e-4 + ) { + throw new Error( + `Pocket audioLevel must be ${expectedLevel}; ` + + `found ${JSON.stringify(status.audioLevel)}`, + ); + } + } +} + export function validatePocketHealthBody(body: JsonObject): number { if (!isJsonObject(body.status)) { throw new Error("Pocket health status must be an object"); @@ -608,9 +669,25 @@ export function validatePocketHealthBody(body: JsonObject): number { ) { throw new Error("Pocket health renderFps must be a finite number > 1"); } + validatePocketFrameCount(body); return status.renderFps; } +export function validatePocketFrameCount(body: JsonObject): number { + if (!isJsonObject(body.status)) { + throw new Error("Pocket health status must be an object"); + } + const frameCount = body.status.renderFrameCount; + if ( + typeof frameCount !== "number" || + !Number.isSafeInteger(frameCount) || + frameCount < 1 + ) { + throw new Error("Pocket health renderFrameCount must be a positive integer"); + } + return frameCount; +} + async function probeHealth( spec: LaunchSpec, child: SpawnedProcess, @@ -1318,6 +1395,7 @@ async function benchmarkTarget( launched_at: new Date().toISOString(), ready_at: null, readiness_health: null, + settled_activity_health: null, reference_cdp: null, frame_receipt: null, settled_at: null, @@ -1331,6 +1409,7 @@ async function benchmarkTarget( activeRun = { target: spec.target, child }; let referenceCdp: CdpClient | null = null; let frameMeasurement: ActiveReferenceFrameMeasurement | null = null; + let previousPocketFrameCount: number | null = null; try { console.error( @@ -1339,6 +1418,9 @@ async function benchmarkTarget( const readiness = await waitForReadyHealth(spec, child); run.ready_at = readiness.captured_at; run.readiness_health = readiness; + if (spec.target === "pocket") { + previousPocketFrameCount = validatePocketFrameCount(readiness.body); + } if (spec.target === "reference") { const prepared = await prepareReferenceCdp(spec, child); referenceCdp = prepared.client; @@ -1348,8 +1430,9 @@ async function benchmarkTarget( `${prepared.receipt.canvas.canvas.backing.join("x")}`, ); } + await driveBenchmarkActivity(spec, child, options.activity); console.error( - `bench-persona: ${spec.target} ready; settling ${options.settleSeconds}s`, + `bench-persona: ${spec.target} ${options.activity}; settling ${options.settleSeconds}s`, ); await delayWhileRunning( child, @@ -1357,6 +1440,17 @@ async function benchmarkTarget( `${spec.target} exited during settle`, ); run.settled_at = new Date().toISOString(); + run.settled_activity_health = await captureRequiredHealth( + spec, + child, + "settled activity", + options.activity, + ); + if (spec.target === "pocket") { + previousPocketFrameCount = validatePocketFrameCount( + run.settled_activity_health.body, + ); + } if (referenceCdp != null) { frameMeasurement = await startReferenceFrameMeasurement( @@ -1387,7 +1481,21 @@ async function benchmarkTarget( spec, child, `sample ${index + 1}`, + options.activity, ); + if (spec.target === "pocket") { + const frameCount = validatePocketFrameCount(health.body); + if ( + previousPocketFrameCount != null && + frameCount <= previousPocketFrameCount + ) { + throw new Error( + `pocket sample ${index + 1} renderFrameCount did not advance: ` + + `${previousPocketFrameCount} -> ${frameCount}`, + ); + } + previousPocketFrameCount = frameCount; + } const sample = buildIntervalSample( index + 1, samplingStart, @@ -1434,6 +1542,67 @@ async function benchmarkTarget( } } +async function driveBenchmarkActivity( + spec: LaunchSpec, + child: SpawnedProcess, + activity: BenchmarkActivity, +): Promise { + await postBenchmarkEvent(spec, child, { + type: "state", + state: { + phase: activity === "speaking" ? "active" : "inactive", + activity, + microphoneMuted: false, + outputMuted: false, + }, + }); + await postBenchmarkEvent(spec, child, { + type: "audio-level", + level: activity === "speaking" ? 0.35 : 0, + }); +} + +async function postBenchmarkEvent( + spec: LaunchSpec, + child: SpawnedProcess, + body: unknown, +): Promise { + const url = `http://127.0.0.1:${spec.bridgePort}/events`; + let lastFailure = "event request did not run"; + for (let attempt = 1; attempt <= 5; attempt++) { + assertProcessAlive(child.pid, `${spec.target} exited before activity setup`); + try { + const response = await fetch(url, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(EVENT_REQUEST_TIMEOUT_MS), + }); + const text = await response.text(); + if (response.status === 202) { + const parsed = JSON.parse(text) as unknown; + if (isJsonObject(parsed) && parsed.accepted === true) return; + lastFailure = `event was not accepted: ${text}`; + } else { + lastFailure = `HTTP ${response.status}: ${text.slice(0, 200)}`; + if (response.status !== 502 && response.status !== 503) break; + } + } catch (error) { + lastFailure = errorMessage(error); + } + if (attempt < 5) { + await delayWhileRunning( + child, + 75, + `${spec.target} exited during activity setup`, + ); + } + } + throw new Error( + `${spec.target} activity event ${JSON.stringify(body)} failed: ${lastFailure}`, + ); +} + async function captureProcessTree(rootPid: number): Promise { const clockMs = performance.now(); const capturedAt = new Date().toISOString(); @@ -1943,6 +2112,7 @@ function parseArgs(argv: string[]): Options { "--bundle", "--max-fps", "--max-texture-dim", + "--activity", "--settle", "--samples", "--interval", @@ -2009,6 +2179,11 @@ function parseArgs(argv: string[]): Options { throw new Error("--max-texture-dim must be an integer > 0"); } + const activityRaw = values.get("--activity") ?? "idle"; + if (activityRaw !== "idle" && activityRaw !== "speaking") { + throw new Error("--activity must be idle or speaking"); + } + const intervalSeconds = numberValue( "--interval", DEFAULT_INTERVAL_SECONDS, @@ -2030,6 +2205,7 @@ function parseArgs(argv: string[]): Options { bundle: absolutePath(required("--bundle")), maxFps, maxTextureDim, + activity: activityRaw, settleSeconds: numberValue( "--settle", DEFAULT_SETTLE_SECONDS, @@ -2174,13 +2350,14 @@ function usage(): string { return `usage: bun scripts/bench-persona.ts \\ --reference-bin PATH --reference-root PATH \\ --pocket-bin PATH --library PATH --bundle PATH \\ - [--max-fps FPS] [--max-texture-dim PIXELS] \\ + [--max-fps FPS] [--max-texture-dim PIXELS] [--activity idle|speaking] \\ [--settle 15] [--samples 13] [--interval 5] [--out PATH] Runs the Electron reference first, terminates its full process tree, then runs the Pocket binary. Each target gets an isolated loopback bridge port and must return {ok:true} from /health within ${READINESS_TIMEOUT_SECONDS}s. Only then -does it settle, capture a cumulative CPU-time baseline, and record resource plus +does it drive the requested idle/speaking activity, settle, capture a +cumulative CPU-time baseline, and record resource plus health samples after each --interval. Reference also requires a non-settings CDP page, validated 430x680 DPR 1.5 WebGL canvas, and a whole-window rAF receipt; Pocket health must report a configured, visible model and renderFps > 1. Use diff --git a/tests/persona-bench.test.ts b/tests/persona-bench.test.ts index 0fc03cc..01b24f8 100644 --- a/tests/persona-bench.test.ts +++ b/tests/persona-bench.test.ts @@ -4,6 +4,7 @@ import { parseCpuTimeSeconds, parseReferenceCanvasReceipt, parseReferenceFrameSample, + validateBenchmarkActivityHealth, validatePocketHealthBody, } from "../scripts/bench-persona"; @@ -16,6 +17,7 @@ describe("Persona benchmark receipts", () => { modelConfigured: true, windowVisible: true, renderFps: 59.8, + renderFrameCount: 123, }, }), ).toBe(59.8); @@ -24,8 +26,61 @@ describe("Persona benchmark receipts", () => { modelConfigured: true, windowVisible: true, renderFps: 59.8, + renderFrameCount: 123, }), ).toThrow("status must be an object"); + expect(() => + validatePocketHealthBody({ + status: { + modelConfigured: true, + windowVisible: true, + renderFps: 59.8, + }, + }), + ).toThrow("renderFrameCount must be a positive integer"); + }); + + test("requires the requested activity to reach both targets", () => { + const speakingState = { + phase: "active", + activity: "speaking", + microphoneMuted: false, + outputMuted: false, + }; + expect(() => + validateBenchmarkActivityHealth( + "reference", + { ok: true, lastState: speakingState }, + "speaking", + ), + ).not.toThrow(); + expect(() => + validateBenchmarkActivityHealth( + "pocket", + { + ok: true, + status: { + voiceState: speakingState, + activeAnimation: "speaking", + audioLevel: 0.349999994, + }, + }, + "speaking", + ), + ).not.toThrow(); + expect(() => + validateBenchmarkActivityHealth( + "pocket", + { + status: { + voiceState: speakingState, + activeAnimation: "idle", + audioLevel: 0, + }, + }, + "speaking", + ), + ).toThrow("activeAnimation must be speaking"); }); test("requires the reference viewport, backing size, and WebGL context", () => { diff --git a/vendor/pocketjs b/vendor/pocketjs index 581aa35..0341db0 160000 --- a/vendor/pocketjs +++ b/vendor/pocketjs @@ -1 +1 @@ -Subproject commit 581aa35c896dc129e1ddf4cb2f0133700d7a5506 +Subproject commit 0341db00a92b9c8a040288a25bd690af7e55fe61