diff --git a/apps/desktop/scripts/electron-launcher.mjs b/apps/desktop/scripts/electron-launcher.mjs index 07fb87b051f..20f535d603a 100644 --- a/apps/desktop/scripts/electron-launcher.mjs +++ b/apps/desktop/scripts/electron-launcher.mjs @@ -228,6 +228,14 @@ function patchMainBundleInfoPlist(appBundlePath, iconPath, executableName) { setPlistString(infoPlistPath, "CFBundleIdentifier", APP_BUNDLE_ID); setPlistString(infoPlistPath, "CFBundleExecutable", executableName); setPlistString(infoPlistPath, "CFBundleIconFile", "icon.icns"); + // Without this key macOS denies every Apple Event with errAEEventNotPermitted + // (-1743) and never shows the Automation prompt, so Codex Computer Use and any + // other MCP server we spawn silently fail to drive other apps. + setPlistString( + infoPlistPath, + "NSAppleEventsUsageDescription", + "This app needs to control other apps to run Computer Use automations you approve.", + ); setPlistJson(infoPlistPath, "CFBundleURLTypes", [ { CFBundleURLName: APP_BUNDLE_ID, diff --git a/apps/desktop/src/computerUse/permissions.test.ts b/apps/desktop/src/computerUse/permissions.test.ts new file mode 100644 index 00000000000..1e11e11be13 --- /dev/null +++ b/apps/desktop/src/computerUse/permissions.test.ts @@ -0,0 +1,99 @@ +// @effect-diagnostics nodeBuiltinImport:off - Fixture reads of Chrome Secure Preferences stay synchronous. +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import { assert, describe, it } from "@effect/vitest"; +import { vi } from "vite-plus/test"; + +vi.mock("electron", () => ({ + systemPreferences: { + isTrustedAccessibilityClient: vi.fn(() => false), + getMediaAccessStatus: vi.fn(() => "denied"), + }, + shell: { + openExternal: vi.fn(async () => undefined), + }, +})); + +import * as Electron from "electron"; + +import { openComputerUsePrivacySettings, readComputerUsePermissions } from "./permissions.ts"; + +describe("computerUse permissions", () => { + it("reports accessibility and screen recording on darwin", () => { + const previous = process.platform; + Object.defineProperty(process, "platform", { value: "darwin" }); + try { + const state = readComputerUsePermissions(); + assert.equal(state.platform, "darwin"); + assert.deepEqual( + state.permissions.map((permission) => permission.kind), + ["accessibility", "screenRecording"], + ); + assert.equal(state.permissions[0]?.status, "denied"); + assert.equal(state.permissions[1]?.status, "denied"); + } finally { + Object.defineProperty(process, "platform", { value: previous }); + } + }); + + it("marks privacy permissions not required off macOS", () => { + const previous = process.platform; + Object.defineProperty(process, "platform", { value: "linux" }); + try { + const state = readComputerUsePermissions(); + assert.equal(state.platform, "linux"); + assert.isTrue(state.permissions.every((permission) => permission.status === "notRequired")); + } finally { + Object.defineProperty(process, "platform", { value: previous }); + } + }); + + it("opens the Accessibility privacy pane and prompts trust", async () => { + const previous = process.platform; + Object.defineProperty(process, "platform", { value: "darwin" }); + try { + const opened = await openComputerUsePrivacySettings("accessibility"); + assert.isTrue(opened); + assert.equal( + vi.mocked(Electron.systemPreferences.isTrustedAccessibilityClient).mock.calls.at(-1)?.[0], + true, + ); + assert.equal( + vi.mocked(Electron.shell.openExternal).mock.calls.at(-1)?.[0], + "x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility", + ); + } finally { + Object.defineProperty(process, "platform", { value: previous }); + } + }); + + it("detects an unpacked extension from Secure Preferences without state", () => { + const previous = process.platform; + Object.defineProperty(process, "platform", { value: "darwin" }); + const home = NodeOS.homedir(); + const securePath = NodePath.join( + home, + "Library/Application Support/Google/Chrome/Default/Secure Preferences", + ); + try { + if (!NodeFS.existsSync(securePath)) return; + const raw = NodeFS.readFileSync(securePath, "utf8"); + const parsed = JSON.parse(raw) as { + extensions?: { settings?: Record }; + }; + const hasExtension = Boolean(parsed.extensions?.settings?.kgdolgnijopbghhomnblabjkmjhnoage); + const state = readComputerUsePermissions(); + if (hasExtension) { + assert.deepEqual(state.chromeExtension, { + status: "installed", + detail: "Browser extension installed", + }); + } else { + assert.isTrue(["missing", "unknown"].includes(state.chromeExtension.status)); + } + } finally { + Object.defineProperty(process, "platform", { value: previous }); + } + }); +}); diff --git a/apps/desktop/src/computerUse/permissions.ts b/apps/desktop/src/computerUse/permissions.ts new file mode 100644 index 00000000000..dff57048464 --- /dev/null +++ b/apps/desktop/src/computerUse/permissions.ts @@ -0,0 +1,288 @@ +// @effect-diagnostics nodeBuiltinImport:off - Sync Chrome preference reads for IPC must stay off the Effect runtime. +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import type { + DesktopChromeExtensionStatus, + DesktopComputerUsePermission, + DesktopComputerUsePermissionsState, + DesktopComputerUsePrivacyPane, +} from "@t3tools/contracts"; +import * as Electron from "electron"; + +const CHROME_EXTENSION_ID = "kgdolgnijopbghhomnblabjkmjhnoage"; + +function platformTag(): DesktopComputerUsePermissionsState["platform"] { + switch (process.platform) { + case "darwin": + return "darwin"; + case "win32": + return "win32"; + case "linux": + return "linux"; + default: + return "other"; + } +} + +function screenRecordingStatus(): DesktopComputerUsePermission["status"] { + try { + const status = Electron.systemPreferences.getMediaAccessStatus("screen"); + switch (status) { + case "granted": + return "granted"; + case "denied": + case "restricted": + return "denied"; + case "not-determined": + return "notDetermined"; + default: + return "unknown"; + } + } catch { + return "unknown"; + } +} + +function accessibilityStatus(): DesktopComputerUsePermission["status"] { + try { + return Electron.systemPreferences.isTrustedAccessibilityClient(false) ? "granted" : "denied"; + } catch { + return "unknown"; + } +} + +function chromeProfileRoots(): string[] { + const home = NodeOS.homedir(); + switch (process.platform) { + case "darwin": + return [ + NodePath.join(home, "Library/Application Support/Google/Chrome"), + NodePath.join(home, "Library/Application Support/Google/Chrome Beta"), + NodePath.join(home, "Library/Application Support/Google/Chrome Canary"), + NodePath.join(home, "Library/Application Support/Chromium"), + ]; + case "win32": { + const local = process.env.LOCALAPPDATA ?? NodePath.join(home, "AppData", "Local"); + return [ + NodePath.join(local, "Google", "Chrome", "User Data"), + NodePath.join(local, "Google", "Chrome Beta", "User Data"), + NodePath.join(local, "Chromium", "User Data"), + ]; + } + default: { + const config = process.env.XDG_CONFIG_HOME ?? NodePath.join(home, ".config"); + return [ + NodePath.join(config, "google-chrome"), + NodePath.join(config, "google-chrome-beta"), + NodePath.join(config, "chromium"), + ]; + } + } +} + +function profileDirectories(root: string): string[] { + try { + return NodeFS.readdirSync(root, { withFileTypes: true }) + .filter((entry) => { + if (!entry.isDirectory()) { + return false; + } + // Chrome uses "Default" plus "Profile N"; also pick up Guest/System if present. + return ( + entry.name === "Default" || + entry.name.startsWith("Profile ") || + entry.name === "Guest Profile" || + entry.name === "System Profile" + ); + }) + .map((entry) => NodePath.join(root, entry.name)); + } catch { + return []; + } +} + +function chromeExtensionInstalledInPreferences(preferencesPath: string): boolean { + try { + const raw = NodeFS.readFileSync(preferencesPath, "utf8"); + const parsed = JSON.parse(raw) as { + extensions?: { + settings?: Record< + string, + { + state?: number; + location?: number; + path?: string; + disable_reasons?: unknown; + was_installed_by_default?: boolean; + } + >; + }; + }; + const entry = parsed.extensions?.settings?.[CHROME_EXTENSION_ID]; + if (!entry) return false; + + // Explicit disable reasons mean it is present but turned off. + if (entry.disable_reasons !== undefined && entry.disable_reasons !== null) { + const reasons = entry.disable_reasons; + if (typeof reasons === "number" && reasons !== 0) return false; + if (typeof reasons === "object" && Object.keys(reasons as object).length > 0) return false; + } + + // Chromium: 0 = disabled, 1 = enabled. Unpacked installs often omit `state` + // in Secure Preferences while still being active (location 4 = UNPACKED). + if (typeof entry.state === "number") return entry.state === 1; + if (typeof entry.path === "string" && entry.path.length > 0) return true; + if (entry.location === 4) return true; + return true; + } catch { + return false; + } +} + +function nativeHostRegistered(root: string): boolean { + const hostPath = NodePath.join(root, "NativeMessagingHosts", "com.t3tools.t3code.desktop.json"); + try { + return NodeFS.statSync(hostPath).isFile(); + } catch { + return false; + } +} + +/** Windows registers the host via the registry + a support-dir manifest. */ +function nativeHostRegisteredWindows(): boolean { + const local = process.env.LOCALAPPDATA; + if (!local) return false; + const hostPath = NodePath.join(local, "t3-desktop-mcp", "com.t3tools.t3code.desktop.json"); + try { + return NodeFS.statSync(hostPath).isFile(); + } catch { + return false; + } +} + +function resolveChromeExtensionStatus(): { + status: DesktopChromeExtensionStatus; + detail: string; +} { + let sawChrome = false; + let hostRegistered = process.platform === "win32" && nativeHostRegisteredWindows(); + + for (const root of chromeProfileRoots()) { + try { + if (!NodeFS.statSync(root).isDirectory()) continue; + } catch { + continue; + } + sawChrome = true; + if (nativeHostRegistered(root)) hostRegistered = true; + + for (const profile of profileDirectories(root)) { + const preferenceFiles = [ + NodePath.join(profile, "Secure Preferences"), + NodePath.join(profile, "Preferences"), + ]; + for (const preferencesPath of preferenceFiles) { + if (chromeExtensionInstalledInPreferences(preferencesPath)) { + return { + status: "installed", + detail: "Browser extension installed", + }; + } + } + } + } + + if (!sawChrome) { + return { + status: "unknown", + detail: "Chrome profile not found on this machine", + }; + } + if (hostRegistered) { + return { + status: "missing", + detail: "Native host registered — load the unpacked extension in chrome://extensions", + }; + } + return { + status: "missing", + detail: "Browser extension not installed", + }; +} + +export function readComputerUsePermissions(): DesktopComputerUsePermissionsState { + const platform = platformTag(); + const chromeExtension = resolveChromeExtensionStatus(); + + if (platform !== "darwin") { + return { + platform, + permissions: [ + { + kind: "accessibility", + status: "notRequired", + label: "Accessibility", + }, + { + kind: "screenRecording", + status: "notRequired", + label: "Screen Recording", + }, + ], + chromeExtension, + }; + } + + return { + platform, + permissions: [ + { + kind: "accessibility", + status: accessibilityStatus(), + label: "Accessibility", + }, + { + kind: "screenRecording", + status: screenRecordingStatus(), + label: "Screen Recording", + }, + ], + chromeExtension, + }; +} + +function privacySettingsUrl(pane: DesktopComputerUsePrivacyPane): string { + // Legacy Security preference pane anchors still open the right list on + // modern macOS and are more reliable than the Settings app deep links. + switch (pane) { + case "accessibility": + return "x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility"; + case "screenRecording": + return "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture"; + } +} + +export async function openComputerUsePrivacySettings( + pane: DesktopComputerUsePrivacyPane, +): Promise { + if (process.platform !== "darwin") { + return false; + } + + // Prompting trust adds this app to the Accessibility list when missing. + if (pane === "accessibility") { + try { + Electron.systemPreferences.isTrustedAccessibilityClient(true); + } catch { + // Still open Settings even if the prompt API fails. + } + } + + try { + await Electron.shell.openExternal(privacySettingsUrl(pane)); + return true; + } catch { + return false; + } +} diff --git a/apps/desktop/src/ipc/DesktopIpcHandlers.ts b/apps/desktop/src/ipc/DesktopIpcHandlers.ts index 37fd873a1b0..777de1ce65d 100644 --- a/apps/desktop/src/ipc/DesktopIpcHandlers.ts +++ b/apps/desktop/src/ipc/DesktopIpcHandlers.ts @@ -13,6 +13,10 @@ import { setServerExposureMode, setTailscaleServeEnabled, } from "./methods/serverExposure.ts"; +import { + getComputerUsePermissions, + openComputerUsePrivacySettings, +} from "./methods/computerUse.ts"; import { bootstrapSshBearerSession, disconnectSshEnvironment, @@ -87,6 +91,8 @@ export const installDesktopIpcHandlers = Effect.fn("desktop.ipc.installHandlers" yield* ipc.handle(showContextMenu); yield* ipc.handle(openExternal); yield* ipc.handle(probeRemoteEditors); + yield* ipc.handle(getComputerUsePermissions); + yield* ipc.handle(openComputerUsePrivacySettings); yield* ipc.handle(getUpdateState); yield* ipc.handle(setUpdateChannel); yield* ipc.handle(downloadUpdate); diff --git a/apps/desktop/src/ipc/channels.ts b/apps/desktop/src/ipc/channels.ts index 02f9ad0df36..4c9605f9720 100644 --- a/apps/desktop/src/ipc/channels.ts +++ b/apps/desktop/src/ipc/channels.ts @@ -4,6 +4,9 @@ export const SET_THEME_CHANNEL = "desktop:set-theme"; export const CONTEXT_MENU_CHANNEL = "desktop:context-menu"; export const OPEN_EXTERNAL_CHANNEL = "desktop:open-external"; export const PROBE_REMOTE_EDITORS_CHANNEL = "desktop:probe-remote-editors"; +export const GET_COMPUTER_USE_PERMISSIONS_CHANNEL = "desktop:get-computer-use-permissions"; +export const OPEN_COMPUTER_USE_PRIVACY_SETTINGS_CHANNEL = + "desktop:open-computer-use-privacy-settings"; export const MENU_ACTION_CHANNEL = "desktop:menu-action"; export const QUIT_SHORTCUT_CHANNEL = "desktop:quit-shortcut"; export const GET_WINDOW_FULLSCREEN_STATE_CHANNEL = "desktop:get-window-fullscreen-state"; diff --git a/apps/desktop/src/ipc/methods/computerUse.ts b/apps/desktop/src/ipc/methods/computerUse.ts new file mode 100644 index 00000000000..64fa5eb3228 --- /dev/null +++ b/apps/desktop/src/ipc/methods/computerUse.ts @@ -0,0 +1,31 @@ +import { + DesktopComputerUsePermissionsStateSchema, + DesktopComputerUsePrivacyPaneSchema, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; + +import { + openComputerUsePrivacySettings as openPrivacySettings, + readComputerUsePermissions, +} from "../../computerUse/permissions.ts"; +import * as IpcChannels from "../channels.ts"; +import * as DesktopIpc from "../DesktopIpc.ts"; + +export const getComputerUsePermissions = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.GET_COMPUTER_USE_PERMISSIONS_CHANNEL, + payload: Schema.Undefined, + result: DesktopComputerUsePermissionsStateSchema, + handler: Effect.fn("desktop.ipc.computerUse.getComputerUsePermissions")(function* () { + return readComputerUsePermissions(); + }), +}); + +export const openComputerUsePrivacySettings = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.OPEN_COMPUTER_USE_PRIVACY_SETTINGS_CHANNEL, + payload: DesktopComputerUsePrivacyPaneSchema, + result: Schema.Boolean, + handler: Effect.fn("desktop.ipc.computerUse.openComputerUsePrivacySettings")(function* (pane) { + return yield* Effect.promise(() => openPrivacySettings(pane)); + }), +}); diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index 6741e392239..af774879603 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -110,6 +110,10 @@ contextBridge.exposeInMainWorld("desktopBridge", { }), openExternal: (url: string) => ipcRenderer.invoke(IpcChannels.OPEN_EXTERNAL_CHANNEL, url), probeRemoteEditors: () => ipcRenderer.invoke(IpcChannels.PROBE_REMOTE_EDITORS_CHANNEL, undefined), + getComputerUsePermissions: () => + ipcRenderer.invoke(IpcChannels.GET_COMPUTER_USE_PERMISSIONS_CHANNEL, undefined), + openComputerUsePrivacySettings: (pane) => + ipcRenderer.invoke(IpcChannels.OPEN_COMPUTER_USE_PRIVACY_SETTINGS_CHANNEL, pane), onMenuAction: (listener) => { const wrappedListener = (_event: Electron.IpcRendererEvent, action: unknown) => { if (typeof action !== "string") return; diff --git a/apps/server/src/desktopControl/desktopMcpBinary.test.ts b/apps/server/src/desktopControl/desktopMcpBinary.test.ts new file mode 100644 index 00000000000..826a137e63a --- /dev/null +++ b/apps/server/src/desktopControl/desktopMcpBinary.test.ts @@ -0,0 +1,103 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { HostProcessEnvironment, HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import { assert, describe, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; + +import { resolveDesktopMcpPath } from "./desktopMcpBinary.ts"; + +describe("desktopMcpBinary", () => { + it.effect("resolves the override path on macOS", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const baseDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-desktop-mcp-binary-", + }); + const binaryPath = `${baseDir}/t3-desktop-mcp`; + yield* fileSystem.writeFileString(binaryPath, "binary"); + yield* fileSystem.chmod(binaryPath, 0o755); + + const resolved = yield* resolveDesktopMcpPath().pipe( + Effect.provideService(HostProcessPlatform, "darwin"), + Effect.provideService(HostProcessEnvironment, { + T3CODE_DESKTOP_MCP_PATH: binaryPath, + }), + ); + + assert.equal(resolved, binaryPath); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("resolves the override on Linux and Windows too", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const baseDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-desktop-mcp-binary-", + }); + + // Windows ships the .exe; the other platforms do not. + for (const [platform, name] of [ + ["linux", "t3-desktop-mcp"], + ["win32", "t3-desktop-mcp.exe"], + ] as const) { + const binaryPath = `${baseDir}/${name}`; + yield* fileSystem.writeFileString(binaryPath, "binary"); + if (platform !== "win32") { + yield* fileSystem.chmod(binaryPath, 0o755); + } + + const resolved = yield* resolveDesktopMcpPath().pipe( + Effect.provideService(HostProcessPlatform, platform), + Effect.provideService(HostProcessEnvironment, { + T3CODE_DESKTOP_MCP_PATH: binaryPath, + }), + ); + assert.equal(resolved, binaryPath); + } + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("returns undefined on platforms with no desktop backend", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const baseDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-desktop-mcp-binary-", + }); + const binaryPath = `${baseDir}/t3-desktop-mcp`; + yield* fileSystem.writeFileString(binaryPath, "binary"); + + // Neither backend covers these, so the tools must not be offered even + // when someone points the override at a binary. + for (const platform of ["freebsd", "aix"] as const) { + const resolved = yield* resolveDesktopMcpPath().pipe( + Effect.provideService(HostProcessPlatform, platform), + Effect.provideService(HostProcessEnvironment, { + T3CODE_DESKTOP_MCP_PATH: binaryPath, + }), + ); + assert.equal(resolved, undefined); + } + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("returns undefined when nothing is built or overridden", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const baseDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-desktop-mcp-binary-", + }); + + const resolved = yield* resolveDesktopMcpPath().pipe( + Effect.provideService(HostProcessPlatform, "darwin"), + Effect.provideService(HostProcessEnvironment, { + T3CODE_DESKTOP_MCP_PATH: `${baseDir}/does-not-exist`, + }), + ); + + // A dev checkout that has built the binary will resolve a bundled + // candidate; otherwise nothing matches. Both are valid — the contract is + // that a missing override never throws and never returns the bad path. + assert.notEqual(resolved, `${baseDir}/does-not-exist`); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); +}); diff --git a/apps/server/src/desktopControl/desktopMcpBinary.ts b/apps/server/src/desktopControl/desktopMcpBinary.ts new file mode 100644 index 00000000000..4284aea20b1 --- /dev/null +++ b/apps/server/src/desktopControl/desktopMcpBinary.ts @@ -0,0 +1,96 @@ +import { HostProcessEnvironment, HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; + +export const DESKTOP_MCP_EXECUTABLE_NAME = "t3-desktop-mcp"; + +/** + * Locate the bundled desktop-control MCP server. + * + * macOS ships the Swift package in `native/t3-desktop-mcp`. Windows and Linux + * ship the Rust crate in `native/t3-desktop-mcp-rs`. Candidate lists are + * platform-specific so a stray Rust binary on macOS (or Swift on Windows) is + * never preferred over the functional backend. Resolves to undefined when the + * binary is absent — callers treat that as "do not offer the tools". + */ +export const resolveDesktopMcpPath = Effect.fn("desktopControl.resolveDesktopMcpPath")( + function* () { + const platform = yield* HostProcessPlatform; + if (platform !== "darwin" && platform !== "win32" && platform !== "linux") { + return undefined; + } + + const environment = yield* HostProcessEnvironment; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + + const override = environment.T3CODE_DESKTOP_MCP_PATH; + // Windows keeps the extension; the staged directory name does not. + const executableName = + platform === "win32" ? `${DESKTOP_MCP_EXECUTABLE_NAME}.exe` : DESKTOP_MCP_EXECUTABLE_NAME; + + const packaged = [ + // Packaged: staged into app Resources beside the server bundle. + path.resolve(import.meta.dirname, DESKTOP_MCP_EXECUTABLE_NAME, executableName), + path.resolve(import.meta.dirname, "..", DESKTOP_MCP_EXECUTABLE_NAME, executableName), + ]; + + const rustDev = [ + path.resolve( + import.meta.dirname, + "../../../../native/t3-desktop-mcp-rs/target/release", + executableName, + ), + path.resolve( + import.meta.dirname, + "../../../native/t3-desktop-mcp-rs/target/release", + executableName, + ), + ]; + + const swiftDev = [ + path.resolve( + import.meta.dirname, + "../../../../native/t3-desktop-mcp/.build/apple/Products/Release", + DESKTOP_MCP_EXECUTABLE_NAME, + ), + path.resolve( + import.meta.dirname, + "../../../../native/t3-desktop-mcp/.build/release", + DESKTOP_MCP_EXECUTABLE_NAME, + ), + path.resolve( + import.meta.dirname, + "../../../native/t3-desktop-mcp/.build/apple/Products/Release", + DESKTOP_MCP_EXECUTABLE_NAME, + ), + ]; + + const candidates = [ + ...(override ? [override] : []), + ...packaged, + ...(platform === "darwin" ? swiftDev : rustDev), + ]; + + for (const candidate of candidates) { + const exists = yield* fileSystem.exists(candidate).pipe(Effect.orElseSucceed(() => false)); + if (!exists) { + continue; + } + const stat = yield* fileSystem.stat(candidate).pipe(Effect.option); + if (Option.isNone(stat) || stat.value.type !== "File") { + continue; + } + // Windows does not use POSIX execute bits the same way; existence of a + // regular file is enough. On POSIX, skip non-executable paths so a bad + // override does not block a valid packaged binary. + if (platform !== "win32" && (stat.value.mode & 0o111) === 0) { + continue; + } + return candidate; + } + return undefined; + }, +); diff --git a/apps/server/src/desktopControl/desktopMcpLaunch.test.ts b/apps/server/src/desktopControl/desktopMcpLaunch.test.ts new file mode 100644 index 00000000000..efdf38ab583 --- /dev/null +++ b/apps/server/src/desktopControl/desktopMcpLaunch.test.ts @@ -0,0 +1,105 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { HostProcessEnvironment, HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import { ServerSettingsError } from "@t3tools/contracts"; +import { assert, describe, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Stream from "effect/Stream"; + +import * as ServerSettings from "../serverSettings.ts"; +import { resolveEnabledDesktopMcp } from "./desktopMcpLaunch.ts"; + +describe("resolveEnabledDesktopMcp", () => { + it.effect("omits the MCP when Computer Use is disabled", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const baseDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-desktop-mcp-enabled-", + }); + const binaryPath = `${baseDir}/t3-desktop-mcp`; + yield* fileSystem.writeFileString(binaryPath, "binary"); + yield* fileSystem.chmod(binaryPath, 0o755); + + const resolved = yield* resolveEnabledDesktopMcp().pipe( + Effect.provideService(HostProcessPlatform, "darwin"), + Effect.provideService(HostProcessEnvironment, { + T3CODE_DESKTOP_MCP_PATH: binaryPath, + }), + Effect.provide(ServerSettings.layerTest({ desktopControl: { enabled: false } })), + ); + + assert.equal(resolved, undefined); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("passes env when agent cursor or browser control is off", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const baseDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-desktop-mcp-enabled-", + }); + const binaryPath = `${baseDir}/t3-desktop-mcp`; + yield* fileSystem.writeFileString(binaryPath, "binary"); + yield* fileSystem.chmod(binaryPath, 0o755); + + const resolved = yield* resolveEnabledDesktopMcp().pipe( + Effect.provideService(HostProcessPlatform, "darwin"), + Effect.provideService(HostProcessEnvironment, { + T3CODE_DESKTOP_MCP_PATH: binaryPath, + }), + Effect.provide( + ServerSettings.layerTest({ + desktopControl: { + enabled: true, + agentCursorEnabled: false, + browserControlEnabled: false, + }, + }), + ), + ); + + assert.isDefined(resolved); + assert.equal(resolved?.path, binaryPath); + assert.deepEqual(resolved?.env, [ + { name: "T3_DESKTOP_AGENT_CURSOR", value: "0" }, + { name: "T3_DESKTOP_BROWSER", value: "0" }, + ]); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("omits the MCP when settings cannot be read", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const baseDir = yield* fileSystem.makeTempDirectoryScoped({ + prefix: "t3-desktop-mcp-enabled-", + }); + const binaryPath = `${baseDir}/t3-desktop-mcp`; + yield* fileSystem.writeFileString(binaryPath, "binary"); + yield* fileSystem.chmod(binaryPath, 0o755); + + const settingsError = new ServerSettingsError({ + settingsPath: `${baseDir}/settings.json`, + operation: "read-file", + cause: "boom", + }); + const failingService = { + start: Effect.void, + ready: Effect.void, + getSettings: Effect.fail(settingsError), + updateSettings: () => Effect.fail(settingsError), + streamChanges: Stream.empty, + subscribeChanges: Effect.succeed(Stream.empty), + }; + + const resolved = yield* resolveEnabledDesktopMcp().pipe( + Effect.provideService(HostProcessPlatform, "darwin"), + Effect.provideService(HostProcessEnvironment, { + T3CODE_DESKTOP_MCP_PATH: binaryPath, + }), + Effect.provideService(ServerSettings.ServerSettingsService, failingService as never), + ); + + assert.equal(resolved, undefined); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); +}); diff --git a/apps/server/src/desktopControl/desktopMcpLaunch.ts b/apps/server/src/desktopControl/desktopMcpLaunch.ts new file mode 100644 index 00000000000..dba5627b813 --- /dev/null +++ b/apps/server/src/desktopControl/desktopMcpLaunch.ts @@ -0,0 +1,90 @@ +/** + * Resolves the desktop MCP binary only when Computer Use is enabled in + * server settings. Settings lookup failures fail closed (tools omitted). + */ +import { HostProcessEnvironment, HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; + +import * as ServerSettings from "../serverSettings.ts"; +import { resolveDesktopMcpPath } from "./desktopMcpBinary.ts"; + +export type DesktopMcpLaunch = { + readonly path: string; + readonly env: ReadonlyArray<{ readonly name: string; readonly value: string }>; +}; + +type DesktopControlFlags = { + readonly enabled: boolean; + readonly agentCursorEnabled: boolean; + readonly browserControlEnabled: boolean; +}; + +const disabledDesktopControl = { + enabled: false, + agentCursorEnabled: false, + browserControlEnabled: false, +} as const satisfies DesktopControlFlags; + +/** + * Always acquire `ServerSettings.ServerSettingsService` from the Effect + * environment. Callers that need R=never session methods should use + * `makeResolveEnabledDesktopMcp` instead of yielding this effect directly. + */ +const readDesktopControlFlags = Effect.fn("desktopControl.readDesktopControlFlags")(function* () { + const settings = yield* ServerSettings.ServerSettingsService; + return yield* settings.getSettings.pipe( + Effect.map((snapshot): DesktopControlFlags => snapshot.desktopControl), + // Fail closed: never inject desktop MCP when we cannot confirm enablement. + Effect.orElseSucceed((): DesktopControlFlags => disabledDesktopControl), + ); +}); + +export const resolveEnabledDesktopMcp = Effect.fn("desktopControl.resolveEnabledDesktopMcp")( + function* () { + const path = yield* resolveDesktopMcpPath(); + if (path === undefined) { + return undefined; + } + + const desktopControl = yield* readDesktopControlFlags(); + if (!desktopControl.enabled) { + return undefined; + } + + const env: Array<{ name: string; value: string }> = []; + if (!desktopControl.agentCursorEnabled) { + env.push({ name: "T3_DESKTOP_AGENT_CURSOR", value: "0" }); + } + if (!desktopControl.browserControlEnabled) { + env.push({ name: "T3_DESKTOP_BROWSER", value: "0" }); + } + + return { path, env } satisfies DesktopMcpLaunch; + }, +); + +/** + * Capture desktop-MCP resolution dependencies once at adapter construction so + * each session can re-read settings without widening `startSession`'s Effect + * context (adapters require `R = never` on session methods). + */ +export const makeResolveEnabledDesktopMcp = Effect.fn( + "desktopControl.makeResolveEnabledDesktopMcp", +)(function* () { + const settings = yield* ServerSettings.ServerSettingsService; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const platform = yield* HostProcessPlatform; + const environment = yield* HostProcessEnvironment; + + return () => + resolveEnabledDesktopMcp().pipe( + Effect.provideService(ServerSettings.ServerSettingsService, settings), + Effect.provideService(FileSystem.FileSystem, fileSystem), + Effect.provideService(Path.Path, path), + Effect.provideService(HostProcessPlatform, platform), + Effect.provideService(HostProcessEnvironment, environment), + ); +}); diff --git a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.approval.test.ts b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.approval.test.ts index 05370781c0d..6e9e2fcceba 100644 --- a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.approval.test.ts +++ b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.approval.test.ts @@ -30,4 +30,52 @@ describe("runtimeEventToActivities approval details", () => { expect(activity?.kind).toBe("approval.requested"); expect((activity?.payload as Record | undefined)?.detail).toBe(detail); }); + + it("preserves permission approvals for the client", () => { + const event = { + type: "request.opened", + eventId: EventId.make("evt-permissions-opened"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-08-10T00:00:00.000Z", + threadId: ThreadId.make("thread-1"), + requestId: RuntimeRequestId.make("permissions-1"), + payload: { + requestType: "permissions_approval", + detail: "Control the desktop", + }, + } satisfies ProviderRuntimeEvent; + + const [activity] = runtimeEventToActivities(event); + + expect(activity?.summary).toBe("Permission requested"); + expect((activity?.payload as Record | undefined)?.requestKind).toBe( + "permissions", + ); + expect((activity?.payload as Record | undefined)?.requestType).toBe( + "permissions_approval", + ); + }); + + it("preserves generic MCP tool approvals for the client", () => { + const event = { + type: "request.opened", + eventId: EventId.make("evt-tool-opened"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-08-10T00:00:00.000Z", + threadId: ThreadId.make("thread-1"), + requestId: RuntimeRequestId.make("tool-1"), + payload: { + requestType: "tool_approval", + detail: "Allow node_repl to run this tool call?", + }, + } satisfies ProviderRuntimeEvent; + + const [activity] = runtimeEventToActivities(event); + + expect(activity?.summary).toBe("Tool approval requested"); + expect((activity?.payload as Record | undefined)?.requestKind).toBe("tool"); + expect((activity?.payload as Record | undefined)?.requestType).toBe( + "tool_approval", + ); + }); }); diff --git a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts index e24825d3d1f..973471a726d 100644 --- a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts +++ b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts @@ -298,7 +298,7 @@ function sessionStatusAllowsActiveTurn( function requestKindFromCanonicalRequestType( requestType: string | undefined, -): "command" | "file-read" | "file-change" | undefined { +): "command" | "file-read" | "file-change" | "tool" | "permissions" | undefined { switch (requestType) { case "command_execution_approval": case "exec_command_approval": @@ -308,6 +308,10 @@ function requestKindFromCanonicalRequestType( case "file_change_approval": case "apply_patch_approval": return "file-change"; + case "permissions_approval": + return "permissions"; + case "tool_approval": + return "tool"; default: return undefined; } @@ -388,7 +392,11 @@ export function runtimeEventToActivities( ? "File-read approval requested" : requestKind === "file-change" ? "File-change approval requested" - : "Approval requested", + : requestKind === "tool" + ? "Tool approval requested" + : requestKind === "permissions" + ? "Permission requested" + : "Approval requested", payload: { requestId: toApprovalRequestId(event.requestId), ...(requestKind ? { requestKind } : {}), diff --git a/apps/server/src/provider/Layers/ClaudeAdapter.ts b/apps/server/src/provider/Layers/ClaudeAdapter.ts index 5715b68a1e4..62afb8ae2ec 100644 --- a/apps/server/src/provider/Layers/ClaudeAdapter.ts +++ b/apps/server/src/provider/Layers/ClaudeAdapter.ts @@ -74,6 +74,7 @@ import * as Stream from "effect/Stream"; import { resolveAttachmentPath } from "../../attachmentStore.ts"; import { ServerConfig } from "../../config.ts"; +import { makeResolveEnabledDesktopMcp } from "../../desktopControl/desktopMcpLaunch.ts"; import * as McpProviderSession from "../../mcp/McpProviderSession.ts"; import { resolveClaudeSdkExecutablePath } from "../Drivers/ClaudeExecutable.ts"; import { makeClaudeEnvironment } from "../Drivers/ClaudeHome.ts"; @@ -1676,6 +1677,8 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( : undefined); const managedNativeEventLogger = options?.nativeEventLogger === undefined ? nativeEventLogger : undefined; + // Re-resolve per session (settings toggles) without widening startSession's R. + const resolveDesktopMcp = yield* makeResolveEnabledDesktopMcp(); const createQuery = options?.createQuery ?? @@ -4151,6 +4154,38 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( ...(input.cwd ? [input.cwd] : []), serverConfig.attachmentsDir, ]; + // Desktop MCP is offered when the platform binary resolves and Computer + // Use is enabled. Re-resolve per session so Settings toggles apply without + // restarting the app. + const desktopMcp = yield* resolveDesktopMcp(); + const mcpServers = { + ...(mcpSession + ? { + "t3-code": { + type: "http" as const, + url: mcpSession.endpoint, + headers: { + Authorization: mcpSession.authorizationHeader, + }, + }, + } + : {}), + ...(desktopMcp + ? { + "t3-desktop": { + type: "stdio" as const, + command: desktopMcp.path, + ...(desktopMcp.env.length > 0 + ? { + env: Object.fromEntries( + desktopMcp.env.map((entry) => [entry.name, entry.value]), + ), + } + : {}), + }, + } + : {}), + }; const queryOptions: ClaudeQueryOptions = { ...(input.cwd ? { cwd: input.cwd } : {}), ...(apiModelId ? { model: apiModelId } : {}), @@ -4176,19 +4211,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( env: claudeEnvironment, additionalDirectories, ...(Object.keys(extraArgs).length > 0 ? { extraArgs } : {}), - ...(mcpSession - ? { - mcpServers: { - "t3-code": { - type: "http", - url: mcpSession.endpoint, - headers: { - Authorization: mcpSession.authorizationHeader, - }, - }, - }, - } - : {}), + ...(Object.keys(mcpServers).length > 0 ? { mcpServers } : {}), }; yield* Effect.annotateCurrentSpan({ diff --git a/apps/server/src/provider/Layers/CodexAdapter.test.ts b/apps/server/src/provider/Layers/CodexAdapter.test.ts index 5358716aabe..87836725d6a 100644 --- a/apps/server/src/provider/Layers/CodexAdapter.test.ts +++ b/apps/server/src/provider/Layers/CodexAdapter.test.ts @@ -936,6 +936,129 @@ lifecycleLayer("CodexAdapterLive lifecycle", (it) => { }), ); + it.effect("maps Computer Use permission requests and resolutions to canonical approvals", () => + Effect.gen(function* () { + const { adapter, runtime } = yield* startLifecycleRuntime(); + const eventsFiber = yield* Stream.runCollect(Stream.take(adapter.streamEvents, 2)).pipe( + Effect.forkChild, + ); + + yield* runtime.emit({ + id: asEventId("evt-permissions-requested"), + kind: "request", + provider: ProviderDriverKind.make("codex"), + threadId: asThreadId("thread-1"), + turnId: asTurnId("turn-1"), + itemId: asItemId("permissions-1"), + createdAt: "2026-01-01T00:00:00.000Z", + method: "mcpServer/elicitation/request", + requestKind: "permissions", + requestId: ApprovalRequestId.make("req-permissions-1"), + payload: { + _meta: { + codex_approval_kind: "mcp_tool_call", + connector_id: "computer-use", + }, + message: "Control the desktop", + mode: "form", + requestedSchema: { type: "object", properties: {} }, + serverName: "computer-use", + threadId: "provider-thread-1", + turnId: "turn-1", + }, + } satisfies ProviderEvent); + yield* runtime.emit({ + id: asEventId("evt-permissions-resolved"), + kind: "notification", + provider: ProviderDriverKind.make("codex"), + threadId: asThreadId("thread-1"), + turnId: asTurnId("turn-1"), + itemId: asItemId("permissions-1"), + createdAt: "2026-01-01T00:00:01.000Z", + method: "item/requestApproval/decision", + requestKind: "permissions", + requestId: ApprovalRequestId.make("req-permissions-1"), + payload: { + requestId: "req-permissions-1", + requestKind: "permissions", + decision: "accept", + }, + } satisfies ProviderEvent); + + const events = Array.from(yield* Fiber.join(eventsFiber)); + NodeAssert.equal(events[0]?.type, "request.opened"); + if (events[0]?.type === "request.opened") { + NodeAssert.equal(events[0].payload.requestType, "permissions_approval"); + NodeAssert.equal(events[0].payload.detail, "Control the desktop"); + } + NodeAssert.equal(events[1]?.type, "request.resolved"); + if (events[1]?.type === "request.resolved") { + NodeAssert.equal(events[1].payload.requestType, "permissions_approval"); + NodeAssert.equal(events[1].payload.decision, "accept"); + } + }), + ); + + it.effect("maps generic MCP tool requests and resolutions to canonical tool approvals", () => + Effect.gen(function* () { + const { adapter, runtime } = yield* startLifecycleRuntime(); + const eventsFiber = yield* Stream.runCollect(Stream.take(adapter.streamEvents, 2)).pipe( + Effect.forkChild, + ); + + yield* runtime.emit({ + id: asEventId("evt-tool-requested"), + kind: "request", + provider: ProviderDriverKind.make("codex"), + threadId: asThreadId("thread-1"), + turnId: asTurnId("turn-1"), + createdAt: "2026-01-01T00:00:00.000Z", + method: "mcpServer/elicitation/request", + requestKind: "tool", + requestId: ApprovalRequestId.make("req-tool-1"), + payload: { + _meta: { + codex_approval_kind: "mcp_tool_call", + }, + message: "Allow node_repl to run this tool call?", + mode: "form", + requestedSchema: { type: "object", properties: {} }, + serverName: "node_repl", + threadId: "provider-thread-1", + turnId: "turn-1", + }, + } satisfies ProviderEvent); + yield* runtime.emit({ + id: asEventId("evt-tool-resolved"), + kind: "notification", + provider: ProviderDriverKind.make("codex"), + threadId: asThreadId("thread-1"), + turnId: asTurnId("turn-1"), + createdAt: "2026-01-01T00:00:01.000Z", + method: "item/requestApproval/decision", + requestKind: "tool", + requestId: ApprovalRequestId.make("req-tool-1"), + payload: { + requestId: "req-tool-1", + requestKind: "tool", + decision: "accept", + }, + } satisfies ProviderEvent); + + const events = Array.from(yield* Fiber.join(eventsFiber)); + NodeAssert.equal(events[0]?.type, "request.opened"); + if (events[0]?.type === "request.opened") { + NodeAssert.equal(events[0].payload.requestType, "tool_approval"); + NodeAssert.equal(events[0].payload.detail, "Allow node_repl to run this tool call?"); + } + NodeAssert.equal(events[1]?.type, "request.resolved"); + if (events[1]?.type === "request.resolved") { + NodeAssert.equal(events[1].payload.requestType, "tool_approval"); + NodeAssert.equal(events[1].payload.decision, "accept"); + } + }), + ); + it.effect("preserves explicit empty multi-select user-input answers", () => Effect.gen(function* () { const { adapter, runtime } = yield* startLifecycleRuntime(); diff --git a/apps/server/src/provider/Layers/CodexAdapter.ts b/apps/server/src/provider/Layers/CodexAdapter.ts index 065156d3647..371d0a5a253 100644 --- a/apps/server/src/provider/Layers/CodexAdapter.ts +++ b/apps/server/src/provider/Layers/CodexAdapter.ts @@ -41,6 +41,7 @@ import * as EffectCodexSchema from "effect-codex-app-server/schema"; import { getModelSelectionStringOptionValue } from "@t3tools/shared/model"; import { getCodexServiceTierOptionValue } from "../../codexModelOptions.ts"; +import { makeResolveEnabledDesktopMcp } from "../../desktopControl/desktopMcpLaunch.ts"; import * as McpProviderSession from "../../mcp/McpProviderSession.ts"; import { @@ -302,6 +303,8 @@ function toRequestTypeFromMethod(method: string): CanonicalRequestType { return "file_read_approval"; case "item/fileChange/requestApproval": return "file_change_approval"; + case "item/permissions/requestApproval": + return "permissions_approval"; case "applyPatchApproval": return "apply_patch_approval"; case "execCommandApproval": @@ -325,6 +328,10 @@ function toRequestTypeFromKind(kind: ProviderRequestKind | undefined): Canonical return "file_read_approval"; case "file-change": return "file_change_approval"; + case "permissions": + return "permissions_approval"; + case "tool": + return "tool_approval"; default: return "unknown"; } @@ -819,6 +826,20 @@ function mapToRuntimeEvents( ); return payload?.reason ?? undefined; } + case "item/permissions/requestApproval": { + const payload = readPayload( + EffectCodexSchema.ServerRequest__PermissionsRequestApprovalParams, + event.payload, + ); + return payload?.reason ?? undefined; + } + case "mcpServer/elicitation/request": { + const payload = readPayload( + EffectCodexSchema.ServerRequest__McpServerElicitationRequestParams, + event.payload, + ); + return payload?.message ?? undefined; + } case "applyPatchApproval": { const payload = readPayload( EffectCodexSchema.ServerRequest__ApplyPatchApprovalParams, @@ -850,7 +871,9 @@ function mapToRuntimeEvents( ...runtimeEventBase(event, canonicalThreadId), type: "request.opened", payload: { - requestType: toRequestTypeFromMethod(event.method), + requestType: event.requestKind + ? toRequestTypeFromKind(event.requestKind) + : toRequestTypeFromMethod(event.method), ...(detail ? { detail } : {}), ...(event.payload !== undefined ? { args: event.payload } : {}), }, @@ -1626,6 +1649,7 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( options?: CodexAdapterLiveOptions, ) { const boundInstanceId = options?.instanceId ?? ProviderInstanceId.make("codex"); + const resolveDesktopMcp = yield* makeResolveEnabledDesktopMcp(); const fileSystem = yield* FileSystem.FileSystem; const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; const crypto = yield* Crypto.Crypto; @@ -1663,6 +1687,40 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( ? getCodexServiceTierOptionValue(input.modelSelection) : undefined; const mcpSession = McpProviderSession.readMcpProviderSession(input.threadId); + // Offer Computer Use tools whenever the desktop MCP resolves and is + // enabled — same as Claude/Cursor/Grok. Wire via Codex `-c mcp_servers.*`. + const desktopMcp = yield* resolveDesktopMcp(); + const appServerArgs: string[] = []; + if (mcpSession) { + appServerArgs.push( + "-c", + `mcp_servers.t3-code.url=${mcpSession.endpoint}`, + "-c", + 'mcp_servers.t3-code.bearer_token_env_var="T3_MCP_BEARER_TOKEN"', + ); + } + if (desktopMcp) { + // Prefer TOML literal strings so Windows paths keep backslashes; + // fall back to escaped basic strings when a value contains `'`. + const quoteToml = (value: string): string => { + if (!value.includes("'") && !value.includes("\n") && !value.includes("\r")) { + return `'${value}'`; + } + return `"${value + .replace(/\\/g, "\\\\") + .replace(/"/g, '\\"') + .replace(/\n/g, "\\n") + .replace(/\r/g, "\\r") + .replace(/\t/g, "\\t")}"`; + }; + appServerArgs.push("-c", `mcp_servers.t3-desktop.command=${quoteToml(desktopMcp.path)}`); + for (const entry of desktopMcp.env) { + appServerArgs.push( + "-c", + `mcp_servers.t3-desktop.env.${entry.name}=${quoteToml(entry.value)}`, + ); + } + } const runtimeInput: CodexSessionRuntimeOptions = { threadId: input.threadId, providerInstanceId: boundInstanceId, @@ -1679,18 +1737,20 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( ? { model: input.modelSelection.model } : {}), ...(serviceTier ? { serviceTier } : {}), - ...(mcpSession + ...(mcpSession || desktopMcp ? { environment: { ...(options?.environment ?? process.env), - T3_MCP_BEARER_TOKEN: mcpSession.authorizationHeader.replace(/^Bearer\s+/, ""), + ...(mcpSession + ? { + T3_MCP_BEARER_TOKEN: mcpSession.authorizationHeader.replace( + /^Bearer\s+/, + "", + ), + } + : {}), }, - appServerArgs: [ - "-c", - `mcp_servers.t3-code.url=${mcpSession.endpoint}`, - "-c", - 'mcp_servers.t3-code.bearer_token_env_var="T3_MCP_BEARER_TOKEN"', - ], + appServerArgs, } : {}), }; diff --git a/apps/server/src/provider/Layers/CodexProvider.test.ts b/apps/server/src/provider/Layers/CodexProvider.test.ts index 26e77f82a79..3f0843fc700 100644 --- a/apps/server/src/provider/Layers/CodexProvider.test.ts +++ b/apps/server/src/provider/Layers/CodexProvider.test.ts @@ -2,10 +2,18 @@ import { assert, it } from "@effect/vitest"; import { applyPreferredCodexDefaultModel, + buildCodexInitializeParams, isLegacyCodexModel, mapCodexModelCapabilities, } from "./CodexProvider.ts"; +it("advertises form elicitation support to Codex App Server sessions", () => { + assert.deepStrictEqual(buildCodexInitializeParams().capabilities, { + experimentalApi: true, + mcpServerOpenaiFormElicitation: true, + }); +}); + it("keeps only the GPT-5.6 Codex family out of legacy models", () => { assert.deepStrictEqual( ["gpt-5.6-luna", "gpt-5.6-terra", "gpt-5.6-sol", "gpt-5.4"].map((model) => [ diff --git a/apps/server/src/provider/Layers/CodexProvider.ts b/apps/server/src/provider/Layers/CodexProvider.ts index 5c0f76dff4e..7d694bf7c0c 100644 --- a/apps/server/src/provider/Layers/CodexProvider.ts +++ b/apps/server/src/provider/Layers/CodexProvider.ts @@ -315,6 +315,7 @@ export function buildCodexInitializeParams(): CodexSchema.V1InitializeParams { }, capabilities: { experimentalApi: true, + mcpServerOpenaiFormElicitation: true, }, }; } diff --git a/apps/server/src/provider/Layers/CodexSessionRuntime.test.ts b/apps/server/src/provider/Layers/CodexSessionRuntime.test.ts index 2a2de683920..890726ee376 100644 --- a/apps/server/src/provider/Layers/CodexSessionRuntime.test.ts +++ b/apps/server/src/provider/Layers/CodexSessionRuntime.test.ts @@ -16,10 +16,18 @@ import { } from "../CodexDeveloperInstructions.ts"; import { codexSessionAppServerArgs } from "./codexLaunchArgs.ts"; import { + buildMcpApprovalResponse, + buildPermissionsApprovalResponse, buildTurnStartParams, hasConfiguredMcpServer, + isComputerUseMcpApproval, + isMcpToolApproval, isRecoverableThreadResumeError, +<<<<<<< HEAD makeMemoryConsolidationNotificationFilter, +======= + mcpApprovalRequestKind, +>>>>>>> a279d4081 (feat(codex): surface Computer Use and MCP tool approvals; Windows native bridge) openCodexThread, } from "./CodexSessionRuntime.ts"; const isCodexAppServerRequestError = Schema.is(CodexErrors.CodexAppServerRequestError); @@ -248,6 +256,116 @@ describe("buildTurnStartParams", () => { }); }); +describe("buildPermissionsApprovalResponse", () => { + const permissions = { + network: { enabled: true }, + fileSystem: { + entries: [{ access: "write" as const, path: { type: "path" as const, path: "/tmp" } }], + }, + }; + + it("grants the requested execution context for this turn", () => { + NodeAssert.deepStrictEqual(buildPermissionsApprovalResponse(permissions, "accept"), { + permissions, + scope: "turn", + }); + }); + + it("persists an accepted execution context only for acceptForSession", () => { + NodeAssert.deepStrictEqual(buildPermissionsApprovalResponse(permissions, "acceptForSession"), { + permissions, + scope: "session", + }); + }); + + it("denies every requested capability on decline or cancellation", () => { + for (const decision of ["decline", "cancel"] as const) { + NodeAssert.deepStrictEqual(buildPermissionsApprovalResponse(permissions, decision), { + permissions: {}, + scope: "turn", + }); + } + }); +}); + +describe("MCP tool approval", () => { + const request = { + _meta: { + codex_approval_kind: "mcp_tool_call", + connector_id: "computer-use", + persist: ["session", "always"], + }, + message: "Allow Computer Use to control this desktop?", + mode: "form" as const, + requestedSchema: { type: "object" as const, properties: {} }, + serverName: "computer-use", + threadId: "provider-thread-1", + turnId: "turn-1", + }; + + it("recognizes only the Computer Use connector approval", () => { + NodeAssert.equal(isComputerUseMcpApproval(request), true); + NodeAssert.equal( + isComputerUseMcpApproval({ + ...request, + _meta: { ...request._meta, connector_id: "calendar" }, + }), + false, + ); + }); + + it("recognizes generic MCP tool guardian approvals without a connector id", () => { + const genericRequest = { + ...request, + _meta: { codex_approval_kind: "mcp_tool_call" as const }, + message: "Allow node_repl to run this tool call?", + serverName: "node_repl", + }; + + NodeAssert.equal(isMcpToolApproval(genericRequest), true); + NodeAssert.equal(isComputerUseMcpApproval(genericRequest), false); + NodeAssert.equal(mcpApprovalRequestKind(genericRequest), "tool"); + NodeAssert.equal(mcpApprovalRequestKind(request), "permissions"); + }); + + it("does not recognize URL or unrelated form elicitations as MCP tool approvals", () => { + NodeAssert.equal( + isMcpToolApproval({ + ...request, + mode: "url", + url: "https://example.com/approve", + elicitationId: "elicitation-1", + }), + false, + ); + const unrelatedRequest = { + ...request, + _meta: { connector_id: "computer-use" }, + }; + NodeAssert.equal(isMcpToolApproval(unrelatedRequest), false); + NodeAssert.equal(mcpApprovalRequestKind(unrelatedRequest), undefined); + NodeAssert.equal( + mcpApprovalRequestKind({ + ...request, + mode: "url", + url: "https://example.com/approve", + elicitationId: "elicitation-1", + }), + undefined, + ); + }); + + it("maps approval decisions to MCP actions and session persistence", () => { + NodeAssert.deepStrictEqual(buildMcpApprovalResponse("accept"), { action: "accept" }); + NodeAssert.deepStrictEqual(buildMcpApprovalResponse("acceptForSession"), { + action: "accept", + _meta: { persist: "session" }, + }); + NodeAssert.deepStrictEqual(buildMcpApprovalResponse("decline"), { action: "decline" }); + NodeAssert.deepStrictEqual(buildMcpApprovalResponse("cancel"), { action: "cancel" }); + }); +}); + describe("buildCodexDeveloperInstructions", () => { it("appends runtime info after the mode instructions", () => { const instructions = buildCodexDeveloperInstructions("default", { diff --git a/apps/server/src/provider/Layers/CodexSessionRuntime.ts b/apps/server/src/provider/Layers/CodexSessionRuntime.ts index 8f887c1b592..88516810852 100644 --- a/apps/server/src/provider/Layers/CodexSessionRuntime.ts +++ b/apps/server/src/provider/Layers/CodexSessionRuntime.ts @@ -60,6 +60,15 @@ const RECOVERABLE_THREAD_RESUME_ERROR_SNIPPETS = [ "does not exist", "no rollout found", ]; +const ComputerUseMcpApprovalMeta = Schema.Struct({ + codex_approval_kind: Schema.Literal("mcp_tool_call"), + connector_id: Schema.Literal("computer-use"), +}); +const McpToolApprovalMeta = Schema.Struct({ + codex_approval_kind: Schema.Literal("mcp_tool_call"), +}); +const isComputerUseApprovalMeta = Schema.is(ComputerUseMcpApprovalMeta); +const isMcpToolApprovalMeta = Schema.is(McpToolApprovalMeta); export function hasConfiguredMcpServer(appServerArgs: ReadonlyArray | undefined): boolean { return appServerArgs?.some((argument) => argument.includes("mcp_servers.")) === true; @@ -119,6 +128,8 @@ export interface CodexSessionRuntimeSendTurnInput { readonly serviceTier?: CodexServiceTier | undefined; readonly effort?: EffectCodexSchema.V2TurnStartParams__ReasoningEffort | undefined; readonly interactionMode?: ProviderInteractionMode; + /** Preloaded Computer History context from the Effect-owned loader. */ + readonly computerHistoryContext?: string; } export interface CodexThreadTurnSnapshot { @@ -154,6 +165,50 @@ export interface CodexSessionRuntimeShape { readonly close: Effect.Effect; } +export function buildPermissionsApprovalResponse( + permissions: EffectCodexSchema.PermissionsRequestApprovalParams["permissions"], + decision: ProviderApprovalDecision, +): EffectCodexSchema.PermissionsRequestApprovalResponse { + return { + permissions: decision === "accept" || decision === "acceptForSession" ? permissions : {}, + scope: decision === "acceptForSession" ? "session" : "turn", + }; +} + +export function isComputerUseMcpApproval( + payload: EffectCodexSchema.McpServerElicitationRequestParams, +): boolean { + return payload.mode !== "url" && isComputerUseApprovalMeta(payload._meta); +} + +export function isMcpToolApproval( + payload: EffectCodexSchema.McpServerElicitationRequestParams, +): boolean { + return payload.mode !== "url" && isMcpToolApprovalMeta(payload._meta); +} + +export function mcpApprovalRequestKind( + payload: EffectCodexSchema.McpServerElicitationRequestParams, +): ProviderRequestKind | undefined { + if (!isMcpToolApproval(payload)) return undefined; + return isComputerUseMcpApproval(payload) ? "permissions" : "tool"; +} + +export function buildMcpApprovalResponse( + decision: ProviderApprovalDecision, +): EffectCodexSchema.McpServerElicitationRequestResponse { + switch (decision) { + case "accept": + return { action: "accept" }; + case "acceptForSession": + return { action: "accept", _meta: { persist: "session" } }; + case "decline": + return { action: "decline" }; + case "cancel": + return { action: "cancel" }; + } +} + export type CodexSessionRuntimeError = | CodexErrors.CodexAppServerError | CodexSessionRuntimePendingApprovalNotFoundError @@ -340,21 +395,31 @@ function buildCodexCollaborationMode(input: { readonly interactionMode?: ProviderInteractionMode; readonly model?: string; readonly effort?: EffectCodexSchema.V2TurnStartParams__ReasoningEffort; + readonly computerHistoryContext?: string; }): EffectCodexSchema.V2TurnStartParams__CollaborationMode | undefined { - if (input.interactionMode === undefined) { + // Mode-less turns still need Computer History when present — otherwise ordinary + // sendTurn calls silently drop the loaded context. + if (input.interactionMode === undefined && !input.computerHistoryContext) { return undefined; } + const interactionMode = input.interactionMode ?? "default"; const model = normalizeCodexModelSlug(input.model) ?? DEFAULT_MODEL; const reasoningEffort = input.effort ?? "medium"; return { - mode: input.interactionMode, + mode: interactionMode, settings: { model, reasoning_effort: reasoningEffort, - developer_instructions: buildCodexDeveloperInstructions(input.interactionMode, { - model, - reasoningEffort, - }), + developer_instructions: buildCodexDeveloperInstructions( + interactionMode, + { + model, + reasoningEffort, + }, + input.computerHistoryContext + ? { computerHistoryContext: input.computerHistoryContext } + : undefined, + ), }, }; } @@ -371,6 +436,7 @@ export function buildTurnStartParams(input: { readonly serviceTier?: CodexServiceTier; readonly effort?: EffectCodexSchema.V2TurnStartParams__ReasoningEffort; readonly interactionMode?: ProviderInteractionMode; + readonly computerHistoryContext?: string; }): Effect.Effect< CodexTurnStartParamsWithCollaborationMode, CodexErrors.CodexAppServerProtocolParseError @@ -391,6 +457,9 @@ export function buildTurnStartParams(input: { ...(input.interactionMode ? { interactionMode: input.interactionMode } : {}), ...(input.model ? { model: input.model } : {}), ...(input.effort ? { effort: input.effort } : {}), + ...(input.computerHistoryContext + ? { computerHistoryContext: input.computerHistoryContext } + : {}), }); return decodeCodexTurnStartParamsWithCollaborationMode({ @@ -1599,6 +1668,127 @@ export const makeCodexSessionRuntime = ( }), ); + yield* client.handleServerRequest("item/permissions/requestApproval", (payload) => + Effect.gen(function* () { + const requestId = ApprovalRequestId.make( + yield* randomUUIDv4("permissions-approval-request"), + ); + const turnId = TurnId.make(payload.turnId); + const itemId = ProviderItemId.make(payload.itemId); + const decision = yield* Deferred.make(); + + yield* Ref.update(pendingApprovalsRef, (current) => { + const next = new Map(current); + next.set(requestId, { + requestId, + jsonRpcId: payload.itemId, + requestKind: "permissions", + turnId, + itemId, + decision, + }); + return next; + }); + yield* Ref.update(approvalCorrelationsRef, (current) => { + const next = new Map(current); + next.set(payload.itemId, { + requestId, + requestKind: "permissions", + turnId, + itemId, + }); + return next; + }); + + yield* emitEvent({ + kind: "request", + threadId: options.threadId, + method: "item/permissions/requestApproval", + requestId, + requestKind: "permissions", + turnId, + itemId, + payload, + }); + + const resolved = yield* Deferred.await(decision).pipe( + Effect.ensuring( + Ref.update(pendingApprovalsRef, (current) => { + const next = new Map(current); + next.delete(requestId); + return next; + }), + ), + ); + return buildPermissionsApprovalResponse(payload.permissions, resolved); + }), + ); + + yield* client.handleServerRequest("mcpServer/elicitation/request", (payload) => + Effect.gen(function* () { + const requestKind = mcpApprovalRequestKind(payload); + if (!requestKind) { + // We advertise form elicitation support at initialize. Unrecognized + // forms (and URL elicitations) must be answered, not methodNotFound, + // or Codex can stall the turn. + return { action: "decline" } as const; + } + + const requestId = ApprovalRequestId.make(yield* randomUUIDv4("mcp-approval-request")); + const turnId = payload.turnId ? TurnId.make(payload.turnId) : undefined; + const decision = yield* Deferred.make(); + // Concurrent elicitations share a serverName; correlate by the JSON-RPC + // request id (fiber-local) so serverRequest/resolved cannot collide. + const incomingRequestId = yield* CodexClient.CurrentServerRequestId; + const correlationKey = + incomingRequestId !== undefined ? String(incomingRequestId) : String(requestId); + + yield* Ref.update(pendingApprovalsRef, (current) => { + const next = new Map(current); + next.set(requestId, { + requestId, + jsonRpcId: correlationKey, + requestKind, + turnId, + itemId: undefined, + decision, + }); + return next; + }); + yield* Ref.update(approvalCorrelationsRef, (current) => { + const next = new Map(current); + next.set(correlationKey, { + requestId, + requestKind, + turnId, + itemId: undefined, + }); + return next; + }); + + yield* emitEvent({ + kind: "request", + threadId: options.threadId, + method: "mcpServer/elicitation/request", + requestId, + requestKind, + ...(turnId ? { turnId } : {}), + payload, + }); + + const resolved = yield* Deferred.await(decision).pipe( + Effect.ensuring( + Ref.update(pendingApprovalsRef, (current) => { + const next = new Map(current); + next.delete(requestId); + return next; + }), + ), + ); + return buildMcpApprovalResponse(resolved); + }), + ); + yield* client.handleServerRequest("item/tool/requestUserInput", (payload) => Effect.gen(function* () { const requestId = ApprovalRequestId.make(yield* randomUUIDv4("user-input-request")); @@ -1813,6 +2003,7 @@ export const makeCodexSessionRuntime = ( const normalizedModel = normalizeCodexModelSlug( input.model ?? (yield* Ref.get(sessionRef)).model, ); + const computerHistoryContext = input.computerHistoryContext; const params = yield* buildTurnStartParams({ threadId: providerThreadId, runtimeMode: options.runtimeMode, @@ -1822,6 +2013,7 @@ export const makeCodexSessionRuntime = ( ...(input.serviceTier ? { serviceTier: input.serviceTier } : {}), ...(input.effort ? { effort: input.effort } : {}), ...(input.interactionMode ? { interactionMode: input.interactionMode } : {}), + ...(computerHistoryContext ? { computerHistoryContext } : {}), }); const rawResponse = yield* client.raw.request("turn/start", params); const response = yield* decodeV2TurnStartResponse(rawResponse).pipe( diff --git a/apps/server/src/provider/Layers/CursorAdapter.ts b/apps/server/src/provider/Layers/CursorAdapter.ts index 30c173d8fae..39c109354ca 100644 --- a/apps/server/src/provider/Layers/CursorAdapter.ts +++ b/apps/server/src/provider/Layers/CursorAdapter.ts @@ -42,6 +42,7 @@ import type * as EffectAcpSchema from "effect-acp/schema"; import { resolveAttachmentPath } from "../../attachmentStore.ts"; import { ServerConfig } from "../../config.ts"; +import { makeResolveEnabledDesktopMcp } from "../../desktopControl/desktopMcpLaunch.ts"; import * as McpProviderSession from "../../mcp/McpProviderSession.ts"; import { ProviderAdapterProcessError, @@ -316,6 +317,7 @@ export function makeCursorAdapter( ) { return Effect.gen(function* () { const boundInstanceId = options?.instanceId ?? ProviderInstanceId.make("cursor"); + const resolveDesktopMcp = yield* makeResolveEnabledDesktopMcp(); const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; @@ -532,6 +534,37 @@ export function makeCursorAdapter( : cursorSettings; const mcpSession = McpProviderSession.readMcpProviderSession(input.threadId); + // Desktop MCP whenever the binary resolves and Computer Use is enabled. + // Re-resolve per session so Settings toggles apply without restart. + // Do not gate on the HTTP t3-code MCP session (fresh Cursor threads). + const desktopMcp = yield* resolveDesktopMcp(); + const mcpServers = [ + ...(mcpSession + ? [ + { + type: "http" as const, + name: "t3-code", + url: mcpSession.endpoint, + headers: [ + { + name: "Authorization", + value: mcpSession.authorizationHeader, + }, + ], + }, + ] + : []), + ...(desktopMcp + ? [ + { + name: "t3-desktop", + command: desktopMcp.path, + args: [] as string[], + env: [...desktopMcp.env], + }, + ] + : []), + ]; const acp = yield* makeCursorAcpRuntime({ cursorSettings: effectiveCursorSettings, ...(options?.environment ? { environment: options.environment } : {}), @@ -539,23 +572,7 @@ export function makeCursorAdapter( cwd, ...(resumeSessionId ? { resumeSessionId } : {}), clientInfo: { name: "t3-code", version: "0.0.0" }, - ...(mcpSession - ? { - mcpServers: [ - { - type: "http" as const, - name: "t3-code", - url: mcpSession.endpoint, - headers: [ - { - name: "Authorization", - value: mcpSession.authorizationHeader, - }, - ], - }, - ], - } - : {}), + ...(mcpServers.length > 0 ? { mcpServers } : {}), ...acpNativeLoggers, }).pipe( Effect.provideService(Crypto.Crypto, crypto), diff --git a/apps/server/src/provider/Layers/GrokAdapter.test.ts b/apps/server/src/provider/Layers/GrokAdapter.test.ts index 6cb71660a74..e5abfa3aa69 100644 --- a/apps/server/src/provider/Layers/GrokAdapter.test.ts +++ b/apps/server/src/provider/Layers/GrokAdapter.test.ts @@ -26,6 +26,7 @@ import { } from "@t3tools/contracts"; import { ServerConfig } from "../../config.ts"; +import { ServerSettingsService } from "../../serverSettings.ts"; import { grokPromptSettlementBelongsToContext, makeGrokAdapter } from "./GrokAdapter.ts"; const decodeGrokSettings = Schema.decodeSync(GrokSettings); @@ -84,7 +85,10 @@ async function readJsonLines(filePath: string) { const grokAdapterTestLayer = ServerConfig.layerTest(process.cwd(), { prefix: "t3code-grok-adapter-test-", -}).pipe(Layer.provideMerge(NodeServices.layer)); +}).pipe( + Layer.provideMerge(ServerSettingsService.layerTest()), + Layer.provideMerge(NodeServices.layer), +); const makeTestAdapter = (binaryPath: string, options?: Parameters[1]) => makeGrokAdapter(decodeGrokSettings({ binaryPath }), options).pipe(Effect.orDie); diff --git a/apps/server/src/provider/Layers/GrokAdapter.ts b/apps/server/src/provider/Layers/GrokAdapter.ts index 858d862e6d5..c8eb55a9e8d 100644 --- a/apps/server/src/provider/Layers/GrokAdapter.ts +++ b/apps/server/src/provider/Layers/GrokAdapter.ts @@ -34,6 +34,7 @@ import type * as EffectAcpSchema from "effect-acp/schema"; import { resolveAttachmentPath } from "../../attachmentStore.ts"; import { ServerConfig } from "../../config.ts"; +import { makeResolveEnabledDesktopMcp } from "../../desktopControl/desktopMcpLaunch.ts"; import * as McpProviderSession from "../../mcp/McpProviderSession.ts"; import { ProviderAdapterProcessError, @@ -227,6 +228,7 @@ export function grokPromptSettlementBelongsToContext(input: { export function makeGrokAdapter(grokSettings: GrokSettings, options?: GrokAdapterLiveOptions) { return Effect.gen(function* () { const boundInstanceId = options?.instanceId ?? ProviderInstanceId.make("grok"); + const resolveDesktopMcp = yield* makeResolveEnabledDesktopMcp(); const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; @@ -570,6 +572,37 @@ export function makeGrokAdapter(grokSettings: GrokSettings, options?: GrokAdapte }); const mcpSession = McpProviderSession.readMcpProviderSession(input.threadId); + // Desktop MCP whenever the binary resolves and Computer Use is enabled. + // Re-resolve per session so Settings toggles apply without restart. + // Do not gate on the HTTP t3-code MCP session (fresh Grok threads). + const desktopMcp = yield* resolveDesktopMcp(); + const mcpServers = [ + ...(mcpSession + ? [ + { + type: "http" as const, + name: "t3-code", + url: mcpSession.endpoint, + headers: [ + { + name: "Authorization", + value: mcpSession.authorizationHeader, + }, + ], + }, + ] + : []), + ...(desktopMcp + ? [ + { + name: "t3-desktop", + command: desktopMcp.path, + args: [] as string[], + env: [...desktopMcp.env], + }, + ] + : []), + ]; const acp = yield* makeGrokAcpRuntime({ grokSettings, ...(options?.environment ? { environment: options.environment } : {}), @@ -577,23 +610,7 @@ export function makeGrokAdapter(grokSettings: GrokSettings, options?: GrokAdapte cwd, ...(resumeSessionId ? { resumeSessionId } : {}), clientInfo: { name: "t3-code", version: "0.0.0" }, - ...(mcpSession - ? { - mcpServers: [ - { - type: "http" as const, - name: "t3-code", - url: mcpSession.endpoint, - headers: [ - { - name: "Authorization", - value: mcpSession.authorizationHeader, - }, - ], - }, - ], - } - : {}), + ...(mcpServers.length > 0 ? { mcpServers } : {}), ...acpNativeLoggers, }).pipe( Effect.provideService(Crypto.Crypto, crypto), diff --git a/apps/web/src/assets/computer-use/agent-cursor-badge.png b/apps/web/src/assets/computer-use/agent-cursor-badge.png new file mode 100644 index 00000000000..e972c52d9b2 Binary files /dev/null and b/apps/web/src/assets/computer-use/agent-cursor-badge.png differ diff --git a/apps/web/src/assets/computer-use/agent-cursor-icon.png b/apps/web/src/assets/computer-use/agent-cursor-icon.png new file mode 100644 index 00000000000..e972c52d9b2 Binary files /dev/null and b/apps/web/src/assets/computer-use/agent-cursor-icon.png differ diff --git a/apps/web/src/assets/computer-use/brave.svg b/apps/web/src/assets/computer-use/brave.svg new file mode 100644 index 00000000000..e192b2d839a --- /dev/null +++ b/apps/web/src/assets/computer-use/brave.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/apps/web/src/assets/computer-use/edge.svg b/apps/web/src/assets/computer-use/edge.svg new file mode 100644 index 00000000000..82159a6adbc --- /dev/null +++ b/apps/web/src/assets/computer-use/edge.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/apps/web/src/assets/computer-use/firefox.svg b/apps/web/src/assets/computer-use/firefox.svg new file mode 100644 index 00000000000..6c2333f5148 --- /dev/null +++ b/apps/web/src/assets/computer-use/firefox.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/apps/web/src/assets/computer-use/google-chrome.svg b/apps/web/src/assets/computer-use/google-chrome.svg new file mode 100644 index 00000000000..e053d777e94 --- /dev/null +++ b/apps/web/src/assets/computer-use/google-chrome.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/apps/web/src/components/chat/ComposerPendingApprovalPanel.test.tsx b/apps/web/src/components/chat/ComposerPendingApprovalPanel.test.tsx index de67505a55e..8cdee28b46e 100644 --- a/apps/web/src/components/chat/ComposerPendingApprovalPanel.test.tsx +++ b/apps/web/src/components/chat/ComposerPendingApprovalPanel.test.tsx @@ -28,4 +28,39 @@ describe("ComposerPendingApprovalPanel", () => { expect(markup).toContain("max-w-full"); expect(markup).toContain("[overflow-wrap:anywhere]"); }); + + it("labels permission requests clearly", () => { + const markup = renderToStaticMarkup( + , + ); + + expect(markup).toContain("Permission requested"); + expect(markup).toContain('aria-label="Requested permissions"'); + }); + + it("distinguishes generic tool approvals from permission requests", () => { + const markup = renderToStaticMarkup( + , + ); + + expect(markup).toContain("Tool approval requested"); + expect(markup).toContain('aria-label="Tool request"'); + expect(markup).not.toContain("Permission requested"); + }); }); diff --git a/apps/web/src/components/chat/ComposerPendingApprovalPanel.tsx b/apps/web/src/components/chat/ComposerPendingApprovalPanel.tsx index 391546355b2..9bb57867b12 100644 --- a/apps/web/src/components/chat/ComposerPendingApprovalPanel.tsx +++ b/apps/web/src/components/chat/ComposerPendingApprovalPanel.tsx @@ -15,13 +15,21 @@ export const ComposerPendingApprovalPanel = memo(function ComposerPendingApprova ? "Command approval requested" : approval.requestKind === "file-read" ? "File-read approval requested" - : "File-change approval requested"; + : approval.requestKind === "file-change" + ? "File-change approval requested" + : approval.requestKind === "tool" + ? "Tool approval requested" + : "Permission requested"; const detailLabel = approval.requestKind === "command" ? "Command" : approval.requestKind === "file-read" ? "File to read" - : "File change"; + : approval.requestKind === "file-change" + ? "File change" + : approval.requestKind === "tool" + ? "Tool request" + : "Requested permissions"; return (
diff --git a/apps/web/src/components/chat/MessagesTimeline.tsx b/apps/web/src/components/chat/MessagesTimeline.tsx index c90aa771f8d..d9088d045f9 100644 --- a/apps/web/src/components/chat/MessagesTimeline.tsx +++ b/apps/web/src/components/chat/MessagesTimeline.tsx @@ -58,6 +58,7 @@ import { MousePointerClickIcon, PaintbrushIcon, MinusIcon, + ShieldIcon, SquarePenIcon, TerminalIcon, Undo2Icon, @@ -1947,6 +1948,7 @@ type WorkEntryIconName = | "globe" | "hammer" | "message-circle" + | "shield" | "square-pen" | "terminal" | "wrench" @@ -1969,6 +1971,8 @@ function WorkEntryIconSvg({ name, className }: { name: WorkEntryIconName; classN return ; case "message-circle": return ; + case "shield": + return ; case "square-pen": return ; case "terminal": @@ -2076,6 +2080,8 @@ function workEntryIconName(workEntry: TimelineWorkEntry): WorkEntryIconName { if (workEntry.requestKind === "command") return "terminal"; if (workEntry.requestKind === "file-read") return "eye"; if (workEntry.requestKind === "file-change") return "square-pen"; + if (workEntry.requestKind === "tool") return "wrench"; + if (workEntry.requestKind === "permissions") return "shield"; if (workEntry.itemType === "command_execution" || workEntry.command) { return "terminal"; diff --git a/apps/web/src/components/settings/ComputerUseMoreBrowsers.tsx b/apps/web/src/components/settings/ComputerUseMoreBrowsers.tsx new file mode 100644 index 00000000000..ee434b50e7a --- /dev/null +++ b/apps/web/src/components/settings/ComputerUseMoreBrowsers.tsx @@ -0,0 +1,60 @@ +import { useState } from "react"; + +import { Button } from "../ui/button"; +import { Collapsible, CollapsiblePanel, CollapsibleTrigger } from "../ui/collapsible"; +import { BraveIcon, EdgeIcon, FirefoxIcon } from "./browserBrandIcons"; +import { SettingRowTitle, SettingsRow } from "./settingsLayout"; + +/** Chromium/Firefox setup disclosure for Computer Use settings. */ +export function ComputerUseMoreBrowsers() { + const [open, setOpen] = useState(false); + + return ( + + }>More browsers + } + description="Set up the same extension in other Chromium browsers." + control={ + } + aria-controls="computer-use-more-browsers" + > + {open ? "Hide" : "Show"} + + } + > + +
+
+ +
+

Microsoft Edge

+

+ Load the unpacked extension from edge://extensions +

+
+
+
+ +
+

Brave

+

+ Same extension via brave://extensions +

+
+
+
+ +
+

Firefox

+

Not supported yet

+
+
+
+
+
+
+ ); +} diff --git a/apps/web/src/components/settings/ComputerUseSettings.tsx b/apps/web/src/components/settings/ComputerUseSettings.tsx new file mode 100644 index 00000000000..1bfb45fa58d --- /dev/null +++ b/apps/web/src/components/settings/ComputerUseSettings.tsx @@ -0,0 +1,465 @@ +import { MonitorIcon } from "lucide-react"; +import { type ReactNode, useCallback, useEffect, useState } from "react"; +import type { + DesktopComputerUsePermission, + DesktopComputerUsePermissionsState, + DesktopComputerUsePrivacyPane, +} from "@t3tools/contracts"; +import { DEFAULT_UNIFIED_SETTINGS } from "@t3tools/contracts/settings"; + +import { usePrimarySettings, useUpdatePrimarySettings } from "../../hooks/useSettings"; +import { cn } from "../../lib/utils"; +import { Button } from "../ui/button"; +import { + Dialog, + DialogClose, + DialogDescription, + DialogFooter, + DialogHeader, + DialogPanel, + DialogPopup, + DialogTitle, +} from "../ui/dialog"; +import { Switch } from "../ui/switch"; +import { AgentCursorIcon, ChromeIcon } from "./browserBrandIcons"; +import { ComputerUseMoreBrowsers } from "./ComputerUseMoreBrowsers"; +import { + SettingResetButton, + SettingRowTitle, + SettingsPageContainer, + SettingsRow, + SettingsSection, +} from "./settingsLayout"; +import { searchableSetting } from "./settingsSearch"; + +/** Shown when the desktop host lacks the Computer Use permissions bridge API. */ +const BRIDGE_UNSUPPORTED_MESSAGE = "Update T3 Code to check Computer Use permissions"; + +function isDesktopHost(): boolean { + return typeof window !== "undefined" && window.desktopBridge !== undefined; +} + +function isBridgeSupported(): boolean { + return ( + typeof window !== "undefined" && window.desktopBridge?.getComputerUsePermissions !== undefined + ); +} + +function ExtensionStatus({ + tone, + children, +}: { + tone: "ok" | "warn" | "muted"; + children: ReactNode; +}) { + return ( + + + {children} + + ); +} + +function permissionTone(status: DesktopComputerUsePermission["status"]): "ok" | "warn" | "muted" { + if (status === "granted" || status === "notRequired") return "ok"; + if (status === "denied" || status === "notDetermined") return "warn"; + return "muted"; +} + +function permissionLabel(status: DesktopComputerUsePermission["status"]): string { + switch (status) { + case "granted": + return "Granted"; + case "denied": + return "Not granted"; + case "notDetermined": + return "Not determined"; + case "notRequired": + return "Not required on this platform"; + case "unknown": + return "Unknown"; + } +} + +export function ComputerUseSettings() { + const settings = usePrimarySettings(); + const updateSettings = useUpdatePrimarySettings(); + const desktop = settings.desktopControl; + const defaults = DEFAULT_UNIFIED_SETTINGS.desktopControl; + const onDesktop = isDesktopHost(); + const [manageOpen, setManageOpen] = useState(false); + const [permissionPrompt, setPermissionPrompt] = useState( + null, + ); + const [permState, setPermState] = useState(null); + const [permError, setPermError] = useState(null); + + const refreshPermissions = useCallback(async () => { + const bridge = window.desktopBridge; + if (!bridge?.getComputerUsePermissions) { + setPermState(null); + setPermError(BRIDGE_UNSUPPORTED_MESSAGE); + return; + } + try { + const next = await bridge.getComputerUsePermissions(); + setPermState(next); + setPermError(null); + } catch (error) { + setPermError(error instanceof Error ? error.message : "Could not read permissions"); + } + }, []); + + useEffect(() => { + if (!onDesktop) return; + const bridgeSupported = isBridgeSupported(); + if (!bridgeSupported) { + setPermError(BRIDGE_UNSUPPORTED_MESSAGE); + return; + } + void refreshPermissions(); + const onFocus = () => void refreshPermissions(); + window.addEventListener("focus", onFocus); + const id = window.setInterval(() => void refreshPermissions(), 4000); + return () => { + window.removeEventListener("focus", onFocus); + window.clearInterval(id); + }; + }, [onDesktop, refreshPermissions]); + + const openPrivacyPane = async (pane: DesktopComputerUsePrivacyPane): Promise => { + const bridge = window.desktopBridge; + if (!bridge?.openComputerUsePrivacySettings) { + setPermError(BRIDGE_UNSUPPORTED_MESSAGE); + return false; + } + try { + const opened = await bridge.openComputerUsePrivacySettings(pane); + if (!opened) { + setPermError("Could not open System Settings. Open Privacy & Security manually."); + return false; + } + window.setTimeout(() => void refreshPermissions(), 1500); + return true; + } catch { + setPermError("Could not open System Settings. Open Privacy & Security manually."); + return false; + } + }; + + const chromeStatus = permState?.chromeExtension; + const needsMacPrivacy = permState?.platform === "darwin"; + const bridgeSupported = onDesktop && isBridgeSupported(); + + return ( + + + {!onDesktop ? ( +
+ +

+ You are connected to a remote environment. Computer Use settings apply on the host + running the T3 Code desktop app. +

+
+ ) : null} + + + updateSettings({ + desktopControl: { ...desktop, enabled: defaults.enabled }, + }) + } + /> + ) : null + } + control={ + + updateSettings({ + desktopControl: { ...desktop, enabled: Boolean(checked) }, + }) + } + aria-label="Enable Computer Use" + /> + } + /> + + }> + {searchableSetting("computer-use-agent-cursor").title} + + } + description="Show the agent pointer overlay while it works, without moving your mouse." + resetAction={ + desktop.agentCursorEnabled !== defaults.agentCursorEnabled ? ( + + updateSettings({ + desktopControl: { + ...desktop, + agentCursorEnabled: defaults.agentCursorEnabled, + }, + }) + } + /> + ) : null + } + control={ + + updateSettings({ + desktopControl: { + ...desktop, + agentCursorEnabled: Boolean(checked), + }, + }) + } + aria-label="Show agent cursor overlay" + /> + } + /> + + }> + {searchableSetting("computer-use-browser").title} + + } + description="Drive an agent-owned tab group in your signed-in Chrome." + status={ + chromeStatus ? ( + + {chromeStatus.detail} + + ) : onDesktop ? ( + + {bridgeSupported ? "Checking extension…" : BRIDGE_UNSUPPORTED_MESSAGE} + + ) : null + } + resetAction={ + desktop.browserControlEnabled !== defaults.browserControlEnabled ? ( + + updateSettings({ + desktopControl: { + ...desktop, + browserControlEnabled: defaults.browserControlEnabled, + }, + }) + } + /> + ) : null + } + control={ +
+ + + updateSettings({ + desktopControl: { + ...desktop, + browserControlEnabled: Boolean(checked), + }, + }) + } + aria-label="Enable Google Chrome browser control" + /> +
+ } + /> + + +
+ + {onDesktop ? ( + + {(permState?.permissions ?? []).map((permission) => { + const canOpen = + needsMacPrivacy && + permission.status !== "granted" && + permission.status !== "notRequired"; + const search = + permission.kind === "accessibility" + ? searchableSetting("computer-use-accessibility") + : searchableSetting("computer-use-screen-recording"); + return ( + + {permissionLabel(permission.status)} + + } + control={ + canOpen ? ( + + ) : null + } + /> + ); + })} + {permError ?

{permError}

: null} + {!permState && !permError ? ( +

Checking permissions…

+ ) : null} + {needsMacPrivacy ? ( +

+ After enabling a permission in System Settings, return here — status refreshes + automatically. +

+ ) : null} +
+ ) : null} + + + + + Google Chrome + + Load the T3 Code extension so agents can open and drive tabs in a labelled group + without taking over your browsing. + + + +
    +
  1. + Open chrome://extensions in + Chrome and turn on Developer mode. +
  2. +
  3. + Choose Load unpacked and select + the{" "} + + native/t3-chrome-extension + {" "} + folder from this repo (or the copy bundled with the desktop app). +
  4. +
  5. + Confirm the extension id is{" "} + + kgdolgnijopbghhomnblabjkmjhnoage + + . +
  6. +
+ {chromeStatus ? ( + + {chromeStatus.detail} + + ) : null} +
+ + }>Close + +
+
+ + { + if (!open) setPermissionPrompt(null); + }} + > + + + Allow {permissionPrompt?.label ?? "permission"} + + T3 Code needs this macOS privacy permission for Computer Use. System Settings will + open to the right Privacy & Security list — turn on the switch for T3 Code. + + + + {permissionPrompt?.kind === "accessibility" ? ( +

+ Accessibility lets the agent click and type in other apps without stealing your + mouse. +

+ ) : ( +

Screen Recording lets the agent take screenshots of apps and displays.

+ )} +
+ + }>Cancel + + +
+
+
+ ); +} diff --git a/apps/web/src/components/settings/SettingsSidebarNav.tsx b/apps/web/src/components/settings/SettingsSidebarNav.tsx index 174c9e9fe97..2f817d0a8c2 100644 --- a/apps/web/src/components/settings/SettingsSidebarNav.tsx +++ b/apps/web/src/components/settings/SettingsSidebarNav.tsx @@ -14,6 +14,7 @@ import { GitBranchIcon, KeyboardIcon, Link2Icon, + MonitorIcon, PaletteIcon, SearchIcon, Settings2Icon, @@ -51,6 +52,7 @@ const SETTINGS_SECTION_ICONS: Readonly< "/settings/providers": BotIcon, "/settings/source-control": GitBranchIcon, "/settings/connections": Link2Icon, + "/settings/computer-use": MonitorIcon, "/settings/archived": ArchiveIcon, }; diff --git a/apps/web/src/components/settings/browserBrandIcons.tsx b/apps/web/src/components/settings/browserBrandIcons.tsx new file mode 100644 index 00000000000..4bf0a316652 --- /dev/null +++ b/apps/web/src/components/settings/browserBrandIcons.tsx @@ -0,0 +1,47 @@ +import type { ImgHTMLAttributes } from "react"; + +import { cn } from "../../lib/utils"; +import agentCursorUrl from "../../assets/computer-use/agent-cursor-badge.png"; +import braveUrl from "../../assets/computer-use/brave.svg"; +import chromeUrl from "../../assets/computer-use/google-chrome.svg"; +import edgeUrl from "../../assets/computer-use/edge.svg"; +import firefoxUrl from "../../assets/computer-use/firefox.svg"; + +type BrandIconProps = Omit, "src" | "alt"> & { + alt?: string; +}; + +function BrandImg({ src, alt = "", className, ...props }: BrandIconProps & { src: string }) { + return ( + {alt} + ); +} + +export function ChromeIcon({ alt = "", ...props }: BrandIconProps) { + return ; +} + +export function EdgeIcon({ alt = "", ...props }: BrandIconProps) { + return ; +} + +export function BraveIcon({ alt = "", ...props }: BrandIconProps) { + return ; +} + +export function FirefoxIcon({ alt = "", ...props }: BrandIconProps) { + return ; +} + +/** Purple rounded badge with soft radial glow fade (no nested square crop). */ +export function AgentCursorIcon({ alt = "", className, ...props }: BrandIconProps) { + return ( + + ); +} diff --git a/apps/web/src/components/settings/settingsLayout.tsx b/apps/web/src/components/settings/settingsLayout.tsx index 880e03e5475..42493bee8d3 100644 --- a/apps/web/src/components/settings/settingsLayout.tsx +++ b/apps/web/src/components/settings/settingsLayout.tsx @@ -83,6 +83,24 @@ function useSettingsSearchTarget(id: string | undefined) return targetRef; } +/** Shared micro icon control used by settings reset/remove/info actions. */ +export function SettingIconAction({ + className, + ...props +}: ComponentPropsWithoutRef) { + return + } /> @@ -211,9 +229,7 @@ export function SettingResetButton({ { @@ -222,7 +238,7 @@ export function SettingResetButton({ }} > - + } /> Reset to default diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts index e3aef670566..09a5270a2bd 100644 --- a/apps/web/src/components/settings/settingsSearch.ts +++ b/apps/web/src/components/settings/settingsSearch.ts @@ -7,6 +7,7 @@ export type SettingsPath = | "/settings/providers" | "/settings/source-control" | "/settings/connections" + | "/settings/computer-use" | "/settings/archived"; export interface SettingsSearchItem { @@ -30,6 +31,7 @@ export const SETTINGS_SECTION_LABELS: Readonly> = { "/settings/providers": "Providers", "/settings/source-control": "Source Control", "/settings/connections": "Connections", + "/settings/computer-use": "Computer Use", "/settings/archived": "Archive", }; @@ -205,6 +207,31 @@ export const SETTINGS_SEARCH_ITEMS = [ title: "Remote environments", to: "/settings/connections", }, + { + id: "computer-use-enabled", + title: "Enable Computer Use", + to: "/settings/computer-use", + }, + { + id: "computer-use-browser", + title: "Google Chrome", + to: "/settings/computer-use", + }, + { + id: "computer-use-agent-cursor", + title: "Agent cursor overlay", + to: "/settings/computer-use", + }, + { + id: "computer-use-accessibility", + title: "Accessibility", + to: "/settings/computer-use", + }, + { + id: "computer-use-screen-recording", + title: "Screen Recording", + to: "/settings/computer-use", + }, { id: "archive", title: "Archived threads", diff --git a/apps/web/src/routeTree.gen.ts b/apps/web/src/routeTree.gen.ts index 697eb607c4d..793833c4c0c 100644 --- a/apps/web/src/routeTree.gen.ts +++ b/apps/web/src/routeTree.gen.ts @@ -21,6 +21,7 @@ import { Route as SettingsKeybindingsRouteImport } from './routes/settings.keybi import { Route as SettingsGeneralRouteImport } from './routes/settings.general' import { Route as SettingsDiagnosticsRouteImport } from './routes/settings.diagnostics' import { Route as SettingsConnectionsRouteImport } from './routes/settings.connections' +import { Route as SettingsComputerUseRouteImport } from './routes/settings.computer-use' import { Route as SettingsArchivedRouteImport } from './routes/settings.archived' import { Route as SettingsAppearanceRouteImport } from './routes/settings.appearance' import { Route as ProjectsProjectKeyRouteImport } from './routes/projects.$projectKey' @@ -88,6 +89,11 @@ const SettingsConnectionsRoute = SettingsConnectionsRouteImport.update({ path: '/connections', getParentRoute: () => SettingsRoute, } as any) +const SettingsComputerUseRoute = SettingsComputerUseRouteImport.update({ + id: '/computer-use', + path: '/computer-use', + getParentRoute: () => SettingsRoute, +} as any) const SettingsArchivedRoute = SettingsArchivedRouteImport.update({ id: '/archived', path: '/archived', @@ -136,6 +142,7 @@ export interface FileRoutesByFullPath { '/projects/$projectKey': typeof ProjectsProjectKeyRoute '/settings/appearance': typeof SettingsAppearanceRoute '/settings/archived': typeof SettingsArchivedRoute + '/settings/computer-use': typeof SettingsComputerUseRoute '/settings/connections': typeof SettingsConnectionsRoute '/settings/diagnostics': typeof SettingsDiagnosticsRoute '/settings/general': typeof SettingsGeneralRoute @@ -155,6 +162,7 @@ export interface FileRoutesByTo { '/projects/$projectKey': typeof ProjectsProjectKeyRoute '/settings/appearance': typeof SettingsAppearanceRoute '/settings/archived': typeof SettingsArchivedRoute + '/settings/computer-use': typeof SettingsComputerUseRoute '/settings/connections': typeof SettingsConnectionsRoute '/settings/diagnostics': typeof SettingsDiagnosticsRoute '/settings/general': typeof SettingsGeneralRoute @@ -177,6 +185,7 @@ export interface FileRoutesById { '/projects/$projectKey': typeof ProjectsProjectKeyRoute '/settings/appearance': typeof SettingsAppearanceRoute '/settings/archived': typeof SettingsArchivedRoute + '/settings/computer-use': typeof SettingsComputerUseRoute '/settings/connections': typeof SettingsConnectionsRoute '/settings/diagnostics': typeof SettingsDiagnosticsRoute '/settings/general': typeof SettingsGeneralRoute @@ -200,6 +209,7 @@ export interface FileRouteTypes { | '/projects/$projectKey' | '/settings/appearance' | '/settings/archived' + | '/settings/computer-use' | '/settings/connections' | '/settings/diagnostics' | '/settings/general' @@ -219,6 +229,7 @@ export interface FileRouteTypes { | '/projects/$projectKey' | '/settings/appearance' | '/settings/archived' + | '/settings/computer-use' | '/settings/connections' | '/settings/diagnostics' | '/settings/general' @@ -240,6 +251,7 @@ export interface FileRouteTypes { | '/projects/$projectKey' | '/settings/appearance' | '/settings/archived' + | '/settings/computer-use' | '/settings/connections' | '/settings/diagnostics' | '/settings/general' @@ -347,6 +359,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof SettingsConnectionsRouteImport parentRoute: typeof SettingsRoute } + '/settings/computer-use': { + id: '/settings/computer-use' + path: '/computer-use' + fullPath: '/settings/computer-use' + preLoaderRoute: typeof SettingsComputerUseRouteImport + parentRoute: typeof SettingsRoute + } '/settings/archived': { id: '/settings/archived' path: '/archived' @@ -418,6 +437,7 @@ const ChatRouteWithChildren = ChatRoute._addFileChildren(ChatRouteChildren) interface SettingsRouteChildren { SettingsAppearanceRoute: typeof SettingsAppearanceRoute SettingsArchivedRoute: typeof SettingsArchivedRoute + SettingsComputerUseRoute: typeof SettingsComputerUseRoute SettingsConnectionsRoute: typeof SettingsConnectionsRoute SettingsDiagnosticsRoute: typeof SettingsDiagnosticsRoute SettingsGeneralRoute: typeof SettingsGeneralRoute @@ -429,6 +449,7 @@ interface SettingsRouteChildren { const SettingsRouteChildren: SettingsRouteChildren = { SettingsAppearanceRoute: SettingsAppearanceRoute, SettingsArchivedRoute: SettingsArchivedRoute, + SettingsComputerUseRoute: SettingsComputerUseRoute, SettingsConnectionsRoute: SettingsConnectionsRoute, SettingsDiagnosticsRoute: SettingsDiagnosticsRoute, SettingsGeneralRoute: SettingsGeneralRoute, diff --git a/apps/web/src/routes/settings.computer-use.tsx b/apps/web/src/routes/settings.computer-use.tsx new file mode 100644 index 00000000000..cef1f869459 --- /dev/null +++ b/apps/web/src/routes/settings.computer-use.tsx @@ -0,0 +1,7 @@ +import { createFileRoute } from "@tanstack/react-router"; + +import { ComputerUseSettings } from "../components/settings/ComputerUseSettings"; + +export const Route = createFileRoute("/settings/computer-use")({ + component: ComputerUseSettings, +}); diff --git a/apps/web/src/session-logic.test.ts b/apps/web/src/session-logic.test.ts index f5effff6602..09ceff0aadc 100644 --- a/apps/web/src/session-logic.test.ts +++ b/apps/web/src/session-logic.test.ts @@ -156,6 +156,58 @@ describe("derivePendingApprovals", () => { ]); }); + it("derives MCP guardian requests as tool approvals", () => { + const activities: OrchestrationThreadActivity[] = [ + makeActivity({ + id: "approval-open-tool", + createdAt: "2026-02-23T00:00:01.000Z", + kind: "approval.requested", + summary: "Tool approval requested", + tone: "approval", + payload: { + requestId: "req-tool", + requestType: "tool_approval", + detail: "Allow node_repl to run?", + }, + }), + ]; + + expect(derivePendingApprovals(activities)).toEqual([ + { + requestId: "req-tool", + requestKind: "tool", + createdAt: "2026-02-23T00:00:01.000Z", + detail: "Allow node_repl to run?", + }, + ]); + }); + + it("derives Computer Use permission requests as actionable approvals", () => { + const activities: OrchestrationThreadActivity[] = [ + makeActivity({ + id: "approval-open-permissions", + createdAt: "2026-02-23T00:00:01.000Z", + kind: "approval.requested", + summary: "Permission requested", + tone: "approval", + payload: { + requestId: "req-permissions", + requestType: "permissions_approval", + detail: "Allow Computer Use to view and control the desktop", + }, + }), + ]; + + expect(derivePendingApprovals(activities)).toEqual([ + { + requestId: "req-permissions", + requestKind: "permissions", + createdAt: "2026-02-23T00:00:01.000Z", + detail: "Allow Computer Use to view and control the desktop", + }, + ]); + }); + it("clears stale pending approvals when provider reports unknown pending request", () => { const activities: OrchestrationThreadActivity[] = [ makeActivity({ @@ -742,6 +794,32 @@ describe("deriveWorkLogEntries", () => { expect(entries.map((entry) => entry.id)).toEqual(["tool-complete"]); }); + it("preserves generic tool and Computer Use approval kinds in the work log", () => { + const activities: OrchestrationThreadActivity[] = [ + makeActivity({ + id: "approval-tool", + createdAt: "2026-02-23T00:00:01.000Z", + kind: "approval.requested", + summary: "Tool approval requested", + tone: "approval", + payload: { requestId: "req-tool", requestType: "tool_approval" }, + }), + makeActivity({ + id: "approval-computer-use", + createdAt: "2026-02-23T00:00:02.000Z", + kind: "approval.requested", + summary: "Permission requested", + tone: "approval", + payload: { requestId: "req-computer-use", requestKind: "permissions" }, + }), + ]; + + expect(deriveWorkLogEntries(activities).map((entry) => entry.requestKind)).toEqual([ + "tool", + "permissions", + ]); + }); + it("omits task.started but shows task.progress and task.completed", () => { const activities: OrchestrationThreadActivity[] = [ makeActivity({ diff --git a/apps/web/src/session-logic.ts b/apps/web/src/session-logic.ts index cd5e2aa11d7..8734725db53 100644 --- a/apps/web/src/session-logic.ts +++ b/apps/web/src/session-logic.ts @@ -107,7 +107,7 @@ interface DerivedWorkLogEntry extends WorkLogEntry { export interface PendingApproval { requestId: ApprovalRequestId; - requestKind: "command" | "file-read" | "file-change"; + requestKind: "command" | "file-read" | "file-change" | "tool" | "permissions"; createdAt: string; detail?: string; } @@ -362,6 +362,10 @@ function requestKindFromRequestType(requestType: unknown): PendingApproval["requ case "file_change_approval": case "apply_patch_approval": return "file-change"; + case "tool_approval": + return "tool"; + case "permissions_approval": + return "permissions"; default: return null; } @@ -402,7 +406,9 @@ export function derivePendingApprovals( payload && (payload.requestKind === "command" || payload.requestKind === "file-read" || - payload.requestKind === "file-change") + payload.requestKind === "file-change" || + payload.requestKind === "tool" || + payload.requestKind === "permissions") ? payload.requestKind : payload ? requestKindFromRequestType(payload.requestType) @@ -1547,7 +1553,9 @@ function extractWorkLogRequestKind( if ( payload?.requestKind === "command" || payload?.requestKind === "file-read" || - payload?.requestKind === "file-change" + payload?.requestKind === "file-change" || + payload?.requestKind === "tool" || + payload?.requestKind === "permissions" ) { return payload.requestKind; } diff --git a/native/t3-chrome-extension/background.js b/native/t3-chrome-extension/background.js new file mode 100644 index 00000000000..61ade6b7e2a --- /dev/null +++ b/native/t3-chrome-extension/background.js @@ -0,0 +1,733 @@ +// T3 Code desktop control — Chrome side. +// +// The agent works only in tabs it created, collected into a labelled tab group, +// so the user's own tabs are never touched and they can keep browsing while a +// task runs. Page interaction goes through the DevTools protocol rather than +// synthetic mouse input, which is what makes it work in a *background* tab: a +// window only renders its active tab, so anything coordinate-based would be +// blind the moment the user switches away. +// +// Commands arrive from the desktop app over native messaging; every reply +// carries the originating request id. + +const HOST = "com.t3tools.t3code.desktop"; +const GROUP_TITLE = "T3 Code"; +const OWNED_STATE_KEY = "ownedState"; + +/** Tabs this extension owns, and the group holding them. */ +let ownedTabs = new Set(); +let groupId = null; +/** Tabs we have attached the debugger to, so we detach exactly once. */ +const attached = new Set(); +let port = null; +/** True after the native host has delivered at least one message this session. */ +let hadLiveSession = false; +/** When the current native-host port was opened (ms). */ +let connectedAt = 0; +/** + * A host that stays connected this long is treated as a live desktop session, + * even if it has not sent a command yet (idle reconnect while MCP is up). + * Shorter disconnects are the usual "MCP not listening yet" race. + */ +const LIVE_SESSION_DWELL_MS = 2000; +let stateReady = null; + +async function persistOwnedState() { + try { + await chrome.storage.session.set({ + [OWNED_STATE_KEY]: { + tabs: Array.from(ownedTabs), + groupId, + }, + }); + } catch { + // Storage can fail in restricted contexts; ownership still works in-memory. + } +} + +async function restoreOwnedState() { + try { + const stored = await chrome.storage.session.get(OWNED_STATE_KEY); + const state = stored?.[OWNED_STATE_KEY]; + if (!state || typeof state !== "object") return; + + const next = new Set(); + for (const tabId of Array.isArray(state.tabs) ? state.tabs : []) { + if (typeof tabId !== "number") continue; + try { + await chrome.tabs.get(tabId); + next.add(tabId); + } catch { + // Tab closed while the service worker was asleep. + } + } + ownedTabs = next; + + groupId = typeof state.groupId === "number" ? state.groupId : null; + if (groupId !== null) { + try { + await chrome.tabGroups.get(groupId); + } catch { + groupId = null; + } + } + await persistOwnedState(); + } catch { + // Fresh start if session storage is unavailable. + } +} + +function ensureStateReady() { + if (!stateReady) stateReady = restoreOwnedState(); + return stateReady; +} + +// ── native messaging ──────────────────────────────────────────────────────── + +function connect() { + if (port) return; + try { + port = chrome.runtime.connectNative(HOST); + } catch { + port = null; + return; + } + const sessionPort = port; + connectedAt = Date.now(); + hadLiveSession = false; + sessionPort.onMessage.addListener((msg) => { + // A command proves the MCP bridge is up. + hadLiveSession = true; + void handleCommand(msg, sessionPort); + }); + sessionPort.onDisconnect.addListener(() => { + // Reading lastError here keeps "Native host has exited" out of the error + // list while the desktop app simply is not running yet. + void chrome.runtime.lastError; + const livedMs = connectedAt ? Date.now() - connectedAt : 0; + // Tear down tabs when a real session ends: either we saw traffic, or the + // host stayed up long enough that this was not a connectNative race. + // Immediate disconnects (MCP pipe not bound yet) keep restored tabs. + const wasLive = hadLiveSession || livedMs >= LIVE_SESSION_DWELL_MS; + const tabsToClose = wasLive ? Array.from(ownedTabs) : []; + const groupToClear = wasLive ? groupId : null; + if (port === sessionPort) { + port = null; + hadLiveSession = false; + connectedAt = 0; + } + if (tabsToClose.length) { + for (const tabId of tabsToClose) { + void hideCursor(tabId); + } + void closeOwnedTabs(tabsToClose, groupToClear); + } + }); +} + +// The desktop app comes and goes with the user's session, so reconnect on a +// schedule. An alarm rather than setTimeout: a service worker is terminated +// when idle and timers do not survive that, which would strand the connection +// until the user reloaded the extension by hand. +// Chrome clamps alarm periods to a minute, so ask for what we will get. +chrome.alarms.create("t3-reconnect", { periodInMinutes: 1 }); +chrome.alarms.onAlarm.addListener((alarm) => { + if (alarm.name === "t3-reconnect") connect(); +}); +chrome.runtime.onStartup.addListener(connect); +chrome.runtime.onInstalled.addListener(connect); +// Connect as soon as the service worker evaluates. onStartup/onInstalled alone +// can miss unpacked loads; content-script pings also wake us via onMessage. +chrome.runtime.onMessage.addListener((msg) => { + if (msg && msg.type === "t3-wake") connect(); +}); +connect(); + +function reply(portRef, id, result) { + try { + portRef?.postMessage({ id, ok: true, result }); + } catch { + // Port went away mid-command; drop the reply. + } +} + +function replyError(portRef, id, message) { + try { + portRef?.postMessage({ id, ok: false, error: String(message) }); + } catch { + // Port went away mid-command; drop the reply. + } +} + +// ── tab + group management ────────────────────────────────────────────────── + +/** Serialize group mutation so concurrent open_tab calls share one group. */ +const groupQueue = (() => { + let chain = Promise.resolve(); + return (task) => { + const run = chain.then(task, task); + chain = run.then( + () => undefined, + () => undefined, + ); + return run; + }; +})(); + +async function ensureGroup(tabId) { + return groupQueue(async () => { + // Re-create the group if the user dismissed it or Chrome dropped it. + if (groupId !== null) { + try { + await chrome.tabGroups.get(groupId); + } catch { + groupId = null; + } + } + if (groupId === null) { + groupId = await chrome.tabs.group({ tabIds: [tabId] }); + await chrome.tabGroups.update(groupId, { title: GROUP_TITLE, color: "blue" }); + } else { + await chrome.tabs.group({ groupId, tabIds: [tabId] }); + } + // Agent tabs get the pointer favicon (not the T3 toolbar logo) as soon as + // they join the group, so the strip reads as "agent-owned" before the first click. + await markTab(tabId); + await persistOwnedState(); + return groupId; + }); +} + +async function openTab(url) { + // active:false is the whole point — the user stays on whatever they were doing. + const tab = await chrome.tabs.create({ url: url || "about:blank", active: false }); + ownedTabs.add(tab.id); + await ensureGroup(tab.id); + await persistOwnedState(); + // Pages replace their favicon on load (Spotify, YouTube, …). Re-apply the + // pointer whenever the document finishes, and also when the tab's own icon + // changes, so the strip stays on the agent cursor rather than the site logo. + chrome.tabs.onUpdated.addListener(function badge(id, info) { + if (id !== tab.id) return; + if (info.status === "complete" || info.favIconUrl) markTab(tab.id); + if (!ownedTabs.has(tab.id)) chrome.tabs.onUpdated.removeListener(badge); + }); + return { tabId: tab.id, url: tab.url, title: tab.title }; +} + +async function listTabs() { + const out = []; + // Snapshot first: the loop drops ids for tabs the user closed behind us. + const known = Array.from(ownedTabs); + for (const tabId of known) { + try { + const tab = await chrome.tabs.get(tabId); + out.push({ tabId, title: tab.title, url: tab.url, active: tab.active }); + } catch { + ownedTabs.delete(tabId); // closed behind our back + } + } + return { groupId, tabs: out }; +} + +/// Close a captured set of agent tabs from a past session. Only mutates +/// `ownedTabs` / `groupId` for those ids so a newer reconnect's tabs survive. +async function closeOwnedTabs(ids, expectedGroupId) { + for (const id of ids) { + ownedTabs.delete(id); + attached.delete(id); + try { + await chrome.tabs.remove(id); + } catch { + // Already closed by the user; nothing to do. + } + } + if (expectedGroupId !== null && groupId === expectedGroupId) { + try { + const remaining = await chrome.tabs.query({ groupId: expectedGroupId }); + // Ungroup stragglers that are not part of the current owned set — a + // reconnect may already have placed new agent tabs in this same group. + const leftover = remaining.filter((t) => !ownedTabs.has(t.id)); + if (leftover.length) await chrome.tabs.ungroup(leftover.map((t) => t.id)); + } catch { + // The group is already gone. + } + // Keep groupId when a newer session still owns tabs in it; only drop the + // handle once nothing we track remains there. + if (groupId === expectedGroupId && ownedTabs.size === 0) { + groupId = null; + } + } + await persistOwnedState(); + return { closed: ids.length }; +} + +async function closeAllTabs() { + return closeOwnedTabs(Array.from(ownedTabs), groupId); +} + +function assertOwned(tabId) { + if (!ownedTabs.has(tabId)) { + throw new Error(`tab ${tabId} is not one of the agent's tabs`); + } +} + +// ── DevTools protocol ─────────────────────────────────────────────────────── + +async function attach(tabId) { + if (attached.has(tabId)) return; + await chrome.debugger.attach({ tabId }, "1.3"); + attached.add(tabId); +} + +async function send(tabId, method, params = {}) { + await attach(tabId); + return chrome.debugger.sendCommand({ tabId }, method, params); +} + +/// A compact outline of the interactive elements on the page, with ids the +/// agent can click. Mirrors the accessibility-tree tools on the desktop side. +const SNAPSHOT_JS = `(() => { + const out = []; + const sel = 'a,button,input,textarea,select,[role=button],[role=link],[role=textbox],[contenteditable=true],summary'; + let i = 0; + for (const el of document.querySelectorAll(sel)) { + const r = el.getBoundingClientRect(); + if (r.width < 2 || r.height < 2) continue; + const style = getComputedStyle(el); + if (style.visibility === 'hidden' || style.display === 'none') continue; + const label = (el.getAttribute('aria-label') || el.innerText || el.value || + el.getAttribute('title') || el.getAttribute('placeholder') || '') + .replace(/\\s+/g, ' ').trim().slice(0, 90); + el.setAttribute('data-t3-idx', String(i)); + out.push({ + i: i++, + tag: el.tagName.toLowerCase(), + label, + x: Math.round(r.left + r.width / 2), + y: Math.round(r.top + r.height / 2), + inView: r.top >= 0 && r.bottom <= innerHeight, + }); + if (i >= 250) break; + } + return { title: document.title, url: location.href, elements: out }; +})()`; + +async function snapshot(tabId) { + const res = await send(tabId, "Runtime.evaluate", { + expression: SNAPSHOT_JS, + returnByValue: true, + }); + if (res?.exceptionDetails) throw new Error(res.exceptionDetails.text || "evaluate failed"); + return res.result.value; +} + +async function clickAt(tabId, x, y) { + // Show the same agent pointer the desktop overlay uses, painted into the page. + const cursor = await paintCursor(tabId, x, y); + // A hover first, then press/release carrying the button bitmask. Single-page + // apps route clicks through pointer/hover handlers, and without the leading + // mouseMoved (or with buttons unset) the press lands on nothing. + await send(tabId, "Input.dispatchMouseEvent", { + type: "mouseMoved", + x, + y, + button: "none", + buttons: 0, + pointerType: "mouse", + }); + await send(tabId, "Input.dispatchMouseEvent", { + type: "mousePressed", + x, + y, + button: "left", + buttons: 1, + clickCount: 1, + pointerType: "mouse", + }); + await send(tabId, "Input.dispatchMouseEvent", { + type: "mouseReleased", + x, + y, + button: "left", + buttons: 0, + clickCount: 1, + pointerType: "mouse", + }); + await markTab(tabId); + return { clicked: { x, y }, cursor }; +} + +/// The agent cursor, drawn into the page itself so a controlled tab shows the +/// same pointer as the desktop overlay. Fixed-position, pointer-events:none and +/// max z-index, so it is purely decorative and cannot intercept anything. +/// +/// Uses the PNG exported from BubbleView in AgentCursor.swift (not a hand-traced +/// SVG) so Chrome and desktop stay pixel-matched: same glow, fill, rim, shape. +/// Motion mirrors the desktop overlay: slow fade-in, cubic flight with tip +/// following path tangent, and fade-out after Computer Use tools stop (not a +/// short idle after the last pixel move). +const CURSOR_IMG_URL = chrome.runtime.getURL("icons/cursor-112.png"); +const CURSOR_HOTSPOT = 56; // OverlayController.hotspot — tip at centre of 112×112 +const CURSOR_FADE_IN_MS = 500; +const CURSOR_FADE_OUT_MS = 350; +/** Match desktop `T3_DESKTOP_AGENT_CURSOR_TASK_FADE_SECS` default (8s). */ +const CURSOR_TASK_FADE_MS = 8000; + +const PAINT_CURSOR_JS = ` + (function paint(x, y, src, fadeInMs, fadeOutMs, taskFadeMs, hotspot) { + const ID = '__t3AgentCursor'; + const easeInOut = (t) => t * t * (3 - 2 * t); + const bezier = (p0, p1, p2, p3, t) => { + const u = 1 - t; + return u*u*u*p0 + 3*u*u*t*p1 + 3*u*t*t*p2 + t*t*t*p3; + }; + const bezierTan = (p0, p1, p2, p3, t) => { + const u = 1 - t; + return 3*u*u*(p1-p0) + 6*u*t*(p2-p1) + 3*t*t*(p3-p2); + }; + + let el = document.getElementById(ID); + if (!el) { + el = document.createElement('div'); + el.id = ID; + el.style.cssText = 'position:fixed;left:0;top:0;width:112px;height:112px;' + + 'pointer-events:none;z-index:2147483647;opacity:0;will-change:transform,opacity;' + + 'transform-origin:' + hotspot + 'px ' + hotspot + 'px;'; + const img = document.createElement('img'); + img.src = src; + img.width = 112; + img.height = 112; + img.alt = ''; + img.draggable = false; + img.style.cssText = 'display:block;width:112px;height:112px;'; + el.appendChild(img); + (document.documentElement || document.body).appendChild(el); + el.__t3 = { x: x, y: y, tilt: 0, arc: 1, raf: 0 }; + } else { + const img = el.querySelector('img'); + if (img && img.src !== src) img.src = src; + } + + const st = el.__t3 || (el.__t3 = { x: x, y: y, tilt: 0, arc: 1, raf: 0 }); + if (st.raf) { cancelAnimationFrame(st.raf); st.raf = 0; } + clearTimeout(el.__t3hide); + + const place = (px, py, tilt) => { + st.x = px; st.y = py; st.tilt = tilt; + el.style.transform = 'translate(' + (px - hotspot) + 'px,' + (py - hotspot) + + 'px) rotate(' + tilt + 'rad)'; + }; + + const fromX = st.x; + const fromY = st.y; + const dx = x - fromX; + const dy = y - fromY; + const dist = Math.hypot(dx, dy); + const fresh = parseFloat(getComputedStyle(el).opacity) < 0.05; + + let waitMs = 80; + if (fresh) { + place(x, y, 0); + el.style.transition = 'opacity ' + fadeInMs + 'ms ease-out'; + // Force style flush so the opacity transition runs from 0. + void el.offsetWidth; + el.style.opacity = '1'; + waitMs = fadeInMs + 40; + } else if (dist < 3) { + el.style.transition = 'opacity ' + fadeInMs + 'ms ease-out'; + el.style.opacity = '1'; + place(x, y, 0); + waitMs = 60; + } else { + el.style.transition = 'opacity 120ms linear'; + el.style.opacity = '1'; + st.arc *= -1; + const handle = Math.min(72, Math.max(22, dist * 0.18)); + const nx = -dy / dist; + const ny = dx / dist; + let sdx, sdy; + if (Math.abs(st.tilt) > 0.08) { + sdx = Math.sin(-st.tilt); + sdy = -Math.cos(-st.tilt); + } else { + sdx = dx / dist; + sdy = dy / dist; + } + const depart = Math.min(handle, dist * 0.28); + const c1x = fromX + sdx * depart + nx * Math.min(36, dist * 0.10) * st.arc; + const c1y = fromY + sdy * depart + ny * Math.min(36, dist * 0.10) * st.arc; + // Approach from below so final tangent is screen-up → tip upright on land. + const approach = Math.min(handle * 0.85, Math.max(20, dist * 0.16)); + const c2x = x; + const c2y = y + approach; + const duration = Math.min(0.85, Math.max(0.28, 0.20 + dist / 1100.0)); + waitMs = Math.round(duration * 1000) + 40; + const t0 = performance.now(); + const tick = (now) => { + const u = Math.min(1, (now - t0) / (duration * 1000)); + const t = easeInOut(u); + const px = bezier(fromX, c1x, c2x, x, t); + const py = bezier(fromY, c1y, c2y, y, t); + const tx = bezierTan(fromX, c1x, c2x, x, t); + const ty = bezierTan(fromY, c1y, c2y, y, t); + let tilt = st.tilt; + const len = Math.hypot(tx, ty); + if (len > 0.001) { + const desired = -Math.atan2(tx, -ty); + let delta = desired - tilt; + while (delta > Math.PI) delta -= Math.PI * 2; + while (delta < -Math.PI) delta += Math.PI * 2; + tilt += delta * Math.min(1, 0.12 + t * 0.55); + } + if (u >= 1) tilt = 0; + place(px, py, tilt); + if (u < 1) { + st.raf = requestAnimationFrame(tick); + } else { + st.raf = 0; + place(x, y, 0); + } + }; + st.raf = requestAnimationFrame(tick); + } + + el.__t3hide = setTimeout(function () { + el.style.transition = 'opacity ' + fadeOutMs + 'ms ease'; + el.style.opacity = '0'; + }, taskFadeMs); + + return { + ok: true, + waitMs: waitMs, + fresh: fresh, + dist: dist + }; + }) +`; + +async function paintCursor(tabId, x, y) { + try { + const res = await send(tabId, "Runtime.evaluate", { + expression: + `(() => {` + + ` const r = (${PAINT_CURSOR_JS})(${Number(x)}, ${Number(y)}, ${JSON.stringify(CURSOR_IMG_URL)},` + + ` ${CURSOR_FADE_IN_MS}, ${CURSOR_FADE_OUT_MS}, ${CURSOR_TASK_FADE_MS}, ${CURSOR_HOTSPOT});` + + ` const el = document.getElementById('__t3AgentCursor');` + + ` if (!el) return { ok: false, reason: 'paint produced no element' };` + + ` const img = el.querySelector('img');` + + ` return Object.assign({}, r, {` + + ` hasGlow: !!(img && /cursor-112\\.png/.test(img.src)),` + + ` darkFill: !!(img && /cursor-112\\.png/.test(img.src)),` + + ` transform: el.style.transform || ''` + + ` });` + + `})()`, + returnByValue: true, + }); + if (res?.exceptionDetails) { + return { ok: false, reason: res.exceptionDetails.text || "paint evaluate failed" }; + } + const value = res?.result?.value || { ok: false, reason: "empty paint result" }; + const waitMs = Math.max(0, Math.min(1200, Number(value.waitMs) || 0)); + if (waitMs > 0) { + await new Promise((resolve) => setTimeout(resolve, waitMs)); + } + return value; + } catch (e) { + // Decorative only — a paint failure must never fail the click. + return { ok: false, reason: e && e.message ? e.message : String(e) }; + } +} + +async function hideCursor(tabId) { + try { + await send(tabId, "Runtime.evaluate", { + expression: + `(() => {` + + ` const el = document.getElementById('__t3AgentCursor');` + + ` if (!el) return false;` + + ` clearTimeout(el.__t3hide);` + + ` if (el.__t3 && el.__t3.raf) cancelAnimationFrame(el.__t3.raf);` + + ` el.style.transition = 'opacity ${CURSOR_FADE_OUT_MS}ms ease';` + + ` el.style.opacity = '0';` + + ` return true;` + + `})()`, + returnByValue: true, + }); + } catch { + // Tab may already be gone. + } +} + +const CLICK_JS = (index) => `(() => { + const el = document.querySelector('[data-t3-idx="${index}"]'); + if (!el) return { ok: false, reason: 'element ${index} is no longer on the page' }; + el.scrollIntoView({ block: 'center', inline: 'nearest' }); + const r = el.getBoundingClientRect(); + const cx = r.left + r.width / 2; + const cy = r.top + r.height / 2; + const opts = { bubbles: true, cancelable: true, composed: true, view: window, + clientX: cx, clientY: cy, button: 0 }; + el.dispatchEvent(new PointerEvent('pointerover', opts)); + el.dispatchEvent(new MouseEvent('mouseover', opts)); + el.dispatchEvent(new PointerEvent('pointerdown', opts)); + el.dispatchEvent(new MouseEvent('mousedown', opts)); + el.focus?.(); + el.dispatchEvent(new PointerEvent('pointerup', opts)); + el.dispatchEvent(new MouseEvent('mouseup', opts)); + el.click(); + return { ok: true, tag: el.tagName.toLowerCase(), href: el.href || null, x: cx, y: cy }; +})()`; + +/// Click a snapshotted element by invoking it in the page. +/// +/// Coordinate dispatch is unreliable here: a background tab is not composited, +/// so hit-testing a point finds nothing and the click silently does nothing. +/// Driving the node directly works regardless of whether the tab is rendered, +/// which is the whole point of working in a tab the user is not looking at. +async function clickElement(tabId, index) { + const res = await send(tabId, "Runtime.evaluate", { + expression: CLICK_JS(index), + returnByValue: true, + userGesture: true, + }); + if (res?.exceptionDetails) throw new Error(res.exceptionDetails.text || "click failed"); + const value = res.result.value || {}; + if (!value.ok) throw new Error(value.reason || "click failed"); + const cursor = await paintCursor(tabId, value.x, value.y); + await markTab(tabId); + return { ...value, cursor }; +} + +async function typeText(tabId, text) { + await send(tabId, "Input.insertText", { text }); + await markTab(tabId); + return { typed: text.length }; +} + +async function pressKey(tabId, key) { + const map = { + Enter: { windowsVirtualKeyCode: 13, key: "Enter", text: "\r" }, + Tab: { windowsVirtualKeyCode: 9, key: "Tab" }, + Escape: { windowsVirtualKeyCode: 27, key: "Escape" }, + Backspace: { windowsVirtualKeyCode: 8, key: "Backspace" }, + }; + const spec = map[key]; + if (!spec) throw new Error(`unsupported key: ${key}`); + await send(tabId, "Input.dispatchKeyEvent", { type: "keyDown", ...spec }); + await send(tabId, "Input.dispatchKeyEvent", { type: "keyUp", ...spec }); + return { pressed: key }; +} + +async function screenshot(tabId) { + // Page.captureScreenshot works on a background tab; captureVisibleTab does not. + const res = await send(tabId, "Page.captureScreenshot", { format: "png" }); + return { data: res.data }; +} + +async function navigate(tabId, url) { + await chrome.tabs.update(tabId, { url }); + return { tabId, url }; +} + +// ── "the agent is using this tab" indicator ───────────────────────────────── +// +// Toolbar icon = T3 logo (manifest icons/). Tab favicon = Computer Use badge +// (icons/pointer-*.png), matching Settings → Agent cursor overlay — so a tab +// in the "T3 Code" group is visually distinct from the extension itself. +// +// An extension cannot set a tab's favicon directly, but it can replace the +// page's icon link, which is what Chrome renders in the tab strip. Pages +// rewrite their own favicon (YouTube does it for notifications), so this is +// re-applied on group join, load, favicon changes, and after each interaction. + +function applyFavicon(url) { + for (const link of document.querySelectorAll("link[rel~='icon'], link[rel='shortcut icon']")) { + link.remove(); + } + const link = document.createElement("link"); + link.rel = "icon"; + link.type = "image/png"; + link.href = url; + document.head.appendChild(link); +} + +async function markTab(tabId) { + try { + await chrome.scripting.executeScript({ + target: { tabId }, + func: applyFavicon, + args: [chrome.runtime.getURL("icons/pointer-64.png")], + }); + } catch { + // Chrome's own pages (chrome://, the Web Store) refuse injection; the tab + // still works, it just cannot show the badge. + } +} + +// ── dispatch ──────────────────────────────────────────────────────────────── + +const handlers = { + ping: async () => ({ pong: true }), + open_tab: async (p) => openTab(p.url), + list_tabs: async () => listTabs(), + select_tab: async (p) => { + assertOwned(p.tabId); + await chrome.tabs.update(p.tabId, { active: true }); + return { tabId: p.tabId }; + }, + close_all_tabs: async () => closeAllTabs(), + close_tab: async (p) => { + assertOwned(p.tabId); + await chrome.tabs.remove(p.tabId); + ownedTabs.delete(p.tabId); + attached.delete(p.tabId); + await persistOwnedState(); + return { closed: p.tabId }; + }, + navigate: async (p) => { + assertOwned(p.tabId); + return navigate(p.tabId, p.url); + }, + snapshot: async (p) => { + assertOwned(p.tabId); + return snapshot(p.tabId); + }, + click: async (p) => { + assertOwned(p.tabId); + return p.index !== undefined ? clickElement(p.tabId, p.index) : clickAt(p.tabId, p.x, p.y); + }, + type: async (p) => { + assertOwned(p.tabId); + return typeText(p.tabId, p.text); + }, + press: async (p) => { + assertOwned(p.tabId); + return pressKey(p.tabId, p.key); + }, + screenshot: async (p) => { + assertOwned(p.tabId); + return screenshot(p.tabId); + }, +}; + +async function handleCommand(msg, replyPort = port) { + await ensureStateReady(); + const { id, command, params } = msg || {}; + const handler = handlers[command]; + if (!handler) return replyError(replyPort, id, `unknown command: ${command}`); + try { + reply(replyPort, id, await handler(params || {})); + } catch (e) { + replyError(replyPort, id, e && e.message ? e.message : e); + } +} + +chrome.tabs.onRemoved.addListener((tabId) => { + if (!ownedTabs.has(tabId) && !attached.has(tabId)) return; + ownedTabs.delete(tabId); + attached.delete(tabId); + void persistOwnedState(); +}); + +void ensureStateReady().then(connect); diff --git a/native/t3-chrome-extension/icons/cursor-112.png b/native/t3-chrome-extension/icons/cursor-112.png new file mode 100644 index 00000000000..19d8498e657 Binary files /dev/null and b/native/t3-chrome-extension/icons/cursor-112.png differ diff --git a/native/t3-chrome-extension/icons/icon-128.png b/native/t3-chrome-extension/icons/icon-128.png new file mode 100644 index 00000000000..9f1e1895706 Binary files /dev/null and b/native/t3-chrome-extension/icons/icon-128.png differ diff --git a/native/t3-chrome-extension/icons/icon-16.png b/native/t3-chrome-extension/icons/icon-16.png new file mode 100644 index 00000000000..841a2e479cb Binary files /dev/null and b/native/t3-chrome-extension/icons/icon-16.png differ diff --git a/native/t3-chrome-extension/icons/icon-32.png b/native/t3-chrome-extension/icons/icon-32.png new file mode 100644 index 00000000000..96b9d50107a Binary files /dev/null and b/native/t3-chrome-extension/icons/icon-32.png differ diff --git a/native/t3-chrome-extension/icons/icon-48.png b/native/t3-chrome-extension/icons/icon-48.png new file mode 100644 index 00000000000..4c243db6e46 Binary files /dev/null and b/native/t3-chrome-extension/icons/icon-48.png differ diff --git a/native/t3-chrome-extension/icons/pointer-16.png b/native/t3-chrome-extension/icons/pointer-16.png new file mode 100644 index 00000000000..bc9a6f9e6e6 Binary files /dev/null and b/native/t3-chrome-extension/icons/pointer-16.png differ diff --git a/native/t3-chrome-extension/icons/pointer-32.png b/native/t3-chrome-extension/icons/pointer-32.png new file mode 100644 index 00000000000..22d08855596 Binary files /dev/null and b/native/t3-chrome-extension/icons/pointer-32.png differ diff --git a/native/t3-chrome-extension/icons/pointer-48.png b/native/t3-chrome-extension/icons/pointer-48.png new file mode 100644 index 00000000000..c1726e3b58d Binary files /dev/null and b/native/t3-chrome-extension/icons/pointer-48.png differ diff --git a/native/t3-chrome-extension/icons/pointer-64.png b/native/t3-chrome-extension/icons/pointer-64.png new file mode 100644 index 00000000000..073ff1848d0 Binary files /dev/null and b/native/t3-chrome-extension/icons/pointer-64.png differ diff --git a/native/t3-chrome-extension/install.ps1 b/native/t3-chrome-extension/install.ps1 new file mode 100644 index 00000000000..afe6bb9cc70 --- /dev/null +++ b/native/t3-chrome-extension/install.ps1 @@ -0,0 +1,81 @@ +# Register the native messaging host so Chrome can reach the desktop server. +# +# The Windows counterpart to install.sh. Chrome finds hosts through the registry +# here rather than a directory of manifests, and it runs the host with no +# arguments, so the manifest points at a small .cmd that re-execs the server in +# host mode. +# +# The extension id is pinned by the "key" in manifest.json, which is why this +# can be registered before the extension is ever loaded. + +$ErrorActionPreference = 'Stop' + +$ExtensionId = 'kgdolgnijopbghhomnblabjkmjhnoage' +$HostName = 'com.t3tools.t3code.desktop' + +$here = Split-Path -Parent $MyInvocation.MyCommand.Path +$binary = $env:T3CODE_DESKTOP_MCP_PATH +if (-not $binary) { + $binary = Join-Path $here '..\t3-desktop-mcp-rs\target\release\t3-desktop-mcp.exe' +} +if (-not (Test-Path $binary)) { + Write-Error "desktop server binary not found at: $binary`nbuild it first: cargo build --release --manifest-path native/t3-desktop-mcp-rs/Cargo.toml" +} +$binary = (Resolve-Path $binary).Path + +$support = Join-Path $env:LOCALAPPDATA 't3-desktop-mcp' +New-Item -ItemType Directory -Force -Path $support | Out-Null + +# Chrome passes no arguments to a host, so the wrapper supplies the mode. +# Write UTF-8 without BOM: ASCII would corrupt non-ASCII path characters, and +# PowerShell's "UTF8" encoding inserts a BOM that some hosts mishandle. +$wrapper = Join-Path $support 'native-host.cmd' +$utf8NoBom = New-Object System.Text.UTF8Encoding $false +[System.IO.File]::WriteAllText( + $wrapper, + "@echo off`r`n`"$binary`" native-host", + $utf8NoBom +) + +$manifestPath = Join-Path $support "$HostName.json" +$manifest = [ordered]@{ + name = $HostName + description = 'T3 Code desktop control bridge' + path = $wrapper + type = 'stdio' + allowed_origins = @("chrome-extension://$ExtensionId/") +} +# Chrome rejects native-host manifests with a UTF-8 BOM (PowerShell's UTF8 +# encoding inserts one). Write UTF-8 without BOM explicitly. +[System.IO.File]::WriteAllText( + $manifestPath, + ($manifest | ConvertTo-Json -Depth 4), + $utf8NoBom +) + +# Chrome and Chromium read separate registry trees; register wherever the +# browser is actually installed. +$installed = 0 +foreach ($vendor in @('Google\Chrome', 'Google\Chrome Beta', 'Chromium')) { + $key = "HKCU:\Software\$vendor\NativeMessagingHosts\$HostName" + try { + New-Item -Path $key -Force | Out-Null + Set-ItemProperty -Path $key -Name '(default)' -Value $manifestPath + Write-Host "registered host in: HKCU\Software\$vendor" + $installed++ + } catch { + # A browser that is not installed simply has no tree; not an error. + } +} + +if ($installed -eq 0) { + Write-Error 'no Chrome registry location could be written' +} + +Write-Host '' +Write-Host 'Next, load the extension once:' +Write-Host ' 1. open chrome://extensions' +Write-Host ' 2. turn on Developer mode' +Write-Host " 3. Load unpacked -> $here" +Write-Host '' +Write-Host "It should appear with id $ExtensionId." diff --git a/native/t3-chrome-extension/install.sh b/native/t3-chrome-extension/install.sh new file mode 100755 index 00000000000..2806e53f024 --- /dev/null +++ b/native/t3-chrome-extension/install.sh @@ -0,0 +1,87 @@ +#!/bin/sh +# Register the native messaging host so Chrome can reach the desktop server. +# +# Chrome runs the host itself and passes no arguments, so it points at a small +# wrapper that re-execs the server binary in host mode. The extension id is +# pinned by the "key" in manifest.json, which is why this can be registered +# before the extension is ever loaded. +set -eu + +EXTENSION_ID="kgdolgnijopbghhomnblabjkmjhnoage" +HOST_NAME="com.t3tools.t3code.desktop" + +here=$(cd "$(dirname "$0")" && pwd) +# macOS builds the Swift package; Linux builds the Rust crate that also covers +# Windows. Either way the binary is called t3-desktop-mcp. +case "$(uname -s)" in + Darwin) default_binary="$here/../t3-desktop-mcp/.build/apple/Products/Release/t3-desktop-mcp" ;; + *) default_binary="$here/../t3-desktop-mcp-rs/target/release/t3-desktop-mcp" ;; +esac +binary="${T3CODE_DESKTOP_MCP_PATH:-$default_binary}" +if [ ! -x "$binary" ]; then + echo "desktop server binary not found at: $binary" >&2 + echo "build it first: pnpm build:desktop-mcp" >&2 + exit 1 +fi + +case "$(uname -s)" in + Darwin) support="$HOME/Library/Application Support/t3-desktop-mcp" ;; + *) support="${XDG_DATA_HOME:-$HOME/.local/share}/t3-desktop-mcp" ;; +esac +mkdir -p "$support" +wrapper="$support/native-host" +cat > "$wrapper" < "$dir/$HOST_NAME.json" <&2 + exit 1 +fi + +echo +echo "Next, load the extension once:" +echo " 1. open chrome://extensions" +echo " 2. turn on Developer mode" +echo " 3. Load unpacked -> $here" +echo +echo "It should appear with id $EXTENSION_ID." diff --git a/native/t3-chrome-extension/manifest.json b/native/t3-chrome-extension/manifest.json new file mode 100644 index 00000000000..431b5bb3295 --- /dev/null +++ b/native/t3-chrome-extension/manifest.json @@ -0,0 +1,47 @@ +{ + "manifest_version": 3, + "name": "T3 Code Desktop Control", + "version": "0.2.5", + "description": "Lets T3 Code agents work in their own tab group, in your signed-in browser, without touching your tabs.", + "key": "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAlKKDWohdbduAaQO52AeI5OdBddEOMr4t2bu/jMeefnDjKLvmPNfjiOUUAKO10LkZNAyQL/IYJRizD2Ps1DqMIHADTdl/ihLQe5GPxOsswNo75oqpVcvpcfKsVRk0g5c3bAnedTaHe8zu37cyLIovKjdlrlcvtavKngJmG9JJfKKPmjZUqriGNA6mRqcUYo75+xppeKg7UJGyVSDvJOyx7SQHebbSN5bxWtT7cdvaNHq1MyqcIML/5LlQxh9vznZlKie7tQQAeDQENPrWzJ51X0YvoJUZWjhog967Wbasi7zVgsrFfcVe0VZBi6Ccsxg2K6DUCa6ZcAI5S/lAZ7uC8QIDAQAB", + "minimum_chrome_version": "116", + "permissions": [ + "tabs", + "tabGroups", + "debugger", + "nativeMessaging", + "scripting", + "alarms", + "storage" + ], + "host_permissions": [""], + "background": { + "service_worker": "background.js" + }, + "content_scripts": [ + { + "matches": [""], + "js": ["wake.js"], + "run_at": "document_start", + "all_frames": false + } + ], + "icons": { + "16": "icons/icon-16.png", + "32": "icons/icon-32.png", + "48": "icons/icon-48.png", + "128": "icons/icon-128.png" + }, + "web_accessible_resources": [ + { + "resources": [ + "icons/cursor-112.png", + "icons/pointer-16.png", + "icons/pointer-32.png", + "icons/pointer-48.png", + "icons/pointer-64.png" + ], + "matches": [""] + } + ] +} diff --git a/native/t3-chrome-extension/wake.js b/native/t3-chrome-extension/wake.js new file mode 100644 index 00000000000..fb1170902dd --- /dev/null +++ b/native/t3-chrome-extension/wake.js @@ -0,0 +1,2 @@ +// Wake the MV3 service worker on navigation so connectNative can run. +chrome.runtime.sendMessage({ type: "t3-wake" }).catch(() => {}); diff --git a/native/t3-desktop-mcp-rs/.gitignore b/native/t3-desktop-mcp-rs/.gitignore new file mode 100644 index 00000000000..ea8c4bf7f35 --- /dev/null +++ b/native/t3-desktop-mcp-rs/.gitignore @@ -0,0 +1 @@ +/target diff --git a/native/t3-desktop-mcp-rs/Cargo.lock b/native/t3-desktop-mcp-rs/Cargo.lock new file mode 100644 index 00000000000..23765eaab64 --- /dev/null +++ b/native/t3-desktop-mcp-rs/Cargo.lock @@ -0,0 +1,2719 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "annotate-snippets" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "710e8eae58854cdc1790fcb56cca04d712a17be849eeb81da2a724bf4bae2bc4" +dependencies = [ + "anstyle", + "unicode-width", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "async-broadcast" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435a87a52755b8f27fcf321ac4f04b2802e337c8c4872923137471ec39c37532" +dependencies = [ + "event-listener", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-executor" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96bf972d85afc50bf5ab8fe2d54d1586b4e0b46c97c50a0c9e71e2f7bcd812a" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand", + "futures-lite", + "pin-project-lite", + "slab", +] + +[[package]] +name = "async-io" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc" +dependencies = [ + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite", + "parking", + "polling", + "rustix 1.1.4", + "slab", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-process" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc50921ec0055cdd8a16de48773bfeec5c972598674347252c0399676be7da75" +dependencies = [ + "async-channel", + "async-io", + "async-lock", + "async-signal", + "async-task", + "blocking", + "cfg-if", + "event-listener", + "futures-lite", + "rustix 1.1.4", +] + +[[package]] +name = "async-recursion" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "async-signal" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52b5aaafa020cf5053a01f2a60e8ff5dccf550f0f77ec54a4e47285ac2bab485" +dependencies = [ + "async-io", + "async-lock", + "atomic-waker", + "cfg-if", + "futures-core", + "futures-io", + "rustix 1.1.4", + "signal-hook-registry", + "slab", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "atspi" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bf601cccedfffec598ec2db1f9d6745885458bccc0e8916d7023f017c94b3d0" +dependencies = [ + "atspi-common", + "atspi-connection", + "atspi-proxies", +] + +[[package]] +name = "atspi-common" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8a79bed3f5b408ce3152f36e07327a845e6ed5d7e2821a89264037dbcc11daf" +dependencies = [ + "enumflags2", + "serde", + "static_assertions", + "zbus", + "zbus-lockstep", + "zbus-lockstep-macros", + "zbus_names", + "zvariant", +] + +[[package]] +name = "atspi-connection" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fab8e4f574f5a7d3af280b38eff25fb6f47a537dac9ae39ce152f52b19fb10b" +dependencies = [ + "atspi-common", + "atspi-proxies", + "futures-lite", + "zbus", +] + +[[package]] +name = "atspi-proxies" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53403acd3ab2fdb5914f6558da22e540fc07656fce5510f8c02be0e6ef68413e" +dependencies = [ + "atspi-common", + "serde", + "zbus", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "annotate-snippets", + "bitflags", + "cexpr", + "clang-sys", + "itertools", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex 1.3.0", + "syn 2.0.119", +] + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2", +] + +[[package]] +name = "blocking" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21" +dependencies = [ + "async-channel", + "async-task", + "futures-io", + "futures-lite", + "piper", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0e56a716f1e132ff6bf4bdac1c944a3fcdc1cae65f70a4a2a1ac3b401d2d1f" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + +[[package]] +name = "cc" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" +dependencies = [ + "find-msvc-tools", + "shlex 2.0.1", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom 7.1.3", +] + +[[package]] +name = "cfg-expr" +version = "0.20.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb693542bcafa528e198be0ebd9d3632ca5b7c93dbe7237460e199910835997c" +dependencies = [ + "smallvec", + "target-lexicon", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "clang-sys" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "cookie-factory" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9885fa71e26b8ab7855e2ec7cae6e9b380edff76cd052e07c683a0319d51b3a2" + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags", + "block2", + "libc", + "objc2", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "dlib" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab8ecd87370524b461f8557c119c405552c396ed91fc0a8eec68679eab26f94a" +dependencies = [ + "libloading", +] + +[[package]] +name = "doctest-file" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2db04e74f0a9a93103b50e90b96024c9b2bdca8bce6a632ec71b88736d3d359" + +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + +[[package]] +name = "drm" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80bc8c5c6c2941f70a55c15f8d9f00f9710ebda3ffda98075f996a0e6c92756f" +dependencies = [ + "bitflags", + "bytemuck", + "drm-ffi", + "drm-fourcc", + "libc", + "rustix 0.38.44", +] + +[[package]] +name = "drm" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a41816e58f47f49acfd956651055ddcf137c4882c2098c30c448817af21183a" +dependencies = [ + "bitflags", + "bytemuck", + "bytemuck_derive", + "drm-ffi", + "drm-fourcc", + "libc", + "rustix 1.1.4", +] + +[[package]] +name = "drm-ffi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51a91c9b32ac4e8105dec255e849e0d66e27d7c34d184364fb93e469db08f690" +dependencies = [ + "drm-sys", + "rustix 1.1.4", +] + +[[package]] +name = "drm-fourcc" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aafbcdb8afc29c1a7ee5fbe53b5d62f4565b35a042a662ca9fecd0b54dae6f4" + +[[package]] +name = "drm-sys" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8e1361066d91f5ffccff060a3c3be9c3ecde15be2959c1937595f7a82a9f8" +dependencies = [ + "libc", + "linux-raw-sys 0.9.4", +] + +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + +[[package]] +name = "endi" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66b7e2430c6dff6a955451e2cfc438f09cea1965a9d6f87f7e3b90decc014099" + +[[package]] +name = "enumflags2" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1027f7680c853e056ebcec683615fb6fbbc07dbaa13b4d5d9442b146ded4ecef" +dependencies = [ + "enumflags2_derive", + "serde", +] + +[[package]] +name = "enumflags2_derive" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "gbm" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce852e998d3ca5e4a97014fb31c940dc5ef344ec7d364984525fd11e8a547e6a" +dependencies = [ + "bitflags", + "drm 0.14.1", + "drm-fourcc", + "gbm-sys", + "libc", + "wayland-backend", + "wayland-server", +] + +[[package]] +name = "gbm-sys" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13a5f2acc785d8fb6bf6b7ab6bfb0ef5dad4f4d97e8e70bb8e470722312f76f" +dependencies = [ + "libc", +] + +[[package]] +name = "gethostname" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" +dependencies = [ + "rustix 1.1.4", + "windows-link", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "rand_core", +] + +[[package]] +name = "gl" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a94edab108827d67608095e269cf862e60d920f144a5026d3dbcfd8b877fb404" +dependencies = [ + "gl_generator", +] + +[[package]] +name = "gl_generator" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a95dfc23a2b4a9a2f5ab41d194f8bfda3cabec42af4e39f08c339eb2a0c124d" +dependencies = [ + "khronos_api", + "log", + "xml-rs", +] + +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "image" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck", + "byteorder-lite", + "moxcms", + "num-traits", + "png", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "interprocess" +version = "2.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "798de1433ba514cc6c04c4144c2469af81396e4906195218737c776d47769572" +dependencies = [ + "doctest-file", + "libc", + "recvmsg", + "widestring", + "windows-sys 0.61.2", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "khronos-egl" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6aae1df220ece3c0ada96b8153459b67eebe9ae9212258bb0134ae60416fdf76" +dependencies = [ + "libc", + "pkg-config", +] + +[[package]] +name = "khronos_api" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2db585e1d738fc771bf08a151420d3ed193d9d895a36df7f6f8a9456b911ddc" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "libspa" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2909f3be29d674e7f10604aff18d1bbe1bb03c4cd61c8a8ba19c0b1d162f7d4e" +dependencies = [ + "bitflags", + "cc", + "cookie-factory", + "libc", + "libspa-sys", + "nom 8.0.0", + "rustix 1.1.4", + "system-deps", +] + +[[package]] +name = "libspa-sys" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69ad52764fca54818486f3cf75afec844d1f1a1568c24dcee25d41b1ab007dda" +dependencies = [ + "bindgen", + "cc", + "system-deps", +] + +[[package]] +name = "libwayshot-xcap" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea8a46e4d016ef464e386e4970f5415fe0939c2d63abe0c2f539c05ef88c5fe5" +dependencies = [ + "drm 0.15.0", + "gbm", + "gl", + "image", + "khronos-egl", + "memmap2", + "rustix 1.1.4", + "thiserror", + "tracing", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "wayland-protocols-wlr", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "linux-raw-sys" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-app-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d49e936b501e5c5bf01fda3a9452ff86dc3ea98ad5f283e1455153142d97518c" +dependencies = [ + "bitflags", + "block2", + "libc", + "objc2", + "objc2-cloud-kit", + "objc2-core-data", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-core-image", + "objc2-core-text", + "objc2-core-video", + "objc2-foundation", + "objc2-quartz-core", +] + +[[package]] +name = "objc2-av-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478ae33fcac9df0a18db8302387c666b8ef08a3e2d62b510ca4fc278a384b6c0" +dependencies = [ + "bitflags", + "block2", + "dispatch2", + "objc2", + "objc2-avf-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-core-image", + "objc2-core-video", + "objc2-foundation", + "objc2-image-io", + "objc2-media-toolbox", + "objc2-quartz-core", +] + +[[package]] +name = "objc2-avf-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13a380031deed8e99db00065c45937da434ca987c034e13b87e4441f9e4090be" +dependencies = [ + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-cloud-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ad74d880bb43877038da939b7427bba67e9dd42004a18b809ba7d87cee241c" +dependencies = [ + "bitflags", + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1eebcea8b0dbff5f7c8504f3107c68fc061a3eb44932051c8cf8a68d969c3b2" +dependencies = [ + "dispatch2", + "objc2", + "objc2-core-audio-types", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-core-audio-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a89f2ec274a0cf4a32642b2991e8b351a404d290da87bb6a9a9d8632490bd1c" +dependencies = [ + "bitflags", + "objc2", +] + +[[package]] +name = "objc2-core-data" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b402a653efbb5e82ce4df10683b6b28027616a2715e90009947d50b8dd298fa" +dependencies = [ + "bitflags", + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags", + "block2", + "dispatch2", + "libc", + "objc2", +] + +[[package]] +name = "objc2-core-graphics" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e022c9d066895efa1345f8e33e584b9f958da2fd4cd116792e15e07e4720a807" +dependencies = [ + "bitflags", + "block2", + "dispatch2", + "libc", + "objc2", + "objc2-core-foundation", + "objc2-io-surface", + "objc2-metal", +] + +[[package]] +name = "objc2-core-image" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d563b38d2b97209f8e861173de434bd0214cf020e3423a52624cd1d989f006" +dependencies = [ + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-media" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ec576860167a15dd9fce7fbee7512beb4e31f532159d3482d1f9c6caedf31d" +dependencies = [ + "bitflags", + "block2", + "dispatch2", + "objc2", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-core-video", +] + +[[package]] +name = "objc2-core-text" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cde0dfb48d25d2b4862161a4d5fcc0e3c24367869ad306b0c9ec0073bfed92d" +dependencies = [ + "bitflags", + "objc2", + "objc2-core-foundation", + "objc2-core-graphics", +] + +[[package]] +name = "objc2-core-video" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d425caf1df73233f29fd8a5c3e5edbc30d2d4307870f802d18f00d83dc5141a6" +dependencies = [ + "bitflags", + "block2", + "objc2", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-io-surface", + "objc2-metal", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" +dependencies = [ + "bitflags", + "block2", + "libc", + "objc2", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-image-io" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b0446e98cf4a784cc7a0177715ff317eeaa8463841c616cfc78aa4f953c4ea" +dependencies = [ + "objc2", + "objc2-core-foundation", + "objc2-core-graphics", +] + +[[package]] +name = "objc2-io-surface" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180788110936d59bab6bd83b6060ffdfffb3b922ba1396b312ae795e1de9d81d" +dependencies = [ + "bitflags", + "objc2", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-media-toolbox" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd9fdde720df3da7046bb9097811000c1e7ab5cd579fa89d96b27d56781fb30" +dependencies = [ + "objc2", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-core-media", +] + +[[package]] +name = "objc2-metal" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0125f776a10d00af4152d74616409f0d4a2053a6f57fa5b7d6aa2854ac04794" +dependencies = [ + "bitflags", + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-quartz-core" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96c1358452b371bf9f104e21ec536d37a650eb10f7ee379fff67d2e08d537f1f" +dependencies = [ + "bitflags", + "objc2", + "objc2-foundation", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "ordered-stream" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aa2b01e1d916879f73a53d01d1d6cee68adbb31d6d9177a8cfce093cced1d50" +dependencies = [ + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "piper" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c835479a4443ded371d6c535cbfd8d31ad92c5d23ae9770a61bc155e4992a3c1" +dependencies = [ + "atomic-waker", + "fastrand", + "futures-io", +] + +[[package]] +name = "pipewire" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8585aba8a52ad74ccc633b8e293c1dc4277976bd5d510b925533f34fd6685f38" +dependencies = [ + "bitflags", + "libc", + "libspa", + "libspa-sys", + "pipewire-sys", + "rustix 1.1.4", +] + +[[package]] +name = "pipewire-sys" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2089f245b548723e60325773c27f586b7a2372c79ea941b246cd0d654706adc" +dependencies = [ + "bindgen", + "libspa-sys", + "system-deps", +] + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + +[[package]] +name = "polling" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi", + "pin-project-lite", + "rustix 1.1.4", + "windows-sys 0.61.2", +] + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pxfm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" + +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "recvmsg" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3edd4d5d42c92f0a659926464d4cce56b562761267ecf0f469d85b7de384175" + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.12.1", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_repr" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "system-deps" +version = "7.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "396a35feb67335377e0251fcbc1092fc85c484bd4e3a7a54319399da127796e7" +dependencies = [ + "cfg-expr", + "heck", + "pkg-config", + "toml", + "version-compare", +] + +[[package]] +name = "t3-desktop-mcp-rs" +version = "0.1.0" +dependencies = [ + "atspi", + "base64", + "futures-lite", + "image", + "interprocess", + "libc", + "serde", + "serde_json", + "uiautomation", + "windows", + "x11rb", + "xcap", + "zbus", +] + +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix 1.1.4", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "toml" +version = "1.1.4+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3aace63f4bbcdfc2c965b059de67119c89c4017a70d633be6c104910f67056f5" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.13+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_writer" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "uds_windows" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e" +dependencies = [ + "memoffset", + "tempfile", + "windows-sys 0.61.2", +] + +[[package]] +name = "uiautomation" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c68495a701b9f2f21f29353ac446f0d27dd0d7ce97aa9ccf9061bca0446cd744" +dependencies = [ + "chrono", + "uiautomation_derive", + "windows", + "windows-core", +] + +[[package]] +name = "uiautomation_derive" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffcc4d404aa1c03a848f95cf5feadc3e63946d7f095bf388770b85550093d388" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" +dependencies = [ + "getrandom", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "version-compare" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c2856837ef78f57382f06b2b8563a2f512f7185d732608fd9176cb3b8edf0e" + +[[package]] +name = "wasm-bindgen" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wayland-backend" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "016ccf01d1c58b6f8999612813e17c9b2390f7d70671428869913310f83f54b8" +dependencies = [ + "cc", + "downcast-rs", + "rustix 1.1.4", + "scoped-tls", + "smallvec", + "wayland-sys", +] + +[[package]] +name = "wayland-client" +version = "0.31.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c36a0f861ad76d0901f2800b46321410d9f73f2ea88aac0650d86c32688073" +dependencies = [ + "bitflags", + "rustix 1.1.4", + "wayland-backend", + "wayland-scanner", +] + +[[package]] +name = "wayland-protocols" +version = "0.32.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d0c813de3daa2ed6520af85a3bd49b0e722a3078506899aa9686fea58dc4b6" +dependencies = [ + "bitflags", + "wayland-backend", + "wayland-client", + "wayland-scanner", +] + +[[package]] +name = "wayland-protocols-wlr" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb04e52f7836d7c7976c78ca0250d61e33873c34156a2a1fc9474828ec268234" +dependencies = [ + "bitflags", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "wayland-scanner", +] + +[[package]] +name = "wayland-scanner" +version = "0.31.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "338e30461b3a2b67d70eb30a6d89f8e0c93a833e07d2ae89085cd070c4a00ac0" +dependencies = [ + "proc-macro2", + "quick-xml", + "quote", +] + +[[package]] +name = "wayland-server" +version = "0.31.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dde9c29be0f723a573977de51ee455bf3dfa03652730a74f9dd3b337e374d75" +dependencies = [ + "bitflags", + "downcast-rs", + "rustix 1.1.4", + "wayland-backend", + "wayland-scanner", +] + +[[package]] +name = "wayland-sys" +version = "0.31.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8eab23fefc9e41f8e841df4a9c707e8a8c4ed26e944ef69297184de2785e3be" +dependencies = [ + "dlib", + "libc", + "log", + "memoffset", + "pkg-config", +] + +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "x11rb" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a8885a854a8bfdf87a301e53e41b17c5f8f33639903131338b997b1eb614f44" +dependencies = [ + "gethostname", + "rustix 1.1.4", + "x11rb-protocol", +] + +[[package]] +name = "x11rb-protocol" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acf4d1bc32aa46eec18caa634ec3cf4c05bfa151f12b93b510b15190f69a1ca8" + +[[package]] +name = "xcap" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da270fd7c8581d43d731cb690bcc791fc0a7b3bc96e131a4b6552d0379640a9e" +dependencies = [ + "dispatch2", + "image", + "libwayshot-xcap", + "log", + "objc2", + "objc2-app-kit", + "objc2-av-foundation", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-core-media", + "objc2-core-video", + "objc2-foundation", + "percent-encoding", + "pipewire", + "rand", + "scopeguard", + "serde", + "thiserror", + "url", + "widestring", + "windows", + "xcb", + "zbus", +] + +[[package]] +name = "xcb" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6c2ad15e0e922856ee89afe862b8992334bbe7953adad56cd1199358cb30566" +dependencies = [ + "bitflags", + "libc", + "quick-xml", +] + +[[package]] +name = "xml-rs" +version = "0.8.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e450f9b2ed1dff33c94c12589a87338689467b9c4f5d8a5710bd09a847d2c8a7" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zbus" +version = "5.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5db4be7c075cb421e4b7ee645541604239bd243ba7c357511f4ff3a74b555907" +dependencies = [ + "async-broadcast", + "async-executor", + "async-io", + "async-lock", + "async-process", + "async-recursion", + "async-task", + "async-trait", + "blocking", + "enumflags2", + "event-listener", + "futures-core", + "futures-lite", + "hex", + "libc", + "ordered-stream", + "rustix 1.1.4", + "serde", + "serde_repr", + "tracing", + "uds_windows", + "uuid", + "windows-sys 0.61.2", + "winnow", + "zbus_macros", + "zbus_names", + "zvariant", +] + +[[package]] +name = "zbus-lockstep" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6998de05217a084b7578728a9443d04ea4cd80f2a0839b8d78770b76ccd45863" +dependencies = [ + "zbus_xml", + "zvariant", +] + +[[package]] +name = "zbus-lockstep-macros" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10da05367f3a7b7553c8cdf8fa91aee6b64afebe32b51c95177957efc47ca3a0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "zbus-lockstep", + "zbus_xml", + "zvariant", +] + +[[package]] +name = "zbus_macros" +version = "5.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2990635d09ade6df1868f72f8cac69a876a90981e8bd3c40b1be413f8dc88f40" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 3.0.3", + "zbus_names", + "zvariant", + "zvariant_utils", +] + +[[package]] +name = "zbus_names" +version = "4.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8bf88b4a3ff53e883001e0e0115b297a9d53c31b9c1edd2bfdd853e3428624e" +dependencies = [ + "serde", + "winnow", + "zvariant", +] + +[[package]] +name = "zbus_xml" +version = "5.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1586c021a01ca0a9216dcd874e546382e156a5cbab5fab6cb5f10087e22682a" +dependencies = [ + "serde", + "winnow", + "zbus_names", + "zvariant", +] + +[[package]] +name = "zcheapstr" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1afec51604565183aeb5c54c20aeab286120d4e4460f7f76e3e8bb8c0d99473" +dependencies = [ + "serde", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zvariant" +version = "5.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e28c25bd8bb8da5a1f3e7065d0c156b9ee9a7973adf78b0e35eaefdf3b1b5c" +dependencies = [ + "endi", + "enumflags2", + "serde", + "winnow", + "zcheapstr", + "zvariant_derive", + "zvariant_utils", +] + +[[package]] +name = "zvariant_derive" +version = "5.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d496a145685283b67e232bd9e47377f6b60ad9d51e3601b23867f77c42477f96" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 3.0.3", + "zvariant_utils", +] + +[[package]] +name = "zvariant_utils" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "629d80ece222cad20fe0e8741be493c4ab166acf3b85341bdc2cdbcfd8f3c2d6" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "syn 3.0.3", + "winnow", +] diff --git a/native/t3-desktop-mcp-rs/Cargo.toml b/native/t3-desktop-mcp-rs/Cargo.toml new file mode 100644 index 00000000000..df1ce9ac962 --- /dev/null +++ b/native/t3-desktop-mcp-rs/Cargo.toml @@ -0,0 +1,58 @@ +[package] +name = "t3-desktop-mcp-rs" +version = "0.1.0" +edition = "2024" +license = "MIT" +publish = false + +# The macOS server is a separate Swift package that talks to the Accessibility +# API. This crate covers the platforms Swift cannot reach, and ships the same +# binary name so the server's resolver treats all three identically. +[[bin]] +name = "t3-desktop-mcp" +path = "src/main.rs" + +[dependencies] +serde = { version = "1.0.228", features = ["derive"] } +serde_json = "1.0.150" +base64 = "0.22.1" +# Window capture on both targets; encodes to PNG through `image`. +xcap = "0.9.8" +# Local IPC for the Chrome bridge: named pipes on Windows, Unix sockets on Linux. +interprocess = "2.4.3" +image = { version = "0.25.10", default-features = false, features = ["png"] } + +[target.'cfg(windows)'.dependencies] +# Friendly wrapper over the UI Automation COM API. Raw `windows` bindings would +# put several hundred lines of COM plumbing between us and every tool call. +uiautomation = "0.25.0" +windows = { version = "0.62.2", features = [ + "Win32_Foundation", + "Win32_UI_Input_KeyboardAndMouse", + "Win32_UI_WindowsAndMessaging", + "Win32_System_Threading", + "Win32_System_LibraryLoader", + "Win32_Graphics_Gdi", +] } + +[target.'cfg(unix)'.dependencies] +libc = "0.2" + +[target.'cfg(target_os = "linux")'.dependencies] +# AT-SPI is the only general accessibility surface on Linux. It is async-first, +# so tool handlers block on a tiny executor rather than dragging tokio in. +atspi = { version = "0.30.0", default-features = false, features = ["connection", "proxies"] } +# Resolve AT-SPI bus names to Unix PIDs via org.freedesktop.DBus. +zbus = { version = "5", default-features = false } +futures-lite = "2.6.1" +# XTEST for synthetic input. Wayland deliberately refuses this, which +# `platform::linux` reports as an actionable error rather than a silent no-op. +x11rb = { version = "0.14.0", features = ["xtest", "shape"] } + +[profile.release] +codegen-units = 1 +lto = "thin" +# Deliberately NOT panic = "abort": xcap panics on compositors it does not +# recognise (WSLg among them), and a screenshot must not take the whole server +# down with it. `capture::guarded` turns those panics into tool errors. +strip = true diff --git a/native/t3-desktop-mcp-rs/src/apps.rs b/native/t3-desktop-mcp-rs/src/apps.rs new file mode 100644 index 00000000000..d3aa22eff2d --- /dev/null +++ b/native/t3-desktop-mcp-rs/src/apps.rs @@ -0,0 +1,165 @@ +//! Running-application discovery, shared by the Windows and Linux backends. +//! +//! Both platforms can enumerate windows through `xcap`, and a window list is +//! exactly what `list_apps` reports: an app with no window is not something the +//! model can drive, so grouping windows by pid gives the right answer on both +//! without touching Win32 or X11 directly. + +use std::collections::HashMap; + +use xcap::Window; + +use crate::platform::{AppInfo, DesktopError, Result}; + +struct Group { + name: String, + pid: u32, + windows: usize, + focused: bool, +} + +fn grouped_windows() -> Result> { + // xcap's `Window::all` enumerates via X11/`xcb` when `DISPLAY` is set — no + // process-wide `WAYLAND_DISPLAY` mutation needed (that is UB with threads). + // Same panic hazard as capture: xcap aborts on compositors it cannot read. + let windows = std::panic::catch_unwind(Window::all) + .map_err(|_| DesktopError::new("window enumeration is not supported by this display server"))? + .map_err(|error| DesktopError::new(format!("failed to enumerate windows: {error}")))?; + + let mut groups: HashMap = HashMap::new(); + for window in windows { + let pid = window.pid().unwrap_or(0); + if pid == 0 { + continue; + } + // Some compositors report zero-sized shadow windows; they are not + // something a model can act on and would inflate the window count. + if window.width().unwrap_or(0) == 0 || window.height().unwrap_or(0) == 0 { + continue; + } + + let name = window + .app_name() + .ok() + .filter(|name| !name.is_empty()) + .or_else(|| window.title().ok().filter(|title| !title.is_empty())) + .unwrap_or_else(|| format!("pid {pid}")); + let focused = window.is_focused().unwrap_or(false); + + groups + .entry(pid) + .and_modify(|group| { + group.windows += 1; + group.focused |= focused; + }) + .or_insert(Group { + name, + pid, + windows: 1, + focused, + }); + } + + Ok(groups.into_values().collect()) +} + +pub fn list_apps() -> Result> { + Ok(grouped_windows() + .map(|groups| { + groups + .into_iter() + .map(|group| AppInfo { + id: group.name.to_lowercase().replace(' ', "-"), + name: group.name, + pid: group.pid, + windows: group.windows, + frontmost: group.focused, + }) + .collect() + })?) +} + +/// Resolve an app query to a pid. +/// +/// Accepts a literal pid, an exact name, or a unique case-insensitive substring. +/// An ambiguous substring is an error listing the candidates rather than a guess, +/// because silently driving the wrong window is worse than asking again. +pub fn resolve_pid(query: &str) -> Result { + let query = query.trim(); + if query.is_empty() { + return Err(DesktopError::new( + "app query is empty — call list_apps, or pass a numeric pid", + )); + } + if let Ok(pid) = query.parse::() { + return Ok(pid); + } + + // Minimal window managers (WSLg among them) do not publish the EWMH + // properties window enumeration needs. Keep the guidance rather than + // surfacing an X11 property name the model can do nothing with. + let apps = list_apps().map_err(|error| { + DesktopError::new(format!( + "cannot enumerate windows on this session ({}) — call list_apps, or pass a numeric pid", + error.0 + )) + })?; + let lowered = query.to_lowercase(); + + let exact: Vec<&AppInfo> = apps + .iter() + .filter(|app| app.name.to_lowercase() == lowered || app.id == lowered) + .collect(); + if !exact.is_empty() { + return pick_from_matches(query, &exact); + } + + let matches: Vec<&AppInfo> = apps + .iter() + .filter(|app| app.name.to_lowercase().contains(&lowered)) + .collect(); + pick_from_matches(query, &matches) +} + +fn pick_from_matches(query: &str, matches: &[&AppInfo]) -> Result { + match matches { + [single] => Ok(single.pid), + [] => Err(DesktopError::new(format!( + "no running app matches '{query}' — call list_apps to see what is open" + ))), + several => { + let names: Vec = several + .iter() + .map(|app| format!("{} (pid {})", app.name, app.pid)) + .collect(); + Err(DesktopError::new(format!( + "'{query}' matches several apps: {} — pass a pid to pick one", + names.join(", ") + ))) + } + } +} + +#[cfg(test)] +mod tests { + use super::resolve_pid; + + #[test] + fn a_numeric_query_is_taken_as_a_pid_without_enumerating() { + // Must hold on a headless CI box where window enumeration returns nothing. + assert_eq!(resolve_pid("4321").unwrap(), 4321); + assert_eq!(resolve_pid(" 4321 ").unwrap(), 4321); + } + + #[test] + fn an_unmatched_name_points_at_list_apps() { + let error = resolve_pid("definitely-not-running-xyzzy").unwrap_err().0; + assert!(error.contains("list_apps"), "unhelpful: {error}"); + } + + #[test] + fn whitespace_only_query_does_not_resolve_to_the_sole_app() { + let error = resolve_pid(" ").unwrap_err().0; + assert!(error.contains("empty"), "unhelpful: {error}"); + } +} diff --git a/native/t3-desktop-mcp-rs/src/browser.rs b/native/t3-desktop-mcp-rs/src/browser.rs new file mode 100644 index 00000000000..5ee02b4df69 --- /dev/null +++ b/native/t3-desktop-mcp-rs/src/browser.rs @@ -0,0 +1,772 @@ +//! Agent-owned Chrome tabs, via the T3 Code Chrome extension. +//! +//! Chrome owns the lifetime of a native messaging host: it spawns the host when +//! the extension connects and speaks 4-byte-length-prefixed JSON over that +//! process's stdio. The MCP server is a different process with its own +//! lifetime, so the two are joined by a local socket: +//! +//! ```text +//! Chrome ──stdio(length-prefixed)──▶ `t3-desktop-mcp native-host` +//! │ local socket +//! ▼ +//! MCP server (this process) +//! ``` +//! +//! This mirrors the macOS Swift bridge exactly, including the wire messages, so +//! one extension build serves all three platforms. The server binds the socket, +//! so the first live server claims the browser and later ones fall back to the +//! accessibility tools. + +use std::io::{BufRead, BufReader, Write}; +#[cfg(unix)] +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::mpsc::{Receiver, Sender, channel}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use interprocess::local_socket::{ListenerOptions, SendHalf, Stream}; +#[cfg(unix)] +use interprocess::local_socket::{GenericFilePath, ToFsName}; +#[cfg(windows)] +use interprocess::local_socket::{GenericNamespaced, ToNsName}; +// Imported anonymously: the traits share their names with the enums above. +use interprocess::local_socket::traits::{Listener as _, Stream as _}; +use serde_json::{Value, json}; + +/// Timeout for extension replies; a stuck call must not wedge a turn. +const CALL_TIMEOUT: Duration = Duration::from_secs(20); + +/// User-private filesystem socket (Unix) or user-scoped named pipe (Windows). +/// Abstract / global names are intentionally avoided — they have no ownership. +#[cfg(unix)] +fn bridge_socket_path() -> Option { + let dir = if let Some(runtime) = std::env::var_os("XDG_RUNTIME_DIR") { + PathBuf::from(runtime).join("t3-desktop-mcp") + } else if let Some(home) = std::env::var_os("HOME") { + PathBuf::from(home).join(".local/share/t3-desktop-mcp") + } else { + // Prefer a UID-owned private dir over a USER-named /tmp path another + // local account can pre-create. Fail closed if we cannot claim it. + let uid = unsafe { libc::getuid() }; + std::env::temp_dir().join(format!("t3-desktop-mcp-{uid}")) + }; + + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + // `create_dir_all` / `metadata` / `set_permissions` follow symlinks. A + // pre-planted `/tmp/t3-desktop-mcp-{uid}` → victim-dir symlink would let us + // chmod someone else's directory and drop `bridge.sock` there. Reject + // symlinks via `symlink_metadata` before and after create. + match std::fs::symlink_metadata(&dir) { + Ok(meta) if meta.file_type().is_symlink() => { + eprintln!("t3-desktop-mcp: bridge dir is a symlink; refusing"); + return None; + } + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + if let Err(error) = std::fs::create_dir_all(&dir) { + eprintln!("t3-desktop-mcp: bridge dir create failed: {error}"); + return None; + } + } + Err(error) => { + eprintln!("t3-desktop-mcp: bridge dir metadata failed: {error}"); + return None; + } + } + + let metadata = match std::fs::symlink_metadata(&dir) { + Ok(meta) if meta.file_type().is_symlink() => { + eprintln!("t3-desktop-mcp: bridge dir became a symlink; refusing"); + return None; + } + Ok(metadata) => metadata, + Err(error) => { + eprintln!("t3-desktop-mcp: bridge dir metadata failed: {error}"); + return None; + } + }; + if !metadata.is_dir() { + eprintln!("t3-desktop-mcp: bridge path is not a directory"); + return None; + } + if metadata.uid() != unsafe { libc::getuid() } { + eprintln!("t3-desktop-mcp: bridge dir not owned by current user"); + return None; + } + if let Err(error) = std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)) { + eprintln!("t3-desktop-mcp: bridge dir chmod failed: {error}"); + return None; + } + // Re-check mode after chmod — refuse a sticky/world-writable directory. + let mode = match std::fs::symlink_metadata(&dir) { + Ok(meta) if meta.file_type().is_symlink() => { + eprintln!("t3-desktop-mcp: bridge dir became a symlink after chmod; refusing"); + return None; + } + Ok(metadata) => metadata.mode() & 0o777, + Err(error) => { + eprintln!("t3-desktop-mcp: bridge dir re-stat failed: {error}"); + return None; + } + }; + if mode != 0o700 { + eprintln!("t3-desktop-mcp: bridge dir mode {mode:o} is not 0700"); + return None; + } + Some(dir.join("bridge.sock")) +} + +#[cfg(windows)] +fn bridge_pipe_name() -> String { + let user = std::env::var("USERNAME") + .or_else(|_| std::env::var("USER")) + .unwrap_or_else(|_| "user".into()); + // Named-pipe namespace is global; embed the username so sessions do not collide. + format!("t3-desktop-mcp-bridge-{user}") +} + +pub struct BrowserBridge { + /// Writer half of the accepted connection, once the extension shows up. + outgoing: Arc>>, + replies: Receiver, + next_id: AtomicU64, + /// Bumped on every accept so disconnect sentinels from a prior host are ignored. + connection_gen: Arc, +} + +impl BrowserBridge { + pub fn new() -> Self { + let outgoing: Arc>> = Arc::new(Mutex::new(None)); + let connection_gen = Arc::new(AtomicU64::new(0)); + let (sender, replies) = channel(); + spawn_listener(Arc::clone(&outgoing), Arc::clone(&connection_gen), sender); + Self { + outgoing, + replies, + next_id: AtomicU64::new(1), + connection_gen, + } + } + + /// No listener — used when browser control is disabled so this process does + /// not claim the single per-user bridge socket/pipe. + pub fn inert() -> Self { + let outgoing: Arc>> = Arc::new(Mutex::new(None)); + let connection_gen = Arc::new(AtomicU64::new(0)); + let (_sender, replies) = channel(); + Self { + outgoing, + replies, + next_id: AtomicU64::new(1), + connection_gen, + } + } + + fn connected(&self) -> bool { + self.outgoing.lock().is_ok_and(|guard| guard.is_some()) + } + + /// Dispatch a `browser_*` call. `command` has the `browser_` prefix stripped. + pub fn call(&mut self, command: &str, args: &Value) -> Result { + if !self.connected() { + return Err(format!( + "browser_{command} needs the T3 Code Chrome extension, which is not connected. \ + Install it from native/t3-chrome-extension, or use the desktop tools instead: \ + get_app_state on the browser window, then click" + )); + } + + let command = if command == "press_key" { "press" } else { command }; + let params = self.params_for(command, args)?; + let result = self.dispatch(command, params)?; + Ok(describe(command, &result, args)) + } + + /// Build extension params, resolving 1-based `index` to an owned `tabId` + /// for select_tab / close_tab when `tab_id` was omitted. + fn params_for(&mut self, command: &str, args: &Value) -> Result { + let mut params = normalise(command, args); + if matches!(command, "select_tab" | "close_tab") { + let needs_tab = params + .get("tabId") + .and_then(Value::as_i64) + .is_none(); + if needs_tab { + if let Some(index) = args.get("index").and_then(Value::as_i64) { + let tab_id = self.tab_id_for_index(index)?; + if let Some(map) = params.as_object_mut() { + map.insert("tabId".into(), json!(tab_id)); + map.remove("index"); + } + } + } + } + Ok(params) + } + + fn tab_id_for_index(&mut self, index: i64) -> Result { + if index < 1 { + return Err("index must be a 1-based tab position from browser_list_tabs".into()); + } + let listed = self.dispatch("list_tabs", json!({}))?; + let tabs = listed + .get("tabs") + .and_then(Value::as_array) + .ok_or_else(|| "the extension returned no tab list".to_string())?; + let idx = (index - 1) as usize; + tabs.get(idx) + .and_then(|tab| tab.get("tabId").and_then(Value::as_i64)) + .ok_or_else(|| { + format!( + "no agent tab at index {index} — call browser_list_tabs ({} open)", + tabs.len() + ) + }) + } + + fn dispatch(&mut self, command: &str, params: Value) -> Result { + let id = self.next_id.fetch_add(1, Ordering::Relaxed); + let request = json!({ "id": id, "command": command, "params": params }); + + // Sample connection generation under the outgoing lock so a reconnect + // between load and write cannot pair a new SendHalf with an old gen. + let gen_at_send = { + let mut guard = self + .outgoing + .lock() + .map_err(|_| "the browser bridge is poisoned".to_string())?; + let stream = guard + .as_mut() + .ok_or_else(|| "the extension disconnected".to_string())?; + let generation = self.connection_gen.load(Ordering::SeqCst); + writeln!(stream, "{request}").map_err(|error| format!("could not reach the extension: {error}"))?; + stream + .flush() + .map_err(|error| format!("could not reach the extension: {error}"))?; + generation + }; + + // Replies carry the originating id, so a slow answer to an earlier call + // cannot be mistaken for this one's. Disconnect sentinels are scoped to + // connection_gen so a prior host drop cannot fail a call on the new socket. + let deadline = std::time::Instant::now() + CALL_TIMEOUT; + loop { + let remaining = deadline.saturating_duration_since(std::time::Instant::now()); + if remaining.is_zero() { + return Err(format!("browser_{command} timed out waiting for the extension")); + } + let reply = self + .replies + .recv_timeout(remaining) + .map_err(|_| format!("browser_{command} timed out waiting for the extension"))?; + if reply.get("disconnected").and_then(Value::as_bool) == Some(true) { + let reply_gen = reply.get("connectionGen").and_then(Value::as_u64); + if reply_gen == Some(gen_at_send) { + return Err("the extension disconnected".to_string()); + } + continue; + } + if reply.get("id").and_then(Value::as_u64) != Some(id) { + continue; + } + if reply.get("ok").and_then(Value::as_bool) == Some(true) { + return Ok(reply.get("result").cloned().unwrap_or(json!({}))); + } + return Err(reply + .get("error") + .and_then(Value::as_str) + .unwrap_or("the extension reported an error") + .to_string()); + } + } +} + +impl Default for BrowserBridge { + fn default() -> Self { + Self::new() + } +} + +/// Whether a Unix bridge socket path is owned by a live listener. +#[cfg(unix)] +fn bridge_socket_is_live(path: &std::path::Path) -> Result { + use std::os::unix::net::UnixStream; + if !path.exists() { + return Ok(false); + } + match UnixStream::connect(path) { + Ok(_) => Ok(true), + Err(error) + if error.kind() == std::io::ErrorKind::NotFound + || error.kind() == std::io::ErrorKind::ConnectionRefused => + { + Ok(false) + } + Err(error) if error.raw_os_error() == Some(107) => + { + // ECONNREFUSED on platforms that map it oddly. + Ok(false) + } + Err(_) => Err(()), + } +} + +#[cfg(unix)] +fn unlink_stale_bridge_socket(path: &std::path::Path) { + match bridge_socket_is_live(path) { + Ok(false) => { + let _ = std::fs::remove_file(path); + } + Ok(true) | Err(()) => {} + } +} + +#[cfg(unix)] +struct BridgeSocketCleanup(std::path::PathBuf); + +#[cfg(unix)] +impl Drop for BridgeSocketCleanup { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.0); + } +} + +/// Accept the native host and pump its replies onto `sender`. +fn spawn_listener( + outgoing: Arc>>, + connection_gen: Arc, + sender: Sender, +) { + std::thread::spawn(move || { + #[cfg(unix)] + let path = match bridge_socket_path() { + Some(path) => path, + None => return, + }; + #[cfg(unix)] + let name = match path.as_os_str().to_fs_name::() { + Ok(name) => name, + Err(_) => return, + }; + #[cfg(windows)] + let pipe = bridge_pipe_name(); + #[cfg(windows)] + let name = match pipe.to_ns_name::() { + Ok(name) => name, + Err(_) => return, + }; + // Create-first: never unlink based on a probe that can race another + // server binding between `bridge_socket_is_live` and `remove_file`. + let listener = match ListenerOptions::new().name(name).create_sync() { + Ok(listener) => listener, + Err(_) => { + #[cfg(unix)] + { + unlink_stale_bridge_socket(&path); + let Ok(name) = path.as_os_str().to_fs_name::() else { + return; + }; + let Ok(listener) = ListenerOptions::new().name(name).create_sync() else { + return; + }; + listener + } + #[cfg(not(unix))] + { + // Another server already owns the browser; accessibility + // tools still work, so this is not worth reporting. + return; + } + } + }; + #[cfg(unix)] + let _cleanup = BridgeSocketCleanup(path.clone()); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + if let Err(error) = + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)) + { + eprintln!("t3-desktop-mcp: bridge socket chmod failed: {error}"); + return; + } + } + + let mut accept_failures: u32 = 0; + loop { + let stream = match listener.accept() { + Ok(stream) => { + accept_failures = 0; + stream + } + Err(_) => { + accept_failures = accept_failures.saturating_add(1); + if accept_failures >= 8 { + // Persistent accept errors (listener torn down) — exit + // instead of spinning a CPU core. + return; + } + std::thread::sleep(Duration::from_millis(50 * u64::from(accept_failures))); + continue; + } + }; + let (recv, send) = stream.split(); + // New generation: prior disconnect sentinels become stale and are + // ignored by dispatch (they carry the old connectionGen). + let generation = connection_gen.fetch_add(1, Ordering::SeqCst) + 1; + if let Ok(mut guard) = outgoing.lock() { + *guard = Some(send); + } + + let reader = BufReader::new(recv); + for line in reader.lines() { + let Ok(line) = line else { break }; + if let Ok(value) = serde_json::from_str::(&line) + && sender.send(value).is_err() + { + return; + } + } + + // The host went away; drop the writer so `call` reports honestly, + // and wake any in-flight `dispatch` wait instead of letting it sit + // until CALL_TIMEOUT. Tag with this connection's generation. + if let Ok(mut guard) = outgoing.lock() { + *guard = None; + } + let _ = sender.send(json!({ + "disconnected": true, + "connectionGen": generation, + "ok": false, + "error": "the extension disconnected", + })); + } + }); +} + +/// Translate tool arguments into the extension's parameter names. +fn normalise(_command: &str, args: &Value) -> Value { + let mut params = json!({}); + let map = params.as_object_mut().expect("just built an object"); + if let Some(tab) = args.get("tab_id").and_then(Value::as_i64) { + map.insert("tabId".into(), json!(tab)); + } + for key in ["url", "text", "key", "index", "x", "y"] { + if let Some(value) = args.get(key) { + map.insert(key.into(), value.clone()); + } + } + // Keep `index` in the wire params for commands that still accept it; select_tab + // / close_tab resolve index → tabId in `BrowserBridge::params_for` before dispatch. + params +} + +/// Render a reply as the tool text the macOS server produces. +fn describe(command: &str, result: &Value, args: &Value) -> String { + match command { + // A freshly opened tab has not loaded yet, so the reply usually carries + // no title and no url. Echo the requested address instead of rendering + // an empty pair the model would read as a failed open. + "open_tab" => { + let tab = result.get("tabId").and_then(Value::as_i64).unwrap_or(-1); + let title = result.get("title").and_then(Value::as_str).unwrap_or(""); + let url = result + .get("url") + .and_then(Value::as_str) + .filter(|url| !url.is_empty() && *url != "about:blank") + .or_else(|| args.get("url").and_then(Value::as_str)) + .unwrap_or("about:blank"); + if title.is_empty() { + format!("opened {url} in the agent tab group (tab_id={tab})") + } else { + format!("opened tab_id={tab} — {title} [{url}]") + } + } + "list_tabs" => describe_tabs(result), + "snapshot" => describe_snapshot(result), + "close_all_tabs" => { + let closed = result.get("closed").and_then(Value::as_i64).unwrap_or(0); + if closed == 0 { + "nothing to clean up — the agent had no tabs open".to_string() + } else { + format!( + "closed {closed} agent tab{} and removed the tab group", + if closed == 1 { "" } else { "s" } + ) + } + } + // The remaining commands have no interesting payload, so the useful + // confirmation is what was done and where. Worded as the macOS server + // words it, so a model reads the same feedback on either platform. + other => { + let tab = match other { + "select_tab" => result.get("tabId").and_then(Value::as_i64), + "close_tab" => result + .get("closed") + .and_then(Value::as_i64) + .or_else(|| result.get("tabId").and_then(Value::as_i64)), + _ => None, + } + .or_else(|| args.get("tab_id").and_then(Value::as_i64)) + .or_else(|| args.get("index").and_then(Value::as_i64)) + .unwrap_or(-1); + match other { + "click" => format!("clicked in tab {tab}"), + "type" => format!( + "typed {} characters into tab {tab}", + args.get("text").and_then(Value::as_str).unwrap_or("").chars().count() + ), + "press" => format!( + "pressed {} in tab {tab}", + args.get("key").and_then(Value::as_str).unwrap_or("?") + ), + "navigate" => format!( + "navigated tab {tab} to {}", + args.get("url").and_then(Value::as_str).unwrap_or("") + ), + "select_tab" => format!("switched the agent group to tab {tab}"), + "close_tab" => format!("closed tab {tab}"), + _ => format!("{other} ok"), + } + } + } +} + +fn describe_tabs(result: &Value) -> String { + let tabs = result.get("tabs").and_then(Value::as_array).cloned().unwrap_or_default(); + if tabs.is_empty() { + return "the agent has no tabs open yet — call browser_open_tab".to_string(); + } + let mut lines = vec![format!( + "agent tab group ({} tab{}):", + tabs.len(), + if tabs.len() == 1 { "" } else { "s" } + )]; + for tab in tabs { + lines.push(format!( + "{}tab_id={} {} [{}]", + if tab.get("active").and_then(Value::as_bool) == Some(true) { + "* " + } else { + " " + }, + tab.get("tabId").and_then(Value::as_i64).unwrap_or(-1), + tab.get("title").and_then(Value::as_str).unwrap_or(""), + tab.get("url").and_then(Value::as_str).unwrap_or("") + )); + } + lines.join("\n") +} + +fn describe_snapshot(result: &Value) -> String { + let mut lines = vec![format!( + "{} [{}]", + result.get("title").and_then(Value::as_str).unwrap_or("?"), + result.get("url").and_then(Value::as_str).unwrap_or("") + )]; + for element in result + .get("elements") + .and_then(Value::as_array) + .cloned() + .unwrap_or_default() + { + let label = element.get("label").and_then(Value::as_str).unwrap_or(""); + lines.push(format!( + " [{}] {}{}{}", + element.get("i").and_then(Value::as_i64).unwrap_or(-1), + element.get("tag").and_then(Value::as_str).unwrap_or("?"), + if label.is_empty() { + String::new() + } else { + format!(" \"{label}\"") + }, + if element.get("inView").and_then(Value::as_bool) == Some(false) { + " (scrolled out of view)" + } else { + "" + } + )); + } + lines.join("\n") +} + +/// Relay mode: Chrome on stdio, the MCP server on the local socket. +/// +/// Chrome frames each message with a 4-byte native-endian length; the socket +/// side is newline-delimited JSON, which keeps the server's reader trivial. +pub fn run_native_host() -> std::io::Result<()> { + #[cfg(unix)] + let path = bridge_socket_path().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::NotFound, + "no safe bridge socket path (HOME/XDG_RUNTIME_DIR unset or chmod failed)", + ) + })?; + #[cfg(unix)] + let name = path.as_os_str().to_fs_name::()?; + #[cfg(windows)] + let pipe = bridge_pipe_name(); + #[cfg(windows)] + let name = pipe.to_ns_name::()?; + let stream = Stream::connect(name)?; + let (recv, mut writer) = stream.split(); + + // Server → Chrome. + std::thread::spawn(move || { + let reader = BufReader::new(recv); + let mut stdout = std::io::stdout(); + for line in reader.lines() { + let Ok(line) = line else { break }; + let bytes = line.as_bytes(); + if stdout + .write_all(&(bytes.len() as u32).to_ne_bytes()) + .and_then(|()| stdout.write_all(bytes)) + .and_then(|()| stdout.flush()) + .is_err() + { + break; + } + } + }); + + // Chrome → server. + let mut stdin = std::io::stdin().lock(); + loop { + let mut header = [0u8; 4]; + if std::io::Read::read_exact(&mut stdin, &mut header).is_err() { + return Ok(()); + } + let length = u32::from_ne_bytes(header) as usize; + // Chrome caps messages well below this; a wild length means a desync. + if length == 0 || length > 64 * 1024 * 1024 { + return Ok(()); + } + let mut body = vec![0u8; length]; + if std::io::Read::read_exact(&mut stdin, &mut body).is_err() { + return Ok(()); + } + writer.write_all(&body)?; + writer.write_all(b"\n")?; + writer.flush()?; + } +} + +#[cfg(test)] +mod tests { + use super::{BrowserBridge, describe, describe_snapshot, describe_tabs, normalise}; + use serde_json::json; + + #[test] + fn an_unconnected_bridge_points_at_the_working_alternative() { + let mut bridge = BrowserBridge::new(); + let error = bridge.call("open_tab", &json!({})).unwrap_err(); + + assert!(error.contains("browser_open_tab"), "names the tool: {error}"); + assert!(error.contains("get_app_state"), "offers a path: {error}"); + } + + #[test] + fn tool_arguments_are_renamed_for_the_extension() { + // The tools speak snake_case; the extension speaks camelCase. + let params = normalise("snapshot", &json!({ "tab_id": 7, "text": "hi" })); + assert_eq!(params["tabId"], json!(7)); + assert_eq!(params["text"], json!("hi")); + assert!(params.get("tab_id").is_none()); + } + + #[test] + fn tab_commands_keep_index_in_normalise_for_bridge_resolution() { + // `normalise` leaves index alone; `params_for` resolves it to tabId via list_tabs. + let params = normalise("select_tab", &json!({ "index": 2 })); + assert_eq!(params.get("index"), Some(&json!(2))); + assert!(params.get("tabId").is_none()); + let params = normalise("close_tab", &json!({ "index": 1 })); + assert!(params.get("tabId").is_none()); + } + + #[test] + fn an_empty_tab_list_tells_the_model_what_to_do_next() { + assert!(describe_tabs(&json!({ "tabs": [] })).contains("browser_open_tab")); + } + + #[test] + fn tab_lists_mark_the_active_tab() { + let rendered = describe_tabs(&json!({ + "tabs": [ + { "tabId": 1, "title": "One", "url": "https://one", "active": false }, + { "tabId": 2, "title": "Two", "url": "https://two", "active": true } + ] + })); + assert!(rendered.contains(" tab_id=1"), "{rendered}"); + assert!(rendered.contains("* tab_id=2"), "{rendered}"); + } + + #[test] + fn snapshots_flag_offscreen_elements() { + let rendered = describe_snapshot(&json!({ + "title": "Page", + "url": "https://example", + "elements": [ + { "i": 0, "tag": "button", "label": "Go", "inView": true }, + { "i": 1, "tag": "a", "label": "Hidden", "inView": false } + ] + })); + assert!(rendered.contains("[0] button \"Go\""), "{rendered}"); + assert!(rendered.contains("(scrolled out of view)"), "{rendered}"); + } + + #[test] + fn closing_nothing_is_reported_as_nothing() { + let no_args = json!({}); + assert!(describe("close_all_tabs", &json!({ "closed": 0 }), &no_args).contains("nothing to clean up")); + assert!(describe("close_all_tabs", &json!({ "closed": 1 }), &no_args).contains("closed 1 agent tab ")); + assert!(describe("close_all_tabs", &json!({ "closed": 3 }), &no_args).contains("closed 3 agent tabs")); + } + + #[test] + fn a_freshly_opened_tab_echoes_the_requested_url() { + // Chrome answers before the tab loads, so title and url come back empty; + // rendering that verbatim reads like the open failed. + let rendered = describe( + "open_tab", + &json!({ "tabId": 42 }), + &json!({ "url": "https://example.com" }), + ); + assert!(rendered.contains("https://example.com"), "{rendered}"); + assert!(rendered.contains("tab_id=42"), "{rendered}"); + assert!(!rendered.contains("[]"), "empty url pair leaked: {rendered}"); + } + + #[test] + fn action_confirmations_name_the_tab_they_acted_on() { + // "click ok" tells a model nothing; these mirror the macOS wording. + let tab = json!({ "tab_id": 9 }); + assert_eq!(describe("click", &json!({}), &tab), "clicked in tab 9"); + assert_eq!(describe("select_tab", &json!({}), &tab), "switched the agent group to tab 9"); + assert_eq!(describe("close_tab", &json!({}), &tab), "closed tab 9"); + assert_eq!( + describe("press", &json!({}), &json!({ "tab_id": 9, "key": "Enter" })), + "pressed Enter in tab 9" + ); + assert_eq!( + describe("type", &json!({}), &json!({ "tab_id": 9, "text": "hello" })), + "typed 5 characters into tab 9" + ); + assert_eq!( + describe("navigate", &json!({}), &json!({ "tab_id": 9, "url": "https://a.test" })), + "navigated tab 9 to https://a.test" + ); + } + + #[test] + fn a_loaded_tab_reports_its_own_title() { + let rendered = describe( + "open_tab", + &json!({ "tabId": 7, "title": "Example Domain", "url": "https://example.com/" }), + &json!({}), + ); + assert!(rendered.contains("Example Domain"), "{rendered}"); + } +} diff --git a/native/t3-desktop-mcp-rs/src/capture.rs b/native/t3-desktop-mcp-rs/src/capture.rs new file mode 100644 index 00000000000..04424299d61 --- /dev/null +++ b/native/t3-desktop-mcp-rs/src/capture.rs @@ -0,0 +1,183 @@ +//! Screen and window capture, shared by every non-macOS backend. +//! +//! `xcap` already abstracts Windows' DXGI/GDI path and Linux's X11 path, so the +//! only platform-aware part left is which window belongs to which pid. +//! +//! On Linux hybrid sessions (Wayland + X11), we never mutate `WAYLAND_DISPLAY`: +//! that is UB with concurrent threads. Window enumeration already goes through +//! X11/`xcb` when `DISPLAY` is set. Display capture uses `xcap` only; list and +//! capture stay consistent (no grim-only displays advertised without capture). + +use image::{ImageEncoder, RgbaImage, codecs::png::PngEncoder, imageops::FilterType}; +use xcap::{Monitor, Window}; + +use crate::platform::{DesktopError, Result}; + +/// Run a capture call that may panic inside `xcap`. +/// +/// `xcap` panics rather than erroring on unsupported compositors and protocol +/// versions. Those are ordinary conditions for us — a headless box, an old +/// Wayland — so they become tool errors instead of killing the process. +fn guarded(what: &str, call: impl FnOnce() -> Result) -> Result { + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(call)) { + Ok(result) => result, + Err(_) => Err(DesktopError::new(format!( + "{what} is not supported by this display server — the Wayland screenshot protocols \ + vary by compositor. Use get_app_state to read the UI instead; it does not need a \ + screen capture" + ))), + } +} + +/// Matches the macOS server's default, which keeps a full-screen capture around +/// 200-400 KB of base64 — large enough to read UI text, small enough to not +/// dominate a model's context window. +pub const DEFAULT_MAX_WIDTH: u32 = 1400; + +fn encode_png(image: RgbaImage, max_width: u32) -> Result> { + let image = if max_width > 0 && image.width() > max_width { + let height = ((image.height() as f64) * (max_width as f64) / (image.width() as f64)) + .round() + .max(1.0) as u32; + image::imageops::resize(&image, max_width, height, FilterType::Triangle) + } else { + image + }; + + let mut buffer = Vec::new(); + PngEncoder::new(&mut buffer) + .write_image( + image.as_raw(), + image.width(), + image.height(), + image::ExtendedColorType::Rgba8, + ) + .map_err(|error| DesktopError::new(format!("failed to encode PNG: {error}")))?; + Ok(buffer) +} + +/// Whether the session is Wayland, matching how `xcap` decides. +pub(crate) fn on_wayland() -> bool { + cfg!(target_os = "linux") + && (std::env::var("XDG_SESSION_TYPE").is_ok_and(|value| value == "wayland") + || std::env::var("WAYLAND_DISPLAY").is_ok_and(|value| !value.is_empty())) +} + +pub fn list_displays() -> Result { + guarded("display enumeration", list_displays_inner) +} + +fn list_displays_inner() -> Result { + let monitors = Monitor::all() + .map_err(|error| DesktopError::new(format!("failed to enumerate displays: {error}")))?; + if monitors.is_empty() { + return Ok("no displays detected".to_string()); + } + + let mut lines = Vec::new(); + for (index, monitor) in monitors.iter().enumerate() { + let name = monitor.name().unwrap_or_else(|_| format!("display {index}")); + let width = monitor.width().unwrap_or(0); + let height = monitor.height().unwrap_or(0); + let x = monitor.x().unwrap_or(0); + let y = monitor.y().unwrap_or(0); + let primary = monitor.is_primary().unwrap_or(false); + lines.push(format!( + "[{index}] {name} {width}x{height} at ({x},{y}){}", + if primary { " PRIMARY" } else { "" } + )); + } + Ok(lines.join("\n")) +} + +pub fn capture_display(index: usize, max_width: u32) -> Result> { + guarded("display capture", || capture_display_inner(index, max_width)) +} + +fn capture_display_inner(index: usize, max_width: u32) -> Result> { + let monitors = Monitor::all() + .map_err(|error| DesktopError::new(format!("failed to enumerate displays: {error}")))?; + let monitor = monitors.get(index).ok_or_else(|| { + DesktopError::new(format!( + "display {index} does not exist — call list_displays ({} attached)", + monitors.len() + )) + })?; + let image = monitor + .capture_image() + .map_err(|error| DesktopError::new(format!("failed to capture display: {error}")))?; + encode_png(image, max_width) +} + +/// Capture the largest window owned by `pid`. +/// +/// Largest rather than frontmost: a foreground app often also owns tooltips and +/// tiny helper windows, and the biggest one is reliably the document window the +/// model means. Returns the window title alongside the PNG so the tool text can +/// name what it captured. +pub fn capture_app_window(pid: u32, max_width: u32) -> Result<(Vec, String)> { + guarded("window capture", || capture_app_window_inner(pid, max_width)) +} + +fn capture_app_window_inner(pid: u32, max_width: u32) -> Result<(Vec, String)> { + let windows = Window::all() + .map_err(|error| DesktopError::new(format!("failed to enumerate windows: {error}")))?; + + let mut best: Option<(u32, &Window)> = None; + for window in &windows { + if window.pid().unwrap_or(0) != pid || window.is_minimized().unwrap_or(false) { + continue; + } + let area = window.width().unwrap_or(0).saturating_mul(window.height().unwrap_or(0)); + if area == 0 { + continue; + } + if best.as_ref().is_none_or(|(best_area, _)| area > *best_area) { + best = Some((area, window)); + } + } + + let (_, window) = best.ok_or_else(|| { + DesktopError::new(format!( + "pid {pid} has no capturable window — it may be minimized or have no UI" + )) + })?; + let title = window.title().unwrap_or_default(); + let image = window + .capture_image() + .map_err(|error| DesktopError::new(format!("failed to capture window: {error}")))?; + Ok((encode_png(image, max_width)?, title)) +} + +#[cfg(test)] +mod tests { + use super::{DEFAULT_MAX_WIDTH, encode_png}; + use image::RgbaImage; + + #[test] + fn encodes_a_png_signature() { + let png = encode_png(RgbaImage::new(4, 4), DEFAULT_MAX_WIDTH).expect("encodes"); + assert_eq!(&png[..8], b"\x89PNG\r\n\x1a\n"); + } + + #[test] + fn downscales_only_when_wider_than_the_limit() { + // Narrower than the cap: dimensions must survive untouched, since + // upscaling would waste tokens without adding detail. + let small = encode_png(RgbaImage::new(100, 50), 400).expect("encodes"); + let decoded = image::load_from_memory(&small).expect("decodes"); + assert_eq!((decoded.width(), decoded.height()), (100, 50)); + + // Wider than the cap: scaled down, aspect ratio preserved. + let large = encode_png(RgbaImage::new(1000, 500), 400).expect("encodes"); + let decoded = image::load_from_memory(&large).expect("decodes"); + assert_eq!((decoded.width(), decoded.height()), (400, 200)); + } + + #[test] + fn a_zero_max_width_disables_downscaling() { + let png = encode_png(RgbaImage::new(80, 20), 0).expect("encodes"); + let decoded = image::load_from_memory(&png).expect("decodes"); + assert_eq!((decoded.width(), decoded.height()), (80, 20)); + } +} diff --git a/native/t3-desktop-mcp-rs/src/main.rs b/native/t3-desktop-mcp-rs/src/main.rs new file mode 100644 index 00000000000..ebe12b66dc7 --- /dev/null +++ b/native/t3-desktop-mcp-rs/src/main.rs @@ -0,0 +1,462 @@ +//! Desktop-control MCP server for Windows and Linux. +//! +//! The macOS half of this feature is a Swift package (`native/t3-desktop-mcp`) +//! built on the Accessibility API. This crate covers the other two platforms +//! and speaks the identical MCP dialect — same tool names, same argument shapes, +//! same tool text — so a model needs no per-platform knowledge. +//! +//! Transport is newline-delimited JSON-RPC over stdio, which is what the MCP +//! stdio transport expects. stdout carries protocol only; anything diagnostic +//! goes to stderr so it cannot corrupt a response. + +mod apps; +mod browser; +mod capture; +mod platform; +mod tools; + +use std::io::{self, BufRead, Write}; + +use base64::Engine as _; +use serde_json::{Value, json}; + +use platform::{Desktop, DesktopError, Point, ScrollDirection}; + +const PROTOCOL_VERSION: &str = "2024-11-05"; +const SERVER_NAME: &str = "t3-desktop"; +const SERVER_VERSION: &str = "0.1.0"; + +/// Keeps the agent pointer up for the duration of a `tools/call`, then +/// schedules a fade once Computer Use tools stop for the task. +#[cfg(any(windows, target_os = "linux"))] +struct DesktopToolGuard; + +#[cfg(any(windows, target_os = "linux"))] +use std::sync::atomic::{AtomicUsize, Ordering}; + +#[cfg(any(windows, target_os = "linux"))] +static DESKTOP_TOOL_DEPTH: AtomicUsize = AtomicUsize::new(0); + +#[cfg(any(windows, target_os = "linux"))] +impl DesktopToolGuard { + fn enter() -> Self { + if DESKTOP_TOOL_DEPTH.fetch_add(1, Ordering::SeqCst) == 0 { + platform::agent_cursor::AgentCursor::shared().note_desktop_tool_started(); + } + Self + } +} + +#[cfg(any(windows, target_os = "linux"))] +impl Drop for DesktopToolGuard { + fn drop(&mut self) { + if DESKTOP_TOOL_DEPTH.fetch_sub(1, Ordering::SeqCst) == 1 { + platform::agent_cursor::AgentCursor::shared().note_desktop_tool_finished(); + } + } +} + +fn main() { + // Chrome spawns this same binary as its native messaging host; in that mode + // the process is a relay, not a server. + if std::env::args().nth(1).as_deref() == Some("native-host") { + if let Err(error) = browser::run_native_host() { + eprintln!("t3-desktop-mcp: native host stopped: {error}"); + } + return; + } + + + let stdin = io::stdin(); + let mut stdout = io::stdout(); + + // A backend failure must not kill the process: `initialize` and `tools/list` + // still have to answer so the client can surface a useful error, and the + // reason is far more actionable than a closed pipe. + let mut desktop = match platform::backend() { + Ok(backend) => Some(backend), + Err(error) => { + eprintln!("t3-desktop-mcp: desktop backend unavailable: {error}"); + None + } + }; + let mut browser = if tools::browser_control_enabled() { + browser::BrowserBridge::new() + } else { + browser::BrowserBridge::inert() + }; + + for line in stdin.lock().lines() { + let line = match line { + Ok(line) => line, + Err(error) => { + eprintln!("t3-desktop-mcp: stdin closed: {error}"); + break; + } + }; + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + + let request: Value = match serde_json::from_str(trimmed) { + Ok(value) => value, + Err(error) => { + eprintln!("t3-desktop-mcp: malformed JSON: {error}"); + let response = json!({ + "jsonrpc": "2.0", + "id": null, + "error": { + "code": -32700, + "message": format!("Parse error: {error}") + } + }); + if writeln!(stdout, "{response}").is_err() || stdout.flush().is_err() { + break; + } + continue; + } + }; + + let method = request + .get("method") + .and_then(Value::as_str) + .unwrap_or_default() + .to_string(); + + // Notifications carry no id and must never be answered. + let Some(id) = request.get("id").cloned() else { + if method == "notifications/cancelled" { + #[cfg(any(windows, target_os = "linux"))] + platform::agent_cursor::AgentCursor::shared().hide(); + } + continue; + }; + let params = request.get("params").cloned().unwrap_or(json!({})); + + #[cfg(any(windows, target_os = "linux"))] + let _tool_guard = (method == "tools/call").then(|| DesktopToolGuard::enter()); + + let outcome = dispatch(&method, ¶ms, desktop.as_deref_mut(), &mut browser); + let response = match outcome { + Ok(result) => json!({ "jsonrpc": "2.0", "id": id, "result": result }), + Err(error) => json!({ + "jsonrpc": "2.0", + "id": id, + "error": { "code": error.0, "message": error.1 } + }), + }; + + if writeln!(stdout, "{response}").is_err() || stdout.flush().is_err() { + break; + } + } + + #[cfg(any(windows, target_os = "linux"))] + platform::agent_cursor::AgentCursor::shared().hide(); +} + +/// A JSON-RPC level failure: the request itself was unusable. +struct RpcError(i64, String); + +fn method_not_found(method: &str) -> RpcError { + RpcError(-32601, format!("unknown method '{method}'")) +} + +fn dispatch( + method: &str, + params: &Value, + desktop: Option<&mut (dyn Desktop + '_)>, + browser: &mut browser::BrowserBridge, +) -> Result { + match method { + "initialize" => Ok(json!({ + "protocolVersion": PROTOCOL_VERSION, + "capabilities": { "tools": { "listChanged": false } }, + "serverInfo": { "name": SERVER_NAME, "version": SERVER_VERSION } + })), + "tools/list" => Ok(json!({ "tools": tools::tool_defs() })), + "tools/call" => Ok(call_tool(params, desktop, browser)), + // Ping is part of the base protocol and some clients probe with it. + "ping" => Ok(json!({})), + other => Err(method_not_found(other)), + } +} + +/// Tool failures are reported inside the result as `isError`, not as JSON-RPC +/// errors, so the model reads them as feedback and can retry differently. +fn text_result(text: impl Into, is_error: bool) -> Value { + json!({ + "isError": is_error, + "content": [{ "type": "text", "text": text.into() }] + }) +} + +fn image_result(png: Vec, caption: String) -> Value { + let encoded = base64::engine::general_purpose::STANDARD.encode(png); + json!({ + "isError": false, + "content": [ + { "type": "text", "text": caption }, + { "type": "image", "data": encoded, "mimeType": "image/png" } + ] + }) +} + +fn arg_str<'a>(args: &'a Value, key: &str) -> Option<&'a str> { + args.get(key).and_then(Value::as_str) +} + +fn arg_i64(args: &Value, key: &str) -> Option { + args.get(key).and_then(Value::as_i64) +} + +fn arg_f64(args: &Value, key: &str) -> Option { + args.get(key).and_then(Value::as_f64) +} + +/// Parse an `e12`-style element id into its numeric handle. +fn element_id(raw: &str) -> Result { + raw.trim() + .trim_start_matches(['e', 'E']) + .parse::() + .map_err(|_| { + DesktopError::new(format!( + "'{raw}' is not an element id — pass one from get_app_state, like e12" + )) + }) +} + +/// Resolve the element-or-coordinates pair the pointer tools accept. +fn point_from(args: &Value, element_key: &str, x_key: &str, y_key: &str) -> Result { + if let Some(raw) = arg_str(args, element_key) { + return Ok(Point::Element(element_id(raw)?)); + } + match (arg_f64(args, x_key), arg_f64(args, y_key)) { + (Some(x), Some(y)) => Ok(Point::Screen(x, y)), + _ => Err(DesktopError::new(format!( + "provide {element_key} from get_app_state, or both {x_key} and {y_key}" + ))), + } +} + +fn call_tool( + params: &Value, + desktop: Option<&mut (dyn Desktop + '_)>, + browser: &mut browser::BrowserBridge, +) -> Value { + let name = params + .get("name") + .and_then(Value::as_str) + .unwrap_or_default() + .to_string(); + let args = params.get("arguments").cloned().unwrap_or(json!({})); + + if let Some(rest) = name.strip_prefix("browser_") { + if !tools::browser_control_enabled() { + return text_result( + "error: browser control is disabled in Computer Use settings", + true, + ); + } + return match browser.call(rest, &args) { + Ok(text) => text_result(text, false), + Err(error) => text_result(format!("error: {error}"), true), + }; + } + + // Display listing and whole-display screenshots need no accessibility + // backend, so answer them even when the backend failed to start — they help + // diagnose a headless session. + if name == "list_displays" { + return match capture::list_displays() { + Ok(text) => text_result(text, false), + Err(error) => text_result(format!("error: {error}"), true), + }; + } + if name == "screenshot" + && let Some(display) = arg_i64(&args, "display") + { + let max_width = arg_i64(&args, "max_width") + .unwrap_or(capture::DEFAULT_MAX_WIDTH as i64) + .clamp(0, 8000) as u32; + return match usize::try_from(display) { + Ok(index) => match capture::capture_display(index, max_width) { + Ok(png) => image_result(png, format!("display {index}")), + Err(error) => text_result(format!("error: {error}"), true), + }, + Err(_) => text_result("error: display index must be zero or greater", true), + }; + } + + let Some(desktop) = desktop else { + return text_result( + "error: the desktop backend is unavailable on this host — see stderr for the reason", + true, + ); + }; + + match run_desktop_tool(&name, &args, desktop) { + Ok(value) => value, + Err(error) => text_result(format!("error: {error}"), true), + } +} + +fn run_desktop_tool( + name: &str, + args: &Value, + desktop: &mut dyn Desktop, +) -> Result { + let text = match name { + "list_apps" => desktop.list_apps()?, + "get_app_state" => { + let app = arg_str(args, "app") + .ok_or_else(|| DesktopError::new("missing required argument 'app'"))?; + let max_depth = arg_i64(args, "max_depth").unwrap_or(18).clamp(1, 60) as usize; + let max_elements = arg_i64(args, "max_elements").unwrap_or(800).clamp(1, 5000) as usize; + desktop.get_app_state(app, max_depth, max_elements)? + } + "activate_app" => { + let app = arg_str(args, "app") + .ok_or_else(|| DesktopError::new("missing required argument 'app'"))?; + desktop.activate_app(app)? + } + "click" => { + let count = arg_i64(args, "click_count").unwrap_or(1).clamp(1, 3) as u32; + desktop.click(point_from(args, "element_id", "x", "y")?, count)? + } + "right_click" => desktop.right_click(point_from(args, "element_id", "x", "y")?)?, + "drag" => { + let from = point_from(args, "from_element_id", "from_x", "from_y")?; + let to = point_from(args, "to_element_id", "to_x", "to_y")?; + desktop.drag(from, to)? + } + "type_text" => { + let text = arg_str(args, "text") + .ok_or_else(|| DesktopError::new("missing required argument 'text'"))?; + let element = arg_str(args, "element_id").map(element_id).transpose()?; + desktop.type_text(text, element)? + } + "press_key" => { + let key = arg_str(args, "key") + .ok_or_else(|| DesktopError::new("missing required argument 'key'"))?; + let modifiers: Vec = args + .get("modifiers") + .and_then(Value::as_array) + .map(|values| { + values + .iter() + .filter_map(Value::as_str) + .map(str::to_string) + .collect() + }) + .unwrap_or_default(); + desktop.press_key(key, &modifiers)? + } + "scroll" => { + let direction = ScrollDirection::parse(arg_str(args, "direction").unwrap_or("down"))?; + let amount = arg_i64(args, "amount").unwrap_or(5).clamp(1, 100) as i32; + let element = arg_str(args, "element_id").map(element_id).transpose()?; + desktop.scroll(direction, amount, element)? + } + "set_value" => { + let element = element_id( + arg_str(args, "element_id") + .ok_or_else(|| DesktopError::new("missing required argument 'element_id'"))?, + )?; + let value = arg_str(args, "value") + .ok_or_else(|| DesktopError::new("missing required argument 'value'"))?; + desktop.set_value(element, value)? + } + "select_text" => { + let element = element_id( + arg_str(args, "element_id") + .ok_or_else(|| DesktopError::new("missing required argument 'element_id'"))?, + )?; + let start = arg_i64(args, "start").unwrap_or(0).max(0) as usize; + let length = arg_i64(args, "length").filter(|value| *value >= 0).map(|v| v as usize); + if let Some(len) = length + && start.checked_add(len).is_none() + { + return Err(DesktopError::new("start + length overflows")); + } + desktop.select_text(element, start, length)? + } + "screenshot" => { + let max_width = arg_i64(args, "max_width") + .unwrap_or(capture::DEFAULT_MAX_WIDTH as i64) + .clamp(0, 8000) as u32; + if let Some(display) = arg_i64(args, "display") { + let index = usize::try_from(display).map_err(|_| { + DesktopError::new("display index must be zero or greater") + })?; + let png = capture::capture_display(index, max_width)?; + return Ok(image_result(png, format!("display {index}"))); + } + let app = arg_str(args, "app").ok_or_else(|| { + DesktopError::new("provide 'app' to capture a window, or 'display' for a whole screen") + })?; + let pid = desktop.resolve_pid(app)?; + let (png, title) = capture::capture_app_window(pid, max_width)?; + return Ok(image_result(png, format!("{app} — \"{title}\""))); + } + other => { + return Err(DesktopError::new(format!("unknown tool '{other}'"))); + } + }; + Ok(text_result(text, false)) +} + +#[cfg(test)] +mod tests { + use super::{element_id, point_from, text_result}; + use crate::platform::Point; + use serde_json::json; + + #[test] + fn element_ids_accept_the_advertised_form() { + assert_eq!(element_id("e12").unwrap(), 12); + assert_eq!(element_id("E7").unwrap(), 7); + // Bare numbers are tolerated because models often drop the prefix. + assert_eq!(element_id("3").unwrap(), 3); + assert!(element_id("button").is_err()); + } + + #[test] + fn a_bad_element_id_names_the_tool_that_produces_them() { + let message = element_id("nope").unwrap_err().0; + assert!(message.contains("get_app_state"), "unhelpful: {message}"); + } + + #[test] + fn points_prefer_element_ids_over_coordinates() { + let args = json!({ "element_id": "e5", "x": 10.0, "y": 20.0 }); + assert!(matches!( + point_from(&args, "element_id", "x", "y").unwrap(), + Point::Element(5) + )); + } + + #[test] + fn points_fall_back_to_coordinates() { + let args = json!({ "x": 10.5, "y": 20.5 }); + match point_from(&args, "element_id", "x", "y").unwrap() { + Point::Screen(x, y) => assert_eq!((x, y), (10.5, 20.5)), + other => panic!("expected screen coordinates, got {other:?}"), + } + } + + #[test] + fn a_lone_coordinate_is_rejected_rather_than_guessed() { + // Clicking at (x, 0) because y was forgotten would be worse than an error. + let args = json!({ "x": 10.0 }); + assert!(point_from(&args, "element_id", "x", "y").is_err()); + } + + #[test] + fn tool_errors_are_reported_in_band() { + let result = text_result("error: nope", true); + assert_eq!(result["isError"], json!(true)); + assert_eq!(result["content"][0]["type"], json!("text")); + } +} diff --git a/native/t3-desktop-mcp-rs/src/platform/agent_cursor.rs b/native/t3-desktop-mcp-rs/src/platform/agent_cursor.rs new file mode 100644 index 00000000000..739712b3801 --- /dev/null +++ b/native/t3-desktop-mcp-rs/src/platform/agent_cursor.rs @@ -0,0 +1,14 @@ +//! Agent cursor overlay — Mac parity on Windows and Linux (X11). +//! +//! macOS lives in the Swift `t3-desktop-mcp` package. This module covers the +//! Rust desktop MCP platforms. + +#[cfg(windows)] +#[path = "agent_cursor_windows.rs"] +mod imp; + +#[cfg(target_os = "linux")] +#[path = "agent_cursor_linux.rs"] +mod imp; + +pub use imp::*; diff --git a/native/t3-desktop-mcp-rs/src/platform/agent_cursor_linux.rs b/native/t3-desktop-mcp-rs/src/platform/agent_cursor_linux.rs new file mode 100644 index 00000000000..5a889683c87 --- /dev/null +++ b/native/t3-desktop-mcp-rs/src/platform/agent_cursor_linux.rs @@ -0,0 +1,1096 @@ +//! Linux agent-cursor overlay — Windows/Mac parity (X11). +//! +//! Soft lavender glow, rounded arrow PNG, curved Bezier flight with heading +//! that follows the path tangent (frozen on land), idle breathe. No click ring. +//! Disabled with `T3_DESKTOP_AGENT_CURSOR=0`. +//! +//! Fade is driven by Computer Use `tools/call` activity (see +//! `note_desktop_tool_*`), not a wall-clock idle after the last move. +//! +//! Own X11 connection on a dedicated UI thread (separate from `LinuxDesktop`). +//! Override-redirect 112×112 topmost window; ShapeInput empty so clicks pass +//! through. Prefers a 32-bit ARGB visual; falls back to opaque-ish PutImage. + +use std::sync::Mutex; +use std::sync::OnceLock; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::mpsc::{self, Receiver, Sender, TryRecvError}; +use std::thread; +use std::time::{Duration, Instant}; + +use x11rb::connection::Connection; +use x11rb::protocol::shape::{ConnectionExt as _, SK, SO}; +use x11rb::protocol::xproto::{ + ClipOrdering, ColormapAlloc, ConfigureWindowAux, ConnectionExt as _, CreateGCAux, + CreateWindowAux, ImageFormat, ImageOrder, StackMode, VisualClass, Visualtype, WindowClass, +}; +use x11rb::rust_connection::RustConnection; + +const SIDE: i32 = 112; +const HOTSPOT: f64 = 56.0; +/// Brief grace after the last desktop tools/call before fading — cancelled if +/// another tools/call starts. Override with `T3_DESKTOP_AGENT_CURSOR_TASK_FADE_SECS`. +const DEFAULT_TASK_FADE: Duration = Duration::from_secs(8); +const FADE_OUT_MS: f64 = 350.0; +const FADE_IN_MS: f64 = 500.0; +const TICK_MS: u64 = 16; // ~60fps + +static ENABLED: AtomicBool = AtomicBool::new(true); +static CURSOR: OnceLock = OnceLock::new(); +static LAST_POINT: Mutex> = Mutex::new(None); +static TASK_HIDE_GEN: AtomicU64 = AtomicU64::new(0); +static FADE_TARGET_GEN: AtomicU64 = AtomicU64::new(0); +static FADE_DEADLINE_MS: AtomicU64 = AtomicU64::new(0); +static FADE_WATCHER_STARTED: AtomicBool = AtomicBool::new(false); +static CMD_TX: OnceLock> = OnceLock::new(); +static UI_LIVE: AtomicBool = AtomicBool::new(false); + +enum Cmd { + Move { x: f64, y: f64, press: bool }, + Hide, +} + +pub struct AgentCursor; + +impl AgentCursor { + pub fn shared() -> &'static Self { + CURSOR.get_or_init(|| { + ENABLED.store(agent_cursor_enabled(), Ordering::Relaxed); + if ENABLED.load(Ordering::Relaxed) { + let (tx, rx) = mpsc::channel(); + let _ = CMD_TX.set(tx); + let _ = thread::Builder::new() + .name("t3-agent-cursor".into()) + .spawn(move || ui_thread(rx)); + thread::sleep(Duration::from_millis(120)); + } + Self + }) + } + + pub fn show(&self, x: f64, y: f64) { + if ENABLED.load(Ordering::Relaxed) { + move_and_wait(x, y, false); + } + } + + pub fn press(&self, x: f64, y: f64) { + if ENABLED.load(Ordering::Relaxed) { + move_and_wait(x, y, true); + } + } + + /// Non-blocking hop for mid-drag visuals (must not sleep while a button is down). + pub fn glide(&self, x: f64, y: f64) { + if ENABLED.load(Ordering::Relaxed) { + move_no_wait(x, y); + } + } + + pub fn hide(&self) { + if !ENABLED.load(Ordering::Relaxed) { + return; + } + TASK_HIDE_GEN.fetch_add(1, Ordering::Relaxed); + if let Ok(mut last) = LAST_POINT.lock() { + *last = None; + } + post(Cmd::Hide); + } + + /// A Computer Use `tools/call` is starting — keep the pointer up. + pub fn note_desktop_tool_started(&self) { + if !ENABLED.load(Ordering::Relaxed) { + return; + } + // Cancel any armed fade before bumping the generation so an expired + // watcher cannot hide after this call. + FADE_DEADLINE_MS.store(0, Ordering::SeqCst); + TASK_HIDE_GEN.fetch_add(1, Ordering::SeqCst); + } + + /// A Computer Use `tools/call` finished. Fade once tools stop for this task. + pub fn note_desktop_tool_finished(&self) { + if !ENABLED.load(Ordering::Relaxed) { + return; + } + let used = LAST_POINT + .lock() + .map(|g| g.is_some()) + .unwrap_or(false); + if !used { + return; + } + let generation = TASK_HIDE_GEN.fetch_add(1, Ordering::SeqCst) + 1; + let delay = task_fade_grace(); + FADE_TARGET_GEN.store(generation, Ordering::SeqCst); + FADE_DEADLINE_MS.store( + now_unix_ms().saturating_add(delay.as_millis() as u64), + Ordering::SeqCst, + ); + ensure_fade_watcher(); + } +} + +fn now_unix_ms() -> u64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +fn ensure_fade_watcher() { + if FADE_WATCHER_STARTED + .compare_exchange(false, true, Ordering::Relaxed, Ordering::Relaxed) + .is_err() + { + return; + } + let _ = thread::Builder::new() + .name("t3-agent-cursor-fade".into()) + .spawn(|| { + loop { + thread::sleep(Duration::from_millis(50)); + let deadline = FADE_DEADLINE_MS.load(Ordering::SeqCst); + if deadline == 0 || now_unix_ms() < deadline { + continue; + } + let target = FADE_TARGET_GEN.load(Ordering::SeqCst); + if FADE_DEADLINE_MS + .compare_exchange(deadline, 0, Ordering::SeqCst, Ordering::SeqCst) + .is_err() + { + continue; + } + // Re-check after disarming: `note_desktop_tool_started` may have + // bumped the generation between the load and the CAS. + if TASK_HIDE_GEN.load(Ordering::SeqCst) == target { + AgentCursor::shared().hide(); + } + } + }); +} + +fn task_fade_grace() -> Duration { + match std::env::var("T3_DESKTOP_AGENT_CURSOR_TASK_FADE_SECS") { + Ok(raw) => { + if let Ok(secs) = raw.trim().parse::() { + // from_secs_f64 panics on inf/NaN/overflow — reject those. + if secs.is_finite() && (0.0..3600.0).contains(&secs) { + return Duration::from_secs_f64(secs); + } + } + DEFAULT_TASK_FADE + } + Err(_) => DEFAULT_TASK_FADE, + } +} + +fn move_and_wait(x: f64, y: f64, press: bool) { + if !UI_LIVE.load(Ordering::Relaxed) { + return; + } + let wait = { + let mut last = LAST_POINT.lock().unwrap_or_else(|e| e.into_inner()); + let micros = travel_wait_micros(*last, x, y); + *last = Some((x, y)); + micros + }; + post(Cmd::Move { x, y, press }); + if wait > 0 { + thread::sleep(Duration::from_micros(wait)); + } +} + +/// Fire-and-forget move for mid-drag hops — never blocks with the button held. +fn move_no_wait(x: f64, y: f64) { + if let Ok(mut last) = LAST_POINT.lock() { + *last = Some((x, y)); + } + post(Cmd::Move { + x, + y, + press: false, + }); +} + +/// Approximate flight time for the curved path so clicks wait until landing. +fn travel_wait_micros(from: Option<(f64, f64)>, x: f64, y: f64) -> u64 { + let Some((fx, fy)) = from else { + return 100_000; + }; + let dist = (x - fx).hypot(y - fy); + if dist < 2.0 { + return 60_000; + } + let seconds = (0.18 + dist / 900.0).clamp(0.28, 0.95); + ((seconds + 0.05) * 1_000_000.0) as u64 +} + +fn agent_cursor_enabled() -> bool { + match std::env::var("T3_DESKTOP_AGENT_CURSOR") { + Ok(value) => { + let v = value.trim(); + !(v == "0" || v.eq_ignore_ascii_case("false") || v.eq_ignore_ascii_case("off")) + } + Err(_) => true, + } +} + +fn post(cmd: Cmd) { + if let Some(tx) = CMD_TX.get() { + let _ = tx.send(cmd); + } +} + +struct Anim { + current: Option<(f64, f64)>, + target: (f64, f64), + vel: (f64, f64), + path_from: (f64, f64), + path_c1: (f64, f64), + path_c2: (f64, f64), + path_to: (f64, f64), + path_elapsed: f64, + path_duration: f64, + path_active: bool, + arc_sign: f64, + phase: f64, + tilt: f64, + alpha: f64, + fading: bool, + visible: bool, +} + +impl Anim { + fn new() -> Self { + Self { + current: None, + target: (0.0, 0.0), + vel: (0.0, 0.0), + path_from: (0.0, 0.0), + path_c1: (0.0, 0.0), + path_c2: (0.0, 0.0), + path_to: (0.0, 0.0), + path_elapsed: 0.0, + path_duration: 0.0, + path_active: false, + arc_sign: 1.0, + phase: 0.0, + tilt: 0.0, + alpha: 0.0, + fading: false, + visible: false, + } + } +} + +struct Overlay { + conn: RustConnection, + screen_num: usize, + win: u32, + gc: u32, + depth: u8, + visual: Visualtype, + /// True when depth is 32 and unused bits can carry alpha. + argb: bool, + byte_order: ImageOrder, + bitmap_bit_order: ImageOrder, + bitmap_scanline_pad: u8, + bits_per_pixel: u8, + mapped: bool, + /// Premultiplied BGRA scratch (same layout as Windows). + pixels: Vec, + /// Packed server pixels for PutImage. + put_buf: Vec, +} + +fn ui_thread(rx: Receiver) { + let Ok((conn, screen_num)) = x11rb::connect(None) else { + // No DISPLAY / connect failed — keep shared() alive; show/press are no-ops. + drain_forever(rx); + return; + }; + let setup = conn.setup().clone(); + let screen = &setup.roots[screen_num]; + let byte_order = setup.image_byte_order; + + let (depth, visual, argb) = match find_argb_visual(screen) { + Some((d, v)) => (d, v, true), + None => { + let root_visual = screen + .allowed_depths + .iter() + .flat_map(|d| d.visuals.iter().map(move |v| (d.depth, *v))) + .find(|(_, v)| v.visual_id == screen.root_visual) + .map(|(d, v)| (d, v)) + .unwrap_or(( + screen.root_depth, + Visualtype { + visual_id: screen.root_visual, + class: VisualClass::TRUE_COLOR, + bits_per_rgb_value: 8, + colormap_entries: 256, + red_mask: 0xFF0000, + green_mask: 0x00FF00, + blue_mask: 0x0000FF, + }, + )); + (root_visual.0, root_visual.1, false) + } + }; + + let win = match conn.generate_id() { + Ok(id) => id, + Err(_) => { + drain_forever(rx); + return; + } + }; + let gc = match conn.generate_id() { + Ok(id) => id, + Err(_) => { + drain_forever(rx); + return; + } + }; + let cmap = match conn.generate_id() { + Ok(id) => id, + Err(_) => { + drain_forever(rx); + return; + } + }; + + if conn + .create_colormap(ColormapAlloc::NONE, cmap, screen.root, visual.visual_id) + .is_err() + { + drain_forever(rx); + return; + } + + let aux = CreateWindowAux::new() + .override_redirect(1) + .colormap(cmap) + .border_pixel(0) + .background_pixel(0); + + if conn + .create_window( + depth, + win, + screen.root, + 0, + 0, + SIDE as u16, + SIDE as u16, + 0, + WindowClass::INPUT_OUTPUT, + visual.visual_id, + &aux, + ) + .is_err() + { + let _ = conn.free_colormap(cmap); + drain_forever(rx); + return; + } + + // Clicks pass through: empty ShapeInput region. Without Shape, a topmost + // override-redirect window would eat clicks under the hotspot — fail closed. + if conn.shape_query_version().is_err() + || conn + .shape_rectangles( + SO::SET, + SK::INPUT, + ClipOrdering::UNSORTED, + win, + 0, + 0, + &[], + ) + .is_err() + { + let _ = conn.destroy_window(win); + let _ = conn.free_colormap(cmap); + drain_forever(rx); + return; + } + + if conn + .create_gc(gc, win, &CreateGCAux::new().graphics_exposures(0)) + .is_err() + { + let _ = conn.destroy_window(win); + let _ = conn.free_colormap(cmap); + drain_forever(rx); + return; + } + let _ = conn.flush(); + + let bits_per_pixel = setup + .pixmap_formats + .iter() + .find(|f| f.depth == depth) + .map(|f| f.bits_per_pixel) + .unwrap_or(if depth == 32 { 32 } else { 24 }); + + let mut overlay = Overlay { + conn, + screen_num, + win, + gc, + depth, + visual, + argb, + byte_order, + bitmap_bit_order: setup.bitmap_format_bit_order, + bitmap_scanline_pad: setup.bitmap_format_scanline_pad, + bits_per_pixel, + mapped: false, + pixels: vec![0u8; (SIDE * SIDE * 4) as usize], + put_buf: Vec::new(), + }; + let mut state = Anim::new(); + UI_LIVE.store(true, Ordering::Relaxed); + + loop { + let tick_start = Instant::now(); + + // Drain pending commands. + let mut got = false; + loop { + match rx.try_recv() { + Ok(Cmd::Move { x, y, press }) => { + got = true; + begin(&mut state, x, y, press); + ensure_shown(&mut overlay, &state); + tick(&mut overlay, &mut state); + } + Ok(Cmd::Hide) => { + got = true; + state.fading = true; + tick(&mut overlay, &mut state); + } + Err(TryRecvError::Empty) => break, + Err(TryRecvError::Disconnected) => return, + } + } + + let busy = if !got { + tick(&mut overlay, &mut state) + } else { + state.visible || state.fading || state.path_active || state.alpha > 0.0 + }; + + // Swallow X events so the queue does not fill. A connection error means + // the overlay can never recover — clear UI_LIVE so callers stop waiting. + match overlay.conn.poll_for_event() { + Ok(Some(_)) => { + while overlay.conn.poll_for_event().ok().flatten().is_some() {} + } + Ok(None) => {} + Err(_) => { + UI_LIVE.store(false, Ordering::Relaxed); + return; + } + } + + let elapsed = tick_start.elapsed(); + let frame = Duration::from_millis(TICK_MS); + if busy { + if elapsed < frame { + thread::sleep(frame - elapsed); + } + } else { + // Idle: block until the next command (or a long poll). + match rx.recv_timeout(Duration::from_secs(3600)) { + Ok(Cmd::Move { x, y, press }) => { + begin(&mut state, x, y, press); + ensure_shown(&mut overlay, &state); + tick(&mut overlay, &mut state); + } + Ok(Cmd::Hide) => { + state.fading = true; + tick(&mut overlay, &mut state); + } + Err(mpsc::RecvTimeoutError::Timeout) => {} + Err(mpsc::RecvTimeoutError::Disconnected) => return, + } + } + } +} + +fn drain_forever(rx: Receiver) { + while rx.recv().is_ok() {} +} + +fn find_argb_visual( + screen: &x11rb::protocol::xproto::Screen, +) -> Option<(u8, Visualtype)> { + for depth in &screen.allowed_depths { + if depth.depth != 32 { + continue; + } + for visual in &depth.visuals { + if visual.class == VisualClass::TRUE_COLOR || visual.class == VisualClass::DIRECT_COLOR + { + let rgb = visual.red_mask | visual.green_mask | visual.blue_mask; + // Prefer visuals with spare high bits for alpha. + if rgb.count_ones() <= 24 { + return Some((depth.depth, *visual)); + } + } + } + } + None +} + +fn begin(state: &mut Anim, x: f64, y: f64, _popping: bool) { + state.target = (x, y); + let fresh = state.current.is_none() || !state.visible || state.alpha < 0.05; + if fresh { + state.current = Some((x, y)); + state.vel = (0.0, 0.0); + state.path_active = false; + state.tilt = 0.0; + // Fade in — never pop to full opacity. + state.alpha = 0.0; + state.fading = false; + state.visible = true; + return; + } + + let from = state.current.unwrap_or((x, y)); + let dx = x - from.0; + let dy = y - from.1; + let dist = dx.hypot(dy); + state.alpha = 1.0; + state.fading = false; + state.visible = true; + + if dist < 2.0 { + state.current = Some((x, y)); + state.vel = (0.0, 0.0); + state.path_active = false; + state.tilt = 0.0; + return; + } + + // Cubic flight: bank through cruise, flare upright into the target. + state.arc_sign *= -1.0; + let handle = (dist * 0.35).clamp(40.0, 180.0); + let nx = -dy / dist; + let ny = dx / dist; + let start_dir = if state.tilt.abs() > 0.05 { + let ang = -state.tilt; + (ang.sin(), -ang.cos()) + } else { + (dx / dist, dy / dist) + }; + let depart = handle.min(dist * 0.45); + state.path_from = from; + state.path_to = (x, y); + state.path_c1 = ( + from.0 + start_dir.0 * depart + nx * (dist * 0.18).min(90.0) * state.arc_sign, + from.1 + start_dir.1 * depart + ny * (dist * 0.18).min(90.0) * state.arc_sign, + ); + let approach = (handle * 0.9).min((dist * 0.28).max(28.0)); + state.path_c2 = (x, y + approach); + state.path_duration = (0.18 + dist / 900.0).clamp(0.28, 0.95); + state.path_elapsed = 0.0; + state.path_active = true; + state.vel = (0.0, 0.0); +} + +fn cubic_bezier( + p0: (f64, f64), + p1: (f64, f64), + p2: (f64, f64), + p3: (f64, f64), + t: f64, +) -> (f64, f64) { + let o = 1.0 - t; + let o2 = o * o; + let t2 = t * t; + ( + o2 * o * p0.0 + 3.0 * o2 * t * p1.0 + 3.0 * o * t2 * p2.0 + t2 * t * p3.0, + o2 * o * p0.1 + 3.0 * o2 * t * p1.1 + 3.0 * o * t2 * p2.1 + t2 * t * p3.1, + ) +} + +fn cubic_bezier_tangent( + p0: (f64, f64), + p1: (f64, f64), + p2: (f64, f64), + p3: (f64, f64), + t: f64, +) -> (f64, f64) { + let o = 1.0 - t; + ( + 3.0 * o * o * (p1.0 - p0.0) + 6.0 * o * t * (p2.0 - p1.0) + 3.0 * t * t * (p3.0 - p2.0), + 3.0 * o * o * (p1.1 - p0.1) + 6.0 * o * t * (p2.1 - p1.1) + 3.0 * t * t * (p3.1 - p2.1), + ) +} + +fn ensure_shown(overlay: &mut Overlay, state: &Anim) { + if let Some((cx, cy)) = state.current { + move_window(overlay, cx, cy); + if !overlay.mapped { + let _ = overlay.conn.map_window(overlay.win); + overlay.mapped = true; + let _ = overlay.conn.flush(); + } + } +} + +fn move_window(overlay: &mut Overlay, cx: f64, cy: f64) { + let x = (cx - HOTSPOT).round() as i32; + let y = (cy - HOTSPOT).round() as i32; + let _ = overlay.conn.configure_window( + overlay.win, + &ConfigureWindowAux::new() + .x(x) + .y(y) + .stack_mode(StackMode::ABOVE), + ); +} + +/// Returns false when the animation can sleep. +fn tick(overlay: &mut Overlay, state: &mut Anim) -> bool { + let mut busy = false; + + if state.fading { + state.alpha = (state.alpha - (TICK_MS as f64) / FADE_OUT_MS).max(0.0); + busy = state.alpha > 0.0; + if state.alpha <= 0.0 { + state.visible = false; + state.current = None; + if overlay.mapped { + let _ = overlay.conn.unmap_window(overlay.win); + overlay.mapped = false; + let _ = overlay.conn.flush(); + } + return false; + } + } else if state.visible && state.alpha < 1.0 { + // Fade in on first appear (and after a prior fade-out). + state.alpha = (state.alpha + (TICK_MS as f64) / FADE_IN_MS).min(1.0); + busy = true; + } + + if let Some(mut cur) = state.current { + if state.path_active { + let dt = TICK_MS as f64 / 1000.0; + state.path_elapsed += dt; + let u = (state.path_elapsed / state.path_duration.max(0.001)).min(1.0); + let t = u * u * (3.0 - 2.0 * u); + let pos = cubic_bezier( + state.path_from, + state.path_c1, + state.path_c2, + state.path_to, + t, + ); + let tan = cubic_bezier_tangent( + state.path_from, + state.path_c1, + state.path_c2, + state.path_to, + t, + ); + state.vel = ((pos.0 - cur.0) / dt, (pos.1 - cur.1) / dt); + cur = pos; + state.current = Some(cur); + + let tan_len = tan.0.hypot(tan.1); + if tan_len > 0.001 { + let desired = -tan.0.atan2(-tan.1); + let mut delta = desired - state.tilt; + while delta > std::f64::consts::PI { + delta -= std::f64::consts::TAU; + } + while delta < -std::f64::consts::PI { + delta += std::f64::consts::TAU; + } + let follow = ((0.12 + t * 0.55) + dt * 6.0).min(1.0); + state.tilt += delta * follow; + } + + if u >= 1.0 { + state.current = Some(state.path_to); + state.vel = (0.0, 0.0); + state.tilt = 0.0; // path flared upright + state.path_active = false; + } + busy = true; + + let (cx, cy) = state.current.unwrap_or(cur); + move_window(overlay, cx, cy); + } else { + move_window(overlay, cur.0, cur.1); + } + } + + if state.visible && state.alpha > 0.05 { + state.phase += 0.08; + busy = true; + } + + if state.visible || state.alpha > 0.0 { + render(&mut overlay.pixels, state); + present(overlay, state.alpha); + } + + busy || state.visible +} + +fn present(overlay: &mut Overlay, alpha: f64) { + let a_scale = alpha.clamp(0.0, 1.0); + // Match the server pixmap format exactly — 15/16-bit displays use 2 bytes/pixel. + // Do not force a minimum of 3; PutImage size must match bits_per_pixel. + let bpp = (overlay.bits_per_pixel as usize).div_ceil(8).max(1); + let n = (SIDE * SIDE) as usize; + overlay.put_buf.resize(n * bpp, 0); + + for i in 0..n { + let bi = i * 4; + let b = overlay.pixels[bi] as f64; + let g = overlay.pixels[bi + 1] as f64; + let r = overlay.pixels[bi + 2] as f64; + let a = overlay.pixels[bi + 3] as f64 * a_scale; + // Buffer is premultiplied; re-scale by global fade. + let r8 = (r * a_scale).round().clamp(0.0, 255.0) as u8; + let g8 = (g * a_scale).round().clamp(0.0, 255.0) as u8; + let b8 = (b * a_scale).round().clamp(0.0, 255.0) as u8; + let a8 = a.round().clamp(0.0, 255.0) as u8; + + let pixel = pack_pixel(r8, g8, b8, a8, &overlay.visual, overlay.argb, overlay.byte_order); + let dest = &mut overlay.put_buf[i * bpp..i * bpp + bpp.min(4)]; + let take = dest.len().min(4); + dest.copy_from_slice(&pixel[..take]); + } + + let _ = overlay.conn.put_image( + ImageFormat::Z_PIXMAP, + overlay.win, + overlay.gc, + SIDE as u16, + SIDE as u16, + 0, + 0, + 0, + overlay.depth, + &overlay.put_buf, + ); + + // Without compositing (or without an ARGB visual), opaque PutImage paints a + // black square. Clip via Shape when needed — `argb` alone only proves the + // visual has an alpha channel, not that a CM is compositing it. + if !overlay.argb || !compositing_manager_running(&overlay.conn, overlay.screen_num) { + apply_alpha_bounding_shape(overlay, a_scale); + } + + let _ = overlay.conn.flush(); +} + +fn compositing_manager_running(conn: &RustConnection, screen_num: usize) -> bool { + let name = format!("_NET_WM_CM_S{screen_num}"); + let Ok(atom) = conn.intern_atom(false, name.as_bytes()) else { + return false; + }; + let Ok(atom) = atom.reply() else { + return false; + }; + let Ok(owner) = conn.get_selection_owner(atom.atom) else { + return false; + }; + owner.reply().is_ok_and(|reply| reply.owner != 0) +} + +fn apply_alpha_bounding_shape(overlay: &mut Overlay, a_scale: f64) { + let Ok(pixmap) = overlay.conn.generate_id() else { + return; + }; + let Ok(mask_gc) = overlay.conn.generate_id() else { + return; + }; + if overlay + .conn + .create_pixmap(1, pixmap, overlay.win, SIDE as u16, SIDE as u16) + .is_err() + { + return; + } + // XYBitmap paints set bits with GC foreground and clear bits with + // background. X11 defaults those to 0/1, which inverts Shape polarity + // (1 = inside the window). Force foreground=1, background=0 so opaque + // cursor pixels stay in the BOUNDING region. + if overlay + .conn + .create_gc( + mask_gc, + pixmap, + &CreateGCAux::new() + .graphics_exposures(0) + .foreground(1) + .background(0), + ) + .is_err() + { + let _ = overlay.conn.free_pixmap(pixmap); + return; + } + + let width = SIDE as usize; + let height = SIDE as usize; + // XYBitmap scanlines are padded to bitmap_format_scanline_pad bits. + let pad_bits = usize::from(overlay.bitmap_scanline_pad).max(8); + let stride = width.div_ceil(pad_bits) * (pad_bits / 8); + let mut bits = vec![0u8; stride * height]; + let msb_first = overlay.bitmap_bit_order == ImageOrder::MSB_FIRST; + for y in 0..height { + for x in 0..width { + let a = overlay.pixels[(y * width + x) * 4 + 3] as f64 * a_scale; + if a < 8.0 { + continue; + } + let bit = if msb_first { + 7 - (x % 8) + } else { + x % 8 + }; + bits[y * stride + x / 8] |= 1 << bit; + } + } + + let _ = overlay.conn.put_image( + ImageFormat::XY_BITMAP, + pixmap, + mask_gc, + SIDE as u16, + SIDE as u16, + 0, + 0, + 0, + 1, + &bits, + ); + let _ = overlay.conn.shape_mask( + SO::SET, + SK::BOUNDING, + overlay.win, + 0, + 0, + pixmap, + ); + let _ = overlay.conn.free_gc(mask_gc); + let _ = overlay.conn.free_pixmap(pixmap); +} + +fn place_component(component: u8, mask: u32) -> u32 { + if mask == 0 { + return 0; + } + let shift = mask.trailing_zeros(); + let bits = mask.count_ones(); + let max = (1u32 << bits) - 1; + let scaled = (u32::from(component) * max) / 255; + scaled << shift +} + +fn pack_pixel( + r: u8, + g: u8, + b: u8, + a: u8, + visual: &Visualtype, + argb: bool, + byte_order: ImageOrder, +) -> [u8; 4] { + let mut pixel = + place_component(r, visual.red_mask) | place_component(g, visual.green_mask) | place_component(b, visual.blue_mask); + if argb { + let alpha_mask = !(visual.red_mask | visual.green_mask | visual.blue_mask); + pixel |= place_component(a, alpha_mask); + } + if byte_order == ImageOrder::MSB_FIRST { + pixel.to_be_bytes() + } else { + pixel.to_le_bytes() + } +} + +fn put_px(buf: &mut [u8], x: i32, y: i32, r: u8, g: u8, b: u8, a: u8) { + if x < 0 || y < 0 || x >= SIDE || y >= SIDE || a == 0 { + return; + } + let i = ((y * SIDE + x) * 4) as usize; + // Premultiplied BGRA (same as Windows UpdateLayeredWindow path). + let af = a as u16; + let dst_b = buf[i] as u16; + let dst_g = buf[i + 1] as u16; + let dst_r = buf[i + 2] as u16; + let dst_a = buf[i + 3] as u16; + let inv = 255u16.saturating_sub(af); + let out_a = af + (dst_a * inv + 127) / 255; + let out_b = (b as u16 * af + dst_b * inv + 127) / 255; + let out_g = (g as u16 * af + dst_g * inv + 127) / 255; + let out_r = (r as u16 * af + dst_r * inv + 127) / 255; + buf[i] = out_b.min(255) as u8; + buf[i + 1] = out_g.min(255) as u8; + buf[i + 2] = out_r.min(255) as u8; + buf[i + 3] = out_a.min(255) as u8; +} + +fn radial_glow(buf: &mut [u8], cx: f64, cy: f64, radius: f64) { + // lavender → purple → transparent, matching Mac gradient stops. + let min_x = (cx - radius).floor() as i32; + let max_x = (cx + radius).ceil() as i32; + let min_y = (cy - radius).floor() as i32; + let max_y = (cy + radius).ceil() as i32; + for y in min_y..=max_y { + for x in min_x..=max_x { + let dx = x as f64 + 0.5 - cx; + let dy = y as f64 + 0.5 - cy; + let t = ((dx * dx + dy * dy).sqrt() / radius).clamp(0.0, 1.0); + let (rf, gf, bf, af) = if t < 0.30 { + let u = t / 0.30; + lerp4((0.76, 0.72, 0.99, 0.72), (0.76, 0.72, 0.99, 0.38), u) + } else if t < 0.65 { + let u = (t - 0.30) / 0.35; + lerp4((0.76, 0.72, 0.99, 0.38), (0.58, 0.52, 0.94, 0.14), u) + } else { + let u = (t - 0.65) / 0.35; + lerp4((0.58, 0.52, 0.94, 0.14), (0.58, 0.52, 0.94, 0.0), u) + }; + if af > 0.002 { + put_px( + buf, + x, + y, + (rf * 255.0) as u8, + (gf * 255.0) as u8, + (bf * 255.0) as u8, + (af * 255.0) as u8, + ); + } + } + } +} + +fn lerp4(a: (f64, f64, f64, f64), b: (f64, f64, f64, f64), t: f64) -> (f64, f64, f64, f64) { + ( + a.0 + (b.0 - a.0) * t, + a.1 + (b.1 - a.1) * t, + a.2 + (b.2 - a.2) * t, + a.3 + (b.3 - a.3) * t, + ) +} + +fn render(buf: &mut [u8], state: &Anim) { + buf.fill(0); + + let tip = (HOTSPOT, HOTSPOT); + let breathe = 1.0 + 0.03 * state.phase.sin(); + + // Soft lavender wash with idle breathe (no click ring). + radial_glow(buf, tip.0 + 6.0, tip.1 + 9.0, 34.0 * breathe); + + // Pure 2D: heading rotation only — no squash/stretch. + let sx = 1.0; + let sy = 1.0; + let cos_t = state.tilt.cos(); + let sin_t = state.tilt.sin(); + + blit_cursor_png(buf, tip, sx, sy, cos_t, sin_t); +} + +fn cursor_rgba() -> &'static [(u8, u8, u8, u8)] { + use std::sync::OnceLock; + static PIXELS: OnceLock> = OnceLock::new(); + PIXELS.get_or_init(|| { + let bytes = include_bytes!("cursor_112.png"); + let img = image::load_from_memory(bytes) + .expect("cursor_112.png") + .into_rgba8(); + assert_eq!(img.width(), SIDE as u32); + assert_eq!(img.height(), SIDE as u32); + img.pixels() + .map(|p| { + let [r, g, b, a] = p.0; + (r, g, b, a) + }) + .collect() + }) +} + +fn blit_cursor_png(buf: &mut [u8], tip: (f64, f64), sx: f64, sy: f64, cos_t: f64, sin_t: f64) { + let pixels = cursor_rgba(); + let sx = sx.max(0.01); + let sy = sy.max(0.01); + let inv_det = 1.0 / (sx * sy); + let isx = sy * inv_det; + let isy = sx * inv_det; + let radius = (SIDE as f64) * 0.55 * sx.max(sy); + let min_x = (tip.0 - radius).floor().max(0.0) as i32; + let max_x = (tip.0 + radius).ceil().min((SIDE - 1) as f64) as i32; + let min_y = (tip.1 - radius).floor().max(0.0) as i32; + let max_y = (tip.1 + radius).ceil().min((SIDE - 1) as f64) as i32; + + for y in min_y..=max_y { + for x in min_x..=max_x { + let dx = x as f64 + 0.5 - tip.0; + let dy = y as f64 + 0.5 - tip.1; + let rx = dx * cos_t + dy * sin_t; + let ry = -dx * sin_t + dy * cos_t; + let u = rx * isx + HOTSPOT; + let v = ry * isy + HOTSPOT; + if u < 0.0 || v < 0.0 || u >= (SIDE as f64) - 1.0 || v >= (SIDE as f64) - 1.0 { + continue; + } + let x0 = u.floor() as i32; + let y0 = v.floor() as i32; + let fx = u - x0 as f64; + let fy = v - y0 as f64; + let sample = |xx: i32, yy: i32| -> (f64, f64, f64, f64) { + if xx < 0 || yy < 0 || xx >= SIDE || yy >= SIDE { + return (0.0, 0.0, 0.0, 0.0); + } + let (r, g, b, a) = pixels[(yy * SIDE + xx) as usize]; + (r as f64, g as f64, b as f64, a as f64) + }; + let c00 = sample(x0, y0); + let c10 = sample(x0 + 1, y0); + let c01 = sample(x0, y0 + 1); + let c11 = sample(x0 + 1, y0 + 1); + let mix = |a: f64, b: f64, t: f64| a + (b - a) * t; + let r0 = ( + mix(c00.0, c10.0, fx), + mix(c00.1, c10.1, fx), + mix(c00.2, c10.2, fx), + mix(c00.3, c10.3, fx), + ); + let r1 = ( + mix(c01.0, c11.0, fx), + mix(c01.1, c11.1, fx), + mix(c01.2, c11.2, fx), + mix(c01.3, c11.3, fx), + ); + let a = mix(r0.3, r1.3, fy); + if a > 1.0 { + put_px( + buf, + x, + y, + mix(r0.0, r1.0, fy) as u8, + mix(r0.1, r1.1, fy) as u8, + mix(r0.2, r1.2, fy) as u8, + a as u8, + ); + } + } + } +} diff --git a/native/t3-desktop-mcp-rs/src/platform/agent_cursor_windows.rs b/native/t3-desktop-mcp-rs/src/platform/agent_cursor_windows.rs new file mode 100644 index 00000000000..aa6ea5b063d --- /dev/null +++ b/native/t3-desktop-mcp-rs/src/platform/agent_cursor_windows.rs @@ -0,0 +1,902 @@ +//! Windows agent-cursor overlay — Mac parity. +//! +//! Matches `native/t3-desktop-mcp/Sources/AgentCursor.swift`: +//! soft lavender glow, rounded arrow, curved Bezier flight with heading that +//! follows the path tangent (frozen on land — no upright settle wiggle), +//! idle breathe. No click ring. Disabled with `T3_DESKTOP_AGENT_CURSOR=0`. +//! +//! Fade is driven by Computer Use `tools/call` activity (see +//! `note_desktop_tool_*`), not a wall-clock idle after the last move. + +use std::sync::Mutex; +use std::sync::OnceLock; +use std::sync::atomic::{AtomicBool, AtomicIsize, AtomicU64, Ordering}; +use std::thread; +use std::time::Duration; + +use windows::Win32::Foundation::{COLORREF, HINSTANCE, HWND, LPARAM, LRESULT, POINT, RECT, SIZE, WPARAM}; +use windows::Win32::Graphics::Gdi::{ + AC_SRC_ALPHA, AC_SRC_OVER, BI_RGB, BITMAPINFO, BITMAPINFOHEADER, BLENDFUNCTION, + CreateCompatibleDC, CreateDIBSection, DIB_RGB_COLORS, DeleteDC, DeleteObject, GetDC, HBITMAP, + HGDIOBJ, ReleaseDC, SelectObject, +}; +use windows::Win32::System::LibraryLoader::GetModuleHandleW; +use windows::Win32::UI::WindowsAndMessaging::{ + CreateWindowExW, DefWindowProcW, DispatchMessageW, GetMessageW, GetWindowRect, KillTimer, MSG, + PostMessageW, PostQuitMessage, RegisterClassExW, SetTimer, SetWindowPos, ShowWindow, + TranslateMessage, UpdateLayeredWindow, CS_HREDRAW, CS_VREDRAW, HWND_TOPMOST, SWP_NOACTIVATE, + SWP_NOSIZE, SWP_SHOWWINDOW, SW_HIDE, SW_SHOWNOACTIVATE, ULW_ALPHA, WM_DESTROY, WM_TIMER, + WM_USER, WNDCLASSEXW, WS_EX_LAYERED, WS_EX_NOACTIVATE, WS_EX_TOOLWINDOW, WS_EX_TOPMOST, + WS_EX_TRANSPARENT, WS_POPUP, +}; +use windows::core::w; + +const SIDE: i32 = 112; +const HOTSPOT: f64 = 56.0; +const WM_AGENT_MOVE: u32 = WM_USER + 40; +const WM_AGENT_PRESS: u32 = WM_USER + 41; +const WM_AGENT_HIDE: u32 = WM_USER + 42; +/// Brief grace after the last desktop tools/call before fading — cancelled if +/// another tools/call starts. Override with `T3_DESKTOP_AGENT_CURSOR_TASK_FADE_SECS`. +const DEFAULT_TASK_FADE: Duration = Duration::from_secs(8); +const FADE_OUT_MS: f64 = 350.0; +const FADE_IN_MS: f64 = 500.0; +const TICK_MS: u32 = 16; // ~60fps + +static ENABLED: AtomicBool = AtomicBool::new(true); +static HWND_PTR: AtomicIsize = AtomicIsize::new(0); +static CURSOR: OnceLock = OnceLock::new(); +static LAST_POINT: Mutex> = Mutex::new(None); +static TASK_HIDE_GEN: AtomicU64 = AtomicU64::new(0); +/// Generation that should trigger hide when `FADE_DEADLINE_MS` elapses. +static FADE_TARGET_GEN: AtomicU64 = AtomicU64::new(0); +/// Deadline (ms since unix epoch) for the single fade watcher thread. +static FADE_DEADLINE_MS: AtomicU64 = AtomicU64::new(0); +static FADE_WATCHER_STARTED: AtomicBool = AtomicBool::new(false); + +pub struct AgentCursor; + +impl AgentCursor { + pub fn shared() -> &'static Self { + CURSOR.get_or_init(|| { + ENABLED.store(agent_cursor_enabled(), Ordering::Relaxed); + if ENABLED.load(Ordering::Relaxed) { + let _ = thread::Builder::new() + .name("t3-agent-cursor".into()) + .spawn(ui_thread); + thread::sleep(Duration::from_millis(120)); + } + Self + }) + } + + pub fn show(&self, x: f64, y: f64) { + if ENABLED.load(Ordering::Relaxed) { + move_and_wait(x, y, false); + } + } + + pub fn press(&self, x: f64, y: f64) { + if ENABLED.load(Ordering::Relaxed) { + move_and_wait(x, y, true); + } + } + + /// Non-blocking hop for mid-drag visuals (must not sleep while a button is down). + pub fn glide(&self, x: f64, y: f64) { + if ENABLED.load(Ordering::Relaxed) { + move_no_wait(x, y); + } + } + + pub fn hide(&self) { + if !ENABLED.load(Ordering::Relaxed) { + return; + } + TASK_HIDE_GEN.fetch_add(1, Ordering::Relaxed); + if let Ok(mut last) = LAST_POINT.lock() { + *last = None; + } + let hwnd = HWND(HWND_PTR.load(Ordering::Relaxed) as *mut _); + if hwnd.0.is_null() { + return; + } + unsafe { + let _ = PostMessageW(Some(hwnd), WM_AGENT_HIDE, WPARAM(0), LPARAM(0)); + } + } + + /// A Computer Use `tools/call` is starting — keep the pointer up. + pub fn note_desktop_tool_started(&self) { + if !ENABLED.load(Ordering::Relaxed) { + return; + } + // Cancel any armed fade before bumping the generation so an expired + // watcher cannot hide after this call. + FADE_DEADLINE_MS.store(0, Ordering::SeqCst); + TASK_HIDE_GEN.fetch_add(1, Ordering::SeqCst); + } + + /// A Computer Use `tools/call` finished. Fade once tools stop for this task. + pub fn note_desktop_tool_finished(&self) { + if !ENABLED.load(Ordering::Relaxed) { + return; + } + let used = LAST_POINT + .lock() + .map(|g| g.is_some()) + .unwrap_or(false); + if !used { + return; + } + let generation = TASK_HIDE_GEN.fetch_add(1, Ordering::SeqCst) + 1; + let delay = task_fade_grace(); + FADE_TARGET_GEN.store(generation, Ordering::SeqCst); + FADE_DEADLINE_MS.store( + now_unix_ms().saturating_add(delay.as_millis() as u64), + Ordering::SeqCst, + ); + ensure_fade_watcher(); + } +} + +fn now_unix_ms() -> u64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +/// One long-lived watcher thread — avoids spawning a sleeper per tools/call. +fn ensure_fade_watcher() { + if FADE_WATCHER_STARTED + .compare_exchange(false, true, Ordering::Relaxed, Ordering::Relaxed) + .is_err() + { + return; + } + let _ = thread::Builder::new() + .name("t3-agent-cursor-fade".into()) + .spawn(|| { + loop { + thread::sleep(Duration::from_millis(50)); + let deadline = FADE_DEADLINE_MS.load(Ordering::SeqCst); + if deadline == 0 || now_unix_ms() < deadline { + continue; + } + let target = FADE_TARGET_GEN.load(Ordering::SeqCst); + // Clear only if this deadline is still armed (a newer finish + // may have replaced it while we slept). + if FADE_DEADLINE_MS + .compare_exchange(deadline, 0, Ordering::SeqCst, Ordering::SeqCst) + .is_err() + { + continue; + } + // Re-check after disarming: `note_desktop_tool_started` may have + // bumped the generation between the load and the CAS. + if TASK_HIDE_GEN.load(Ordering::SeqCst) == target { + AgentCursor::shared().hide(); + } + } + }); +} + +fn task_fade_grace() -> Duration { + match std::env::var("T3_DESKTOP_AGENT_CURSOR_TASK_FADE_SECS") { + Ok(raw) => { + if let Ok(secs) = raw.trim().parse::() { + // from_secs_f64 panics on inf/NaN/overflow — reject those. + if secs.is_finite() && (0.0..3600.0).contains(&secs) { + return Duration::from_secs_f64(secs); + } + } + DEFAULT_TASK_FADE + } + Err(_) => DEFAULT_TASK_FADE, + } +} + +fn move_and_wait(x: f64, y: f64, press: bool) { + let wait = { + let mut last = LAST_POINT.lock().unwrap_or_else(|e| e.into_inner()); + let micros = travel_wait_micros(*last, x, y); + *last = Some((x, y)); + micros + }; + post( + if press { + WM_AGENT_PRESS + } else { + WM_AGENT_MOVE + }, + x, + y, + ); + if wait > 0 { + thread::sleep(Duration::from_micros(wait)); + } +} + +/// Fire-and-forget move for mid-drag hops — never blocks with the button held. +fn move_no_wait(x: f64, y: f64) { + if let Ok(mut last) = LAST_POINT.lock() { + *last = Some((x, y)); + } + post(WM_AGENT_MOVE, x, y); +} + +/// Approximate flight time for the curved path so clicks wait until landing. +fn travel_wait_micros(from: Option<(f64, f64)>, x: f64, y: f64) -> u64 { + let Some((fx, fy)) = from else { + return 100_000; + }; + let dist = (x - fx).hypot(y - fy); + if dist < 2.0 { + return 60_000; + } + let seconds = (0.18 + dist / 900.0).clamp(0.28, 0.95); + ((seconds + 0.05) * 1_000_000.0) as u64 +} + +fn agent_cursor_enabled() -> bool { + match std::env::var("T3_DESKTOP_AGENT_CURSOR") { + Ok(value) => { + let v = value.trim(); + !(v == "0" || v.eq_ignore_ascii_case("false") || v.eq_ignore_ascii_case("off")) + } + Err(_) => true, + } +} + +fn post(msg: u32, x: f64, y: f64) { + let hwnd = HWND(HWND_PTR.load(Ordering::Relaxed) as *mut _); + if hwnd.0.is_null() { + return; + } + let xi = x.round().clamp(i16::MIN as f64, i16::MAX as f64) as i16 as u16 as isize; + let yi = y.round().clamp(i16::MIN as f64, i16::MAX as f64) as i16 as u16 as isize; + unsafe { + let _ = PostMessageW(Some(hwnd), msg, WPARAM(0), LPARAM((yi << 16) | xi)); + } +} + +struct Anim { + current: Option<(f64, f64)>, + target: (f64, f64), + vel: (f64, f64), + path_from: (f64, f64), + path_c1: (f64, f64), + path_c2: (f64, f64), + path_to: (f64, f64), + path_elapsed: f64, + path_duration: f64, + path_active: bool, + arc_sign: f64, + phase: f64, + tilt: f64, + alpha: f64, + fading: bool, + visible: bool, +} + +impl Anim { + fn new() -> Self { + Self { + current: None, + target: (0.0, 0.0), + vel: (0.0, 0.0), + path_from: (0.0, 0.0), + path_c1: (0.0, 0.0), + path_c2: (0.0, 0.0), + path_to: (0.0, 0.0), + path_elapsed: 0.0, + path_duration: 0.0, + path_active: false, + arc_sign: 1.0, + phase: 0.0, + tilt: 0.0, + alpha: 0.0, + fading: false, + visible: false, + } + } +} + +struct Framebuf { + bits: *mut u8, + hdc: windows::Win32::Graphics::Gdi::HDC, + dib: HBITMAP, + old: HGDIOBJ, +} + +fn ui_thread() { + unsafe { + let class = w!("T3AgentCursorOverlay"); + let module = GetModuleHandleW(None).unwrap_or_default(); + let wc = WNDCLASSEXW { + cbSize: std::mem::size_of::() as u32, + style: CS_HREDRAW | CS_VREDRAW, + lpfnWndProc: Some(wnd_proc), + hInstance: HINSTANCE(module.0), + lpszClassName: class, + ..Default::default() + }; + let _ = RegisterClassExW(&wc); + + let hwnd = match CreateWindowExW( + WS_EX_LAYERED | WS_EX_TRANSPARENT | WS_EX_TOPMOST | WS_EX_TOOLWINDOW | WS_EX_NOACTIVATE, + class, + w!("T3 Agent Cursor"), + WS_POPUP, + 0, + 0, + SIDE, + SIDE, + None, + None, + Some(HINSTANCE(module.0)), + None, + ) { + Ok(hwnd) => hwnd, + Err(_) => return, + }; + HWND_PTR.store(hwnd.0 as isize, Ordering::Relaxed); + let _ = ShowWindow(hwnd, SW_HIDE); + + let mut message = MSG::default(); + while GetMessageW(&mut message, None, 0, 0).as_bool() { + let _ = TranslateMessage(&message); + DispatchMessageW(&message); + } + HWND_PTR.store(0, Ordering::Relaxed); + } +} + +thread_local! { + static STATE: std::cell::RefCell = std::cell::RefCell::new(Anim::new()); + static FB: std::cell::RefCell> = const { std::cell::RefCell::new(None) }; +} + +unsafe extern "system" fn wnd_proc( + hwnd: HWND, + msg: u32, + wparam: WPARAM, + lparam: LPARAM, +) -> LRESULT { + unsafe { + match msg { + WM_AGENT_MOVE | WM_AGENT_PRESS => { + let x = (lparam.0 & 0xFFFF) as i16 as f64; + let y = ((lparam.0 >> 16) & 0xFFFF) as i16 as f64; + let popping = msg == WM_AGENT_PRESS; + STATE.with(|cell| { + let mut state = cell.borrow_mut(); + begin(&mut state, x, y, popping); + ensure_shown(hwnd, &state); + }); + FB.with(|cell| { + let mut slot = cell.borrow_mut(); + if slot.is_none() { + *slot = make_framebuf(hwnd); + } + }); + let _ = SetTimer(Some(hwnd), 1, TICK_MS, None); + STATE.with(|state_cell| { + FB.with(|fb_cell| { + let mut state = state_cell.borrow_mut(); + let mut fb = fb_cell.borrow_mut(); + tick(hwnd, &mut state, fb.as_mut()); + }); + }); + LRESULT(0) + } + WM_AGENT_HIDE => { + STATE.with(|cell| { + let mut state = cell.borrow_mut(); + state.fading = true; + }); + let _ = SetTimer(Some(hwnd), 1, TICK_MS, None); + LRESULT(0) + } + WM_TIMER => { + let keep = STATE.with(|state_cell| { + FB.with(|fb_cell| { + let mut state = state_cell.borrow_mut(); + let mut fb = fb_cell.borrow_mut(); + tick(hwnd, &mut state, fb.as_mut()) + }) + }); + if !keep { + let _ = KillTimer(Some(hwnd), 1); + } + LRESULT(0) + } + WM_DESTROY => { + FB.with(|cell| { + if let Some(fb) = cell.borrow_mut().take() { + destroy_framebuf(fb); + } + }); + PostQuitMessage(0); + LRESULT(0) + } + _ => DefWindowProcW(hwnd, msg, wparam, lparam), + } + } +} + +fn begin(state: &mut Anim, x: f64, y: f64, _popping: bool) { + state.target = (x, y); + let fresh = state.current.is_none() || !state.visible || state.alpha < 0.05; + if fresh { + state.current = Some((x, y)); + state.vel = (0.0, 0.0); + state.path_active = false; + state.tilt = 0.0; + // Fade in — never pop to full opacity. + state.alpha = 0.0; + state.fading = false; + state.visible = true; + return; + } + + let from = state.current.unwrap_or((x, y)); + let dx = x - from.0; + let dy = y - from.1; + let dist = dx.hypot(dy); + state.alpha = 1.0; + state.fading = false; + state.visible = true; + + if dist < 2.0 { + state.current = Some((x, y)); + state.vel = (0.0, 0.0); + state.path_active = false; + state.tilt = 0.0; + return; + } + + // Cubic flight: bank through cruise, flare upright into the target. + state.arc_sign *= -1.0; + let handle = (dist * 0.35).clamp(40.0, 180.0); + let nx = -dy / dist; + let ny = dx / dist; + let start_dir = if state.tilt.abs() > 0.05 { + let ang = -state.tilt; + (ang.sin(), -ang.cos()) + } else { + (dx / dist, dy / dist) + }; + let depart = handle.min(dist * 0.45); + state.path_from = from; + state.path_to = (x, y); + state.path_c1 = ( + from.0 + start_dir.0 * depart + nx * (dist * 0.18).min(90.0) * state.arc_sign, + from.1 + start_dir.1 * depart + ny * (dist * 0.18).min(90.0) * state.arc_sign, + ); + let approach = (handle * 0.9).min((dist * 0.28).max(28.0)); + state.path_c2 = (x, y + approach); + state.path_duration = (0.18 + dist / 900.0).clamp(0.28, 0.95); + state.path_elapsed = 0.0; + state.path_active = true; + state.vel = (0.0, 0.0); +} + +fn cubic_bezier( + p0: (f64, f64), + p1: (f64, f64), + p2: (f64, f64), + p3: (f64, f64), + t: f64, +) -> (f64, f64) { + let o = 1.0 - t; + let o2 = o * o; + let t2 = t * t; + ( + o2 * o * p0.0 + 3.0 * o2 * t * p1.0 + 3.0 * o * t2 * p2.0 + t2 * t * p3.0, + o2 * o * p0.1 + 3.0 * o2 * t * p1.1 + 3.0 * o * t2 * p2.1 + t2 * t * p3.1, + ) +} + +fn cubic_bezier_tangent( + p0: (f64, f64), + p1: (f64, f64), + p2: (f64, f64), + p3: (f64, f64), + t: f64, +) -> (f64, f64) { + let o = 1.0 - t; + ( + 3.0 * o * o * (p1.0 - p0.0) + 6.0 * o * t * (p2.0 - p1.0) + 3.0 * t * t * (p3.0 - p2.0), + 3.0 * o * o * (p1.1 - p0.1) + 6.0 * o * t * (p2.1 - p1.1) + 3.0 * t * t * (p3.1 - p2.1), + ) +} + +unsafe fn ensure_shown(hwnd: HWND, state: &Anim) { + if let Some((cx, cy)) = state.current { + unsafe { + let _ = SetWindowPos( + hwnd, + Some(HWND_TOPMOST), + (cx - HOTSPOT).round() as i32, + (cy - HOTSPOT).round() as i32, + 0, + 0, + SWP_NOSIZE | SWP_NOACTIVATE | SWP_SHOWWINDOW, + ); + let _ = ShowWindow(hwnd, SW_SHOWNOACTIVATE); + } + } +} + +/// Returns false when the animation can sleep (timer may stop). +unsafe fn tick(hwnd: HWND, state: &mut Anim, fb: Option<&mut Framebuf>) -> bool { + let mut busy = false; + + if state.fading { + state.alpha = (state.alpha - (TICK_MS as f64) / FADE_OUT_MS).max(0.0); + busy = state.alpha > 0.0; + if state.alpha <= 0.0 { + state.visible = false; + state.current = None; + unsafe { + let _ = ShowWindow(hwnd, SW_HIDE); + } + return false; + } + } else if state.visible && state.alpha < 1.0 { + // Fade in on first appear (and after a prior fade-out). + state.alpha = (state.alpha + (TICK_MS as f64) / FADE_IN_MS).min(1.0); + busy = true; + } + + if let Some(mut cur) = state.current { + if state.path_active { + let dt = TICK_MS as f64 / 1000.0; + state.path_elapsed += dt; + let u = (state.path_elapsed / state.path_duration.max(0.001)).min(1.0); + let t = u * u * (3.0 - 2.0 * u); + let pos = cubic_bezier( + state.path_from, + state.path_c1, + state.path_c2, + state.path_to, + t, + ); + let tan = cubic_bezier_tangent( + state.path_from, + state.path_c1, + state.path_c2, + state.path_to, + t, + ); + state.vel = ((pos.0 - cur.0) / dt, (pos.1 - cur.1) / dt); + cur = pos; + state.current = Some(cur); + + let tan_len = tan.0.hypot(tan.1); + if tan_len > 0.001 { + let desired = -tan.0.atan2(-tan.1); + let mut delta = desired - state.tilt; + while delta > std::f64::consts::PI { + delta -= std::f64::consts::TAU; + } + while delta < -std::f64::consts::PI { + delta += std::f64::consts::TAU; + } + let follow = ((0.12 + t * 0.55) + dt * 6.0).min(1.0); + state.tilt += delta * follow; + } + + if u >= 1.0 { + state.current = Some(state.path_to); + state.vel = (0.0, 0.0); + state.tilt = 0.0; // path flared upright + state.path_active = false; + } + busy = true; + + unsafe { + let (cx, cy) = state.current.unwrap_or(cur); + let _ = SetWindowPos( + hwnd, + Some(HWND_TOPMOST), + (cx - HOTSPOT).round() as i32, + (cy - HOTSPOT).round() as i32, + 0, + 0, + SWP_NOSIZE | SWP_NOACTIVATE | SWP_SHOWWINDOW, + ); + } + } else { + unsafe { + let _ = SetWindowPos( + hwnd, + Some(HWND_TOPMOST), + (cur.0 - HOTSPOT).round() as i32, + (cur.1 - HOTSPOT).round() as i32, + 0, + 0, + SWP_NOSIZE | SWP_NOACTIVATE | SWP_SHOWWINDOW, + ); + } + } + } + + if state.visible && state.alpha > 0.05 { + state.phase += 0.08; + busy = true; + } + + if let Some(fb) = fb { + unsafe { + render(fb, state); + present(hwnd, fb, state.alpha); + } + } + + busy || state.visible +} + +unsafe fn make_framebuf(hwnd: HWND) -> Option { + unsafe { + let screen = GetDC(Some(hwnd)); + if screen.is_invalid() { + return None; + } + let hdc = CreateCompatibleDC(Some(screen)); + let _ = ReleaseDC(Some(hwnd), screen); + if hdc.is_invalid() { + return None; + } + + let mut info = BITMAPINFO { + bmiHeader: BITMAPINFOHEADER { + biSize: std::mem::size_of::() as u32, + biWidth: SIDE, + biHeight: -SIDE, // top-down + biPlanes: 1, + biBitCount: 32, + biCompression: BI_RGB.0, + ..Default::default() + }, + ..Default::default() + }; + let mut bits: *mut std::ffi::c_void = std::ptr::null_mut(); + let dib = match CreateDIBSection(Some(hdc), &info, DIB_RGB_COLORS, &mut bits, None, 0) { + Ok(dib) if !bits.is_null() => dib, + _ => { + let _ = DeleteDC(hdc); + return None; + } + }; + let old = SelectObject(hdc, HGDIOBJ(dib.0)); + Some(Framebuf { + bits: bits as *mut u8, + hdc, + dib, + old, + }) + } +} + +unsafe fn destroy_framebuf(fb: Framebuf) { + unsafe { + let _ = SelectObject(fb.hdc, fb.old); + let _ = DeleteObject(fb.dib.into()); + let _ = DeleteDC(fb.hdc); + } +} + +unsafe fn present(hwnd: HWND, fb: &Framebuf, alpha: f64) { + unsafe { + let mut src = POINT { x: 0, y: 0 }; + let mut size = SIZE { + cx: SIDE, + cy: SIDE, + }; + let mut dst = POINT { x: 0, y: 0 }; + let mut rect = RECT::default(); + let _ = GetWindowRect(hwnd, &mut rect); + dst.x = rect.left; + dst.y = rect.top; + + let blend = BLENDFUNCTION { + BlendOp: AC_SRC_OVER as u8, + BlendFlags: 0, + SourceConstantAlpha: (alpha.clamp(0.0, 1.0) * 255.0).round() as u8, + AlphaFormat: AC_SRC_ALPHA as u8, + }; + let _ = UpdateLayeredWindow( + hwnd, + None, + Some(&dst), + Some(&size), + Some(fb.hdc), + Some(&src), + COLORREF(0), + Some(&blend), + ULW_ALPHA, + ); + } +} + +fn put_px(buf: &mut [u8], x: i32, y: i32, r: u8, g: u8, b: u8, a: u8) { + if x < 0 || y < 0 || x >= SIDE || y >= SIDE || a == 0 { + return; + } + let i = ((y * SIDE + x) * 4) as usize; + // Premultiplied BGRA for UpdateLayeredWindow + let af = a as u16; + let dst_b = buf[i] as u16; + let dst_g = buf[i + 1] as u16; + let dst_r = buf[i + 2] as u16; + let dst_a = buf[i + 3] as u16; + let inv = 255u16.saturating_sub(af); + let out_a = af + (dst_a * inv + 127) / 255; + let out_b = (b as u16 * af + dst_b * inv + 127) / 255; + let out_g = (g as u16 * af + dst_g * inv + 127) / 255; + let out_r = (r as u16 * af + dst_r * inv + 127) / 255; + buf[i] = out_b.min(255) as u8; + buf[i + 1] = out_g.min(255) as u8; + buf[i + 2] = out_r.min(255) as u8; + buf[i + 3] = out_a.min(255) as u8; +} + +fn radial_glow(buf: &mut [u8], cx: f64, cy: f64, radius: f64) { + // lavender → purple → transparent, matching Mac gradient stops. + let min_x = (cx - radius).floor() as i32; + let max_x = (cx + radius).ceil() as i32; + let min_y = (cy - radius).floor() as i32; + let max_y = (cy + radius).ceil() as i32; + for y in min_y..=max_y { + for x in min_x..=max_x { + let dx = x as f64 + 0.5 - cx; + let dy = y as f64 + 0.5 - cy; + let t = ((dx * dx + dy * dy).sqrt() / radius).clamp(0.0, 1.0); + // stops: 0→0.72 lavender, 0.30→0.38, 0.65→0.14 purple, 1→0 + let (rf, gf, bf, af) = if t < 0.30 { + let u = t / 0.30; + lerp4((0.76, 0.72, 0.99, 0.72), (0.76, 0.72, 0.99, 0.38), u) + } else if t < 0.65 { + let u = (t - 0.30) / 0.35; + lerp4((0.76, 0.72, 0.99, 0.38), (0.58, 0.52, 0.94, 0.14), u) + } else { + let u = (t - 0.65) / 0.35; + lerp4((0.58, 0.52, 0.94, 0.14), (0.58, 0.52, 0.94, 0.0), u) + }; + if af > 0.002 { + put_px( + buf, + x, + y, + (rf * 255.0) as u8, + (gf * 255.0) as u8, + (bf * 255.0) as u8, + (af * 255.0) as u8, + ); + } + } + } +} + +fn lerp4(a: (f64, f64, f64, f64), b: (f64, f64, f64, f64), t: f64) -> (f64, f64, f64, f64) { + ( + a.0 + (b.0 - a.0) * t, + a.1 + (b.1 - a.1) * t, + a.2 + (b.2 - a.2) * t, + a.3 + (b.3 - a.3) * t, + ) +} + +unsafe fn render(fb: &mut Framebuf, state: &Anim) { + let len = (SIDE * SIDE * 4) as usize; + let buf = unsafe { std::slice::from_raw_parts_mut(fb.bits, len) }; + buf.fill(0); + + let tip = (HOTSPOT, HOTSPOT); + let breathe = 1.0 + 0.03 * state.phase.sin(); + + // Soft lavender wash with idle breathe (no click ring). + radial_glow(buf, tip.0 + 6.0, tip.1 + 9.0, 34.0 * breathe); + + // Pure 2D: heading rotation only — no squash/stretch. + let sx = 1.0; + let sy = 1.0; + let cos_t = state.tilt.cos(); + let sin_t = state.tilt.sin(); + + // Exact Mac artwork (shared with chrome extension icons/cursor-112.png). + blit_cursor_png(buf, tip, sx, sy, cos_t, sin_t); +} + +fn cursor_rgba() -> &'static [(u8, u8, u8, u8)] { + use std::sync::OnceLock; + static PIXELS: OnceLock> = OnceLock::new(); + PIXELS.get_or_init(|| { + let bytes = include_bytes!("cursor_112.png"); + let img = image::load_from_memory(bytes) + .expect("cursor_112.png") + .into_rgba8(); + assert_eq!(img.width(), SIDE as u32); + assert_eq!(img.height(), SIDE as u32); + img.pixels() + .map(|p| { + let [r, g, b, a] = p.0; + (r, g, b, a) + }) + .collect() + }) +} + +fn blit_cursor_png(buf: &mut [u8], tip: (f64, f64), sx: f64, sy: f64, cos_t: f64, sin_t: f64) { + let pixels = cursor_rgba(); + // Large travel spikes can drive scale ≤ 0 (div-by-zero / mirrored sprite) + // or make the scan radius cover millions of off-frame pixels. + let sx = sx.max(0.01); + let sy = sy.max(0.01); + let inv_det = 1.0 / (sx * sy); + let isx = sy * inv_det; + let isy = sx * inv_det; + let radius = (SIDE as f64) * 0.55 * sx.max(sy); + let min_x = (tip.0 - radius).floor().max(0.0) as i32; + let max_x = (tip.0 + radius).ceil().min((SIDE - 1) as f64) as i32; + let min_y = (tip.1 - radius).floor().max(0.0) as i32; + let max_y = (tip.1 + radius).ceil().min((SIDE - 1) as f64) as i32; + + for y in min_y..=max_y { + for x in min_x..=max_x { + let dx = x as f64 + 0.5 - tip.0; + let dy = y as f64 + 0.5 - tip.1; + let rx = dx * cos_t + dy * sin_t; + let ry = -dx * sin_t + dy * cos_t; + let u = rx * isx + HOTSPOT; + let v = ry * isy + HOTSPOT; + if u < 0.0 || v < 0.0 || u >= (SIDE as f64) - 1.0 || v >= (SIDE as f64) - 1.0 { + continue; + } + let x0 = u.floor() as i32; + let y0 = v.floor() as i32; + let fx = u - x0 as f64; + let fy = v - y0 as f64; + let sample = |xx: i32, yy: i32| -> (f64, f64, f64, f64) { + if xx < 0 || yy < 0 || xx >= SIDE || yy >= SIDE { + return (0.0, 0.0, 0.0, 0.0); + } + let (r, g, b, a) = pixels[(yy * SIDE + xx) as usize]; + (r as f64, g as f64, b as f64, a as f64) + }; + let c00 = sample(x0, y0); + let c10 = sample(x0 + 1, y0); + let c01 = sample(x0, y0 + 1); + let c11 = sample(x0 + 1, y0 + 1); + let mix = |a: f64, b: f64, t: f64| a + (b - a) * t; + let r0 = ( + mix(c00.0, c10.0, fx), + mix(c00.1, c10.1, fx), + mix(c00.2, c10.2, fx), + mix(c00.3, c10.3, fx), + ); + let r1 = ( + mix(c01.0, c11.0, fx), + mix(c01.1, c11.1, fx), + mix(c01.2, c11.2, fx), + mix(c01.3, c11.3, fx), + ); + let a = mix(r0.3, r1.3, fy); + if a > 1.0 { + put_px( + buf, + x, + y, + mix(r0.0, r1.0, fy) as u8, + mix(r0.1, r1.1, fy) as u8, + mix(r0.2, r1.2, fy) as u8, + a as u8, + ); + } + } + } +} diff --git a/native/t3-desktop-mcp-rs/src/platform/cursor_112.png b/native/t3-desktop-mcp-rs/src/platform/cursor_112.png new file mode 100644 index 00000000000..19d8498e657 Binary files /dev/null and b/native/t3-desktop-mcp-rs/src/platform/cursor_112.png differ diff --git a/native/t3-desktop-mcp-rs/src/platform/linux.rs b/native/t3-desktop-mcp-rs/src/platform/linux.rs new file mode 100644 index 00000000000..5f891230455 --- /dev/null +++ b/native/t3-desktop-mcp-rs/src/platform/linux.rs @@ -0,0 +1,1496 @@ +//! Linux backend: AT-SPI for the accessibility tree, XTEST for synthetic input. +//! +//! Two caveats shape this file, and both are reported to the model rather than +//! hidden: +//! +//! * AT-SPI is opt-in. Toolkits expose a tree only when accessibility is +//! enabled, so an app can be running and still have nothing to read. +//! * Wayland refuses synthetic input by design. XTEST reaches X11 and XWayland +//! clients; a native Wayland client will ignore it, so we say so instead of +//! silently doing nothing. + +use std::collections::HashMap; + +use atspi::proxy::accessible::AccessibleProxy; +use atspi::{connection::AccessibilityConnection, Role}; +use futures_lite::future::block_on; +use x11rb::connection::Connection; +use x11rb::protocol::xproto::{ + ClientMessageEvent, ConfigureWindowAux, ConnectionExt as _, EventMask, GetKeyboardMappingReply, + InputFocus, Keycode, StackMode, +}; +use x11rb::protocol::xtest::ConnectionExt as _; +use x11rb::rust_connection::RustConnection; +use xcap::Window; + +use super::{Desktop, DesktopError, Point, Result, ScrollDirection, format_app_list}; +use super::agent_cursor::AgentCursor; +use crate::apps; + +/// X11 button numbers. +const BUTTON_LEFT: u8 = 1; +const BUTTON_RIGHT: u8 = 3; +const BUTTON_SCROLL_UP: u8 = 4; +const BUTTON_SCROLL_DOWN: u8 = 5; +const BUTTON_SCROLL_LEFT: u8 = 6; +const BUTTON_SCROLL_RIGHT: u8 = 7; + +/// Where an element lives on the AT-SPI bus. Proxies borrow their connection, +/// so the registry stores addresses and rebuilds a proxy on demand. +#[derive(Clone)] +struct ElementRef { + bus: String, + path: String, +} + +pub struct LinuxDesktop { + accessibility: Option, + x11: Option<(RustConnection, usize)>, + registry: HashMap, +} + +impl LinuxDesktop { + pub fn new() -> Result { + // Neither half is fatal on its own: a session with no a11y bus can still + // click by coordinate, and a session with no X11 can still read a tree. + // AT-SPI starts unset and is retried lazily — a bus that appears after + // process start must not leave accessibility permanently disabled. + let x11 = x11rb::connect(None).ok().map(|(conn, screen)| (conn, screen)); + Ok(Self { + accessibility: None, + x11, + registry: HashMap::new(), + }) + } + + fn ensure_accessibility(&mut self) { + if self.accessibility.is_none() { + self.accessibility = block_on(AccessibilityConnection::new()).ok(); + } + } + + fn bus(&self) -> Result<&AccessibilityConnection> { + self.accessibility.as_ref().ok_or_else(|| { + DesktopError::new( + "no AT-SPI bus on this session — start at-spi2-core (and set \ + GTK_MODULES=gail:atk-bridge for GTK apps) to read accessibility trees; \ + screenshot and coordinate clicks still work", + ) + }) + } + + fn x11(&self) -> Result<&(RustConnection, usize)> { + self.x11.as_ref().ok_or_else(|| { + DesktopError::new( + "no X11 display — synthetic input needs X11 or XWayland (native Wayland \ + refuses it by design). Set DISPLAY, or interact through the app's own UI", + ) + }) + } + + fn proxy<'a>( + &'a self, + element: &ElementRef, + ) -> Result> { + let connection = self.bus()?; + block_on( + AccessibleProxy::builder(connection.connection()) + .destination(element.bus.clone()) + .and_then(|builder| builder.path(element.path.clone())) + .map_err(|error| DesktopError::new(format!("bad element address: {error}")))? + .build(), + ) + .map_err(|error| DesktopError::new(format!("element is gone: {error}"))) + } + + fn element(&self, id: u32) -> Result { + self.registry.get(&id).cloned().ok_or_else(|| { + DesktopError::new(format!( + "element e{id} is not in the current snapshot — call get_app_state again, ids are per-snapshot" + )) + }) + } + + /// Screen rectangle centre of an element, via the Component interface. + fn center(&self, element: &ElementRef) -> Result<(f64, f64)> { + let proxy = self.proxy(element)?; + let component = block_on( + atspi::proxy::component::ComponentProxy::builder(self.bus()?.connection()) + .destination(element.bus.clone()) + .and_then(|builder| builder.path(element.path.clone())) + .map_err(|error| DesktopError::new(format!("bad element address: {error}")))? + .build(), + ) + .map_err(|error| DesktopError::new(format!("element has no geometry: {error}")))?; + drop(proxy); + + let extents = block_on(component.get_extents(atspi::CoordType::Screen)) + .map_err(|error| DesktopError::new(format!("could not read bounds: {error}")))?; + if extents.2 <= 0 || extents.3 <= 0 { + return Err(DesktopError::new( + "element is not visible on screen — scroll it into view first", + )); + } + Ok(( + f64::from(extents.0) + f64::from(extents.2) / 2.0, + f64::from(extents.1) + f64::from(extents.3) / 2.0, + )) + } + + /// Write text straight into an element through AT-SPI. + /// + /// Preferred over XTEST wherever it works: synthetic keys go to whatever + /// currently holds X11 focus, which under a compositor is not reliably the + /// element we were asked to type into. This addresses the element directly. + fn insert_text(&self, element: &ElementRef, text: &str, replace: bool) -> Result<()> { + let editable = block_on( + atspi::proxy::editable_text::EditableTextProxy::builder(self.bus()?.connection()) + .destination(element.bus.clone()) + .and_then(|builder| builder.path(element.path.clone())) + .map_err(|error| DesktopError::new(format!("bad element address: {error}")))? + .build(), + ) + .map_err(|error| DesktopError::new(format!("element is not editable: {error}")))?; + + if replace { + return match block_on(editable.set_text_contents(text)) { + Ok(true) => Ok(()), + Ok(false) => Err(DesktopError::new("the element refused the new contents")), + Err(error) => Err(DesktopError::new(format!("write failed: {error}"))), + }; + } + + // Append at the caret when AT-SPI exposes it. If caret_offset fails, + // append at the end (character_count) rather than silently prepending + // at 0. If that also fails, propagate the error so type_text can use + // the focus+keystroke path (which refuses unsafe Wayland clicks). + let caret = match self.caret_offset(element) { + Ok(offset) => offset, + Err(_) => self.character_count(element)?, + }; + match block_on(editable.insert_text(caret, text, text.chars().count() as i32)) { + Ok(true) => Ok(()), + Ok(false) => Err(DesktopError::new("the element refused the text")), + Err(error) => Err(DesktopError::new(format!("write failed: {error}"))), + } + } + + fn text_proxy<'a>( + &'a self, + element: &ElementRef, + ) -> Result> { + block_on( + atspi::proxy::text::TextProxy::builder(self.bus()?.connection()) + .destination(element.bus.clone()) + .and_then(|builder| builder.path(element.path.clone())) + .map_err(|error| DesktopError::new(format!("bad element address: {error}")))? + .build(), + ) + .map_err(|error| DesktopError::new(format!("element exposes no text: {error}"))) + } + + fn caret_offset(&self, element: &ElementRef) -> Result { + let text = self.text_proxy(element)?; + block_on(text.caret_offset()) + .map_err(|error| DesktopError::new(format!("could not read the caret: {error}"))) + } + + fn character_count(&self, element: &ElementRef) -> Result { + let text = self.text_proxy(element)?; + block_on(text.character_count()) + .map_err(|error| DesktopError::new(format!("could not read character count: {error}"))) + } + + /// Ask the toolkit to focus an element, which also raises its window on most + /// desktops — the closest portable equivalent to activating an app. + fn grab_focus(&self, element: &ElementRef) -> Result { + let component = block_on( + atspi::proxy::component::ComponentProxy::builder(self.bus()?.connection()) + .destination(element.bus.clone()) + .and_then(|builder| builder.path(element.path.clone())) + .map_err(|error| DesktopError::new(format!("bad element address: {error}")))? + .build(), + ) + .map_err(|error| DesktopError::new(format!("element cannot take focus: {error}")))?; + block_on(component.grab_focus()) + .map_err(|error| DesktopError::new(format!("focus refused: {error}"))) + } + + /// Raise and focus a window via EWMH `_NET_ACTIVE_WINDOW` + X11 focus. + /// + /// AT-SPI `grab_focus` is enough on many GTK apps, but dialogs (zenity) and + /// some WMs refuse it. Matching Windows' `SetForegroundWindow`, this asks + /// the window manager over X11 — which covers X11 and XWayland sessions. + fn raise_x11_window(&self, pid: u32) -> Result<()> { + let window_id = largest_window_id_for_pid(pid)?; + let (connection, screen) = self.x11()?; + let root = connection.setup().roots[*screen].root; + + // Best-effort restore/raise before the EWMH request — harmless if the + // window is already mapped and on top. + let _ = connection.map_window(window_id).and_then(|cookie| cookie.check()); + connection + .configure_window( + window_id, + &ConfigureWindowAux::new().stack_mode(StackMode::ABOVE), + ) + .map_err(|error| DesktopError::new(format!("could not raise window: {error}")))? + .check() + .map_err(|error| DesktopError::new(format!("could not raise window: {error}")))?; + + let atom = connection + .intern_atom(false, b"_NET_ACTIVE_WINDOW") + .map_err(|error| DesktopError::new(format!("could not intern _NET_ACTIVE_WINDOW: {error}")))? + .reply() + .map_err(|error| { + DesktopError::new(format!("could not intern _NET_ACTIVE_WINDOW: {error}")) + })? + .atom; + // data[0]=1 (application), data[1]=CurrentTime, data[2]=0 (no requestor). + let event = ClientMessageEvent::new(32, window_id, atom, [1u32, 0, 0, 0, 0]); + connection + .send_event( + false, + root, + EventMask::SUBSTRUCTURE_REDIRECT | EventMask::SUBSTRUCTURE_NOTIFY, + event, + ) + .map_err(|error| { + DesktopError::new(format!("could not send _NET_ACTIVE_WINDOW: {error}")) + })? + .check() + .map_err(|error| { + DesktopError::new(format!("could not send _NET_ACTIVE_WINDOW: {error}")) + })?; + + // Nudge for WMs (Openbox under Xvfb) that ignore the client message. + connection + .set_input_focus(InputFocus::PARENT, window_id, 0u32) + .map_err(|error| DesktopError::new(format!("could not set input focus: {error}")))? + .check() + .map_err(|error| DesktopError::new(format!("could not set input focus: {error}")))?; + connection + .flush() + .map_err(|error| DesktopError::new(format!("X11 flush failed: {error}")))?; + Ok(()) + } + + /// Run a click/press accessible action when one is advertised. + fn invoke(&self, element: &ElementRef) -> Result<()> { + let action = block_on( + atspi::proxy::action::ActionProxy::builder(self.bus()?.connection()) + .destination(element.bus.clone()) + .and_then(|builder| builder.path(element.path.clone())) + .map_err(|error| DesktopError::new(format!("bad element address: {error}")))? + .build(), + ) + .map_err(|error| DesktopError::new(format!("element exposes no actions: {error}")))?; + + let actions = block_on(action.get_actions()).map_err(|error| { + DesktopError::new(format!("could not list element actions: {error}")) + })?; + let index = actions + .iter() + .position(|entry| { + let name = entry.name.to_lowercase(); + name == "press" + || name == "click" + || name == "activate" + || name.contains("press") + || name.contains("click") + }) + .ok_or_else(|| { + DesktopError::new( + "element exposes no press/click action — use coordinates or set_value", + ) + })?; + + match block_on(action.do_action(index as i32)) { + Ok(true) => Ok(()), + Ok(false) => Err(DesktopError::new("the element refused its press action")), + Err(error) => Err(DesktopError::new(format!("press failed: {error}"))), + } + } + + fn point_coordinates(&self, target: Point) -> Result<(f64, f64)> { + match target { + Point::Screen(x, y) => { + if crate::capture::on_wayland() && !self.focused_app_is_x11_backed() { + return Err(DesktopError::new( + "screen-coordinate clicks are not supported for native Wayland apps — \ + focus an X11/XWayland client or use an element id", + )); + } + Ok((x, y)) + } + Point::Element(id) => { + let reference = self.element(id)?; + // Native Wayland clients report window-relative geometry; XTEST + // would treat those as absolute screen coords. XWayland/X11 apps + // still publish real screen extents and appear in `_NET_CLIENT_LIST`. + if crate::capture::on_wayland() && !self.element_is_x11_backed(&reference) { + return Err(DesktopError::new(format!( + "e{id} cannot be targeted via coordinates on Wayland — AT-SPI bounds \ + are window-relative. Use screenshot + absolute screen coordinates, \ + invoke a single-click action, or activate an X11/XWayland client" + ))); + } + self.center(&reference) + } + } + } + + /// True when the element's screen geometry lies inside an X11/XWayland + /// window owned by its pid. Native Wayland apps report window-relative + /// bounds that do not match any X11 window — fail closed in that case. + fn element_is_x11_backed(&self, element: &ElementRef) -> bool { + let pid = self.pid_for_bus(&element.bus); + if pid == 0 { + return false; + } + let Ok((x, y)) = self.center(element) else { + return false; + }; + point_in_x11_window_for_pid(pid, x, y) + } + + /// True when the frontmost app owns an X11/XWayland window, so XTEST can + /// reach it even on a Wayland session. + fn focused_app_is_x11_backed(&self) -> bool { + let Ok(apps) = apps::list_apps() else { + return false; + }; + let Some(focused) = apps.into_iter().find(|app| app.frontmost) else { + return false; + }; + focused.pid != 0 && largest_window_id_for_pid(focused.pid).is_ok() + } + + fn pid_for_bus(&self, bus: &str) -> u32 { + let Ok(connection) = self.bus() else { + return 0; + }; + block_on(async { + let Ok(dbus) = zbus::fdo::DBusProxy::new(connection.connection()).await else { + return 0u32; + }; + let Ok(bus_name) = zbus::names::BusName::try_from(bus) else { + return 0; + }; + dbus.get_connection_unix_process_id(bus_name) + .await + .unwrap_or(0) + }) + } + + fn move_pointer(&self, x: f64, y: f64) -> Result<()> { + // XTEST MotionNotify takes i16 screen coords — reject out-of-range or + // non-finite values instead of silently clamping via `as i16`. + let xi = to_xtest_coord(x, "x")?; + let yi = to_xtest_coord(y, "y")?; + let (connection, screen) = self.x11()?; + let root = connection.setup().roots[*screen].root; + connection + .xtest_fake_input(6 /* MotionNotify */, 0, 0, root, xi, yi, 0) + .map_err(|error| DesktopError::new(format!("could not move pointer: {error}")))? + .check() + .map_err(|error| DesktopError::new(format!("could not move pointer: {error}")))?; + connection + .flush() + .map_err(|error| DesktopError::new(format!("X11 flush failed: {error}")))?; + Ok(()) + } + + fn button(&self, button: u8, press: bool) -> Result<()> { + let (connection, screen) = self.x11()?; + let root = connection.setup().roots[*screen].root; + // 4 = ButtonPress, 5 = ButtonRelease + connection + .xtest_fake_input(if press { 4 } else { 5 }, button, 0, root, 0, 0, 0) + .map_err(|error| DesktopError::new(format!("could not send button event: {error}")))?; + connection + .flush() + .map_err(|error| DesktopError::new(format!("X11 flush failed: {error}")))?; + Ok(()) + } + + fn tap_button(&self, button: u8) -> Result<()> { + self.button(button, true)?; + self.button(button, false) + } + + fn keyboard_mapping(&self) -> Result<(GetKeyboardMappingReply, u8)> { + let (connection, _) = self.x11()?; + let setup = connection.setup(); + let first = setup.min_keycode; + let count = setup.max_keycode - setup.min_keycode + 1; + let mapping = connection + .get_keyboard_mapping(first, count) + .map_err(|error| DesktopError::new(format!("could not read keymap: {error}")))? + .reply() + .map_err(|error| DesktopError::new(format!("could not read keymap: {error}")))?; + Ok((mapping, first)) + } + + /// Find a keycode (and whether shift is needed) producing `keysym`. + fn keycode_for(&self, keysym: u32) -> Result> { + let (mapping, first) = self.keyboard_mapping()?; + let per = mapping.keysyms_per_keycode as usize; + for (index, chunk) in mapping.keysyms.chunks(per).enumerate() { + if chunk.first().copied() == Some(keysym) { + return Ok(Some((first + index as u8, false))); + } + if per > 1 && chunk.get(1).copied() == Some(keysym) { + return Ok(Some((first + index as u8, true))); + } + } + Ok(None) + } + + fn tap_keycode(&self, keycode: Keycode, shift: bool) -> Result<()> { + let shift_code = if shift { + // Prefer Shift_L, then Shift_R — never send an unshifted key when + // the caller asked for Shift (that types the wrong character). + Some( + match self.keycode_for(0xffe1 /* Shift_L */)? { + Some((code, _)) => code, + None => self + .keycode_for(0xffe2 /* Shift_R */)? + .map(|(code, _)| code) + .ok_or_else(|| { + DesktopError::new( + "Shift is required for this character but no Shift key is available \ + on the current keyboard layout", + ) + })?, + }, + ) + } else { + None + }; + if let Some(code) = shift_code { + self.key(code, true)?; + } + let tapped = self.key(keycode, true).and_then(|()| self.key(keycode, false)); + // Always release Shift if we pressed it, even when the key tap fails. + let released = shift_code.map(|code| self.key(code, false)); + match (tapped, released) { + (Err(error), _) => Err(error), + (Ok(()), Some(Err(error))) => Err(error), + (Ok(()), _) => Ok(()), + } + } + + fn key(&self, keycode: Keycode, press: bool) -> Result<()> { + let (connection, screen) = self.x11()?; + let root = connection.setup().roots[*screen].root; + // 2 = KeyPress, 3 = KeyRelease + connection + .xtest_fake_input(if press { 2 } else { 3 }, keycode, 0, root, 0, 0, 0) + .map_err(|error| DesktopError::new(format!("could not send key event: {error}")))?; + connection + .flush() + .map_err(|error| DesktopError::new(format!("X11 flush failed: {error}")))?; + Ok(()) + } + + /// Type one character, remapping a scratch keycode when the current layout + /// cannot produce it. This is how xdotool handles accented and non-Latin + /// text, and without it typing would silently drop characters. + fn type_char(&self, character: char) -> Result<()> { + let keysym = char_to_keysym(character); + if let Some((keycode, shift)) = self.keycode_for(keysym)? { + return self.tap_keycode(keycode, shift); + } + + let (connection, _) = self.x11()?; + let (mapping, first) = self.keyboard_mapping()?; + let per = mapping.keysyms_per_keycode as usize; + // A keycode whose every slot is NoSymbol is free to borrow. + let scratch = mapping + .keysyms + .chunks(per) + .position(|chunk| chunk.iter().all(|symbol| *symbol == 0)) + .map(|index| first + index as u8) + .ok_or_else(|| { + DesktopError::new(format!( + "'{character}' is not on the current keyboard layout and no spare keycode is free" + )) + })?; + + let replacement = vec![keysym; per]; + let remap = connection + .change_keyboard_mapping(1, scratch, per as u8, &replacement) + .map_err(|error| DesktopError::new(format!("could not remap keycode: {error}"))) + .and_then(|cookie| { + cookie + .check() + .map_err(|error| DesktopError::new(format!("could not remap keycode: {error}"))) + }) + .and_then(|()| { + connection + .flush() + .map_err(|error| DesktopError::new(format!("X11 flush failed: {error}"))) + }); + + let result = remap.and_then(|()| self.tap_keycode(scratch, false)); + + // Always hand the keycode back, even if the tap failed, or the user's + // keyboard keeps our borrowed mapping. Surface cleanup failures after + // the tap result so a successful type never leaves a remapped key. + let cleared = vec![0u32; per]; + let restore = connection + .change_keyboard_mapping(1, scratch, per as u8, &cleared) + .map_err(|error| DesktopError::new(format!("could not restore keycode: {error}"))) + .and_then(|cookie| { + cookie + .check() + .map_err(|error| DesktopError::new(format!("could not restore keycode: {error}"))) + }) + .and_then(|()| { + connection + .flush() + .map_err(|error| DesktopError::new(format!("X11 flush failed: {error}"))) + }); + match (result, restore) { + (Ok(()), Ok(())) => Ok(()), + (Err(tap), _) => Err(tap), + (Ok(()), Err(cleanup)) => Err(cleanup), + } + } + + fn walk( + &mut self, + element: &ElementRef, + depth: usize, + max_depth: usize, + max_elements: usize, + next_id: &mut u32, + visited: &mut usize, + lines: &mut Vec, + ) { + // Count every node we inspect — unnamed non-interactive parents still + // cost D-Bus round-trips and must not bypass `max_elements`. + if depth > max_depth || *visited >= max_elements { + return; + } + // Read everything the proxy can tell us, then drop it: it borrows + // `self`, and the registry below needs that borrow released. + let Some((name, role, children)) = ({ + match self.proxy(element) { + Ok(proxy) => { + let name = block_on(proxy.name()).unwrap_or_default(); + let role = block_on(proxy.get_role()).unwrap_or(Role::Invalid); + let children = block_on(proxy.get_children()).unwrap_or_default(); + Some((name, role, children)) + } + Err(_) => None, + } + }) else { + return; + }; + *visited += 1; + + let interactive = matches!( + role, + Role::Button + | Role::CheckBox + | Role::ComboBox + | Role::Entry + | Role::Link + | Role::ListItem + | Role::MenuItem + | Role::PasswordText + | Role::RadioButton + | Role::Slider + | Role::Text + | Role::ToggleButton + | Role::TreeItem + ); + + if !name.is_empty() || interactive { + let mut row = " ".repeat(depth); + if interactive { + *next_id += 1; + row.push_str(&format!("[e{next_id}] ")); + self.registry.insert(*next_id, element.clone()); + } + row.push_str(&format!("{role:?}")); + if !name.is_empty() { + row.push_str(&format!(" \"{}\"", truncate(&name, 120))); + } + // Screen bounds let a model fall back to coordinates when an element + // has no usable action, and make a wrong-looking click diagnosable. + if interactive && let Ok((x, y)) = self.center(element) { + row.push_str(&format!(" @({x:.0},{y:.0})")); + } + lines.push(row); + } + + for child in children { + if *visited >= max_elements { + lines.push(format!( + "{}… truncated at {max_elements} elements — raise max_elements or target a child", + " ".repeat(depth + 1) + )); + return; + } + let reference = ElementRef { + bus: child.name().map(|name| name.to_string()).unwrap_or_default(), + path: child.path().to_string(), + }; + self.walk( + &reference, + depth + 1, + max_depth, + max_elements, + next_id, + visited, + lines, + ); + } + } + + /// Top-level application objects on the a11y bus, with their pids. + fn applications(&mut self) -> Result> { + self.ensure_accessibility(); + let connection = self.bus()?; + let root = AccessibleProxy::builder(connection.connection()) + .destination("org.a11y.atspi.Registry") + .and_then(|builder| builder.path("/org/a11y/atspi/accessible/root")) + .map_err(|error| DesktopError::new(format!("bad registry address: {error}")))?; + let root = block_on(root.build()) + .map_err(|error| DesktopError::new(format!("no a11y registry: {error}")))?; + + let children = block_on(root.get_children()) + .map_err(|error| DesktopError::new(format!("could not list applications: {error}")))?; + + let mut applications = Vec::new(); + for child in children { + let reference = ElementRef { + bus: child.name().map(|name| name.to_string()).unwrap_or_default(), + path: child.path().to_string(), + }; + let Ok(proxy) = self.proxy(&reference) else { + continue; + }; + let name = block_on(proxy.name()).unwrap_or_default(); + // Resolve the a11y bus name to a Unix PID via D-Bus. Application.id + // is a registry-assigned token, not a process id. + let pid = block_on(async { + let Ok(dbus) = zbus::fdo::DBusProxy::new(connection.connection()).await else { + return 0u32; + }; + let Ok(bus_name) = zbus::names::BusName::try_from(reference.bus.as_str()) else { + return 0; + }; + dbus.get_connection_unix_process_id(bus_name) + .await + .unwrap_or(0) + }); + applications.push((reference, name, pid)); + } + Ok(applications) + } +} + +fn truncate(value: &str, limit: usize) -> String { + let cleaned = value.replace(['\n', '\r'], " "); + if cleaned.chars().count() <= limit { + return cleaned; + } + cleaned.chars().take(limit).collect::() + "…" +} + +/// Prefer PID when the query is numeric, then an exact AT-SPI name match; +/// otherwise require a unique substring hit so `Code` cannot silently activate +/// `Visual Studio Code`. Duplicate exact names must be selected by pid. +fn match_application( + applications: &[(ElementRef, String, u32)], + query: &str, +) -> Result<(ElementRef, String, u32)> { + let trimmed = query.trim(); + if let Ok(pid) = trimmed.parse::() { + if pid != 0 { + if let Some(hit) = applications.iter().find(|(_, _, app_pid)| *app_pid == pid) { + return Ok(hit.clone()); + } + return Err(DesktopError::new(format!( + "no app on the accessibility bus has pid {pid}" + ))); + } + } + + let lowered = trimmed.to_lowercase(); + let exact: Vec<_> = applications + .iter() + .filter(|(_, name, _)| name.eq_ignore_ascii_case(trimmed)) + .cloned() + .collect(); + if exact.len() == 1 { + return Ok(exact[0].clone()); + } + if exact.len() > 1 { + let pids = exact + .iter() + .map(|(_, _, pid)| pid.to_string()) + .collect::>() + .join(", "); + return Err(DesktopError::new(format!( + "'{trimmed}' matches several apps exactly (pids {pids}) — pass a pid from list_apps" + ))); + } + let partial: Vec<_> = applications + .iter() + .filter(|(_, name, _)| name.to_lowercase().contains(&lowered)) + .cloned() + .collect(); + match partial.as_slice() { + [single] => Ok(single.clone()), + [] => Err(DesktopError::new(format!( + "no app on the accessibility bus matches '{trimmed}'. Toolkits only publish a tree \ + when accessibility is enabled — try screenshot plus coordinate clicks instead" + ))), + many => { + let detail = many + .iter() + .map(|(_, name, pid)| { + if *pid == 0 { + name.clone() + } else { + format!("{name} (pid {pid})") + } + }) + .collect::>() + .join(", "); + Err(DesktopError::new(format!( + "'{trimmed}' matches several apps: {detail} — pass an exact name or pid from list_apps" + ))) + } + } +} + +/// True when `(x, y)` lies inside an X11/XWayland window owned by `pid`. +fn point_in_x11_window_for_pid(pid: u32, x: f64, y: f64) -> bool { + let Ok(windows) = std::panic::catch_unwind(Window::all) else { + return false; + }; + let Ok(windows) = windows else { + return false; + }; + let xi = x.round() as i32; + let yi = y.round() as i32; + for window in windows { + if window.pid().unwrap_or(0) != pid { + continue; + } + let Ok(wx) = window.x() else { + continue; + }; + let Ok(wy) = window.y() else { + continue; + }; + let width = window.width().unwrap_or(0); + let height = window.height().unwrap_or(0); + if width == 0 || height == 0 { + continue; + } + if xi >= wx && xi < wx + width as i32 && yi >= wy && yi < wy + height as i32 { + return true; + } + } + false +} + +/// Largest window owned by `pid` (including minimized), for EWMH activation. +fn largest_window_id_for_pid(pid: u32) -> Result { + // `Window::all` uses X11/`xcb` when `DISPLAY` is set — do not mutate + // `WAYLAND_DISPLAY` (UB with concurrent threads). + let windows = std::panic::catch_unwind(Window::all) + .map_err(|_| DesktopError::new("window enumeration is not supported by this display server"))? + .map_err(|error| DesktopError::new(format!("failed to enumerate windows: {error}")))?; + + let mut best: Option<(u32, u32, bool)> = None; // (area, id, minimized) + for window in windows { + if window.pid().unwrap_or(0) != pid { + continue; + } + let width = window.width().unwrap_or(0); + let height = window.height().unwrap_or(0); + // Prefer real geometry; minimized windows may report 0×0 — still keep as fallback. + let minimized = window.is_minimized().unwrap_or(false); + let area = if width == 0 || height == 0 { + 0 + } else { + width.saturating_mul(height) + }; + let Ok(id) = window.id() else { + continue; + }; + let better = match best { + None => true, + Some((best_area, _, best_min)) => { + (!minimized && best_min) || (minimized == best_min && area > best_area) + } + }; + if better { + best = Some((area, id, minimized)); + } + } + best.map(|(_, id, _)| id).ok_or_else(|| { + DesktopError::new(format!("pid {pid} has no raisable window — it may have no UI")) + }) +} + +/// Map a character to an X11 keysym. +/// +/// Latin-1 is its own keysym range; everything else uses the Unicode range +/// X11 reserves for exactly this purpose. +fn char_to_keysym(character: char) -> u32 { + match character { + '\n' => 0xff0a, + '\t' => 0xff09, + '\r' => 0xff0d, + other => { + let code = other as u32; + if (0x20..=0xff).contains(&code) { + code + } else { + 0x0100_0000 + code + } + } + } +} + +/// Named keys the model can send, in X11 keysym terms. +fn named_keysym(key: &str) -> Option { + Some(match key.to_lowercase().as_str() { + "return" | "enter" => 0xff0d, + "tab" => 0xff09, + "escape" | "esc" => 0xff1b, + "space" => 0x0020, + "backspace" => 0xff08, + "delete" => 0xffff, + "up" => 0xff52, + "down" => 0xff54, + "left" => 0xff51, + "right" => 0xff53, + "home" => 0xff50, + "end" => 0xff57, + "page_up" | "pageup" => 0xff55, + "page_down" | "pagedown" => 0xff56, + "f1" => 0xffbe, + "f2" => 0xffbf, + "f3" => 0xffc0, + "f4" => 0xffc1, + "f5" => 0xffc2, + "f6" => 0xffc3, + "f7" => 0xffc4, + "f8" => 0xffc5, + "f9" => 0xffc6, + "f10" => 0xffc7, + "f11" => 0xffc8, + "f12" => 0xffc9, + _ => return None, + }) +} + +/// Modifier names to keysyms. `cmd` becomes Super, matching the Windows path. +fn modifier_keysym(modifier: &str) -> Option { + Some(match modifier.to_lowercase().as_str() { + "ctrl" | "control" => 0xffe3, + "shift" => 0xffe1, + "alt" | "option" => 0xffe9, + "cmd" | "command" | "super" | "meta" | "win" => 0xffeb, + _ => return None, + }) +} + +fn to_xtest_coord(value: f64, axis: &str) -> Result { + if !value.is_finite() { + return Err(DesktopError::new(format!( + "pointer {axis} coordinate must be finite (got {value})" + ))); + } + if value < f64::from(i16::MIN) || value > f64::from(i16::MAX) { + return Err(DesktopError::new(format!( + "pointer {axis} coordinate {value} is outside the XTEST i16 range \ + ({min}..={max})", + min = i16::MIN, + max = i16::MAX, + ))); + } + Ok(value as i16) +} + +impl Desktop for LinuxDesktop { + fn list_apps(&mut self) -> Result { + // Window enumeration needs EWMH properties that minimal window managers + // (WSLg included) do not publish, and the accessibility bus is the more + // relevant view here anyway: an app absent from it cannot be driven. + let focused = apps::list_apps().ok().and_then(|apps| { + apps.into_iter().find(|app| app.frontmost) + }); + let focused_name = focused + .as_ref() + .map(|app| app.name.to_lowercase()) + .unwrap_or_default(); + let focused_pid = focused.map(|app| app.pid); + + if let Ok(applications) = self.applications() + && !applications.is_empty() + { + let mut lines: Vec = applications + .into_iter() + .filter(|(_, name, _)| !name.is_empty()) + .map(|(_, name, pid)| { + // Prefer pid equality — xcap and AT-SPI names often disagree + // (`Code` vs `Visual Studio Code`). Exact name is the fallback. + let is_frontmost = focused_pid + .filter(|front| *front != 0 && *front == pid) + .is_some() + || (!focused_name.is_empty() && name.to_lowercase() == focused_name); + + let marker = if is_frontmost { " FRONTMOST" } else { "" }; + if pid == 0 { + format!("{name} [a11y]{marker}") + } else { + format!("{name} [a11y] pid={pid}{marker}") + } + }) + .collect(); + if !lines.is_empty() { + lines.sort_by_key(|line| (!line.contains("FRONTMOST"), line.to_lowercase())); + return Ok(lines.join("\n")); + } + } + Ok(format_app_list(apps::list_apps()?)) + } + + fn resolve_pid(&mut self, app: &str) -> Result { + // Prefer AT-SPI's application list (same names `list_apps` shows when + // a11y is up). Ambiguous exact matches must stay errors — falling through + // to xcap can pick the wrong process. Only fall back when a11y is down + // or the name is simply absent from the bus. + match self.applications() { + Ok(applications) => match match_application(&applications, app) { + Ok((_, _, pid)) if pid != 0 => Ok(pid), + Ok((_, _, 0)) => apps::resolve_pid(app), + Err(error) if error.0.contains("matches several apps") => Err(error), + Err(_) => apps::resolve_pid(app), + }, + Err(_) => apps::resolve_pid(app), + } + } + + fn get_app_state(&mut self, app: &str, max_depth: usize, max_elements: usize) -> Result { + // Invalidate prior snapshot IDs even if this refresh fails to find `app`. + self.registry.clear(); + let applications = self.applications()?; + let (reference, name, _) = match_application(&applications, app)?; + + let mut next_id = 0u32; + let mut visited = 0usize; + let mut lines = vec![format!("{name}")]; + self.walk( + &reference, + 0, + max_depth, + max_elements, + &mut next_id, + &mut visited, + &mut lines, + ); + if next_id == 0 { + lines.push( + "no interactive elements exposed — use screenshot and click with coordinates" + .to_string(), + ); + } + Ok(lines.join("\n")) + } + + fn activate_app(&mut self, app: &str) -> Result { + // Prefer AT-SPI grab_focus (portable), then fall back to EWMH raise — + // dialogs often refuse Component.grab_focus even when X11 can activate. + if app.trim().is_empty() { + return Err(DesktopError::new( + "missing required argument 'app' — pass an app name from list_apps", + )); + } + let applications = match self.applications() { + Ok(apps) => apps, + Err(_) => { + // list_apps can still enumerate X11 windows when AT-SPI is down; + // activate those via EWMH instead of failing on the missing bus. + let pid = apps::resolve_pid(app)?; + let name = apps::list_apps() + .ok() + .and_then(|apps| { + apps.into_iter() + .find(|entry| entry.pid == pid) + .map(|entry| entry.name) + }) + .unwrap_or_else(|| app.to_string()); + return match self.raise_x11_window(pid) { + Ok(()) => Ok(format!("activated {name} (pid {pid})")), + Err(error) => Err(DesktopError::new(format!( + "{name} refused focus ({error}) — accessibility is unavailable and no \ + X11 window could be raised" + ))), + }; + } + }; + let (reference, name, a11y_pid) = match match_application(&applications, app) { + Ok(hit) => hit, + Err(error) => { + // Ambiguous matches must stay fail-closed (ask for a pid). Only + // fall back to X11 when AT-SPI has no match at all. + if error.0.contains("matches several") { + return Err(error); + } + let pid = apps::resolve_pid(app)?; + let name = apps::list_apps() + .ok() + .and_then(|apps| { + apps.into_iter() + .find(|entry| entry.pid == pid) + .map(|entry| entry.name) + }) + .unwrap_or_else(|| app.to_string()); + return match self.raise_x11_window(pid) { + Ok(()) => Ok(format!("activated {name} (pid {pid})")), + Err(raise_error) => Err(DesktopError::new(format!( + "{name} refused focus ({raise_error}) — no AT-SPI match and no X11 window \ + could be raised" + ))), + }; + } + }; + + // The application object itself cannot take focus; its first frame can. + let frames = { + match self.proxy(&reference) { + Ok(proxy) => block_on(proxy.get_children()).unwrap_or_default(), + Err(_) => Vec::new(), + } + }; + for frame in frames { + let child = ElementRef { + bus: frame.name().map(|name| name.to_string()).unwrap_or_default(), + path: frame.path().to_string(), + }; + if self.grab_focus(&child).unwrap_or(false) { + return Ok(format!("activated {name}")); + } + } + + let pid = if a11y_pid != 0 { + a11y_pid + } else { + // AT-SPI often omits a usable pid; xcap window grouping still can. + apps::resolve_pid(&name) + .or_else(|_| apps::resolve_pid(app)) + .map_err(|error| { + DesktopError::new(format!( + "{name} refused AT-SPI focus and no X11 window matched ({error})" + )) + })? + }; + + match self.raise_x11_window(pid) { + Ok(()) => Ok(format!("activated {name} (pid {pid})")), + Err(error) => Err(DesktopError::new(format!( + "{name} refused focus ({error}) — window managers vary here. The other tools do \ + not need it focused, so carry on without activating it" + ))), + } + } + + fn click(&mut self, target: Point, click_count: u32) -> Result { + self.ensure_accessibility(); + // Prefer the element's own action. Wayland clients cannot learn their + // absolute screen position, so AT-SPI reports geometry relative to the + // window and synthetic clicks would land in the wrong place. Invoking + // the action sidesteps coordinates entirely, and matches what the + // Windows backend does with the Invoke pattern. + if let Point::Element(id) = target { + let reference = self.element(id)?; + if click_count <= 1 { + // Wait for the agent pointer to land before invoking, matching Win/Mac. + // Skip native Wayland — AT-SPI bounds are window-relative and would mis-fly. + if !crate::capture::on_wayland() || self.element_is_x11_backed(&reference) { + if let Ok((x, y)) = self.center(&reference) { + AgentCursor::shared().press(x, y); + } + } + if self.invoke(&reference).is_ok() { + return Ok(format!("pressed e{id}")); + } + } + // Native Wayland clients report window-relative geometry; XTEST + // clicks would land on the wrong place. XWayland/X11 apps are fine. + if crate::capture::on_wayland() && !self.element_is_x11_backed(&reference) { + return Err(DesktopError::new(format!( + "e{id} cannot be clicked via coordinates on Wayland — AT-SPI bounds are \ + window-relative. Use screenshot + absolute screen coordinates, invoke a \ + single-click action, or activate an X11/XWayland client" + ))); + } + } + + let (x, y) = self.point_coordinates(target)?; + AgentCursor::shared().press(x, y); + self.move_pointer(x, y)?; + for _ in 0..click_count.max(1) { + self.tap_button(BUTTON_LEFT)?; + } + Ok(format!( + "clicked at ({x:.0}, {y:.0}){}", + if click_count > 1 { + format!(" x{click_count}") + } else { + String::new() + } + )) + } + + fn right_click(&mut self, target: Point) -> Result { + self.ensure_accessibility(); + let (x, y) = self.point_coordinates(target)?; + AgentCursor::shared().press(x, y); + self.move_pointer(x, y)?; + self.button(BUTTON_RIGHT, true)?; + if let Err(error) = self.button(BUTTON_RIGHT, false) { + let _ = self.button(BUTTON_RIGHT, false); + return Err(error); + } + Ok(format!("right-clicked at ({x:.0}, {y:.0})")) + } + + fn drag(&mut self, from: Point, to: Point) -> Result { + self.ensure_accessibility(); + let (from_x, from_y) = self.point_coordinates(from)?; + let (to_x, to_y) = self.point_coordinates(to)?; + AgentCursor::shared().show(from_x, from_y); + self.move_pointer(from_x, from_y)?; + // Fly the overlay to the destination *before* button-down so the + // blocking wait cannot hold a stationary press (Bugbot: drag timing). + AgentCursor::shared().press(to_x, to_y); + self.button(BUTTON_LEFT, true)?; + // A single jump can read as a click to apps that track motion, so step. + // Release the button before propagating any motion error, or the + // session is left mid-drag. + let motion = (|| -> Result<()> { + for step in 1..=10 { + let progress = f64::from(step) / 10.0; + let x = from_x + (to_x - from_x) * progress; + let y = from_y + (to_y - from_y) * progress; + if step % 3 == 0 { + AgentCursor::shared().glide(x, y); + } + self.move_pointer(x, y)?; + } + Ok(()) + })(); + let release = self.button(BUTTON_LEFT, false); + motion?; + release?; + Ok(format!( + "dragged ({from_x:.0}, {from_y:.0}) → ({to_x:.0}, {to_y:.0})" + )) + } + + fn type_text(&mut self, text: &str, element: Option) -> Result { + self.ensure_accessibility(); + // With a target element, write through AT-SPI: it does not depend on + // which window the compositor considers focused, so it is reliable where + // synthetic keys are not. + if let Some(id) = element { + let reference = self.element(id)?; + if self.insert_text(&reference, text, false).is_ok() { + return Ok(format!("typed {} characters into e{id}", text.chars().count())); + } + // Fall back to focusing and using the keyboard. Only click when + // focus failed — clicking a focused field can move the caret. + // On Wayland, never use coordinate clicks for native clients + // (bounds are window-relative). XWayland still has absolute geometry. + let focused = self.grab_focus(&reference).unwrap_or(false); + if !focused { + if crate::capture::on_wayland() && !self.element_is_x11_backed(&reference) { + return Err(DesktopError::new(format!( + "could not focus e{id} for typing on Wayland — click the field first, \ + or use set_value (coordinate fallback is unsafe here)" + ))); + } + let clicked = if let Ok((x, y)) = self.center(&reference) { + self.move_pointer(x, y) + .and_then(|()| self.tap_button(BUTTON_LEFT)) + .is_ok() + } else { + false + }; + if !clicked { + return Err(DesktopError::new(format!( + "could not focus e{id} for typing — click the field first, or use set_value" + ))); + } + } + // Focus alone is not enough on native Wayland — XTEST keys never arrive. + if crate::capture::on_wayland() && !self.element_is_x11_backed(&reference) { + return Err(DesktopError::new(format!( + "type_text cannot deliver keys to native Wayland e{id} — use set_value, \ + or focus an X11/XWayland client" + ))); + } + } else if crate::capture::on_wayland() && !self.focused_app_is_x11_backed() { + return Err(DesktopError::new( + "type_text is not supported for native Wayland apps without an element — \ + pass an element id, use set_value, or focus an X11/XWayland client", + )); + } + // Force a round trip so the server has drained anything queued, then let + // the target settle. Some toolkits still swallow the opening character + // under XWayland; if the first keystroke goes missing, type it twice or + // click the field first. + if let Ok((connection, _)) = self.x11() { + let _ = connection.get_input_focus().and_then(|cookie| Ok(cookie.reply())); + } + std::thread::sleep(std::time::Duration::from_millis(60)); + for character in text.chars() { + self.type_char(character)?; + // xdotool uses the same default gap; typing flat out makes some + // toolkits coalesce or drop events. + std::thread::sleep(std::time::Duration::from_millis(12)); + } + Ok(format!("typed {} characters", text.chars().count())) + } + + fn press_key(&mut self, key: &str, modifiers: &[String]) -> Result { + // XTEST reaches X11/XWayland clients only. On Wayland, allow when the + // focused app is X11-backed (same heuristic as clicks); refuse native + // Wayland clients rather than silently dropping keys. + if crate::capture::on_wayland() && !self.focused_app_is_x11_backed() { + return Err(DesktopError::new( + "press_key is not supported for native Wayland apps — synthetic XTEST keys do not \ + reach them. Focus an X11/XWayland client, use type_text or set_value on an \ + element, or run under X11", + )); + } + let keysym = if let Some(named) = named_keysym(key) { + named + } else { + let mut chars = key.chars(); + let Some(first) = chars.next() else { + return Err(DesktopError::new("missing required argument 'key'")); + }; + if chars.next().is_some() { + return Err(DesktopError::new(format!( + "unsupported key '{key}' — use a single character or a named key (enter, escape, …)" + ))); + } + char_to_keysym(first) + }; + let (keycode, needs_shift) = self.keycode_for(keysym)?.ok_or_else(|| { + DesktopError::new(format!("'{key}' is not on the current keyboard layout")) + })?; + + let mut held = Vec::new(); + for modifier in modifiers { + // `fn` has no X11 equivalent; ignore only that known no-op. + if modifier.eq_ignore_ascii_case("fn") { + continue; + } + let symbol = modifier_keysym(modifier).ok_or_else(|| { + DesktopError::new(format!( + "unsupported modifier '{modifier}' — use ctrl, shift, alt, or cmd" + )) + })?; + let (code, _) = self.keycode_for(symbol)?.ok_or_else(|| { + DesktopError::new(format!( + "modifier '{modifier}' is not available on the current keyboard layout" + )) + })?; + held.push(code); + } + if needs_shift { + let shift = match self.keycode_for(0xffe1 /* Shift_L */)? { + Some((code, _)) => code, + None => self + .keycode_for(0xffe2 /* Shift_R */)? + .map(|(code, _)| code) + .ok_or_else(|| { + DesktopError::new( + "Shift is required for this key but no Shift key is available \ + on the current keyboard layout", + ) + })?, + }; + held.push(shift); + } + + let press_modifiers = (|| -> Result<()> { + for code in &held { + self.key(*code, true)?; + } + Ok(()) + })(); + if let Err(error) = press_modifiers { + for code in held.iter().rev() { + let _ = self.key(*code, false); + } + return Err(error); + } + let tapped = self.key(keycode, true).and_then(|()| self.key(keycode, false)); + // Release modifiers even if the tap failed, or the session is left with + // ctrl stuck down. + for code in held.iter().rev() { + let _ = self.key(*code, false); + } + tapped?; + + Ok(if modifiers.is_empty() { + format!("pressed {key}") + } else { + format!("pressed {}+{key}", modifiers.join("+")) + }) + } + + fn scroll( + &mut self, + direction: ScrollDirection, + amount: i32, + element: Option, + ) -> Result { + self.ensure_accessibility(); + if let Some(id) = element { + // Route through point_coordinates so Wayland refuses window-relative + // AT-SPI bounds the same way click / right_click / drag do. + let (x, y) = self.point_coordinates(Point::Element(id))?; + AgentCursor::shared().show(x, y); + self.move_pointer(x, y)?; + } + let button = match direction { + ScrollDirection::Up => BUTTON_SCROLL_UP, + ScrollDirection::Down => BUTTON_SCROLL_DOWN, + ScrollDirection::Left => BUTTON_SCROLL_LEFT, + ScrollDirection::Right => BUTTON_SCROLL_RIGHT, + }; + for _ in 0..amount.max(1) { + self.tap_button(button)?; + } + Ok(format!("scrolled {direction:?} by {amount}").to_lowercase()) + } + + fn set_value(&mut self, element: u32, value: &str) -> Result { + self.ensure_accessibility(); + let reference = self.element(element)?; + self.insert_text(&reference, value, true).map_err(|error| { + DesktopError::new(format!( + "{error} — not every toolkit allows a direct write; click e{element}, select all \ + with press_key('a', ['ctrl']), then type_text" + )) + })?; + Ok(format!("set e{element} to \"{}\"", truncate(value, 80))) + } + + fn select_text(&mut self, element: u32, start: usize, length: Option) -> Result { + self.ensure_accessibility(); + let reference = self.element(element)?; + let text = self.text_proxy(&reference)?; + let total = block_on(text.character_count()) + .map_err(|error| { + DesktopError::new(format!( + "could not read character count for e{element}: {error}" + )) + })? + .max(0); + let start = i32::try_from(start).unwrap_or(i32::MAX).min(total); + let end = length + .and_then(|count| i32::try_from(count).ok()) + .map_or(total, |count| start.saturating_add(count).min(total)); + + // Replace selection 0 when one exists; otherwise create it. + let applied = block_on(text.set_selection(0, start, end)) + .unwrap_or(false) + || block_on(text.add_selection(start, end)) + .map_err(|error| DesktopError::new(format!("selection failed: {error}")))?; + if !applied { + return Err(DesktopError::new(format!( + "e{element} refused the selection — click it then use press_key('a', ['ctrl'])" + ))); + } + Ok(format!("selected {} characters in e{element}", end - start)) + } +} + +#[cfg(test)] +mod tests { + use super::{char_to_keysym, modifier_keysym, named_keysym, to_xtest_coord, truncate}; + + #[test] + fn latin1_characters_map_to_themselves() { + assert_eq!(char_to_keysym('a'), 0x61); + assert_eq!(char_to_keysym(' '), 0x20); + assert_eq!(char_to_keysym('ÿ'), 0xff); + } + + #[test] + fn xtest_coords_reject_non_finite_and_out_of_range() { + assert_eq!(to_xtest_coord(12.9, "x").unwrap(), 12); + assert!(to_xtest_coord(f64::NAN, "x").is_err()); + assert!(to_xtest_coord(40_000.0, "x").is_err()); + assert!(to_xtest_coord(-40_000.0, "y").is_err()); + } + + #[test] + fn other_characters_use_the_unicode_keysym_range() { + // Without this, emoji and CJK would silently fail to type. + assert_eq!(char_to_keysym('€'), 0x0100_0000 + 0x20ac); + assert_eq!(char_to_keysym('日'), 0x0100_0000 + 0x65e5); + } + + #[test] + fn named_keys_cover_what_the_tool_advertises() { + assert_eq!(named_keysym("return"), Some(0xff0d)); + assert_eq!(named_keysym("Escape"), Some(0xff1b)); + assert_eq!(named_keysym("nonsense"), None); + } + + #[test] + fn cmd_maps_to_super_like_the_windows_backend() { + assert_eq!(modifier_keysym("cmd"), modifier_keysym("super")); + assert_eq!(modifier_keysym("ctrl"), Some(0xffe3)); + assert_eq!(modifier_keysym("fn"), None); + } + + #[test] + fn truncation_collapses_newlines() { + assert_eq!(truncate("a\nb", 10), "a b"); + } + + #[test] + fn match_application_prefers_pid_and_rejects_ambiguous_names() { + let a = ( + ElementRef { + bus: ":1.1".into(), + path: "/a".into(), + }, + "Terminal".into(), + 11, + ); + let b = ( + ElementRef { + bus: ":1.2".into(), + path: "/b".into(), + }, + "Terminal".into(), + 22, + ); + let apps = vec![a.clone(), b.clone()]; + let by_pid = match_application(&apps, "22").expect("pid"); + assert_eq!(by_pid.2, 22); + let err = match_application(&apps, "Terminal").unwrap_err().0; + assert!(err.contains("pids"), "{err}"); + } +} diff --git a/native/t3-desktop-mcp-rs/src/platform/mod.rs b/native/t3-desktop-mcp-rs/src/platform/mod.rs new file mode 100644 index 00000000000..a632a4db0c5 --- /dev/null +++ b/native/t3-desktop-mcp-rs/src/platform/mod.rs @@ -0,0 +1,258 @@ +//! Platform-specific desktop control. +//! +//! Each backend returns finished tool text rather than structured data, matching +//! the macOS server: the text *is* the contract the model reads, so keeping it +//! next to the platform quirks that shape it avoids a lossy intermediate layer. +//! +//! Screen capture and display enumeration are shared (see [`crate::capture`]); +//! only the accessibility tree and synthetic input genuinely differ. + +use std::fmt; + +#[cfg(target_os = "linux")] +pub mod linux; +#[cfg(any(windows, target_os = "linux"))] +pub mod agent_cursor; +#[cfg(windows)] +pub mod windows; + +/// A tool failure that is worth showing the model verbatim. +/// +/// These are expected outcomes — a missing window, a refused permission — not +/// bugs, so they render as `error: ...` tool text instead of JSON-RPC errors. +/// The model can usually recover by picking a different target. +#[derive(Debug)] +pub struct DesktopError(pub String); + +impl fmt::Display for DesktopError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "{}", self.0) + } +} + +impl std::error::Error for DesktopError {} + +impl DesktopError { + pub fn new(message: impl Into) -> Self { + Self(message.into()) + } +} + +pub type Result = std::result::Result; + +/// Where a pointer action should land. +#[derive(Debug, Clone, Copy)] +pub enum Point { + /// An element from the most recent `get_app_state` snapshot. + Element(u32), + /// Absolute screen coordinates in logical pixels. + Screen(f64, f64), +} + +/// Scroll axis and sign, already normalised away from the tool's string enum. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ScrollDirection { + Up, + Down, + Left, + Right, +} + +impl ScrollDirection { + pub fn parse(raw: &str) -> Result { + match raw.to_ascii_lowercase().as_str() { + "up" | "u" => Ok(Self::Up), + "down" | "d" => Ok(Self::Down), + "left" | "l" => Ok(Self::Left), + "right" | "r" => Ok(Self::Right), + other => Err(DesktopError::new(format!( + "unknown direction '{other}' — use up, down, left, or right" + ))), + } + } + + /// Horizontal and vertical deltas in wheel notches for `amount` lines. + pub fn deltas(self, amount: i32) -> (i32, i32) { + match self { + Self::Up => (0, amount), + Self::Down => (0, -amount), + Self::Left => (-amount, 0), + Self::Right => (amount, 0), + } + } +} + +/// A running application, as reported by `list_apps`. +pub struct AppInfo { + pub name: String, + /// Bundle-id equivalent: executable path stem on Windows, desktop id on Linux. + pub id: String, + pub pid: u32, + pub windows: usize, + pub frontmost: bool, +} + +/// Escape app name/id tokens so `format_app_list` lines stay one line and +/// `parse_app_line` can locate the trailing ` [id]` marker reliably. +pub fn escape_app_field(value: &str) -> String { + let mut out = String::with_capacity(value.len()); + for ch in value.chars() { + match ch { + '\\' => out.push_str("\\\\"), + '[' => out.push_str("\\["), + ']' => out.push_str("\\]"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + other => out.push(other), + } + } + out +} + +/// Reverse `escape_app_field` after splitting a `format_app_list` line. +pub fn unescape_app_field(value: &str) -> String { + let mut out = String::with_capacity(value.len()); + let mut chars = value.chars().peekable(); + while let Some(ch) = chars.next() { + if ch == '\\' { + match chars.next() { + Some('\\') => out.push('\\'), + Some('[') => out.push('['), + Some(']') => out.push(']'), + Some('n') => out.push('\n'), + Some('r') => out.push('\r'), + Some(other) => { + out.push('\\'); + out.push(other); + } + None => out.push('\\'), + } + } else { + out.push(ch); + } + } + out +} + +/// Renders `list_apps` output identically to the macOS server so the model sees +/// one format everywhere. +pub fn format_app_list(mut apps: Vec) -> String { + if apps.is_empty() { + return "no running applications with windows".to_string(); + } + apps.sort_by(|left, right| left.name.to_lowercase().cmp(&right.name.to_lowercase())); + apps.iter() + .map(|app| { + format!( + "{} [{}] pid={} windows={}{}", + escape_app_field(&app.name), + escape_app_field(&app.id), + app.pid, + app.windows, + if app.frontmost { " FRONTMOST" } else { "" } + ) + }) + .collect::>() + .join("\n") +} + +/// The operations every backend must provide. +/// +/// `&mut self` throughout because `get_app_state` refreshes the element +/// registry that later calls resolve ids against. +pub trait Desktop { + fn list_apps(&mut self) -> Result; + fn get_app_state(&mut self, app: &str, max_depth: usize, max_elements: usize) -> Result; + fn activate_app(&mut self, app: &str) -> Result; + fn click(&mut self, target: Point, click_count: u32) -> Result; + fn right_click(&mut self, target: Point) -> Result; + fn drag(&mut self, from: Point, to: Point) -> Result; + fn type_text(&mut self, text: &str, element: Option) -> Result; + fn press_key(&mut self, key: &str, modifiers: &[String]) -> Result; + fn scroll( + &mut self, + direction: ScrollDirection, + amount: i32, + element: Option, + ) -> Result; + fn set_value(&mut self, element: u32, value: &str) -> Result; + fn select_text(&mut self, element: u32, start: usize, length: Option) -> Result; + /// Resolve an app query to a pid so shared capture can find its windows. + fn resolve_pid(&mut self, app: &str) -> Result; +} + +/// Build the backend for the host platform. +pub fn backend() -> Result> { + #[cfg(windows)] + { + Ok(Box::new(windows::WindowsDesktop::new()?)) + } + #[cfg(target_os = "linux")] + { + Ok(Box::new(linux::LinuxDesktop::new()?)) + } + #[cfg(not(any(windows, target_os = "linux")))] + { + Err(DesktopError::new( + "t3-desktop-mcp-rs supports Windows and Linux; macOS uses the Swift t3-desktop-mcp server", + )) + } +} + +#[cfg(test)] +mod tests { + use super::{AppInfo, ScrollDirection, format_app_list}; + + #[test] + fn scroll_directions_accept_short_and_long_forms() { + assert_eq!(ScrollDirection::parse("up").unwrap(), ScrollDirection::Up); + assert_eq!(ScrollDirection::parse("D").unwrap(), ScrollDirection::Down); + assert!(ScrollDirection::parse("sideways").is_err()); + } + + #[test] + fn scrolling_down_moves_content_up() { + // Wheel deltas are inverted relative to the direction the content moves; + // getting this backwards is an easy and very confusing bug. + assert_eq!(ScrollDirection::Down.deltas(5), (0, -5)); + assert_eq!(ScrollDirection::Up.deltas(5), (0, 5)); + assert_eq!(ScrollDirection::Right.deltas(3), (3, 0)); + } + + #[test] + fn app_list_sorts_case_insensitively_and_marks_frontmost() { + let rendered = format_app_list(vec![ + AppInfo { + name: "zed".into(), + id: "zed".into(), + pid: 2, + windows: 1, + frontmost: false, + }, + AppInfo { + name: "Chrome".into(), + id: "chrome".into(), + pid: 1, + windows: 3, + frontmost: true, + }, + ]); + + let lines: Vec<&str> = rendered.lines().collect(); + assert!(lines[0].starts_with("Chrome [chrome] pid=1 windows=3 FRONTMOST")); + assert!(lines[1].starts_with("zed")); + } + + #[test] + fn app_list_escapes_newlines_in_names() { + let rendered = format_app_list(vec![AppInfo { + name: "Foo\nBar".into(), + id: "com.foo".into(), + pid: 1, + windows: 1, + frontmost: false, + }]); + assert!(!rendered.contains('\n') || rendered.lines().count() == 1); + assert!(rendered.contains("Foo\\nBar")); + } +} diff --git a/native/t3-desktop-mcp-rs/src/platform/windows.rs b/native/t3-desktop-mcp-rs/src/platform/windows.rs new file mode 100644 index 00000000000..b256d658d2c --- /dev/null +++ b/native/t3-desktop-mcp-rs/src/platform/windows.rs @@ -0,0 +1,697 @@ +//! Windows backend, built on UI Automation. +//! +//! UI Automation is the direct counterpart to the macOS Accessibility API: the +//! same tree of roles, names and values, and the same patterns (Invoke, Value, +//! Text) that let us press a button properly instead of guessing at pixels. +//! Coordinates remain available as a fallback for canvas-style UIs that expose +//! nothing useful. + +use std::collections::HashMap; + +use uiautomation::UIAutomation; +use uiautomation::UIElement; +use uiautomation::inputs::{Keyboard, Mouse, MouseButton}; +use uiautomation::patterns::{UIInvokePattern, UITextPattern, UIValuePattern}; +use uiautomation::types::{Handle, Point as UIPoint}; +use windows::Win32::Foundation::{HWND, LPARAM, POINT, WPARAM}; +use windows::core::BOOL; +use windows::Win32::Graphics::Gdi::ScreenToClient; +use windows::Win32::UI::Input::KeyboardAndMouse::{ + INPUT, INPUT_0, INPUT_MOUSE, MOUSEEVENTF_HWHEEL, MOUSEEVENTF_WHEEL, MOUSEINPUT, SendInput, +}; +use windows::Win32::UI::WindowsAndMessaging::{ + ChildWindowFromPointEx, CWP_SKIPDISABLED, CWP_SKIPINVISIBLE, EnumWindows, GetClassNameW, + GetWindowLongW, GetWindowThreadProcessId, IsWindowVisible, PostMessageW, SW_RESTORE, + SetForegroundWindow, ShowWindow, WindowFromPoint, WM_LBUTTONDOWN, WM_LBUTTONUP, WM_MOUSEMOVE, + WM_RBUTTONDOWN, WM_RBUTTONUP, GWL_STYLE, +}; + +use super::agent_cursor::AgentCursor; +use super::{Desktop, DesktopError, Point, Result, ScrollDirection, format_app_list}; +use crate::apps; + +/// One wheel notch, as Windows defines it. +const WHEEL_DELTA: i32 = 120; + +pub struct WindowsDesktop { + automation: UIAutomation, + /// Element handles from the most recent `get_app_state`, keyed by the + /// numeric part of the `e12` ids handed to the model. + registry: HashMap, +} + +impl WindowsDesktop { + pub fn new() -> Result { + let automation = UIAutomation::new().map_err(|error| { + DesktopError::new(format!("failed to initialise UI Automation: {error}")) + })?; + Ok(Self { + automation, + registry: HashMap::new(), + }) + } + + fn element(&self, id: u32) -> Result<&UIElement> { + self.registry.get(&id).ok_or_else(|| { + DesktopError::new(format!( + "element e{id} is not in the current snapshot — call get_app_state again, ids are per-snapshot" + )) + }) + } + + /// Centre of an element in screen coordinates. + fn center(element: &UIElement) -> Result<(f64, f64)> { + let rect = element.get_bounding_rectangle().map_err(|error| { + DesktopError::new(format!("element has no on-screen bounds: {error}")) + })?; + let width = rect.get_right() - rect.get_left(); + let height = rect.get_bottom() - rect.get_top(); + if width <= 0 || height <= 0 { + return Err(DesktopError::new( + "element is not visible on screen — scroll it into view first", + )); + } + Ok(( + f64::from(rect.get_left()) + f64::from(width) / 2.0, + f64::from(rect.get_top()) + f64::from(height) / 2.0, + )) + } + + fn point_coordinates(&self, target: Point) -> Result<(f64, f64)> { + match target { + Point::Screen(x, y) => Ok((x, y)), + Point::Element(id) => Self::center(self.element(id)?), + } + } + + /// Top-level visible windows belonging to `pid`. + fn top_level_windows(pid: u32) -> Vec { + struct Search { + pid: u32, + found: Vec, + } + + unsafe extern "system" fn visit(window: HWND, param: LPARAM) -> BOOL { + // SAFETY: `param` is the `&mut Search` handed to EnumWindows below, + // which outlives the enumeration. + let search = unsafe { &mut *(param.0 as *mut Search) }; + let mut owner = 0u32; + unsafe { GetWindowThreadProcessId(window, Some(&mut owner)) }; + if owner == search.pid && unsafe { IsWindowVisible(window) }.as_bool() { + search.found.push(window); + } + // Non-zero keeps the enumeration going. + BOOL(1) + } + + let mut search = Search { + pid, + found: Vec::new(), + }; + let _ = unsafe { + EnumWindows( + Some(visit), + LPARAM(&mut search as *mut Search as isize), + ) + }; + search.found + } + + /// Pack client coordinates into an `lParam` for mouse window messages. + fn pack_client_lparam(x: i32, y: i32) -> LPARAM { + let lo = (x as u16) as u32; + let hi = (y as u16) as u32; + LPARAM(((hi << 16) | lo) as isize) + } + + /// Resolve the deepest visible child HWND under a screen point. + fn hwnd_at_screen(x: i32, y: i32) -> Option { + let point = POINT { x, y }; + let mut hwnd = unsafe { WindowFromPoint(point) }; + if hwnd.0.is_null() { + return None; + } + // Walk into nested children — a single ChildWindowFromPointEx only + // returns the immediate child, which misses grandchildren (Bot: nested + // Win32 controls). + loop { + let mut client = point; + if !unsafe { ScreenToClient(hwnd, &mut client) }.as_bool() { + break; + } + let child = unsafe { + ChildWindowFromPointEx(hwnd, client, CWP_SKIPINVISIBLE | CWP_SKIPDISABLED) + }; + if child.0.is_null() || child.0 == hwnd.0 { + break; + } + hwnd = child; + } + Some(hwnd) + } + + /// Only post mouse messages to control classes known to honor them. + /// Everything else (Chromium, Qt, DirectInput, unknown) falls through to + /// the real cursor path so we never report a false click success. + fn accepts_posted_mouse(hwnd: HWND) -> bool { + let mut buf = [0u16; 256]; + let len = unsafe { GetClassNameW(hwnd, &mut buf) }; + if len == 0 { + return false; + } + let class = String::from_utf16_lossy(&buf[..len as usize]); + if class.starts_with("Chrome_") || class.starts_with("Chrome_WidgetWin") { + return false; + } + if class == "Static" { + // Static labels ignore mouse messages unless SS_NOTIFY is set. + const SS_NOTIFY: i32 = 0x0001_0000; + let style = unsafe { GetWindowLongW(hwnd, GWL_STYLE) }; + return style & SS_NOTIFY != 0; + } + matches!( + class.as_str(), + "Button" + | "Edit" + | "ComboBox" + | "ComboLBox" + | "ListBox" + | "SysListView32" + | "SysTreeView32" + | "SysTabControl32" + | "ToolbarWindow32" + | "msctls_trackbar32" + | "msctls_updown32" + | "ScrollBar" + | "#32770" + ) || class.starts_with("WindowsForms") + } + + /// Deliver a left/right click via posted mouse messages so the system + /// cursor does not move. Only used for known-good Win32 classes; callers + /// fall back to the cursor path when this returns false. + fn background_click(x: f64, y: f64, right: bool) -> bool { + let sx = x.round() as i32; + let sy = y.round() as i32; + let Some(hwnd) = Self::hwnd_at_screen(sx, sy) else { + return false; + }; + if !Self::accepts_posted_mouse(hwnd) { + return false; + } + let mut client = POINT { x: sx, y: sy }; + if !unsafe { ScreenToClient(hwnd, &mut client) }.as_bool() { + return false; + } + let lp = Self::pack_client_lparam(client.x, client.y); + // MK_LBUTTON = 0x0001, MK_RBUTTON = 0x0002 + let (down, up, mk) = if right { + (WM_RBUTTONDOWN, WM_RBUTTONUP, 0x0002usize) + } else { + (WM_LBUTTONDOWN, WM_LBUTTONUP, 0x0001usize) + }; + // Prime hover state; some controls ignore down without a prior move. + let _ = unsafe { PostMessageW(Some(hwnd), WM_MOUSEMOVE, WPARAM(0), lp) }; + let down_ok = unsafe { PostMessageW(Some(hwnd), down, WPARAM(mk), lp) }.is_ok(); + let up_ok = unsafe { PostMessageW(Some(hwnd), up, WPARAM(0), lp) }.is_ok(); + down_ok && up_ok + } + + fn scroll_wheel(horizontal: bool, notches: i32) -> Result<()> { + let input = INPUT { + r#type: INPUT_MOUSE, + Anonymous: INPUT_0 { + mi: MOUSEINPUT { + dx: 0, + dy: 0, + mouseData: (notches * WHEEL_DELTA) as u32, + dwFlags: if horizontal { + MOUSEEVENTF_HWHEEL + } else { + MOUSEEVENTF_WHEEL + }, + time: 0, + dwExtraInfo: 0, + }, + }, + }; + let sent = unsafe { SendInput(&[input], std::mem::size_of::() as i32) }; + if sent == 0 { + return Err(DesktopError::new( + "the system rejected synthetic scrolling — another app may be holding an input grab", + )); + } + Ok(()) + } + + /// Render one element as an outline row, registering it when it is + /// interactive enough to be worth an id. + fn describe(&mut self, element: &UIElement, depth: usize, next_id: &mut u32) -> Option { + let control_type = element + .get_control_type() + .map(|kind| format!("{kind:?}")) + .unwrap_or_else(|_| "Unknown".to_string()); + let name = element.get_name().unwrap_or_default(); + let value = element + .get_pattern::() + .ok() + .and_then(|pattern| pattern.get_value().ok()) + .filter(|value| !value.is_empty()); + + // Rows with nothing to say are noise in an already large tree. + if name.is_empty() && value.is_none() && control_type == "Pane" { + return None; + } + + let interactive = element.is_enabled().unwrap_or(false) + && matches!( + control_type.as_str(), + "Button" + | "CheckBox" + | "ComboBox" + | "Edit" + | "Document" + | "Hyperlink" + | "ListItem" + | "MenuItem" + | "RadioButton" + | "Slider" + | "SplitButton" + | "Tab" + | "TabItem" + | "Text" + | "Tree" + | "TreeItem" + ); + + let mut row = " ".repeat(depth); + if interactive { + *next_id += 1; + row.push_str(&format!("[e{next_id}] ")); + self.registry.insert(*next_id, element.clone()); + } + row.push_str(&control_type); + if !name.is_empty() { + row.push_str(&format!(" \"{}\"", truncate(&name, 120))); + } + if let Some(value) = value { + row.push_str(&format!(" = \"{}\"", truncate(&value, 120))); + } + if !element.is_enabled().unwrap_or(true) { + row.push_str(" (disabled)"); + } + Some(row) + } + + #[allow(clippy::too_many_arguments)] + fn walk( + &mut self, + element: &UIElement, + depth: usize, + max_depth: usize, + max_elements: usize, + next_id: &mut u32, + lines: &mut Vec, + ) { + if depth > max_depth || lines.len() >= max_elements { + return; + } + if let Some(row) = self.describe(element, depth, next_id) { + lines.push(row); + } + // At max_depth we still describe this node, but skip child enumeration — + // walking siblings would only waste UIA work with no lines added. + if depth == max_depth { + return; + } + + let walker = match self.automation.create_tree_walker() { + Ok(walker) => walker, + Err(_) => return, + }; + let mut child = walker.get_first_child(element).ok(); + while let Some(current) = child { + if lines.len() >= max_elements { + lines.push(format!( + "{}… truncated at {max_elements} elements — raise max_elements or target a child", + " ".repeat(depth + 1) + )); + return; + } + self.walk(¤t, depth + 1, max_depth, max_elements, next_id, lines); + child = walker.get_next_sibling(¤t).ok(); + } + } +} + +fn truncate(value: &str, limit: usize) -> String { + let cleaned = value.replace(['\n', '\r'], " "); + if cleaned.chars().count() <= limit { + return cleaned; + } + cleaned.chars().take(limit).collect::() + "…" +} + +/// Translate the tool's modifier names into the `uiautomation` key syntax. +/// +/// `cmd` maps to Win rather than failing: models trained on macOS reach for it +/// constantly, and Win is the closest analogue. +/// +/// Unknown modifiers are rejected (except `fn`, which has no synthetic +/// equivalent and is intentionally ignored) so a typo like `ctl` cannot +/// silently send the bare key while reporting success. +fn key_sequence(key: &str, modifiers: &[String]) -> Result { + let mut sequence = String::new(); + for modifier in modifiers { + let token = match modifier.to_lowercase().as_str() { + "cmd" | "command" | "win" | "super" | "meta" => "{win}", + "ctrl" | "control" => "{ctrl}", + "alt" | "option" => "{alt}", + "shift" => "{shift}", + // `fn` has no synthetic equivalent on Windows; dropping it is better + // than refusing an otherwise valid chord. + "fn" => "", + other => { + return Err(format!( + "unsupported modifier '{other}' — use ctrl, shift, alt, or cmd" + )); + } + }; + sequence.push_str(token); + } + sequence.push_str(&match key.to_lowercase().as_str() { + "return" | "enter" => "{enter}".to_string(), + "tab" => "{tab}".to_string(), + "escape" | "esc" => "{esc}".to_string(), + "space" => " ".to_string(), + "backspace" => "{backspace}".to_string(), + "delete" => "{delete}".to_string(), + "up" => "{up}".to_string(), + "down" => "{down}".to_string(), + "left" => "{left}".to_string(), + "right" => "{right}".to_string(), + "home" => "{home}".to_string(), + "end" => "{end}".to_string(), + other => other.to_string(), + }); + Ok(sequence) +} + +impl Desktop for WindowsDesktop { + fn list_apps(&mut self) -> Result { + Ok(format_app_list(apps::list_apps()?)) + } + + fn resolve_pid(&mut self, app: &str) -> Result { + apps::resolve_pid(app) + } + + fn get_app_state(&mut self, app: &str, max_depth: usize, max_elements: usize) -> Result { + // Ids are per-snapshot, so previous handles must not resolve — clear + // before resolve_pid so a failed lookup cannot leave stale ids. + self.registry.clear(); + let pid = apps::resolve_pid(app)?; + let windows = Self::top_level_windows(pid); + if windows.is_empty() { + return Err(DesktopError::new(format!( + "{app} (pid {pid}) has no visible window" + ))); + } + + let mut next_id = 0u32; + let mut lines = vec![format!("{app} (pid {pid}), {} window(s)", windows.len())]; + // One shared element budget across every window — recreating the walk + // buffer per window would let multi-window apps emit + // windows.len() * max_elements rows. + let mut element_lines = Vec::new(); + + for (index, window) in windows.iter().enumerate() { + let element = match self + .automation + .element_from_handle(Handle::from(window.0 as isize)) + { + Ok(element) => element, + Err(_) => continue, + }; + let title = element.get_name().unwrap_or_default(); + lines.push(String::new()); + lines.push(format!("── window {index}: \"{title}\"")); + let window_start = element_lines.len(); + self.walk( + &element, + 0, + max_depth, + max_elements, + &mut next_id, + &mut element_lines, + ); + lines.extend(element_lines[window_start..].iter().cloned()); + } + + if next_id == 0 { + lines.push(String::new()); + lines.push( + "no interactive elements found — the app may render its own UI, so use screenshot \ + and click with coordinates" + .to_string(), + ); + } + Ok(lines.join("\n")) + } + + fn activate_app(&mut self, app: &str) -> Result { + let pid = apps::resolve_pid(app)?; + let windows = Self::top_level_windows(pid); + let window = windows.first().ok_or_else(|| { + DesktopError::new(format!("{app} (pid {pid}) has no window to activate")) + })?; + unsafe { + let _ = ShowWindow(*window, SW_RESTORE); + } + let raised = unsafe { SetForegroundWindow(*window) }; + if !raised.as_bool() { + // Windows refuses foreground changes from background processes in + // some states; say so rather than claim a success the model can see + // is false in the next screenshot. + return Err(DesktopError::new(format!( + "Windows refused to bring {app} forward — click its taskbar button, or try again \ + after interacting with the desktop" + ))); + } + Ok(format!("activated {app} (pid {pid})")) + } + + fn click(&mut self, target: Point, click_count: u32) -> Result { + // An element press goes through the control's own Invoke handler, which + // is far more reliable than a synthetic click landing on the right pixel. + if let Point::Element(id) = target + && click_count == 1 + && let Ok(element) = self.element(id) + && let Ok(invoke) = element.get_pattern::() + { + // Wait for the agent pointer to land before invoking, matching Mac. + if let Ok((x, y)) = Self::center(element) { + AgentCursor::shared().press(x, y); + } + if invoke.invoke().is_ok() { + return Ok(format!("pressed e{id}")); + } + } + + let (x, y) = self.point_coordinates(target)?; + AgentCursor::shared().press(x, y); + // Prefer window-message delivery so the user's cursor stays put. + if click_count <= 1 && Self::background_click(x, y, false) { + return Ok(format!("clicked at ({x:.0}, {y:.0}) in background")); + } + + let mouse = Mouse::default(); + let point = UIPoint::new(x as i32, y as i32); + for _ in 0..click_count.max(1) { + mouse + .click(&point) + .map_err(|error| DesktopError::new(format!("click failed: {error}")))?; + } + Ok(format!( + "clicked at ({:.0}, {:.0}) via cursor{}", + x, + y, + if click_count > 1 { + format!(" x{click_count}") + } else { + String::new() + } + )) + } + + fn right_click(&mut self, target: Point) -> Result { + let (x, y) = self.point_coordinates(target)?; + AgentCursor::shared().press(x, y); + if Self::background_click(x, y, true) { + return Ok(format!("right-clicked at ({x:.0}, {y:.0}) in background")); + } + Mouse::default() + .right_click(&UIPoint::new(x as i32, y as i32)) + .map_err(|error| DesktopError::new(format!("right click failed: {error}")))?; + Ok(format!("right-clicked at ({x:.0}, {y:.0}) via cursor")) + } + + fn drag(&mut self, from: Point, to: Point) -> Result { + let (from_x, from_y) = self.point_coordinates(from)?; + let (to_x, to_y) = self.point_coordinates(to)?; + AgentCursor::shared().show(from_x, from_y); + let mouse = Mouse::default(); + mouse + .move_to(&UIPoint::new(from_x as i32, from_y as i32)) + .map_err(|error| DesktopError::new(format!("could not reach the drag origin: {error}")))?; + // Fly overlay to the end before the real drag starts (button still up). + AgentCursor::shared().press(to_x, to_y); + mouse + .drag_to(MouseButton::LEFT, &UIPoint::new(to_x as i32, to_y as i32)) + .map_err(|error| DesktopError::new(format!("drag failed: {error}")))?; + Ok(format!( + "dragged ({from_x:.0}, {from_y:.0}) → ({to_x:.0}, {to_y:.0})" + )) + } + + fn type_text(&mut self, text: &str, element: Option) -> Result { + if let Some(id) = element { + self.element(id)? + .set_focus() + .map_err(|error| DesktopError::new(format!("could not focus e{id}: {error}")))?; + } + Keyboard::default() + .send_text(text) + .map_err(|error| DesktopError::new(format!("typing failed: {error}")))?; + Ok(format!("typed {} characters", text.chars().count())) + } + + fn press_key(&mut self, key: &str, modifiers: &[String]) -> Result { + let sequence = key_sequence(key, modifiers).map_err(DesktopError::new)?; + Keyboard::default() + .send_keys(&sequence) + .map_err(|error| DesktopError::new(format!("key press failed: {error}")))?; + Ok(if modifiers.is_empty() { + format!("pressed {key}") + } else { + format!("pressed {}+{key}", modifiers.join("+")) + }) + } + + fn scroll( + &mut self, + direction: ScrollDirection, + amount: i32, + element: Option, + ) -> Result { + // The wheel goes to whatever is under the cursor, so move there first. + if let Some(id) = element { + let (x, y) = Self::center(self.element(id)?)?; + AgentCursor::shared().show(x, y); + Mouse::default() + .move_to(&UIPoint::new(x as i32, y as i32)) + .map_err(|error| DesktopError::new(format!("could not move cursor: {error}")))?; + } + let (horizontal, vertical) = direction.deltas(amount); + if horizontal != 0 { + Self::scroll_wheel(true, horizontal)?; + } + if vertical != 0 { + Self::scroll_wheel(false, vertical)?; + } + Ok(format!("scrolled {direction:?} by {amount}").to_lowercase()) + } + + fn set_value(&mut self, element: u32, value: &str) -> Result { + let target = self.element(element)?; + let pattern = target.get_pattern::().map_err(|_| { + DesktopError::new(format!( + "e{element} does not accept a value directly — click it and use type_text" + )) + })?; + pattern + .set_value(value) + .map_err(|error| DesktopError::new(format!("could not set e{element}: {error}")))?; + Ok(format!("set e{element} to \"{}\"", truncate(value, 80))) + } + + fn select_text(&mut self, element: u32, start: usize, length: Option) -> Result { + let target = self.element(element)?; + let pattern = target.get_pattern::().map_err(|_| { + DesktopError::new(format!("e{element} does not expose selectable text")) + })?; + let document = pattern + .get_document_range() + .map_err(|error| DesktopError::new(format!("could not read e{element}: {error}")))?; + let text = document.get_text(-1).map_err(|error| { + DesktopError::new(format!("could not read e{element}: {error}")) + })?; + let total = text.chars().count(); + let start = start.min(total); + let end = length.map_or(total, |count| (start + count).min(total)); + + let range = document.clone(); + range + .move_endpoint_by_unit( + uiautomation::types::TextPatternRangeEndpoint::Start, + uiautomation::types::TextUnit::Character, + start as i32, + ) + .and_then(|_| { + range.move_endpoint_by_unit( + uiautomation::types::TextPatternRangeEndpoint::End, + uiautomation::types::TextUnit::Character, + -((total - end) as i32), + ) + }) + .and_then(|_| range.select()) + .map_err(|error| DesktopError::new(format!("could not select in e{element}: {error}")))?; + Ok(format!("selected {} characters in e{element}", end - start)) + } +} + +#[cfg(test)] +mod tests { + use super::{key_sequence, truncate}; + + #[test] + fn cmd_is_translated_to_the_windows_key() { + // Models trained on macOS send cmd constantly; refusing it would make + // every save and copy fail on Windows. + assert_eq!(key_sequence("s", &["cmd".to_string()]).unwrap(), "{win}s"); + assert_eq!(key_sequence("s", &["ctrl".to_string()]).unwrap(), "{ctrl}s"); + } + + #[test] + fn named_keys_become_uiautomation_tokens() { + assert_eq!(key_sequence("return", &[]).unwrap(), "{enter}"); + assert_eq!(key_sequence("Escape", &[]).unwrap(), "{esc}"); + assert_eq!( + key_sequence("a", &["ctrl".to_string(), "shift".to_string()]).unwrap(), + "{ctrl}{shift}a" + ); + } + + #[test] + fn fn_modifier_is_dropped_rather_than_breaking_the_chord() { + assert_eq!( + key_sequence("c", &["fn".to_string(), "ctrl".to_string()]).unwrap(), + "{ctrl}c" + ); + } + + #[test] + fn unrecognized_modifiers_are_rejected() { + let error = key_sequence("c", &["ctl".to_string()]).unwrap_err(); + assert!(error.contains("unsupported modifier")); + assert!(error.contains("ctl")); + } + + #[test] + fn truncation_collapses_newlines_and_marks_elision() { + assert_eq!(truncate("one\ntwo", 40), "one two"); + let long = truncate(&"x".repeat(200), 10); + assert_eq!(long.chars().count(), 11, "10 chars plus the ellipsis"); + assert!(long.ends_with('…')); + } +} diff --git a/native/t3-desktop-mcp-rs/src/tools.rs b/native/t3-desktop-mcp-rs/src/tools.rs new file mode 100644 index 00000000000..a634f03bdc5 --- /dev/null +++ b/native/t3-desktop-mcp-rs/src/tools.rs @@ -0,0 +1,355 @@ +//! Tool schemas, kept byte-compatible with the macOS Swift server's `toolDefs`. +//! +//! A model that learned the tools on one platform must not have to relearn them +//! on another, so the names, argument shapes and descriptions are deliberately +//! identical. Behavioural differences belong in the tool text, not the schema. + +use serde_json::{Value, json}; + +/// Host settings pass `T3_DESKTOP_BROWSER=0` when browser control is off. +pub fn env_flag_disabled(name: &str) -> bool { + match std::env::var(name) { + Ok(raw) => { + let trimmed = raw.trim().to_ascii_lowercase(); + matches!(trimmed.as_str(), "0" | "false" | "off" | "no") + } + Err(_) => false, + } +} + +pub fn browser_control_enabled() -> bool { + !env_flag_disabled("T3_DESKTOP_BROWSER") +} + +pub fn tool_defs() -> Value { + let defs = all_tool_defs(); + if browser_control_enabled() { + return defs; + } + let Some(array) = defs.as_array() else { + return defs; + }; + Value::Array( + array + .iter() + .filter(|tool| { + tool.get("name") + .and_then(Value::as_str) + .is_none_or(|name| !name.starts_with("browser_")) + }) + .cloned() + .collect(), + ) +} + +fn all_tool_defs() -> Value { + json!([ + { + "name": "list_apps", + "description": "List running applications with their bundle id, pid, window count, and which is frontmost. Note that one app can have several running instances and only some may own windows.", + "inputSchema": { "type": "object", "properties": {} } + }, + { + "name": "get_app_state", + "description": "Read an app's accessibility tree as an indented outline. Interactive elements are prefixed with an id like [e12] that you pass to click/type_text/scroll. Call this before interacting, and again after the UI changes, since ids are per-snapshot.", + "inputSchema": { + "type": "object", + "properties": { + "app": { "type": "string", "description": "App name, bundle id, or pid" }, + "max_depth": { "type": "integer", "description": "Max tree depth (default 18)" }, + "max_elements": { "type": "integer", "description": "Max elements to emit (default 800)" } + }, + "required": ["app"] + } + }, + { + "name": "click", + "description": "Click an element by element_id (preferred, uses the accessibility press action) or at absolute screen coordinates.", + "inputSchema": { + "type": "object", + "properties": { + "element_id": { "type": "string", "description": "Element id from get_app_state, e.g. e12" }, + "x": { "type": "number" }, + "y": { "type": "number" }, + "click_count": { "type": "integer", "description": "1 for single, 2 for double-click" } + } + } + }, + { + "name": "type_text", + "description": "Type literal text into the focused element, optionally focusing element_id first.", + "inputSchema": { + "type": "object", + "properties": { + "text": { "type": "string" }, + "element_id": { "type": "string", "description": "Focus this element before typing" } + }, + "required": ["text"] + } + }, + { + "name": "press_key", + "description": "Press a named key with optional modifiers, e.g. key='s' modifiers=['ctrl'] to save, or key='return'.", + "inputSchema": { + "type": "object", + "properties": { + "key": { "type": "string" }, + "modifiers": { + "type": "array", + "items": { "type": "string" }, + "description": "Any of cmd, shift, alt, ctrl, fn. cmd maps to the Windows/Super key off macOS." + } + }, + "required": ["key"] + } + }, + { + "name": "scroll", + "description": "Scroll up, down, left, or right, optionally positioning the cursor over element_id first.", + "inputSchema": { + "type": "object", + "properties": { + "direction": { "type": "string", "enum": ["up", "down", "left", "right"] }, + "amount": { "type": "integer", "description": "Scroll lines (default 5)" }, + "element_id": { "type": "string" } + } + } + }, + { + "name": "activate_app", + "description": "Bring an app to the foreground.", + "inputSchema": { + "type": "object", + "properties": { "app": { "type": "string" } }, + "required": ["app"] + } + }, + { + "name": "screenshot", + "description": "Capture the app's largest window as a PNG image. Prefer get_app_state for interaction, which is cheaper and gives clickable element ids; use a screenshot when you need to see rendered content the accessibility tree does not describe, such as canvas or video.", + "inputSchema": { + "type": "object", + "properties": { + "app": { "type": "string", "description": "App name, bundle id, or pid" }, + "display": { "type": "integer", "description": "Capture a whole display by index (see list_displays) instead of an app window" }, + "max_width": { "type": "integer", "description": "Downscale to this width in pixels (default 1400)" } + } + } + }, + { + "name": "list_displays", + "description": "List every attached display with its index, resolution and position, for use with screenshot(display: N).", + "inputSchema": { "type": "object", "properties": {} } + }, + { + "name": "right_click", + "description": "Right-click (secondary click) an element or screen position to open a context menu.", + "inputSchema": { + "type": "object", + "properties": { + "element_id": { "type": "string" }, + "x": { "type": "number" }, + "y": { "type": "number" } + } + } + }, + { + "name": "drag", + "description": "Press at one point, drag, and release at another. Accepts element ids or coordinates on each end.", + "inputSchema": { + "type": "object", + "properties": { + "from_element_id": { "type": "string" }, + "to_element_id": { "type": "string" }, + "from_x": { "type": "number" }, + "from_y": { "type": "number" }, + "to_x": { "type": "number" }, + "to_y": { "type": "number" } + } + } + }, + { + "name": "set_value", + "description": "Replace a text field's contents directly. More reliable than select-all-then-type for long values, though some fields reject it and need click + type_text.", + "inputSchema": { + "type": "object", + "properties": { + "element_id": { "type": "string" }, + "value": { "type": "string" } + }, + "required": ["element_id", "value"] + } + }, + { + "name": "select_text", + "description": "Select a character range inside a text element. Defaults to selecting from 'start' to the end of the value.", + "inputSchema": { + "type": "object", + "properties": { + "element_id": { "type": "string" }, + "start": { "type": "integer", "description": "Start offset (default 0)" }, + "length": { "type": "integer", "description": "Characters to select (default: to end)" } + }, + "required": ["element_id"] + } + }, + { + "name": "browser_open_tab", + "description": "Open a URL in a new background tab inside the agent's own labelled tab group, in the user's signed-in Chrome. The user keeps browsing their tabs undisturbed. Returns a tab_id for browser_snapshot / browser_click.", + "inputSchema": { + "type": "object", + "properties": { "url": { "type": "string", "description": "URL to open (default about:blank)" } } + } + }, + { + "name": "browser_list_tabs", + "description": "List the tabs in the agent's own Chrome window, marking the active one.", + "inputSchema": { "type": "object", "properties": {} } + }, + { + "name": "browser_select_tab", + "description": "Make one of the agent's tabs the visible one. Does not affect the user's tabs.", + "inputSchema": { + "type": "object", + "properties": { + "tab_id": { "type": "integer", "description": "From browser_list_tabs" }, + "index": { "type": "integer", "description": "1-based index, fallback mode only" } + } + } + }, + { + "name": "browser_close_tab", + "description": "Close one of the agent's tabs.", + "inputSchema": { + "type": "object", + "properties": { + "tab_id": { "type": "integer", "description": "From browser_list_tabs" }, + "index": { "type": "integer", "description": "1-based index, fallback mode only" } + } + } + }, + { + "name": "browser_snapshot", + "description": "List the interactive elements on a page in one of the agent's tabs, with indices to pass to browser_click. Works on a background tab, so the user can be looking at something else.", + "inputSchema": { + "type": "object", + "properties": { "tab_id": { "type": "integer", "description": "From browser_open_tab or browser_list_tabs" } }, + "required": ["tab_id"] + } + }, + { + "name": "browser_click", + "description": "Click in one of the agent's tabs, by element index from browser_snapshot or by page coordinates. Works on a background tab.", + "inputSchema": { + "type": "object", + "properties": { + "tab_id": { "type": "integer" }, + "index": { "type": "integer", "description": "Element index from browser_snapshot" }, + "x": { "type": "number" }, + "y": { "type": "number" } + }, + "required": ["tab_id"] + } + }, + { + "name": "browser_type", + "description": "Type text into the focused field of one of the agent's tabs. Click the field first.", + "inputSchema": { + "type": "object", + "properties": { "tab_id": { "type": "integer" }, "text": { "type": "string" } }, + "required": ["tab_id", "text"] + } + }, + { + "name": "browser_press_key", + "description": "Press Enter, Tab, Escape or Backspace in one of the agent's tabs.", + "inputSchema": { + "type": "object", + "properties": { + "tab_id": { "type": "integer" }, + "key": { "type": "string", "enum": ["Enter", "Tab", "Escape", "Backspace"] } + }, + "required": ["tab_id", "key"] + } + }, + { + "name": "browser_close_all_tabs", + "description": "Close every tab the agent opened and remove its tab group. Call this when finished with the browser so no empty group is left in the user's tab strip.", + "inputSchema": { "type": "object", "properties": {} } + }, + { + "name": "browser_navigate", + "description": "Point one of the agent's tabs at a different URL.", + "inputSchema": { + "type": "object", + "properties": { "tab_id": { "type": "integer" }, "url": { "type": "string" } }, + "required": ["tab_id", "url"] + } + } + ]) +} + +#[cfg(test)] +mod tests { + use super::{all_tool_defs, tool_defs}; + + /// The macOS server advertises exactly these 23 tools. Drifting apart would + /// silently give a model different capabilities per platform. + #[test] + fn advertises_the_macos_tool_surface() { + let defs = all_tool_defs(); + let names: Vec<&str> = defs + .as_array() + .expect("tool defs are an array") + .iter() + .map(|tool| tool["name"].as_str().expect("tool has a name")) + .collect(); + + assert_eq!(names.len(), 23, "tool count drifted from the macOS server"); + for expected in [ + "list_apps", + "get_app_state", + "click", + "type_text", + "press_key", + "scroll", + "activate_app", + "screenshot", + "list_displays", + "right_click", + "drag", + "set_value", + "select_text", + "browser_open_tab", + "browser_list_tabs", + "browser_select_tab", + "browser_close_tab", + "browser_snapshot", + "browser_click", + "browser_type", + "browser_press_key", + "browser_close_all_tabs", + "browser_navigate", + ] { + assert!(names.contains(&expected), "missing tool {expected}"); + } + } + + #[test] + fn every_tool_declares_an_object_input_schema() { + for tool in tool_defs().as_array().expect("tool defs are an array") { + let schema = &tool["inputSchema"]; + assert_eq!( + schema["type"].as_str(), + Some("object"), + "{} has a non-object input schema", + tool["name"] + ); + assert!( + schema["properties"].is_object(), + "{} is missing properties", + tool["name"] + ); + } + } +} diff --git a/native/t3-desktop-mcp/.gitignore b/native/t3-desktop-mcp/.gitignore new file mode 100644 index 00000000000..30bcfa4ed5c --- /dev/null +++ b/native/t3-desktop-mcp/.gitignore @@ -0,0 +1 @@ +.build/ diff --git a/native/t3-desktop-mcp/Package.swift b/native/t3-desktop-mcp/Package.swift new file mode 100644 index 00000000000..7cdbbcb5dab --- /dev/null +++ b/native/t3-desktop-mcp/Package.swift @@ -0,0 +1,18 @@ +// swift-tools-version:5.9 +import PackageDescription + +// Desktop control MCP server. macOS only: it talks to the Accessibility API, +// which has no counterpart on other platforms, so the build is gated on darwin +// by the desktop artifact script rather than by a runtime check here. +let package = Package( + name: "t3-desktop-mcp", + // macOS 14 for SCScreenshotManager, which replaces the deprecated + // CGWindowListCreateImage path for window capture. + platforms: [.macOS(.v14)], + targets: [ + .executableTarget( + name: "t3-desktop-mcp", + path: "Sources", + ), + ], +) diff --git a/native/t3-desktop-mcp/Sources/AgentCursor.swift b/native/t3-desktop-mcp/Sources/AgentCursor.swift new file mode 100644 index 00000000000..e9f25564c0b --- /dev/null +++ b/native/t3-desktop-mcp/Sources/AgentCursor.swift @@ -0,0 +1,1022 @@ +import AppKit +import Foundation + +// The agent's own pointer. +// +// Desktop control should never fight the person sitting at the machine for +// their mouse, so clicks go straight to a window (see `backgroundClick`) and +// the system cursor is left alone. That leaves nothing on screen to show where +// the agent is working, which is unnerving to watch — so we draw our own +// pointer instead. +// +// AppKit needs a real application bundle to put a window up: a bare executable +// started with `Process` never finishes launching, so the overlay stays +// invisible and silent. The pointer therefore lives in a minimal +// `T3AgentCursor.app`. Preferred launch is `NSWorkspace` (registers with +// Launch Services); if that fails we fall back to `Process` aimed at the +// bundled executable, which still gets a real `Bundle.main`. Move/hide +// commands ride a Unix socket: +// +// {"x": 400, "y": 260} move (screen coordinates, top-left origin) +// {"x": 400, "y": 260, "press": true} move (no click ring) +// {"hide": true} fade out until the next move +// +// Fade is driven by Computer Use tool activity (see noteDesktopTool*), +// not a wall-clock idle after the last move. The overlay stays up across +// mid-task pauses; it fades once desktop tools/call traffic stops. +// +// The look is the soft translucent bubble (lavender glow, rounded +// arrow, spring follow with tilt/squash, idle breathe) — never a +// system-style pointer. No click ring and no settle wobble. + +private let overlayAppName = "T3AgentCursor.app" +private let overlayExecutableName = "T3AgentCursor" +private let overlayBundleIdentifier = "com.t3tools.t3code.agent-cursor" + +/// Client side: owns the overlay process and speaks to it. +final class AgentCursor { + static let shared = AgentCursor() + + private var connection: FileHandle? + private var listenerSource: DispatchSourceRead? + private var listenerFD: Int32 = -1 + private var socketPath: String? + private var pending: [[String: Any]] = [] + private var process: Process? + private let lock = NSLock() + /// Last Quartz point we told the overlay to visit — used to time clicks + /// so the real action waits for the spring animation to land. + private var lastPoint: CGPoint? + /// Bumped to cancel a pending post-task fade when another tools/call starts. + private var taskHideGeneration: UInt64 = 0 + private static var desktopToolDepth: Int = 0 + private var taskHideWork: DispatchWorkItem? + /// Bumped on each `ensureRunning` so a prior attempt's timeout cannot tear + /// down a later startup on the same socket path. + private var startupGeneration: UInt64 = 0 + + /// Show the agent pointer at a screen point, starting the overlay if needed. + /// + /// Failures are deliberately silent toward the tool caller: the overlay is + /// a courtesy, and a missing pointer must never turn a working click into a + /// failed tool call. Launch problems still go to stderr so they are + /// diagnosable without poisoning the MCP response. + /// + /// Blocks until the spring follow would have settled on `point`, so callers + /// that click afterward land in sync with the visible pointer. + func show(at point: CGPoint) { + guard agentCursorEnabled else { return } + moveAndWait(to: point, press: false) + } + + /// Move the agent pointer to a screen point, waiting for the animation. + func press(at point: CGPoint) { + guard agentCursorEnabled else { return } + moveAndWait(to: point, press: true) + } + + /// Non-blocking hop for mid-drag visuals (must not sleep while a button is down). + func glide(at point: CGPoint) { + guard agentCursorEnabled else { return } + moveNoWait(to: point) + } + + func hide() { + lock.lock() + defer { lock.unlock() } + taskHideGeneration += 1 + taskHideWork?.cancel() + taskHideWork = nil + lastPoint = nil + guard connection != nil || listenerFD >= 0 else { return } + sendLocked(["hide": true]) + } + + /// A Computer Use `tools/call` is starting — keep the pointer up. + func noteDesktopToolStarted() { + guard agentCursorEnabled else { return } + lock.lock() + defer { lock.unlock() } + let depth = Self.desktopToolDepth + Self.desktopToolDepth = depth + 1 + guard depth == 0 else { return } + taskHideGeneration += 1 + taskHideWork?.cancel() + taskHideWork = nil + } + + /// A Computer Use `tools/call` finished. If nothing else starts soon, the + /// task is done and the pointer should fade — not N seconds after the last + /// pixel move while the agent is still working. + func noteDesktopToolFinished() { + guard agentCursorEnabled else { return } + lock.lock() + defer { lock.unlock() } + guard Self.desktopToolDepth > 0 else { return } + Self.desktopToolDepth -= 1 + guard Self.desktopToolDepth == 0 else { return } + // Only schedule if the pointer was actually used for this task. + guard lastPoint != nil else { return } + taskHideGeneration += 1 + let generation = taskHideGeneration + taskHideWork?.cancel() + let work = DispatchWorkItem { [weak self] in + guard let self else { return } + self.lock.lock() + let shouldHide = self.taskHideGeneration == generation + self.lock.unlock() + if shouldHide { self.hide() } + } + taskHideWork = work + let delay = Self.taskFadeGraceSeconds() + DispatchQueue.global(qos: .utility).asyncAfter(deadline: .now() + delay, execute: work) + } + + /// Brief grace so a follow-up tool in the same turn cancels before fade. + /// Override with `T3_DESKTOP_AGENT_CURSOR_TASK_FADE_SECS`. + private static func taskFadeGraceSeconds() -> TimeInterval { + if let raw = ProcessInfo.processInfo.environment["T3_DESKTOP_AGENT_CURSOR_TASK_FADE_SECS"], + let value = Double(raw.trimmingCharacters(in: .whitespacesAndNewlines)), + value.isFinite, value >= 0, value < 3600 + { + return value + } + // Long enough to absorb normal model latency between chained desktop + // tools; short enough that the pointer does not linger after the turn. + return 8.0 + } + + private func moveAndWait(to point: CGPoint, press: Bool) { + guard Self.isRepresentableScreenPoint(point) else { return } + let wait: useconds_t + var needsStartupSlack = false + lock.lock() + wait = travelWaitMicros(to: point) + ensureRunning() + needsStartupSlack = connection == nil + // If the overlay could not start, drop the event instead of queuing + // forever and growing `pending` for the lifetime of the MCP server. + if connection != nil || listenerFD >= 0 { + var message: [String: Any] = ["x": Int(point.x), "y": Int(point.y)] + if press { message["press"] = true } + sendLocked(message) + lastPoint = point + } + lock.unlock() + + var total = wait + if needsStartupSlack { total += 220_000 } + if total > 0 { usleep(total) } + } + + private func moveNoWait(to point: CGPoint) { + guard Self.isRepresentableScreenPoint(point) else { return } + lock.lock() + ensureRunning() + if connection != nil || listenerFD >= 0 { + sendLocked(["x": Int(point.x), "y": Int(point.y)]) + lastPoint = point + } + lock.unlock() + } + + /// Overlay messages use `Int` coordinates — reject non-finite / out-of-range + /// values so `Int(point.x)` cannot trap the MCP process. + private static func isRepresentableScreenPoint(_ point: CGPoint) -> Bool { + let x = Double(point.x) + let y = Double(point.y) + guard x.isFinite, y.isFinite else { return false } + // `Double(Int.max)` is not exact, so an inclusive `<= Double(Int.max)` + // bound can still accept values that trap on `Int(...)`. Require the + // truncated coordinate to round-trip through `Int(exactly:)`. + let ix = x.rounded(.towardZero) + let iy = y.rounded(.towardZero) + return Int(exactly: ix) != nil && Int(exactly: iy) != nil + } + + /// Approximate flight time matching OverlayController's cubic path. + private func travelWaitMicros(to point: CGPoint) -> useconds_t { + guard let from = lastPoint else { + return 100_000 + } + let dist = hypot(point.x - from.x, point.y - from.y) + if dist < 2 { return 60_000 } + // Same duration formula as the overlay flight. + let seconds = min(0.85, max(0.28, 0.20 + Double(dist) / 1100.0)) + return useconds_t((seconds + 0.04) * 1_000_000) + } + + private func ensureRunning() { + if connection != nil { return } + if listenerFD >= 0 { return } + + guard let appURL = OverlayBundle.ensureApp() else { + fputs("t3-desktop-mcp: agent cursor: could not materialise T3AgentCursor.app\n", stderr) + pending.removeAll() + return + } + + // sockaddr_un.sun_path is only 104 bytes on macOS; NSTemporaryDirectory() + // under /var/folders/... plus a UUID blows past that and bind() fails, + // which is why the overlay never started from the MCP server. + let path = "/tmp/t3ac-\(getpid()).sock" + startupGeneration &+= 1 + let generation = startupGeneration + guard startListening(at: path) else { + fputs("t3-desktop-mcp: agent cursor: could not listen on \(path)\n", stderr) + pending.removeAll() + return + } + socketPath = path + + let executable = appURL + .appendingPathComponent("Contents", isDirectory: true) + .appendingPathComponent("MacOS", isDirectory: true) + .appendingPathComponent(overlayExecutableName) + + // Fresh copies need an LS registration before openApplication will + // resolve the bundle; without this the completion returns an error and + // the pointer never appears after a rebuild. + LSRegisterURL(appURL as CFURL, true) + + let configuration = NSWorkspace.OpenConfiguration() + configuration.arguments = ["cursor-overlay", "--socket", path] + configuration.activates = false + configuration.addsToRecentItems = false + configuration.createsNewApplicationInstance = true + + NSWorkspace.shared.openApplication(at: appURL, configuration: configuration) { [weak self] _, error in + guard let self else { return } + if let error { + fputs( + "t3-desktop-mcp: agent cursor: NSWorkspace open failed (\(error.localizedDescription)); falling back to Process\n", + stderr + ) + self.lock.lock() + self.launchViaProcess(executable: executable, socketPath: path) + self.lock.unlock() + } + } + + // If NSWorkspace is slow or silent, arm a Process fallback shortly. + DispatchQueue.global(qos: .userInitiated).asyncAfter(deadline: .now() + 0.8) { [weak self] in + guard let self else { return } + self.lock.lock() + defer { self.lock.unlock() } + if self.connection == nil, self.process?.isRunning != true, + self.socketPath == path, self.startupGeneration == generation + { + fputs("t3-desktop-mcp: agent cursor: NSWorkspace timed out; falling back to Process\n", stderr) + self.launchViaProcess(executable: executable, socketPath: path) + } + } + + // If nothing connects, tear down the listener so later show/press retries + // startup instead of queuing forever into a dead socket. + DispatchQueue.global(qos: .utility).asyncAfter(deadline: .now() + 5.0) { [weak self] in + guard let self else { return } + self.lock.lock() + defer { self.lock.unlock() } + if self.connection == nil, self.socketPath == path, self.startupGeneration == generation { + fputs("t3-desktop-mcp: agent cursor: overlay never connected; resetting\n", stderr) + self.tearDownLocked() + } + } + } + + private func launchViaProcess(executable: URL, socketPath: String) { + if process?.isRunning == true { return } + if connection != nil { return } + let child = Process() + child.executableURL = executable + child.arguments = ["cursor-overlay", "--socket", socketPath] + child.standardInput = FileHandle.nullDevice + child.standardOutput = FileHandle.nullDevice + child.standardError = FileHandle.nullDevice + do { + try child.run() + process = child + } catch { + fputs("t3-desktop-mcp: agent cursor: Process launch failed (\(error.localizedDescription))\n", stderr) + tearDownLocked() + } + } + + private func sendLocked(_ message: [String: Any]) { + guard let data = try? JSONSerialization.data(withJSONObject: message) else { return } + var line = data + line.append(0x0A) + if let connection { + // The overlay may have been killed by the user; a broken pipe raises + // here, which we swallow and retry on the next call. + do { + try connection.write(contentsOf: line) + } catch { + tearDownLocked() + } + return + } + pending.append(message) + } + + private func startListening(at path: String) -> Bool { + unlink(path) + let fd = socket(AF_UNIX, SOCK_STREAM, 0) + guard fd >= 0 else { return false } + + var address = sockaddr_un() + address.sun_family = sa_family_t(AF_UNIX) + let pathBytes = path.utf8CString + guard pathBytes.count <= MemoryLayout.size(ofValue: address.sun_path) else { + close(fd) + return false + } + withUnsafeMutablePointer(to: &address.sun_path) { ptr in + ptr.withMemoryRebound(to: CChar.self, capacity: pathBytes.count) { dest in + for (index, byte) in pathBytes.enumerated() { + dest[index] = byte + } + } + } + + let bindResult = withUnsafePointer(to: &address) { ptr in + ptr.withMemoryRebound(to: sockaddr.self, capacity: 1) { sockPtr in + bind(fd, sockPtr, socklen_t(MemoryLayout.size)) + } + } + guard bindResult == 0, listen(fd, 1) == 0 else { + close(fd) + unlink(path) + return false + } + // Owner-only: any local process can otherwise connect and permanently + // become `connection`, blocking the real overlay. + _ = chmod(path, 0o600) + + let source = DispatchSource.makeReadSource(fileDescriptor: fd, queue: .main) + source.setEventHandler { [weak self] in + self?.acceptConnection() + } + source.setCancelHandler { + close(fd) + } + source.resume() + listenerFD = fd + listenerSource = source + return true + } + + private func acceptConnection() { + lock.lock() + defer { lock.unlock() } + guard listenerFD >= 0 else { return } + let client = accept(listenerFD, nil, nil) + guard client >= 0 else { return } + enableNoSigPipe(client) + + var peerUid: uid_t = 0 + var peerGid: gid_t = 0 + if getpeereid(client, &peerUid, &peerGid) != 0 || peerUid != getuid() { + close(client) + return + } + + listenerSource?.cancel() + listenerSource = nil + listenerFD = -1 + if let socketPath { + unlink(socketPath) + self.socketPath = nil + } + + let handle = FileHandle(fileDescriptor: client, closeOnDealloc: true) + connection = handle + let queued = pending + pending.removeAll() + for message in queued { + sendLocked(message) + } + } + + private func tearDownLocked() { + // Cancel handler owns closing the listener FD — do not double-close. + if let source = listenerSource { + listenerSource = nil + listenerFD = -1 + source.cancel() + } else if listenerFD >= 0 { + close(listenerFD) + listenerFD = -1 + } + if let socketPath { + unlink(socketPath) + self.socketPath = nil + } + try? connection?.close() + connection = nil + if let process, process.isRunning { + process.terminate() + } + process = nil + pending.removeAll() + } +} + +/// Builds or locates the overlay `.app` next to the MCP binary (or under +/// Application Support for a bare SwiftPM build). +private enum OverlayBundle { + static func ensureApp() -> URL? { + let fm = FileManager.default + let selfURL = URL(fileURLWithPath: CommandLine.arguments[0]).resolvingSymlinksInPath() + + // Staged artifact: `…/t3-desktop-mcp/T3AgentCursor.app` beside the binary. + let sibling = selfURL.deletingLastPathComponent().appendingPathComponent(overlayAppName) + if isValidApp(sibling) { + do { + try refreshExecutable(in: sibling, from: selfURL) + return sibling + } catch { + return isValidApp(sibling) ? sibling : nil + } + } + + // Dev / unsigned: materialise under Application Support so Launch Services + // sees a stable path across rebuilds. + guard + let support = fm.urls(for: .applicationSupportDirectory, in: .userDomainMask).first + else { return nil } + let dir = support.appendingPathComponent("t3-desktop-mcp", isDirectory: true) + let appURL = dir.appendingPathComponent(overlayAppName, isDirectory: true) + do { + try fm.createDirectory(at: dir, withIntermediateDirectories: true) + try materialize(at: appURL, executable: selfURL) + return appURL + } catch { + return nil + } + } + + private static func isValidApp(_ appURL: URL) -> Bool { + let fm = FileManager.default + let exe = appURL + .appendingPathComponent("Contents", isDirectory: true) + .appendingPathComponent("MacOS", isDirectory: true) + .appendingPathComponent(overlayExecutableName) + let plist = appURL + .appendingPathComponent("Contents", isDirectory: true) + .appendingPathComponent("Info.plist") + return fm.fileExists(atPath: exe.path) && fm.fileExists(atPath: plist.path) + } + + private static func materialize(at appURL: URL, executable: URL) throws { + let fm = FileManager.default + let contents = appURL.appendingPathComponent("Contents", isDirectory: true) + let macOS = contents.appendingPathComponent("MacOS", isDirectory: true) + try fm.createDirectory(at: macOS, withIntermediateDirectories: true) + + let plistURL = contents.appendingPathComponent("Info.plist") + if !fm.fileExists(atPath: plistURL.path) { + try overlayInfoPlist().write(to: plistURL, atomically: true, encoding: .utf8) + } + + try refreshExecutable(in: appURL, from: executable) + } + + /// Keep the bundled binary in sync with the running MCP server so a rebuild + /// is picked up without a manual wipe of Application Support. + private static func refreshExecutable(in appURL: URL, from executable: URL) throws { + let fm = FileManager.default + let dest = appURL + .appendingPathComponent("Contents", isDirectory: true) + .appendingPathComponent("MacOS", isDirectory: true) + .appendingPathComponent(overlayExecutableName) + // Skip the copy when we *are* the bundled binary (overlay relaunching). + if executable.resolvingSymlinksInPath() == dest.resolvingSymlinksInPath() { return } + + let needsCopy: Bool + if !fm.fileExists(atPath: dest.path) { + needsCopy = true + } else { + // Compare size + contents — equal mtimes after a rebuild must not + // leave a stale overlay binary in place. + let srcData = try Data(contentsOf: executable) + let dstData = (try? Data(contentsOf: dest)) ?? Data() + needsCopy = srcData != dstData + } + guard needsCopy else { return } + try fm.createDirectory(at: dest.deletingLastPathComponent(), withIntermediateDirectories: true) + // Unique temp name so concurrent MCP sessions cannot clobber each other. + let temp = dest.deletingLastPathComponent() + .appendingPathComponent(".\(overlayExecutableName).\(getpid()).\(UUID().uuidString).new") + defer { try? fm.removeItem(at: temp) } + try fm.copyItem(at: executable, to: temp) + try fm.setAttributes([.posixPermissions: 0o755], ofItemAtPath: temp.path) + // Atomic replace: overwrite dest in place via replaceItem when possible. + if fm.fileExists(atPath: dest.path) { + _ = try fm.replaceItemAt(dest, withItemAt: temp) + } else { + try fm.moveItem(at: temp, to: dest) + } + } + + private static func overlayInfoPlist() -> String { + """ + + + + + CFBundleDevelopmentRegion + en + CFBundleExecutable + \(overlayExecutableName) + CFBundleIdentifier + \(overlayBundleIdentifier) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + T3 Agent Cursor + CFBundlePackageType + APPL + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1 + LSMinimumSystemVersion + 14.0 + LSUIElement + + NSHighResolutionCapable + + NSPrincipalClass + NSApplication + + + """ + } +} + +/// The overlay process itself. +enum AgentCursorOverlay { + static func run(socketPath: String) -> Never { + let application = NSApplication.shared + // .accessory keeps it out of the Dock and stops it stealing focus. + // LSUIElement in Info.plist does the same for Launch Services. + application.setActivationPolicy(.accessory) + let controller = OverlayController(socketPath: socketPath) + application.delegate = controller + // Build the window here rather than waiting for + // applicationDidFinishLaunching: even inside a bundle the callback can + // race the first move command, and an empty window list meant the + // pointer never appeared for that click. + controller.makeWindow() + controller.listen() + // NSApplication.delegate is weak; keep the controller alive for the run loop. + withExtendedLifetime(controller) { + application.run() + } + exit(0) + } +} + +private final class OverlayController: NSObject, NSApplicationDelegate { + private let socketPath: String + private var panel: NSPanel? + private var view: BubbleView? + private var socketHandle: FileHandle? + private var socketBuffer = Data() + private var animation: Timer? + + /// Generous panel so the glow, squash and travel lean have room. + private let side: CGFloat = 112 + /// Distance from the panel's top-left corner to the cursor's hot point. + fileprivate static let hotspot: CGFloat = 56 + + /// Plane-style cubic flight in Quartz screen coordinates. + /// Tip follows path tangent the whole way; path flares upright into the + /// target so reorientation happens on approach — not after landing. + private var current: CGPoint? + private var target: CGPoint = .zero + private var velocity: CGVector = .zero + private var pathFrom: CGPoint = .zero + private var pathC1: CGPoint = .zero + private var pathC2: CGPoint = .zero + private var pathTo: CGPoint = .zero + private var pathElapsed: CFTimeInterval = 0 + private var pathDuration: CFTimeInterval = 0 + private var pathActive = false + private var arcSign: CGFloat = 1 + private var lastTickAt: CFTimeInterval? + /// Bumped on each fadeOut / begin so a stale fade completion cannot orderOut + /// a pointer that already reappeared. + private var fadeGeneration: UInt64 = 0 + + init(socketPath: String) { + self.socketPath = socketPath + super.init() + } + + func applicationDidFinishLaunching(_ notification: Notification) { + makeWindow() + } + + func makeWindow() { + guard panel == nil else { return } + let panel = NSPanel( + contentRect: NSRect(x: 0, y: 0, width: side, height: side), + styleMask: [.borderless, .nonactivatingPanel], + backing: .buffered, + defer: false + ) + panel.isOpaque = false + panel.backgroundColor = .clear + panel.hasShadow = false + panel.ignoresMouseEvents = true + // Above ordinary windows and full-screen apps, but still below system + // alerts so it can never hide something the user must answer. + panel.level = .screenSaver + panel.collectionBehavior = [.canJoinAllSpaces, .stationary, .fullScreenAuxiliary, .ignoresCycle] + let view = BubbleView(frame: NSRect(x: 0, y: 0, width: side, height: side)) + panel.contentView = view + panel.alphaValue = 0 + self.panel = panel + self.view = view + } + + /// Connect to the server's socket and read move/hide commands without + /// blocking the run loop. + func listen() { + let fd = socket(AF_UNIX, SOCK_STREAM, 0) + guard fd >= 0 else { + NSApplication.shared.terminate(nil) + return + } + + var address = sockaddr_un() + address.sun_family = sa_family_t(AF_UNIX) + let pathBytes = socketPath.utf8CString + guard pathBytes.count <= MemoryLayout.size(ofValue: address.sun_path) else { + close(fd) + NSApplication.shared.terminate(nil) + return + } + withUnsafeMutablePointer(to: &address.sun_path) { ptr in + ptr.withMemoryRebound(to: CChar.self, capacity: pathBytes.count) { dest in + for (index, byte) in pathBytes.enumerated() { + dest[index] = byte + } + } + } + + // The parent listens before openApplication returns; retry briefly in + // case Launch Services schedules us first. + var connected = false + for _ in 0..<50 { + let result = withUnsafePointer(to: &address) { ptr in + ptr.withMemoryRebound(to: sockaddr.self, capacity: 1) { sockPtr in + connect(fd, sockPtr, socklen_t(MemoryLayout.size)) + } + } + if result == 0 { + connected = true + break + } + usleep(20_000) + } + guard connected else { + close(fd) + NSApplication.shared.terminate(nil) + return + } + + let handle = FileHandle(fileDescriptor: fd, closeOnDealloc: true) + socketHandle = handle + handle.readabilityHandler = { [weak self] handle in + guard let self else { return } + let data = handle.availableData + if data.isEmpty { + // The server exited; take the pointer with it. + DispatchQueue.main.async { NSApplication.shared.terminate(nil) } + return + } + self.socketBuffer.append(data) + while let newline = self.socketBuffer.firstIndex(of: 0x0A) { + let line = self.socketBuffer[self.socketBuffer.startIndex.. 0.08 { + let ang = -view.tilt + startDir = CGVector(dx: sin(ang), dy: -cos(ang)) + } else { + startDir = CGVector(dx: dx / dist, dy: dy / dist) + } + + pathFrom = from + pathTo = point + let depart = min(handle, dist * 0.28) + pathC1 = CGPoint( + x: from.x + startDir.dx * depart + nx * min(36, dist * 0.10) * arcSign, + y: from.y + startDir.dy * depart + ny * min(36, dist * 0.10) * arcSign + ) + // Approach from "below" (Quartz Y-down) so final tangent is screen-up + // → tip already upright as it arrives. + let approach = min(handle * 0.85, max(20, dist * 0.16)) + pathC2 = CGPoint(x: point.x, y: point.y + approach) + + pathDuration = min(0.85, max(0.28, 0.20 + Double(dist) / 1100.0)) + pathElapsed = 0 + pathActive = true + velocity = .zero + lastTickAt = nil + startAnimating() + } + + private func ensurePanel() -> NSPanel { + if let panel { return panel } + makeWindow() + return panel! + } + + private func startAnimating() { + guard animation == nil else { return } + let timer = Timer(timeInterval: 1.0 / 60.0, repeats: true) { [weak self] _ in self?.tick() } + RunLoop.main.add(timer, forMode: .common) + animation = timer + } + + private func tick() { + guard let view else { return } + var busy = false + + let now = CACurrentMediaTime() + let dt = min(1.0 / 30.0, max(1.0 / 120.0, lastTickAt.map { now - $0 } ?? (1.0 / 60.0))) + lastTickAt = now + + if pathActive, var cur = current { + pathElapsed += dt + let u = min(1.0, pathElapsed / max(0.001, pathDuration)) + // Ease-in-out along the flight path. + let t = u * u * (3 - 2 * u) + let pos = Self.cubicBezier(pathFrom, pathC1, pathC2, pathTo, CGFloat(t)) + let tan = Self.cubicBezierTangent(pathFrom, pathC1, pathC2, pathTo, CGFloat(t)) + velocity = CGVector( + dx: (pos.x - cur.x) / CGFloat(dt), + dy: (pos.y - cur.y) / CGFloat(dt) + ) + cur = pos + current = cur + view.velocity = velocity + + // Tip tracks path tangent continuously — the turn into upright is + // the last part of the curve, not a settle spin after arrival. + let tanLen = hypot(tan.dx, tan.dy) + if tanLen > 0.001 { + let desired = -atan2(tan.dx, -tan.dy) + var delta = desired - view.tilt + while delta > .pi { delta -= 2 * .pi } + while delta < -.pi { delta += 2 * .pi } + // Slight lag early; tighten on final flare so tip matches path. + let follow = min(1, 0.16 + CGFloat(t) * 0.55 + CGFloat(dt) * 7) + view.tilt += delta * follow + } + + if u >= 1 { + current = pathTo + velocity = .zero + view.velocity = .zero + view.tilt = 0 + pathActive = false + } + busy = true + place(current ?? pathTo) + } + + if panel?.isVisible == true, (panel?.alphaValue ?? 0) > 0.05 { + view.phase += 0.08 + busy = true + } + view.needsDisplay = true + + if !busy { + animation?.invalidate() + animation = nil + lastTickAt = nil + } + } + + private static func cubicBezier( + _ p0: CGPoint, _ p1: CGPoint, _ p2: CGPoint, _ p3: CGPoint, _ t: CGFloat + ) -> CGPoint { + let o = 1 - t + let o2 = o * o + let t2 = t * t + return CGPoint( + x: o2 * o * p0.x + 3 * o2 * t * p1.x + 3 * o * t2 * p2.x + t2 * t * p3.x, + y: o2 * o * p0.y + 3 * o2 * t * p1.y + 3 * o * t2 * p2.y + t2 * t * p3.y + ) + } + + private static func cubicBezierTangent( + _ p0: CGPoint, _ p1: CGPoint, _ p2: CGPoint, _ p3: CGPoint, _ t: CGFloat + ) -> CGVector { + let o = 1 - t + return CGVector( + dx: 3 * o * o * (p1.x - p0.x) + 6 * o * t * (p2.x - p1.x) + 3 * t * t * (p3.x - p2.x), + dy: 3 * o * o * (p1.y - p0.y) + 6 * o * t * (p2.y - p1.y) + 3 * t * t * (p3.y - p2.y) + ) + } + + private func place(_ point: CGPoint) { + guard let panel else { return } + let primary = + NSScreen.screens.first(where: { $0.frame.origin == .zero }) + ?? NSScreen.main + ?? NSScreen.screens.first + guard let primary else { return } + let flippedY = primary.frame.maxY - point.y + panel.setFrameOrigin(NSPoint( + x: point.x - OverlayController.hotspot, + y: flippedY - side + OverlayController.hotspot + )) + panel.orderFrontRegardless() + } + + private func fadeOut() { + guard let panel, panel.isVisible else { return } + pathActive = false + fadeGeneration &+= 1 + let generation = fadeGeneration + NSAnimationContext.runAnimationGroup({ ctx in + ctx.duration = 0.35 + panel.animator().alphaValue = 0 + }, completionHandler: { [weak self] in + guard let self else { return } + // Ignore completions from a fade that was superseded by a new move. + guard generation == self.fadeGeneration else { return } + if panel.alphaValue < 0.05 { + panel.orderOut(nil) + self.animation?.invalidate() + self.animation = nil + } + }) + } +} + +/// Soft translucent bubble: lavender glow, rounded arrow, path heading, +/// idle breathe. No click ring. +private final class BubbleView: NSView { + var phase: CGFloat = 0 + /// Unused for drawing now (no squash); kept so motion code can still assign it. + var velocity: CGVector = .zero + /// Path heading in radians (2D spin only); upright (0) when landed. + var tilt: CGFloat = 0 + + override func draw(_ dirtyRect: NSRect) { + guard let ctx = NSGraphicsContext.current?.cgContext else { return } + let tip = CGPoint(x: OverlayController.hotspot, y: bounds.maxY - OverlayController.hotspot) + + let lavender = NSColor(calibratedRed: 0.76, green: 0.72, blue: 0.99, alpha: 1) + let purple = NSColor(calibratedRed: 0.58, green: 0.52, blue: 0.94, alpha: 1) + let breathe = 1 + 0.03 * sin(phase) + + if let wash = CGGradient( + colorsSpace: CGColorSpaceCreateDeviceRGB(), + colors: [ + lavender.withAlphaComponent(0.72).cgColor, + lavender.withAlphaComponent(0.38).cgColor, + purple.withAlphaComponent(0.14).cgColor, + purple.withAlphaComponent(0).cgColor, + ] as CFArray, + locations: [0, 0.30, 0.65, 1] + ) { + let glowR: CGFloat = 34 * breathe + let center = CGPoint(x: tip.x + 6, y: tip.y - 9) + ctx.drawRadialGradient( + wash, + startCenter: center, startRadius: 0, + endCenter: center, endRadius: glowR, + options: [] + ) + } + + ctx.saveGState() + ctx.translateBy(x: tip.x, y: tip.y) + // Pure 2D: rotate in the plane only — never squash/stretch (reads as 3D). + ctx.rotate(by: tilt) + + let corners = [ + NSPoint(x: 0, y: 0), + NSPoint(x: 24, y: -11), + NSPoint(x: 14.5, y: -16.5), + NSPoint(x: 7, y: -28), + ] + let radius: CGFloat = 2.6 + let arrow = NSBezierPath() + func midpoint(_ a: NSPoint, _ b: NSPoint) -> NSPoint { + NSPoint(x: (a.x + b.x) / 2, y: (a.y + b.y) / 2) + } + arrow.move(to: midpoint(corners[corners.count - 1], corners[0])) + for i in 0.. 100 { + return shortFallback + } + return candidate +}() + +/// Reply from the extension. A custom type rather than `Result` because the +/// failure carries a human-readable message, not an `Error`. +enum BridgeOutcome { + case success([String: Any]) + case failure(String) +} + +// MARK: - Length-prefixed framing (Chrome side) + +enum NativeMessaging { + /// Chrome native messaging rejects messages larger than 1 MiB. + static let maxPayloadBytes = 1_048_576 + + /// Read exactly `count` bytes, treating an empty read as EOF and a short + /// non-empty read as a fragment to keep accumulating. + private static func readExact(_ handle: FileHandle, count: Int) -> Data? { + var data = Data() + data.reserveCapacity(count) + while data.count < count { + let needed = count - data.count + guard let chunk = try? handle.read(upToCount: needed) else { return nil } + if chunk.isEmpty { + return nil + } + data.append(chunk) + } + return data + } + + /// Read one message: 4-byte little-endian length, then that many UTF-8 bytes. + static func read(_ handle: FileHandle) -> Data? { + guard let header = readExact(handle, count: 4) else { return nil } + var lengthLE: UInt32 = 0 + _ = withUnsafeMutableBytes(of: &lengthLE) { dest in + header.copyBytes(to: dest, count: 4) + } + let length = UInt32(littleEndian: lengthLE) + guard length > 0, length < 64 * 1024 * 1024 else { return nil } + return readExact(handle, count: Int(length)) + } + + static func write(_ handle: FileHandle, _ payload: Data) { + guard payload.count <= maxPayloadBytes else { + fputs( + "t3-desktop-mcp: native messaging payload exceeds \(maxPayloadBytes) bytes\n", + stderr + ) + return + } + var lengthLE = UInt32(payload.count).littleEndian + var framed = Data() + withUnsafeBytes(of: &lengthLE) { framed.append(contentsOf: $0) } + framed.append(payload) + try? handle.write(contentsOf: framed) + } +} + +/// Write every byte, retrying EINTR and failing on other errors / short EOF. +/// Fully blocking — safe for `NativeHost`, which shares this socket with a +/// concurrent reader that must not see `O_NONBLOCK` / `EAGAIN`. +func writeAll(_ fd: Int32, _ data: Data) -> Bool { + data.withUnsafeBytes { rawBuffer -> Bool in + guard var ptr = rawBuffer.bindMemory(to: UInt8.self).baseAddress else { + return data.isEmpty + } + var remaining = data.count + while remaining > 0 { + let n = Darwin.write(fd, ptr, remaining) + if n < 0 { + if errno == EINTR { continue } + return false + } + if n == 0 { return false } + ptr += n + remaining -= n + } + return true + } +} + +/// Deadline-bounded write for MCP `call` that must not hang forever. +/// +/// Uses `send(..., MSG_DONTWAIT)` so the socket's blocking mode is unchanged for +/// the concurrent NativeHost reader. Retries on `EINTR` / `EAGAIN` via `poll`. +func writeAll(_ fd: Int32, _ data: Data, deadline: DispatchTime) -> Bool { + data.withUnsafeBytes { rawBuffer -> Bool in + guard var ptr = rawBuffer.bindMemory(to: UInt8.self).baseAddress else { + return data.isEmpty + } + var remaining = data.count + while remaining > 0 { + if DispatchTime.now() >= deadline { + return false + } + let n = Darwin.send(fd, ptr, remaining, Int32(MSG_DONTWAIT)) + if n < 0 { + if errno == EINTR { continue } + if errno == EAGAIN || errno == EWOULDBLOCK { + var pollFd = pollfd(fd: fd, events: Int16(POLLOUT), revents: 0) + let now = DispatchTime.now().uptimeNanoseconds + let end = deadline.uptimeNanoseconds + if end <= now { return false } + let waitMs = Int32(min((end - now) / 1_000_000, UInt64(Int32.max))) + let ready = poll(&pollFd, 1, waitMs) + if ready < 0 { + if errno == EINTR { continue } + return false + } + if ready == 0 { return false } + continue + } + return false + } + if n == 0 { return false } + ptr += n + remaining -= n + } + return true + } +} + +func enableNoSigPipe(_ fd: Int32) { + var on: Int32 = 1 + _ = setsockopt(fd, SOL_SOCKET, SO_NOSIGPIPE, &on, socklen_t(MemoryLayout.size)) +} + +/// Fill in a `sockaddr_un` for the bridge path. +func bridgeAddress() -> sockaddr_un { + var addr = sockaddr_un() + addr.sun_family = sa_family_t(AF_UNIX) + _ = withUnsafeMutablePointer(to: &addr.sun_path) { pathPtr in + bridgeSocketPath.withCString { src in + strncpy(UnsafeMutableRawPointer(pathPtr).assumingMemoryBound(to: CChar.self), src, 103) + } + } + return addr +} + +/// Whether a server is already listening on the bridge socket. +/// +/// The socket file outlives the process that made it, so its presence proves +/// nothing — only a successful connect distinguishes a live owner from a stale +/// file left behind by a crash. +enum BridgeSocketProbe { + case live + case stale + case unknown +} + +func probeBridgeSocket() -> BridgeSocketProbe { + guard FileManager.default.fileExists(atPath: bridgeSocketPath) else { return .stale } + let fd = socket(AF_UNIX, SOCK_STREAM, 0) + guard fd >= 0 else { return .unknown } + defer { close(fd) } + var addr = bridgeAddress() + let size = socklen_t(MemoryLayout.size) + let result = withUnsafePointer(to: &addr) { + $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { connect(fd, $0, size) } + } + if result == 0 { return .live } + switch errno { + case ENOENT, ECONNREFUSED: + return .stale + default: + return .unknown + } +} + +func bridgeSocketIsLive() -> Bool { + probeBridgeSocket() == .live +} + +// MARK: - Host mode + +/// `t3-desktop-mcp native-host` — relays between Chrome's stdio and the socket. +/// Chrome launches this; it is not the MCP server. +enum NativeHost { + static func run() -> Never { + let input = FileHandle.standardInput + let output = FileHandle.standardOutput + + let fd = socket(AF_UNIX, SOCK_STREAM, 0) + guard fd >= 0 else { exit(1) } + enableNoSigPipe(fd) + var addr = bridgeAddress() + let size = socklen_t(MemoryLayout.size) + let connected = withUnsafePointer(to: &addr) { + $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { connect(fd, $0, size) } + } + guard connected == 0 else { exit(1) } + + // Socket → Chrome. Server speaks newline-delimited JSON. + DispatchQueue.global().async { + var buffer = Data() + var chunk = [UInt8](repeating: 0, count: 65536) + while true { + let n = Darwin.read(fd, &chunk, chunk.count) + if n <= 0 { exit(0) } + buffer.append(contentsOf: chunk[0.. Void] = [:] + /// Serializes newline-delimited JSON writes so concurrent `call`s cannot interleave. + private let writeLock = NSLock() + + var isConnected: Bool { + lock.lock(); defer { lock.unlock() } + return clientFD >= 0 + } + + private var ownershipLockPath: String { bridgeSocketPath + ".lock" } + + /// Bind the socket and accept the host connection. Silently does nothing if + /// another server already owns it. + func start() { + // Cross-process exclusive lock closes the live-check / unlink / bind + // race where two servers could both think they own the bridge. + let lockFd = open(ownershipLockPath, O_CREAT | O_RDWR, 0o600) + guard lockFd >= 0 else { return } + if flock(lockFd, LOCK_EX | LOCK_NB) != 0 { + close(lockFd) + return + } + + switch probeBridgeSocket() { + case .live: + flock(lockFd, LOCK_UN) + close(lockFd) + return + case .unknown: + flock(lockFd, LOCK_UN) + close(lockFd) + return + case .stale: + unlink(bridgeSocketPath) + } + let fd = socket(AF_UNIX, SOCK_STREAM, 0) + guard fd >= 0 else { + flock(lockFd, LOCK_UN) + close(lockFd) + return + } + var addr = bridgeAddress() + let size = socklen_t(MemoryLayout.size) + let bound = withUnsafePointer(to: &addr) { + $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { bind(fd, $0, size) } + } + // Backlog of several: Chrome relaunches the host on every extension + // reload, and a full queue makes the next connect fail outright. + guard bound == 0, listen(fd, 8) == 0 else { + close(fd) + flock(lockFd, LOCK_UN) + close(lockFd) + return + } + listenFD = fd + ownershipLockFD = lockFd + + DispatchQueue.global(qos: .utility).async { [weak self] in + while true { + let client = accept(fd, nil, nil) + if client < 0 { + // A transient error must not retire the listener for good — + // that silently strands every later host launch. + if errno == EINTR || errno == ECONNABORTED { continue } + return + } + enableNoSigPipe(client) + self?.serve(client) + } + } + } + + private func serve(_ fd: Int32) { + lock.lock() + clientFD = fd + connectionGeneration += 1 + lock.unlock() + var buffer = Data() + var chunk = [UInt8](repeating: 0, count: 65536) + while true { + let n = Darwin.read(fd, &chunk, chunk.count) + if n <= 0 { break } + buffer.append(contentsOf: chunk[0.. BridgeOutcome + { + let semaphore = DispatchSemaphore(value: 0) + var outcome: BridgeOutcome = .failure("timed out") + + lock.lock() + guard clientFD >= 0 else { + lock.unlock() + return .failure("the T3 Code Chrome extension is not connected") + } + nextID += 1 + let id = nextID + let fd = clientFD + let generation = connectionGeneration + let payload: [String: Any] = ["id": id, "command": command, "params": params] + guard var data = try? JSONSerialization.data(withJSONObject: payload) else { + lock.unlock() + return .failure("could not encode the command") + } + data.append(0x0A) + // Register before unlocking so a disconnect that races the write still + // drains this waiter with a disconnect failure instead of a timeout. + pending[id] = { result in + outcome = result + semaphore.signal() + } + // Release `lock` before writing so `serve` can still drain replies or a + // disconnect while the write waits on a full socket buffer. + lock.unlock() + + let writeDeadline = DispatchTime.now() + timeout + writeLock.lock() + lock.lock() + guard clientFD == fd && connectionGeneration == generation else { + pending.removeValue(forKey: id) + lock.unlock() + writeLock.unlock() + return .failure("the browser extension disconnected") + } + lock.unlock() + let wrote = writeAll(fd, data, deadline: writeDeadline) + writeLock.unlock() + + if !wrote { + // A partial write corrupts newline framing for every later request — + // shut the socket down so `serve` tears down and the host reconnects. + lock.lock() + if clientFD == fd && connectionGeneration == generation { + pending.removeValue(forKey: id) + let stranded = pending + pending.removeAll() + clientFD = -1 + lock.unlock() + _ = Darwin.shutdown(fd, SHUT_RDWR) + for (_, resume) in stranded { + resume(.failure("the browser extension disconnected")) + } + } else { + pending.removeValue(forKey: id) + lock.unlock() + } + return .failure("the browser extension disconnected") + } + + if semaphore.wait(timeout: writeDeadline) == .timedOut { + lock.lock(); pending.removeValue(forKey: id); lock.unlock() + return .failure("the extension did not respond in \(Int(timeout))s") + } + return outcome + } +} diff --git a/native/t3-desktop-mcp/Sources/main.swift b/native/t3-desktop-mcp/Sources/main.swift new file mode 100644 index 00000000000..3bb82f1edbe --- /dev/null +++ b/native/t3-desktop-mcp/Sources/main.swift @@ -0,0 +1,2396 @@ +import AppKit +import ApplicationServices +import CoreGraphics +import Foundation +import ScreenCaptureKit + +// t3-desktop-mcp — a macOS computer-use MCP server built on the Accessibility API. +// +// Design notes: +// * Speaks newline-delimited JSON-RPC over stdio (MCP stdio transport). +// * Uses AXUIElement directly, never AppleScript/System Events. AppleScript would +// require a per-target-app kTCCServiceAppleEvents grant that macOS frequently +// refuses to prompt for; AX needs only Accessibility. +// * Ships as a bare executable so it runs as a child of the host app and inherits +// the host's TCC grants. A separate .app bundle would get its own TCC identity +// and require its own permissions. The agent-cursor overlay is the exception: +// it is a minimal LSUIElement .app (no Accessibility needed) launched via +// NSWorkspace — a bare Process child never gets a real window. + +// MARK: - AX helpers + +/// Host settings pass `T3_DESKTOP_AGENT_CURSOR=0` / `T3_DESKTOP_BROWSER=0` when +/// the matching Computer Use toggle is off. Missing or empty means enabled. +func envFlagDisabled(_ name: String) -> Bool { + guard let raw = ProcessInfo.processInfo.environment[name]? + .trimmingCharacters(in: .whitespacesAndNewlines) + .lowercased(), + !raw.isEmpty + else { + return false + } + return raw == "0" || raw == "false" || raw == "off" || raw == "no" +} + +var agentCursorEnabled: Bool { !envFlagDisabled("T3_DESKTOP_AGENT_CURSOR") } +var browserControlEnabled: Bool { !envFlagDisabled("T3_DESKTOP_BROWSER") } + +func axCopy(_ el: AXUIElement, _ attr: String) -> AnyObject? { + var value: AnyObject? + return AXUIElementCopyAttributeValue(el, attr as CFString, &value) == .success ? value : nil +} + +func axString(_ el: AXUIElement, _ attr: String) -> String? { + guard let v = axCopy(el, attr) else { return nil } + if let s = v as? String { return s.isEmpty ? nil : s } + if let n = v as? NSNumber { return n.stringValue } + return nil +} + +func axBool(_ el: AXUIElement, _ attr: String) -> Bool? { + (axCopy(el, attr) as? NSNumber)?.boolValue +} + +func axChildren(_ el: AXUIElement) -> [AXUIElement] { + (axCopy(el, kAXChildrenAttribute as String) as? [AXUIElement]) ?? [] +} + +func axActions(_ el: AXUIElement) -> [String] { + var names: CFArray? + guard AXUIElementCopyActionNames(el, &names) == .success else { return [] } + return (names as? [String]) ?? [] +} + +func axPoint(_ el: AXUIElement, _ attr: String) -> CGPoint? { + guard let v = axCopy(el, attr), CFGetTypeID(v) == AXValueGetTypeID() else { return nil } + var p = CGPoint.zero + return AXValueGetValue(v as! AXValue, .cgPoint, &p) ? p : nil +} + +func axSize(_ el: AXUIElement, _ attr: String) -> CGSize? { + guard let v = axCopy(el, attr), CFGetTypeID(v) == AXValueGetTypeID() else { return nil } + var s = CGSize.zero + return AXValueGetValue(v as! AXValue, .cgSize, &s) ? s : nil +} + +/// Read an attribute that should hold another element, checking the type first. +/// A blind `as!` here would crash on any app that returns something unexpected. +func axElement(_ el: AXUIElement, _ attr: String) -> AXUIElement? { + guard let v = axCopy(el, attr), CFGetTypeID(v) == AXUIElementGetTypeID() else { return nil } + return (v as! AXUIElement) +} + +func elementCenter(_ el: AXUIElement) -> CGPoint? { + guard let p = axPoint(el, kAXPositionAttribute as String), + let s = axSize(el, kAXSizeAttribute as String) else { return nil } + return CGPoint(x: p.x + s.width / 2, y: p.y + s.height / 2) +} + +// MARK: - Element registry +// +// Snapshots hand out short ids ("e12") that later calls reference, so the model +// clicks a named element instead of guessing pixel coordinates. + +final class Registry { + static var map: [String: AXUIElement] = [:] + static var counter = 0 + /// App most recently inspected. Subsequent input is delivered to this + /// process by default, so interaction stays in the background. + static var targetPid: pid_t? + + static func reset() { + map.removeAll() + counter = 0 + targetPid = nil + } + + static func add(_ el: AXUIElement) -> String { + counter += 1 + let id = "e\(counter)" + map[id] = el + return id + } + + static func get(_ id: String) -> AXUIElement? { map[id] } +} + +// MARK: - App resolution + +struct ResolvedApp { + let app: NSRunningApplication + let note: String? +} + +/// Resolve an app by name, bundle id, or pid. +/// +/// A single bundle id can have several running instances — Chrome routinely does. +/// Only some of them own windows, so prefer an instance that actually has one; +/// picking blindly is what makes System Events report "Invalid index". +func resolveApp(_ query: String) -> ResolvedApp? { + let trimmed = query.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return nil } + let running = NSWorkspace.shared.runningApplications + let lowered = trimmed.lowercased() + + var matches: [NSRunningApplication] + if let pid = Int32(trimmed), running.contains(where: { $0.processIdentifier == pid }) { + matches = running.filter { $0.processIdentifier == pid } + } else if Int32(trimmed) != nil { + // Numeric query that is not a live PID (e.g. app name "2048"): exact name + // only — never substring, or short pids like "1" bind unrelated apps. + matches = running.filter { ($0.localizedName ?? "").lowercased() == lowered } + } else { + matches = running.filter { $0.bundleIdentifier?.lowercased() == lowered } + if matches.isEmpty { + matches = running.filter { ($0.localizedName ?? "").lowercased() == lowered } + } + if matches.isEmpty { + matches = running.filter { ($0.localizedName ?? "").lowercased().contains(lowered) } + } + } + guard !matches.isEmpty else { return nil } + if matches.count == 1 { return ResolvedApp(app: matches[0], note: nil) } + + // Count windows only. An app element always has children (the menu bar, at + // minimum), so testing children here would happily select a windowless instance. + func windowCount(_ instance: NSRunningApplication) -> Int { + let ax = AXUIElementCreateApplication(instance.processIdentifier) + return ((axCopy(ax, kAXWindowsAttribute as String) as? [AXUIElement]) ?? []).count + } + + // Prefer frontmost among instances that own windows, so the choice matches + // what the user is actually looking at. + let withWindows = matches.filter { windowCount($0) > 0 } + let chosen = withWindows.first(where: { $0.isActive }) ?? withWindows.first ?? matches.first! + let n = windowCount(chosen) + let note = "\(matches.count) running instances of \(query); selected pid \(chosen.processIdentifier) " + + (n > 0 ? "(\(n) window\(n == 1 ? "" : "s"))" : "(no instance has windows)") + return ResolvedApp(app: chosen, note: note) +} + +// MARK: - Tree walking + +let interactiveRoles: Set = [ + "AXButton", "AXTextField", "AXTextArea", "AXCheckBox", "AXRadioButton", + "AXPopUpButton", "AXMenuItem", "AXMenuButton", "AXLink", "AXComboBox", + "AXSlider", "AXDisclosureTriangle", "AXSegmentedControl", "AXSearchField", + "AXTabGroup", "AXIncrementor", "AXColorWell", "AXCell", +] + +func truncate(_ s: String, _ n: Int) -> String { + let flat = s.replacingOccurrences(of: "\n", with: " ") + return flat.count <= n ? flat : String(flat.prefix(n)) + "…" +} + +func walk(_ el: AXUIElement, depth: Int, lines: inout [String], budget: inout Int, maxDepth: Int) { + guard budget > 0, depth <= maxDepth else { return } + + let role = axString(el, kAXRoleAttribute as String) ?? "AXUnknown" + let title = axString(el, kAXTitleAttribute as String) + let desc = axString(el, kAXDescriptionAttribute as String) + let value = axString(el, kAXValueAttribute as String) + let actions = axActions(el).filter { $0 != "AXShowMenu" } + let isInteractive = interactiveRoles.contains(role) || !actions.isEmpty + let label = title ?? desc ?? value + + // Emit a node only if it carries information: something actionable, or text. + // Pure layout containers are traversed but not printed, which keeps the + // outline small enough to be worth putting in a prompt. + if isInteractive || label != nil { + var parts = ["\(String(repeating: " ", count: depth))"] + if isInteractive { + parts.append("[\(Registry.add(el))] ") + } else { + parts.append(" ") + } + parts.append(role.replacingOccurrences(of: "AX", with: "")) + if let l = label { parts.append(" \"\(truncate(l, 120))\"") } + // Show the current contents whenever they are not already the label. + // Fields commonly label themselves with AXDescription ("Address and + // search bar") and keep the typed text in AXValue, so gating this on + // AXTitle hid what the field actually contains. + if let v = value, v != label { + parts.append(" value=\"\(truncate(v, 80))\"") + } + if axBool(el, kAXEnabledAttribute as String) == false { parts.append(" (disabled)") } + if axBool(el, kAXFocusedAttribute as String) == true { parts.append(" (focused)") } + lines.append(parts.joined()) + budget -= 1 + } + + for child in axChildren(el) { + walk(child, depth: depth + 1, lines: &lines, budget: &budget, maxDepth: maxDepth) + } +} + +// MARK: - Input synthesis + +// MOUSE_TARGETING +// +// Coordinate mouse events reach a background window through SkyLight, so the +// agent can click in one app while the user works in another and the physical +// cursor never moves. Three things are all required — miss any one and the event +// is silently dropped: +// +// 1. Window addressing. The event carries the target window id in fields +// 51/91/92 plus window-local coordinates via CGEventSetWindowLocation. +// 2. SLEventSetIntegerValueField, NOT CGEvent.setIntegerValueField. The public +// setter takes a CGEventField enum and CGEventField(rawValue:) returns nil +// for the undocumented fields (51/58/91/92), so those stamps vanish. +// 3. activate_without_raise. A background window will not accept routed input +// until its AppKit-active state is flipped, which is done without raising +// the window or switching Spaces. +// +// Delivery goes through both SLEventPostToPid (reaches Chromium/Catalyst, which +// ignore the public path because it skips the activity-monitor tickle) and +// CGEvent.postToPid (lands on AppKit targets where the SkyLight path drops). +// +// Ported from trycua/cua's cua-driver, which in turn takes focus-without-raise +// from yabai. These are private SPIs resolved by dlsym: if any fail to resolve +// we fall back to the global HID tap, which works but moves the user's cursor. +// +// Summary: +// * type_text / press_key -> postToPid, background-safe +// * click by element_id -> AXPress, background-safe, no cursor movement +// * click/drag by coordinates -> SkyLight background path, cursor stays put +// * any of the above, degraded -> global HID tap, takes over the pointer + +/// Deliver an event to a specific process when we know one, otherwise to the +/// global HID tap. +/// +/// Targeting a pid is what lets the agent work in the background: the event goes +/// straight to that application, so the physical cursor does not jump, focus is +/// not stolen, and the user can keep working in another app meanwhile. The global +/// tap is a fallback for raw-coordinate calls where no app is known, and it does +/// take over the machine. +func post(_ event: CGEvent?, to pid: pid_t?) { + guard let event else { return } + if let pid { + event.postToPid(pid) + } else { + event.post(tap: .cghidEventTap) + } +} + +func pidOf(_ element: AXUIElement) -> pid_t? { + var pid: pid_t = 0 + return AXUIElementGetPid(element, &pid) == .success ? pid : nil +} + +// MARK: - SkyLight background input + +/// Private SPIs behind background mouse delivery. All optional: when a symbol +/// stops resolving on a future macOS the caller degrades to the global HID tap +/// rather than failing. +enum SkyLight { + typealias PostToPidFn = @convention(c) (pid_t, UnsafeMutableRawPointer) -> Void + typealias SetIntFieldFn = @convention(c) (UnsafeMutableRawPointer, UInt32, Int64) -> Void + typealias SetWindowLocFn = @convention(c) (UnsafeMutableRawPointer, CGPoint) -> Void + typealias PostEventRecordFn = @convention(c) (UnsafeMutableRawPointer, UnsafeMutablePointer) -> Int32 + typealias GetFrontProcessFn = @convention(c) (UnsafeMutableRawPointer) -> Int32 + typealias GetProcessForPIDFn = @convention(c) (pid_t, UnsafeMutablePointer) -> OSStatus + typealias AXGetWindowFn = @convention(c) (AXUIElement, UnsafeMutablePointer) -> AXError + + static let skyHandle = dlopen( + "/System/Library/PrivateFrameworks/SkyLight.framework/Versions/A/SkyLight", RTLD_LAZY) + static let appServices = dlopen( + "/System/Library/Frameworks/ApplicationServices.framework/ApplicationServices", RTLD_LAZY) + + static let postToPid: PostToPidFn? = load("SLEventPostToPid", skyHandle) + static let setIntField: SetIntFieldFn? = load("SLEventSetIntegerValueField", skyHandle) + static let setWindowLocation: SetWindowLocFn? = load("CGEventSetWindowLocation", skyHandle) + ?? load("CGEventSetWindowLocation", dlopen("/System/Library/Frameworks/CoreGraphics.framework/CoreGraphics", RTLD_LAZY)) + static let postEventRecord: PostEventRecordFn? = load("SLPSPostEventRecordTo", skyHandle) + static let getFrontProcess: GetFrontProcessFn? = load("_SLPSGetFrontProcess", skyHandle) + static let getProcessForPID: GetProcessForPIDFn? = load("GetProcessForPID", appServices) + static let axGetWindow: AXGetWindowFn? = load("_AXUIElementGetWindow", appServices) + + static func load(_ name: String, _ handle: UnsafeMutableRawPointer?) -> T? { + guard let handle, let sym = dlsym(handle, name) else { return nil } + return unsafeBitCast(sym, to: T.self) + } + + static var available: Bool { + // setWindowLocation is required: without window-local coordinates, + // background mouse events are delivered but never hit-test, so callers + // would report success while clicks/scrolls do nothing. + postToPid != nil && setIntField != nil && setWindowLocation != nil + && postEventRecord != nil && getFrontProcess != nil + && getProcessForPID != nil && axGetWindow != nil + } + + static func windowID(_ window: AXUIElement) -> UInt32? { + guard let fn = axGetWindow else { return nil } + var wid: UInt32 = 0 + return fn(window, &wid) == .success ? wid : nil + } + + /// Make the target window able to accept routed input without raising it or + /// switching Spaces. Deliberately skips SLPSSetFrontProcessWithOptions — + /// omitting it keeps Chromium's user-activation gate open. + @discardableResult + static func activateWithoutRaise(pid: pid_t, wid: UInt32) -> Bool { + guard let post = postEventRecord, let front = getFrontProcess, let forPID = getProcessForPID + else { return false } + + // PSNs are 8 raw bytes here, not the Swift struct's layout guarantees. + var previous = [UInt8](repeating: 0, count: 8) + var target = [UInt8](repeating: 0, count: 8) + let gotPrevious = previous.withUnsafeMutableBufferPointer { + front(UnsafeMutableRawPointer($0.baseAddress!)) == 0 + } + guard gotPrevious else { return false } + + var psn = ProcessSerialNumber() + guard forPID(pid, &psn) == 0 else { return false } + withUnsafeBytes(of: &psn) { raw in for i in 0..<8 { target[i] = raw[i] } } + + var record = [UInt8](repeating: 0, count: 0xF8) + record[0x04] = 0xF8 + record[0x08] = 0x0D + record[0x3C] = UInt8(wid & 0xFF) + record[0x3D] = UInt8((wid >> 8) & 0xFF) + record[0x3E] = UInt8((wid >> 16) & 0xFF) + record[0x3F] = UInt8((wid >> 24) & 0xFF) + + record[0x8A] = 0x02 // defocus the outgoing front process + let defocused = previous.withUnsafeMutableBufferPointer { p in + record.withUnsafeMutableBufferPointer { r in + post(UnsafeMutableRawPointer(p.baseAddress!), r.baseAddress!) == 0 + } + } + record[0x8A] = 0x01 // focus the target + let focused = target.withUnsafeMutableBufferPointer { p in + record.withUnsafeMutableBufferPointer { r in + post(UnsafeMutableRawPointer(p.baseAddress!), r.baseAddress!) == 0 + } + } + return defocused && focused + } + + /// Stamp the window-routing fields and deliver down both paths. + static func postMouse( + _ event: CGEvent, pid: pid_t, wid: UInt32, windowOrigin: CGPoint, + screen: CGPoint, clickState: Int64, button: Int64, subtype: Int64, groupID: Int64 + ) { + guard let post = postToPid, let setField = setIntField, let setWindowLocation else { return } + let ptr = Unmanaged.passUnretained(event).toOpaque() + setWindowLocation(ptr, CGPoint(x: screen.x - windowOrigin.x, y: screen.y - windowOrigin.y)) + let w = Int64(wid) + setField(ptr, 1, clickState) // click state + setField(ptr, 3, button) // button number + setField(ptr, 7, subtype) // subtype: 3 touch for clicks, 0 for drags + setField(ptr, 51, w) // window number + setField(ptr, 58, groupID) // click-group id, coalesces the gesture + setField(ptr, 91, w) // window under mouse pointer + setField(ptr, 92, w) // ...that can handle this event + setField(ptr, 40, Int64(pid)) // target pid (Chromium synthetic filter) + post(pid, ptr) + event.postToPid(pid) + } +} + +/// A window that background mouse events can be addressed to. +struct WindowTarget { + let pid: pid_t + let wid: UInt32 + let frame: CGRect + var origin: CGPoint { frame.origin } +} + +func makeWindowTarget(pid: pid_t, window: AXUIElement) -> WindowTarget? { + guard let wid = SkyLight.windowID(window) else { return nil } + guard let origin = axPoint(window, kAXPositionAttribute as String) else { return nil } + guard let size = axSize(window, kAXSizeAttribute as String), + size.width > 0, size.height > 0 + else { return nil } + return WindowTarget(pid: pid, wid: wid, frame: CGRect(origin: origin, size: size)) +} + +func windowTarget(for element: AXUIElement) -> WindowTarget? { + guard let pid = pidOf(element) else { return nil } + let window = axElement(element, kAXWindowAttribute as String) + ?? (axCopy(AXUIElementCreateApplication(pid), kAXWindowsAttribute as String) as? [AXUIElement])?.first + guard let window else { return nil } + return makeWindowTarget(pid: pid, window: window) +} + +/// The frontmost on-screen window containing `point`. +/// +/// `CGWindowListCopyWindowInfo` returns windows front to back, so the first +/// hit is the one a person clicking there would reach. +func windowTarget(under point: CGPoint) -> WindowTarget? { + let options: CGWindowListOption = [.optionOnScreenOnly, .excludeDesktopElements] + guard let windows = CGWindowListCopyWindowInfo(options, kCGNullWindowID) as? [[String: Any]] else { + return nil + } + for window in windows { + // These arrive as NSNumber, which does not bridge straight to pid_t or + // UInt32 — casting directly returns nil and the lookup silently fails. + guard let bounds = window[kCGWindowBounds as String] as? [String: Any], + let pidValue = window[kCGWindowOwnerPID as String] as? NSNumber, + let numberValue = window[kCGWindowNumber as String] as? NSNumber, + let x = (bounds["X"] as? NSNumber)?.doubleValue, + let y = (bounds["Y"] as? NSNumber)?.doubleValue, + let width = (bounds["Width"] as? NSNumber)?.doubleValue, + let height = (bounds["Height"] as? NSNumber)?.doubleValue + else { continue } + let pid = pid_t(pidValue.int32Value) + let number = numberValue.uint32Value + // Skip this process and the separate T3AgentCursor overlay, which sits + // above the click point by design and would steal hit-testing. + if pid == getpid() { continue } + if let owner = window[kCGWindowOwnerName as String] as? String, + owner == "T3AgentCursor" || owner.hasPrefix("T3AgentCursor") + { + continue + } + if let app = NSRunningApplication(processIdentifier: pid), + app.bundleIdentifier == "com.t3tools.t3code.agent-cursor" + { + continue + } + if CGRect(x: x, y: y, width: width, height: height).contains(point) { + return WindowTarget( + pid: pid, + wid: number, + frame: CGRect(x: x, y: y, width: width, height: height) + ) + } + } + return nil +} + +func windowTarget(forPid pid: pid_t, containing point: CGPoint? = nil) -> WindowTarget? { + let windows = (axCopy(AXUIElementCreateApplication(pid), kAXWindowsAttribute as String) as? [AXUIElement]) ?? [] + if let point { + for window in windows { + guard let target = makeWindowTarget(pid: pid, window: window) else { continue } + if target.frame.contains(point) { + return target + } + } + } + guard let window = windows.first else { return nil } + return makeWindowTarget(pid: pid, window: window) +} + +/// Whether an element lives inside rendered web content. +/// +/// Chromium exposes AXPress on web elements and returns success without doing +/// anything, so callers need to know when to bypass it and click for real. +func isInWebContent(_ element: AXUIElement) -> Bool { + var node: AXUIElement? = element + while let current = node { + if let role = axString(current, kAXRoleAttribute as String), + role == "AXWebArea" { return true } + node = axElement(current, kAXParentAttribute as String) + } + return false +} + +/// Centre of the part of an element that is actually on screen. +/// +/// A scrollable element reports its *content* frame, which can be far taller +/// than the window showing it — the raw centre of a long document's text area +/// lands below the window entirely, and the click misses. Clipping to the +/// window keeps the point somewhere clickable. +func visibleCenter(of element: AXUIElement) -> CGPoint? { + guard let position = axPoint(element, kAXPositionAttribute as String), + let size = axSize(element, kAXSizeAttribute as String) else { return nil } + let elementRect = CGRect(origin: position, size: size) + guard let target = windowTarget(for: element), !target.frame.isEmpty else { + return CGPoint(x: elementRect.midX, y: elementRect.midY) + } + let visible = elementRect.intersection(target.frame) + // Entirely off-window (scrolled away / off-screen) — no clickable target. + guard !visible.isNull, !visible.isEmpty else { return nil } + return CGPoint(x: visible.midX, y: visible.midY) +} + +var clickGroupCounter: Int64 = 0x4000 + +/// Background click. Returns false if the SkyLight path is unavailable, so the +/// caller can fall back to the cursor-moving global tap. +func backgroundClick(_ target: WindowTarget, at point: CGPoint, clickCount: Int) -> Bool { + guard SkyLight.available else { return false } + CursorOverlay.shared.press(at: point) + guard SkyLight.activateWithoutRaise(pid: target.pid, wid: target.wid) else { return false } + usleep(80_000) + + clickGroupCounter += 1 + let group = clickGroupCounter + let src = CGEventSource(stateID: .combinedSessionState) + + // A background window has stale cursor-tracking state, so a bare mouseDown + // hit-tests "outside" the control and never fires. + if let move = CGEvent(mouseEventSource: src, mouseType: .mouseMoved, mouseCursorPosition: point, mouseButton: .left) { + SkyLight.postMouse(move, pid: target.pid, wid: target.wid, windowOrigin: target.origin, + screen: point, clickState: 0, button: 0, subtype: 3, groupID: group) + } + usleep(12_000) + var delivered = false + guard clickCount > 0 else { return false } + for i in 1...clickCount { + if let down = CGEvent(mouseEventSource: src, mouseType: .leftMouseDown, mouseCursorPosition: point, mouseButton: .left) { + SkyLight.postMouse(down, pid: target.pid, wid: target.wid, windowOrigin: target.origin, + screen: point, clickState: Int64(i), button: 0, subtype: 3, groupID: group) + delivered = true + } + usleep(28_000) + if let up = CGEvent(mouseEventSource: src, mouseType: .leftMouseUp, mouseCursorPosition: point, mouseButton: .left) { + SkyLight.postMouse(up, pid: target.pid, wid: target.wid, windowOrigin: target.origin, + screen: point, clickState: Int64(i), button: 0, subtype: 3, groupID: group) + delivered = true + } + if i < clickCount { usleep(80_000) } + } + return delivered +} + +func backgroundScroll(_ target: WindowTarget, at point: CGPoint, dx: Int32, dy: Int32, steps: Int) -> Bool { + guard SkyLight.available, let post = SkyLight.postToPid, let setField = SkyLight.setIntField, + let setWindowLocation = SkyLight.setWindowLocation + else { return false } + CursorOverlay.shared.show(at: point) + guard SkyLight.activateWithoutRaise(pid: target.pid, wid: target.wid) else { return false } + usleep(80_000) + clickGroupCounter += 1 + let group = clickGroupCounter + + // Prime the window's hit-test location. A background window keeps a stale + // one, and the wheel then lands on nothing even though it is delivered. + if let move = CGEvent(mouseEventSource: CGEventSource(stateID: .hidSystemState), + mouseType: .mouseMoved, mouseCursorPosition: point, mouseButton: .left) { + SkyLight.postMouse(move, pid: target.pid, wid: target.wid, windowOrigin: target.origin, + screen: point, clickState: 0, button: 0, subtype: 3, groupID: group) + } + usleep(12_000) + + let local = CGPoint(x: point.x - target.origin.x, y: point.y - target.origin.y) + var delivered = 0 + for _ in 0.. 0 +} + +func backgroundRightClick(_ target: WindowTarget, at point: CGPoint) -> Bool { + guard SkyLight.available else { return false } + CursorOverlay.shared.press(at: point) + guard SkyLight.activateWithoutRaise(pid: target.pid, wid: target.wid) else { return false } + usleep(80_000) + clickGroupCounter += 1 + let group = clickGroupCounter + let src = CGEventSource(stateID: .combinedSessionState) + // Prime hit-testing the same way left-click and scroll do; a bare + // rightMouseDown against a background window often lands outside the control. + if let moved = CGEvent(mouseEventSource: src, mouseType: .mouseMoved, mouseCursorPosition: point, mouseButton: .left) { + SkyLight.postMouse(moved, pid: target.pid, wid: target.wid, windowOrigin: target.origin, + screen: point, clickState: 0, button: 0, subtype: 3, groupID: group) + } + usleep(12_000) + var delivered = false + if let down = CGEvent(mouseEventSource: src, mouseType: .rightMouseDown, mouseCursorPosition: point, mouseButton: .right) { + SkyLight.postMouse(down, pid: target.pid, wid: target.wid, windowOrigin: target.origin, + screen: point, clickState: 1, button: 1, subtype: 3, groupID: group) + delivered = true + } + usleep(28_000) + if let up = CGEvent(mouseEventSource: src, mouseType: .rightMouseUp, mouseCursorPosition: point, mouseButton: .right) { + SkyLight.postMouse(up, pid: target.pid, wid: target.wid, windowOrigin: target.origin, + screen: point, clickState: 1, button: 1, subtype: 3, groupID: group) + delivered = true + } + return delivered +} + +func backgroundDrag(_ target: WindowTarget, from start: CGPoint, to end: CGPoint) -> Bool { + guard SkyLight.available else { return false } + // SkyLight posts are addressed to one window. A mouseUp aimed at another + // window (or the desktop) would still be delivered to `target`, so refuse + // cross-window background drags instead of mis-routing the release. + if !target.frame.contains(end) { + guard let dest = windowTarget(under: end), dest.wid == target.wid, dest.pid == target.pid else { + return false + } + } + CursorOverlay.shared.press(at: start) + guard SkyLight.activateWithoutRaise(pid: target.pid, wid: target.wid) else { return false } + usleep(80_000) + clickGroupCounter += 1 + let group = clickGroupCounter + let src = CGEventSource(stateID: .combinedSessionState) + var delivered = false + + func send(_ type: CGEventType, _ point: CGPoint, _ clickState: Int64, _ subtype: Int64) { + guard let e = CGEvent(mouseEventSource: src, mouseType: type, mouseCursorPosition: point, mouseButton: .left) + else { return } + SkyLight.postMouse(e, pid: target.pid, wid: target.wid, windowOrigin: target.origin, + screen: point, clickState: clickState, button: 0, subtype: subtype, groupID: group) + delivered = true + } + + send(.mouseMoved, start, 0, 3) + usleep(12_000) + send(.leftMouseDown, start, 1, 3) + usleep(28_000) + // Drags carry the normal subtype rather than touch. + let steps = 24 + for i in 1...steps { + let t = Double(i) / Double(steps) + let step = CGPoint(x: start.x + (end.x - start.x) * t, y: start.y + (end.y - start.y) * t) + send(.leftMouseDragged, step, 1, 0) + if i % 4 == 0 { CursorOverlay.shared.glide(at: step) } + usleep(15_000) + } + usleep(40_000) + send(.leftMouseUp, end, 1, 3) + CursorOverlay.shared.press(at: end) + return delivered +} + +func postClick(at point: CGPoint, clickCount: Int = 1, pid: pid_t?) { + CursorOverlay.shared.press(at: point) + let src = CGEventSource(stateID: .combinedSessionState) + for i in 1...clickCount { + let down = CGEvent(mouseEventSource: src, mouseType: .leftMouseDown, mouseCursorPosition: point, mouseButton: .left) + let up = CGEvent(mouseEventSource: src, mouseType: .leftMouseUp, mouseCursorPosition: point, mouseButton: .left) + down?.setIntegerValueField(.mouseEventClickState, value: Int64(i)) + up?.setIntegerValueField(.mouseEventClickState, value: Int64(i)) + post(down, to: pid) + post(up, to: pid) + if i < clickCount { usleep(80_000) } + } +} + +func typeText(_ text: String, pid: pid_t?) { + let src = CGEventSource(stateID: .combinedSessionState) + // Send in small UTF-16 chunks: keyboardSetUnicodeString has a length cap, + // and per-chunk events keep long strings from being dropped. + for chunk in Array(text).chunked(into: 16) { + var utf16 = Array(String(chunk).utf16) + guard let down = CGEvent(keyboardEventSource: src, virtualKey: 0, keyDown: true), + let up = CGEvent(keyboardEventSource: src, virtualKey: 0, keyDown: false) else { continue } + down.keyboardSetUnicodeString(stringLength: utf16.count, unicodeString: &utf16) + up.keyboardSetUnicodeString(stringLength: utf16.count, unicodeString: &utf16) + post(down, to: pid) + post(up, to: pid) + usleep(8_000) + } +} + +extension Array { + func chunked(into size: Int) -> [[Element]] { + stride(from: 0, to: count, by: size).map { Array(self[$0.. String? { + guard let code = keyCodes[key.lowercased()] else { return "unknown key: \(key)" } + var flags: CGEventFlags = [] + for m in modifiers.map({ $0.lowercased() }) { + switch m { + case "cmd", "command": flags.insert(.maskCommand) + case "shift": flags.insert(.maskShift) + case "alt", "option": flags.insert(.maskAlternate) + case "ctrl", "control": flags.insert(.maskControl) + case "fn": flags.insert(.maskSecondaryFn) + default: return "unknown modifier: \(m)" + } + } + let src = CGEventSource(stateID: .combinedSessionState) + let down = CGEvent(keyboardEventSource: src, virtualKey: code, keyDown: true) + let up = CGEvent(keyboardEventSource: src, virtualKey: code, keyDown: false) + down?.flags = flags + up?.flags = flags + post(down, to: pid) + post(up, to: pid) + return nil +} + +// MARK: - Tool implementations + +func toolListApps() -> String { + var out: [String] = [] + let apps = NSWorkspace.shared.runningApplications + .filter { $0.activationPolicy == .regular } + .sorted { ($0.localizedName ?? "") < ($1.localizedName ?? "") } + + for app in apps { + let ax = AXUIElementCreateApplication(app.processIdentifier) + let windows = (axCopy(ax, kAXWindowsAttribute as String) as? [AXUIElement]) ?? [] + var line = "\(app.localizedName ?? "?") [\(app.bundleIdentifier ?? "-")] pid=\(app.processIdentifier) windows=\(windows.count)" + if app.isActive { line += " FRONTMOST" } + out.append(line) + } + return out.isEmpty ? "No apps found." : out.joined(separator: "\n") +} + +func toolGetAppState(_ args: [String: Any]) -> String { + guard let query = args["app"] as? String else { return "error: missing required argument 'app'" } + guard let resolved = resolveApp(query) else { return "error: no running app matching \(query)" } + + let app = resolved.app + let maxDepth = (args["max_depth"] as? Int) ?? 18 + var budget = (args["max_elements"] as? Int) ?? 800 + + Registry.reset() + Registry.targetPid = app.processIdentifier + let ax = AXUIElementCreateApplication(app.processIdentifier) + var windows = (axCopy(ax, kAXWindowsAttribute as String) as? [AXUIElement]) ?? [] + + var header = "\(app.localizedName ?? "?") [\(app.bundleIdentifier ?? "-")] pid=\(app.processIdentifier) frontmost=\(app.isActive) windows=\(windows.count)" + if let note = resolved.note { header += "\nnote: \(note)" } + + // Narrow to one window. "agent" is the Chrome window this server owns, which + // keeps the tree (and any clicks derived from it) off the user's own tabs. + if let scope = args["window"] { + if let name = scope as? String, name == "agent" { + guard let agent = Chrome.agentAXWindow() else { + return header + "\n\n(no agent window yet — call browser_open_tab first)" + } + windows = [agent.element] + Registry.targetPid = agent.pid + header += "\nscope: agent window only" + } else if let index = scope as? Int { + guard index >= 0, index < windows.count else { + return header + "\n\n(window \(index) is out of range)" + } + windows = [windows[index]] + header += "\nscope: window \(index) only" + } + } + + if windows.isEmpty { + return header + "\n\n(this process has no accessibility windows — if you expected one, another instance of the same app may own it; check list_apps)" + } + + var lines: [String] = [] + for (i, w) in windows.enumerated() { + let title = axString(w, kAXTitleAttribute as String) ?? "" + lines.append("── window \(i): \"\(title)\"") + walk(w, depth: 1, lines: &lines, budget: &budget, maxDepth: maxDepth) + } + if budget <= 0 { + lines.append("… element budget reached; raise max_elements for more") + } + return header + "\n\n" + lines.joined(separator: "\n") +} + +/// Which process should receive synthetic input. +/// +/// Order matters: an element knows its own owner, an explicit `app` argument is +/// the caller's intent, and the last inspected app is the sensible default. +/// Returning nil means global delivery, which moves the real cursor. +/// An explicit `app` that does not resolve is an error — never fall through. +func resolveTargetPid(_ args: [String: Any], element: AXUIElement? = nil) -> Result { + if let element, let pid = pidOf(element) { return .success(pid) } + if let query = args["app"] as? String { + guard let resolved = resolveApp(query) else { + return .failure("error: no running app matching \(query)") + } + return .success(resolved.app.processIdentifier) + } + return .success(Registry.targetPid) +} + +func toolClick(_ args: [String: Any]) -> String { + let clickCount = (args["click_count"] as? Int) ?? 1 + guard clickCount > 0, clickCount <= 3 else { + return "error: click_count must be an integer between 1 and 3" + } + + if let id = args["element_id"] as? String { + guard let el = Registry.get(id) else { + return "error: unknown element_id \(id) — call get_app_state again to refresh ids" + } + // Prefer the semantic action; it works even when the element is scrolled + // out of view or overlapped, where a synthetic click would hit the wrong thing. + // + // Web content is the exception: Blink reports AXPress as supported and + // returns success, but does not act on it — a link "pressed" this way + // never navigates. Inside a web area, go straight to a real click. + // Show the pointer before acting, not after: AXPress returns early, so + // placing this later meant the overlay never appeared for the common + // case of pressing a button. + let elementCenter = visibleCenter(of: el) + // Point the overlay at the element's own frame, not its visible rect: + // visibleCenter is nil whenever the window is occluded, which is the + // normal case for background control and meant the pointer never showed. + if let origin = axPoint(el, kAXPositionAttribute as String), + let size = axSize(el, kAXSizeAttribute as String), size.width > 0, size.height > 0 + { + AgentCursor.shared.press( + at: CGPoint(x: origin.x + size.width / 2, y: origin.y + size.height / 2) + ) + } else if let elementCenter { + AgentCursor.shared.press(at: elementCenter) + } + if axActions(el).contains(kAXPressAction as String), clickCount == 1, !isInWebContent(el) { + if AXUIElementPerformAction(el, kAXPressAction as CFString) == .success { + let label = axString(el, kAXTitleAttribute as String) ?? axString(el, kAXDescriptionAttribute as String) ?? id + return "pressed \(id) \"\(label)\" via AXPress" + } + } + // Coordinate fallback: see MOUSE_TARGETING. + guard let center = elementCenter else { + return "error: \(id) is not visible in its window — scroll it into view and call get_app_state again" + } + if let target = windowTarget(for: el), backgroundClick(target, at: center, clickCount: clickCount) { + return "clicked \(id) at (\(Int(center.x)), \(Int(center.y))) in background" + } + postClick(at: center, clickCount: clickCount, pid: nil) + return "clicked \(id) at (\(Int(center.x)), \(Int(center.y))) via cursor" + } + + if let x = args["x"] as? Double, let y = args["y"] as? Double { + guard Int(exactly: x.rounded(.towardZero)) != nil, + Int(exactly: y.rounded(.towardZero)) != nil else { + return "error: coordinates must be finite and representable as integers" + } + let point = CGPoint(x: x, y: y) + AgentCursor.shared.press(at: point) + // Prefer the window under the point. Only constrain to an app PID when the + // caller passed `app` explicitly — Registry.targetPid from get_app_state + // must not discard a same-desktop under-point window. + let under = windowTarget(under: point) + let target: WindowTarget? + if let query = args["app"] as? String { + guard let resolved = resolveApp(query) else { + return "error: no running app matching \(query)" + } + let appPid = resolved.app.processIdentifier + target = under.flatMap { $0.pid == appPid ? $0 : nil } + ?? windowTarget(forPid: appPid, containing: point) + } else { + switch resolveTargetPid(args) { + case .failure(let message): + return message + case .success(let pid): + target = under ?? pid.flatMap { windowTarget(forPid: $0, containing: point) } + } + } + if let target, backgroundClick(target, at: point, clickCount: clickCount) { + return "clicked at (\(Int(x)), \(Int(y))) in background" + } + postClick(at: point, clickCount: clickCount, pid: nil) + return "clicked at (\(Int(x)), \(Int(y))) via cursor" + } + return "error: provide either element_id, or both x and y" +} + +func toolTypeText(_ args: [String: Any]) -> String { + guard let text = args["text"] as? String else { return "error: missing required argument 'text'" } + var element: AXUIElement? + if let id = args["element_id"] as? String { + guard let el = Registry.get(id) else { return "error: unknown element_id \(id)" } + element = el + // Focus the field within its own app rather than raising the app, so a + // background window still receives the text. + AXUIElementSetAttributeValue(el, kAXFocusedAttribute as CFString, kCFBooleanTrue) + usleep(60_000) + } + let pid: pid_t? + switch resolveTargetPid(args, element: element) { + case .failure(let message): + return message + case .success(let resolved): + pid = resolved + } + typeText(text, pid: pid) + return "typed \(text.count) characters" +} + +func toolPressKey(_ args: [String: Any]) -> String { + guard let key = args["key"] as? String else { return "error: missing required argument 'key'" } + let mods = (args["modifiers"] as? [String]) ?? [] + let pid: pid_t? + switch resolveTargetPid(args) { + case .failure(let message): + return message + case .success(let resolved): + pid = resolved + } + if let err = pressKey(key, modifiers: mods, pid: pid) { return "error: \(err)" } + return "pressed \(mods.isEmpty ? key : mods.joined(separator: "+") + "+" + key)" +} + +func toolScroll(_ args: [String: Any]) -> String { + let direction = ((args["direction"] as? String) ?? "down").lowercased() + let amount = (args["amount"] as? Int) ?? 5 + guard amount != Int.min else { return "error: amount is out of range" } + + var dy: Int32 = 0 + var dx: Int32 = 0 + switch direction { + case "up": dy = 1 + case "down": dy = -1 + case "left": dx = 1 + case "right": dx = -1 + default: return "error: direction must be up, down, left, or right" + } + + if let elementID = args["element_id"] as? String, Registry.get(elementID) == nil { + return "error: unknown element_id \(elementID) — call get_app_state again to refresh ids" + } + let element = (args["element_id"] as? String).flatMap { Registry.get($0) } + let target: WindowTarget? + if let element { + target = windowTarget(for: element) + } else { + switch resolveTargetPid(args) { + case .failure(let message): + return message + case .success(let pid): + target = pid.flatMap { windowTarget(forPid: $0) } + } + } + + if let target { + // Scroll follows the pointer, so aim at the element when given one and + // otherwise at the middle of the window. + let point = element.flatMap { visibleCenter(of: $0) } + ?? CGPoint(x: target.origin.x + 200, y: target.origin.y + 200) + if backgroundScroll(target, at: point, dx: dx, dy: dy, steps: abs(amount)) { + return "scrolled \(direction) by \(amount) in background" + } + } + + // Fallback: drive the real pointer. + if let el = element, let center = elementCenter(el) { + CGWarpMouseCursorPosition(center) + usleep(30_000) + } + let src = CGEventSource(stateID: .combinedSessionState) + for _ in 0.. String { + guard let query = args["app"] as? String else { return "error: missing required argument 'app'" } + guard let resolved = resolveApp(query) else { return "error: no running app matching \(query)" } + // Requests are handled off the main thread; NSRunningApplication.activate is + // AppKit and belongs on main. + DispatchQueue.main.sync { resolved.app.activate(options: []) } + usleep(250_000) + return "activated \(resolved.app.localizedName ?? query) (pid \(resolved.app.processIdentifier))" +} + +// MARK: - Screen capture + +/// Synchronizes capture results so a timed-out waiter never races a late write. +private final class CaptureBox: @unchecked Sendable { + private let lock = NSLock() + private var value: Data? + func set(_ data: Data?) { + lock.lock() + value = data + lock.unlock() + } + func get() -> Data? { + lock.lock() + defer { lock.unlock() } + return value + } +} + +/// Capture a window as PNG. Runs the async ScreenCaptureKit call on a background +/// executor and blocks the JSON-RPC loop until it lands, with a timeout so a +/// wedged capture can never hang the server. +func captureWindowPNG(pid: pid_t, maxWidth: Int) -> Data? { + let semaphore = DispatchSemaphore(value: 0) + let box = CaptureBox() + + let task = Task.detached { + defer { semaphore.signal() } + do { + let content = try await SCShareableContent.excludingDesktopWindows( + false, onScreenWindowsOnly: true) + // Largest on-screen window belonging to the target process; smaller + // ones are usually palettes or overlays rather than the main UI. + let candidates = content.windows + .filter { $0.owningApplication?.processID == pid } + .sorted { ($0.frame.width * $0.frame.height) > ($1.frame.width * $1.frame.height) } + guard let window = candidates.first else { return } + + let config = SCStreamConfiguration() + let scale: Double + if maxWidth > 0 { + scale = min(1.0, Double(maxWidth) / max(1.0, Double(window.frame.width))) + } else { + scale = 1.0 + } + config.width = Int(window.frame.width * scale) + config.height = Int(window.frame.height * scale) + config.showsCursor = false + + let image = try await SCScreenshotManager.captureImage( + contentFilter: SCContentFilter(desktopIndependentWindow: window), + configuration: config) + let png = NSBitmapImageRep(cgImage: image).representation(using: .png, properties: [:]) + box.set(png) + } catch { + box.set(nil) + } + } + + let waited = semaphore.wait(timeout: .now() + 15) + if waited == .timedOut { + task.cancel() + // Do not read `box` after cancel — the task may still be writing. + return nil + } + return box.get() +} + +/// Capture a whole display. Window capture covers one app; this is for seeing +/// the desktop as a whole, including every monitor the user has attached. +func captureDisplayPNG(index: Int, maxWidth: Int) -> (data: Data, width: Int, height: Int)? { + let semaphore = DispatchSemaphore(value: 0) + let lock = NSLock() + var result: (Data, Int, Int)? + Task.detached { + defer { semaphore.signal() } + do { + let content = try await SCShareableContent.excludingDesktopWindows( + false, onScreenWindowsOnly: true) + let displays = content.displays + guard index >= 0, index < displays.count else { return } + let display = displays[index] + let config = SCStreamConfiguration() + let scale: Double + if maxWidth > 0 { + scale = min(1.0, Double(maxWidth) / max(1.0, Double(display.width))) + } else { + scale = 1.0 + } + config.width = Int(Double(display.width) * scale) + config.height = Int(Double(display.height) * scale) + config.showsCursor = false + let image = try await SCScreenshotManager.captureImage( + contentFilter: SCContentFilter(display: display, excludingWindows: []), + configuration: config) + if let png = NSBitmapImageRep(cgImage: image).representation(using: .png, properties: [:]) { + lock.lock() + result = (png, display.width, display.height) + lock.unlock() + } + } catch { + lock.lock() + result = nil + lock.unlock() + } + } + // On timeout the task may still write `result` — do not read it. + if semaphore.wait(timeout: .now() + 20) == .timedOut { + return nil + } + lock.lock() + defer { lock.unlock() } + return result +} + +func toolListDisplays(_ args: [String: Any]) -> String { + let semaphore = DispatchSemaphore(value: 0) + let lock = NSLock() + var lines: [String]? + let task = Task.detached { + defer { semaphore.signal() } + var collected: [String] = [] + if let content = try? await SCShareableContent.excludingDesktopWindows( + false, onScreenWindowsOnly: true) { + for (i, display) in content.displays.enumerated() { + let frame = display.frame + collected.append("[\(i)] \(display.width)x\(display.height) " + + "at (\(Int(frame.origin.x)), \(Int(frame.origin.y)))") + } + } + lock.lock() + lines = collected + lock.unlock() + } + if semaphore.wait(timeout: .now() + 20) == .timedOut { + task.cancel() + // On timeout the task may still write `lines` — do not read it. + return "error: could not enumerate displays" + } + lock.lock() + let snapshot = lines + lock.unlock() + guard let snapshot, !snapshot.isEmpty else { + return "error: could not enumerate displays" + } + return "\(snapshot.count) display\(snapshot.count == 1 ? "" : "s"):\n" + snapshot.joined(separator: "\n") +} + +// MARK: - Additional input synthesis + +func postRightClick(at point: CGPoint, pid: pid_t?) { + CursorOverlay.shared.press(at: point) + let src = CGEventSource(stateID: .combinedSessionState) + post(CGEvent(mouseEventSource: src, mouseType: .rightMouseDown, mouseCursorPosition: point, mouseButton: .right), to: pid) + post(CGEvent(mouseEventSource: src, mouseType: .rightMouseUp, mouseCursorPosition: point, mouseButton: .right), to: pid) +} + +func postDrag(from start: CGPoint, to end: CGPoint, pid: pid_t?) { + let src = CGEventSource(stateID: .combinedSessionState) + // Deliver a move to the press location first: many views only begin drag + // tracking when the press arrives where the pointer already is, and without + // it the gesture degrades into a plain click. + // + // When targeting a pid this is a synthetic move sent to that app only, so + // the user's real cursor stays put. Only the no-pid fallback warps it. + if pid == nil { + CGWarpMouseCursorPosition(start) + } + post(CGEvent(mouseEventSource: src, mouseType: .mouseMoved, mouseCursorPosition: start, mouseButton: .left), to: pid) + usleep(80_000) + post(CGEvent(mouseEventSource: src, mouseType: .leftMouseDown, mouseCursorPosition: start, mouseButton: .left), to: pid) + usleep(80_000) + // Interpolate: a single jump often reads as a click, since many views need + // intermediate drag events to start tracking. + let steps = 24 + for i in 1...steps { + let t = Double(i) / Double(steps) + let point = CGPoint(x: start.x + (end.x - start.x) * t, y: start.y + (end.y - start.y) * t) + post(CGEvent(mouseEventSource: src, mouseType: .leftMouseDragged, mouseCursorPosition: point, mouseButton: .left), to: pid) + usleep(15_000) + } + usleep(80_000) + post(CGEvent(mouseEventSource: src, mouseType: .leftMouseUp, mouseCursorPosition: end, mouseButton: .left), to: pid) +} + +// MARK: - Additional tools + +func resolvePoint(_ args: [String: Any], xKey: String, yKey: String, idKey: String) -> Result { + if let id = args[idKey] as? String { + guard let el = Registry.get(id) else { + return .failure("error: unknown element_id \(id) — call get_app_state again to refresh ids") + } + guard let point = visibleCenter(of: el) else { + return .failure( + "error: \(id) is not visible in its window — scroll it into view and call get_app_state again" + ) + } + return .success(point) + } + if let x = args[xKey] as? Double, let y = args[yKey] as? Double { + guard x.isFinite, y.isFinite, + Int(exactly: x.rounded(.towardZero)) != nil, + Int(exactly: y.rounded(.towardZero)) != nil else { + return .failure("error: coordinates must be finite and representable as integers") + } + return .success(CGPoint(x: x, y: y)) + } + return .failure("error: provide either \(idKey), or both \(xKey) and \(yKey)") +} + +func toolRightClick(_ args: [String: Any]) -> String { + let point: CGPoint + switch resolvePoint(args, xKey: "x", yKey: "y", idKey: "element_id") { + case .failure(let message): + return message + case .success(let resolved): + point = resolved + } + let element = (args["element_id"] as? String).flatMap { Registry.get($0) } + let target: WindowTarget? + if let element { + target = windowTarget(for: element) + } else { + switch resolveTargetPid(args) { + case .failure(let message): + return message + case .success(let pid): + target = pid.flatMap { windowTarget(forPid: $0, containing: point) } + } + } + if let target, backgroundRightClick(target, at: point) { + return "right-clicked at (\(Int(point.x)), \(Int(point.y))) in background" + } + postRightClick(at: point, pid: nil) + return "right-clicked at (\(Int(point.x)), \(Int(point.y))) via cursor" +} + +func toolDrag(_ args: [String: Any]) -> String { + let start: CGPoint + switch resolvePoint(args, xKey: "from_x", yKey: "from_y", idKey: "from_element_id") { + case .failure(let message): + return message + case .success(let resolved): + start = resolved + } + let end: CGPoint + switch resolvePoint(args, xKey: "to_x", yKey: "to_y", idKey: "to_element_id") { + case .failure(let message): + return message + case .success(let resolved): + end = resolved + } + let element = (args["from_element_id"] as? String).flatMap { Registry.get($0) } + let underStart = windowTarget(under: start) + let target: WindowTarget? + if let element { + target = windowTarget(for: element) + } else if let query = args["app"] as? String { + // Resolve the named app directly — never fall through to Registry.targetPid. + guard let resolved = resolveApp(query) else { + return "error: no running app matching \(query)" + } + let appPid = resolved.app.processIdentifier + target = underStart.flatMap { $0.pid == appPid ? $0 : nil } + ?? windowTarget(forPid: appPid, containing: start) + } else { + target = underStart + } + // Reject element→element drags across windows even when the destination + // center still lies inside the source frame (overlapping windows). + if let target, + let toElementID = args["to_element_id"] as? String, + let destinationElement = Registry.get(toElementID), + let destination = windowTarget(for: destinationElement), + destination.pid != target.pid || destination.wid != target.wid + { + return "error: cross-window drag is not supported — keep the drag inside one window" + } + // Only treat as cross-window when the endpoint is outside the source frame. + // `windowTarget(under:)` is frontmost-first, so using it for every drag would + // reject legitimate background drags under an occluding window. + if let target, !target.frame.contains(end) { + if let dest = windowTarget(under: end), dest.wid != target.wid || dest.pid != target.pid { + return "error: cross-window drag is not supported — keep the drag inside one window" + } + if windowTarget(under: end) == nil { + return "error: drag destination is outside the source window" + } + } + if let target, backgroundDrag(target, from: start, to: end) { + return "dragged from (\(Int(start.x)), \(Int(start.y))) to (\(Int(end.x)), \(Int(end.y))) in background" + } + postDrag(from: start, to: end, pid: nil) + return "dragged from (\(Int(start.x)), \(Int(start.y))) to (\(Int(end.x)), \(Int(end.y))) via cursor" +} + +func toolSetValue(_ args: [String: Any]) -> String { + guard let id = args["element_id"] as? String else { return "error: missing required argument 'element_id'" } + guard let value = args["value"] as? String else { return "error: missing required argument 'value'" } + guard let el = Registry.get(id) else { return "error: unknown element_id \(id)" } + // Setting AXValue replaces field contents atomically, which is far more + // reliable than select-all-then-type for long strings. + let err = AXUIElementSetAttributeValue(el, kAXValueAttribute as CFString, value as CFString) + if err != .success { + return "error: could not set value on \(id) (AX error \(err.rawValue)); try click + type_text instead" + } + return "set \(id) to \(value.count) characters" +} + +func toolSelectText(_ args: [String: Any]) -> String { + guard let id = args["element_id"] as? String else { return "error: missing required argument 'element_id'" } + guard let el = Registry.get(id) else { return "error: unknown element_id \(id)" } + + let text = axString(el, kAXValueAttribute as String) ?? "" + let start = (args["start"] as? Int) ?? 0 + let length = (args["length"] as? Int) ?? max(0, text.count - start) + var range = CFRange(location: start, length: length) + guard let axRange = AXValueCreate(.cfRange, &range) else { return "error: could not build range" } + + let err = AXUIElementSetAttributeValue(el, kAXSelectedTextRangeAttribute as CFString, axRange) + if err != .success { return "error: could not select text on \(id) (AX error \(err.rawValue))" } + let selected = axString(el, kAXSelectedTextAttribute as String) ?? "" + return "selected \(selected.count) characters in \(id)" +} + +// MARK: - Chrome agent window +// +// The agent gets its own Chrome window and only ever drives tabs inside it, so +// the user can keep browsing their own tabs undisturbed. Tab management goes +// through Chrome's scripting interface (the same surface a browser extension +// would use); page interaction stays on the AX + SkyLight path, addressed to the +// agent window's id, so it never touches the user's window. + +/// A script result. `Result` is not used because its failure type must conform +/// to `Error`, and these are human-readable messages headed straight into a +/// tool response. +enum ScriptOutcome { + case success(String) + case failure(String) +} + +enum WindowOutcome { + case success(Int) + case failure(String) +} + +enum Chrome { + /// Persisted so a restarted server reattaches to the same window instead of + /// stranding it and opening another. MCP servers are spawned per session; + /// the browser window outlives them. + static let stateURL: URL = { + let base = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask).first + ?? URL(fileURLWithPath: NSTemporaryDirectory()) + let dir = base.appendingPathComponent("t3-desktop-mcp", isDirectory: true) + try? FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + return dir.appendingPathComponent("agent-window") + }() + + private static var cachedWindowID: Int? + private static var cachedChromePid: pid_t? + /// Process start time for `cachedChromePid` — PIDs alone are reusable after relaunch. + private static var cachedChromeLaunch: TimeInterval? + private static var didLoadState = false + + private static func chromePid() -> pid_t? { + NSWorkspace.shared.runningApplications + .first(where: { $0.bundleIdentifier == "com.google.Chrome" })? + .processIdentifier + } + + private static func chromeApp(pid: pid_t) -> NSRunningApplication? { + NSWorkspace.shared.runningApplications.first { + $0.processIdentifier == pid && $0.bundleIdentifier == "com.google.Chrome" + } + } + + private static func launchInterval(for app: NSRunningApplication) -> TimeInterval? { + app.launchDate?.timeIntervalSince1970 + } + + private static func loadState() { + guard !didLoadState else { return } + didLoadState = true + guard + let data = try? Data(contentsOf: stateURL), + let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let windowId = object["windowId"] as? Int, + let chromePid = object["chromePid"] as? Int, + chromePid >= Int(pid_t.min), chromePid <= Int(pid_t.max) + else { + // Legacy plain-integer files from older builds are intentionally + // discarded: a reused window id after Chrome restart is unsafe. + // Out-of-range chromePid would trap on pid_t conversion. + try? FileManager.default.removeItem(at: stateURL) + cachedWindowID = nil + cachedChromePid = nil + cachedChromeLaunch = nil + return + } + cachedWindowID = windowId + cachedChromePid = pid_t(chromePid) + cachedChromeLaunch = object["chromeLaunch"] as? TimeInterval + } + + private static func persistState() { + guard let windowId = cachedWindowID, let chromePid = cachedChromePid else { + try? FileManager.default.removeItem(at: stateURL) + return + } + var payload: [String: Any] = ["windowId": windowId, "chromePid": Int(chromePid)] + if let launch = cachedChromeLaunch { + payload["chromeLaunch"] = launch + } + guard let data = try? JSONSerialization.data(withJSONObject: payload) else { return } + try? data.write(to: stateURL, options: .atomic) + } + + static var agentWindowID: Int? { + get { + withStateLock { + loadState() + return cachedWindowID + } + } + set { + withStateLock { + didLoadState = true + cachedWindowID = newValue + if let id = newValue { + // Prefer the Chrome process that owns this window, not the first + // com.google.Chrome in the process list (multi-instance safe). + if let frame = boundsOf(id), let match = axWindow(matching: frame) { + cachedChromePid = match.pid + if let app = chromeApp(pid: match.pid) { + cachedChromeLaunch = launchInterval(for: app) + } else { + cachedChromeLaunch = nil + } + } else { + cachedChromePid = nil + cachedChromeLaunch = nil + } + } else { + cachedChromePid = nil + cachedChromeLaunch = nil + } + persistState() + } + } + } + + private static func clearAgentWindowState() { + cachedWindowID = nil + cachedChromePid = nil + cachedChromeLaunch = nil + try? FileManager.default.removeItem(at: stateURL) + } + + /// The stored id, or nil if that window (or this Chrome instance) is gone. + /// Caller must hold the agent-window state lock. + private static func liveAgentWindowIDLocked() -> Int? { + loadState() + guard let id = cachedWindowID else { return nil } + guard let expectedPid = cachedChromePid, let app = chromeApp(pid: expectedPid) else { + clearAgentWindowState() + return nil + } + guard let expectedLaunch = cachedChromeLaunch, + let liveLaunch = launchInterval(for: app), + abs(expectedLaunch - liveLaunch) <= 0.5 + else { + clearAgentWindowState() + return nil + } + guard windowExists(id) else { + clearAgentWindowState() + return nil + } + if let frame = boundsOf(id), + let match = axWindow(matching: frame, pid: expectedPid) + { + cachedChromePid = match.pid + cachedChromeLaunch = launchInterval(for: app) + return id + } + // AX can miss briefly while AppleScript still sees the window — keep it + // so Computer Use does not drop ownership and spawn orphans on retry. + return id + } + + /// The stored id, or nil if that window (or this Chrome instance) is gone. + static func liveAgentWindowID() -> Int? { + withStateLock { liveAgentWindowIDLocked() } + } + + /// NSAppleScript is not thread-safe and the JSON-RPC loop runs off-main. + static func run(_ source: String) -> ScriptOutcome { + var result: ScriptOutcome = .failure("script did not run") + let work = { + guard let script = NSAppleScript(source: source) else { + result = .failure("could not compile script") + return + } + var error: NSDictionary? + let value = script.executeAndReturnError(&error) + if let error { + result = .failure((error[NSAppleScript.errorMessage] as? String) ?? "\(error)") + } else { + result = .success(value.stringValue ?? "") + } + } + if Thread.isMainThread { work() } else { DispatchQueue.main.sync(execute: work) } + return result + } + + /// Run browser work without leaving Chrome in front. Chrome raises itself on + /// window creation and on tab changes, so every browser tool restores the + /// app the user was in and pushes the agent window back down the stack. + static func preservingFocus(_ body: () -> T) -> T { + let previous = NSWorkspace.shared.frontmostApplication + let previousWindow = frontWindowID() + let result = body() + if let previousWindow, previousWindow != agentWindowID { + raiseWindow(previousWindow) + } + if let previous, + previous.processIdentifier != NSWorkspace.shared.frontmostApplication?.processIdentifier { + DispatchQueue.main.sync { previous.activate(options: []) } + usleep(220_000) + } + return result + } + + /// Chrome's scripting `index` property does not actually reorder windows, so + /// the user's window is brought back to the front with the accessibility + /// raise action instead. That reorders within Chrome without activating it. + static func raiseWindow(_ id: Int) { + guard let frame = boundsOf(id), let match = axWindow(matching: frame) else { return } + AXUIElementPerformAction(match.element, kAXRaiseAction as CFString) + } + + static func frontWindowID() -> Int? { + guard case .success(let s) = run(""" + tell application "Google Chrome" + if (count windows) is 0 then return "" + return (id of window 1) as string + end tell + """) else { return nil } + return Int(s.trimmingCharacters(in: CharacterSet.whitespacesAndNewlines)) + } + + static func windowExists(_ id: Int) -> Bool { + if case .success(let s) = run(""" + tell application "Google Chrome" to return (exists window id \(id)) as string + """) { return s == "true" } + return false + } + + /// Cross-process lock around agent-window state so concurrent MCP servers + /// cannot each create a window after both observing a missing one. + private static func withStateLock(_ body: () -> T) -> T { + let lockPath = stateURL.path + ".lock" + let lockFd = open(lockPath, O_CREAT | O_RDWR, 0o600) + guard lockFd >= 0 else { return body() } + _ = flock(lockFd, LOCK_EX) + defer { + flock(lockFd, LOCK_UN) + close(lockFd) + } + return body() + } + + /// Return the agent's window id, creating the window if needed. + static func ensureAgentWindow() -> WindowOutcome { + withStateLock { + // Another MCP process may have created and persisted a window while + // this process held a stale in-memory cache — reload under the lock. + didLoadState = false + if let id = liveAgentWindowIDLocked() { return .success(id) } + + let created = run(""" + tell application "Google Chrome" + set w to make new window + return id of w as string + end tell + """) + switch created { + case .failure(let e): return .failure(e) + case .success(let s): + guard let id = Int(s.trimmingCharacters(in: CharacterSet.whitespacesAndNewlines)) else { + return .failure("unexpected window id: \(s)") + } + didLoadState = true + cachedWindowID = id + if let frame = boundsOf(id), let match = axWindow(matching: frame) { + cachedChromePid = match.pid + if let app = chromeApp(pid: match.pid) { + cachedChromeLaunch = launchInterval(for: app) + } else { + cachedChromeLaunch = nil + } + } else { + // Close the orphan so the next retry does not create another window. + _ = run("tell application \"Google Chrome\" to close window id \(id)") + clearAgentWindowState() + return .failure( + "created agent window \(id) but could not pair it with accessibility — retry ensureAgentWindow" + ) + } + persistState() + return .success(id) + } + } + } + + /// Screen frame of the agent window, used to pair it with its AX window. + static func agentWindowFrame() -> CGRect? { + guard let id = liveAgentWindowID() else { return nil } + return boundsOf(id) + } + + static func boundsOf(_ id: Int) -> CGRect? { + guard case .success(let s) = run(""" + tell application "Google Chrome" + set b to bounds of window id \(id) + return ((item 1 of b) as string) & "," & ((item 2 of b) as string) & "," ¬ + & ((item 3 of b) as string) & "," & ((item 4 of b) as string) + end tell + """) else { return nil } + let parts = s.split(separator: ",").compactMap { Double($0.trimmingCharacters(in: CharacterSet.whitespaces)) } + guard parts.count == 4 else { return nil } + return CGRect(x: parts[0], y: parts[1], width: parts[2] - parts[0], height: parts[3] - parts[1]) + } + + /// The AX window for the agent's Chrome window. + /// + /// Chrome's scripting ids and accessibility elements are separate worlds with + /// no shared handle, so they are paired by screen position — the closest + /// origin wins, which is unambiguous unless two windows are exactly stacked. + static func agentAXWindow() -> (element: AXUIElement, pid: pid_t)? { + guard let frame = agentWindowFrame() else { return nil } + loadState() + return axWindow(matching: frame, pid: cachedChromePid) + } + + /// Pair a scripting window with its accessibility element by screen frame. + /// Chrome cascades new windows only ~28px apart, so origin alone is not + /// enough to tell them apart — size is folded into the distance and the + /// tolerance is tight. When `pid` is set, only that Chrome process is searched. + static func axWindow(matching frame: CGRect, pid: pid_t? = nil) -> (element: AXUIElement, pid: pid_t)? { + var best: (AXUIElement, pid_t, CGFloat)? + var tied = false + for app in NSWorkspace.shared.runningApplications + where app.bundleIdentifier == "com.google.Chrome" { + if let pid, app.processIdentifier != pid { continue } + let ax = AXUIElementCreateApplication(app.processIdentifier) + for window in (axCopy(ax, kAXWindowsAttribute as String) as? [AXUIElement]) ?? [] { + guard let origin = axPoint(window, kAXPositionAttribute as String), + let size = axSize(window, kAXSizeAttribute as String) else { continue } + let distance = hypot(origin.x - frame.origin.x, origin.y - frame.origin.y) + + hypot(size.width - frame.width, size.height - frame.height) + if best == nil || distance + 0.5 < best!.2 { + best = (window, app.processIdentifier, distance) + tied = false + } else if let current = best, abs(distance - current.2) <= 0.5 { + tied = true + } + } + } + // Equal-distance matches are ambiguous (stacked / identical frames). + guard let best, !tied, best.2 < 12 else { return nil } + return (best.0, best.1) + } +} + +func toolBrowserOpenTab(_ args: [String: Any]) -> String { + let url = (args["url"] as? String) ?? "about:blank" + // The extension is the good path: it opens an inactive tab in a labelled + // group inside the user's own signed-in Chrome. Without it, fall back to a + // separate window driven through the accessibility API. + if BrowserBridge.shared.isConnected { + return bridgeText(BrowserBridge.shared.call("open_tab", ["url": url])) { payload in + "opened \(url) in the agent tab group (tab_id=\(payload["tabId"] as? Int ?? -1))" + } + } + return Chrome.preservingFocus { + switch Chrome.ensureAgentWindow() { + case .failure(let e): + return "error: could not open the agent window: \(e)" + case .success(let id): + let escaped = url + .replacingOccurrences(of: "\\", with: "\\\\") + .replacingOccurrences(of: "\"", with: "\\\"") + switch Chrome.run(""" + tell application "Google Chrome" + set w to window id \(id) + make new tab at end of tabs of w with properties {URL:"\(escaped)"} + set active tab index of w to (count tabs of w) + return ((count tabs of w) as string) + end tell + """) { + case .failure(let e): + return "error: \(e)" + case .success(let count): + let n = count.trimmingCharacters(in: CharacterSet.whitespacesAndNewlines) + return "opened \(url) as tab \(n) in the agent window (id \(id))" + } + } + } +} + +func toolBrowserListTabs(_ args: [String: Any]) -> String { + if BrowserBridge.shared.isConnected { + return bridgeText(BrowserBridge.shared.call("list_tabs"), describeTabs) + } + guard let id = Chrome.liveAgentWindowID() else { + return "no agent window yet — call browser_open_tab first" + } + return Chrome.preservingFocus { + switch Chrome.run(""" + tell application "Google Chrome" + set w to window id \(id) + set activeIndex to active tab index of w + set out to "" + repeat with i from 1 to (count tabs of w) + set t to tab i of w + set marker to " " + if i is activeIndex then set marker to "* " + set out to out & marker & (i as string) & ". " & (title of t) & " [" & (URL of t) & "]" & linefeed + end repeat + return out + end tell + """) { + case .failure(let e): + return "error: \(e)" + case .success(let s): + return "agent window \(id) (* = active):\n" + (s.isEmpty ? " (no tabs)" : s) + } + } +} + +func toolBrowserSelectTab(_ args: [String: Any]) -> String { + if BrowserBridge.shared.isConnected { + guard let tabId = args["tab_id"] as? Int ?? args["index"] as? Int else { + return "error: missing required argument 'tab_id'" + } + return bridgeText(BrowserBridge.shared.call("select_tab", ["tabId": tabId])) { _ in + "switched the agent group to tab \(tabId)" + } + } + guard let index = args["index"] as? Int else { return "error: missing required argument 'index'" } + guard let id = Chrome.liveAgentWindowID() else { + return "error: no agent window yet — call browser_open_tab first" + } + return Chrome.preservingFocus { + switch Chrome.run(""" + tell application "Google Chrome" + set w to window id \(id) + if \(index) < 1 or \(index) > (count tabs of w) then return "out of range" + set active tab index of w to \(index) + return title of active tab of w + end tell + """) { + case .failure(let e): + return "error: \(e)" + case .success(let title): + return title == "out of range" + ? "error: tab \(index) is out of range for the agent window" + : "switched the agent window to tab \(index): \(title)" + } + } +} + +func toolBrowserCloseTab(_ args: [String: Any]) -> String { + if BrowserBridge.shared.isConnected { + guard let tabId = args["tab_id"] as? Int ?? args["index"] as? Int else { + return "error: missing required argument 'tab_id'" + } + return bridgeText(BrowserBridge.shared.call("close_tab", ["tabId": tabId])) { _ in + "closed tab \(tabId)" + } + } + guard let index = args["index"] as? Int else { return "error: missing required argument 'index'" } + guard let id = Chrome.liveAgentWindowID() else { + return "error: no agent window yet" + } + return Chrome.preservingFocus { + switch Chrome.run(""" + tell application "Google Chrome" + set w to window id \(id) + if \(index) < 1 or \(index) > (count tabs of w) then return "out of range" + close tab \(index) of w + return "ok" + end tell + """) { + case .failure(let e): + return "error: \(e)" + case .success(let s): + return s == "out of range" ? "error: tab \(index) is out of range" : "closed tab \(index)" + } + } +} + + +// MARK: - Browser tools over the extension + +/// Render a bridge reply as tool text, or the failure as an error line. +func bridgeText(_ result: BridgeOutcome, _ describe: ([String: Any]) -> String) -> String { + switch result { + case .failure(let message): return "error: \(message)" + case .success(let payload): return describe(payload) + } +} + +func toolBrowserSnapshot(_ args: [String: Any]) -> String { + guard let tabId = args["tab_id"] as? Int else { return "error: missing required argument 'tab_id'" } + return bridgeText(BrowserBridge.shared.call("snapshot", ["tabId": tabId])) { payload in + let elements = payload["elements"] as? [[String: Any]] ?? [] + var lines = ["\(payload["title"] as? String ?? "?") [\(payload["url"] as? String ?? "")]"] + for element in elements { + let index = element["i"] as? Int ?? -1 + let tag = element["tag"] as? String ?? "?" + let label = element["label"] as? String ?? "" + let offscreen = (element["inView"] as? Bool == false) ? " (scrolled out of view)" : "" + lines.append(" [\(index)] \(tag)\(label.isEmpty ? "" : " \"\(label)\"")\(offscreen)") + } + return lines.joined(separator: "\n") + } +} + +func toolBrowserClick(_ args: [String: Any]) -> String { + guard let tabId = args["tab_id"] as? Int else { return "error: missing required argument 'tab_id'" } + var params: [String: Any] = ["tabId": tabId] + if let index = args["index"] as? Int { + params["index"] = index + } else if let x = args["x"] as? Double, let y = args["y"] as? Double { + params["x"] = x + params["y"] = y + } else { + return "error: provide either index (from browser_snapshot), or both x and y" + } + // The Chrome extension paints the same agent pointer into the page. Keep + // that as the source of truth for tab clicks — background tabs are not + // composited, so a desktop overlay at guessed screen coords would lie. + return bridgeText(BrowserBridge.shared.call("click", params)) { payload in + var line = "clicked in tab \(tabId)" + if let cursor = payload["cursor"] as? [String: Any] { + if cursor["ok"] as? Bool == true { + let glow = cursor["hasGlow"] as? Bool == true ? "glow" : "no-glow" + let fill = cursor["darkFill"] as? Bool == true ? "dark-fill" : "fill" + line += " (pointer \(glow), \(fill))" + } else if let reason = cursor["reason"] as? String { + line += " (pointer missing: \(reason))" + } + } + return line + } +} + +func toolBrowserType(_ args: [String: Any]) -> String { + guard let tabId = args["tab_id"] as? Int else { return "error: missing required argument 'tab_id'" } + guard let text = args["text"] as? String else { return "error: missing required argument 'text'" } + return bridgeText(BrowserBridge.shared.call("type", ["tabId": tabId, "text": text])) { _ in + "typed \(text.count) characters into tab \(tabId)" + } +} + +func toolBrowserPressKey(_ args: [String: Any]) -> String { + guard let tabId = args["tab_id"] as? Int else { return "error: missing required argument 'tab_id'" } + guard let key = args["key"] as? String else { return "error: missing required argument 'key'" } + return bridgeText(BrowserBridge.shared.call("press", ["tabId": tabId, "key": key])) { _ in + "pressed \(key) in tab \(tabId)" + } +} + +func toolBrowserCloseAllTabs(_ args: [String: Any]) -> String { + guard BrowserBridge.shared.isConnected else { + return "error: the T3 Code Chrome extension is not connected" + } + return bridgeText(BrowserBridge.shared.call("close_all_tabs")) { payload in + let closed = payload["closed"] as? Int ?? 0 + return closed == 0 + ? "nothing to clean up — the agent had no tabs open" + : "closed \(closed) agent tab\(closed == 1 ? "" : "s") and removed the tab group" + } +} + +func toolBrowserNavigate(_ args: [String: Any]) -> String { + guard let tabId = args["tab_id"] as? Int else { return "error: missing required argument 'tab_id'" } + guard let url = args["url"] as? String else { return "error: missing required argument 'url'" } + return bridgeText(BrowserBridge.shared.call("navigate", ["tabId": tabId, "url": url])) { _ in + "navigated tab \(tabId) to \(url)" + } +} + +func describeTabs(_ payload: [String: Any]) -> String { + let tabs = payload["tabs"] as? [[String: Any]] ?? [] + if tabs.isEmpty { return "the agent has no tabs open yet — call browser_open_tab" } + var lines = ["agent tab group (\(tabs.count) tab\(tabs.count == 1 ? "" : "s")):"] + for tab in tabs { + let marker = (tab["active"] as? Bool == true) ? "* " : " " + lines.append("\(marker)tab_id=\(tab["tabId"] as? Int ?? -1) \(tab["title"] as? String ?? "")" + + " [\(tab["url"] as? String ?? "")]") + } + return lines.joined(separator: "\n") +} + +// MARK: - Tool schemas + +func obj(_ d: [String: Any]) -> [String: Any] { d } + +let toolDefs: [[String: Any]] = [ + [ + "name": "list_apps", + "description": "List running applications with their bundle id, pid, window count, and which is frontmost. Note that one app can have several running instances and only some may own windows.", + "inputSchema": ["type": "object", "properties": [:] as [String: Any]], + ], + [ + "name": "get_app_state", + "description": "Read an app's accessibility tree as an indented outline. Interactive elements are prefixed with an id like [e12] that you pass to click/type_text/scroll. Call this before interacting, and again after the UI changes, since ids are per-snapshot.", + "inputSchema": [ + "type": "object", + "properties": [ + "app": ["type": "string", "description": "App name, bundle id, or pid"], + "max_depth": ["type": "integer", "description": "Max tree depth (default 18)"], + "max_elements": ["type": "integer", "description": "Max elements to emit (default 800)"], + "window": ["description": "Limit to one window: a 0-based index, or \"agent\" for the browser window this agent owns"], + ], + "required": ["app"], + ], + ], + [ + "name": "click", + "description": "Click an element by element_id (preferred, uses the accessibility press action) or at absolute screen coordinates.", + "inputSchema": [ + "type": "object", + "properties": [ + "element_id": ["type": "string", "description": "Element id from get_app_state, e.g. e12"], + "x": ["type": "number"], "y": ["type": "number"], + "click_count": ["type": "integer", "description": "1 for single, 2 for double-click"], + ], + ], + ], + [ + "name": "type_text", + "description": "Type literal text into the focused element, optionally focusing element_id first.", + "inputSchema": [ + "type": "object", + "properties": [ + "text": ["type": "string"], + "element_id": ["type": "string", "description": "Focus this element before typing"], + ], + "required": ["text"], + ], + ], + [ + "name": "press_key", + "description": "Press a named key with optional modifiers, e.g. key='s' modifiers=['cmd'] to save, or key='return'.", + "inputSchema": [ + "type": "object", + "properties": [ + "key": ["type": "string"], + "modifiers": ["type": "array", "items": ["type": "string"], + "description": "Any of cmd, shift, alt, ctrl, fn"], + ], + "required": ["key"], + ], + ], + [ + "name": "scroll", + "description": "Scroll up, down, left, or right, optionally positioning the cursor over element_id first.", + "inputSchema": [ + "type": "object", + "properties": [ + "direction": ["type": "string", "enum": ["up", "down", "left", "right"]], + "amount": ["type": "integer", "description": "Scroll lines (default 5)"], + "element_id": ["type": "string"], + ], + ], + ], + [ + "name": "activate_app", + "description": "Bring an app to the foreground.", + "inputSchema": [ + "type": "object", + "properties": ["app": ["type": "string"]], + "required": ["app"], + ], + ], + [ + "name": "screenshot", + "description": "Capture the app's largest window as a PNG image. Prefer get_app_state for interaction, which is cheaper and gives clickable element ids; use a screenshot when you need to see rendered content the accessibility tree does not describe, such as canvas or video.", + "inputSchema": [ + "type": "object", + "properties": [ + "app": ["type": "string", "description": "App name, bundle id, or pid"], + "display": ["type": "integer", "description": "Capture a whole display by index (see list_displays) instead of an app window"], + "max_width": ["type": "integer", "description": "Downscale to this width in pixels (default 1400)"], + ], + ], + ], + [ + "name": "list_displays", + "description": "List every attached display with its index, resolution and position, for use with screenshot(display: N).", + "inputSchema": ["type": "object", "properties": [:] as [String: Any]], + ], + [ + "name": "right_click", + "description": "Right-click (secondary click) an element or screen position to open a context menu.", + "inputSchema": [ + "type": "object", + "properties": [ + "element_id": ["type": "string"], + "x": ["type": "number"], "y": ["type": "number"], + ], + ], + ], + [ + "name": "drag", + "description": "Press at one point, drag, and release at another. Accepts element ids or coordinates on each end.", + "inputSchema": [ + "type": "object", + "properties": [ + "from_element_id": ["type": "string"], "to_element_id": ["type": "string"], + "from_x": ["type": "number"], "from_y": ["type": "number"], + "to_x": ["type": "number"], "to_y": ["type": "number"], + ], + ], + ], + [ + "name": "set_value", + "description": "Replace a text field's contents directly. More reliable than select-all-then-type for long values, though some fields reject it and need click + type_text.", + "inputSchema": [ + "type": "object", + "properties": [ + "element_id": ["type": "string"], + "value": ["type": "string"], + ], + "required": ["element_id", "value"], + ], + ], + [ + "name": "browser_open_tab", + "description": "Open a URL in a new background tab inside the agent's own labelled tab group, in the user's signed-in Chrome. The user keeps browsing their tabs undisturbed. Returns a tab_id for browser_snapshot / browser_click.", + "inputSchema": [ + "type": "object", + "properties": ["url": ["type": "string", "description": "URL to open (default about:blank)"]], + ], + ], + [ + "name": "browser_list_tabs", + "description": "List the tabs in the agent's own Chrome window, marking the active one.", + "inputSchema": ["type": "object", "properties": [:] as [String: Any]], + ], + [ + "name": "browser_select_tab", + "description": "Make one of the agent's tabs the visible one. Does not affect the user's tabs.", + "inputSchema": [ + "type": "object", + "properties": [ + "tab_id": ["type": "integer", "description": "From browser_list_tabs"], + "index": ["type": "integer", "description": "1-based index, fallback mode only"], + ], + ], + ], + [ + "name": "browser_close_tab", + "description": "Close one of the agent's tabs.", + "inputSchema": [ + "type": "object", + "properties": [ + "tab_id": ["type": "integer", "description": "From browser_list_tabs"], + "index": ["type": "integer", "description": "1-based index, fallback mode only"], + ], + ], + ], + [ + "name": "browser_snapshot", + "description": "List the interactive elements on a page in one of the agent's tabs, with indices to pass to browser_click. Works on a background tab, so the user can be looking at something else.", + "inputSchema": [ + "type": "object", + "properties": ["tab_id": ["type": "integer", "description": "From browser_open_tab or browser_list_tabs"]], + "required": ["tab_id"], + ], + ], + [ + "name": "browser_click", + "description": "Click in one of the agent's tabs, by element index from browser_snapshot or by page coordinates. Works on a background tab.", + "inputSchema": [ + "type": "object", + "properties": [ + "tab_id": ["type": "integer"], + "index": ["type": "integer", "description": "Element index from browser_snapshot"], + "x": ["type": "number"], "y": ["type": "number"], + ], + "required": ["tab_id"], + ], + ], + [ + "name": "browser_type", + "description": "Type text into the focused field of one of the agent's tabs. Click the field first.", + "inputSchema": [ + "type": "object", + "properties": ["tab_id": ["type": "integer"], "text": ["type": "string"]], + "required": ["tab_id", "text"], + ], + ], + [ + "name": "browser_press_key", + "description": "Press Enter, Tab, Escape or Backspace in one of the agent's tabs.", + "inputSchema": [ + "type": "object", + "properties": [ + "tab_id": ["type": "integer"], + "key": ["type": "string", "enum": ["Enter", "Tab", "Escape", "Backspace"]], + ], + "required": ["tab_id", "key"], + ], + ], + [ + "name": "browser_close_all_tabs", + "description": "Close every tab the agent opened and remove its tab group. Call this when finished with the browser so no empty group is left in the user's tab strip.", + "inputSchema": ["type": "object", "properties": [:] as [String: Any]], + ], + [ + "name": "browser_navigate", + "description": "Point one of the agent's tabs at a different URL.", + "inputSchema": [ + "type": "object", + "properties": ["tab_id": ["type": "integer"], "url": ["type": "string"]], + "required": ["tab_id", "url"], + ], + ], + [ + "name": "select_text", + "description": "Select a character range inside a text element. Defaults to selecting from 'start' to the end of the value.", + "inputSchema": [ + "type": "object", + "properties": [ + "element_id": ["type": "string"], + "start": ["type": "integer", "description": "Start offset (default 0)"], + "length": ["type": "integer", "description": "Characters to select (default: to end)"], + ], + "required": ["element_id"], + ], + ], +] + +func advertisedToolDefs() -> [[String: Any]] { + if browserControlEnabled { return toolDefs } + return toolDefs.filter { tool in + guard let name = tool["name"] as? String else { return true } + return !name.hasPrefix("browser_") + } +} + +func dispatch(_ name: String, _ args: [String: Any]) -> String { + if name.hasPrefix("browser_"), !browserControlEnabled { + return "error: browser control is disabled in Computer Use settings" + } + switch name { + case "list_apps": return toolListApps() + case "get_app_state": return toolGetAppState(args) + case "click": return toolClick(args) + case "type_text": return toolTypeText(args) + case "press_key": return toolPressKey(args) + case "scroll": return toolScroll(args) + case "activate_app": return toolActivateApp(args) + case "list_displays": return toolListDisplays(args) + case "right_click": return toolRightClick(args) + case "drag": return toolDrag(args) + case "set_value": return toolSetValue(args) + case "select_text": return toolSelectText(args) + case "browser_open_tab": return toolBrowserOpenTab(args) + case "browser_list_tabs": return toolBrowserListTabs(args) + case "browser_select_tab": return toolBrowserSelectTab(args) + case "browser_close_tab": return toolBrowserCloseTab(args) + case "browser_snapshot": return toolBrowserSnapshot(args) + case "browser_click": return toolBrowserClick(args) + case "browser_type": return toolBrowserType(args) + case "browser_press_key": return toolBrowserPressKey(args) + case "browser_navigate": return toolBrowserNavigate(args) + case "browser_close_all_tabs": return toolBrowserCloseAllTabs(args) + default: return "error: unknown tool \(name)" + } +} + +// MARK: - Agent cursor overlay +// +// The drawing lives in the T3AgentCursor.app child (see AgentCursor.swift). +// This facade keeps the older call sites (`CursorOverlay.shared.press`) pointed +// at the bundle that actually puts a window up. + +final class CursorOverlay { + static let shared = CursorOverlay() + + /// Move the agent cursor to a Quartz screen point. + func show(at point: CGPoint) { AgentCursor.shared.show(at: point) } + + /// Move the agent pointer. + func press(at point: CGPoint) { AgentCursor.shared.press(at: point) } + + /// Non-blocking hop for mid-drag visuals. + func glide(at point: CGPoint) { AgentCursor.shared.glide(at: point) } +} + +// MARK: - JSON-RPC over stdio + +func send(_ payload: [String: Any]) { + guard let data = try? JSONSerialization.data(withJSONObject: payload), + let line = String(data: data, encoding: .utf8) else { return } + print(line) + fflush(stdout) +} + +func respond(id: Any, result: [String: Any]) { + send(["jsonrpc": "2.0", "id": id, "result": result]) +} + +func respondError(id: Any, code: Int, message: String) { + send(["jsonrpc": "2.0", "id": id, "error": ["code": code, "message": message]]) +} + +func textResult(_ s: String, isError: Bool = false) -> [String: Any] { + ["content": [["type": "text", "text": s]], "isError": isError] +} + +// Chrome launches this same binary as its native messaging host; in that mode +// it is a relay, not an MCP server. +if CommandLine.arguments.contains("native-host") { NativeHost.run() } +// Computer History background recorder (Skysight-style interaction events). +if CommandLine.arguments.contains("computer-history") { + let args = CommandLine.arguments + if let flag = args.firstIndex(of: "--root"), args.index(after: flag) < args.endIndex { + ComputerHistoryDaemon.run(root: args[args.index(after: flag)]) + } + fputs("t3-desktop-mcp: computer-history requires --root \n", stderr) + exit(2) +} +// The agent pointer is a separate LSUIElement .app (see AgentCursor.swift) +// launched via NSWorkspace with `--socket ` for move/hide commands. +if CommandLine.arguments.contains("cursor-overlay") { + let args = CommandLine.arguments + if let flag = args.firstIndex(of: "--socket"), args.index(after: flag) < args.endIndex { + AgentCursorOverlay.run(socketPath: args[args.index(after: flag)]) + } + fputs("t3-desktop-mcp: cursor-overlay requires --socket \n", stderr) + exit(2) +} + +BrowserBridge.shared.start() + +// ScreenCaptureKit talks to the window server, which asserts (did_initialize) +// unless the process has been initialised as a GUI app. `.accessory` keeps it +// out of the Dock and app switcher while still allowing the cursor overlay +// panel; `.prohibited` would forbid windows entirely. +_ = NSApplication.shared +NSApp.setActivationPolicy(.accessory) + +setvbuf(stdout, nil, _IOLBF, 0) + +// The JSON-RPC loop blocks on readLine, so it cannot own the main thread: AppKit +// needs the main run loop to draw the overlay. Requests are handled on a +// background queue and UI work hops back to main. +func runJSONRPCLoop() { +while let line = readLine(strippingNewline: true) { + if line.trimmingCharacters(in: CharacterSet.whitespaces).isEmpty { continue } + guard let data = line.data(using: .utf8), + let msg = (try? JSONSerialization.jsonObject(with: data)) as? [String: Any], + let method = msg["method"] as? String else { continue } + + let id = msg["id"] + + switch method { + case "initialize": + respond(id: id ?? NSNull(), result: [ + "protocolVersion": "2024-11-05", + "capabilities": ["tools": ["listChanged": false]], + "serverInfo": ["name": "t3-desktop", "version": "0.1.0"], + ]) + + case "tools/list": + respond(id: id ?? NSNull(), result: ["tools": advertisedToolDefs()]) + + case "tools/call": + // Pointer fade is keyed to Computer Use tool traffic: stay up while + // tools are in flight / chained, fade once the task stops calling. + do { + AgentCursor.shared.noteDesktopToolStarted() + defer { AgentCursor.shared.noteDesktopToolFinished() } + + guard let id else { break } + let params = msg["params"] as? [String: Any] ?? [:] + guard let name = params["name"] as? String else { + respondError(id: id, code: -32602, message: "missing tool name") + break + } + let args = params["arguments"] as? [String: Any] ?? [:] + + // Handled ahead of the Accessibility check: screen capture is gated by + // Screen Recording, a separate permission, so screenshots should still + // work if only that one is granted. + if name == "screenshot" { + if let display = args["display"] as? Int { + let maxWidth = (args["max_width"] as? Int) ?? 1400 + guard let shot = captureDisplayPNG(index: display, maxWidth: maxWidth) else { + respond(id: id, result: textResult( + "error: could not capture display \(display) — check Screen Recording " + + "permission, or call list_displays for valid indices.", isError: true)) + break + } + respond(id: id, result: [ + "content": [[ + "type": "image", "data": shot.data.base64EncodedString(), + "mimeType": "image/png", + ]], + "isError": false, + ]) + break + } + guard let query = args["app"] as? String, let resolved = resolveApp(query) else { + respond(id: id, result: textResult( + "error: no running app matching \(args["app"] as? String ?? "")", + isError: true)) + break + } + let maxWidth = (args["max_width"] as? Int) ?? 1400 + guard let png = captureWindowPNG(pid: resolved.app.processIdentifier, maxWidth: maxWidth) else { + respond(id: id, result: textResult( + "error: screen capture failed. The host app may be missing Screen Recording " + + "permission, or this app may have no on-screen window.", + isError: true)) + break + } + respond(id: id, result: [ + "content": [[ + "type": "image", + "data": png.base64EncodedString(), + "mimeType": "image/png", + ]], + "isError": false, + ]) + break + } + + // list_displays / browser_* do not need Accessibility — Screen + // Recording / Chrome bridge only. Keep them ahead of the AX gate so + // the Screen Recording-only flow can still recover (Bot finding). + if name == "list_displays" || name.hasPrefix("browser_") { + let out = dispatch(name, args) + respond(id: id, result: textResult(out, isError: out.hasPrefix("error:"))) + break + } + + if !AXIsProcessTrusted() { + respond(id: id, result: textResult( + "Accessibility permission is not granted to the host app. Enable it in " + + "System Settings → Privacy & Security → Accessibility, then restart the app.", + isError: true)) + break + } + let out = dispatch(name, args) + respond(id: id, result: textResult(out, isError: out.hasPrefix("error:"))) + } + + case "ping": + respond(id: id ?? NSNull(), result: [:]) + + case "notifications/cancelled": + // Host aborted the turn — drop the pointer immediately. + AgentCursor.shared.hide() + + default: + // Notifications carry no id and require no reply. + if let id { respondError(id: id, code: -32601, message: "method not found: \(method)") } + } +} + // stdin closed: the client is gone, so the process should follow. + AgentCursor.shared.hide() + exit(0) +} + +DispatchQueue.global(qos: .userInitiated).async { runJSONRPCLoop() } +NSApp.run() diff --git a/package.json b/package.json index 3fc66d0dd02..89b7968ad6f 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "build:marketing": "vp run --filter @t3tools/marketing build", "build:desktop": "vp run --filter @t3tools/desktop --filter t3 build", "build:resource-monitor": "cargo build --locked --release --manifest-path native/resource-monitor/Cargo.toml", + "build:desktop-mcp": "swift build -c release --arch arm64 --arch x86_64 --package-path native/t3-desktop-mcp", "typecheck": "vp run -r --concurrency-limit 2 typecheck", "tc": "vp run -r --concurrency-limit 2 typecheck", "lint": "vp lint --report-unused-disable-directives", diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index d872a422a51..909c44a8ad8 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -500,6 +500,48 @@ export const DesktopWslStateSchema = Schema.Struct({ preflightError: Schema.NullOr(Schema.String), }); +/** macOS Privacy panes Computer Use needs; other platforms report notRequired. */ +export const DesktopComputerUsePrivacyPaneSchema = Schema.Literals([ + "accessibility", + "screenRecording", +]); +export type DesktopComputerUsePrivacyPane = typeof DesktopComputerUsePrivacyPaneSchema.Type; + +export const DesktopComputerUsePermissionStatusSchema = Schema.Literals([ + "granted", + "denied", + "notDetermined", + "notRequired", + "unknown", +]); +export type DesktopComputerUsePermissionStatus = + typeof DesktopComputerUsePermissionStatusSchema.Type; + +export const DesktopComputerUsePermissionSchema = Schema.Struct({ + kind: DesktopComputerUsePrivacyPaneSchema, + status: DesktopComputerUsePermissionStatusSchema, + label: Schema.String, +}); +export type DesktopComputerUsePermission = typeof DesktopComputerUsePermissionSchema.Type; + +export const DesktopChromeExtensionStatusSchema = Schema.Literals([ + "installed", + "missing", + "unknown", +]); +export type DesktopChromeExtensionStatus = typeof DesktopChromeExtensionStatusSchema.Type; + +export const DesktopComputerUsePermissionsStateSchema = Schema.Struct({ + platform: Schema.Literals(["darwin", "win32", "linux", "other"]), + permissions: Schema.Array(DesktopComputerUsePermissionSchema), + chromeExtension: Schema.Struct({ + status: DesktopChromeExtensionStatusSchema, + detail: Schema.String, + }), +}); +export type DesktopComputerUsePermissionsState = + typeof DesktopComputerUsePermissionsStateSchema.Type; + /** * Renderer-facing snapshot of a desktop preview tab. Mirrors the main-process * PreviewTabState shape but uses serialisable primitives only. @@ -1086,6 +1128,17 @@ export interface DesktopBridge { * builds lack it; callers fall back to VS Code only. */ probeRemoteEditors?: () => Promise; + /** + * Computer Use TCC / extension readiness. Optional so older desktop builds + * still load the settings page without crashing. + */ + getComputerUsePermissions?: () => Promise; + /** + * Open the OS privacy pane for a Computer Use permission (macOS System + * Settings → Privacy & Security). Also prompts Accessibility trust when + * needed so the app appears in the list. + */ + openComputerUsePrivacySettings?: (pane: DesktopComputerUsePrivacyPane) => Promise; onMenuAction: (listener: (action: string) => void) => () => void; /** * Hold-to-quit hint pushes: "down" when the quit shortcut is first pressed, diff --git a/packages/contracts/src/orchestration.ts b/packages/contracts/src/orchestration.ts index cd9f3a74787..8fcf67eeaa1 100644 --- a/packages/contracts/src/orchestration.ts +++ b/packages/contracts/src/orchestration.ts @@ -127,7 +127,13 @@ export const DEFAULT_RUNTIME_MODE: RuntimeMode = "full-access"; export const ProviderInteractionMode = Schema.Literals(["default", "plan"]); export type ProviderInteractionMode = typeof ProviderInteractionMode.Type; export const DEFAULT_PROVIDER_INTERACTION_MODE: ProviderInteractionMode = "default"; -export const ProviderRequestKind = Schema.Literals(["command", "file-read", "file-change"]); +export const ProviderRequestKind = Schema.Literals([ + "command", + "file-read", + "file-change", + "tool", + "permissions", +]); export type ProviderRequestKind = typeof ProviderRequestKind.Type; export const AssistantDeliveryMode = Schema.Literals(["buffered", "streaming"]); export type AssistantDeliveryMode = typeof AssistantDeliveryMode.Type; diff --git a/packages/contracts/src/providerRuntime.ts b/packages/contracts/src/providerRuntime.ts index bd525e6542e..3bd2e970740 100644 --- a/packages/contracts/src/providerRuntime.ts +++ b/packages/contracts/src/providerRuntime.ts @@ -138,6 +138,8 @@ export const CanonicalRequestType = Schema.Literals([ "file_change_approval", "apply_patch_approval", "exec_command_approval", + "tool_approval", + "permissions_approval", "tool_user_input", "dynamic_tool_call", "auth_tokens_refresh", diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 22ce210ed89..3e5a9d383f5 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -541,6 +541,17 @@ export const BackgroundActivitySettings = Schema.Struct({ }).pipe(Schema.withDecodingDefault(Effect.succeed({}))); export type BackgroundActivitySettings = typeof BackgroundActivitySettings.Type; +/** Local desktop / browser computer-use MCP (`t3-desktop`). */ +export const DesktopControlSettings = Schema.Struct({ + /** When false, providers do not inject the t3-desktop MCP server. Default on. */ + enabled: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))), + /** Show the agent pointer overlay while controlling the desktop. */ + agentCursorEnabled: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))), + /** Allow browser_* tools via the Chrome extension bridge. */ + browserControlEnabled: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))), +}).pipe(Schema.withDecodingDefault(Effect.succeed({}))); +export type DesktopControlSettings = typeof DesktopControlSettings.Type; + export const ServerSettings = Schema.Struct({ // Legacy token-by-token assistant output. Deliberately a fresh key (was // `enableAssistantStreaming`): decoding drops the old key, so everyone, @@ -615,6 +626,10 @@ export const ServerSettings = Schema.Struct({ Schema.withDecodingDefault(Effect.succeed({})), ), observability: ObservabilitySettings.pipe(Schema.withDecodingDefault(Effect.succeed({}))), + // Local computer-use MCP (`t3-desktop`): agents can drive the desktop and an + // agent-owned Chrome tab group. Off disables injection even when the binary + // is present. Sub-flags are passed through to the MCP process as env. + desktopControl: DesktopControlSettings.pipe(Schema.withDecodingDefault(Effect.succeed({}))), }); export type ServerSettings = typeof ServerSettings.Type; @@ -741,6 +756,13 @@ export const ServerSettingsPatch = Schema.Struct({ otlpMetricsUrl: Schema.optionalKey(TrimmedString), }), ), + desktopControl: Schema.optionalKey( + Schema.Struct({ + enabled: Schema.optionalKey(Schema.Boolean), + agentCursorEnabled: Schema.optionalKey(Schema.Boolean), + browserControlEnabled: Schema.optionalKey(Schema.Boolean), + }), + ), providers: Schema.optionalKey( Schema.Struct({ codex: Schema.optionalKey(CodexSettingsPatch), diff --git a/packages/effect-codex-app-server/src/client.ts b/packages/effect-codex-app-server/src/client.ts index c0cb5b1dc23..a0fbadf89c8 100644 --- a/packages/effect-codex-app-server/src/client.ts +++ b/packages/effect-codex-app-server/src/client.ts @@ -18,6 +18,17 @@ import { } from "./_internal/shared.ts"; import { makeChildStdio, makeTerminationError } from "./_internal/stdio.ts"; +/** + * JSON-RPC id of the Codex server request currently being handled on this + * fiber. Handlers that need a stable correlation key (for example concurrent + * MCP elicitations that share a `serverName`) should read this instead of a + * non-unique payload field. + */ +export const CurrentServerRequestId = Context.Reference( + "effect-codex-app-server/CurrentServerRequestId", + { defaultValue: () => undefined }, +); + export interface CodexAppServerClientOptions { readonly logIncoming?: boolean; readonly logOutgoing?: boolean; @@ -176,6 +187,7 @@ export const make = Effect.fn("effect-codex-app-server/CodexAppServerClient.make return decodeOptionalPayload(method, payloadSchema, request.params).pipe( Effect.flatMap((decoded) => runHandler(handler, decoded, method)), Effect.flatMap((result) => encodeOptionalPayload(method, responseSchema, result)), + Effect.provideService(CurrentServerRequestId, request.id), ); } diff --git a/packages/effect-codex-app-server/src/errors.ts b/packages/effect-codex-app-server/src/errors.ts index f0e0945d352..dda358a871c 100644 --- a/packages/effect-codex-app-server/src/errors.ts +++ b/packages/effect-codex-app-server/src/errors.ts @@ -118,6 +118,8 @@ export const CodexAppServerIdentifierPurpose = Schema.Literals([ "provider-event", "command-approval-request", "file-change-approval-request", + "permissions-approval-request", + "mcp-approval-request", "user-input-request", ]); export type CodexAppServerIdentifierPurpose = typeof CodexAppServerIdentifierPurpose.Type; diff --git a/scripts/build-desktop-artifact.test.ts b/scripts/build-desktop-artifact.test.ts index b7383de236a..55a9efd279f 100644 --- a/scripts/build-desktop-artifact.test.ts +++ b/scripts/build-desktop-artifact.test.ts @@ -17,6 +17,7 @@ import { createStageWorkspaceConfig, createStagePatchedDependencies, createBuildConfig, + desktopMcpExecutableName, DESKTOP_ELECTRON_LANGUAGES, DESKTOP_FILE_EXCLUSIONS, DESKTOP_EXTRA_RESOURCES, @@ -1019,6 +1020,66 @@ it.layer(NodeServices.layer)("build-desktop-artifact", (it) => { }).pipe(Effect.provide(ConfigProvider.layer(ConfigProvider.fromEnv({ env: {} })))), ); + it("suffixes the desktop server executable only on Windows", () => { + // The server's resolver builds the same name; if these drift the packaged + // app silently offers no desktop tools. + assert.equal(desktopMcpExecutableName("win"), "t3-desktop-mcp.exe"); + assert.equal(desktopMcpExecutableName("mac"), "t3-desktop-mcp"); + assert.equal(desktopMcpExecutableName("linux"), "t3-desktop-mcp"); + }); + + it("builds the desktop server for the same Rust targets as the resource monitor", () => { + // stageDesktopMcpRust reuses this mapping, so a Windows or Linux artifact + // build compiles the crate for exactly the architectures it ships. + assert.deepStrictEqual(resolveResourceMonitorRustTargets("win", "x64"), [ + "x86_64-pc-windows-msvc", + ]); + assert.deepStrictEqual(resolveResourceMonitorRustTargets("linux", "arm64"), [ + "aarch64-unknown-linux-gnu", + ]); + }); + + it.effect("declares the Apple Events usage description on macOS builds", () => + Effect.gen(function* () { + const config = yield* createBuildConfig( + "mac", + "dmg", + "1.2.3", + false, + false, + undefined, + undefined, + ); + + const mac = config.mac as Record; + const extendInfo = mac.extendInfo as Record; + // Without this key macOS denies Apple Events with errAEEventNotPermitted + // (-1743) and never prompts, so desktop automation fails silently. + assert.isString(extendInfo.NSAppleEventsUsageDescription); + assert.isNotEmpty(extendInfo.NSAppleEventsUsageDescription as string); + }).pipe(Effect.provide(ConfigProvider.layer(ConfigProvider.fromEnv({ env: {} })))), + ); + + it.effect("keeps the Apple Events usage description off non-macOS builds", () => + Effect.gen(function* () { + for (const [platform, target] of [ + ["win", "nsis"], + ["linux", "AppImage"], + ] as const) { + const config = yield* createBuildConfig( + platform, + target, + "1.2.3", + false, + false, + undefined, + undefined, + ); + assert.notProperty(config, "mac"); + } + }).pipe(Effect.provide(ConfigProvider.layer(ConfigProvider.fromEnv({ env: {} })))), + ); + it.effect("keeps executable resource editing enabled for unsigned Windows builds", () => Effect.gen(function* () { const config = yield* createBuildConfig( diff --git a/scripts/build-desktop-artifact.ts b/scripts/build-desktop-artifact.ts index bf36029bc75..7cd20ef5654 100644 --- a/scripts/build-desktop-artifact.ts +++ b/scripts/build-desktop-artifact.ts @@ -300,6 +300,31 @@ export class ResourceMonitorBuildOutputMissingError extends Schema.TaggedErrorCl } } +export const DESKTOP_MCP_EXECUTABLE_NAME = "t3-desktop-mcp"; + +/** + * On-disk name of the desktop-control server for a platform. + * + * The staged directory keeps the bare name on every platform; only the + * executable inside it carries Windows' suffix. The server's resolver has to + * agree with this exactly or it will look for a file that is not there. + */ +export function desktopMcpExecutableName(platform: typeof BuildPlatform.Type): string { + return platform === "win" ? `${DESKTOP_MCP_EXECUTABLE_NAME}.exe` : DESKTOP_MCP_EXECUTABLE_NAME; +} + +export class DesktopMcpBuildOutputMissingError extends Schema.TaggedErrorClass()( + "DesktopMcpBuildOutputMissingError", + { + candidates: Schema.Array(Schema.String), + arch: BuildArch, + }, +) { + override get message(): string { + return `Desktop MCP build for ${this.arch} produced no binary at any of: ${this.candidates.join(", ")}.`; + } +} + const desktopIconPlatformNames = { mac: "macOS", linux: "Linux", @@ -1643,6 +1668,74 @@ const verifyPackagedBundleIsSelfContained = Effect.fn("verifyPackagedBundleIsSel }, ); +/** + * Build and stage the Windows/Linux desktop-control MCP server. + * + * macOS is served by the Swift package in `native/t3-desktop-mcp`; this is the + * Rust crate covering the other two. Both emit a binary called + * `t3-desktop-mcp`, so the server's resolver treats every platform the same. + */ +const stageDesktopMcpRust = Effect.fn("stageDesktopMcpRust")(function* (input: { + readonly repoRoot: string; + readonly stageResourcesDir: string; + readonly platform: typeof BuildPlatform.Type; + readonly arch: typeof BuildArch.Type; + readonly verbose: boolean; +}) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const manifestPath = path.join(input.repoRoot, "native/t3-desktop-mcp-rs/Cargo.toml"); + const executableName = desktopMcpExecutableName(input.platform); + // The desktop server has the same per-platform target matrix as the resource + // monitor, so it reuses that mapping rather than growing a parallel one. + const rustTargets = resolveResourceMonitorRustTargets(input.platform, input.arch); + + const destinationDirectory = path.join(input.stageResourcesDir, DESKTOP_MCP_EXECUTABLE_NAME); + const destinationPath = path.join(destinationDirectory, executableName); + yield* fs.remove(destinationDirectory, { recursive: true, force: true }).pipe(Effect.ignore); + yield* fs.makeDirectory(destinationDirectory, { recursive: true }); + + for (const rustTarget of rustTargets) { + const spawnCommand = yield* resolveSpawnCommand("cargo", [ + "build", + "--locked", + "--release", + "--manifest-path", + manifestPath, + "--target", + rustTarget, + ]); + yield* runCommand( + ChildProcess.make(spawnCommand.command, spawnCommand.args, { + cwd: input.repoRoot, + shell: spawnCommand.shell, + }), + { + label: `cargo build desktop mcp (${rustTarget})`, + verbose: input.verbose, + }, + ); + + const binaryPath = path.join( + input.repoRoot, + "native/t3-desktop-mcp-rs/target", + rustTarget, + "release", + executableName, + ); + if (!(yield* fs.exists(binaryPath))) { + return yield* new DesktopMcpBuildOutputMissingError({ + candidates: [binaryPath], + arch: input.arch, + }); + } + yield* fs.copyFile(binaryPath, destinationPath); + if (input.platform !== "win") { + yield* fs.chmod(destinationPath, 0o755); + } + } +}); + const stageResourceMonitor = Effect.fn("stageResourceMonitor")(function* (input: { readonly repoRoot: string; readonly stageResourcesDir: string; @@ -1718,6 +1811,115 @@ const stageResourceMonitor = Effect.fn("stageResourceMonitor")(function* (input: } }); +// macOS Swift desktop MCP. Windows/Linux stage the Rust binary via +// `stageDesktopMcpRust` instead — this helper is the Darwin path only. +const stageDesktopMcp = Effect.fn("stageDesktopMcp")(function* (input: { + readonly repoRoot: string; + readonly stageResourcesDir: string; + readonly arch: typeof BuildArch.Type; + readonly verbose: boolean; +}) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const packagePath = path.join(input.repoRoot, "native/t3-desktop-mcp"); + // SwiftPM emits a fat binary directly when handed several --arch flags, so + // this needs no separate lipo step the way the Rust monitor does. + const archArgs = + input.arch === "universal" + ? ["--arch", "arm64", "--arch", "x86_64"] + : ["--arch", input.arch === "arm64" ? "arm64" : "x86_64"]; + const spawnCommand = yield* resolveSpawnCommand("swift", [ + "build", + "-c", + "release", + "--package-path", + packagePath, + ...archArgs, + ]); + yield* runCommand( + ChildProcess.make(spawnCommand.command, spawnCommand.args, { + cwd: input.repoRoot, + shell: spawnCommand.shell, + }), + { + label: `swift build desktop mcp (${input.arch})`, + verbose: input.verbose, + }, + ); + + // Multi-arch builds land under .build/apple/Products/Release; single-arch + // builds land under .build/release. + const candidates = [ + path.join(packagePath, ".build/apple/Products/Release", DESKTOP_MCP_EXECUTABLE_NAME), + path.join(packagePath, ".build/release", DESKTOP_MCP_EXECUTABLE_NAME), + ]; + let binaryPath: string | undefined; + for (const candidate of candidates) { + if (yield* fs.exists(candidate)) { + binaryPath = candidate; + break; + } + } + if (binaryPath === undefined) { + return yield* new DesktopMcpBuildOutputMissingError({ candidates, arch: input.arch }); + } + + const destinationDirectory = path.join(input.stageResourcesDir, DESKTOP_MCP_EXECUTABLE_NAME); + const destinationPath = path.join(destinationDirectory, DESKTOP_MCP_EXECUTABLE_NAME); + yield* fs.remove(destinationDirectory, { recursive: true, force: true }).pipe(Effect.ignore); + yield* fs.makeDirectory(destinationDirectory, { recursive: true }); + yield* fs.copyFile(binaryPath, destinationPath); + yield* fs.chmod(destinationPath, 0o755); + + // Agent cursor overlay: a minimal LSUIElement .app so AppKit will actually + // put the pointer window up. The MCP server itself stays a bare executable + // so it keeps inheriting the host app's TCC grants; only the overlay needs a + // bundle identity. Same binary, different launch path (see AgentCursor.swift). + const overlayAppName = "T3AgentCursor.app"; + const overlayExecutableName = "T3AgentCursor"; + const overlayAppDir = path.join(destinationDirectory, overlayAppName); + const overlayMacOSDir = path.join(overlayAppDir, "Contents", "MacOS"); + const overlayPlistPath = path.join(overlayAppDir, "Contents", "Info.plist"); + const overlayExecutablePath = path.join(overlayMacOSDir, overlayExecutableName); + yield* fs.makeDirectory(overlayMacOSDir, { recursive: true }); + yield* fs.copyFile(binaryPath, overlayExecutablePath); + yield* fs.chmod(overlayExecutablePath, 0o755); + yield* fs.writeFileString( + overlayPlistPath, + ` + + + + CFBundleDevelopmentRegion + en + CFBundleExecutable + ${overlayExecutableName} + CFBundleIdentifier + com.t3tools.t3code.agent-cursor + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + T3 Agent Cursor + CFBundlePackageType + APPL + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1 + LSMinimumSystemVersion + 14.0 + LSUIElement + + NSHighResolutionCapable + + NSPrincipalClass + NSApplication + + +`, + ); +}); + function generateMacIconSet( sourcePng: string, targetIcns: string, @@ -2069,6 +2271,13 @@ export const createBuildConfig = Effect.fn("createBuildConfig")(function* ( target: target === "dmg" ? [target, "zip"] : [target], icon: "icon.icns", category: "public.app-category.developer-tools", + // Without this key macOS denies every Apple Event with errAEEventNotPermitted + // (-1743) and never shows the Automation prompt, so Codex Computer Use and any + // other MCP server we spawn silently fail to drive other apps. + extendInfo: { + NSAppleEventsUsageDescription: + "This app needs to control other apps to run Computer Use automations you approve.", + }, protocols: [ { name: "T3 Code", @@ -2825,6 +3034,22 @@ const buildDesktopArtifact = Effect.fn("buildDesktopArtifact")(function* ( arch: options.arch, verbose: options.verbose, }); + if (options.platform === "mac") { + yield* stageDesktopMcp({ + repoRoot, + stageResourcesDir, + arch: options.arch, + verbose: options.verbose, + }); + } else { + yield* stageDesktopMcpRust({ + repoRoot, + stageResourcesDir, + platform: options.platform, + arch: options.arch, + verbose: options.verbose, + }); + } yield* assertPlatformBuildResources( options.platform, diff --git a/vite.config.ts b/vite.config.ts index 585428028d3..219e87f7845 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -22,7 +22,11 @@ export default defineConfig({ }, staged: { // Formatter only for now — no lint or typecheck on commit. - "*": "vp fmt", + // + // Matched to what oxfmt can parse rather than "*": handed only files it + // does not recognise it exits non-zero ("Expected at least one target + // file"), which failed every commit touching just native/ sources. + "*.{js,jsx,mjs,cjs,ts,tsx,mts,cts,json,jsonc,md,css,html,yml,yaml}": "vp fmt", }, fmt: { ignorePatterns: [