diff --git a/multimodal/gui-agent/operator-adb/src/AdbOperator.ts b/multimodal/gui-agent/operator-adb/src/AdbOperator.ts index b287c4cb73..4adec22710 100644 --- a/multimodal/gui-agent/operator-adb/src/AdbOperator.ts +++ b/multimodal/gui-agent/operator-adb/src/AdbOperator.ts @@ -110,7 +110,7 @@ export class AdbOperator extends Operator { throw new Error('point is required when click'); } const { realX, realY } = await this.calculateRealCoords(point); - this.handleSwipe({ x: realX, y: realY }, { x: realX, y: realY }, 1500); + await this.handleSwipe({ x: realX, y: realY }, { x: realX, y: realY }, 1500); break; } case 'swipe': @@ -124,7 +124,7 @@ export class AdbOperator extends Operator { } const { realX: startX, realY: startY } = await this.calculateRealCoords(startPoint); const { realX: endX, realY: endY } = await this.calculateRealCoords(endPoint); - this.handleSwipe({ x: startX, y: startY }, { x: endX, y: endY }, 300); + await this.handleSwipe({ x: startX, y: startY }, { x: endX, y: endY }, 300); break; } case 'scroll': { @@ -132,12 +132,12 @@ export class AdbOperator extends Operator { if (!direction) { throw new Error(`Direction required when scroll`); } - this.handleScroll(direction, point); + await this.handleScroll(direction, point); break; } case 'type': { const { content } = actionInputs; - this.handleType(content); + await this.handleType(content); break; } case 'hotkey': { @@ -332,7 +332,7 @@ export class AdbOperator extends Operator { if (!keyCode) { throw new Error(`Unsupported key: ${keyStr}`); } - this._adb!.keyevent(keyCode); + await this._adb!.keyevent(keyCode); } private async handleSwipe( @@ -370,7 +370,7 @@ export class AdbOperator extends Operator { default: throw new Error(`Unsupported scroll direction: ${direction}`); } - this.handleSwipe({ x: startX, y: startY }, { x: endX, y: endY }, 300); + await this.handleSwipe({ x: startX, y: startY }, { x: endX, y: endY }, 300); } /** diff --git a/multimodal/gui-agent/operator-adb/test/AdbOperator.test.ts b/multimodal/gui-agent/operator-adb/test/AdbOperator.test.ts new file mode 100644 index 0000000000..e492e6c0f4 --- /dev/null +++ b/multimodal/gui-agent/operator-adb/test/AdbOperator.test.ts @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2025 Bytedance, Inc. and its affiliates. + * SPDX-License-Identifier: Apache-2.0 + */ + +import type { ConsoleLogger } from '@agent-infra/logger'; +import type { BaseAction } from '@gui-agent/shared/types'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { AdbOperator } from '../src/AdbOperator'; + +type AdbMethod = 'shell' | 'inputText' | 'keyevent'; + +type Deferred = { + promise: Promise; + resolve: () => void; + reject: (error: Error) => void; +}; + +function createDeferred(): Deferred { + let resolve!: () => void; + let reject!: (error: Error) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +} + +function createLogger(): ConsoleLogger { + const logger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }; + return { + ...logger, + spawn: vi.fn(() => logger), + } as unknown as ConsoleLogger; +} + +function createOperator(deferred: Deferred, method: AdbMethod) { + const operator = new AdbOperator(createLogger()); + const adb = { + shell: vi.fn().mockResolvedValue(undefined), + inputText: vi.fn().mockResolvedValue(undefined), + keyevent: vi.fn().mockResolvedValue(undefined), + }; + adb[method].mockReturnValueOnce(deferred.promise); + + Reflect.set(operator, '_adb', adb); + vi.spyOn(operator, 'doInitialize').mockResolvedValue(undefined); + vi.spyOn(operator, 'getScreenContext').mockResolvedValue({ + screenWidth: 1080, + screenHeight: 1920, + scaleX: 1, + scaleY: 1, + }); + + return { operator, adb }; +} + +const point = (x: number, y: number) => ({ raw: { x, y } }); + +const cases: Array<{ + name: string; + action: BaseAction; + method: AdbMethod; +}> = [ + { + name: 'long_press', + action: { type: 'long_press', inputs: { point: point(100, 200) } }, + method: 'shell', + }, + { + name: 'swipe', + action: { + type: 'swipe', + inputs: { start: point(100, 200), end: point(300, 400), direction: 'down' }, + }, + method: 'shell', + }, + { + name: 'drag', + action: { + type: 'drag', + inputs: { start: point(100, 200), end: point(300, 400) }, + }, + method: 'shell', + }, + { + name: 'scroll', + action: { type: 'scroll', inputs: { direction: 'down' } }, + method: 'shell', + }, + { + name: 'type', + action: { type: 'type', inputs: { content: 'hello' } }, + method: 'inputText', + }, + { + name: 'hotkey', + action: { type: 'hotkey', inputs: { key: 'home' } }, + method: 'keyevent', + }, +]; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe.each(cases)('$name action', ({ action, method }) => { + it('waits for the ADB command and reports its failure', async () => { + vi.spyOn(console, 'error').mockImplementation(() => undefined); + const deferred = createDeferred(); + const { operator, adb } = createOperator(deferred, method); + const execution = operator.doExecute({ actions: [action] }); + + await vi.waitFor(() => { + expect(adb[method]).toHaveBeenCalledOnce(); + }); + + try { + const state = await Promise.race([ + execution.then(() => 'settled'), + new Promise<'pending'>((resolve) => setTimeout(() => resolve('pending'), 0)), + ]); + expect(state).toBe('pending'); + + deferred.reject(new Error('adb failed')); + await expect(execution).resolves.toEqual({ + status: 'failed', + errorMessage: 'adb failed', + }); + } finally { + deferred.resolve(); + } + }); +}); + +describe('multi-action ordering', () => { + it('does not start the next action until the current ADB command finishes', async () => { + const deferred = createDeferred(); + const { operator, adb } = createOperator(deferred, 'inputText'); + const execution = operator.doExecute({ + actions: [ + { type: 'type', inputs: { content: 'hello' } }, + { type: 'hotkey', inputs: { key: 'home' } }, + ], + }); + + await vi.waitFor(() => { + expect(adb.inputText).toHaveBeenCalledOnce(); + }); + expect(adb.keyevent).not.toHaveBeenCalled(); + + deferred.resolve(); + + await expect(execution).resolves.toEqual({ status: 'success' }); + expect(adb.keyevent).toHaveBeenCalledOnce(); + }); +});