diff --git a/commands/screenrecord.go b/commands/screenrecord.go index 33692f2..86014b9 100644 --- a/commands/screenrecord.go +++ b/commands/screenrecord.go @@ -19,9 +19,23 @@ type ScreenRecordRequest struct { OutputPath string TimeLimit int // max recording duration in seconds, 0 = no limit StopChan <-chan struct{} // when non-nil, stops recording when closed (server mode) + Ready chan<- error // optional (server mode): signaled once, with nil once recording is confirmed live or with an error if it failed to start Silent bool } +// signalReady notifies req.Ready, if present, that the recording is confirmed +// live (err == nil) or failed to start (err != nil). Safe to call more than +// once or with a nil Ready channel — only the first send has any effect. +func (req ScreenRecordRequest) signalReady(err error) { + if req.Ready == nil { + return + } + select { + case req.Ready <- err: + default: + } +} + // ScreenRecordResponse contains the result of a screen recording type ScreenRecordResponse struct { Output string `json:"output"` @@ -33,6 +47,7 @@ type ScreenRecordResponse struct { func ScreenRecordCommand(req ScreenRecordRequest) *CommandResponse { targetDevice, err := FindDeviceOrAutoSelect(req.DeviceID) if err != nil { + req.signalReady(err) return NewErrorResponse(fmt.Errorf("error finding device: %w", err)) } @@ -43,6 +58,7 @@ func ScreenRecordCommand(req ScreenRecordRequest) *CommandResponse { Hook: GetShutdownHook(), }) if err != nil { + req.signalReady(err) return NewErrorResponse(fmt.Errorf("error starting agent: %w", err)) } @@ -55,6 +71,9 @@ func ScreenRecordCommand(req ScreenRecordRequest) *CommandResponse { OnDownloadProgress: progress.downloadProgress, OnDownloaded: progress.downloaded, } + // no async on-device UI step here (unlike real iOS devices) — the + // recording is live as soon as we're about to dispatch it. + req.signalReady(nil) return screenRecordNative(func() error { return dev.ScreenRecord(req.OutputPath, req.TimeLimit, req.StopChan, cb) }, req, progress) @@ -64,23 +83,33 @@ func ScreenRecordCommand(req ScreenRecordRequest) *CommandResponse { case targetDevice.Platform() == "android": dev, ok := targetDevice.(*devices.AndroidDevice) if !ok { - return NewErrorResponse(fmt.Errorf("expected android device")) + err := fmt.Errorf("expected android device") + req.signalReady(err) + return NewErrorResponse(err) } + req.signalReady(nil) return screenRecordNative(func() error { return dev.ScreenRecord(req.OutputPath, req.TimeLimit, req.StopChan) }, req, progress) case targetDevice.Platform() == "ios" && targetDevice.DeviceType() == "simulator": dev, ok := targetDevice.(*devices.SimulatorDevice) if !ok { - return NewErrorResponse(fmt.Errorf("expected simulator device")) + err := fmt.Errorf("expected simulator device") + req.signalReady(err) + return NewErrorResponse(err) } + req.signalReady(nil) return screenRecordNative(func() error { return dev.ScreenRecord(req.OutputPath, req.TimeLimit, req.StopChan) }, req, progress) case targetDevice.Platform() == "ios" && targetDevice.DeviceType() == "real": + // real iOS devices route through DeviceKit + ReplayKit; screenRecordIOSDevice + // signals req.Ready itself once the broadcast picker is confirmed started. return screenRecordIOSDevice(targetDevice, req, progress) default: - return NewErrorResponse(fmt.Errorf("screen recording is not supported for this device type")) + err := fmt.Errorf("screen recording is not supported for this device type") + req.signalReady(err) + return NewErrorResponse(err) } } @@ -165,6 +194,7 @@ func (p *screenRecordProgress) downloaded(speedMBps float64) { func screenRecordIOSDevice(targetDevice devices.ControllableDevice, req ScreenRecordRequest, progress *screenRecordProgress) *CommandResponse { tempFile, err := os.CreateTemp("", "screenrecord-*.avc") if err != nil { + req.signalReady(err) return NewErrorResponse(fmt.Errorf("error creating temp file: %w", err)) } tempPath := tempFile.Name() @@ -188,6 +218,9 @@ func screenRecordIOSDevice(targetDevice devices.ControllableDevice, req ScreenRe OnProgress: func(message string) { utils.Verbose(message) }, + OnReady: func() { + req.signalReady(nil) + }, OnData: withStopChan(func(data []byte) bool { _, writeErr := tempFile.Write(data) return writeErr == nil @@ -203,6 +236,9 @@ func screenRecordIOSDevice(targetDevice devices.ControllableDevice, req ScreenRe tempFile.Close() if err != nil { + // no-op if OnReady already fired above; covers failures that happen + // before DeviceKit/the broadcast picker was ever confirmed live. + req.signalReady(err) return NewErrorResponse(fmt.Errorf("error during screen capture: %w", err)) } diff --git a/devices/common.go b/devices/common.go index a5182f6..21e4fca 100644 --- a/devices/common.go +++ b/devices/common.go @@ -81,6 +81,7 @@ type ScreenCaptureConfig struct { FPS int Bitrate int // bitrate in bits per second, only applies to AVC (0 for default) OnProgress func(message string) // optional progress callback + OnReady func() // optional: called once capture is confirmed live (e.g. after the ReplayKit broadcast picker is clicked), before streaming begins OnData func([]byte) bool // data callback - return false to stop } diff --git a/devices/devicekit/types.go b/devices/devicekit/types.go index f66127c..c10fba8 100644 --- a/devices/devicekit/types.go +++ b/devices/devicekit/types.go @@ -2,6 +2,8 @@ package devicekit import ( "net/http" + "net/url" + "strconv" "strings" "time" ) @@ -29,6 +31,21 @@ func NewDeviceKitClient(hostPort string) *DeviceKitClient { } } +// Port returns the port this client talks to, parsed from its base URL. +func (c *DeviceKitClient) Port() int { + parsed, err := url.Parse(c.baseURL) + if err != nil { + return 0 + } + + port, err := strconv.Atoi(parsed.Port()) + if err != nil { + return 0 + } + + return port +} + type TapAction struct { Type string `json:"type"` Duration int `json:"duration"` diff --git a/devices/ios.go b/devices/ios.go index 0d275a2..7514f5d 100644 --- a/devices/ios.go +++ b/devices/ios.go @@ -999,12 +999,18 @@ func (d *IOSDevice) StartScreenCapture(config ScreenCaptureConfig) error { // start DeviceKit // Note: passing nil registry since this is internal call from StartScreenCapture // ScreenCapture callers should have already registered the device via StartAgent - deviceKitInfo, err = d.StartDeviceKit(nil) + deviceKitInfo, err = d.StartDeviceKitAvc(nil) if err != nil { return fmt.Errorf("failed to start DeviceKit: %w", err) } } + // DeviceKit is confirmed running (either reused or freshly started and the + // broadcast picker was clicked) — safe to tell the caller capture is live. + if config.OnReady != nil { + config.OnReady() + } + if config.OnProgress != nil { config.OnProgress(fmt.Sprintf("Connecting to H.264 stream on localhost:%d", deviceKitInfo.StreamPort)) } @@ -1061,7 +1067,7 @@ func (d *IOSDevice) StartScreenCapture(config ScreenCaptureConfig) error { // mjpeg is served on the same port as the agent HTTP server at /mjpeg d.mu.Lock() - wdaPort, _ := d.portForwarderDeviceKitAgent.GetPorts() + wdaPort := d.deviceKitClient.Port() mjpegURL := buildMjpegURL(wdaPort, config.FPS, config.Scale) d.mjpegClient = mjpeg.NewDeviceKitMjpegClient(mjpegURL) d.mu.Unlock() @@ -1163,7 +1169,10 @@ func (d *IOSDevice) clickStartBroadcastButton() error { // first dump: handle "Press to Start Broadcasting" screen if present firstElements, err := d.DumpSource() if err == nil { - if hasText(firstElements, "Press to Start Broadcasting") { + if hasText(firstElements, "BroadcastUploadExtension") { + // dialog is already open, no need to click anything + utils.Verbose("It seems that the start broadcasting dialog is already visible") + } else if hasText(firstElements, "Press to Start Broadcasting") { utils.Verbose("Found 'Press to Start Broadcasting' screen; tapping the only button.") buttons := filterButtons(firstElements) if len(buttons) != 1 { @@ -1172,6 +1181,7 @@ func (d *IOSDevice) clickStartBroadcastButton() error { centerX := buttons[0].Rect.X + buttons[0].Rect.Width/2 centerY := buttons[0].Rect.Y + buttons[0].Rect.Height/2 + utils.Verbose("Tapping at %f,%f", centerX, centerY) if err = d.Tap(centerX, centerY); err != nil { return fmt.Errorf("failed to tap broadcast button: %w", err) } @@ -1417,98 +1427,123 @@ func (d *IOSDevice) isDeviceKitRunning() bool { return true } -// StartDeviceKit starts the devicekit-ios XCUITest which provides: -// - An HTTP server for tap/dumpUI commands (port 12004) -// - A broadcast extension for H.264 screen streaming (port 12005) -func (d *IOSDevice) StartDeviceKit(hook *ShutdownHook) (*DeviceKitInfo, error) { - // register cleanup hook for this device - if hook != nil { - hookName := fmt.Sprintf("ios-devicekit-%s", d.Udid) - hook.Register(hookName, d.Cleanup) - } - - // Start tunnel if needed (iOS 17+) - err := d.startTunnel() - if err != nil { - return nil, fmt.Errorf("failed to start tunnel: %w", err) - } - - // Broadcast is not running, we need to start it. - utils.Verbose("Broadcast extension not running, starting DeviceKit app...") - - // find DeviceKit main app (not the xctrunner) - apps, err := d.ListApps(true) - if err != nil { - return nil, fmt.Errorf("failed to list apps: %w", err) - } - - var devicekitMainAppBundleId string +// findScreenCaptureAppBundleId finds the DeviceKit H.264 screen capture app +// (not the xctrunner) among the device's installed apps. +func findScreenCaptureAppBundleId(apps []InstalledAppInfo) (string, error) { for _, app := range apps { - // look for the main app, not the test runner - if strings.HasPrefix(app.PackageName, "com.") && strings.Contains(app.PackageName, "devicekit-ios") && !strings.Contains(app.PackageName, "UITests") { + if strings.Contains(app.PackageName, "com.mobilenext.devicekit-h264") { utils.Verbose("DeviceKit main app found, bundle ID: %s", app.PackageName) - devicekitMainAppBundleId = app.PackageName - break + return app.PackageName, nil } } + return "", fmt.Errorf("DeviceKit main app not found. Please install devicekit-ios on the device") +} - if devicekitMainAppBundleId == "" { - return nil, fmt.Errorf("DeviceKit main app not found. Please install devicekit-ios on the device") - } - - // Find available local port for HTTP forwarding and bind immediately. +// startDeviceKitAvcForwarders sets up the HTTP and H.264 stream port forwarders, +// cleaning up any partially created forwarder on failure. +func (d *IOSDevice) startDeviceKitAvcForwarders() (int, int, error) { localHTTPPort, err := findAvailablePortInRange(portRangeStart, portRangeEnd) if err != nil { - return nil, fmt.Errorf("failed to find available port for HTTP: %w", err) + return 0, 0, fmt.Errorf("failed to find available port for HTTP: %w", err) } d.mu.Lock() d.portForwarderDeviceKit = ios.NewPortForwarder(d.ID()) d.mu.Unlock() - err = d.portForwarderDeviceKit.Forward(localHTTPPort, deviceKitHTTPPort) - if err != nil { - return nil, fmt.Errorf("failed to forward HTTP port: %w", err) + if err := d.portForwarderDeviceKit.Forward(localHTTPPort, deviceKitHTTPPort); err != nil { + return 0, 0, fmt.Errorf("failed to forward HTTP port: %w", err) } utils.Verbose("Port forwarding started: localhost:%d -> device:%d (HTTP)", localHTTPPort, deviceKitHTTPPort) - // Find available local port for stream forwarding after HTTP is bound. + localStreamPort, err := findAvailablePortInRange(portRangeStart, portRangeEnd) if err != nil { _ = d.portForwarderDeviceKit.Stop() - return nil, fmt.Errorf("failed to find available port for stream: %w", err) + return 0, 0, fmt.Errorf("failed to find available port for stream: %w", err) } d.mu.Lock() d.portForwarderAvc = ios.NewPortForwarder(d.ID()) d.mu.Unlock() - err = d.portForwarderAvc.Forward(localStreamPort, deviceKitStreamPort) - if err != nil { - // clean up HTTP forwarder on failure + if err := d.portForwarderAvc.Forward(localStreamPort, deviceKitStreamPort); err != nil { _ = d.portForwarderDeviceKit.Stop() - return nil, fmt.Errorf("failed to forward stream port: %w", err) + return 0, 0, fmt.Errorf("failed to forward stream port: %w", err) } utils.Verbose("Port forwarding started: localhost:%d -> device:%d (H.264 stream)", localStreamPort, deviceKitStreamPort) - // Launch the main DeviceKit app - utils.Verbose("Launching DeviceKit app: %s", devicekitMainAppBundleId) - startTime := time.Now() - err = d.LaunchApp(devicekitMainAppBundleId, LaunchOptions{}) - if err != nil { - // clean up port forwarders on failure - _ = d.portForwarderDeviceKit.Stop() - _ = d.portForwarderAvc.Stop() - return nil, fmt.Errorf("failed to launch DeviceKit app: %w", err) + return localHTTPPort, localStreamPort, nil +} + +// stopDeviceKitAvcForwarders stops the HTTP and H.264 stream port forwarders. +func (d *IOSDevice) stopDeviceKitAvcForwarders() { + _ = d.portForwarderDeviceKit.Stop() + _ = d.portForwarderAvc.Stop() +} + +// launchDeviceKitApp launches the DeviceKit app and waits for it to reach the foreground. +func (d *IOSDevice) launchDeviceKitApp(bundleId string) error { + utils.Verbose("Launching DeviceKit app: %s", bundleId) + if err := d.LaunchApp(bundleId, LaunchOptions{}); err != nil { + return fmt.Errorf("failed to launch DeviceKit app: %w", err) } - // wait for the app to be in foreground utils.Verbose("Waiting for DeviceKit app to be in foreground...") - err = d.waitForAppInForeground(devicekitMainAppBundleId, deviceKitAppLaunchTimeout) + if err := d.waitForAppInForeground(bundleId, deviceKitAppLaunchTimeout); err != nil { + return fmt.Errorf("failed to wait for DeviceKit app: %w", err) + } + + return nil +} + +// dismissDeviceKitApp presses HOME a few times to return to the home screen +// after the broadcast has started. +func (d *IOSDevice) dismissDeviceKitApp() { + for i := 0; i < 3; i++ { + if err := d.PressButton("HOME"); err != nil { + utils.Verbose("Failed to press HOME button (attempt %d): %v", i+1, err) + } + time.Sleep(500 * time.Millisecond) + } +} + +// StartDeviceKitAvc starts the devicekit-ios XCUITest which provides: +// - An HTTP server for tap/dumpUI commands (port 12004) +// - A broadcast extension for H.264 screen streaming (port 12005) +func (d *IOSDevice) StartDeviceKitAvc(hook *ShutdownHook) (*DeviceKitInfo, error) { + // register cleanup hook for this device + if hook != nil { + hookName := fmt.Sprintf("ios-devicekit-%s", d.Udid) + hook.Register(hookName, d.Cleanup) + } + + // Start tunnel if needed (iOS 17+) + if err := d.startTunnel(); err != nil { + return nil, fmt.Errorf("failed to start tunnel: %w", err) + } + + // Broadcast is not running, we need to start it. + utils.Verbose("Broadcast extension not running, starting DeviceKit app...") + + apps, err := d.ListApps(true) if err != nil { - // clean up port forwarders on failure - _ = d.portForwarderDeviceKit.Stop() - _ = d.portForwarderAvc.Stop() - return nil, fmt.Errorf("failed to wait for DeviceKit app: %w", err) + return nil, fmt.Errorf("failed to list apps: %w", err) + } + + screenCaptureAppBundleId, err := findScreenCaptureAppBundleId(apps) + if err != nil { + return nil, err + } + + localHTTPPort, localStreamPort, err := d.startDeviceKitAvcForwarders() + if err != nil { + return nil, err + } + + startTime := time.Now() + if err := d.launchDeviceKitApp(screenCaptureAppBundleId); err != nil { + d.stopDeviceKitAvcForwarders() + return nil, err } // Start WebDriverAgent to be able to tap on the screen @@ -1517,20 +1552,14 @@ func (d *IOSDevice) StartDeviceKit(hook *ShutdownHook) (*DeviceKitInfo, error) { utils.Verbose(message) }, }) - if err != nil { - // clean up port forwarders on failure - _ = d.portForwarderDeviceKit.Stop() - _ = d.portForwarderAvc.Stop() + d.stopDeviceKitAvcForwarders() return nil, fmt.Errorf("failed to start agent: %w", err) } // find and tap the "Start Broadcast" button - err = d.clickStartBroadcastButton() - if err != nil { - // clean up port forwarders on failure - _ = d.portForwarderDeviceKit.Stop() - _ = d.portForwarderAvc.Stop() + if err := d.clickStartBroadcastButton(); err != nil { + d.stopDeviceKitAvcForwarders() return nil, fmt.Errorf("failed to click Start Broadcast button: %w", err) } @@ -1542,14 +1571,7 @@ func (d *IOSDevice) StartDeviceKit(hook *ShutdownHook) (*DeviceKitInfo, error) { utils.Verbose("Waiting %v for broadcast TCP server to start...", deviceKitBroadcastTimeout) time.Sleep(deviceKitBroadcastTimeout) - // Press HOME 3 times to dismiss the DeviceKit app and return to home screen - for i := 0; i < 3; i++ { - err = d.PressButton("HOME") - if err != nil { - utils.Verbose("Failed to press HOME button (attempt %d): %v", i+1, err) - } - time.Sleep(500 * time.Millisecond) - } + d.dismissDeviceKitApp() utils.Verbose("DeviceKit broadcast started successfully") diff --git a/server/recording.go b/server/recording.go index 5d74c51..91a5be5 100644 --- a/server/recording.go +++ b/server/recording.go @@ -13,6 +13,7 @@ type RecordingSession struct { Output string StartedAt time.Time StopChan chan struct{} + Ready chan error // signaled once: nil once recording is confirmed live, or an error if it failed to start Done chan *commands.CommandResponse stopped bool // true after StopChan has been closed } @@ -36,6 +37,7 @@ func (rm *recordingManager) start(output string) (*RecordingSession, error) { Output: output, StartedAt: time.Now(), StopChan: make(chan struct{}), + Ready: make(chan error, 1), Done: make(chan *commands.CommandResponse, 1), } rm.session = s diff --git a/server/server.go b/server/server.go index ea2071f..6079751 100644 --- a/server/server.go +++ b/server/server.go @@ -1098,6 +1098,12 @@ func handleAppsUninstall(params json.RawMessage) (any, error) { return response.Data, nil } +// screenRecordReadyTimeout bounds how long handleScreenRecord waits for the +// recording to be confirmed live before giving up. Sized generously above a +// worst-case cold DeviceKit start on real iOS devices (WDA/app launch + +// two 10s broadcast-picker button polls + the 5s post-click TCP wait). +const screenRecordReadyTimeout = 60 * time.Second + func handleScreenRecord(params json.RawMessage) (any, error) { if len(params) == 0 { return nil, fmt.Errorf("'params' is required with fields: deviceId, output") @@ -1122,6 +1128,7 @@ func handleScreenRecord(params json.RawMessage) (any, error) { OutputPath: p.Output, TimeLimit: p.TimeLimit, StopChan: session.StopChan, + Ready: session.Ready, } go func() { @@ -1129,10 +1136,40 @@ func handleScreenRecord(params json.RawMessage) (any, error) { session.Done <- resp }() - return map[string]any{ - "status": "recording", - "output": p.Output, - }, nil + // Don't ack until the recording is actually confirmed live. On real iOS + // devices this waits for the ReplayKit broadcast picker to be clicked, so + // callers never race a still-starting recording with device commands. + select { + case readyErr := <-session.Ready: + if readyErr != nil { + recorder.clear() + return nil, fmt.Errorf("failed to start recording: %w", readyErr) + } + return map[string]any{ + "status": "recording", + "output": p.Output, + }, nil + case resp := <-session.Done: + // recording finished (or failed) before ever confirming it was live + recorder.clear() + if resp.Status == "error" { + return nil, fmt.Errorf("%s", resp.Error) + } + return nil, fmt.Errorf("recording ended before it was confirmed started") + case <-time.After(screenRecordReadyTimeout): + // the command goroutine may still be starting (or even recording); + // ask it to stop and wait for it to exit before freeing the session, + // so it cannot overlap a subsequent capture + if _, stopErr := recorder.stop(); stopErr == nil { + select { + case <-session.Done: + case <-time.After(30 * time.Second): + } + } + + recorder.clear() + return nil, fmt.Errorf("timed out waiting for recording to start") + } } // ScreenRecordStopParams represents the parameters for stopping a screen recording