diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift index da4c6247c7..8a9fcae38d 100644 --- a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift +++ b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift @@ -10,20 +10,18 @@ func captureAppWindowScreenshot(app: NSRunningApplication, outPath: String) thro throw HelperError.commandFailed("app window screenshots require macOS 14 or newer") } let pid = app.processIdentifier - let frontWindowID = frontWindowNumber(pid: pid) + let frontWindow = try frontWindow(pid: pid) + let frontWindowID = frontWindow?.number let content = try awaitShareableContent() - let candidates = content.windows.filter { - $0.owningApplication?.processID == pid && $0.windowLayer == 0 && $0.frame.width > 0 - && $0.frame.height > 0 - } - guard - let window = candidates.first(where: { Int($0.windowID) == frontWindowID }) - ?? candidates.max(by: { $0.frame.width * $0.frame.height < $1.frame.width * $1.frame.height }) - else { - throw HelperError.commandFailed( - "screenshot could not find a window for the app", - details: ["reason": "window-not-found", "bundleId": app.bundleIdentifier ?? ""] - ) + let candidates = content.windows.filter { $0.owningApplication?.processID == pid } + let window: SCWindow + let selection = frontCaptureWindow( + candidates, frontWindowID: frontWindowID, sharingState: frontWindow?.sharingState, windowID: { Int($0.windowID) }) + switch selection { + case .success(let front): + window = front + case .failure(let refusal): + throw screenshotRefusal(refusal, app: app, frontWindowID: frontWindowID) } let filter = SCContentFilter(desktopIndependentWindow: window) @@ -52,6 +50,54 @@ func captureAppWindowScreenshot(app: NSRunningApplication, outPath: String) thro try writePNG(capturedImage, to: outPath) } +/// Why an app screenshot refuses. The capture is the app's front window or nothing: another +/// window of the app would show state the user is not looking at. Pinned with the host by +/// `contracts/fixtures/macos-native-helper-outcomes.json`. +enum AppWindowScreenshotRefusal: String, CaseIterable, Error { + case windowNotFound = "window-not-found" + case frontWindowNotShareable = "front-window-not-shareable" + case frontWindowCaptureExcluded = "front-window-capture-excluded" +} + +/// The shareable candidate whose WindowServer ID is the app's front window. +func frontCaptureWindow( + _ candidates: [Window], + frontWindowID: Int?, + sharingState: Int?, + windowID: (Window) -> Int +) -> Result { + guard let frontWindowID else { return .failure(.windowNotFound) } + guard sharingState != 0 else { return .failure(.frontWindowCaptureExcluded) } + guard let front = candidates.first(where: { windowID($0) == frontWindowID }) else { + return .failure(.frontWindowNotShareable) + } + return .success(front) +} + +private func screenshotRefusal( + _ refusal: AppWindowScreenshotRefusal, + app: NSRunningApplication, + frontWindowID: Int? +) -> HelperError { + var details = ["reason": refusal.rawValue, "bundleId": app.bundleIdentifier ?? ""] + switch refusal { + case .windowNotFound: + return .commandFailed("screenshot could not find an on-screen window for the app", details: details) + case .frontWindowCaptureExcluded: + details["frontWindowId"] = frontWindowID.map(String.init) ?? "" + return .commandFailed( + "screenshot could not capture the app's front window: the app excludes it from capture", + details: details + ) + case .frontWindowNotShareable: + details["frontWindowId"] = frontWindowID.map(String.init) ?? "" + return .commandFailed( + "screenshot could not capture the app's front window: ScreenCaptureKit does not offer it", + details: details + ) + } +} + @available(macOS 14.0, *) private func awaitShareableContent() throws -> SCShareableContent { let semaphore = DispatchSemaphore(value: 0) diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift index 349426f54c..9903b905a0 100644 --- a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift +++ b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift @@ -97,7 +97,7 @@ func pressInBackground( let point = CGPoint(x: request.x, y: request.y) try requireAppWindowPoint(point, app: app) cursor?.move(to: point) - let target = resolvePressTarget(app: app, point: point) + let target = try resolvePressTarget(app: app, point: point) if let target, isInMenuBar(target.element) { throw refusal(.noAccessibleTarget, "the point is on the app's menu bar", app: app) } @@ -172,7 +172,7 @@ func fillInBackground( cursor?.move(to: point) let hit = elementAtPoint(in: app, point: point) let chain = hit.map(pressSearchChain) ?? [] - let fallback = actionWindow(app: app, hit: hit).flatMap { window in + let fallback = try actionWindow(app: app, hit: hit).flatMap { window in smallestElement(in: window, containing: point, where: isTextInput) } if let input = chain.first(where: isTextInput) ?? fallback, isInMenuBar(input) { @@ -206,7 +206,7 @@ func scrollInBackground( guard isVertical || direction == "left" || direction == "right" else { throw HelperError.invalidArgs("scroll requires --direction ") } - guard let frame = frontWindowFrame(pid: app.processIdentifier) else { + guard let frame = try frontWindowFrame(pid: app.processIdentifier) else { throw HelperError.commandFailed( "scroll could not resolve an on-screen window", details: ["reason": "window-not-found", "bundleId": app.bundleIdentifier ?? ""] @@ -276,12 +276,12 @@ private enum PressTarget { /// What a click at a point means. The app's own hit test answers first; web content often answers /// with a wrapper group, so the snapshot geometry decides when the hit names no control. -private func resolvePressTarget(app: NSRunningApplication, point: CGPoint) -> PressTarget? { +private func resolvePressTarget(app: NSRunningApplication, point: CGPoint) throws -> PressTarget? { let hit = elementAtPoint(in: app, point: point) let chain = hit.map(pressSearchChain) ?? [] if let input = chain.first(where: isTextInput) { return .focus(input) } if let control = chain.first(where: isPressableControl) { return .press(control) } - if let found = actionWindow(app: app, hit: hit).flatMap({ + if let found = try actionWindow(app: app, hit: hit).flatMap({ smallestElement(in: $0, containing: point, where: { isTextInput($0) || isPressableControl($0) }) }) { return isTextInput(found) ? .focus(found) : .press(found) @@ -306,10 +306,7 @@ private func perform(_ target: PressTarget) -> BackgroundDeliveryMechanism? { /// own on-screen windows, its menus and sheets included, and never on a menu bar element: the app /// owns its menu bar's window too, so the window check alone does not exclude it. private func requireAppWindowPoint(_ point: CGPoint, app: NSRunningApplication) throws { - guard - let info = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) - as? [[String: Any]] - else { + guard let info = appWindowInfo() else { throw refusal(.noAccessibleTarget, "the app's on-screen windows could not be read", app: app) } let inAppWindow = info.contains { entry in @@ -364,12 +361,12 @@ private func pressSearchChain(from hit: AXUIElement) -> [AXUIElement] { /// The one window a point action may act in: the window the app's hit test landed in, else the /// app's front on-screen window. A fallback target never comes from a window behind it, and the /// response names the window so the host can check it. -private func actionWindow(app: NSRunningApplication, hit: AXUIElement?) -> AXUIElement? { +private func actionWindow(app: NSRunningApplication, hit: AXUIElement?) throws -> AXUIElement? { if let hit { if role(of: hit) == "AXWindow" { return hit } if let window = elementAttribute(hit, attribute: kAXWindowAttribute as String) { return window } } - guard let front = frontWindowFrame(pid: app.processIdentifier) else { return nil } + guard let front = try frontWindowFrame(pid: app.processIdentifier) else { return nil } return windows(of: AXUIElementCreateApplication(app.processIdentifier)).first { window in guard let rect = rectAttribute(window) else { return false } return abs(rect.x - front.minX) < 1 && abs(rect.y - front.minY) < 1 @@ -482,39 +479,55 @@ private func numberAttribute(_ element: AXUIElement, attribute: String) -> Doubl return number.doubleValue } -private struct OnScreenWindow { +struct OnScreenWindow { let number: Int let bounds: CGRect + let sharingState: Int? } -/// The pid's normal-layer windows, front to back. Bounds and owners need no Screen Recording -/// permission; only titles do. -private func onScreenWindows(pid: pid_t) -> [OnScreenWindow] { - guard - let info = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) - as? [[String: Any]] - else { - return [] - } - return info.compactMap { entry in +/// Only normal-layer windows qualify; app-modal alerts at level 8 are excluded. +let frontWindowLayer = 0 + +/// The pid's visible, nonempty normal-layer windows, front to back. +func onScreenWindows(pid: pid_t, windowInfo: [[String: Any]]) -> [OnScreenWindow] { + windowInfo.compactMap { entry in guard (entry[kCGWindowOwnerPID as String] as? Int32) == pid, - (entry[kCGWindowLayer as String] as? Int) == 0, + (entry[kCGWindowLayer as String] as? Int) == frontWindowLayer, + (entry[kCGWindowIsOnscreen as String] as? Bool) == true, + let alpha = entry[kCGWindowAlpha as String] as? Double, alpha > 0, let number = entry[kCGWindowNumber as String] as? Int, let boundsDict = entry[kCGWindowBounds as String] as? NSDictionary, - let bounds = CGRect(dictionaryRepresentation: boundsDict as CFDictionary) + let bounds = CGRect(dictionaryRepresentation: boundsDict as CFDictionary), + bounds.width > 0, bounds.height > 0 else { return nil } - return OnScreenWindow(number: number, bounds: bounds) + return OnScreenWindow( + number: number, bounds: bounds, sharingState: entry[kCGWindowSharingState as String] as? Int) } } -func frontWindowNumber(pid: pid_t) -> Int? { - onScreenWindows(pid: pid).first?.number +func appWindowInfo() -> [[String: Any]]? { + CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) + as? [[String: Any]] +} + +enum WindowListRefusal: String, CaseIterable { + case unreadable = "window-list-unreadable" +} + +func frontWindow(pid: pid_t, windowInfo: [[String: Any]]? = appWindowInfo()) throws -> OnScreenWindow? { + guard let windowInfo else { + throw HelperError.commandFailed( + "the app's on-screen windows could not be read", + details: ["reason": WindowListRefusal.unreadable.rawValue] + ) + } + return onScreenWindows(pid: pid, windowInfo: windowInfo).first } -private func frontWindowFrame(pid: pid_t) -> CGRect? { - onScreenWindows(pid: pid).first?.bounds +func frontWindowFrame(pid: pid_t, windowInfo: [[String: Any]]? = appWindowInfo()) throws -> CGRect? { + try frontWindow(pid: pid, windowInfo: windowInfo)?.bounds } private let returnVirtualKey: CGKeyCode = 36 diff --git a/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift b/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift new file mode 100644 index 0000000000..539c8752e0 --- /dev/null +++ b/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift @@ -0,0 +1,144 @@ +import CoreGraphics +import XCTest + +@testable import AgentDeviceMacOSHelper + +private struct Candidate: Equatable { + let id: Int + let area: Double +} + +/// An app screenshot captures the front window or refuses; it never substitutes another window. +final class AppWindowScreenshotTests: XCTestCase { + func testCaptureIdentityAndScrollAndClickFallbackFrameSelectTheSameEligibleFrontWindow() throws { + let windows = [ + windowInfo(id: 1, pid: 99), + windowInfo(id: 2, layer: 1), + windowInfo(id: 3, onScreen: false), + windowInfo(id: 4, alpha: 0), + windowInfo(id: 5, width: 0), + windowInfo(id: 6, height: 0), + windowInfo(id: 7, alpha: 0.5, width: 20, height: 20), + windowInfo(id: 8), + ] + + let frontID = try frontWindow(pid: 42, windowInfo: windows)?.number + let selected = frontCaptureWindow([7, 8], frontWindowID: frontID, sharingState: 1, windowID: { $0 }) + + // Scroll and the click actionWindow fallback both consume frontWindowFrame. + XCTAssertEqual( + try frontWindowFrame(pid: 42, windowInfo: windows), + CGRect(x: 10, y: 10, width: 20, height: 20)) + XCTAssertEqual(frontID, 7) + XCTAssertEqual(try selected.get(), 7) + } + + func testSkipsAppModalAlertInFrontOfNormalLayerWindow() throws { + let windows = [windowInfo(id: 8, layer: 8), windowInfo(id: 3)] + let front = try frontWindow(pid: 42, windowInfo: windows) + let selected = frontCaptureWindow( + [8, 3], frontWindowID: front?.number, sharingState: front?.sharingState, windowID: { $0 }) + + XCTAssertEqual(front?.number, 3) + XCTAssertEqual(try selected.get(), 3) + XCTAssertEqual(try frontWindowFrame(pid: 42, windowInfo: windows), front?.bounds) + } + + func testRefusesWhenOnlyInvisibleOrEmptyWindowsRemain() throws { + let windows = [ + windowInfo(id: 1, onScreen: false), + windowInfo(id: 2, alpha: 0), + windowInfo(id: 3, width: 0), + windowInfo(id: 4, height: 0), + ] + let frontID = try frontWindow(pid: 42, windowInfo: windows)?.number + let selected = frontCaptureWindow([5], frontWindowID: frontID, sharingState: 1, windowID: { $0 }) + + XCTAssertNil(try frontWindowFrame(pid: 42, windowInfo: windows)) + XCTAssertNil(frontID) + XCTAssertThrowsError(try selected.get()) { error in + XCTAssertEqual(error as? AppWindowScreenshotRefusal, .windowNotFound) + } + } + + func testUnreadableWindowListRefusesBeforeCaptureOrScrollCanReportNoWindow() { + XCTAssertThrowsError(try frontWindow(pid: 42, windowInfo: nil)) { error in + guard case HelperError.commandFailed(_, let details) = error else { + return XCTFail("expected a typed window-list refusal, got \(error)") + } + XCTAssertEqual(details["reason"], "window-list-unreadable") + } + XCTAssertThrowsError(try frontWindowFrame(pid: 42, windowInfo: nil)) { error in + guard case HelperError.commandFailed(_, let details) = error else { + return XCTFail("expected a typed window-list refusal, got \(error)") + } + XCTAssertEqual(details["reason"], "window-list-unreadable") + } + XCTAssertNil(try frontWindow(pid: 42, windowInfo: [])) + XCTAssertNil(try frontWindowFrame(pid: 42, windowInfo: [])) + } + + func testSelectsTheFrontWindowEvenWhenAnotherIsLarger() { + let candidates = [Candidate(id: 7, area: 10_000), Candidate(id: 3, area: 100)] + + let selected = frontCaptureWindow(candidates, frontWindowID: 3, sharingState: 1, windowID: \.id) + + XCTAssertEqual(try selected.get(), Candidate(id: 3, area: 100)) + } + + func testRefusesWhenTheFrontWindowIsNotShareable() { + let candidates = [Candidate(id: 7, area: 10_000), Candidate(id: 8, area: 100)] + + let selected = frontCaptureWindow(candidates, frontWindowID: 3, sharingState: 1, windowID: \.id) + + XCTAssertThrowsError(try selected.get()) { error in + XCTAssertEqual(error as? AppWindowScreenshotRefusal, .frontWindowNotShareable) + } + } + + func testCaptureExclusionSurvivesWindowSelectionAndNeverSubstitutesAnotherWindow() throws { + let windows = [windowInfo(id: 3, sharingState: 0), windowInfo(id: 7)] + let front = try frontWindow(pid: 42, windowInfo: windows) + XCTAssertEqual(front?.number, 3) + XCTAssertEqual(front?.sharingState, 0) + + for candidates in [[3, 7], [7]] { + let selected = frontCaptureWindow( + candidates, frontWindowID: front?.number, sharingState: front?.sharingState, windowID: { $0 }) + XCTAssertThrowsError(try selected.get()) { error in + XCTAssertEqual(error as? AppWindowScreenshotRefusal, .frontWindowCaptureExcluded) + } + } + + var entry = windowInfo(id: 3) + entry.removeValue(forKey: kCGWindowSharingState as String) + let unknown = try frontWindow(pid: 42, windowInfo: [entry]) + let selected = frontCaptureWindow( + [3], frontWindowID: unknown?.number, sharingState: unknown?.sharingState, windowID: { $0 }) + XCTAssertEqual(try selected.get(), 3) + } + + func testRefusesWhenTheAppHasNoOnScreenWindow() { + let selected = frontCaptureWindow( + [Candidate(id: 7, area: 10_000)], frontWindowID: nil, sharingState: nil, windowID: \.id) + + XCTAssertThrowsError(try selected.get()) { error in + XCTAssertEqual(error as? AppWindowScreenshotRefusal, .windowNotFound) + } + } +} + +private func windowInfo( + id: Int, pid: Int32 = 42, layer: Int = 0, onScreen: Bool = true, + alpha: Double = 1, sharingState: Int = 1, width: Double = 100, height: Double = 100 +) -> [String: Any] { + [ + kCGWindowNumber as String: id, + kCGWindowOwnerPID as String: pid, + kCGWindowLayer as String: layer, + kCGWindowIsOnscreen as String: onScreen, + kCGWindowAlpha as String: alpha, + kCGWindowSharingState as String: sharingState, + kCGWindowBounds as String: CGRect(x: 10, y: 10, width: width, height: height).dictionaryRepresentation, + ] +} diff --git a/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/BackgroundInteractionTests.swift b/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/BackgroundInteractionTests.swift index 7b61be5fc3..36bd3766e8 100644 --- a/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/BackgroundInteractionTests.swift +++ b/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/BackgroundInteractionTests.swift @@ -26,6 +26,8 @@ private struct ScrollGestureFixture: Decodable { private struct HelperOutcomesFixture: Decodable { let refusalReasons: [String] let deliveryMechanisms: [String] + let screenshotRefusalReasons: [String] + let windowListRefusalReasons: [String] } /// `#filePath` is `/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/`. @@ -41,8 +43,11 @@ final class BackgroundInteractionTests: XCTestCase { from: Data( contentsOf: repoRoot.appendingPathComponent("contracts/fixtures/macos-native-helper-outcomes.json")) ) + XCTAssertEqual(WindowListRefusal.allCases.map(\.rawValue), fixture.windowListRefusalReasons) XCTAssertEqual(BackgroundRefusal.allCases.map(\.rawValue), fixture.refusalReasons) XCTAssertEqual(BackgroundDeliveryMechanism.allCases.map(\.rawValue), fixture.deliveryMechanisms) + XCTAssertEqual( + AppWindowScreenshotRefusal.allCases.map(\.rawValue), fixture.screenshotRefusalReasons) } /// The native backend scrolls as far as the runner does: its travel agrees with every case of diff --git a/contracts/fixtures/macos-native-helper-outcomes.json b/contracts/fixtures/macos-native-helper-outcomes.json index be60f68c9b..e3c2a751f6 100644 --- a/contracts/fixtures/macos-native-helper-outcomes.json +++ b/contracts/fixtures/macos-native-helper-outcomes.json @@ -1,5 +1,5 @@ { - "description": "Vocabulary the macOS helper reports for app-surface actions. Swift: BackgroundRefusal and BackgroundDeliveryMechanism in apple/macos-helper. TypeScript: the macOS helper client in packages/platform-apple/src/os/macos/helper.ts.", + "description": "Vocabulary the macOS helper reports for app-surface actions. Swift: BackgroundRefusal, BackgroundDeliveryMechanism, WindowListRefusal, and AppWindowScreenshotRefusal in apple/macos-helper. TypeScript: the macOS helper client in packages/platform-apple/src/os/macos/helper.ts.", "refusalReasons": [ "background-pointer-gesture", "no-accessible-target", @@ -14,5 +14,11 @@ "ax-confirm", "ax-scroll-bar", "key-events" - ] + ], + "screenshotRefusalReasons": [ + "window-not-found", + "front-window-not-shareable", + "front-window-capture-excluded" + ], + "windowListRefusalReasons": ["window-list-unreadable"] } diff --git a/packages/platform-apple/src/core/__tests__/display-inventory.test.ts b/packages/platform-apple/src/core/__tests__/display-inventory.test.ts index a476dafec9..416048de0f 100644 --- a/packages/platform-apple/src/core/__tests__/display-inventory.test.ts +++ b/packages/platform-apple/src/core/__tests__/display-inventory.test.ts @@ -266,15 +266,26 @@ describe('buildInventory', () => { } }); - test('names a capture even when no panel reports primary', () => { + test('names the sole lit panel when no panel reports primary', () => { const inventory = buildInventory([ - { ...outerPanel, primary: false }, + { ...outerPanel, power: 'dark', primary: false }, { ...innerPanel, power: 'lit', primary: false }, ]); assert.equal(inventory.multiScreen, true); assert.equal(inventory.ambiguous, true); - // A capture is still named: never the implicit black-producing default. - assert.ok(inventory.activeDisplay); + assert.equal(inventory.activeDisplay?.name, 'LCD-1'); + }); + + test('names no panel when neither primary nor power identifies one', () => { + for (const power of ['lit', 'dark', 'unknown'] as const) { + const inventory = buildInventory([ + { ...outerPanel, power, primary: false }, + { ...innerPanel, power, primary: false }, + ]); + assert.equal(inventory.multiScreen, true); + assert.equal(inventory.ambiguous, true); + assert.equal(inventory.activeDisplay, undefined, power); + } }); test('keeps choosing the lit panel when a third panel is present', () => { @@ -476,6 +487,58 @@ describe('resolveAppleCaptureTarget', () => { assert.equal(target.displayRotation, undefined); }); + /** The closed-Duo payload with each panel's `primary` flag and backlight replaced. */ + function duoReporting(panels: Record) { + return JSON.stringify({ + result: { + displays: IPHONE_DUO_CLOSED.result.displays.map((entry) => ({ + ...entry, + ...panels[entry.displayId], + })), + }, + }); + } + + test('names the primary panel when panel power is ambiguous', async () => { + mockReadHostTextFile.mockResolvedValue( + duoReporting({ + 1: { primary: true, backlightState: 'activeOn' }, + 3: { primary: false, backlightState: 'activeOn' }, + }), + ); + assert.equal((await resolveAppleCaptureTarget(IOS_TEST_SIMULATOR)).display?.name, 'LCD'); + }); + + test('names the sole lit panel when no panel reports primary', async () => { + mockReadHostTextFile.mockResolvedValue( + duoReporting({ + 1: { primary: false, backlightState: 'off' }, + 3: { primary: false, backlightState: 'activeOn' }, + }), + ); + assert.equal((await resolveAppleCaptureTarget(IOS_TEST_SIMULATOR)).display?.name, 'LCD-1'); + }); + + test('refuses instead of picking a panel by list order when no fact identifies one', async () => { + for (const backlightState of ['activeOn', 'off', 'futureState']) { + mockReadHostTextFile.mockResolvedValue( + duoReporting({ + 1: { primary: false, backlightState }, + 3: { primary: false, backlightState }, + }), + ); + await assert.rejects( + resolveAppleCaptureTarget(IOS_TEST_SIMULATOR), + (error: unknown) => + error instanceof AppError && + error.code === 'COMMAND_FAILED' && + error.details?.reason === 'capture-panel-unidentified' && + typeof error.details?.hint === 'string', + backlightState, + ); + } + }); + test('reports nothing when CoreDevice cannot be asked', async () => { mockRunXcrun.mockResolvedValueOnce({ exitCode: 1, diff --git a/packages/platform-apple/src/core/display-inventory.ts b/packages/platform-apple/src/core/display-inventory.ts index 6dd4fbee2a..1eee41c9f1 100644 --- a/packages/platform-apple/src/core/display-inventory.ts +++ b/packages/platform-apple/src/core/display-inventory.ts @@ -1,5 +1,6 @@ import type { DeviceRotation } from '@agent-device/contracts/device'; import type { DeviceInfo } from '@agent-device/kernel/device'; +import { AppError } from '@agent-device/kernel/errors'; import { isCommandTimeoutError } from '@agent-device/host-kit/command'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; @@ -45,6 +46,11 @@ export type AppleDisplayInventory = { * single-screen behavior and never route through a display selector. */ multiScreen: boolean; + /** + * The panel a multi-panel capture names. Absent on a multi-panel device when + * neither panel power nor CoreDevice `primary` identifies a panel: list order is + * not identity, so the capture refuses instead. + */ activeDisplay?: AppleDeviceDisplay; /** * True on a multi-panel device whose lit panel could not be decided, so the @@ -258,7 +264,7 @@ export function buildInventory(displays: AppleDeviceDisplay[]): AppleDisplayInve // choice is the highest screen ID, which is exactly how a foldable capture comes // back all black and exits 0. When panel power is ambiguous the primary panel is // named explicitly and the ambiguity is reported, not hidden. - const captureDisplay = decided ? litPanels[0]! : (primaryPanel ?? litPanels[0] ?? panels[0]!); + const captureDisplay = decided ? litPanels[0]! : primaryPanel; const ambiguous = !decided || primaryPanel === undefined; if (ambiguous) { emitDiagnostic({ @@ -267,20 +273,27 @@ export function buildInventory(displays: AppleDeviceDisplay[]): AppleDisplayInve data: { reason: primaryPanel === undefined ? 'no-primary-panel' : 'lit-panel-count', litPanelCount: litPanels.length, - capturedDisplay: captureDisplay.name, - hint: 'Panel power did not identify exactly one lit panel, so the capture names the primary panel without claiming to know which panel the device is showing. Read the device pose from the app under test with UIHinge.status.', + ...(captureDisplay + ? { + capturedDisplay: captureDisplay.name, + hint: 'Panel power did not identify exactly one lit panel, so the capture names the primary panel without claiming to know which panel the device is showing. Read the device pose from the app under test with UIHinge.status.', + } + : { hint: UNIDENTIFIED_PANEL_HINT }), }, }); } return { displays: panels, multiScreen: true, - activeDisplay: captureDisplay, + ...(captureDisplay ? { activeDisplay: captureDisplay } : {}), ambiguous, unresolved: false, }; } +const UNIDENTIFIED_PANEL_HINT = + 'CoreDevice reported no primary panel and panel power did not identify exactly one lit panel. Run `xcrun devicectl device info displays --device `, then retry after one panel reports a lit backlightState.'; + /** * Sanity-checks the panel geometry against a foldable's shape. * @@ -315,8 +328,9 @@ function warnWhenPrimaryPanelIsNotTheSmallest(panels: AppleDeviceDisplay[]): voi * `simctl io screenshot` without `--display` silently picks the highest screen * ID, which on a closed foldable is the *dark* inner panel: the capture succeeds * and comes back black. A multi-panel device therefore always names a display - * explicitly — even when panel power is ambiguous — and only a genuinely - * single-panel device (or an unreadable toolchain) keeps the historic argv. + * explicitly — even when panel power is ambiguous — or refuses when CoreDevice + * identifies no panel. Only a genuinely single-panel device (or an unreadable + * toolchain) keeps the historic argv. */ export async function resolveAppleCaptureDisplay( device: DeviceInfo, @@ -336,7 +350,17 @@ export async function resolveAppleCaptureTarget( const inventory = await queryAppleDisplayInventory(device, options); if (inventory.unresolved) return {}; if (inventory.multiScreen) { - return inventory.activeDisplay ? { display: inventory.activeDisplay } : {}; + if (inventory.activeDisplay) return { display: inventory.activeDisplay }; + throw new AppError( + 'COMMAND_FAILED', + `CoreDevice did not identify which panel of ${device.name} to capture`, + { + deviceId: device.id, + reason: 'capture-panel-unidentified', + litPanelCount: inventory.displays.filter((display) => display.power === 'lit').length, + hint: UNIDENTIFIED_PANEL_HINT, + }, + ); } // Only a built-in panel is the screen simctl captures; an attached display says nothing about it. const panel = inventory.displays.find((entry) => entry.integrated); diff --git a/packages/platform-apple/src/os/macos/helper.test.ts b/packages/platform-apple/src/os/macos/helper.test.ts index 777c0caa92..baacda831b 100644 --- a/packages/platform-apple/src/os/macos/helper.test.ts +++ b/packages/platform-apple/src/os/macos/helper.test.ts @@ -1,21 +1,26 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { macOsHelperSurface } from '@agent-device/contracts/session'; +import { AppError } from '@agent-device/kernel/errors'; import { createLocalAppleToolProvider, withAppleToolProvider } from '../../core/tool-provider.ts'; import { readFileSync } from 'node:fs'; import { MACOS_DELIVERY_MECHANISMS, MACOS_HELPER_REFUSAL_REASONS, + MACOS_WINDOW_LIST_REFUSAL_REASONS, + MACOS_SCREENSHOT_REFUSAL_REASONS, macOsClickScheduleMs, runMacOsPressAction, runMacOsReadTextAction, runMacOsScreenshotAction, runMacOsSnapshotAction, + runMacOsScrollAction, } from './helper.ts'; const desktop = macOsHelperSurface('desktop', 'xctest')!; const menubar = macOsHelperSurface('menubar', 'xctest')!; const frontmostApp = macOsHelperSurface('frontmost-app', 'xctest')!; +const nativeApp = macOsHelperSurface('app', 'native')!; test('macOS helper snapshot passes cancellation to the helper process', async () => { const controller = new AbortController(); @@ -241,6 +246,56 @@ test('macOS helper screenshot argv carries only --out and --surface', async () = assert.deepEqual(receivedArgs, ['screenshot', '--out', '/tmp/out.png', '--surface', 'desktop']); }); +for (const reason of ['front-window-not-shareable', 'front-window-capture-excluded', undefined]) { + test(`an app screenshot refusal gives the remedy for ${reason}`, async () => { + const provider = createLocalAppleToolProvider({ + macosHelper: { + run: async () => ({ + exitCode: 1, + stdout: JSON.stringify({ + ok: false, + error: { + message: + "screenshot could not capture the app's front window: ScreenCaptureKit does not offer it", + details: { + reason, + bundleId: 'com.apple.TextEdit', + frontWindowId: '4242', + }, + }, + }), + stderr: '', + }), + }, + }); + + const error = await withAppleToolProvider(provider, async () => + runMacOsScreenshotAction('/tmp/out.png', { + surface: nativeApp, + bundleId: 'com.apple.TextEdit', + }).then( + () => undefined, + (error: unknown) => error, + ), + ); + + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.details?.reason, reason); + assert.equal(error.details?.frontWindowId, '4242'); + if (reason === 'front-window-capture-excluded') { + assert.match(String(error.details?.hint), /Run `snapshot` to read the window/); + assert.match(String(error.details?.hint), /retrying the excluded window will not help/); + assert.doesNotMatch(String(error.details?.hint), /retry after/); + } else if (reason === 'front-window-not-shareable') { + assert.match(String(error.details?.hint), /retry after it settles/); + assert.doesNotMatch(String(error.details?.hint), /excludes/); + } else { + assert.equal(error.details?.hint, undefined); + } + }); +} + test('helper entry points accept only an owner-routed surface', () => { // Never invoked: each directive fails typecheck once its entry point widens back to an // unbranded or optional surface. @@ -259,6 +314,51 @@ test('helper entry points accept only an owner-routed surface', () => { assert.equal(typeof widenedCalls, 'function'); }); +for (const action of ['screenshot', 'scroll'] as const) { + for (const reason of ['window-list-unreadable', undefined]) { + test(`${action} maps an unreadable window list by reason, not error text (${reason})`, async () => { + const provider = createLocalAppleToolProvider({ + macosHelper: { + run: async () => ({ + exitCode: 1, + stdout: JSON.stringify({ + ok: false, + error: { + message: "the app's on-screen windows could not be read", + details: { reason }, + }, + }), + stderr: '', + }), + }, + }); + await assert.rejects( + withAppleToolProvider(provider, async () => + action === 'screenshot' + ? runMacOsScreenshotAction('/tmp/out.png', { + surface: nativeApp, + bundleId: 'com.apple.TextEdit', + }) + : runMacOsScrollAction('down', { bundleId: 'com.apple.TextEdit' }), + ), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.details?.reason, reason); + if (reason) { + assert.match(String(error.details?.hint), /window list could not be read/); + assert.match(String(error.details?.hint), /Retry the command/); + assert.doesNotMatch(String(error.details?.hint), /Open or unminimize|XCTest/); + } else { + assert.equal(error.details?.hint, undefined); + } + return true; + }, + ); + }); + } +} + test('the helper vocabulary matches the table the Swift helper is held to', () => { const fixture = JSON.parse( readFileSync( @@ -268,7 +368,14 @@ test('the helper vocabulary matches the table the Swift helper is held to', () = ), 'utf8', ), - ) as { refusalReasons: string[]; deliveryMechanisms: string[] }; + ) as { + refusalReasons: string[]; + deliveryMechanisms: string[]; + screenshotRefusalReasons: string[]; + windowListRefusalReasons: string[]; + }; + assert.deepEqual([...MACOS_WINDOW_LIST_REFUSAL_REASONS], fixture.windowListRefusalReasons); assert.deepEqual([...MACOS_HELPER_REFUSAL_REASONS], fixture.refusalReasons); assert.deepEqual([...MACOS_DELIVERY_MECHANISMS], fixture.deliveryMechanisms); + assert.deepEqual([...MACOS_SCREENSHOT_REFUSAL_REASONS], fixture.screenshotRefusalReasons); }); diff --git a/packages/platform-apple/src/os/macos/helper.ts b/packages/platform-apple/src/os/macos/helper.ts index 97cc51f146..6f395dd41d 100644 --- a/packages/platform-apple/src/os/macos/helper.ts +++ b/packages/platform-apple/src/os/macos/helper.ts @@ -142,6 +142,25 @@ export const MACOS_HELPER_REFUSAL_REASONS = [ 'no-settable-text-input', 'no-scroll-bar', ] as const; +/** + * Why the helper refused an app-surface screenshot. It captures the app's front window or + * nothing, so each refusal names what kept that window out of reach. + */ +export const MACOS_SCREENSHOT_REFUSAL_REASONS = [ + 'window-not-found', + 'front-window-not-shareable', + 'front-window-capture-excluded', +] as const; +type MacOsScreenshotRefusalReason = (typeof MACOS_SCREENSHOT_REFUSAL_REASONS)[number]; +const MACOS_SCREENSHOT_REFUSAL_HINTS: Record = { + 'window-not-found': + 'Open or unminimize a window of the app on the current Space, then retry the screenshot.', + 'front-window-not-shareable': + 'The front window is unavailable to screen capture. If it is opening or closing, retry after it settles.', + 'front-window-capture-excluded': + "The app excludes the front window from screen capture. Run `snapshot` to read the window's UI tree; retrying the excluded window will not help.", +}; +export const MACOS_WINDOW_LIST_REFUSAL_REASONS = ['window-list-unreadable'] as const; export const MACOS_DELIVERY_MECHANISMS = [ 'ax-press', 'ax-focus', @@ -354,6 +373,7 @@ async function runMacOsHelper>( parsed && !parsed.ok ? (parsed.error?.message ?? `macOS helper exited with code ${result.exitCode}`) : stdout || result.stderr.trim() || `macOS helper exited with code ${result.exitCode}`; + const details = parsed && !parsed.ok ? parsed.error?.details : undefined; // exec-guard-allow: the message is already built from the helper's JSON error // envelope (or its output); a stderr excerpt would only degrade it, and the // throw is reachable at exit 0 when the envelope reports ok=false. @@ -363,7 +383,10 @@ async function runMacOsHelper>( stdout: result.stdout, stderr: result.stderr, exitCode: result.exitCode, - ...(parsed && !parsed.ok ? parsed.error?.details : {}), + ...details, + ...(details?.reason === MACOS_WINDOW_LIST_REFUSAL_REASONS[0] + ? { hint: "The app's on-screen window list could not be read. Retry the command." } + : {}), }); } @@ -581,5 +604,25 @@ export async function runMacOsScreenshotAction( }> { const args = ['screenshot', '--out', outPath]; appendMacOsHelperContextArgs(args, options); - return await runMacOsHelper(args, { signal: options.signal }); + try { + return await runMacOsHelper(args, { signal: options.signal }); + } catch (error) { + throw withScreenshotRefusalHint(error); + } +} + +function withScreenshotRefusalHint(error: unknown): unknown { + if (!(error instanceof AppError)) return error; + const reason = error.details?.reason; + if (!isMacOsScreenshotRefusalReason(reason)) return error; + return new AppError( + error.code, + error.message, + { ...error.details, hint: MACOS_SCREENSHOT_REFUSAL_HINTS[reason] }, + error, + ); +} + +function isMacOsScreenshotRefusalReason(value: unknown): value is MacOsScreenshotRefusalReason { + return (MACOS_SCREENSHOT_REFUSAL_REASONS as readonly unknown[]).includes(value); } diff --git a/src/commands/capture/screenshot.ts b/src/commands/capture/screenshot.ts index 13ab745d18..dacf530362 100644 --- a/src/commands/capture/screenshot.ts +++ b/src/commands/capture/screenshot.ts @@ -74,7 +74,7 @@ export const screenshotCommandFacet = defineCommandFacet({ text: { summary: 'Capture a screenshot', cliDetail: - 'Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators and Android emulators), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator chrome, or --no-stabilize for low-latency Android capture loops. On macOS, any --surface other than app (desktop, menubar, frontmost-app) always captures the main display and refuses an explicit --fullscreen. On single-display iOS simulators and Android devices, the result reports displayRotation, the rotation the display rendered the capture in, when the device returns a reading in time; otherwise the field is absent.', + "Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators and Android emulators), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator chrome, or --no-stabilize for low-latency Android capture loops. With the native macOS backend, the front window is the app's frontmost visible, nonempty normal-level window on the current Space; floating palettes are skipped, and a sheet is captured as its own window. Only layer-0 windows count as the front window; app-modal alerts at window level 8 are skipped. On macOS, any --surface other than app (desktop, menubar, frontmost-app) always captures the main display and refuses an explicit --fullscreen. On single-display iOS simulators and Android devices, the result reports displayRotation, the rotation the display rendered the capture in, when the device returns a reading in time; otherwise the field is absent.", }, metadata: screenshotCommandMetadata, run: (client, input) => client.capture.screenshot(input), diff --git a/src/commands/system/index.ts b/src/commands/system/index.ts index 7f17fd3da9..16aad73a84 100644 --- a/src/commands/system/index.ts +++ b/src/commands/system/index.ts @@ -278,7 +278,7 @@ export const foldableHelpTopic = { A foldable Apple device (iPhone Duo) carries two integrated panels, which Apple calls the outer display and the inner display. Only one is lit at a time, and which one is lit is the device pose. Screens are handled for you: - Each iOS simulator capture resolves the CoreDevice display table, captures the lit panel explicitly, and normalizes density with that panel's own point scale. Do not add a screen flag to the normal loop; there is none, because the lit panel is always the only capturable one: the dark panel yields an all-black PNG. + Each iOS simulator capture resolves the CoreDevice display table, selects the sole lit panel when power identifies it, and normalizes density with that panel's own point scale. There is no screen flag. When panel power names no single lit panel, the capture names CoreDevice's primary panel, which may be dark and yield an all-black PNG. Check the pose with UIHinge.status before trusting that capture. With no primary panel either, capture fails with COMMAND_FAILED and reason capture-panel-unidentified rather than pick a panel by list order. The two panels are different sizes (iPhone Duo: 466x678 points closed on the outer panel, 669x951 open on the inner). A pose change therefore invalidates every ref and coordinate. Re-snapshot after any pose change and never carry coordinates or refs across one. Check which panel is lit before trusting a geometry claim: agent-device screenshot reports its point size, and 466x678 versus 669x951 says which panel you captured. diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index e0014faa7f..5a4adfe251 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -321,7 +321,7 @@ agent-device snapshot -i --platform apple --target desktop Set `AGENT_DEVICE_MACOS_APP_BACKEND=native` to run macOS app sessions without XCTest. There is no Automation Mode overlay, the app can stay behind other windows, and your pointer stays yours while a drawn pointer shows each action. -- `click`, `press`, and `fill` use accessibility actions on the element at the target point, which must be inside one of the app's on-screen windows, not its menu bar. `type` inserts at the focused control or falls back to key events. `scroll` moves the scroll bar under the front window's center. Chromium-based apps expose their full tree. `screenshot` captures only the app's front window, even when it is covered. +- `click`, `press`, and `fill` use accessibility actions on the element at the target point, which must be inside one of the app's on-screen windows, not its menu bar. `type` inserts at the focused control or falls back to key events. `scroll` moves the scroll bar under the front window's center. Chromium-based apps expose their full tree. `screenshot` captures only the app's front window, even when it is covered. The front window is the app's frontmost visible, nonempty normal-level window on the current Space; floating palettes are skipped, and a sheet is captured as its own window. Only layer-0 windows count as the front window; app-modal alerts at window level 8 are skipped. If that window cannot be captured, `screenshot` fails; it never captures another window. - `record`, `prepare`, `back`, `longpress`, `--hold-ms`, `--double-tap`, `--button secondary`, gestures, and clicks on an element with no accessibility action fail with `UNSUPPORTED_OPERATION` (`reason: unsupported-device-backend`). Use the default XCTest backend for those. ```bash