From 3fdd3aec9a45cf59db418f12fc2e501251760543 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 22:09:55 +0200 Subject: [PATCH 01/11] fix(macos): refuse an app screenshot whose front window is not shareable ADR 0031 decides that a native macOS app screenshot captures only the app's front window through ScreenCaptureKit (#3188). When the front window's WindowServer ID was missing from the shareable candidates, the helper captured the app's largest window instead, so the image could show a window the user was not looking at. The helper now selects the front window by ID or refuses with a typed reason: `window-not-found` when the app has no on-screen window, `front-window-not-shareable` when ScreenCaptureKit does not offer the front window. The shared outcomes fixture pins both reasons for Swift and TypeScript, and the host adds a hint for each. Evidence: the audit at bd4fd688e7 (AppWindowScreenshot.swift:21); re-verified on 4c1c786968. Part of #3744. --- .../AppWindowScreenshot.swift | 55 ++++++++++++++++--- .../AppWindowScreenshotTests.swift | 38 +++++++++++++ .../BackgroundInteractionTests.swift | 3 + .../macos-native-helper-outcomes.json | 5 +- .../src/os/macos/helper.test.ts | 49 ++++++++++++++++- .../platform-apple/src/os/macos/helper.ts | 37 ++++++++++++- website/docs/docs/commands.md | 2 +- 7 files changed, 176 insertions(+), 13 deletions(-) create mode 100644 apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift index da4c6247c7..d2b2522b07 100644 --- a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift +++ b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift @@ -16,14 +16,14 @@ func captureAppWindowScreenshot(app: NSRunningApplication, outPath: String) thro $0.owningApplication?.processID == pid && $0.windowLayer == 0 && $0.frame.width > 0 && $0.frame.height > 0 } - guard - let window = candidates.first(where: { Int($0.windowID) == frontWindowID }) - ?? candidates.max(by: { $0.frame.width * $0.frame.height < $1.frame.width * $1.frame.height }) - else { - throw HelperError.commandFailed( - "screenshot could not find a window for the app", - details: ["reason": "window-not-found", "bundleId": app.bundleIdentifier ?? ""] - ) + let window: SCWindow + let selection = frontCaptureWindow( + candidates, frontWindowID: frontWindowID, windowID: { Int($0.windowID) }) + switch selection { + case .success(let front): + window = front + case .failure(let refusal): + throw screenshotRefusal(refusal, app: app, frontWindowID: frontWindowID) } let filter = SCContentFilter(desktopIndependentWindow: window) @@ -52,6 +52,45 @@ func captureAppWindowScreenshot(app: NSRunningApplication, outPath: String) thro try writePNG(capturedImage, to: outPath) } +/// Why an app screenshot refuses. The capture is the app's front window or nothing: another +/// window of the app would show state the user is not looking at. Pinned with the host by +/// `contracts/fixtures/macos-native-helper-outcomes.json`. +enum AppWindowScreenshotRefusal: String, CaseIterable, Error { + case windowNotFound = "window-not-found" + case frontWindowNotShareable = "front-window-not-shareable" +} + +/// The shareable candidate whose WindowServer ID is the app's front window. +func frontCaptureWindow( + _ candidates: [Window], + frontWindowID: Int?, + windowID: (Window) -> Int +) -> Result { + guard let frontWindowID else { return .failure(.windowNotFound) } + guard let front = candidates.first(where: { windowID($0) == frontWindowID }) else { + return .failure(.frontWindowNotShareable) + } + return .success(front) +} + +private func screenshotRefusal( + _ refusal: AppWindowScreenshotRefusal, + app: NSRunningApplication, + frontWindowID: Int? +) -> HelperError { + var details = ["reason": refusal.rawValue, "bundleId": app.bundleIdentifier ?? ""] + switch refusal { + case .windowNotFound: + return .commandFailed("screenshot could not find an on-screen window for the app", details: details) + case .frontWindowNotShareable: + details["frontWindowId"] = frontWindowID.map(String.init) ?? "" + return .commandFailed( + "screenshot could not capture the app's front window: ScreenCaptureKit does not offer it", + details: details + ) + } +} + @available(macOS 14.0, *) private func awaitShareableContent() throws -> SCShareableContent { let semaphore = DispatchSemaphore(value: 0) diff --git a/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift b/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift new file mode 100644 index 0000000000..8ab7e3e278 --- /dev/null +++ b/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift @@ -0,0 +1,38 @@ +import XCTest + +@testable import AgentDeviceMacOSHelper + +private struct Candidate: Equatable { + let id: Int + let area: Double +} + +/// An app screenshot captures the front window or refuses; it never substitutes another window. +final class AppWindowScreenshotTests: XCTestCase { + func testSelectsTheFrontWindowEvenWhenAnotherIsLarger() { + let candidates = [Candidate(id: 7, area: 10_000), Candidate(id: 3, area: 100)] + + let selected = frontCaptureWindow(candidates, frontWindowID: 3, windowID: \.id) + + XCTAssertEqual(try selected.get(), Candidate(id: 3, area: 100)) + } + + func testRefusesWhenTheFrontWindowIsNotShareable() { + let candidates = [Candidate(id: 7, area: 10_000), Candidate(id: 8, area: 100)] + + let selected = frontCaptureWindow(candidates, frontWindowID: 3, windowID: \.id) + + XCTAssertThrowsError(try selected.get()) { error in + XCTAssertEqual(error as? AppWindowScreenshotRefusal, .frontWindowNotShareable) + } + } + + func testRefusesWhenTheAppHasNoOnScreenWindow() { + let selected = frontCaptureWindow( + [Candidate(id: 7, area: 10_000)], frontWindowID: nil, windowID: \.id) + + XCTAssertThrowsError(try selected.get()) { error in + XCTAssertEqual(error as? AppWindowScreenshotRefusal, .windowNotFound) + } + } +} diff --git a/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/BackgroundInteractionTests.swift b/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/BackgroundInteractionTests.swift index 7b61be5fc3..20445946df 100644 --- a/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/BackgroundInteractionTests.swift +++ b/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/BackgroundInteractionTests.swift @@ -26,6 +26,7 @@ private struct ScrollGestureFixture: Decodable { private struct HelperOutcomesFixture: Decodable { let refusalReasons: [String] let deliveryMechanisms: [String] + let screenshotRefusalReasons: [String] } /// `#filePath` is `/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/`. @@ -43,6 +44,8 @@ final class BackgroundInteractionTests: XCTestCase { ) XCTAssertEqual(BackgroundRefusal.allCases.map(\.rawValue), fixture.refusalReasons) XCTAssertEqual(BackgroundDeliveryMechanism.allCases.map(\.rawValue), fixture.deliveryMechanisms) + XCTAssertEqual( + AppWindowScreenshotRefusal.allCases.map(\.rawValue), fixture.screenshotRefusalReasons) } /// The native backend scrolls as far as the runner does: its travel agrees with every case of diff --git a/contracts/fixtures/macos-native-helper-outcomes.json b/contracts/fixtures/macos-native-helper-outcomes.json index be60f68c9b..c4337931fd 100644 --- a/contracts/fixtures/macos-native-helper-outcomes.json +++ b/contracts/fixtures/macos-native-helper-outcomes.json @@ -1,5 +1,5 @@ { - "description": "Vocabulary the macOS helper reports for app-surface actions. Swift: BackgroundRefusal and BackgroundDeliveryMechanism in apple/macos-helper. TypeScript: the macOS helper client in packages/platform-apple/src/os/macos/helper.ts.", + "description": "Vocabulary the macOS helper reports for app-surface actions. Swift: BackgroundRefusal, BackgroundDeliveryMechanism, and AppWindowScreenshotRefusal in apple/macos-helper. TypeScript: the macOS helper client in packages/platform-apple/src/os/macos/helper.ts.", "refusalReasons": [ "background-pointer-gesture", "no-accessible-target", @@ -14,5 +14,6 @@ "ax-confirm", "ax-scroll-bar", "key-events" - ] + ], + "screenshotRefusalReasons": ["window-not-found", "front-window-not-shareable"] } diff --git a/packages/platform-apple/src/os/macos/helper.test.ts b/packages/platform-apple/src/os/macos/helper.test.ts index 777c0caa92..92857ed84d 100644 --- a/packages/platform-apple/src/os/macos/helper.test.ts +++ b/packages/platform-apple/src/os/macos/helper.test.ts @@ -1,11 +1,13 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { macOsHelperSurface } from '@agent-device/contracts/session'; +import { AppError } from '@agent-device/kernel/errors'; import { createLocalAppleToolProvider, withAppleToolProvider } from '../../core/tool-provider.ts'; import { readFileSync } from 'node:fs'; import { MACOS_DELIVERY_MECHANISMS, MACOS_HELPER_REFUSAL_REASONS, + MACOS_SCREENSHOT_REFUSAL_REASONS, macOsClickScheduleMs, runMacOsPressAction, runMacOsReadTextAction, @@ -16,6 +18,7 @@ import { const desktop = macOsHelperSurface('desktop', 'xctest')!; const menubar = macOsHelperSurface('menubar', 'xctest')!; const frontmostApp = macOsHelperSurface('frontmost-app', 'xctest')!; +const nativeApp = macOsHelperSurface('app', 'native')!; test('macOS helper snapshot passes cancellation to the helper process', async () => { const controller = new AbortController(); @@ -241,6 +244,45 @@ test('macOS helper screenshot argv carries only --out and --surface', async () = assert.deepEqual(receivedArgs, ['screenshot', '--out', '/tmp/out.png', '--surface', 'desktop']); }); +test('an app screenshot that cannot reach the front window refuses with its reason and a hint', async () => { + const provider = createLocalAppleToolProvider({ + macosHelper: { + run: async () => ({ + exitCode: 1, + stdout: JSON.stringify({ + ok: false, + error: { + message: + "screenshot could not capture the app's front window: ScreenCaptureKit does not offer it", + details: { + reason: 'front-window-not-shareable', + bundleId: 'com.apple.TextEdit', + frontWindowId: '4242', + }, + }, + }), + stderr: '', + }), + }, + }); + + const error = await withAppleToolProvider(provider, async () => + runMacOsScreenshotAction('/tmp/out.png', { + surface: nativeApp, + bundleId: 'com.apple.TextEdit', + }).then( + () => undefined, + (error: unknown) => error, + ), + ); + + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.details?.reason, 'front-window-not-shareable'); + assert.equal(error.details?.frontWindowId, '4242'); + assert.match(String(error.details?.hint), /Retry after the window settles/); +}); + test('helper entry points accept only an owner-routed surface', () => { // Never invoked: each directive fails typecheck once its entry point widens back to an // unbranded or optional surface. @@ -268,7 +310,12 @@ test('the helper vocabulary matches the table the Swift helper is held to', () = ), 'utf8', ), - ) as { refusalReasons: string[]; deliveryMechanisms: string[] }; + ) as { + refusalReasons: string[]; + deliveryMechanisms: string[]; + screenshotRefusalReasons: string[]; + }; assert.deepEqual([...MACOS_HELPER_REFUSAL_REASONS], fixture.refusalReasons); assert.deepEqual([...MACOS_DELIVERY_MECHANISMS], fixture.deliveryMechanisms); + assert.deepEqual([...MACOS_SCREENSHOT_REFUSAL_REASONS], fixture.screenshotRefusalReasons); }); diff --git a/packages/platform-apple/src/os/macos/helper.ts b/packages/platform-apple/src/os/macos/helper.ts index 97cc51f146..8f7de6a1b8 100644 --- a/packages/platform-apple/src/os/macos/helper.ts +++ b/packages/platform-apple/src/os/macos/helper.ts @@ -142,6 +142,21 @@ export const MACOS_HELPER_REFUSAL_REASONS = [ 'no-settable-text-input', 'no-scroll-bar', ] as const; +/** + * Why the helper refused an app-surface screenshot. It captures the app's front window or + * nothing, so each refusal names what kept that window out of reach. + */ +export const MACOS_SCREENSHOT_REFUSAL_REASONS = [ + 'window-not-found', + 'front-window-not-shareable', +] as const; +type MacOsScreenshotRefusalReason = (typeof MACOS_SCREENSHOT_REFUSAL_REASONS)[number]; +const MACOS_SCREENSHOT_REFUSAL_HINTS: Record = { + 'window-not-found': + 'Open or unminimize a window of the app on the current Space, then retry the screenshot.', + 'front-window-not-shareable': + 'The front window may still be opening or closing, or the app excludes it from screen capture. Retry after the window settles.', +}; export const MACOS_DELIVERY_MECHANISMS = [ 'ax-press', 'ax-focus', @@ -581,5 +596,25 @@ export async function runMacOsScreenshotAction( }> { const args = ['screenshot', '--out', outPath]; appendMacOsHelperContextArgs(args, options); - return await runMacOsHelper(args, { signal: options.signal }); + try { + return await runMacOsHelper(args, { signal: options.signal }); + } catch (error) { + throw withScreenshotRefusalHint(error); + } +} + +function withScreenshotRefusalHint(error: unknown): unknown { + if (!(error instanceof AppError)) return error; + const reason = error.details?.reason; + if (!isMacOsScreenshotRefusalReason(reason)) return error; + return new AppError( + error.code, + error.message, + { ...error.details, hint: MACOS_SCREENSHOT_REFUSAL_HINTS[reason] }, + error, + ); +} + +function isMacOsScreenshotRefusalReason(value: unknown): value is MacOsScreenshotRefusalReason { + return (MACOS_SCREENSHOT_REFUSAL_REASONS as readonly unknown[]).includes(value); } diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index e0014faa7f..8ce4a66430 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -321,7 +321,7 @@ agent-device snapshot -i --platform apple --target desktop Set `AGENT_DEVICE_MACOS_APP_BACKEND=native` to run macOS app sessions without XCTest. There is no Automation Mode overlay, the app can stay behind other windows, and your pointer stays yours while a drawn pointer shows each action. -- `click`, `press`, and `fill` use accessibility actions on the element at the target point, which must be inside one of the app's on-screen windows, not its menu bar. `type` inserts at the focused control or falls back to key events. `scroll` moves the scroll bar under the front window's center. Chromium-based apps expose their full tree. `screenshot` captures only the app's front window, even when it is covered. +- `click`, `press`, and `fill` use accessibility actions on the element at the target point, which must be inside one of the app's on-screen windows, not its menu bar. `type` inserts at the focused control or falls back to key events. `scroll` moves the scroll bar under the front window's center. Chromium-based apps expose their full tree. `screenshot` captures only the app's front window, even when it is covered. If that window cannot be captured, `screenshot` fails; it never captures another window. - `record`, `prepare`, `back`, `longpress`, `--hold-ms`, `--double-tap`, `--button secondary`, gestures, and clicks on an element with no accessibility action fail with `UNSUPPORTED_OPERATION` (`reason: unsupported-device-backend`). Use the default XCTest backend for those. ```bash From dec7c9ba418bcc23875a3ea26732304028253a9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 22:10:01 +0200 Subject: [PATCH 02/11] fix(apple): refuse a foldable capture when CoreDevice names no panel ADR 0025 decides that only CoreDevice facts identify a panel: the sole lit panel, otherwise the `primary` panel, and never array position. When no panel reported `primary` and power did not decide one lit panel, the inventory captured the first lit or first integrated panel in list order. That inventory now names no panel, and capture-target resolution refuses with COMMAND_FAILED, reason `capture-panel-unidentified`, and a hint. The sole-lit-panel path and the primary-panel path are unchanged. The foldable help topic states the refusal. Evidence: the audit at bd4fd688e7 (display-inventory.ts:261); re-verified on 4c1c786968. Part of #3744. --- .../core/__tests__/display-inventory.test.ts | 71 +++++++++++++++++-- .../src/core/display-inventory.ts | 38 ++++++++-- src/commands/system/index.ts | 2 +- 3 files changed, 99 insertions(+), 12 deletions(-) diff --git a/packages/platform-apple/src/core/__tests__/display-inventory.test.ts b/packages/platform-apple/src/core/__tests__/display-inventory.test.ts index a476dafec9..416048de0f 100644 --- a/packages/platform-apple/src/core/__tests__/display-inventory.test.ts +++ b/packages/platform-apple/src/core/__tests__/display-inventory.test.ts @@ -266,15 +266,26 @@ describe('buildInventory', () => { } }); - test('names a capture even when no panel reports primary', () => { + test('names the sole lit panel when no panel reports primary', () => { const inventory = buildInventory([ - { ...outerPanel, primary: false }, + { ...outerPanel, power: 'dark', primary: false }, { ...innerPanel, power: 'lit', primary: false }, ]); assert.equal(inventory.multiScreen, true); assert.equal(inventory.ambiguous, true); - // A capture is still named: never the implicit black-producing default. - assert.ok(inventory.activeDisplay); + assert.equal(inventory.activeDisplay?.name, 'LCD-1'); + }); + + test('names no panel when neither primary nor power identifies one', () => { + for (const power of ['lit', 'dark', 'unknown'] as const) { + const inventory = buildInventory([ + { ...outerPanel, power, primary: false }, + { ...innerPanel, power, primary: false }, + ]); + assert.equal(inventory.multiScreen, true); + assert.equal(inventory.ambiguous, true); + assert.equal(inventory.activeDisplay, undefined, power); + } }); test('keeps choosing the lit panel when a third panel is present', () => { @@ -476,6 +487,58 @@ describe('resolveAppleCaptureTarget', () => { assert.equal(target.displayRotation, undefined); }); + /** The closed-Duo payload with each panel's `primary` flag and backlight replaced. */ + function duoReporting(panels: Record) { + return JSON.stringify({ + result: { + displays: IPHONE_DUO_CLOSED.result.displays.map((entry) => ({ + ...entry, + ...panels[entry.displayId], + })), + }, + }); + } + + test('names the primary panel when panel power is ambiguous', async () => { + mockReadHostTextFile.mockResolvedValue( + duoReporting({ + 1: { primary: true, backlightState: 'activeOn' }, + 3: { primary: false, backlightState: 'activeOn' }, + }), + ); + assert.equal((await resolveAppleCaptureTarget(IOS_TEST_SIMULATOR)).display?.name, 'LCD'); + }); + + test('names the sole lit panel when no panel reports primary', async () => { + mockReadHostTextFile.mockResolvedValue( + duoReporting({ + 1: { primary: false, backlightState: 'off' }, + 3: { primary: false, backlightState: 'activeOn' }, + }), + ); + assert.equal((await resolveAppleCaptureTarget(IOS_TEST_SIMULATOR)).display?.name, 'LCD-1'); + }); + + test('refuses instead of picking a panel by list order when no fact identifies one', async () => { + for (const backlightState of ['activeOn', 'off', 'futureState']) { + mockReadHostTextFile.mockResolvedValue( + duoReporting({ + 1: { primary: false, backlightState }, + 3: { primary: false, backlightState }, + }), + ); + await assert.rejects( + resolveAppleCaptureTarget(IOS_TEST_SIMULATOR), + (error: unknown) => + error instanceof AppError && + error.code === 'COMMAND_FAILED' && + error.details?.reason === 'capture-panel-unidentified' && + typeof error.details?.hint === 'string', + backlightState, + ); + } + }); + test('reports nothing when CoreDevice cannot be asked', async () => { mockRunXcrun.mockResolvedValueOnce({ exitCode: 1, diff --git a/packages/platform-apple/src/core/display-inventory.ts b/packages/platform-apple/src/core/display-inventory.ts index 6dd4fbee2a..1eee41c9f1 100644 --- a/packages/platform-apple/src/core/display-inventory.ts +++ b/packages/platform-apple/src/core/display-inventory.ts @@ -1,5 +1,6 @@ import type { DeviceRotation } from '@agent-device/contracts/device'; import type { DeviceInfo } from '@agent-device/kernel/device'; +import { AppError } from '@agent-device/kernel/errors'; import { isCommandTimeoutError } from '@agent-device/host-kit/command'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; @@ -45,6 +46,11 @@ export type AppleDisplayInventory = { * single-screen behavior and never route through a display selector. */ multiScreen: boolean; + /** + * The panel a multi-panel capture names. Absent on a multi-panel device when + * neither panel power nor CoreDevice `primary` identifies a panel: list order is + * not identity, so the capture refuses instead. + */ activeDisplay?: AppleDeviceDisplay; /** * True on a multi-panel device whose lit panel could not be decided, so the @@ -258,7 +264,7 @@ export function buildInventory(displays: AppleDeviceDisplay[]): AppleDisplayInve // choice is the highest screen ID, which is exactly how a foldable capture comes // back all black and exits 0. When panel power is ambiguous the primary panel is // named explicitly and the ambiguity is reported, not hidden. - const captureDisplay = decided ? litPanels[0]! : (primaryPanel ?? litPanels[0] ?? panels[0]!); + const captureDisplay = decided ? litPanels[0]! : primaryPanel; const ambiguous = !decided || primaryPanel === undefined; if (ambiguous) { emitDiagnostic({ @@ -267,20 +273,27 @@ export function buildInventory(displays: AppleDeviceDisplay[]): AppleDisplayInve data: { reason: primaryPanel === undefined ? 'no-primary-panel' : 'lit-panel-count', litPanelCount: litPanels.length, - capturedDisplay: captureDisplay.name, - hint: 'Panel power did not identify exactly one lit panel, so the capture names the primary panel without claiming to know which panel the device is showing. Read the device pose from the app under test with UIHinge.status.', + ...(captureDisplay + ? { + capturedDisplay: captureDisplay.name, + hint: 'Panel power did not identify exactly one lit panel, so the capture names the primary panel without claiming to know which panel the device is showing. Read the device pose from the app under test with UIHinge.status.', + } + : { hint: UNIDENTIFIED_PANEL_HINT }), }, }); } return { displays: panels, multiScreen: true, - activeDisplay: captureDisplay, + ...(captureDisplay ? { activeDisplay: captureDisplay } : {}), ambiguous, unresolved: false, }; } +const UNIDENTIFIED_PANEL_HINT = + 'CoreDevice reported no primary panel and panel power did not identify exactly one lit panel. Run `xcrun devicectl device info displays --device `, then retry after one panel reports a lit backlightState.'; + /** * Sanity-checks the panel geometry against a foldable's shape. * @@ -315,8 +328,9 @@ function warnWhenPrimaryPanelIsNotTheSmallest(panels: AppleDeviceDisplay[]): voi * `simctl io screenshot` without `--display` silently picks the highest screen * ID, which on a closed foldable is the *dark* inner panel: the capture succeeds * and comes back black. A multi-panel device therefore always names a display - * explicitly — even when panel power is ambiguous — and only a genuinely - * single-panel device (or an unreadable toolchain) keeps the historic argv. + * explicitly — even when panel power is ambiguous — or refuses when CoreDevice + * identifies no panel. Only a genuinely single-panel device (or an unreadable + * toolchain) keeps the historic argv. */ export async function resolveAppleCaptureDisplay( device: DeviceInfo, @@ -336,7 +350,17 @@ export async function resolveAppleCaptureTarget( const inventory = await queryAppleDisplayInventory(device, options); if (inventory.unresolved) return {}; if (inventory.multiScreen) { - return inventory.activeDisplay ? { display: inventory.activeDisplay } : {}; + if (inventory.activeDisplay) return { display: inventory.activeDisplay }; + throw new AppError( + 'COMMAND_FAILED', + `CoreDevice did not identify which panel of ${device.name} to capture`, + { + deviceId: device.id, + reason: 'capture-panel-unidentified', + litPanelCount: inventory.displays.filter((display) => display.power === 'lit').length, + hint: UNIDENTIFIED_PANEL_HINT, + }, + ); } // Only a built-in panel is the screen simctl captures; an attached display says nothing about it. const panel = inventory.displays.find((entry) => entry.integrated); diff --git a/src/commands/system/index.ts b/src/commands/system/index.ts index 7f17fd3da9..63ec1914d4 100644 --- a/src/commands/system/index.ts +++ b/src/commands/system/index.ts @@ -278,7 +278,7 @@ export const foldableHelpTopic = { A foldable Apple device (iPhone Duo) carries two integrated panels, which Apple calls the outer display and the inner display. Only one is lit at a time, and which one is lit is the device pose. Screens are handled for you: - Each iOS simulator capture resolves the CoreDevice display table, captures the lit panel explicitly, and normalizes density with that panel's own point scale. Do not add a screen flag to the normal loop; there is none, because the lit panel is always the only capturable one: the dark panel yields an all-black PNG. + Each iOS simulator capture resolves the CoreDevice display table, captures the lit panel explicitly, and normalizes density with that panel's own point scale. Do not add a screen flag to the normal loop; there is none, because the lit panel is always the only capturable one: the dark panel yields an all-black PNG. When panel power names no single lit panel, the capture names CoreDevice's primary panel; with no primary panel either, it fails with COMMAND_FAILED and reason capture-panel-unidentified rather than pick a panel by list order. The two panels are different sizes (iPhone Duo: 466x678 points closed on the outer panel, 669x951 open on the inner). A pose change therefore invalidates every ref and coordinate. Re-snapshot after any pose change and never carry coordinates or refs across one. Check which panel is lit before trusting a geometry claim: agent-device screenshot reports its point size, and 466x678 versus 669x951 says which panel you captured. From fa3fe4f774bf055c8b3b81818a626d5ca6c2c028 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 08:00:16 +0200 Subject: [PATCH 03/11] fix(macos): skip invisible windows when selecting a screenshot Choose the first on-screen normal-layer app window with positive alpha and nonempty bounds. Keep interaction window selection unchanged and refuse if an eligible front window is absent from ScreenCaptureKit. Review: https://github.com/callstack/agent-device/pull/3761#issuecomment-6226107623 --- .../AppWindowScreenshot.swift | 21 +++++++- .../BackgroundInteraction.swift | 4 -- .../AppWindowScreenshotTests.swift | 50 +++++++++++++++++++ 3 files changed, 70 insertions(+), 5 deletions(-) diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift index d2b2522b07..3bad70c649 100644 --- a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift +++ b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift @@ -10,7 +10,9 @@ func captureAppWindowScreenshot(app: NSRunningApplication, outPath: String) thro throw HelperError.commandFailed("app window screenshots require macOS 14 or newer") } let pid = app.processIdentifier - let frontWindowID = frontWindowNumber(pid: pid) + let windowInfo = CGWindowListCopyWindowInfo( + [.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] ?? [] + let frontWindowID = frontCaptureWindowNumber(windowInfo, pid: pid) let content = try awaitShareableContent() let candidates = content.windows.filter { $0.owningApplication?.processID == pid && $0.windowLayer == 0 && $0.frame.width > 0 @@ -52,6 +54,23 @@ func captureAppWindowScreenshot(app: NSRunningApplication, outPath: String) thro try writePNG(capturedImage, to: outPath) } +/// WindowServer returns windows front to back. Invisible and empty windows cannot be capture targets. +func frontCaptureWindowNumber(_ windowInfo: [[String: Any]], pid: pid_t) -> Int? { + for entry in windowInfo { + guard (entry[kCGWindowOwnerPID as String] as? Int32) == pid, + (entry[kCGWindowLayer as String] as? Int) == 0, + (entry[kCGWindowIsOnscreen as String] as? Bool) == true, + let alpha = entry[kCGWindowAlpha as String] as? Double, alpha > 0, + let boundsDict = entry[kCGWindowBounds as String] as? NSDictionary, + let bounds = CGRect(dictionaryRepresentation: boundsDict as CFDictionary), + bounds.width > 0, bounds.height > 0, + let number = entry[kCGWindowNumber as String] as? Int + else { continue } + return number + } + return nil +} + /// Why an app screenshot refuses. The capture is the app's front window or nothing: another /// window of the app would show state the user is not looking at. Pinned with the host by /// `contracts/fixtures/macos-native-helper-outcomes.json`. diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift index 349426f54c..fe6f231124 100644 --- a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift +++ b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift @@ -509,10 +509,6 @@ private func onScreenWindows(pid: pid_t) -> [OnScreenWindow] { } } -func frontWindowNumber(pid: pid_t) -> Int? { - onScreenWindows(pid: pid).first?.number -} - private func frontWindowFrame(pid: pid_t) -> CGRect? { onScreenWindows(pid: pid).first?.bounds } diff --git a/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift b/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift index 8ab7e3e278..76a6e739e3 100644 --- a/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift +++ b/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift @@ -1,3 +1,4 @@ +import CoreGraphics import XCTest @testable import AgentDeviceMacOSHelper @@ -9,6 +10,41 @@ private struct Candidate: Equatable { /// An app screenshot captures the front window or refuses; it never substitutes another window. final class AppWindowScreenshotTests: XCTestCase { + func testSelectsTheFirstVisibleNonemptyNormalLayerWindowOfTheApp() { + let windows = [ + windowInfo(id: 1, pid: 99), + windowInfo(id: 2, layer: 1), + windowInfo(id: 3, onScreen: false), + windowInfo(id: 4, alpha: 0), + windowInfo(id: 5, width: 0), + windowInfo(id: 6, height: 0), + windowInfo(id: 7, alpha: 0.5, width: 20, height: 20), + windowInfo(id: 8), + ] + + let frontID = frontCaptureWindowNumber(windows, pid: 42) + let selected = frontCaptureWindow([7, 8], frontWindowID: frontID, windowID: { $0 }) + + XCTAssertEqual(frontID, 7) + XCTAssertEqual(try selected.get(), 7) + } + + func testRefusesWhenOnlyInvisibleOrEmptyWindowsRemain() { + let windows = [ + windowInfo(id: 1, onScreen: false), + windowInfo(id: 2, alpha: 0), + windowInfo(id: 3, width: 0), + windowInfo(id: 4, height: 0), + ] + let frontID = frontCaptureWindowNumber(windows, pid: 42) + let selected = frontCaptureWindow([5], frontWindowID: frontID, windowID: { $0 }) + + XCTAssertNil(frontID) + XCTAssertThrowsError(try selected.get()) { error in + XCTAssertEqual(error as? AppWindowScreenshotRefusal, .windowNotFound) + } + } + func testSelectsTheFrontWindowEvenWhenAnotherIsLarger() { let candidates = [Candidate(id: 7, area: 10_000), Candidate(id: 3, area: 100)] @@ -36,3 +72,17 @@ final class AppWindowScreenshotTests: XCTestCase { } } } + +private func windowInfo( + id: Int, pid: Int32 = 42, layer: Int = 0, onScreen: Bool = true, + alpha: Double = 1, width: Double = 100, height: Double = 100 +) -> [String: Any] { + [ + kCGWindowNumber as String: id, + kCGWindowOwnerPID as String: pid, + kCGWindowLayer as String: layer, + kCGWindowIsOnscreen as String: onScreen, + kCGWindowAlpha as String: alpha, + kCGWindowBounds as String: CGRect(x: 10, y: 10, width: width, height: height).dictionaryRepresentation, + ] +} From a77578f8ee6c10351b9625e7ecf63c78b81194e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 08:00:16 +0200 Subject: [PATCH 04/11] fix(macos): clarify screenshot capture exclusion hint Review: https://github.com/callstack/agent-device/pull/3761#discussion_r4298949007 --- packages/platform-apple/src/os/macos/helper.test.ts | 3 ++- packages/platform-apple/src/os/macos/helper.ts | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/platform-apple/src/os/macos/helper.test.ts b/packages/platform-apple/src/os/macos/helper.test.ts index 92857ed84d..aa6d017e95 100644 --- a/packages/platform-apple/src/os/macos/helper.test.ts +++ b/packages/platform-apple/src/os/macos/helper.test.ts @@ -280,7 +280,8 @@ test('an app screenshot that cannot reach the front window refuses with its reas assert.equal(error.code, 'COMMAND_FAILED'); assert.equal(error.details?.reason, 'front-window-not-shareable'); assert.equal(error.details?.frontWindowId, '4242'); - assert.match(String(error.details?.hint), /Retry after the window settles/); + assert.match(String(error.details?.hint), /retry after it settles/); + assert.match(String(error.details?.hint), /use a window that permits capture/); }); test('helper entry points accept only an owner-routed surface', () => { diff --git a/packages/platform-apple/src/os/macos/helper.ts b/packages/platform-apple/src/os/macos/helper.ts index 8f7de6a1b8..dba7cdae26 100644 --- a/packages/platform-apple/src/os/macos/helper.ts +++ b/packages/platform-apple/src/os/macos/helper.ts @@ -155,7 +155,7 @@ const MACOS_SCREENSHOT_REFUSAL_HINTS: Record Date: Sat, 17 Oct 2026 08:00:16 +0200 Subject: [PATCH 05/11] docs(foldable): clarify primary panel capture ambiguity Review: https://github.com/callstack/agent-device/pull/3761#discussion_r4298948982 --- src/commands/system/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/system/index.ts b/src/commands/system/index.ts index 63ec1914d4..16aad73a84 100644 --- a/src/commands/system/index.ts +++ b/src/commands/system/index.ts @@ -278,7 +278,7 @@ export const foldableHelpTopic = { A foldable Apple device (iPhone Duo) carries two integrated panels, which Apple calls the outer display and the inner display. Only one is lit at a time, and which one is lit is the device pose. Screens are handled for you: - Each iOS simulator capture resolves the CoreDevice display table, captures the lit panel explicitly, and normalizes density with that panel's own point scale. Do not add a screen flag to the normal loop; there is none, because the lit panel is always the only capturable one: the dark panel yields an all-black PNG. When panel power names no single lit panel, the capture names CoreDevice's primary panel; with no primary panel either, it fails with COMMAND_FAILED and reason capture-panel-unidentified rather than pick a panel by list order. + Each iOS simulator capture resolves the CoreDevice display table, selects the sole lit panel when power identifies it, and normalizes density with that panel's own point scale. There is no screen flag. When panel power names no single lit panel, the capture names CoreDevice's primary panel, which may be dark and yield an all-black PNG. Check the pose with UIHinge.status before trusting that capture. With no primary panel either, capture fails with COMMAND_FAILED and reason capture-panel-unidentified rather than pick a panel by list order. The two panels are different sizes (iPhone Duo: 466x678 points closed on the outer panel, 669x951 open on the inner). A pose change therefore invalidates every ref and coordinate. Re-snapshot after any pose change and never carry coordinates or refs across one. Check which panel is lit before trusting a geometry claim: agent-device screenshot reports its point size, and 466x678 versus 669x951 says which panel you captured. From dd5a35e4880bb03dc61914c274391fd2f7253b27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 09:25:31 +0200 Subject: [PATCH 06/11] fix(macos): share front-window eligibility across capture and actions --- .../AppWindowScreenshot.swift | 26 ++------------- .../BackgroundInteraction.swift | 33 ++++++++++--------- .../AppWindowScreenshotTests.swift | 11 +++++-- 3 files changed, 28 insertions(+), 42 deletions(-) diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift index 3bad70c649..f01f541536 100644 --- a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift +++ b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift @@ -10,14 +10,9 @@ func captureAppWindowScreenshot(app: NSRunningApplication, outPath: String) thro throw HelperError.commandFailed("app window screenshots require macOS 14 or newer") } let pid = app.processIdentifier - let windowInfo = CGWindowListCopyWindowInfo( - [.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] ?? [] - let frontWindowID = frontCaptureWindowNumber(windowInfo, pid: pid) + let frontWindowID = onScreenWindows(pid: pid, windowInfo: appWindowInfo()).first?.number let content = try awaitShareableContent() - let candidates = content.windows.filter { - $0.owningApplication?.processID == pid && $0.windowLayer == 0 && $0.frame.width > 0 - && $0.frame.height > 0 - } + let candidates = content.windows.filter { $0.owningApplication?.processID == pid } let window: SCWindow let selection = frontCaptureWindow( candidates, frontWindowID: frontWindowID, windowID: { Int($0.windowID) }) @@ -54,23 +49,6 @@ func captureAppWindowScreenshot(app: NSRunningApplication, outPath: String) thro try writePNG(capturedImage, to: outPath) } -/// WindowServer returns windows front to back. Invisible and empty windows cannot be capture targets. -func frontCaptureWindowNumber(_ windowInfo: [[String: Any]], pid: pid_t) -> Int? { - for entry in windowInfo { - guard (entry[kCGWindowOwnerPID as String] as? Int32) == pid, - (entry[kCGWindowLayer as String] as? Int) == 0, - (entry[kCGWindowIsOnscreen as String] as? Bool) == true, - let alpha = entry[kCGWindowAlpha as String] as? Double, alpha > 0, - let boundsDict = entry[kCGWindowBounds as String] as? NSDictionary, - let bounds = CGRect(dictionaryRepresentation: boundsDict as CFDictionary), - bounds.width > 0, bounds.height > 0, - let number = entry[kCGWindowNumber as String] as? Int - else { continue } - return number - } - return nil -} - /// Why an app screenshot refuses. The capture is the app's front window or nothing: another /// window of the app would show state the user is not looking at. Pinned with the host by /// `contracts/fixtures/macos-native-helper-outcomes.json`. diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift index fe6f231124..2d01faa742 100644 --- a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift +++ b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift @@ -482,26 +482,24 @@ private func numberAttribute(_ element: AXUIElement, attribute: String) -> Doubl return number.doubleValue } -private struct OnScreenWindow { +struct OnScreenWindow { let number: Int let bounds: CGRect } -/// The pid's normal-layer windows, front to back. Bounds and owners need no Screen Recording -/// permission; only titles do. -private func onScreenWindows(pid: pid_t) -> [OnScreenWindow] { - guard - let info = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) - as? [[String: Any]] - else { - return [] - } - return info.compactMap { entry in +let frontWindowLayer = 0 + +/// The pid's visible, nonempty normal-layer windows, front to back. +func onScreenWindows(pid: pid_t, windowInfo: [[String: Any]]) -> [OnScreenWindow] { + windowInfo.compactMap { entry in guard (entry[kCGWindowOwnerPID as String] as? Int32) == pid, - (entry[kCGWindowLayer as String] as? Int) == 0, + (entry[kCGWindowLayer as String] as? Int) == frontWindowLayer, + (entry[kCGWindowIsOnscreen as String] as? Bool) == true, + let alpha = entry[kCGWindowAlpha as String] as? Double, alpha > 0, let number = entry[kCGWindowNumber as String] as? Int, let boundsDict = entry[kCGWindowBounds as String] as? NSDictionary, - let bounds = CGRect(dictionaryRepresentation: boundsDict as CFDictionary) + let bounds = CGRect(dictionaryRepresentation: boundsDict as CFDictionary), + bounds.width > 0, bounds.height > 0 else { return nil } @@ -509,8 +507,13 @@ private func onScreenWindows(pid: pid_t) -> [OnScreenWindow] { } } -private func frontWindowFrame(pid: pid_t) -> CGRect? { - onScreenWindows(pid: pid).first?.bounds +func appWindowInfo() -> [[String: Any]] { + CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) + as? [[String: Any]] ?? [] +} + +func frontWindowFrame(pid: pid_t, windowInfo: [[String: Any]] = appWindowInfo()) -> CGRect? { + onScreenWindows(pid: pid, windowInfo: windowInfo).first?.bounds } private let returnVirtualKey: CGKeyCode = 36 diff --git a/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift b/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift index 76a6e739e3..f6fb1d354d 100644 --- a/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift +++ b/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift @@ -10,7 +10,7 @@ private struct Candidate: Equatable { /// An app screenshot captures the front window or refuses; it never substitutes another window. final class AppWindowScreenshotTests: XCTestCase { - func testSelectsTheFirstVisibleNonemptyNormalLayerWindowOfTheApp() { + func testCaptureIdentityAndScrollAndClickFallbackFrameSelectTheSameEligibleFrontWindow() { let windows = [ windowInfo(id: 1, pid: 99), windowInfo(id: 2, layer: 1), @@ -22,9 +22,13 @@ final class AppWindowScreenshotTests: XCTestCase { windowInfo(id: 8), ] - let frontID = frontCaptureWindowNumber(windows, pid: 42) + let frontID = onScreenWindows(pid: 42, windowInfo: windows).first?.number let selected = frontCaptureWindow([7, 8], frontWindowID: frontID, windowID: { $0 }) + // Scroll and the click actionWindow fallback both consume frontWindowFrame. + XCTAssertEqual( + frontWindowFrame(pid: 42, windowInfo: windows), + CGRect(x: 10, y: 10, width: 20, height: 20)) XCTAssertEqual(frontID, 7) XCTAssertEqual(try selected.get(), 7) } @@ -36,9 +40,10 @@ final class AppWindowScreenshotTests: XCTestCase { windowInfo(id: 3, width: 0), windowInfo(id: 4, height: 0), ] - let frontID = frontCaptureWindowNumber(windows, pid: 42) + let frontID = onScreenWindows(pid: 42, windowInfo: windows).first?.number let selected = frontCaptureWindow([5], frontWindowID: frontID, windowID: { $0 }) + XCTAssertNil(frontWindowFrame(pid: 42, windowInfo: windows)) XCTAssertNil(frontID) XCTAssertThrowsError(try selected.get()) { error in XCTAssertEqual(error as? AppWindowScreenshotRefusal, .windowNotFound) From c326cafbfd3d44b289d86be0dbb210093030238a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 09:26:15 +0200 Subject: [PATCH 07/11] docs(macos): define the native front-window target --- src/commands/capture/screenshot.ts | 2 +- website/docs/docs/commands.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/commands/capture/screenshot.ts b/src/commands/capture/screenshot.ts index 13ab745d18..e02f1b4f37 100644 --- a/src/commands/capture/screenshot.ts +++ b/src/commands/capture/screenshot.ts @@ -74,7 +74,7 @@ export const screenshotCommandFacet = defineCommandFacet({ text: { summary: 'Capture a screenshot', cliDetail: - 'Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators and Android emulators), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator chrome, or --no-stabilize for low-latency Android capture loops. On macOS, any --surface other than app (desktop, menubar, frontmost-app) always captures the main display and refuses an explicit --fullscreen. On single-display iOS simulators and Android devices, the result reports displayRotation, the rotation the display rendered the capture in, when the device returns a reading in time; otherwise the field is absent.', + "Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators and Android emulators), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator chrome, or --no-stabilize for low-latency Android capture loops. With the native macOS backend, the front window is the app's frontmost visible, nonempty normal-level window on the current Space; floating palettes are skipped, and a sheet is captured as its own window. On macOS, any --surface other than app (desktop, menubar, frontmost-app) always captures the main display and refuses an explicit --fullscreen. On single-display iOS simulators and Android devices, the result reports displayRotation, the rotation the display rendered the capture in, when the device returns a reading in time; otherwise the field is absent.", }, metadata: screenshotCommandMetadata, run: (client, input) => client.capture.screenshot(input), diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 8ce4a66430..6b96a325df 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -321,7 +321,7 @@ agent-device snapshot -i --platform apple --target desktop Set `AGENT_DEVICE_MACOS_APP_BACKEND=native` to run macOS app sessions without XCTest. There is no Automation Mode overlay, the app can stay behind other windows, and your pointer stays yours while a drawn pointer shows each action. -- `click`, `press`, and `fill` use accessibility actions on the element at the target point, which must be inside one of the app's on-screen windows, not its menu bar. `type` inserts at the focused control or falls back to key events. `scroll` moves the scroll bar under the front window's center. Chromium-based apps expose their full tree. `screenshot` captures only the app's front window, even when it is covered. If that window cannot be captured, `screenshot` fails; it never captures another window. +- `click`, `press`, and `fill` use accessibility actions on the element at the target point, which must be inside one of the app's on-screen windows, not its menu bar. `type` inserts at the focused control or falls back to key events. `scroll` moves the scroll bar under the front window's center. Chromium-based apps expose their full tree. `screenshot` captures only the app's front window, even when it is covered. The front window is the app's frontmost visible, nonempty normal-level window on the current Space; floating palettes are skipped, and a sheet is captured as its own window. If that window cannot be captured, `screenshot` fails; it never captures another window. - `record`, `prepare`, `back`, `longpress`, `--hold-ms`, `--double-tap`, `--button secondary`, gestures, and clicks on an element with no accessibility action fail with `UNSUPPORTED_OPERATION` (`reason: unsupported-device-backend`). Use the default XCTest backend for those. ```bash From 0de4a1e2c8ef6c9ca88855ba6452d2c04e053221 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 09:27:08 +0200 Subject: [PATCH 08/11] fix(macos): distinguish windows excluded from screen capture --- .../AppWindowScreenshot.swift | 14 +++- .../BackgroundInteraction.swift | 4 +- .../AppWindowScreenshotTests.swift | 35 ++++++-- .../macos-native-helper-outcomes.json | 6 +- .../src/os/macos/helper.test.ts | 80 +++++++++++-------- .../platform-apple/src/os/macos/helper.ts | 5 +- 6 files changed, 98 insertions(+), 46 deletions(-) diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift index f01f541536..cbb90f0aac 100644 --- a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift +++ b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift @@ -10,12 +10,13 @@ func captureAppWindowScreenshot(app: NSRunningApplication, outPath: String) thro throw HelperError.commandFailed("app window screenshots require macOS 14 or newer") } let pid = app.processIdentifier - let frontWindowID = onScreenWindows(pid: pid, windowInfo: appWindowInfo()).first?.number + let frontWindow = onScreenWindows(pid: pid, windowInfo: appWindowInfo()).first + let frontWindowID = frontWindow?.number let content = try awaitShareableContent() let candidates = content.windows.filter { $0.owningApplication?.processID == pid } let window: SCWindow let selection = frontCaptureWindow( - candidates, frontWindowID: frontWindowID, windowID: { Int($0.windowID) }) + candidates, frontWindowID: frontWindowID, sharingState: frontWindow?.sharingState, windowID: { Int($0.windowID) }) switch selection { case .success(let front): window = front @@ -55,15 +56,18 @@ func captureAppWindowScreenshot(app: NSRunningApplication, outPath: String) thro enum AppWindowScreenshotRefusal: String, CaseIterable, Error { case windowNotFound = "window-not-found" case frontWindowNotShareable = "front-window-not-shareable" + case frontWindowCaptureExcluded = "front-window-capture-excluded" } /// The shareable candidate whose WindowServer ID is the app's front window. func frontCaptureWindow( _ candidates: [Window], frontWindowID: Int?, + sharingState: Int?, windowID: (Window) -> Int ) -> Result { guard let frontWindowID else { return .failure(.windowNotFound) } + guard sharingState != 0 else { return .failure(.frontWindowCaptureExcluded) } guard let front = candidates.first(where: { windowID($0) == frontWindowID }) else { return .failure(.frontWindowNotShareable) } @@ -79,6 +83,12 @@ private func screenshotRefusal( switch refusal { case .windowNotFound: return .commandFailed("screenshot could not find an on-screen window for the app", details: details) + case .frontWindowCaptureExcluded: + details["frontWindowId"] = frontWindowID.map(String.init) ?? "" + return .commandFailed( + "screenshot could not capture the app's front window: the app excludes it from capture", + details: details + ) case .frontWindowNotShareable: details["frontWindowId"] = frontWindowID.map(String.init) ?? "" return .commandFailed( diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift index 2d01faa742..82fe94d357 100644 --- a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift +++ b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift @@ -485,6 +485,7 @@ private func numberAttribute(_ element: AXUIElement, attribute: String) -> Doubl struct OnScreenWindow { let number: Int let bounds: CGRect + let sharingState: Int? } let frontWindowLayer = 0 @@ -503,7 +504,8 @@ func onScreenWindows(pid: pid_t, windowInfo: [[String: Any]]) -> [OnScreenWindow else { return nil } - return OnScreenWindow(number: number, bounds: bounds) + return OnScreenWindow( + number: number, bounds: bounds, sharingState: entry[kCGWindowSharingState as String] as? Int) } } diff --git a/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift b/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift index f6fb1d354d..8417c60027 100644 --- a/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift +++ b/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift @@ -23,7 +23,7 @@ final class AppWindowScreenshotTests: XCTestCase { ] let frontID = onScreenWindows(pid: 42, windowInfo: windows).first?.number - let selected = frontCaptureWindow([7, 8], frontWindowID: frontID, windowID: { $0 }) + let selected = frontCaptureWindow([7, 8], frontWindowID: frontID, sharingState: 1, windowID: { $0 }) // Scroll and the click actionWindow fallback both consume frontWindowFrame. XCTAssertEqual( @@ -41,7 +41,7 @@ final class AppWindowScreenshotTests: XCTestCase { windowInfo(id: 4, height: 0), ] let frontID = onScreenWindows(pid: 42, windowInfo: windows).first?.number - let selected = frontCaptureWindow([5], frontWindowID: frontID, windowID: { $0 }) + let selected = frontCaptureWindow([5], frontWindowID: frontID, sharingState: 1, windowID: { $0 }) XCTAssertNil(frontWindowFrame(pid: 42, windowInfo: windows)) XCTAssertNil(frontID) @@ -53,7 +53,7 @@ final class AppWindowScreenshotTests: XCTestCase { func testSelectsTheFrontWindowEvenWhenAnotherIsLarger() { let candidates = [Candidate(id: 7, area: 10_000), Candidate(id: 3, area: 100)] - let selected = frontCaptureWindow(candidates, frontWindowID: 3, windowID: \.id) + let selected = frontCaptureWindow(candidates, frontWindowID: 3, sharingState: 1, windowID: \.id) XCTAssertEqual(try selected.get(), Candidate(id: 3, area: 100)) } @@ -61,16 +61,38 @@ final class AppWindowScreenshotTests: XCTestCase { func testRefusesWhenTheFrontWindowIsNotShareable() { let candidates = [Candidate(id: 7, area: 10_000), Candidate(id: 8, area: 100)] - let selected = frontCaptureWindow(candidates, frontWindowID: 3, windowID: \.id) + let selected = frontCaptureWindow(candidates, frontWindowID: 3, sharingState: 1, windowID: \.id) XCTAssertThrowsError(try selected.get()) { error in XCTAssertEqual(error as? AppWindowScreenshotRefusal, .frontWindowNotShareable) } } + func testCaptureExclusionSurvivesWindowSelectionAndNeverSubstitutesAnotherWindow() { + let windows = [windowInfo(id: 3, sharingState: 0), windowInfo(id: 7)] + let front = onScreenWindows(pid: 42, windowInfo: windows).first + XCTAssertEqual(front?.number, 3) + XCTAssertEqual(front?.sharingState, 0) + + for candidates in [[3, 7], [7]] { + let selected = frontCaptureWindow( + candidates, frontWindowID: front?.number, sharingState: front?.sharingState, windowID: { $0 }) + XCTAssertThrowsError(try selected.get()) { error in + XCTAssertEqual(error as? AppWindowScreenshotRefusal, .frontWindowCaptureExcluded) + } + } + + var entry = windowInfo(id: 3) + entry.removeValue(forKey: kCGWindowSharingState as String) + let unknown = onScreenWindows(pid: 42, windowInfo: [entry]).first + let selected = frontCaptureWindow( + [3], frontWindowID: unknown?.number, sharingState: unknown?.sharingState, windowID: { $0 }) + XCTAssertEqual(try selected.get(), 3) + } + func testRefusesWhenTheAppHasNoOnScreenWindow() { let selected = frontCaptureWindow( - [Candidate(id: 7, area: 10_000)], frontWindowID: nil, windowID: \.id) + [Candidate(id: 7, area: 10_000)], frontWindowID: nil, sharingState: nil, windowID: \.id) XCTAssertThrowsError(try selected.get()) { error in XCTAssertEqual(error as? AppWindowScreenshotRefusal, .windowNotFound) @@ -80,7 +102,7 @@ final class AppWindowScreenshotTests: XCTestCase { private func windowInfo( id: Int, pid: Int32 = 42, layer: Int = 0, onScreen: Bool = true, - alpha: Double = 1, width: Double = 100, height: Double = 100 + alpha: Double = 1, sharingState: Int = 1, width: Double = 100, height: Double = 100 ) -> [String: Any] { [ kCGWindowNumber as String: id, @@ -88,6 +110,7 @@ private func windowInfo( kCGWindowLayer as String: layer, kCGWindowIsOnscreen as String: onScreen, kCGWindowAlpha as String: alpha, + kCGWindowSharingState as String: sharingState, kCGWindowBounds as String: CGRect(x: 10, y: 10, width: width, height: height).dictionaryRepresentation, ] } diff --git a/contracts/fixtures/macos-native-helper-outcomes.json b/contracts/fixtures/macos-native-helper-outcomes.json index c4337931fd..cdf2fed369 100644 --- a/contracts/fixtures/macos-native-helper-outcomes.json +++ b/contracts/fixtures/macos-native-helper-outcomes.json @@ -15,5 +15,9 @@ "ax-scroll-bar", "key-events" ], - "screenshotRefusalReasons": ["window-not-found", "front-window-not-shareable"] + "screenshotRefusalReasons": [ + "window-not-found", + "front-window-not-shareable", + "front-window-capture-excluded" + ] } diff --git a/packages/platform-apple/src/os/macos/helper.test.ts b/packages/platform-apple/src/os/macos/helper.test.ts index aa6d017e95..8bc96c0911 100644 --- a/packages/platform-apple/src/os/macos/helper.test.ts +++ b/packages/platform-apple/src/os/macos/helper.test.ts @@ -244,45 +244,55 @@ test('macOS helper screenshot argv carries only --out and --surface', async () = assert.deepEqual(receivedArgs, ['screenshot', '--out', '/tmp/out.png', '--surface', 'desktop']); }); -test('an app screenshot that cannot reach the front window refuses with its reason and a hint', async () => { - const provider = createLocalAppleToolProvider({ - macosHelper: { - run: async () => ({ - exitCode: 1, - stdout: JSON.stringify({ - ok: false, - error: { - message: - "screenshot could not capture the app's front window: ScreenCaptureKit does not offer it", - details: { - reason: 'front-window-not-shareable', - bundleId: 'com.apple.TextEdit', - frontWindowId: '4242', +for (const reason of ['front-window-not-shareable', 'front-window-capture-excluded', undefined]) { + test(`an app screenshot refusal gives the remedy for ${reason}`, async () => { + const provider = createLocalAppleToolProvider({ + macosHelper: { + run: async () => ({ + exitCode: 1, + stdout: JSON.stringify({ + ok: false, + error: { + message: + "screenshot could not capture the app's front window: ScreenCaptureKit does not offer it", + details: { + reason, + bundleId: 'com.apple.TextEdit', + frontWindowId: '4242', + }, }, - }, + }), + stderr: '', }), - stderr: '', - }), - }, - }); + }, + }); - const error = await withAppleToolProvider(provider, async () => - runMacOsScreenshotAction('/tmp/out.png', { - surface: nativeApp, - bundleId: 'com.apple.TextEdit', - }).then( - () => undefined, - (error: unknown) => error, - ), - ); + const error = await withAppleToolProvider(provider, async () => + runMacOsScreenshotAction('/tmp/out.png', { + surface: nativeApp, + bundleId: 'com.apple.TextEdit', + }).then( + () => undefined, + (error: unknown) => error, + ), + ); - assert.ok(error instanceof AppError); - assert.equal(error.code, 'COMMAND_FAILED'); - assert.equal(error.details?.reason, 'front-window-not-shareable'); - assert.equal(error.details?.frontWindowId, '4242'); - assert.match(String(error.details?.hint), /retry after it settles/); - assert.match(String(error.details?.hint), /use a window that permits capture/); -}); + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.details?.reason, reason); + assert.equal(error.details?.frontWindowId, '4242'); + if (reason === 'front-window-capture-excluded') { + assert.match(String(error.details?.hint), /Use a window that permits capture/); + assert.match(String(error.details?.hint), /retrying the excluded window will not help/); + assert.doesNotMatch(String(error.details?.hint), /retry after/); + } else if (reason === 'front-window-not-shareable') { + assert.match(String(error.details?.hint), /retry after it settles/); + assert.doesNotMatch(String(error.details?.hint), /excludes/); + } else { + assert.equal(error.details?.hint, undefined); + } + }); +} test('helper entry points accept only an owner-routed surface', () => { // Never invoked: each directive fails typecheck once its entry point widens back to an diff --git a/packages/platform-apple/src/os/macos/helper.ts b/packages/platform-apple/src/os/macos/helper.ts index dba7cdae26..3f34abf8f4 100644 --- a/packages/platform-apple/src/os/macos/helper.ts +++ b/packages/platform-apple/src/os/macos/helper.ts @@ -149,13 +149,16 @@ export const MACOS_HELPER_REFUSAL_REASONS = [ export const MACOS_SCREENSHOT_REFUSAL_REASONS = [ 'window-not-found', 'front-window-not-shareable', + 'front-window-capture-excluded', ] as const; type MacOsScreenshotRefusalReason = (typeof MACOS_SCREENSHOT_REFUSAL_REASONS)[number]; const MACOS_SCREENSHOT_REFUSAL_HINTS: Record = { 'window-not-found': 'Open or unminimize a window of the app on the current Space, then retry the screenshot.', 'front-window-not-shareable': - 'If the front window is opening or closing, retry after it settles. If the app excludes the window from screen capture, use a window that permits capture; retrying the excluded window will not help.', + 'The front window is unavailable to screen capture. If it is opening or closing, retry after it settles.', + 'front-window-capture-excluded': + 'The app excludes the front window from screen capture. Use a window that permits capture; retrying the excluded window will not help.', }; export const MACOS_DELIVERY_MECHANISMS = [ 'ax-press', From bb69e6a301b81f4e4219cf5ffe89e46c13648a2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 09:54:35 +0200 Subject: [PATCH 09/11] refactor(macos): centralize front window selection Share the on-screen window-list query while preserving nil versus empty point refusals. Pin layer-8 alert exclusion at the shared selection boundary. Review: https://github.com/callstack/agent-device/pull/3761#discussion_r4301906142 --- .../AppWindowScreenshot.swift | 2 +- .../BackgroundInteraction.swift | 18 ++++++++++-------- .../AppWindowScreenshotTests.swift | 19 +++++++++++++++---- 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift index cbb90f0aac..b6644d982e 100644 --- a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift +++ b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift @@ -10,7 +10,7 @@ func captureAppWindowScreenshot(app: NSRunningApplication, outPath: String) thro throw HelperError.commandFailed("app window screenshots require macOS 14 or newer") } let pid = app.processIdentifier - let frontWindow = onScreenWindows(pid: pid, windowInfo: appWindowInfo()).first + let frontWindow = frontWindow(pid: pid) let frontWindowID = frontWindow?.number let content = try awaitShareableContent() let candidates = content.windows.filter { $0.owningApplication?.processID == pid } diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift index 82fe94d357..d82b6aedbb 100644 --- a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift +++ b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift @@ -306,10 +306,7 @@ private func perform(_ target: PressTarget) -> BackgroundDeliveryMechanism? { /// own on-screen windows, its menus and sheets included, and never on a menu bar element: the app /// owns its menu bar's window too, so the window check alone does not exclude it. private func requireAppWindowPoint(_ point: CGPoint, app: NSRunningApplication) throws { - guard - let info = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) - as? [[String: Any]] - else { + guard let info = appWindowInfo() else { throw refusal(.noAccessibleTarget, "the app's on-screen windows could not be read", app: app) } let inAppWindow = info.contains { entry in @@ -488,6 +485,7 @@ struct OnScreenWindow { let sharingState: Int? } +/// Only normal-layer windows qualify; app-modal alerts at level 8 are excluded. let frontWindowLayer = 0 /// The pid's visible, nonempty normal-layer windows, front to back. @@ -509,13 +507,17 @@ func onScreenWindows(pid: pid_t, windowInfo: [[String: Any]]) -> [OnScreenWindow } } -func appWindowInfo() -> [[String: Any]] { +func appWindowInfo() -> [[String: Any]]? { CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) - as? [[String: Any]] ?? [] + as? [[String: Any]] +} + +func frontWindow(pid: pid_t, windowInfo: [[String: Any]] = appWindowInfo() ?? []) -> OnScreenWindow? { + onScreenWindows(pid: pid, windowInfo: windowInfo).first } -func frontWindowFrame(pid: pid_t, windowInfo: [[String: Any]] = appWindowInfo()) -> CGRect? { - onScreenWindows(pid: pid, windowInfo: windowInfo).first?.bounds +func frontWindowFrame(pid: pid_t, windowInfo: [[String: Any]] = appWindowInfo() ?? []) -> CGRect? { + frontWindow(pid: pid, windowInfo: windowInfo)?.bounds } private let returnVirtualKey: CGKeyCode = 36 diff --git a/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift b/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift index 8417c60027..685a8554a9 100644 --- a/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift +++ b/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift @@ -22,7 +22,7 @@ final class AppWindowScreenshotTests: XCTestCase { windowInfo(id: 8), ] - let frontID = onScreenWindows(pid: 42, windowInfo: windows).first?.number + let frontID = frontWindow(pid: 42, windowInfo: windows)?.number let selected = frontCaptureWindow([7, 8], frontWindowID: frontID, sharingState: 1, windowID: { $0 }) // Scroll and the click actionWindow fallback both consume frontWindowFrame. @@ -33,6 +33,17 @@ final class AppWindowScreenshotTests: XCTestCase { XCTAssertEqual(try selected.get(), 7) } + func testSkipsAppModalAlertInFrontOfNormalLayerWindow() throws { + let windows = [windowInfo(id: 8, layer: 8), windowInfo(id: 3)] + let front = frontWindow(pid: 42, windowInfo: windows) + let selected = frontCaptureWindow( + [8, 3], frontWindowID: front?.number, sharingState: front?.sharingState, windowID: { $0 }) + + XCTAssertEqual(front?.number, 3) + XCTAssertEqual(try selected.get(), 3) + XCTAssertEqual(frontWindowFrame(pid: 42, windowInfo: windows), front?.bounds) + } + func testRefusesWhenOnlyInvisibleOrEmptyWindowsRemain() { let windows = [ windowInfo(id: 1, onScreen: false), @@ -40,7 +51,7 @@ final class AppWindowScreenshotTests: XCTestCase { windowInfo(id: 3, width: 0), windowInfo(id: 4, height: 0), ] - let frontID = onScreenWindows(pid: 42, windowInfo: windows).first?.number + let frontID = frontWindow(pid: 42, windowInfo: windows)?.number let selected = frontCaptureWindow([5], frontWindowID: frontID, sharingState: 1, windowID: { $0 }) XCTAssertNil(frontWindowFrame(pid: 42, windowInfo: windows)) @@ -70,7 +81,7 @@ final class AppWindowScreenshotTests: XCTestCase { func testCaptureExclusionSurvivesWindowSelectionAndNeverSubstitutesAnotherWindow() { let windows = [windowInfo(id: 3, sharingState: 0), windowInfo(id: 7)] - let front = onScreenWindows(pid: 42, windowInfo: windows).first + let front = frontWindow(pid: 42, windowInfo: windows) XCTAssertEqual(front?.number, 3) XCTAssertEqual(front?.sharingState, 0) @@ -84,7 +95,7 @@ final class AppWindowScreenshotTests: XCTestCase { var entry = windowInfo(id: 3) entry.removeValue(forKey: kCGWindowSharingState as String) - let unknown = onScreenWindows(pid: 42, windowInfo: [entry]).first + let unknown = frontWindow(pid: 42, windowInfo: [entry]) let selected = frontCaptureWindow( [3], frontWindowID: unknown?.number, sharingState: unknown?.sharingState, windowID: { $0 }) XCTAssertEqual(try selected.get(), 3) From 5e16f816b5c2b1f8a0b20131e5cc328ce7d814f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 09:54:35 +0200 Subject: [PATCH 10/11] docs(capture): clarify alert exclusion and refusal remedy --- packages/platform-apple/src/os/macos/helper.test.ts | 2 +- packages/platform-apple/src/os/macos/helper.ts | 2 +- src/commands/capture/screenshot.ts | 2 +- website/docs/docs/commands.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/platform-apple/src/os/macos/helper.test.ts b/packages/platform-apple/src/os/macos/helper.test.ts index 8bc96c0911..b0037ac051 100644 --- a/packages/platform-apple/src/os/macos/helper.test.ts +++ b/packages/platform-apple/src/os/macos/helper.test.ts @@ -282,7 +282,7 @@ for (const reason of ['front-window-not-shareable', 'front-window-capture-exclud assert.equal(error.details?.reason, reason); assert.equal(error.details?.frontWindowId, '4242'); if (reason === 'front-window-capture-excluded') { - assert.match(String(error.details?.hint), /Use a window that permits capture/); + assert.match(String(error.details?.hint), /Run `snapshot` to read the window/); assert.match(String(error.details?.hint), /retrying the excluded window will not help/); assert.doesNotMatch(String(error.details?.hint), /retry after/); } else if (reason === 'front-window-not-shareable') { diff --git a/packages/platform-apple/src/os/macos/helper.ts b/packages/platform-apple/src/os/macos/helper.ts index 3f34abf8f4..b67eee082c 100644 --- a/packages/platform-apple/src/os/macos/helper.ts +++ b/packages/platform-apple/src/os/macos/helper.ts @@ -158,7 +158,7 @@ const MACOS_SCREENSHOT_REFUSAL_HINTS: Record to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators and Android emulators), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator chrome, or --no-stabilize for low-latency Android capture loops. With the native macOS backend, the front window is the app's frontmost visible, nonempty normal-level window on the current Space; floating palettes are skipped, and a sheet is captured as its own window. On macOS, any --surface other than app (desktop, menubar, frontmost-app) always captures the main display and refuses an explicit --fullscreen. On single-display iOS simulators and Android devices, the result reports displayRotation, the rotation the display rendered the capture in, when the device returns a reading in time; otherwise the field is absent.", + "Web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. iOS simulators default to 1x logical-point output; use --pixel-density to request a different screenshot density. macOS app sessions default to the app window; use --fullscreen for full desktop, --scale to downscale, --crop-on to crop the capture to the frame the selector resolves on the same screen (currently iOS simulators and Android emulators), --overlay-refs to annotate current refs, --normalize-status-bar for deterministic iOS simulator chrome, or --no-stabilize for low-latency Android capture loops. With the native macOS backend, the front window is the app's frontmost visible, nonempty normal-level window on the current Space; floating palettes are skipped, and a sheet is captured as its own window. Only layer-0 windows count as the front window; app-modal alerts at window level 8 are skipped. On macOS, any --surface other than app (desktop, menubar, frontmost-app) always captures the main display and refuses an explicit --fullscreen. On single-display iOS simulators and Android devices, the result reports displayRotation, the rotation the display rendered the capture in, when the device returns a reading in time; otherwise the field is absent.", }, metadata: screenshotCommandMetadata, run: (client, input) => client.capture.screenshot(input), diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 6b96a325df..5a4adfe251 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -321,7 +321,7 @@ agent-device snapshot -i --platform apple --target desktop Set `AGENT_DEVICE_MACOS_APP_BACKEND=native` to run macOS app sessions without XCTest. There is no Automation Mode overlay, the app can stay behind other windows, and your pointer stays yours while a drawn pointer shows each action. -- `click`, `press`, and `fill` use accessibility actions on the element at the target point, which must be inside one of the app's on-screen windows, not its menu bar. `type` inserts at the focused control or falls back to key events. `scroll` moves the scroll bar under the front window's center. Chromium-based apps expose their full tree. `screenshot` captures only the app's front window, even when it is covered. The front window is the app's frontmost visible, nonempty normal-level window on the current Space; floating palettes are skipped, and a sheet is captured as its own window. If that window cannot be captured, `screenshot` fails; it never captures another window. +- `click`, `press`, and `fill` use accessibility actions on the element at the target point, which must be inside one of the app's on-screen windows, not its menu bar. `type` inserts at the focused control or falls back to key events. `scroll` moves the scroll bar under the front window's center. Chromium-based apps expose their full tree. `screenshot` captures only the app's front window, even when it is covered. The front window is the app's frontmost visible, nonempty normal-level window on the current Space; floating palettes are skipped, and a sheet is captured as its own window. Only layer-0 windows count as the front window; app-modal alerts at window level 8 are skipped. If that window cannot be captured, `screenshot` fails; it never captures another window. - `record`, `prepare`, `back`, `longpress`, `--hold-ms`, `--double-tap`, `--button secondary`, gestures, and clicks on an element with no accessibility action fail with `UNSUPPORTED_OPERATION` (`reason: unsupported-device-backend`). Use the default XCTest backend for those. ```bash From 70acf8844c71e0de43730edda7c43e818ee70a8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 10:09:09 +0200 Subject: [PATCH 11/11] fix(macos): distinguish unreadable window lists from missing windows Refuse screenshot and scroll with window-list-unreadable when the shared window-list read fails, and preserve a retry hint through the host helper client. Review: https://github.com/callstack/agent-device/pull/3761#discussion_r4301987524 --- .../AppWindowScreenshot.swift | 2 +- .../BackgroundInteraction.swift | 32 +++++++----- .../AppWindowScreenshotTests.swift | 39 ++++++++++----- .../BackgroundInteractionTests.swift | 2 + .../macos-native-helper-outcomes.json | 5 +- .../src/os/macos/helper.test.ts | 49 +++++++++++++++++++ .../platform-apple/src/os/macos/helper.ts | 7 ++- 7 files changed, 110 insertions(+), 26 deletions(-) diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift index b6644d982e..8a9fcae38d 100644 --- a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift +++ b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift @@ -10,7 +10,7 @@ func captureAppWindowScreenshot(app: NSRunningApplication, outPath: String) thro throw HelperError.commandFailed("app window screenshots require macOS 14 or newer") } let pid = app.processIdentifier - let frontWindow = frontWindow(pid: pid) + let frontWindow = try frontWindow(pid: pid) let frontWindowID = frontWindow?.number let content = try awaitShareableContent() let candidates = content.windows.filter { $0.owningApplication?.processID == pid } diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift index d82b6aedbb..9903b905a0 100644 --- a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift +++ b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift @@ -97,7 +97,7 @@ func pressInBackground( let point = CGPoint(x: request.x, y: request.y) try requireAppWindowPoint(point, app: app) cursor?.move(to: point) - let target = resolvePressTarget(app: app, point: point) + let target = try resolvePressTarget(app: app, point: point) if let target, isInMenuBar(target.element) { throw refusal(.noAccessibleTarget, "the point is on the app's menu bar", app: app) } @@ -172,7 +172,7 @@ func fillInBackground( cursor?.move(to: point) let hit = elementAtPoint(in: app, point: point) let chain = hit.map(pressSearchChain) ?? [] - let fallback = actionWindow(app: app, hit: hit).flatMap { window in + let fallback = try actionWindow(app: app, hit: hit).flatMap { window in smallestElement(in: window, containing: point, where: isTextInput) } if let input = chain.first(where: isTextInput) ?? fallback, isInMenuBar(input) { @@ -206,7 +206,7 @@ func scrollInBackground( guard isVertical || direction == "left" || direction == "right" else { throw HelperError.invalidArgs("scroll requires --direction ") } - guard let frame = frontWindowFrame(pid: app.processIdentifier) else { + guard let frame = try frontWindowFrame(pid: app.processIdentifier) else { throw HelperError.commandFailed( "scroll could not resolve an on-screen window", details: ["reason": "window-not-found", "bundleId": app.bundleIdentifier ?? ""] @@ -276,12 +276,12 @@ private enum PressTarget { /// What a click at a point means. The app's own hit test answers first; web content often answers /// with a wrapper group, so the snapshot geometry decides when the hit names no control. -private func resolvePressTarget(app: NSRunningApplication, point: CGPoint) -> PressTarget? { +private func resolvePressTarget(app: NSRunningApplication, point: CGPoint) throws -> PressTarget? { let hit = elementAtPoint(in: app, point: point) let chain = hit.map(pressSearchChain) ?? [] if let input = chain.first(where: isTextInput) { return .focus(input) } if let control = chain.first(where: isPressableControl) { return .press(control) } - if let found = actionWindow(app: app, hit: hit).flatMap({ + if let found = try actionWindow(app: app, hit: hit).flatMap({ smallestElement(in: $0, containing: point, where: { isTextInput($0) || isPressableControl($0) }) }) { return isTextInput(found) ? .focus(found) : .press(found) @@ -361,12 +361,12 @@ private func pressSearchChain(from hit: AXUIElement) -> [AXUIElement] { /// The one window a point action may act in: the window the app's hit test landed in, else the /// app's front on-screen window. A fallback target never comes from a window behind it, and the /// response names the window so the host can check it. -private func actionWindow(app: NSRunningApplication, hit: AXUIElement?) -> AXUIElement? { +private func actionWindow(app: NSRunningApplication, hit: AXUIElement?) throws -> AXUIElement? { if let hit { if role(of: hit) == "AXWindow" { return hit } if let window = elementAttribute(hit, attribute: kAXWindowAttribute as String) { return window } } - guard let front = frontWindowFrame(pid: app.processIdentifier) else { return nil } + guard let front = try frontWindowFrame(pid: app.processIdentifier) else { return nil } return windows(of: AXUIElementCreateApplication(app.processIdentifier)).first { window in guard let rect = rectAttribute(window) else { return false } return abs(rect.x - front.minX) < 1 && abs(rect.y - front.minY) < 1 @@ -512,12 +512,22 @@ func appWindowInfo() -> [[String: Any]]? { as? [[String: Any]] } -func frontWindow(pid: pid_t, windowInfo: [[String: Any]] = appWindowInfo() ?? []) -> OnScreenWindow? { - onScreenWindows(pid: pid, windowInfo: windowInfo).first +enum WindowListRefusal: String, CaseIterable { + case unreadable = "window-list-unreadable" } -func frontWindowFrame(pid: pid_t, windowInfo: [[String: Any]] = appWindowInfo() ?? []) -> CGRect? { - frontWindow(pid: pid, windowInfo: windowInfo)?.bounds +func frontWindow(pid: pid_t, windowInfo: [[String: Any]]? = appWindowInfo()) throws -> OnScreenWindow? { + guard let windowInfo else { + throw HelperError.commandFailed( + "the app's on-screen windows could not be read", + details: ["reason": WindowListRefusal.unreadable.rawValue] + ) + } + return onScreenWindows(pid: pid, windowInfo: windowInfo).first +} + +func frontWindowFrame(pid: pid_t, windowInfo: [[String: Any]]? = appWindowInfo()) throws -> CGRect? { + try frontWindow(pid: pid, windowInfo: windowInfo)?.bounds } private let returnVirtualKey: CGKeyCode = 36 diff --git a/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift b/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift index 685a8554a9..539c8752e0 100644 --- a/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift +++ b/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift @@ -10,7 +10,7 @@ private struct Candidate: Equatable { /// An app screenshot captures the front window or refuses; it never substitutes another window. final class AppWindowScreenshotTests: XCTestCase { - func testCaptureIdentityAndScrollAndClickFallbackFrameSelectTheSameEligibleFrontWindow() { + func testCaptureIdentityAndScrollAndClickFallbackFrameSelectTheSameEligibleFrontWindow() throws { let windows = [ windowInfo(id: 1, pid: 99), windowInfo(id: 2, layer: 1), @@ -22,12 +22,12 @@ final class AppWindowScreenshotTests: XCTestCase { windowInfo(id: 8), ] - let frontID = frontWindow(pid: 42, windowInfo: windows)?.number + let frontID = try frontWindow(pid: 42, windowInfo: windows)?.number let selected = frontCaptureWindow([7, 8], frontWindowID: frontID, sharingState: 1, windowID: { $0 }) // Scroll and the click actionWindow fallback both consume frontWindowFrame. XCTAssertEqual( - frontWindowFrame(pid: 42, windowInfo: windows), + try frontWindowFrame(pid: 42, windowInfo: windows), CGRect(x: 10, y: 10, width: 20, height: 20)) XCTAssertEqual(frontID, 7) XCTAssertEqual(try selected.get(), 7) @@ -35,32 +35,49 @@ final class AppWindowScreenshotTests: XCTestCase { func testSkipsAppModalAlertInFrontOfNormalLayerWindow() throws { let windows = [windowInfo(id: 8, layer: 8), windowInfo(id: 3)] - let front = frontWindow(pid: 42, windowInfo: windows) + let front = try frontWindow(pid: 42, windowInfo: windows) let selected = frontCaptureWindow( [8, 3], frontWindowID: front?.number, sharingState: front?.sharingState, windowID: { $0 }) XCTAssertEqual(front?.number, 3) XCTAssertEqual(try selected.get(), 3) - XCTAssertEqual(frontWindowFrame(pid: 42, windowInfo: windows), front?.bounds) + XCTAssertEqual(try frontWindowFrame(pid: 42, windowInfo: windows), front?.bounds) } - func testRefusesWhenOnlyInvisibleOrEmptyWindowsRemain() { + func testRefusesWhenOnlyInvisibleOrEmptyWindowsRemain() throws { let windows = [ windowInfo(id: 1, onScreen: false), windowInfo(id: 2, alpha: 0), windowInfo(id: 3, width: 0), windowInfo(id: 4, height: 0), ] - let frontID = frontWindow(pid: 42, windowInfo: windows)?.number + let frontID = try frontWindow(pid: 42, windowInfo: windows)?.number let selected = frontCaptureWindow([5], frontWindowID: frontID, sharingState: 1, windowID: { $0 }) - XCTAssertNil(frontWindowFrame(pid: 42, windowInfo: windows)) + XCTAssertNil(try frontWindowFrame(pid: 42, windowInfo: windows)) XCTAssertNil(frontID) XCTAssertThrowsError(try selected.get()) { error in XCTAssertEqual(error as? AppWindowScreenshotRefusal, .windowNotFound) } } + func testUnreadableWindowListRefusesBeforeCaptureOrScrollCanReportNoWindow() { + XCTAssertThrowsError(try frontWindow(pid: 42, windowInfo: nil)) { error in + guard case HelperError.commandFailed(_, let details) = error else { + return XCTFail("expected a typed window-list refusal, got \(error)") + } + XCTAssertEqual(details["reason"], "window-list-unreadable") + } + XCTAssertThrowsError(try frontWindowFrame(pid: 42, windowInfo: nil)) { error in + guard case HelperError.commandFailed(_, let details) = error else { + return XCTFail("expected a typed window-list refusal, got \(error)") + } + XCTAssertEqual(details["reason"], "window-list-unreadable") + } + XCTAssertNil(try frontWindow(pid: 42, windowInfo: [])) + XCTAssertNil(try frontWindowFrame(pid: 42, windowInfo: [])) + } + func testSelectsTheFrontWindowEvenWhenAnotherIsLarger() { let candidates = [Candidate(id: 7, area: 10_000), Candidate(id: 3, area: 100)] @@ -79,9 +96,9 @@ final class AppWindowScreenshotTests: XCTestCase { } } - func testCaptureExclusionSurvivesWindowSelectionAndNeverSubstitutesAnotherWindow() { + func testCaptureExclusionSurvivesWindowSelectionAndNeverSubstitutesAnotherWindow() throws { let windows = [windowInfo(id: 3, sharingState: 0), windowInfo(id: 7)] - let front = frontWindow(pid: 42, windowInfo: windows) + let front = try frontWindow(pid: 42, windowInfo: windows) XCTAssertEqual(front?.number, 3) XCTAssertEqual(front?.sharingState, 0) @@ -95,7 +112,7 @@ final class AppWindowScreenshotTests: XCTestCase { var entry = windowInfo(id: 3) entry.removeValue(forKey: kCGWindowSharingState as String) - let unknown = frontWindow(pid: 42, windowInfo: [entry]) + let unknown = try frontWindow(pid: 42, windowInfo: [entry]) let selected = frontCaptureWindow( [3], frontWindowID: unknown?.number, sharingState: unknown?.sharingState, windowID: { $0 }) XCTAssertEqual(try selected.get(), 3) diff --git a/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/BackgroundInteractionTests.swift b/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/BackgroundInteractionTests.swift index 20445946df..36bd3766e8 100644 --- a/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/BackgroundInteractionTests.swift +++ b/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/BackgroundInteractionTests.swift @@ -27,6 +27,7 @@ private struct HelperOutcomesFixture: Decodable { let refusalReasons: [String] let deliveryMechanisms: [String] let screenshotRefusalReasons: [String] + let windowListRefusalReasons: [String] } /// `#filePath` is `/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/`. @@ -42,6 +43,7 @@ final class BackgroundInteractionTests: XCTestCase { from: Data( contentsOf: repoRoot.appendingPathComponent("contracts/fixtures/macos-native-helper-outcomes.json")) ) + XCTAssertEqual(WindowListRefusal.allCases.map(\.rawValue), fixture.windowListRefusalReasons) XCTAssertEqual(BackgroundRefusal.allCases.map(\.rawValue), fixture.refusalReasons) XCTAssertEqual(BackgroundDeliveryMechanism.allCases.map(\.rawValue), fixture.deliveryMechanisms) XCTAssertEqual( diff --git a/contracts/fixtures/macos-native-helper-outcomes.json b/contracts/fixtures/macos-native-helper-outcomes.json index cdf2fed369..e3c2a751f6 100644 --- a/contracts/fixtures/macos-native-helper-outcomes.json +++ b/contracts/fixtures/macos-native-helper-outcomes.json @@ -1,5 +1,5 @@ { - "description": "Vocabulary the macOS helper reports for app-surface actions. Swift: BackgroundRefusal, BackgroundDeliveryMechanism, and AppWindowScreenshotRefusal in apple/macos-helper. TypeScript: the macOS helper client in packages/platform-apple/src/os/macos/helper.ts.", + "description": "Vocabulary the macOS helper reports for app-surface actions. Swift: BackgroundRefusal, BackgroundDeliveryMechanism, WindowListRefusal, and AppWindowScreenshotRefusal in apple/macos-helper. TypeScript: the macOS helper client in packages/platform-apple/src/os/macos/helper.ts.", "refusalReasons": [ "background-pointer-gesture", "no-accessible-target", @@ -19,5 +19,6 @@ "window-not-found", "front-window-not-shareable", "front-window-capture-excluded" - ] + ], + "windowListRefusalReasons": ["window-list-unreadable"] } diff --git a/packages/platform-apple/src/os/macos/helper.test.ts b/packages/platform-apple/src/os/macos/helper.test.ts index b0037ac051..baacda831b 100644 --- a/packages/platform-apple/src/os/macos/helper.test.ts +++ b/packages/platform-apple/src/os/macos/helper.test.ts @@ -7,12 +7,14 @@ import { readFileSync } from 'node:fs'; import { MACOS_DELIVERY_MECHANISMS, MACOS_HELPER_REFUSAL_REASONS, + MACOS_WINDOW_LIST_REFUSAL_REASONS, MACOS_SCREENSHOT_REFUSAL_REASONS, macOsClickScheduleMs, runMacOsPressAction, runMacOsReadTextAction, runMacOsScreenshotAction, runMacOsSnapshotAction, + runMacOsScrollAction, } from './helper.ts'; const desktop = macOsHelperSurface('desktop', 'xctest')!; @@ -312,6 +314,51 @@ test('helper entry points accept only an owner-routed surface', () => { assert.equal(typeof widenedCalls, 'function'); }); +for (const action of ['screenshot', 'scroll'] as const) { + for (const reason of ['window-list-unreadable', undefined]) { + test(`${action} maps an unreadable window list by reason, not error text (${reason})`, async () => { + const provider = createLocalAppleToolProvider({ + macosHelper: { + run: async () => ({ + exitCode: 1, + stdout: JSON.stringify({ + ok: false, + error: { + message: "the app's on-screen windows could not be read", + details: { reason }, + }, + }), + stderr: '', + }), + }, + }); + await assert.rejects( + withAppleToolProvider(provider, async () => + action === 'screenshot' + ? runMacOsScreenshotAction('/tmp/out.png', { + surface: nativeApp, + bundleId: 'com.apple.TextEdit', + }) + : runMacOsScrollAction('down', { bundleId: 'com.apple.TextEdit' }), + ), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.details?.reason, reason); + if (reason) { + assert.match(String(error.details?.hint), /window list could not be read/); + assert.match(String(error.details?.hint), /Retry the command/); + assert.doesNotMatch(String(error.details?.hint), /Open or unminimize|XCTest/); + } else { + assert.equal(error.details?.hint, undefined); + } + return true; + }, + ); + }); + } +} + test('the helper vocabulary matches the table the Swift helper is held to', () => { const fixture = JSON.parse( readFileSync( @@ -325,7 +372,9 @@ test('the helper vocabulary matches the table the Swift helper is held to', () = refusalReasons: string[]; deliveryMechanisms: string[]; screenshotRefusalReasons: string[]; + windowListRefusalReasons: string[]; }; + assert.deepEqual([...MACOS_WINDOW_LIST_REFUSAL_REASONS], fixture.windowListRefusalReasons); assert.deepEqual([...MACOS_HELPER_REFUSAL_REASONS], fixture.refusalReasons); assert.deepEqual([...MACOS_DELIVERY_MECHANISMS], fixture.deliveryMechanisms); assert.deepEqual([...MACOS_SCREENSHOT_REFUSAL_REASONS], fixture.screenshotRefusalReasons); diff --git a/packages/platform-apple/src/os/macos/helper.ts b/packages/platform-apple/src/os/macos/helper.ts index b67eee082c..6f395dd41d 100644 --- a/packages/platform-apple/src/os/macos/helper.ts +++ b/packages/platform-apple/src/os/macos/helper.ts @@ -160,6 +160,7 @@ const MACOS_SCREENSHOT_REFUSAL_HINTS: Record>( parsed && !parsed.ok ? (parsed.error?.message ?? `macOS helper exited with code ${result.exitCode}`) : stdout || result.stderr.trim() || `macOS helper exited with code ${result.exitCode}`; + const details = parsed && !parsed.ok ? parsed.error?.details : undefined; // exec-guard-allow: the message is already built from the helper's JSON error // envelope (or its output); a stderr excerpt would only degrade it, and the // throw is reachable at exit 0 when the envelope reports ok=false. @@ -381,7 +383,10 @@ async function runMacOsHelper>( stdout: result.stdout, stderr: result.stderr, exitCode: result.exitCode, - ...(parsed && !parsed.ok ? parsed.error?.details : {}), + ...details, + ...(details?.reason === MACOS_WINDOW_LIST_REFUSAL_REASONS[0] + ? { hint: "The app's on-screen window list could not be read. Retry the command." } + : {}), }); }