fix(apple): refuse captures whose window or panel identity is unknown

apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift

@@ -10,20 +10,18 @@ func captureAppWindowScreenshot(app: NSRunningApplication, outPath: String) thro
     throw HelperError.commandFailed("app window screenshots require macOS 14 or newer")
   }
   let pid = app.processIdentifier
-  let frontWindowID = frontWindowNumber(pid: pid)
+  let frontWindow = try frontWindow(pid: pid)
+  let frontWindowID = frontWindow?.number
   let content = try awaitShareableContent()
-  let candidates = content.windows.filter {
-    $0.owningApplication?.processID == pid && $0.windowLayer == 0 && $0.frame.width > 0
-      && $0.frame.height > 0
-  }
-  guard
-    let window = candidates.first(where: { Int($0.windowID) == frontWindowID })
-      ?? candidates.max(by: { $0.frame.width * $0.frame.height < $1.frame.width * $1.frame.height })
-  else {
-    throw HelperError.commandFailed(
-      "screenshot could not find a window for the app",
-      details: ["reason": "window-not-found", "bundleId": app.bundleIdentifier ?? ""]
-    )
+  let candidates = content.windows.filter { $0.owningApplication?.processID == pid }
+  let window: SCWindow
+  let selection = frontCaptureWindow(
+    candidates, frontWindowID: frontWindowID, sharingState: frontWindow?.sharingState, windowID: { Int($0.windowID) })
+  switch selection {
+  case .success(let front):
+    window = front
+  case .failure(let refusal):
+    throw screenshotRefusal(refusal, app: app, frontWindowID: frontWindowID)
   }
 
   let filter = SCContentFilter(desktopIndependentWindow: window)
@@ -52,6 +50,54 @@ func captureAppWindowScreenshot(app: NSRunningApplication, outPath: String) thro
   try writePNG(capturedImage, to: outPath)
 }
 
+/// Why an app screenshot refuses. The capture is the app's front window or nothing: another
+/// window of the app would show state the user is not looking at. Pinned with the host by
+/// `contracts/fixtures/macos-native-helper-outcomes.json`.
+enum AppWindowScreenshotRefusal: String, CaseIterable, Error {
+  case windowNotFound = "window-not-found"
+  case frontWindowNotShareable = "front-window-not-shareable"
+  case frontWindowCaptureExcluded = "front-window-capture-excluded"
+}
+
+/// The shareable candidate whose WindowServer ID is the app's front window.
+func frontCaptureWindow<Window>(
+  _ candidates: [Window],
+  frontWindowID: Int?,
+  sharingState: Int?,
+  windowID: (Window) -> Int
+) -> Result<Window, AppWindowScreenshotRefusal> {
+  guard let frontWindowID else { return .failure(.windowNotFound) }
+  guard sharingState != 0 else { return .failure(.frontWindowCaptureExcluded) }
+  guard let front = candidates.first(where: { windowID($0) == frontWindowID }) else {
+    return .failure(.frontWindowNotShareable)
+  }
+  return .success(front)
+}
+
+private func screenshotRefusal(
+  _ refusal: AppWindowScreenshotRefusal,
+  app: NSRunningApplication,
+  frontWindowID: Int?
+) -> HelperError {
+  var details = ["reason": refusal.rawValue, "bundleId": app.bundleIdentifier ?? ""]
+  switch refusal {
+  case .windowNotFound:
+    return .commandFailed("screenshot could not find an on-screen window for the app", details: details)
+  case .frontWindowCaptureExcluded:
+    details["frontWindowId"] = frontWindowID.map(String.init) ?? ""
+    return .commandFailed(
+      "screenshot could not capture the app's front window: the app excludes it from capture",
+      details: details
+    )
+  case .frontWindowNotShareable:
+    details["frontWindowId"] = frontWindowID.map(String.init) ?? ""
+    return .commandFailed(
+      "screenshot could not capture the app's front window: ScreenCaptureKit does not offer it",
+      details: details
+    )
+  }
+}
+
 @available(macOS 14.0, *)
 private func awaitShareableContent() throws -> SCShareableContent {
   let semaphore = DispatchSemaphore(value: 0)

apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift

@@ -97,7 +97,7 @@ func pressInBackground(
   let point = CGPoint(x: request.x, y: request.y)
   try requireAppWindowPoint(point, app: app)
   cursor?.move(to: point)
-  let target = resolvePressTarget(app: app, point: point)
+  let target = try resolvePressTarget(app: app, point: point)
   if let target, isInMenuBar(target.element) {
     throw refusal(.noAccessibleTarget, "the point is on the app's menu bar", app: app)
   }
@@ -172,7 +172,7 @@ func fillInBackground(
   cursor?.move(to: point)
   let hit = elementAtPoint(in: app, point: point)
   let chain = hit.map(pressSearchChain) ?? []
-  let fallback = actionWindow(app: app, hit: hit).flatMap { window in
+  let fallback = try actionWindow(app: app, hit: hit).flatMap { window in
     smallestElement(in: window, containing: point, where: isTextInput)
   }
   if let input = chain.first(where: isTextInput) ?? fallback, isInMenuBar(input) {
@@ -206,7 +206,7 @@ func scrollInBackground(
   guard isVertical || direction == "left" || direction == "right" else {
     throw HelperError.invalidArgs("scroll requires --direction <up|down|left|right>")
   }
-  guard let frame = frontWindowFrame(pid: app.processIdentifier) else {
+  guard let frame = try frontWindowFrame(pid: app.processIdentifier) else {
     throw HelperError.commandFailed(
       "scroll could not resolve an on-screen window",
       details: ["reason": "window-not-found", "bundleId": app.bundleIdentifier ?? ""]
@@ -276,12 +276,12 @@ private enum PressTarget {
 
 /// What a click at a point means. The app's own hit test answers first; web content often answers
 /// with a wrapper group, so the snapshot geometry decides when the hit names no control.
-private func resolvePressTarget(app: NSRunningApplication, point: CGPoint) -> PressTarget? {
+private func resolvePressTarget(app: NSRunningApplication, point: CGPoint) throws -> PressTarget? {
   let hit = elementAtPoint(in: app, point: point)
   let chain = hit.map(pressSearchChain) ?? []
   if let input = chain.first(where: isTextInput) { return .focus(input) }
   if let control = chain.first(where: isPressableControl) { return .press(control) }
-  if let found = actionWindow(app: app, hit: hit).flatMap({
+  if let found = try actionWindow(app: app, hit: hit).flatMap({
     smallestElement(in: $0, containing: point, where: { isTextInput($0) || isPressableControl($0) })
   }) {
     return isTextInput(found) ? .focus(found) : .press(found)
@@ -306,10 +306,7 @@ private func perform(_ target: PressTarget) -> BackgroundDeliveryMechanism? {
 /// own on-screen windows, its menus and sheets included, and never on a menu bar element: the app
 /// owns its menu bar's window too, so the window check alone does not exclude it.
 private func requireAppWindowPoint(_ point: CGPoint, app: NSRunningApplication) throws {
-  guard
-    let info = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID)
-      as? [[String: Any]]
-  else {
+  guard let info = appWindowInfo() else {
     throw refusal(.noAccessibleTarget, "the app's on-screen windows could not be read", app: app)
   }
   let inAppWindow = info.contains { entry in
@@ -364,12 +361,12 @@ private func pressSearchChain(from hit: AXUIElement) -> [AXUIElement] {
 /// The one window a point action may act in: the window the app's hit test landed in, else the
 /// app's front on-screen window. A fallback target never comes from a window behind it, and the
 /// response names the window so the host can check it.
-private func actionWindow(app: NSRunningApplication, hit: AXUIElement?) -> AXUIElement? {
+private func actionWindow(app: NSRunningApplication, hit: AXUIElement?) throws -> AXUIElement? {
   if let hit {
     if role(of: hit) == "AXWindow" { return hit }
     if let window = elementAttribute(hit, attribute: kAXWindowAttribute as String) { return window }
   }
-  guard let front = frontWindowFrame(pid: app.processIdentifier) else { return nil }
+  guard let front = try frontWindowFrame(pid: app.processIdentifier) else { return nil }
   return windows(of: AXUIElementCreateApplication(app.processIdentifier)).first { window in
     guard let rect = rectAttribute(window) else { return false }
     return abs(rect.x - front.minX) < 1 && abs(rect.y - front.minY) < 1
@@ -482,39 +479,55 @@ private func numberAttribute(_ element: AXUIElement, attribute: String) -> Doubl
   return number.doubleValue
 }
 
-private struct OnScreenWindow {
+struct OnScreenWindow {
   let number: Int
   let bounds: CGRect
+  let sharingState: Int?
 }
 
-/// The pid's normal-layer windows, front to back. Bounds and owners need no Screen Recording
-/// permission; only titles do.
-private func onScreenWindows(pid: pid_t) -> [OnScreenWindow] {
-  guard
-    let info = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID)
-      as? [[String: Any]]
-  else {
-    return []
-  }
-  return info.compactMap { entry in
+/// Only normal-layer windows qualify; app-modal alerts at level 8 are excluded.
+let frontWindowLayer = 0
+
+/// The pid's visible, nonempty normal-layer windows, front to back.
+func onScreenWindows(pid: pid_t, windowInfo: [[String: Any]]) -> [OnScreenWindow] {
+  windowInfo.compactMap { entry in
     guard (entry[kCGWindowOwnerPID as String] as? Int32) == pid,
-      (entry[kCGWindowLayer as String] as? Int) == 0,
+      (entry[kCGWindowLayer as String] as? Int) == frontWindowLayer,
+      (entry[kCGWindowIsOnscreen as String] as? Bool) == true,
+      let alpha = entry[kCGWindowAlpha as String] as? Double, alpha > 0,
       let number = entry[kCGWindowNumber as String] as? Int,
       let boundsDict = entry[kCGWindowBounds as String] as? NSDictionary,
-      let bounds = CGRect(dictionaryRepresentation: boundsDict as CFDictionary)
+      let bounds = CGRect(dictionaryRepresentation: boundsDict as CFDictionary),
+      bounds.width > 0, bounds.height > 0
     else {
       return nil
     }
-    return OnScreenWindow(number: number, bounds: bounds)
+    return OnScreenWindow(
+      number: number, bounds: bounds, sharingState: entry[kCGWindowSharingState as String] as? Int)
   }
 }
 
-func frontWindowNumber(pid: pid_t) -> Int? {
-  onScreenWindows(pid: pid).first?.number
+func appWindowInfo() -> [[String: Any]]? {
+  CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID)
+    as? [[String: Any]]
+}
+
+enum WindowListRefusal: String, CaseIterable {
+  case unreadable = "window-list-unreadable"
+}
+
+func frontWindow(pid: pid_t, windowInfo: [[String: Any]]? = appWindowInfo()) throws -> OnScreenWindow? {
+  guard let windowInfo else {
+    throw HelperError.commandFailed(
+      "the app's on-screen windows could not be read",
+      details: ["reason": WindowListRefusal.unreadable.rawValue]
+    )
+  }
+  return onScreenWindows(pid: pid, windowInfo: windowInfo).first
 }
 
-private func frontWindowFrame(pid: pid_t) -> CGRect? {
-  onScreenWindows(pid: pid).first?.bounds
+func frontWindowFrame(pid: pid_t, windowInfo: [[String: Any]]? = appWindowInfo()) throws -> CGRect? {
+  try frontWindow(pid: pid, windowInfo: windowInfo)?.bounds
 }
 
 private let returnVirtualKey: CGKeyCode = 36

apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift

@@ -0,0 +1,144 @@
+import CoreGraphics
+import XCTest
+
+@testable import AgentDeviceMacOSHelper
+
+private struct Candidate: Equatable {
+  let id: Int
+  let area: Double
+}
+
+/// An app screenshot captures the front window or refuses; it never substitutes another window.
+final class AppWindowScreenshotTests: XCTestCase {
+  func testCaptureIdentityAndScrollAndClickFallbackFrameSelectTheSameEligibleFrontWindow() throws {
+    let windows = [
+      windowInfo(id: 1, pid: 99),
+      windowInfo(id: 2, layer: 1),
+      windowInfo(id: 3, onScreen: false),
+      windowInfo(id: 4, alpha: 0),
+      windowInfo(id: 5, width: 0),
+      windowInfo(id: 6, height: 0),
+      windowInfo(id: 7, alpha: 0.5, width: 20, height: 20),
+      windowInfo(id: 8),
+    ]
+
+    let frontID = try frontWindow(pid: 42, windowInfo: windows)?.number
+    let selected = frontCaptureWindow([7, 8], frontWindowID: frontID, sharingState: 1, windowID: { $0 })
+
+    // Scroll and the click actionWindow fallback both consume frontWindowFrame.
+    XCTAssertEqual(
+      try frontWindowFrame(pid: 42, windowInfo: windows),
+      CGRect(x: 10, y: 10, width: 20, height: 20))
+    XCTAssertEqual(frontID, 7)
+    XCTAssertEqual(try selected.get(), 7)
+  }
+
+  func testSkipsAppModalAlertInFrontOfNormalLayerWindow() throws {
+    let windows = [windowInfo(id: 8, layer: 8), windowInfo(id: 3)]
+    let front = try frontWindow(pid: 42, windowInfo: windows)
+    let selected = frontCaptureWindow(
+      [8, 3], frontWindowID: front?.number, sharingState: front?.sharingState, windowID: { $0 })
+
+    XCTAssertEqual(front?.number, 3)
+    XCTAssertEqual(try selected.get(), 3)
+    XCTAssertEqual(try frontWindowFrame(pid: 42, windowInfo: windows), front?.bounds)
+  }
+
+  func testRefusesWhenOnlyInvisibleOrEmptyWindowsRemain() throws {
+    let windows = [
+      windowInfo(id: 1, onScreen: false),
+      windowInfo(id: 2, alpha: 0),
+      windowInfo(id: 3, width: 0),
+      windowInfo(id: 4, height: 0),
+    ]
+    let frontID = try frontWindow(pid: 42, windowInfo: windows)?.number
+    let selected = frontCaptureWindow([5], frontWindowID: frontID, sharingState: 1, windowID: { $0 })
+
+    XCTAssertNil(try frontWindowFrame(pid: 42, windowInfo: windows))
+    XCTAssertNil(frontID)
+    XCTAssertThrowsError(try selected.get()) { error in
+      XCTAssertEqual(error as? AppWindowScreenshotRefusal, .windowNotFound)
+    }
+  }
+
+  func testUnreadableWindowListRefusesBeforeCaptureOrScrollCanReportNoWindow() {
+    XCTAssertThrowsError(try frontWindow(pid: 42, windowInfo: nil)) { error in
+      guard case HelperError.commandFailed(_, let details) = error else {
+        return XCTFail("expected a typed window-list refusal, got \(error)")
+      }
+      XCTAssertEqual(details["reason"], "window-list-unreadable")
+    }
+    XCTAssertThrowsError(try frontWindowFrame(pid: 42, windowInfo: nil)) { error in
+      guard case HelperError.commandFailed(_, let details) = error else {
+        return XCTFail("expected a typed window-list refusal, got \(error)")
+      }
+      XCTAssertEqual(details["reason"], "window-list-unreadable")
+    }
+    XCTAssertNil(try frontWindow(pid: 42, windowInfo: []))
+    XCTAssertNil(try frontWindowFrame(pid: 42, windowInfo: []))
+  }
+
+  func testSelectsTheFrontWindowEvenWhenAnotherIsLarger() {
+    let candidates = [Candidate(id: 7, area: 10_000), Candidate(id: 3, area: 100)]
+
+    let selected = frontCaptureWindow(candidates, frontWindowID: 3, sharingState: 1, windowID: \.id)
+
+    XCTAssertEqual(try selected.get(), Candidate(id: 3, area: 100))
+  }
+
+  func testRefusesWhenTheFrontWindowIsNotShareable() {
+    let candidates = [Candidate(id: 7, area: 10_000), Candidate(id: 8, area: 100)]
+
+    let selected = frontCaptureWindow(candidates, frontWindowID: 3, sharingState: 1, windowID: \.id)
+
+    XCTAssertThrowsError(try selected.get()) { error in
+      XCTAssertEqual(error as? AppWindowScreenshotRefusal, .frontWindowNotShareable)
+    }
+  }
+
+  func testCaptureExclusionSurvivesWindowSelectionAndNeverSubstitutesAnotherWindow() throws {
+    let windows = [windowInfo(id: 3, sharingState: 0), windowInfo(id: 7)]
+    let front = try frontWindow(pid: 42, windowInfo: windows)
+    XCTAssertEqual(front?.number, 3)
+    XCTAssertEqual(front?.sharingState, 0)
+
+    for candidates in [[3, 7], [7]] {
+      let selected = frontCaptureWindow(
+        candidates, frontWindowID: front?.number, sharingState: front?.sharingState, windowID: { $0 })
+      XCTAssertThrowsError(try selected.get()) { error in
+        XCTAssertEqual(error as? AppWindowScreenshotRefusal, .frontWindowCaptureExcluded)
+      }
+    }
+
+    var entry = windowInfo(id: 3)
+    entry.removeValue(forKey: kCGWindowSharingState as String)
+    let unknown = try frontWindow(pid: 42, windowInfo: [entry])
+    let selected = frontCaptureWindow(
+      [3], frontWindowID: unknown?.number, sharingState: unknown?.sharingState, windowID: { $0 })
+    XCTAssertEqual(try selected.get(), 3)
+  }
+
+  func testRefusesWhenTheAppHasNoOnScreenWindow() {
+    let selected = frontCaptureWindow(
+      [Candidate(id: 7, area: 10_000)], frontWindowID: nil, sharingState: nil, windowID: \.id)
+
+    XCTAssertThrowsError(try selected.get()) { error in
+      XCTAssertEqual(error as? AppWindowScreenshotRefusal, .windowNotFound)
+    }
+  }
+}
+
+private func windowInfo(
+  id: Int, pid: Int32 = 42, layer: Int = 0, onScreen: Bool = true,
+  alpha: Double = 1, sharingState: Int = 1, width: Double = 100, height: Double = 100
+) -> [String: Any] {
+  [
+    kCGWindowNumber as String: id,
+    kCGWindowOwnerPID as String: pid,
+    kCGWindowLayer as String: layer,
+    kCGWindowIsOnscreen as String: onScreen,
+    kCGWindowAlpha as String: alpha,
+    kCGWindowSharingState as String: sharingState,
+    kCGWindowBounds as String: CGRect(x: 10, y: 10, width: width, height: height).dictionaryRepresentation,
+  ]
+}

apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/BackgroundInteractionTests.swift

@@ -26,6 +26,8 @@ private struct ScrollGestureFixture: Decodable {
 private struct HelperOutcomesFixture: Decodable {
   let refusalReasons: [String]
   let deliveryMechanisms: [String]
+  let screenshotRefusalReasons: [String]
+  let windowListRefusalReasons: [String]
 }
 
 /// `#filePath` is `<repo>/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/<file>`.
@@ -41,8 +43,11 @@ final class BackgroundInteractionTests: XCTestCase {
       from: Data(
         contentsOf: repoRoot.appendingPathComponent("contracts/fixtures/macos-native-helper-outcomes.json"))
     )
+    XCTAssertEqual(WindowListRefusal.allCases.map(\.rawValue), fixture.windowListRefusalReasons)
     XCTAssertEqual(BackgroundRefusal.allCases.map(\.rawValue), fixture.refusalReasons)
     XCTAssertEqual(BackgroundDeliveryMechanism.allCases.map(\.rawValue), fixture.deliveryMechanisms)
+    XCTAssertEqual(
+      AppWindowScreenshotRefusal.allCases.map(\.rawValue), fixture.screenshotRefusalReasons)
   }
 
   /// The native backend scrolls as far as the runner does: its travel agrees with every case of