From a7370b2fe95ec2f7886c94919a825a56d9e36553 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 22 Oct 2026 21:51:37 +0200 Subject: [PATCH 01/15] fix(ios): publish a windowed iPad app's capture in screen space (#3899) An iPad window that does not fill the screen reports its own screen box while its subtree arrives anchored at its own top-left. The capture published a screen-space viewport around window-relative rects: the fold clipped content to (w - x) x (h - y), a rect named no screenshot position, and a synthesized tap landed at the point minus the window origin. SnapshotContentTranslation reads the one vector from a carrier's (0,0)-anchored copy of the viewport box to the viewport itself, and SnapshotGeometrySpace.normalized translates the subtree it declares in the same pass that turns a quarter-turned subtree. CoordinateSpaceRotation now turns axes only; the digitizer origin stays where the display put it. Replayed from contracts/fixtures/window-coordinate-space.json in both languages. Refs #3899 --- ...unnerTests+SnapshotPresentationTests.swift | 127 ++++++++++++++ .../SnapshotCoordinateSpace.swift | 112 +++++++++--- .../CoordinateSpaceTests.swift | 162 +++++++++++++++++- .../fixtures/window-coordinate-space.json | 92 +++++++++- .../adr/0004-ios-snapshot-backend-strategy.md | 37 ++++ .../platform-apple/src/snapshot-route.test.ts | 2 +- .../window-coordinate-space.test.ts | 48 +++++- .../window-coordinate-space.ts | 69 +++++++- 8 files changed, 600 insertions(+), 49 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift index 8138eb41fb..1510dce82a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift @@ -662,6 +662,133 @@ extension RunnerTests { XCTAssertNotNil(keyQ, "`q` key must be presented under --depth \(options.depth ?? -1)") } + /// #3899: an iPad window that does not fill the screen publishes screen space. The measured shape is + /// iPad Pro 11-inch (M5, iOS 27.0) with Windowed Apps on and a Calendar window at (224,207,386x682): + /// the app's own window arrives as a (0,0)-anchored copy of that box and its subtree with it, so the + /// Add button reports (335,14,37,36) for a control drawn at (559,221). The one normalization pass + /// reads the declaration and publishes the subtree at its place on the screen, which is what lets the + /// fold keep every node instead of clipping each one to (w − x) × (h − y) and refusing the capture. + func testAWindowedSubtreeIsPublishedInScreenSpaceAndKeepsEveryNode() throws { + let windowOnScreen = CGRect(x: 224, y: 207, width: 386, height: 682) + func node( + _ index: Int, _ type: String, _ label: String, _ rect: CGRect, _ parent: Int?, _ depth: Int + ) -> RawAXNode { + RawAXNode( + index: index, type: type, label: label, identifier: nil, value: nil, + rect: SnapshotRect(x: rect.minX, y: rect.minY, width: rect.width, height: rect.height), + enabled: true, focused: nil, selected: nil, hittable: false, + depth: depth, parentIndex: parent, hiddenContentAbove: nil, hiddenContentBelow: nil + ) + } + let reportedRoot = CGRect(x: 0, y: 0, width: 386, height: 682) + let acquired = [ + node(0, "Application", "Calendar", reportedRoot, nil, 0), + node(1, "Window", "Window", reportedRoot, 0, 1), + node(2, "Other", "Toolbar", reportedRoot, 1, 2), + node(3, "Button", "Add", CGRect(x: 335, y: 14, width: 37, height: 36), 2, 3), + node(4, "Button", "Today", CGRect(x: 8, y: 583, width: 90, height: 38), 2, 3), + // A SpringBoard overlay under the app root reports the screen's own box: its subtree is already + // in screen space and must not move. + node(5, "Window", "Alert", CGRect(x: 0, y: 0, width: 834, height: 1210), 0, 1), + node(6, "Button", "Discard Changes", CGRect(x: 300, y: 500, width: 240, height: 44), 5, 2), + ] + let normalized = SnapshotGeometrySpace.normalized( + nodes: acquired, + viewport: .reported( + box: windowOnScreen, + interfaceOrientation: RunnerInterfaceOrientation.portrait + ) + ) + let options = PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false) + let capture = try XCTUnwrap(try SnapshotPresentation.present( + SnapshotAcquisition( + hint: SnapshotPresentation.captureHint(for: options), + nodes: normalized, + truncated: false, + effectiveDepth: nil, + viewport: .reported(box: windowOnScreen) + ), + options: options + )) + + // One space: the root answers with the window's screen box, and the button with the position the + // screenshot draws. + XCTAssertEqual(capture.nodes.first?.rect, SnapshotRect(windowOnScreen)) + let add = try XCTUnwrap(capture.nodes.first { $0.label == "Add" }) + XCTAssertEqual(add.rect, SnapshotRect(x: 559, y: 221, width: 37, height: 36)) + XCTAssertEqual(add.hittable, true) + // Nothing is dropped for being "outside the clip": the whole acquired tree is still there. The one + // clipped node is the system overlay that really is larger than the app's window, which is the + // fold's normal answer to oversized content — not the mixed-space clip that used to publish + // (w − x) × (h − y) boxes and refuse this capture outright. + XCTAssertEqual(Set(capture.nodes.map(\.label)), Set(acquired.map(\.label))) + let clipNote = capture.qualityNotes.first { $0.code == .regularGeometryClipped } + XCTAssertEqual(clipNote?.droppedNodeCount, 0, "no node is dropped for the window's origin") + XCTAssertEqual(clipNote?.clippedNodeCount, 1, "only the overlay larger than the window is clipped") + XCTAssertEqual( + capture.nodes.first { $0.label == "Alert" }?.rect, + SnapshotRect(x: 224, y: 207, width: 386, height: 682), + "the overlay is clipped to the window's visible box, not to (w − x) × (h − y) of it" + ) + // The system window and its button stay where the platform reported them. + let discard = try XCTUnwrap(capture.nodes.first { $0.label == "Discard Changes" }) + XCTAssertEqual(discard.rect, SnapshotRect(x: 300, y: 500, width: 240, height: 44)) + } + + /// #3899 keeps a full-screen capture byte-identical: the same tree reported from the screen origin is + /// published exactly as reported, with no translation read from its zero-origin viewport. + func testAFullScreenCapturePublishesWhatThePlatformReported() throws { + let screen = CGRect(x: 0, y: 0, width: 834, height: 1210) + func node( + _ index: Int, _ type: String, _ label: String, _ rect: CGRect, _ parent: Int?, _ depth: Int + ) -> RawAXNode { + RawAXNode( + index: index, type: type, label: label, identifier: nil, value: nil, + rect: SnapshotRect(x: rect.minX, y: rect.minY, width: rect.width, height: rect.height), + enabled: true, focused: nil, selected: nil, hittable: false, + depth: depth, parentIndex: parent, hiddenContentAbove: nil, hiddenContentBelow: nil + ) + } + let acquired = [ + node(0, "Application", "Calendar", screen, nil, 0), + node(1, "Window", "Window", screen, 0, 1), + node(2, "Button", "Add", CGRect(x: 335, y: 14, width: 37, height: 36), 1, 2), + ] + let normalized = SnapshotGeometrySpace.normalized( + nodes: acquired, + viewport: .reported( + box: screen, + interfaceOrientation: RunnerInterfaceOrientation.portrait + ) + ) + XCTAssertEqual(normalized.map(\.rect), acquired.map(\.rect)) + XCTAssertEqual( + SnapshotContentTranslation.offset(reportedBox: screen, viewportBox: screen), + nil, + "a viewport at the screen origin carries no translation" + ) + } + + /// #3899 keeps ADR 0025 foldable panel taps: a panel reports its own box from its own display's + /// origin, so no translation is read and the panel's geometry stays exactly as measured. + func testAFoldablePanelReportsNoContentTranslation() { + let panel = CGRect(x: 0, y: 0, width: 951, height: 669) + XCTAssertNil( + SnapshotContentTranslation.offset(reportedBox: panel, viewportBox: panel), + "a panel box anchored at its display's origin names no translation" + ) + // A digitizer point on the panel is passed through, so the fold's display selection stays the only + // window-dependent choice the tap path makes. + XCTAssertEqual( + CoordinateSpaceRotation.native( + point: CGPoint(x: 476, y: 226), + in: panel, + interfaceOrientation: RunnerInterfaceOrientation.portrait + ), + CGPoint(x: 476, y: 226) + ) + } + /// #2891: a capture whose viewport read failed publishes no `hittable` for a node whose answer is /// containment, while a root and a disabled node stay a declared `false`. Pinned on the /// encoded objects rather than on `capture.nodes`, because the promise is about the wire: a test diff --git a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift index 300b8c910e..7718fbabc1 100644 --- a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift +++ b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift @@ -4,6 +4,10 @@ import CoreGraphics // Rotation between an app's interface space and the device's native (portrait-up) space, shared by // synthesized dispatch (forward) and capture (inverse). Rationale and measurements: ADR 0004 // "the coordinate space of a captured subtree" and contracts/fixtures/window-coordinate-space.json. +// +// Rotation moves axes only. It never moves an origin: a placed window's offset from the screen +// belongs to `SnapshotContentTranslation` (#3899), which is the one rule that turns a reported box +// into its screen-space box. Keeping the two apart is what lets a capture publish one space. public enum RunnerInterfaceOrientation { public static let unknown = 0 @@ -14,18 +18,18 @@ public enum RunnerInterfaceOrientation { } public enum CoordinateSpaceRotation { - // Dispatch rotates a point about the reference frame's top-left. The origin is localized away - // and never added back: an XCUIApplication session observes only its own scene surface, so its - // windows report (0,0)-anchored frames (measured on iPhone landscape both ways and on every - // system window in window-coordinate-space.json), and displacement between panels or panes is - // carried by the record's displayID, not by an origin. + // Dispatch turns a point's axes and never moves its origin: the turn says which axis a box runs + // along, not where that box sits (#3899). The digitizer's origin is the display's own, which is + // where a windowed app's screen-space points are already measured from, so a placed window's + // offset from the screen belongs to `SnapshotContentTranslation` — the one rule that reads an + // offset — and not to this function. The `frame` argument names the side lengths the turn swaps. public static func native( point: CGPoint, in frame: CGRect, interfaceOrientation: Int ) -> CGPoint { - let localX = Double(point.x) - Double(frame.minX) - let localY = Double(point.y) - Double(frame.minY) + let localX = Double(point.x) + let localY = Double(point.y) let width = Double(frame.width) let height = Double(frame.height) switch interfaceOrientation { @@ -60,23 +64,23 @@ public enum CoordinateSpaceRotation { ) -> CGPoint { let width = Double(frame.width) let height = Double(frame.height) - let localX: Double - let localY: Double + let x: Double + let y: Double switch interfaceOrientation { case RunnerInterfaceOrientation.landscapeRight: - localX = Double(point.y) - localY = height - Double(point.x) + x = Double(point.y) + y = height - Double(point.x) case RunnerInterfaceOrientation.landscapeLeft: - localX = width - Double(point.y) - localY = Double(point.x) + x = width - Double(point.y) + y = Double(point.x) case RunnerInterfaceOrientation.portraitUpsideDown: - localX = width - Double(point.x) - localY = height - Double(point.y) + x = width - Double(point.x) + y = height - Double(point.y) default: - localX = Double(point.x) - localY = Double(point.y) + x = Double(point.x) + y = Double(point.y) } - return CGPoint(x: localX + Double(frame.minX), y: localY + Double(frame.minY)) + return CGPoint(x: x, y: y) } public static func oriented( @@ -103,9 +107,57 @@ public enum CoordinateSpaceRotation { } } +/// Whether a carrier's own box declares that its subtree reports in the carrier's content space +/// rather than in the space the viewport names (#3899). +/// +/// An iPad window that does not fill the screen (Windowed Apps, Stage Manager) reports its own frame +/// at its place on the screen while its subtree arrives anchored at its own top-left: measured on +/// iPad Pro 11-inch (M5, iOS 27.0) with a Calendar window whose `frame` is `(224,207,386,682)`, the +/// application root reports `(0,0,386,682)` and its Add button `(335,14,37,36)` at screen +/// `(559,221)`. A full-screen window, an iPhone, and each foldable panel report origins at zero on +/// both sides, so the rule reads zero translation there and every published rect is unchanged. +/// +/// The signature is the carrier's box as a `(0,0)`-anchored copy of the viewport box: the side +/// lengths match within the tolerance and the viewport — the app's screen-anchored box — starts +/// somewhere else on the screen. Any other carrier box names no space: it is not the app's own +/// window, so its numbers say nothing about where the app's subtree starts. +public enum SnapshotContentTranslation { + /// The vector from a carrier's reported box to the screen space it is placed at, or `nil` when the + /// carrier's box declares no translation. One box offset from one viewport box, decided from the + /// same tolerance the quarter-turn rule reads; replayed against + /// `contracts/fixtures/window-coordinate-space.json` in `windowContentCases`. + public static func offset(reportedBox: CGRect, viewportBox: CGRect) -> CGSize? { + guard SnapshotGeometry.isPositiveFinite(reportedBox), + SnapshotGeometry.isPositiveFinite(viewportBox) + else { + return nil + } + guard abs(reportedBox.minX) <= SnapshotGeometrySpace.quarterTurnTolerance, + abs(reportedBox.minY) <= SnapshotGeometrySpace.quarterTurnTolerance + else { + return nil + } + guard abs(viewportBox.minX) > SnapshotGeometrySpace.quarterTurnTolerance + || abs(viewportBox.minY) > SnapshotGeometrySpace.quarterTurnTolerance + else { + return nil + } + guard abs(reportedBox.width - viewportBox.width) <= SnapshotGeometrySpace.quarterTurnTolerance, + abs(reportedBox.height - viewportBox.height) <= SnapshotGeometrySpace.quarterTurnTolerance + else { + return nil + } + return CGSize(width: viewportBox.minX, height: viewportBox.minY) + } +} + public enum SnapshotGeometrySpace: Equatable { case appOrientation case deviceNative(appFrame: SnapshotViewport.Box, interfaceOrientation: Int) + /// The subtree was reported in a placed window's own content space and is published translated + /// into the viewport's screen space (`SnapshotContentTranslation`, #3899). The offset is the + /// vector from the window's reported box to the viewport box it sits at on screen. + case windowContent(offset: CGSize) public static let quarterTurnTolerance: Double = 1 @@ -119,10 +171,15 @@ public enum SnapshotGeometrySpace: Equatable { in: appFrame.rect, interfaceOrientation: interfaceOrientation ) + case .windowContent(let offset): + return reportedFrame.offsetBy(dx: offset.width, dy: offset.height) } } - /// Only a `.reported` viewport carries an orientation, so only it can anchor a rotation. + /// A carrier declares its own subtree's space from its own box. A `.reported` viewport is the + /// screen-anchored fact both declarations read against: a quarter turn needs its orientation, and + /// a window-content offset needs its origin. A `.derived` viewport carries no screen anchor, so a + /// subtree under it publishes what the platform reported rather than a guessed space. public static func space( reportedBySurfaceHost isSurfaceHost: Bool, reportedFrame: CGRect, @@ -130,13 +187,20 @@ public enum SnapshotGeometrySpace: Equatable { viewport: SnapshotViewport ) -> SnapshotGeometrySpace { guard isSurfaceHost else { return inherited } - guard case .reported(let appFrame, let interfaceOrientation) = viewport, - namesQuarterTurn(interfaceOrientation), - isQuarterTurned(reportedFrame, relativeTo: appFrame.rect) - else { + guard case .reported(let appFrame, let interfaceOrientation) = viewport else { return .appOrientation } - return .deviceNative(appFrame: appFrame, interfaceOrientation: interfaceOrientation) + if namesQuarterTurn(interfaceOrientation), + isQuarterTurned(reportedFrame, relativeTo: appFrame.rect) { + return .deviceNative(appFrame: appFrame, interfaceOrientation: interfaceOrientation) + } + if let offset = SnapshotContentTranslation.offset( + reportedBox: reportedFrame, + viewportBox: appFrame.rect + ) { + return .windowContent(offset: offset) + } + return .appOrientation } public static func namesQuarterTurn(_ interfaceOrientation: Int) -> Bool { diff --git a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift index c5df2bc22c..82a76fb74a 100644 --- a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift +++ b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift @@ -28,9 +28,22 @@ final class CoordinateSpaceTests: XCTestCase { let oriented: Frame } + struct WindowContentCase: Decodable { + let name: String + let reportedBox: Frame + let viewportBox: Frame + let offset: Offset? + } + + struct Offset: Decodable { + let width: Double + let height: Double + } + let constants: Constants let quarterTurnCases: [QuarterTurnCase] let rotationCases: [RotationCase] + let windowContentCases: [WindowContentCase] } /// Identity in portrait/unknown, 90° per landscape, 180° upside-down. @@ -63,9 +76,9 @@ final class CoordinateSpaceTests: XCTestCase { in: offsetLandscape, interfaceOrientation: RunnerInterfaceOrientation.landscapeLeft ), - CGPoint(x: 268, y: 1040), - "non-zero frame origin is localized before rotation" - ) + CGPoint(x: 288, y: 1030), + "a frame offset from the screen never moves the turned point (#3899)" + ); } func testNativeSynthesizedVectorRotatesByInterfaceOrientation() { @@ -303,7 +316,7 @@ final class CoordinateSpaceTests: XCTestCase { ) } - /// Golden parity table (#2612): every case in contracts/fixtures/window-coordinate-space.json + /// Golden parity table (#2612, #3899): every case in contracts/fixtures/window-coordinate-space.json /// must agree with the vitest twin /// (packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts). Add /// cases there, never fork the rule. Detection is replayed under both landscape turns, because @@ -313,6 +326,22 @@ final class CoordinateSpaceTests: XCTestCase { let fixture = try loadWindowCoordinateSpaceFixture() XCTAssertFalse(fixture.quarterTurnCases.isEmpty, "parity table must not be empty") XCTAssertFalse(fixture.rotationCases.isEmpty, "parity table must not be empty") + XCTAssertFalse(fixture.windowContentCases.isEmpty, "parity table must not be empty") + XCTAssertEqual( + Set(fixture.quarterTurnCases.map(\.name)).count, + fixture.quarterTurnCases.count, + "parity table case names must be unique" + ) + XCTAssertEqual( + Set(fixture.rotationCases.map(\.name)).count, + fixture.rotationCases.count, + "parity table case names must be unique" + ) + XCTAssertEqual( + Set(fixture.windowContentCases.map(\.name)).count, + fixture.windowContentCases.count, + "parity table case names must be unique" + ) XCTAssertEqual( fixture.constants.quarterTurnTolerance, SnapshotGeometrySpace.quarterTurnTolerance, @@ -349,6 +378,17 @@ final class CoordinateSpaceTests: XCTestCase { testCase.name ) } + for testCase in fixture.windowContentCases { + let expected = testCase.offset.map { CGSize(width: $0.width, height: $0.height) } + XCTAssertEqual( + SnapshotContentTranslation.offset( + reportedBox: testCase.reportedBox.cgRect, + viewportBox: testCase.viewportBox.cgRect + ), + expected, + testCase.name + ) + } } private func deviceNative(_ appFrame: CGRect, _ interfaceOrientation: Int) -> SnapshotGeometrySpace { @@ -486,6 +526,120 @@ final class CoordinateSpaceTests: XCTestCase { ) } + /// The measured windowed iPad Calendar window (#3899) driven through the pass: the carrier reports + /// a `(0,0)`-anchored copy of the viewport box, declares its subtree window-relative, and every + /// rect under it comes back at its place on the screen — the root at the window's screen box, the + /// Add button at (559,221) where the screenshot draws it, and a full-screen-capture tree untouched. + func testNormalizedTranslatesAWindowedSubtreeIntoScreenSpace() { + let windowOnScreen = CGRect(x: 224, y: 207, width: 386, height: 682) + let reportedRoot = CGRect(x: 0, y: 0, width: 386, height: 682) + let reportedAddButton = CGRect(x: 335, y: 14, width: 37, height: 36) + let acquired = [ + coordinateNode(0, "Application", reportedRoot, nil, 0, label: "Calendar"), + coordinateNode(1, "Window", reportedRoot, 0, 1), + coordinateNode(2, "Other", reportedRoot, 1, 2), + coordinateNode(3, "Button", reportedAddButton, 2, 3, label: "Add"), + ] + let normalized = SnapshotGeometrySpace.normalized( + nodes: acquired, + viewport: .reported(box: windowOnScreen, interfaceOrientation: RunnerInterfaceOrientation.portrait) + ) + XCTAssertEqual(normalized[0].rect.cgRect, windowOnScreen) + XCTAssertEqual(normalized[1].rect.cgRect, windowOnScreen) + XCTAssertEqual(normalized[2].rect.cgRect, windowOnScreen) + XCTAssertEqual(normalized[3].rect.cgRect, CGRect(x: 559, y: 221, width: 37, height: 36)) + XCTAssertEqual(normalized[3].hittable, true) + + // The same window moved: the subtree follows the new origin. + let moved = SnapshotGeometrySpace.normalized( + nodes: acquired, + viewport: .reported( + box: CGRect(x: 120, y: 90, width: 386, height: 682), + interfaceOrientation: RunnerInterfaceOrientation.portrait + ) + ) + XCTAssertEqual(moved[3].rect.cgRect, CGRect(x: 455, y: 104, width: 37, height: 36)) + } + + /// What must not move: a full-screen capture, and any subtree under a carrier that is not the app's + /// own placed window. Those trees keep exactly the rect the platform reported (#3899 keeps + /// full-screen and ADR 0025 panel behavior identical). + func testNormalizedTranslatesNothingForFullScreenAndForeignCarriers() { + // Full-screen iPad: reported boxes already start at the viewport. + let fullScreen = CGRect(x: 0, y: 0, width: 834, height: 1210) + let acquired = [ + coordinateNode(0, "Application", fullScreen, nil, 0), + coordinateNode(1, "Window", fullScreen, 0, 1), + coordinateNode(2, "Button", CGRect(x: 335, y: 14, width: 37, height: 36), 1, 2), + ] + XCTAssertEqual( + SnapshotGeometrySpace.normalized( + nodes: acquired, + viewport: .reported( + box: fullScreen, + interfaceOrientation: RunnerInterfaceOrientation.portrait + ) + ).map(\.rect), + acquired.map(\.rect) + ); + + // A windowed app, measured shape: the app's own window declares the translation and carries the + // subtree; the keyboard window and the SpringBoard overlay report the screen's own box, so their + // subtrees stay where the platform put them. + let windowOnScreen = CGRect(x: 224, y: 207, width: 386, height: 682) + let reportedRoot = CGRect(x: 0, y: 0, width: 386, height: 682) + let mixed = [ + coordinateNode(0, "Application", reportedRoot, nil, 0), + coordinateNode(1, "Window", reportedRoot, 0, 1, label: "appWindow"), + coordinateNode(2, "Other", reportedRoot, 1, 2), + coordinateNode(3, "Button", CGRect(x: 335, y: 14, width: 37, height: 36), 2, 3, label: "Add"), + coordinateNode(4, "Window", CGRect(x: 0, y: 925, width: 834, height: 282), 0, 1, label: "keyboard"), + coordinateNode(5, "Key", CGRect(x: 300, y: 1100, width: 60, height: 60), 4, 2), + coordinateNode(6, "Window", CGRect(x: 0, y: 0, width: 834, height: 1210), 0, 1, label: "springboard"), + coordinateNode(7, "Alert", CGRect(x: 300, y: 500, width: 240, height: 160), 6, 2), + ] + let normalized = SnapshotGeometrySpace.normalized( + nodes: mixed, + viewport: .reported(box: windowOnScreen, interfaceOrientation: RunnerInterfaceOrientation.portrait) + ) + XCTAssertEqual(normalized[1].rect.cgRect, windowOnScreen, "appWindow") + XCTAssertEqual(normalized[2].rect.cgRect, windowOnScreen) + XCTAssertEqual(normalized[3].rect.cgRect, CGRect(x: 559, y: 221, width: 37, height: 36)) + XCTAssertEqual(normalized[4].rect, mixed[4].rect, "keyboard") + XCTAssertEqual(normalized[5].rect, mixed[5].rect) + XCTAssertEqual(normalized[6].rect, mixed[6].rect, "springboard") + XCTAssertEqual(normalized[7].rect, mixed[7].rect) + } + + /// The dispatch direction for a windowed app: the caller's point is the point touched, so + /// `native(point:)` turns a screen-space point's axes without moving it toward the window origin + /// (#3899). Under portrait — the only orientation measured for a windowed app — it is the identity. + func testNativePointForAWindowedAppKeepsTheCallersScreenPoint() { + let windowOnScreen = CGRect(x: 224, y: 207, width: 386, height: 682) + for orientation in [ + RunnerInterfaceOrientation.portrait, RunnerInterfaceOrientation.unknown + ] { + XCTAssertEqual( + CoordinateSpaceRotation.native( + point: CGPoint(x: 578, y: 239), + in: windowOnScreen, + interfaceOrientation: orientation + ), + CGPoint(x: 578, y: 239), + "interfaceOrientation \(orientation)" + ) + } + // A turn moves axes only: the same point under a landscape window keeps its own origin story. + XCTAssertEqual( + CoordinateSpaceRotation.native( + point: CGPoint(x: 578, y: 239), + in: windowOnScreen, + interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + ), + CGPoint(x: 682 - 239, y: 578) + ) + } + func testNormalizedOfAnEmptyArrayIsEmpty() { XCTAssertEqual( SnapshotGeometrySpace.normalized( diff --git a/contracts/fixtures/window-coordinate-space.json b/contracts/fixtures/window-coordinate-space.json index ca92a01a71..d42ef974ce 100644 --- a/contracts/fixtures/window-coordinate-space.json +++ b/contracts/fixtures/window-coordinate-space.json @@ -1,5 +1,5 @@ { - "description": "Golden table for the coordinate space a captured window's own box declares (#2612), the twin of tap-keyboard-occlusion-policy.json on the geometry that guard reads. iOS hosts some system surfaces in the device's native (portrait-up) space even while the app is rotated, so their whole subtree arrives quarter-turned: measured on iPhone 17 Pro (iOS 26.2) with the system keyboard up over a landscape app frame (0,0,874,402), UIRemoteKeyboardWindow reports its own box as (0,0,402,874) — the app's box with the two side lengths swapped — while the app's own UIWindow, UITextEffectsWindow and a dev-menu overlay window all report (0,0,874,402); in portrait the same keyboard window reports (0,0,402,874) against a (0,0,402,874) app frame, i.e. nothing is turned. RULE 1 (detection) is enforced twice, in packages/contracts/src/window-coordinate-space.ts and in SnapshotGeometrySpace inside apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift, and decides from side lengths alone: a window is quarter-turned relative to an app frame when the app frame is not square (abs(app.width - app.height) > quarterTurnTolerance), both frames are finite, positive and non-empty, and the window's side lengths match the app's on the swapped axes (abs(window.width - app.height) <= quarterTurnTolerance && abs(window.height - app.width) <= quarterTurnTolerance). Origins are deliberately never compared: the turn is a statement about which axis a box runs along, and a window may legitimately be placed elsewhere. A square app frame cannot be told from its own quarter turn, so it is left alone rather than guessed at. The tolerance is 1 because measured captures match the swap to the point and this absorbs float representation only. RULE 2 (rotation) is the capture's way back for the runner's own UIInterfaceOrientation raw values — 1 portrait, 2 portraitUpsideDown, 3 landscapeRight, 4 landscapeLeft, and 0 unknown, where portrait and unknown publish what the platform reported rather than inventing a turn — implemented by CoordinateSpaceRotation.oriented(rect:in:interfaceOrientation:) in that same Swift file, whose rect helper turns opposite corners of the reported box and re-orders them so the result keeps a top-left origin. `rotationCases` are native -> oriented rows: the landscape ones measured on the device above, the 180 degree and identity ones derived from that table. Capture turns back with the exact inverse of what dispatch turns forward, so each row also has to come back to its own `native` box under the forward mapping; that is why one measured-looking landscape-left pairing whose native and oriented boxes are identical is absent — the table computes no such row. Infinity has no JSON spelling, so the unusable frame a platform hands back when it could not resolve a box is named {\"infinite\": true} and decodes to CGRect.infinite on the Swift side and to an infinite rect on the TypeScript side.", + "description": "Golden table for the coordinate space a captured window's own box declares (#2612), the twin of tap-keyboard-occlusion-policy.json on the geometry that guard reads. iOS hosts some system surfaces in the device's native (portrait-up) space even while the app is rotated, so their whole subtree arrives quarter-turned: measured on iPhone 17 Pro (iOS 26.2) with the system keyboard up over a landscape app frame (0,0,874,402), UIRemoteKeyboardWindow reports its own box as (0,0,402,874) — the app's box with the two side lengths swapped — while the app's own UIWindow, UITextEffectsWindow and a dev-menu overlay window all report (0,0,874,402); in portrait the same keyboard window reports (0,0,402,874) against a (0,0,402,874) app frame, i.e. nothing is turned. RULE 1 (detection) is enforced twice, in packages/contracts/src/window-coordinate-space.ts and in SnapshotGeometrySpace inside apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift, and decides from side lengths alone: a window is quarter-turned relative to an app frame when the app frame is not square (abs(app.width - app.height) > quarterTurnTolerance), both frames are finite, positive and non-empty, and the window's side lengths match the app's on the swapped axes (abs(window.width - app.height) <= quarterTurnTolerance && abs(window.height - app.width) <= quarterTurnTolerance). Origins are deliberately never compared: the turn is a statement about which axis a box runs along, and a window may legitimately be placed elsewhere. A square app frame cannot be told from its own quarter turn, so it is left alone rather than guessed at. The tolerance is 1 because measured captures match the swap to the point and this absorbs float representation only. RULE 2 (rotation) is the capture's way back for the runner's own UIInterfaceOrientation raw values — 1 portrait, 2 portraitUpsideDown, 3 landscapeRight, 4 landscapeLeft, and 0 unknown, where portrait and unknown publish what the platform reported rather than inventing a turn — implemented by CoordinateSpaceRotation.oriented(rect:in:interfaceOrientation:) in that same Swift file, whose rect helper turns opposite corners of the reported box and re-orders them so the result keeps a top-left origin. `rotationCases` are native -> oriented rows: the landscape ones measured on the device above, the 180 degree and identity ones derived from that table. Capture turns back with the exact inverse of what dispatch turns forward, so each row also has to come back to its own `native` box under the forward mapping; that is why one measured-looking landscape-left pairing whose native and oriented boxes are identical is absent — the table computes no such row. Infinity has no JSON spelling, so the unusable frame a platform hands back when it could not resolve a box is named {\"infinite\": true} and decodes to CGRect.infinite on the Swift side and to an infinite rect on the TypeScript side. RULE 3 (content translation) is the other space a captured subtree arrives in, and the one rule in this file that reads an origin. An iPad window that does not fill the screen (Windowed Apps, Stage Manager) reports its own screen box while the subtree under it arrives anchored at its own top-left: measured on iPad Pro 11-inch (M5, iOS 27.0) with a Calendar window at screen (224,207,386x682), the application root reports (0,0,386,682) and its Add button (335,14,37,36) for a control drawn at screen (559,221). Published as reported, that tree mixes two spaces, clips its own content, and moves every tap by the window origin (#3899). SnapshotContentTranslation.offset(reportedBox:viewportBox:) reads the carrier box against the viewport box the platform reported, and names the viewport origin as the offset only when the carrier is the app's own window: a (0,0)-anchored box whose side lengths match the viewport's within the same tolerance, while the viewport starts somewhere else on the screen. A carrier that is not the app's own window names no space, so its numbers are left alone: the keyboard window and a SpringBoard overlay under a windowed app report the screen's own box and are already in screen space. A full-screen window, an iPhone, and each foldable panel of ADR 0025 report origins at zero on both sides, so the rule reads no offset there and every published rect stays exactly as it was measured before. `windowContentCases` are reportedBox + viewportBox to offset rows, replayed against that Swift function and against windowContentTranslation in packages/platform-apple/src/snapshot-source/window-coordinate-space.ts; a null offset means the rule declares no translation. Because a translation is the one thing a rotation must not also do, `rotationCases` pin the turn as axes-only: `native` and `oriented` never move a box's origin, and a row whose app frame is offset from the screen comes back at the same place as its own zero-origin twin.", "constants": { "quarterTurnTolerance": 1 }, @@ -126,11 +126,11 @@ "oriented": { "x": 75, "y": 198, "width": 0, "height": 0 } }, { - "name": "landscape right localizes a non-zero app frame origin before the turn and restores it after", + "name": "a non-zero app frame origin does not travel through the turn: the same native box comes back at the same place (#3899)", "interfaceOrientation": 3, "app": { "x": 12, "y": 24, "width": 874, "height": 402 }, "native": { "x": 2, "y": 75, "width": 202, "height": 724 }, - "oriented": { "x": 87, "y": 222, "width": 724, "height": 202 } + "oriented": { "x": 75, "y": 198, "width": 724, "height": 202 } }, { "name": "landscape left turns the same plane the other way, which is where its own native column is", @@ -167,5 +167,91 @@ "native": { "x": 226, "y": 475, "width": 0, "height": 0 }, "oriented": { "x": 476, "y": 226, "width": 0, "height": 0 } } + ], + "windowContentCases": [ + { + "name": "RULE 3 (content translation): a Calendar window placed at (224,207) reports its own screen box and its subtree from its own top-left", + "reportedBox": { "x": 0, "y": 0, "width": 386, "height": 682 }, + "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "offset": { "width": 224, "height": 207 } + }, + { + "name": "the same window moved elsewhere on the screen carries its subtree by the new origin", + "reportedBox": { "x": 0, "y": 0, "width": 386, "height": 682 }, + "viewportBox": { "x": 120, "y": 90, "width": 386, "height": 682 }, + "offset": { "width": 120, "height": 90 } + }, + { + "name": "the app's own window box reports the screen box it is placed at and translates nothing", + "reportedBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "offset": null + }, + { + "name": "a full-screen iPad window fills the screen from (0,0), so its subtree starts where the viewport starts", + "reportedBox": { "x": 0, "y": 0, "width": 834, "height": 1210 }, + "viewportBox": { "x": 0, "y": 0, "width": 834, "height": 1210 }, + "offset": null + }, + { + "name": "a full-screen iPhone window is the same no-op on the smaller screen", + "reportedBox": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "viewportBox": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "offset": null + }, + { + "name": "a Duo panel reports its own box from the origin of its own display, so no translation is read (ADR 0025)", + "reportedBox": { "x": 0, "y": 0, "width": 951, "height": 669 }, + "viewportBox": { "x": 0, "y": 0, "width": 951, "height": 669 }, + "offset": null + }, + { + "name": "the keyboard window under a windowed app reports the screen's box and stays in screen space", + "reportedBox": { "x": 0, "y": 925, "width": 834, "height": 282 }, + "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "offset": null + }, + { + "name": "a SpringBoard overlay under a windowed app reports the whole screen and translates nothing", + "reportedBox": { "x": 0, "y": 0, "width": 834, "height": 1210 }, + "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "offset": null + }, + { + "name": "a half-tolerance origin miss is float representation, not a different placement", + "reportedBox": { "x": 0, "y": 0, "width": 385.5, "height": 682 }, + "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "offset": { "width": 224, "height": 207 } + }, + { + "name": "a carrier two points narrower than the viewport is some other box and declares no space", + "reportedBox": { "x": 0, "y": 0, "width": 384, "height": 682 }, + "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "offset": null + }, + { + "name": "a carrier anchored inside the screen names some content box, not the app's window", + "reportedBox": { "x": 40, "y": 12, "width": 386, "height": 682 }, + "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "offset": null + }, + { + "name": "a viewport the capture derived from its own root carries no screen anchor to translate toward", + "reportedBox": { "x": 0, "y": 0, "width": 386, "height": 682 }, + "viewportBox": { "x": 0, "y": 0, "width": 386, "height": 682 }, + "offset": null + }, + { + "name": "a zero-width carrier carries no geometry to read a space from", + "reportedBox": { "x": 0, "y": 0, "width": 0, "height": 682 }, + "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "offset": null + }, + { + "name": "a non-finite viewport declares nothing", + "reportedBox": { "x": 0, "y": 0, "width": 386, "height": 682 }, + "viewportBox": { "infinite": true }, + "offset": null + } ] } diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index 0d7d1c78c3..d56a194091 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -626,6 +626,43 @@ The rule detects un-normalized arrival, not a standing fact about iOS. The producers that the earlier sections of this ADR describe do normalize. A band taller than it is wide is what the output of a producer that did not normalize looks like. +## Amendment: a window that does not fill the screen (issue #3899) + +An iPad window that does not fill the screen splits a capture in two. +The measurement used iPad Pro 11-inch (M5, iOS 27.0) with Windowed Apps on and a Calendar window at screen `(224,207,386x682)`: + +- The app's root and its subtree report `(0,0)`-anchored frames. The window's `Add` button reads `(335,14,37,36)` and draws at screen `(559,221)`. +- The runner's live `XCUIApplication.windows` read reports the window's screen box `(224,207,386,682)`. +- A system window under the app root — the keyboard at `(0,925,834,282)`, a SpringBoard overlay at `(0,0,834,1210)` — reports the screen's own box. + +The viewport is built from the live read, so the capture published a screen-space viewport around window-relative rects. +That mix produced three failures from one cause: the fold clipped each node to `(w - x) x (h - y)` and the regular capture died with `regular-node-outside-cumulative-clip`; a published rect named no position in the screenshot; and a coordinate tap landed at the point minus the window origin. +The same drift appears in any app the user has placed, so Stage Manager behaves the same way. + +Decision. A window that does not fill the screen publishes screen space in points. +One capture publishes one space, as the #2612 amendment decided, and the space a placed window names is the screen's — because that is the space its viewport, its screenshot, and its digitizer already use. +The conversion runs where the platform's frame is still the platform's frame: inside the same `SnapshotGeometrySpace.normalized` pass that turns a quarter-turned subtree. +No per-command patch reads a window origin. + +A window declares its subtree's space from its own box, and this is the one rule in the table that reads an origin. +The carrier's box must be a `(0,0)`-anchored copy of the viewport box the platform reported, and the viewport must start somewhere else on the screen. +Then the subtree was reported in the carrier's content space, and the viewport's origin is the one vector that carries it to the screen. +Any other carrier box names no space, so the rule leaves its numbers alone: the keyboard window and the SpringBoard overlay above report the screen's own box and are already in screen space. +A full-screen window, an iPhone, and each foldable panel of ADR 0025 report origins at zero on both sides, so the rule reads no vector there and every rect is unchanged. +`SnapshotContentTranslation.offset(reportedBox:viewportBox:)` is that rule, replayed from `contracts/fixtures/window-coordinate-space.json` in both languages. + +Rotation and translation stay separate rules, and the split is the point. +`CoordinateSpaceRotation` turns a box's axes and never moves its origin: the digitizer's origin is the display's own, which is where a placed window's screen-space points are already measured from. +Before this amendment, that function localized a non-zero frame origin away and never added it back. +Every configuration measured before this one has its app frame at the screen origin, so the localization was dead code there and the dispatch bug here. +Translation is the one rule that reads an origin, so it is the only place the fix can live. + +Capture is half of the promise; dispatch is the other half. +The measured dispatch spaces for a placed window are both screen-anchored: the synthesized path lands a point at its screen position, and XCTest's own `XCUICoordinate` on the app anchors at the screen's origin too. +`interactionCoordinate` therefore passes the caller's screen point through, and `resolveRunnerWindow` keeps naming the window a foldable's tap resolves its display on (ADR 0025). +The live `XCUIElement.frame` reads that the tap path already compares — the selector match, the input probe, the tree's sweep candidates — report screen space on a placed window, as the ref-tap path proved on the device. +Only the snapshot channel arrives window-relative, and only that channel converts. + ## Amendment: modal containment in the presentation cut The bridge reads one window and reports every container attached to it. diff --git a/packages/platform-apple/src/snapshot-route.test.ts b/packages/platform-apple/src/snapshot-route.test.ts index ca1a92920c..3872fca846 100644 --- a/packages/platform-apple/src/snapshot-route.test.ts +++ b/packages/platform-apple/src/snapshot-route.test.ts @@ -407,7 +407,7 @@ test.for([ ['foreground-owner-changed', 'while another surface owned the foreground'], [ 'app-window-not-full-screen', - 'while the app window did not fill the screen; XCTest geometry for a windowed app can also be offset from the screen', + "while the app window did not fill the screen; XCTest publishes a windowed app's geometry in screen space (#3899)", ], ] as const)('a %s refusal sends only that capture to the runner', async ([code, consequence]) => { const source = sourceReturning({ stage: 'failed', failure: { kind: 'unsupported', code } }); diff --git a/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts b/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts index 8abe052783..e416406beb 100644 --- a/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts +++ b/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts @@ -6,6 +6,7 @@ import type { Point, Rect } from '@agent-device/kernel/snapshot'; import { isQuarterTurnedWindowFrame, WINDOW_QUARTER_TURN_TOLERANCE, + windowContentTranslation, } from './window-coordinate-space.ts'; // ADR 0011 Layer 2 golden parity table (#2612): the SAME JSON is asserted against the Swift twin @@ -38,10 +39,18 @@ type RotationCase = { oriented: FixtureFrame; }; +type WindowContentCase = { + name: string; + reportedBox: FixtureFrame; + viewportBox: FixtureFrame; + offset: { width: number; height: number } | null; +}; + type Fixture = { constants: { quarterTurnTolerance: number }; quarterTurnCases: QuarterTurnCase[]; rotationCases: RotationCase[]; + windowContentCases: WindowContentCase[]; }; const TABLE_PATH = path.resolve( @@ -82,8 +91,11 @@ test('the quarter-turn window rule agrees with every golden parity table case', const table = loadTable(); assert.ok(table.quarterTurnCases.length > 0, 'parity table must not be empty'); assert.ok(table.rotationCases.length > 0, 'parity table must not be empty'); + assert.ok(table.windowContentCases.length > 0, 'parity table must not be empty'); assertUniqueNames( - [...table.quarterTurnCases, ...table.rotationCases].map((fixture) => fixture.name), + [...table.quarterTurnCases, ...table.rotationCases, ...table.windowContentCases].map( + (fixture) => fixture.name, + ), 'parity table', ); for (const fixture of table.quarterTurnCases) { @@ -95,6 +107,22 @@ test('the quarter-turn window rule agrees with every golden parity table case', } }); +test('the window-content translation rule agrees with every golden parity table case', () => { + for (const fixture of loadTable().windowContentCases) { + const translation = windowContentTranslation( + toRect(fixture.reportedBox), + toRect(fixture.viewportBox), + ); + assert.deepEqual( + translation, + fixture.offset === null + ? null + : { offsetX: fixture.offset.width, offsetY: fixture.offset.height }, + fixture.name, + ); + } +}); + test('the quarter-turn tolerance belongs to the table, not this file', () => { assert.equal(loadTable().constants.quarterTurnTolerance, WINDOW_QUARTER_TURN_TOLERANCE); }); @@ -104,8 +132,12 @@ const LANDSCAPE_RIGHT = 3; const LANDSCAPE_LEFT = 4; /** - * The dispatch direction — an app-space point in the native space a synthesized event is performed in - * — as the runner's own `CoordinateSpaceRotation.native(point:in:interfaceOrientation:)` computes it. + * The dispatch direction — the point a caller passes, in the native space a synthesized event is + * performed in — as the runner's own `CoordinateSpaceRotation.native(point:in:interfaceOrientation:)` + * computes it. + * + * The turn moves a point's axes and never its origin (#3899): the digitizer's origin is the display's + * own, which is where a placed window's screen-space points are already measured from. * * The capture's inverse is NOT implemented here: it lives in Swift alone, and a TypeScript copy of it * would prove nothing about the code that runs. What this proves is that the table cannot be edited @@ -115,17 +147,15 @@ const LANDSCAPE_LEFT = 4; * forward map it ships with. */ function nativePoint(point: Point, app: Rect, interfaceOrientation: number): Point { - const localX = point.x - app.x; - const localY = point.y - app.y; switch (interfaceOrientation) { case LANDSCAPE_RIGHT: - return { x: app.height - localY, y: localX }; + return { x: app.height - point.y, y: point.x }; case LANDSCAPE_LEFT: - return { x: localY, y: app.width - localX }; + return { x: point.y, y: app.width - point.x }; case PORTRAIT_UPSIDE_DOWN: - return { x: app.width - localX, y: app.height - localY }; + return { x: app.width - point.x, y: app.height - point.y }; default: - return { x: localX, y: localY }; + return { x: point.x, y: point.y }; } } diff --git a/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts b/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts index 13c58fa59e..b08c20964f 100644 --- a/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts +++ b/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts @@ -15,17 +15,28 @@ import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; * two spaces coincide and nothing is turned. Rules that read the reported numbers refused app content * the keyboard was nowhere near and let a tap through into a key (#2612). * - * The box is the only evidence, so the rule reads side lengths and nothing else. Origins are not - * compared: a quarter turn says which axis a box runs along, not where it sits, and a window may - * legitimately be placed elsewhere. A square app frame cannot be told from its own quarter turn, so it - * is left alone rather than guessed at — rotating geometry that did not need it is the worse failure. + * A window that does not fill the screen — iPad Windowed Apps or Stage Manager — declares the other + * space. Measured on iPad Pro 11-inch (M5, iOS 27.0) with a Calendar window whose screen box is + * `(224,207,386,682)`, the window itself reports that screen box while the subtree under it arrives + * anchored at its own top-left: the application root reads `(0,0,386,682)` and its `Add` button + * `(335,14,37,36)` — the button's screen position `(559,221)` minus the window origin. Published as + * reported, that tree mixes two spaces, clips its own content, and moves every tap by the window + * origin (#3899). `windowContentTranslation` names the one vector that puts the subtree back. * - * The same rule is enforced in Swift for the capture that produces these trees, and one table proves - * they are one rule: `contracts/fixtures/window-coordinate-space.json` is replayed here and against - * `SnapshotGeometrySpace` in + * A rule that reads a box reads what the box can say. The quarter turn is a statement about which + * axis a box runs along, so it compares side lengths and nothing else. A content translation is a + * statement about where a subtree starts, so it reads all four numbers of the carrier's box against + * the viewport box the platform reported: the carrier must be the app's own window — a `(0,0)`-anchored + * copy of the viewport's side lengths — and the viewport must start somewhere else on the screen. + * A full-screen window, an iPhone, and each foldable panel report origins at zero on both sides, so + * the rule reads no translation there and every published rect stays as the platform reported it. + * + * The same rules are enforced in Swift for the capture that produces these trees, and one table + * proves they are one rule: `contracts/fixtures/window-coordinate-space.json` is replayed here and + * against `SnapshotGeometrySpace` and `SnapshotContentTranslation` in * `apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift`. The table also * pins the way back (`CoordinateSpaceRotation.oriented(rect:in:interfaceOrientation:)`, Swift-only - * today); change either rule only through that table. + * today); change any rule only through that table. * * Why this is a detector and not a repair: the bridge guest cannot read the app's interface * orientation. Measured in the #2659 spike (verdict on the issue, write-up in the diff of #2667): @@ -59,3 +70,45 @@ export function isQuarterTurnedWindowFrame(windowFrame: Rect, appFrame: Rect): b Math.abs(windowFrame.height - appFrame.width) <= WINDOW_QUARTER_TURN_TOLERANCE ); } + +/** The vector that carries a window-relative subtree into the screen space the viewport names. */ +export type WindowContentTranslation = { + offsetX: number; + offsetY: number; +}; + +/** + * The vector from a carrier's own box to the screen space it is placed at, or null when the carrier's + * box declares no translation. The carrier's subtree was reported in the carrier's content space, so + * adding the vector to each reported rect publishes the subtree in the viewport's screen space (#3899). + * + * Null on anything that is not the app's own placed window: a box that is not a `(0,0)`-anchored copy + * of the viewport's side lengths names no space, and a viewport that already starts at the screen + * origin needs no move. Replayed against `windowContentCases` in + * `contracts/fixtures/window-coordinate-space.json`, in the Swift twin and here. + */ +export function windowContentTranslation( + reportedBox: Rect, + viewportBox: Rect, +): WindowContentTranslation | null { + if (!isPositiveFiniteRect(reportedBox) || !isPositiveFiniteRect(viewportBox)) return null; + if ( + Math.abs(reportedBox.x) > WINDOW_QUARTER_TURN_TOLERANCE || + Math.abs(reportedBox.y) > WINDOW_QUARTER_TURN_TOLERANCE + ) { + return null; + } + if ( + Math.abs(viewportBox.x) <= WINDOW_QUARTER_TURN_TOLERANCE && + Math.abs(viewportBox.y) <= WINDOW_QUARTER_TURN_TOLERANCE + ) { + return null; + } + if ( + Math.abs(reportedBox.width - viewportBox.width) > WINDOW_QUARTER_TURN_TOLERANCE || + Math.abs(reportedBox.height - viewportBox.height) > WINDOW_QUARTER_TURN_TOLERANCE + ) { + return null; + } + return { offsetX: viewportBox.x, offsetY: viewportBox.y }; +} From 8b0427f76cb8c9cd24309b32e61affbebcbdc25a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 22 Oct 2026 21:55:21 +0200 Subject: [PATCH 02/15] revert: keep the snapshot-route warning owned by #3914 The refusal wording belongs to the snapshot-route failure policy, not to this fix. Refs #3899 --- packages/platform-apple/src/snapshot-route.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/platform-apple/src/snapshot-route.test.ts b/packages/platform-apple/src/snapshot-route.test.ts index 3872fca846..ca1a92920c 100644 --- a/packages/platform-apple/src/snapshot-route.test.ts +++ b/packages/platform-apple/src/snapshot-route.test.ts @@ -407,7 +407,7 @@ test.for([ ['foreground-owner-changed', 'while another surface owned the foreground'], [ 'app-window-not-full-screen', - "while the app window did not fill the screen; XCTest publishes a windowed app's geometry in screen space (#3899)", + 'while the app window did not fill the screen; XCTest geometry for a windowed app can also be offset from the screen', ], ] as const)('a %s refusal sends only that capture to the runner', async ([code, consequence]) => { const source = sourceReturning({ stage: 'failed', failure: { kind: 'unsupported', code } }); From 6bb1ff8b8faaa722cca19b362ef1802cd6d18f1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 22 Oct 2026 22:17:06 +0200 Subject: [PATCH 03/15] fix(ios): read a placed window's snapshot frames at their screen place (#3899) XCUIElement.frame reports a placed window's content in screen space, but XCUIElementSnapshot.frame reports it in the window's own content space, so the text-input probe saw two spaces on an iPad Windowed Apps window and refused a fill whose point was on the screen. The probe, the focus tap, and the typing reader now read the same SnapshotContentTranslation vector the capture reads, from the two live boxes that name it. Refs #3899 --- .../RunnerTests+TextEntry.swift | 23 +++++++++++---- .../RunnerTests+TextEntryFocus.swift | 5 +++- .../RunnerTests+TextInputProbe.swift | 28 ++++++++++++++++++- .../RunnerTests+TextTyping.swift | 14 ++++++++-- .../adr/0004-ios-snapshot-backend-strategy.md | 5 ++-- 5 files changed, 64 insertions(+), 11 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index 9f8abe79e5..508c711af6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -182,11 +182,18 @@ extension RunnerTests { /// The target moved to the input a resolution read: its element, a refresh point at its center, /// and its identity while the target is unbound. - func withResolvedInput(_ input: ResolvedTextEntryInput?) -> TextEntryTarget { + /// + /// `translation` is the live snapshot-frame vector for the reading session (#3899): the snapshot + /// frame of a placed window's input names the window's content space, while a refresh point is + /// compared against screen-space probes, so the center enters the target translated. + func withResolvedInput( + _ input: ResolvedTextEntryInput?, + translation: CGSize? = nil + ) -> TextEntryTarget { guard let input else { return self } - let frame = input.snapshot?.frame ?? .zero + let frame = snapshotFrameOnScreen(input.snapshot?.frame ?? .zero, translation: translation) return TextEntryTarget( element: input.element, refreshPoint: frame.isEmpty ? refreshPoint : CGPoint(x: frame.midX, y: frame.midY), @@ -259,9 +266,15 @@ extension RunnerTests { private(set) var target: TextEntryTarget private var lastRead: ResolvedTextEntryInput? private let resolve: (TextEntryTarget) -> ResolvedTextEntryInput? + private let translation: CGSize? - init(target: TextEntryTarget, resolve: @escaping (TextEntryTarget) -> ResolvedTextEntryInput?) { - self.target = target.withResolvedInput(target.postFocusRead) + init( + target: TextEntryTarget, + translation: CGSize? = nil, + resolve: @escaping (TextEntryTarget) -> ResolvedTextEntryInput? + ) { + self.translation = translation + self.target = target.withResolvedInput(target.postFocusRead, translation: translation) lastRead = target.postFocusRead self.resolve = resolve } @@ -271,7 +284,7 @@ extension RunnerTests { return lastRead } let read = resolve(target) - target = target.withResolvedInput(read) + target = target.withResolvedInput(read, translation: translation) lastRead = read return read } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift index 34f289cccb..0b009bddaa 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift @@ -134,7 +134,10 @@ extension RunnerTests { let input = textInputAt(app: app, x: x, y: y) let target = input?.element let requestedPoint = CGPoint(x: x, y: y) - if let frame = input?.attributes.frame, !frame.isEmpty { + // The input's frame arrives from a snapshot, which reports a placed window's content in the + // window's own space; the tap below is a screen point (#3899). + if let read = input, !read.attributes.frame.isEmpty { + let frame = snapshotFrameOnScreen(read.attributes.frame, translation: liveContentTranslation(app: app)) _ = tapAt(app: app, x: frame.midX, y: frame.midY) } else { _ = tapAt(app: app, x: x, y: y) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift index 642e9db8f6..4d348c4355 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift @@ -1,4 +1,5 @@ import XCTest +import AgentDeviceSnapshotPresentation final class TextInputProbeIssues { let thread = Thread.current @@ -153,6 +154,7 @@ extension RunnerTests { shouldStop: () -> Bool ) -> [ResolvedTextEntryInput] { var inputs: [ResolvedTextEntryInput] = [] + let translation = liveContentTranslation(app: app) for query in [app.textFields, app.secureTextFields, app.searchFields, app.textViews] { if shouldStop() { break } let candidates = query.allElementsBoundByIndex @@ -168,7 +170,8 @@ extension RunnerTests { if shouldStop() { break } guard let candidate = ResolvedTextEntryInput(readingListed: element) else { continue } let attributes = candidate.attributes - if isCoordinateTextInputCandidate(enabled: attributes.isEnabled, frame: attributes.frame, point: point) { + let frame = snapshotFrameOnScreen(attributes.frame, translation: translation) + if isCoordinateTextInputCandidate(enabled: attributes.isEnabled, frame: frame, point: point) { inputs.append(candidate) } } @@ -176,4 +179,27 @@ extension RunnerTests { guard !shouldStop() else { return [] } return inputs.sorted { smallestElementFirst($0.attributes, $1.attributes) } } + + /// The vector that carries this session's snapshot-read app-content frames to their place on the + /// screen, or nil when the app fills the screen (#3899). + /// + /// The live `XCUIElement.frame` property already reports a placed window's content in screen space, + /// but `XCUIElementSnapshot.frame` reports it in the window's own content space, so the same probe + /// sees two spaces on an iPad Windowed Apps window. The signature and the vector are the capture's + /// rule (`SnapshotContentTranslation`, ADR 0004) read from the two live boxes that name it: the + /// application's `frame` is a `(0,0)`-anchored copy of the box the resolved window reports at its + /// place on the screen. Any other pair of boxes declares no vector, so full-screen, iPhone, and + /// each foldable panel keep their frames exactly as read. + func liveContentTranslation(app: XCUIApplication) -> CGSize? { + safely("LIVE_CONTENT_TRANSLATION", nil) { + guard let resolved = resolveRunnerWindowElement(app: app) else { return nil } + return SnapshotContentTranslation.offset(reportedBox: app.frame, viewportBox: resolved.frame) + } + } +} + +/// `frame` as the screen names it, given `liveContentTranslation`'s vector. +func snapshotFrameOnScreen(_ frame: CGRect, translation: CGSize?) -> CGRect { + guard let translation else { return frame } + return frame.offsetBy(dx: translation.width, dy: translation.height) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift index c0ad58ca1c..760e215661 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift @@ -16,7 +16,13 @@ extension RunnerTests { ) -> TextEntryResult { let totalStartedAt = Date() let resolve = { (target: TextEntryTarget) in self.resolveTextEntryInput(app: app, target: target) } - var reader = TextEntryReader(target: target, resolve: resolve) + // Reads inside the typing loop answer from snapshots, which report a placed window's content in + // the window's own space; the reader's refresh points are screen points (#3899). + var reader = TextEntryReader( + target: target, + translation: liveContentTranslation(app: app), + resolve: resolve + ) guard !text.isEmpty else { // Replacing a field WITH the empty string is the clear-field primitive (#2063): the typing // is vacuous but the clear is not. Returning here unconditionally left the old value in @@ -74,7 +80,11 @@ extension RunnerTests { case .completed(let result): return result case .fallback(let fallbackTarget): - reader = TextEntryReader(target: fallbackTarget, resolve: resolve) + reader = TextEntryReader( + target: fallbackTarget, + translation: liveContentTranslation(app: app), + resolve: resolve + ) } } let shouldUseSynthesizedFirstResponderType = Self.shouldUseSynthesizedFirstResponderType( diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index d56a194091..4e8b143f13 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -660,8 +660,9 @@ Translation is the one rule that reads an origin, so it is the only place the fi Capture is half of the promise; dispatch is the other half. The measured dispatch spaces for a placed window are both screen-anchored: the synthesized path lands a point at its screen position, and XCTest's own `XCUICoordinate` on the app anchors at the screen's origin too. `interactionCoordinate` therefore passes the caller's screen point through, and `resolveRunnerWindow` keeps naming the window a foldable's tap resolves its display on (ADR 0025). -The live `XCUIElement.frame` reads that the tap path already compares — the selector match, the input probe, the tree's sweep candidates — report screen space on a placed window, as the ref-tap path proved on the device. -Only the snapshot channel arrives window-relative, and only that channel converts. +The live `XCUIElement.frame` property reports screen space on a placed window, as the selector-tap path proved on the device: it measured the `Add` button at (641,321) for a window at (324,307). +The one live read that does not is `XCUIElementSnapshot.frame`, which reports the window's content space like the capture channel does. +So the snapshot capture converts in `SnapshotGeometrySpace.normalized`, and the text-input probe converts its snapshot reads through the same `SnapshotContentTranslation` rule read from the two live boxes (`app.frame` against the resolved window's frame); every other live read needs no conversion. ## Amendment: modal containment in the presentation cut From 7fa359023e1132a19fcb69fe83e5e0df1be8edae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 22 Oct 2026 22:51:08 +0200 Subject: [PATCH 04/15] fix(ios): keep caret repositioning and keyboard evidence decision stable on a placed window [ #3899 ] --- .../RunnerTests+TextEntry.swift | 5 ++++- .../RunnerTests+TextTyping.swift | 16 +++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index 508c711af6..d32579e051 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -537,9 +537,12 @@ extension RunnerTests { // zero-size one (how a departed handle reads), there is no point to dispatch — an edge-tap // from a frame the field moved after records a failure or navigates away, the shape // `clearTextInput`'s comment refuses — so degrade to the caller's point-free delete burst. - guard let frame = input.snapshot?.frame, !frame.isEmpty else { + // The snapshot frame names the window's own content space on a placed window, so it enters + // `tapAt` translated (#3899). + guard let read = input.snapshot?.frame, !read.isEmpty else { return } + let frame = snapshotFrameOnScreen(read, translation: liveContentTranslation(app: app)) _ = tapAt( app: app, x: frame.minX + max(2, frame.width - 4), diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift index 760e215661..2328e0c966 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift @@ -87,11 +87,20 @@ extension RunnerTests { ) } } + // The keyboard evidence is decided once, at the command's start. Measured on iPad Pro 11-inch + // (M5, iOS 27.0) with Windowed Apps (#3899): `app.keyboards` answered `false` 50 ms after + // answering `true` for the same on-screen keyboard, while the app's own AX reads kept failing + // with `kAXErrorServerNotFound` under a synthesized post burst. A per-post re-query therefore + // refused a receiver mid-burst and reported `TEXT_INPUT_NOT_FOCUSED` for a focused field, one + // character per call. An app that loses the keyboard mid-command keeps the start-of-command + // answer: the extra posts land on the digitizer and do nothing, the outcome a tap into empty + // space already has. + let softwareKeyboardVisibleAtStart = isKeyboardVisible(app: app) let shouldUseSynthesizedFirstResponderType = Self.shouldUseSynthesizedFirstResponderType( repairMode: repairMode, text: text, fromTapWitness: reader.target.fromTapWitness, - softwareKeyboardVisible: isKeyboardVisible(app: app) + softwareKeyboardVisible: softwareKeyboardVisibleAtStart ) let initialResolveStartedAt = Date() let initialTarget = reader.input() @@ -110,7 +119,7 @@ extension RunnerTests { // Dispatching text through XCTest without a resolved target or evidence of a focused // responder records a test failure. That tears down the long-lived runner and turns a // single invalid request into a restart cascade, so fail before entering that channel. - guard initialTarget != nil || (reader.target.prefersFocusedElement && isKeyboardVisible(app: app)) else { + guard initialTarget != nil || (reader.target.prefersFocusedElement && softwareKeyboardVisibleAtStart) else { logTextEntryPhase( commandId: commandId, phase: "total", @@ -205,7 +214,8 @@ extension RunnerTests { textEntryRoute = "xctest-element" reader.dispatch { currentTarget.element.typeText(value) } return nil - } else if reader.target.boundIdentity == nil && reader.target.prefersFocusedElement && isKeyboardVisible(app: app) { + } else if reader.target.boundIdentity == nil && reader.target.prefersFocusedElement + && softwareKeyboardVisibleAtStart { #if os(iOS) // Two ways this post leaves the synthesized channel, and both hand the text to // application-wide typing: the command's own budget refused it, or XCTest's private synthesis From a8dbd01e273e87f6c9e204ba05ce78b3d9bb7476 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 23 Oct 2026 00:08:37 +0200 Subject: [PATCH 05/15] fix(ios): translate only snapshot frames and pin the unmeasured landscape case [ #3899 ] --- .../RunnerTests+TextEntry.swift | 4 +- .../RunnerTests+TextEntryFocus.swift | 56 +++++++++++++++---- .../RunnerTests+TextInputProbe.swift | 10 +++- .../CoordinateSpaceTests.swift | 4 +- .../adr/0004-ios-snapshot-backend-strategy.md | 5 ++ 5 files changed, 62 insertions(+), 17 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index d32579e051..b928947aeb 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -188,7 +188,7 @@ extension RunnerTests { /// compared against screen-space probes, so the center enters the target translated. func withResolvedInput( _ input: ResolvedTextEntryInput?, - translation: CGSize? = nil + translation: CGSize? ) -> TextEntryTarget { guard let input else { return self @@ -270,7 +270,7 @@ extension RunnerTests { init( target: TextEntryTarget, - translation: CGSize? = nil, + translation: CGSize?, resolve: @escaping (TextEntryTarget) -> ResolvedTextEntryInput? ) { self.translation = translation diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift index 0b009bddaa..1ac9056733 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift @@ -135,9 +135,10 @@ extension RunnerTests { let target = input?.element let requestedPoint = CGPoint(x: x, y: y) // The input's frame arrives from a snapshot, which reports a placed window's content in the - // window's own space; the tap below is a screen point (#3899). - if let read = input, !read.attributes.frame.isEmpty { - let frame = snapshotFrameOnScreen(read.attributes.frame, translation: liveContentTranslation(app: app)) + // window's own space; the tap below is a screen point (#3899). An input read live (no + // snapshot) already names screen space, so only the snapshot frame is translated here. + if let snapshotFrame = input?.snapshot?.frame, !snapshotFrame.isEmpty { + let frame = snapshotFrameOnScreen(snapshotFrame, translation: liveContentTranslation(app: app)) _ = tapAt(app: app, x: frame.midX, y: frame.midY) } else { _ = tapAt(app: app, x: x, y: y) @@ -146,7 +147,11 @@ extension RunnerTests { // belong to a previous field. With a concrete target we type through XCUIElement.typeText, // so after tapping it the iOS readiness timeout cannot prove anything stronger. if keyboardVisibleBeforeTap, let target { - return focusedTextEntryTarget(target, requestedPoint: requestedPoint) + return focusedTextEntryTarget( + target, + requestedPoint: requestedPoint, + translation: liveContentTranslation(app: app) + ) } let stabilized = stabilizeTextInputBeforeTyping( app: app, @@ -160,7 +165,12 @@ extension RunnerTests { stabilized.polls ) if case .inputReportsKeyboardFocus(let read) = stabilized.evidence { - return focusedTextEntryTarget(read.element, read: read, requestedPoint: requestedPoint) + return focusedTextEntryTarget( + read.element, + read: read, + requestedPoint: requestedPoint, + translation: liveContentTranslation(app: app) + ) } let readyTarget = TextEntryTarget( element: stabilized.element ?? target, @@ -170,26 +180,44 @@ extension RunnerTests { let element = stabilized.focusConfirmed ? readyTarget.element : (waitForTextEntryReadiness(app: app, target: readyTarget) ?? readyTarget.element) - return focusedTextEntryTarget(element, requestedPoint: requestedPoint) + return focusedTextEntryTarget( + element, + requestedPoint: requestedPoint, + translation: liveContentTranslation(app: app) + ) } /// The target the focus step hands to text entry, with the read its refresh point came from. - private func focusedTextEntryTarget(_ element: XCUIElement?, requestedPoint: CGPoint) -> TextEntryTarget { + private func focusedTextEntryTarget( + _ element: XCUIElement?, + requestedPoint: CGPoint, + translation: CGSize? + ) -> TextEntryTarget { guard let element else { return TextEntryTarget(element: nil, refreshPoint: requestedPoint, prefersFocusedElement: false) } - return focusedTextEntryTarget(element, read: ResolvedTextEntryInput(reading: element), requestedPoint: requestedPoint) + return focusedTextEntryTarget( + element, + read: ResolvedTextEntryInput(reading: element), + requestedPoint: requestedPoint, + translation: translation + ) } /// `read` is `element` read after the focus tap, or nil when the read found no element. private func focusedTextEntryTarget( _ element: XCUIElement, read: ResolvedTextEntryInput?, - requestedPoint: CGPoint + requestedPoint: CGPoint, + translation: CGSize? ) -> TextEntryTarget { return TextEntryTarget( element: element, - refreshPoint: textEntryRefreshPoint(frame: read?.snapshot?.frame, fallbackTo: requestedPoint), + refreshPoint: textEntryRefreshPoint( + frame: read?.snapshot?.frame, + fallbackTo: requestedPoint, + translation: translation + ), prefersFocusedElement: false, postFocusRead: read?.snapshot == nil ? nil : read ) @@ -206,7 +234,8 @@ extension RunnerTests { /// typing into that other field. private func textEntryRefreshPoint( frame: CGRect?, - fallbackTo requestedPoint: CGPoint + fallbackTo requestedPoint: CGPoint, + translation: CGSize? ) -> CGPoint? { guard let frame else { return nil @@ -214,6 +243,9 @@ extension RunnerTests { guard !frame.isEmpty else { return requestedPoint } - return CGPoint(x: frame.midX, y: frame.midY) + // The read's frame names the window's content space on a placed window; a refresh point is + // compared against screen-space probes, so the center enters the target translated (#3899). + let onScreen = snapshotFrameOnScreen(frame, translation: translation) + return CGPoint(x: onScreen.midX, y: onScreen.midY) } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift index 4d348c4355..9a449a51c9 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift @@ -170,7 +170,15 @@ extension RunnerTests { if shouldStop() { break } guard let candidate = ResolvedTextEntryInput(readingListed: element) else { continue } let attributes = candidate.attributes - let frame = snapshotFrameOnScreen(attributes.frame, translation: translation) + // Only the snapshot names the window's content space; a candidate read live (no snapshot) + // already reports screen space, and translating it again would move the candidate off the + // point it is being matched against (#3899). + let frame: CGRect + if let snapshot = candidate.snapshot { + frame = snapshotFrameOnScreen(snapshot.frame, translation: translation) + } else { + frame = attributes.frame + } if isCoordinateTextInputCandidate(enabled: attributes.isEnabled, frame: frame, point: point) { inputs.append(candidate) } diff --git a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift index 82a76fb74a..1e97adfc83 100644 --- a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift +++ b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift @@ -78,7 +78,7 @@ final class CoordinateSpaceTests: XCTestCase { ), CGPoint(x: 288, y: 1030), "a frame offset from the screen never moves the turned point (#3899)" - ); + ) } func testNativeSynthesizedVectorRotatesByInterfaceOrientation() { @@ -581,7 +581,7 @@ final class CoordinateSpaceTests: XCTestCase { ) ).map(\.rect), acquired.map(\.rect) - ); + ) // A windowed app, measured shape: the app's own window declares the translation and carries the // subtree; the keyboard window and the SpringBoard overlay report the screen's own box, so their diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index 4e8b143f13..ddfb0ea37d 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -663,6 +663,11 @@ The measured dispatch spaces for a placed window are both screen-anchored: the s The live `XCUIElement.frame` property reports screen space on a placed window, as the selector-tap path proved on the device: it measured the `Add` button at (641,321) for a window at (324,307). The one live read that does not is `XCUIElementSnapshot.frame`, which reports the window's content space like the capture channel does. So the snapshot capture converts in `SnapshotGeometrySpace.normalized`, and the text-input probe converts its snapshot reads through the same `SnapshotContentTranslation` rule read from the two live boxes (`app.frame` against the resolved window's frame); every other live read needs no conversion. +Where a probe read has no snapshot and falls back to the live element, its frame is already screen space and is not translated. +`app.frame` unions transformed subtrees, so content pushed off a placed window (a closed drawer at negative x) can stop the two live boxes matching and read no vector; the capture still converts from the snapshot root box, so a probe may then read window-local frames where the capture published screen space. A mid-command window move has the same effect on a vector read at command start. +Both are reads of a moving target, not a second space: a re-opened command reads the current vector. +The host bridge keeps refusing a windowed capture (`app-window-not-full-screen`) rather than converting one, so the TypeScript twin of the rule runs today only in the parity replay. +Portrait is the only orientation measured for a placed window; the quarter-turn table's windowed row pins the arithmetic (a frame origin never travels through the turn), not a device measurement, and landscape under Windowed Apps stays unverified until measured. ## Amendment: modal containment in the presentation cut From ec7aecbf92fb97635445913d9e64603e4473dc24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 23 Oct 2026 00:33:53 +0200 Subject: [PATCH 06/15] refactor(ios): keep the window-content translation with the capture that runs it [ #3899 ] --- .../CoordinateSpaceTests.swift | 8 ++- .../fixtures/window-coordinate-space.json | 2 +- .../adr/0004-ios-snapshot-backend-strategy.md | 2 +- .../window-coordinate-space.test.ts | 20 ++---- .../window-coordinate-space.ts | 66 ++++++------------- 5 files changed, 30 insertions(+), 68 deletions(-) diff --git a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift index 1e97adfc83..ceb6f35cdb 100644 --- a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift +++ b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift @@ -317,9 +317,11 @@ final class CoordinateSpaceTests: XCTestCase { } /// Golden parity table (#2612, #3899): every case in contracts/fixtures/window-coordinate-space.json - /// must agree with the vitest twin - /// (packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts). Add - /// cases there, never fork the rule. Detection is replayed under both landscape turns, because + /// must agree here, and the quarter-turn rows with the vitest twin + /// (packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts). The + /// window-content rows replay here alone: the host channel's guest refuses a windowed tree + /// before its decoder can read one, so this capture is the rule's only production owner. Add + /// cases to the table, never fork the rule. Detection is replayed under both landscape turns, because /// which window declares the native space cannot depend on which way the app is turned; the /// rotation rows carry their own orientation. func testWindowCoordinateSpaceMatchesGoldenParityTable() throws { diff --git a/contracts/fixtures/window-coordinate-space.json b/contracts/fixtures/window-coordinate-space.json index d42ef974ce..1211632d64 100644 --- a/contracts/fixtures/window-coordinate-space.json +++ b/contracts/fixtures/window-coordinate-space.json @@ -1,5 +1,5 @@ { - "description": "Golden table for the coordinate space a captured window's own box declares (#2612), the twin of tap-keyboard-occlusion-policy.json on the geometry that guard reads. iOS hosts some system surfaces in the device's native (portrait-up) space even while the app is rotated, so their whole subtree arrives quarter-turned: measured on iPhone 17 Pro (iOS 26.2) with the system keyboard up over a landscape app frame (0,0,874,402), UIRemoteKeyboardWindow reports its own box as (0,0,402,874) — the app's box with the two side lengths swapped — while the app's own UIWindow, UITextEffectsWindow and a dev-menu overlay window all report (0,0,874,402); in portrait the same keyboard window reports (0,0,402,874) against a (0,0,402,874) app frame, i.e. nothing is turned. RULE 1 (detection) is enforced twice, in packages/contracts/src/window-coordinate-space.ts and in SnapshotGeometrySpace inside apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift, and decides from side lengths alone: a window is quarter-turned relative to an app frame when the app frame is not square (abs(app.width - app.height) > quarterTurnTolerance), both frames are finite, positive and non-empty, and the window's side lengths match the app's on the swapped axes (abs(window.width - app.height) <= quarterTurnTolerance && abs(window.height - app.width) <= quarterTurnTolerance). Origins are deliberately never compared: the turn is a statement about which axis a box runs along, and a window may legitimately be placed elsewhere. A square app frame cannot be told from its own quarter turn, so it is left alone rather than guessed at. The tolerance is 1 because measured captures match the swap to the point and this absorbs float representation only. RULE 2 (rotation) is the capture's way back for the runner's own UIInterfaceOrientation raw values — 1 portrait, 2 portraitUpsideDown, 3 landscapeRight, 4 landscapeLeft, and 0 unknown, where portrait and unknown publish what the platform reported rather than inventing a turn — implemented by CoordinateSpaceRotation.oriented(rect:in:interfaceOrientation:) in that same Swift file, whose rect helper turns opposite corners of the reported box and re-orders them so the result keeps a top-left origin. `rotationCases` are native -> oriented rows: the landscape ones measured on the device above, the 180 degree and identity ones derived from that table. Capture turns back with the exact inverse of what dispatch turns forward, so each row also has to come back to its own `native` box under the forward mapping; that is why one measured-looking landscape-left pairing whose native and oriented boxes are identical is absent — the table computes no such row. Infinity has no JSON spelling, so the unusable frame a platform hands back when it could not resolve a box is named {\"infinite\": true} and decodes to CGRect.infinite on the Swift side and to an infinite rect on the TypeScript side. RULE 3 (content translation) is the other space a captured subtree arrives in, and the one rule in this file that reads an origin. An iPad window that does not fill the screen (Windowed Apps, Stage Manager) reports its own screen box while the subtree under it arrives anchored at its own top-left: measured on iPad Pro 11-inch (M5, iOS 27.0) with a Calendar window at screen (224,207,386x682), the application root reports (0,0,386,682) and its Add button (335,14,37,36) for a control drawn at screen (559,221). Published as reported, that tree mixes two spaces, clips its own content, and moves every tap by the window origin (#3899). SnapshotContentTranslation.offset(reportedBox:viewportBox:) reads the carrier box against the viewport box the platform reported, and names the viewport origin as the offset only when the carrier is the app's own window: a (0,0)-anchored box whose side lengths match the viewport's within the same tolerance, while the viewport starts somewhere else on the screen. A carrier that is not the app's own window names no space, so its numbers are left alone: the keyboard window and a SpringBoard overlay under a windowed app report the screen's own box and are already in screen space. A full-screen window, an iPhone, and each foldable panel of ADR 0025 report origins at zero on both sides, so the rule reads no offset there and every published rect stays exactly as it was measured before. `windowContentCases` are reportedBox + viewportBox to offset rows, replayed against that Swift function and against windowContentTranslation in packages/platform-apple/src/snapshot-source/window-coordinate-space.ts; a null offset means the rule declares no translation. Because a translation is the one thing a rotation must not also do, `rotationCases` pin the turn as axes-only: `native` and `oriented` never move a box's origin, and a row whose app frame is offset from the screen comes back at the same place as its own zero-origin twin.", + "description": "Golden table for the coordinate space a captured window's own box declares (#2612), the twin of tap-keyboard-occlusion-policy.json on the geometry that guard reads. iOS hosts some system surfaces in the device's native (portrait-up) space even while the app is rotated, so their whole subtree arrives quarter-turned: measured on iPhone 17 Pro (iOS 26.2) with the system keyboard up over a landscape app frame (0,0,874,402), UIRemoteKeyboardWindow reports its own box as (0,0,402,874) — the app's box with the two side lengths swapped — while the app's own UIWindow, UITextEffectsWindow and a dev-menu overlay window all report (0,0,874,402); in portrait the same keyboard window reports (0,0,402,874) against a (0,0,402,874) app frame, i.e. nothing is turned. RULE 1 (detection) is enforced twice, in packages/contracts/src/window-coordinate-space.ts and in SnapshotGeometrySpace inside apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift, and decides from side lengths alone: a window is quarter-turned relative to an app frame when the app frame is not square (abs(app.width - app.height) > quarterTurnTolerance), both frames are finite, positive and non-empty, and the window's side lengths match the app's on the swapped axes (abs(window.width - app.height) <= quarterTurnTolerance && abs(window.height - app.width) <= quarterTurnTolerance). Origins are deliberately never compared: the turn is a statement about which axis a box runs along, and a window may legitimately be placed elsewhere. A square app frame cannot be told from its own quarter turn, so it is left alone rather than guessed at. The tolerance is 1 because measured captures match the swap to the point and this absorbs float representation only. RULE 2 (rotation) is the capture's way back for the runner's own UIInterfaceOrientation raw values — 1 portrait, 2 portraitUpsideDown, 3 landscapeRight, 4 landscapeLeft, and 0 unknown, where portrait and unknown publish what the platform reported rather than inventing a turn — implemented by CoordinateSpaceRotation.oriented(rect:in:interfaceOrientation:) in that same Swift file, whose rect helper turns opposite corners of the reported box and re-orders them so the result keeps a top-left origin. `rotationCases` are native -> oriented rows: the landscape ones measured on the device above, the 180 degree and identity ones derived from that table. Capture turns back with the exact inverse of what dispatch turns forward, so each row also has to come back to its own `native` box under the forward mapping; that is why one measured-looking landscape-left pairing whose native and oriented boxes are identical is absent — the table computes no such row. Infinity has no JSON spelling, so the unusable frame a platform hands back when it could not resolve a box is named {\"infinite\": true} and decodes to CGRect.infinite on the Swift side and to an infinite rect on the TypeScript side. RULE 3 (content translation) is the other space a captured subtree arrives in, and the one rule in this file that reads an origin. An iPad window that does not fill the screen (Windowed Apps, Stage Manager) reports its own screen box while the subtree under it arrives anchored at its own top-left: measured on iPad Pro 11-inch (M5, iOS 27.0) with a Calendar window at screen (224,207,386x682), the application root reports (0,0,386,682) and its Add button (335,14,37,36) for a control drawn at screen (559,221). Published as reported, that tree mixes two spaces, clips its own content, and moves every tap by the window origin (#3899). SnapshotContentTranslation.offset(reportedBox:viewportBox:) reads the carrier box against the viewport box the platform reported, and names the viewport origin as the offset only when the carrier is the app's own window: a (0,0)-anchored box whose side lengths match the viewport's within the same tolerance, while the viewport starts somewhere else on the screen. A carrier that is not the app's own window names no space, so its numbers are left alone: the keyboard window and a SpringBoard overlay under a windowed app report the screen's own box and are already in screen space. A full-screen window, an iPhone, and each foldable panel of ADR 0025 report origins at zero on both sides, so the rule reads no offset there and every published rect stays exactly as it was measured before. `windowContentCases` are reportedBox + viewportBox to offset rows, replayed against that Swift function alone; a null offset means the rule declares no translation. The host decoder (packages/platform-apple/src/snapshot-source/window-coordinate-space.ts) replays only the quarter-turn rows: its channel's guest refuses a windowed tree (app-window-not-full-screen, #3899) before that decoder can read one, so the translation has exactly one producer-side owner. Because a translation is the one thing a rotation must not also do, `rotationCases` pin the turn as axes-only: `native` and `oriented` never move a box's origin, and a row whose app frame is offset from the screen comes back at the same place as its own zero-origin twin.", "constants": { "quarterTurnTolerance": 1 }, diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index ddfb0ea37d..11f0799d22 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -649,7 +649,7 @@ The carrier's box must be a `(0,0)`-anchored copy of the viewport box the platfo Then the subtree was reported in the carrier's content space, and the viewport's origin is the one vector that carries it to the screen. Any other carrier box names no space, so the rule leaves its numbers alone: the keyboard window and the SpringBoard overlay above report the screen's own box and are already in screen space. A full-screen window, an iPhone, and each foldable panel of ADR 0025 report origins at zero on both sides, so the rule reads no vector there and every rect is unchanged. -`SnapshotContentTranslation.offset(reportedBox:viewportBox:)` is that rule, replayed from `contracts/fixtures/window-coordinate-space.json` in both languages. +`SnapshotContentTranslation.offset(reportedBox:viewportBox:)` is that rule, replayed from `contracts/fixtures/window-coordinate-space.json` in the capture's language alone: the host bridge refuses a windowed tree (`app-window-not-full-screen`) before its decoder can read one, so the translation has one producer-side owner and the host's parity replay covers only the rules that decoder reads. Rotation and translation stay separate rules, and the split is the point. `CoordinateSpaceRotation` turns a box's axes and never moves its origin: the digitizer's origin is the display's own, which is where a placed window's screen-space points are already measured from. diff --git a/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts b/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts index e416406beb..a95d4908c8 100644 --- a/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts +++ b/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts @@ -6,7 +6,6 @@ import type { Point, Rect } from '@agent-device/kernel/snapshot'; import { isQuarterTurnedWindowFrame, WINDOW_QUARTER_TURN_TOLERANCE, - windowContentTranslation, } from './window-coordinate-space.ts'; // ADR 0011 Layer 2 golden parity table (#2612): the SAME JSON is asserted against the Swift twin @@ -107,21 +106,10 @@ test('the quarter-turn window rule agrees with every golden parity table case', } }); -test('the window-content translation rule agrees with every golden parity table case', () => { - for (const fixture of loadTable().windowContentCases) { - const translation = windowContentTranslation( - toRect(fixture.reportedBox), - toRect(fixture.viewportBox), - ); - assert.deepEqual( - translation, - fixture.offset === null - ? null - : { offsetX: fixture.offset.width, offsetY: fixture.offset.height }, - fixture.name, - ); - } -}); +// `windowContentCases` is replayed in the Swift capture alone (CoordinateSpaceTests): this decoder +// never receives a windowed tree — the bridge guest refuses it first (#3899) — so a second replay +// here would pin arithmetic no production reader runs. The table still loads here, and its rows are +// read for the name-uniqueness guard below. test('the quarter-turn tolerance belongs to the table, not this file', () => { assert.equal(loadTable().constants.quarterTurnTolerance, WINDOW_QUARTER_TURN_TOLERANCE); diff --git a/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts b/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts index b08c20964f..4ce6887031 100644 --- a/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts +++ b/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts @@ -21,7 +21,9 @@ import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; * anchored at its own top-left: the application root reads `(0,0,386,682)` and its `Add` button * `(335,14,37,36)` — the button's screen position `(559,221)` minus the window origin. Published as * reported, that tree mixes two spaces, clips its own content, and moves every tap by the window - * origin (#3899). `windowContentTranslation` names the one vector that puts the subtree back. + * origin (#3899). That translation belongs to the capture that performs it — the runner's Swift + * `SnapshotContentTranslation` — not to this decoder, because the guest refuses a windowed tree + * before this file can read one; see the note at the foot of this file. * * A rule that reads a box reads what the box can say. The quarter turn is a statement about which * axis a box runs along, so it compares side lengths and nothing else. A content translation is a @@ -31,12 +33,13 @@ import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; * A full-screen window, an iPhone, and each foldable panel report origins at zero on both sides, so * the rule reads no translation there and every published rect stays as the platform reported it. * - * The same rules are enforced in Swift for the capture that produces these trees, and one table - * proves they are one rule: `contracts/fixtures/window-coordinate-space.json` is replayed here and - * against `SnapshotGeometrySpace` and `SnapshotContentTranslation` in - * `apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift`. The table also - * pins the way back (`CoordinateSpaceRotation.oriented(rect:in:interfaceOrientation:)`, Swift-only - * today); change any rule only through that table. + * RULE 1 is enforced in both languages, because both channels read it. RULE 3 (the content + * translation) and RULE 2 (the rotation back, `CoordinateSpaceRotation.oriented(rect:in:interfaceOrientation:)`) + * live with the runner's capture alone. One table proves the shared rules are one rule: + * `contracts/fixtures/window-coordinate-space.json` is replayed here and against + * `SnapshotGeometrySpace`, `SnapshotContentTranslation`, and `CoordinateSpaceRotation` in + * `apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift`; + * change any rule only through that table. * * Why this is a detector and not a repair: the bridge guest cannot read the app's interface * orientation. Measured in the #2659 spike (verdict on the issue, write-up in the diff of #2667): @@ -71,44 +74,13 @@ export function isQuarterTurnedWindowFrame(windowFrame: Rect, appFrame: Rect): b ); } -/** The vector that carries a window-relative subtree into the screen space the viewport names. */ -export type WindowContentTranslation = { - offsetX: number; - offsetY: number; -}; - -/** - * The vector from a carrier's own box to the screen space it is placed at, or null when the carrier's - * box declares no translation. The carrier's subtree was reported in the carrier's content space, so - * adding the vector to each reported rect publishes the subtree in the viewport's screen space (#3899). - * - * Null on anything that is not the app's own placed window: a box that is not a `(0,0)`-anchored copy - * of the viewport's side lengths names no space, and a viewport that already starts at the screen - * origin needs no move. Replayed against `windowContentCases` in - * `contracts/fixtures/window-coordinate-space.json`, in the Swift twin and here. +/* + * A window that does not fill the screen (#3899) is translated by the capture that owns it, and this + * file is not that capture. The bridge guest refuses a windowed tree (`app-window-not-full-screen`) + * before this decoder ever sees one, so no host tree arrives in the window's content space and there + * is nothing here to translate. The rule lives with the producer that converts — + * `SnapshotContentTranslation` beside the Swift capture — and `windowContentCases` in + * `contracts/fixtures/window-coordinate-space.json` is replayed there alone. The quarter-turn + * detection above stays in both languages because this decoder does read it: it is what refuses a + * turned tree the reader cannot resolve. */ -export function windowContentTranslation( - reportedBox: Rect, - viewportBox: Rect, -): WindowContentTranslation | null { - if (!isPositiveFiniteRect(reportedBox) || !isPositiveFiniteRect(viewportBox)) return null; - if ( - Math.abs(reportedBox.x) > WINDOW_QUARTER_TURN_TOLERANCE || - Math.abs(reportedBox.y) > WINDOW_QUARTER_TURN_TOLERANCE - ) { - return null; - } - if ( - Math.abs(viewportBox.x) <= WINDOW_QUARTER_TURN_TOLERANCE && - Math.abs(viewportBox.y) <= WINDOW_QUARTER_TURN_TOLERANCE - ) { - return null; - } - if ( - Math.abs(reportedBox.width - viewportBox.width) > WINDOW_QUARTER_TURN_TOLERANCE || - Math.abs(reportedBox.height - viewportBox.height) > WINDOW_QUARTER_TURN_TOLERANCE - ) { - return null; - } - return { offsetX: viewportBox.x, offsetY: viewportBox.y }; -} From 009b65c8d0aa7abbab8fdf0baa4b8a04f9809c49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 24 Oct 2026 10:31:09 +0200 Subject: [PATCH 07/15] refactor(ios): make ResolvedTextEntryInput own the probe-side screen frame (#3899) The readiness retap re-read the input as a bare XCUIElementSnapshot and tapped its raw center through tapAt, which takes screen points, so on a windowed iPad the retap landed outside the window. The live window lookup also ran whole-app queries four to six times per fill on every iOS device. Every snapshot frame that becomes a dispatched point now passes through the one content translation, held on ResolvedTextEntryInput.frameOnScreen. The vector is read once per command and stored on TextEntryTarget.contentTranslation; the probe matching, focus tap, refresh point, caret tap and readiness retap all read that one property. The threaded translation: parameters and the per-read lookups are gone, and the once-per-command keyboard read in the typing command is reverted to the base behavior with the retap fixed. --- .../RunnerTests+CommandExecution.swift | 9 +- .../RunnerTests+Interaction.swift | 23 +++- .../RunnerTests+TextEntry.swift | 123 +++++++++++++----- .../RunnerTests+TextEntryFocus.swift | 82 +++++++----- .../RunnerTests+TextEntryReadiness.swift | 44 ++++++- .../RunnerTests+TextInputProbe.swift | 103 ++++++++++----- .../RunnerTests+TextTyping.swift | 30 +---- .../RunnerTests+TypeExecution.swift | 6 +- ...RunnerTests+CoordinateTextEntryTests.swift | 2 +- .../RunnerTests+TextEntryLatencyTests.swift | 9 +- .../RunnerTests+TextEntryReadinessTests.swift | 57 ++++++++ .../RunnerTests+TextInputProbeTests.swift | 8 +- .../SnapshotCoordinateSpace.swift | 5 + .../CoordinateSpaceTests.swift | 12 +- .../fixtures/window-coordinate-space.json | 2 +- .../adr/0004-ios-snapshot-backend-strategy.md | 4 +- .../platform-apple/src/snapshot-route.test.ts | 2 +- packages/platform-apple/src/snapshot-route.ts | 2 +- .../window-coordinate-space.ts | 10 +- 19 files changed, 364 insertions(+), 169 deletions(-) create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryReadinessTests.swift diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index c8c622cfe6..ea6a445abf 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -155,7 +155,14 @@ extension RunnerTests { ) let textInput: XCUIElement? if !xCTestTextInputProbeSkipped { - textInput = coordinateTapTextInputAt(app: activeApp, x: x, y: y) + // The lookup compares the point, which is a screen point, with candidate frames; on a + // placed window the snapshot frames need the content vector (#3899). + textInput = coordinateTapTextInputAt( + app: activeApp, + x: x, + y: y, + contentTranslation: liveContentTranslation(app: activeApp) + ) } else { textInput = nil // Clear before dispatch so a throwing gesture cannot retain the preceding field's witness. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index ae1407d1d0..4b134a059d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -352,7 +352,13 @@ extension RunnerTests { func readTextAt(app: XCUIApplication, x: Double, y: Double) -> String? { let point = CGPoint(x: x, y: y) - let textInputCandidates = textInputCandidatesAt(app: app, point: point).map(\.element) + // The point is a screen point; candidate snapshot frames of a placed window need the content + // vector to be compared with it (#3899). + let textInputCandidates = textInputCandidatesAt( + app: app, + point: point, + contentTranslation: liveContentTranslation(app: app) + ).map(\.element) for element in textInputCandidates where prefersExpandedTextRead(element) { if let text = readableText(for: element) { return text @@ -617,12 +623,15 @@ extension RunnerTests { #if !os(tvOS) private func interactionCoordinate(app: XCUIApplication, x: Double, y: Double) -> XCUICoordinate { #if os(iOS) - // Coordinate taps, double taps, long presses, and drags anchor at the app origin, not the - // resolved window: on iOS the snapshot hands back app-space points, and the first qualifying - // window is not always the one under the finger — a SpringBoard `alert accept` can live in an - // alert window while the wallpaper or status-bar window qualifies first, so a window-relative - // anchor misses the button. Reference frames and the synthesized display ID still resolve - // through `resolveRunnerWindow`, so a foldable tap keeps its panel and stays on its display. + // A caller's point is a screen point: coordinate taps, double taps, long presses, and drags + // anchor at the application's origin, not the resolved window, so the point is dispatched as + // given. That holds because `app.coordinate(withNormalizedOffset: .zero)` sits at the origin of + // the space `app.frame` names, and for a windowed app the measured `app.frame` starts at the + // screen's origin. The first qualifying window is not always the one under the finger — a + // SpringBoard `alert accept` can live in an alert window while the wallpaper or status-bar + // window qualifies first, so a window-relative anchor misses the button. Reference frames and + // the synthesized display ID still resolve through `resolveRunnerWindow`, so a foldable tap + // keeps its panel and stays on its display (#3899, ADR 0025). let origin = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0)) return origin.withOffset(CGVector(dx: x, dy: y)) #else diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index 079bb14a36..097213aac1 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -149,6 +149,11 @@ extension RunnerTests { let element: XCUIElement? let refreshPoint: CGPoint? let prefersFocusedElement: Bool + /// The live snapshot-frame vector for the reading session, read once when this target was built + /// (#3899). Every snapshot frame the target's reads produce is compared with screen points, so + /// every such read carries this vector (`ResolvedTextEntryInput.frameOnScreen`). Nil on a device + /// whose app fills the screen, where a snapshot frame is already a screen frame. + let contentTranslation: CGSize? let fromTapWitness: Bool /// The input the first resolved element was. Once bound, resolution refuses any other input. let boundIdentity: TextEntryInputIdentity? @@ -166,6 +171,7 @@ extension RunnerTests { element: XCUIElement?, refreshPoint: CGPoint?, prefersFocusedElement: Bool, + contentTranslation: CGSize? = nil, fromTapWitness: Bool = false, boundIdentity: TextEntryInputIdentity? = nil, inputAtRefreshPoint: TextInputAtPoint? = nil, @@ -174,30 +180,34 @@ extension RunnerTests { self.element = element self.refreshPoint = refreshPoint self.prefersFocusedElement = prefersFocusedElement + self.contentTranslation = contentTranslation self.fromTapWitness = fromTapWitness self.boundIdentity = boundIdentity ?? inputAtRefreshPoint?.identity self.inputAtRefreshPoint = inputAtRefreshPoint self.postFocusRead = postFocusRead } - /// The target moved to the input a resolution read: its element, a refresh point at its center, - /// and its identity while the target is unbound. - /// - /// `translation` is the live snapshot-frame vector for the reading session (#3899): the snapshot - /// frame of a placed window's input names the window's content space, while a refresh point is - /// compared against screen-space probes, so the center enters the target translated. - func withResolvedInput( - _ input: ResolvedTextEntryInput?, - translation: CGSize? - ) -> TextEntryTarget { + /// The target moved to the input a resolution read: its element and identity, and — when the + /// read carries a snapshot — a refresh point at that frame's center on the screen (`frameOnScreen` + /// plus this target's content vector, #3899). A read with no snapshot keeps the old refresh + /// point: its center would come from the live `element.frame`, the read a post-focus handle + /// cannot answer without recording a failure (#3060). + func withResolvedInput(_ input: ResolvedTextEntryInput?) -> TextEntryTarget { guard let input else { return self } - let frame = snapshotFrameOnScreen(input.snapshot?.frame ?? .zero, translation: translation) + let nextRefreshPoint: CGPoint? + if let snapshot = input.snapshot { + let frame = input.frameOnScreen + nextRefreshPoint = frame.isEmpty ? refreshPoint : CGPoint(x: frame.midX, y: frame.midY) + } else { + nextRefreshPoint = refreshPoint + } return TextEntryTarget( element: input.element, - refreshPoint: frame.isEmpty ? refreshPoint : CGPoint(x: frame.midX, y: frame.midY), + refreshPoint: nextRefreshPoint, prefersFocusedElement: prefersFocusedElement, + contentTranslation: contentTranslation, fromTapWitness: fromTapWitness, boundIdentity: boundIdentity ?? input.identity, inputAtRefreshPoint: inputAtRefreshPoint @@ -211,20 +221,31 @@ extension RunnerTests { /// Pro, iOS 27.0). Where a live read of an element that stopped answering its query, such as a /// removed input or a focused Flutter password field (#3060), records an XCTest failure that ends /// the runner session, the snapshot only throws. + /// + /// This type owns the probe-side space of #3899 (ADR 0004). `XCUIElementSnapshot.frame` reports a + /// placed window's content in the window's own space, while every point this read feeds — a refresh + /// point, a focus tap, a probe match, a readiness retap — is a screen point. So the read carries + /// the content vector it was taken under, and `frameOnScreen` is the only frame consumers use: a + /// snapshot frame plus the vector, or the live frame when there is no snapshot. Raw frames + /// (`snapshot.frame`) are compared only with each other. struct ResolvedTextEntryInput { let element: XCUIElement /// Nil when the element exists but could not be snapshotted. let snapshot: XCUIElementSnapshot? + /// The live snapshot-frame vector read once for the command that made this read (#3899). Nil + /// when the app fills the screen, where a snapshot frame is already a screen frame. + let contentTranslation: CGSize? - private init(element: XCUIElement, snapshot: XCUIElementSnapshot?) { + private init(element: XCUIElement, snapshot: XCUIElementSnapshot?, contentTranslation: CGSize?) { self.element = element self.snapshot = snapshot + self.contentTranslation = contentTranslation } /// Reads a handle that may name an input the app has removed; nil when it names none or the read /// raised. The snapshot of a missing element throws only after XCTest waits about two seconds for /// a match, so existence is read first. - init?(reading element: XCUIElement) { + init?(reading element: XCUIElement, contentTranslation: CGSize? = nil) { var exists = false var snapshot: XCUIElementSnapshot? if let exception = RunnerObjCExceptionCatcher.catchException({ @@ -235,15 +256,15 @@ extension RunnerTests { return nil } guard exists else { return nil } - self.init(element: element, snapshot: snapshot) + self.init(element: element, snapshot: snapshot, contentTranslation: contentTranslation) } /// Reads a handle its query has just listed, inside the caller's exception containment, so the /// snapshot is read first and existence only when the snapshot fails; nil when it names no element. - init?(readingListed element: XCUIElement) { + init?(readingListed element: XCUIElement, contentTranslation: CGSize? = nil) { let snapshot = try? element.snapshot() guard snapshot != nil || element.exists else { return nil } - self.init(element: element, snapshot: snapshot) + self.init(element: element, snapshot: snapshot, contentTranslation: contentTranslation) } var identity: TextEntryInputIdentity? { @@ -256,25 +277,37 @@ extension RunnerTests { } return element } + + /// The input's box in screen space: the snapshot frame carried by `contentTranslation` to its + /// place on the screen, or the live frame when there is no snapshot — the live + /// `XCUIElement.frame` already reports a placed window's content in screen space (#3899). + var frameOnScreen: CGRect { + guard let snapshot else { + return element.frame + } + guard let contentTranslation else { + return snapshot.frame + } + return snapshot.frame.offsetBy(dx: contentTranslation.width, dy: contentTranslation.height) + } } /// A text entry target and its input as last read. `input()` hands the last read out again until /// `dispatch` runs, and resolves the target again after that, so a bound target refuses an input /// that took its place. Each resolution moves the target to the input it read. Every post and every /// pause between reads goes through `dispatch`; the pause a typing plan takes itself follows a post. + /// The reader hands the target's content vector to every read it resolves, so one vector rules the + /// command (#3899). struct TextEntryReader { private(set) var target: TextEntryTarget private var lastRead: ResolvedTextEntryInput? private let resolve: (TextEntryTarget) -> ResolvedTextEntryInput? - private let translation: CGSize? init( target: TextEntryTarget, - translation: CGSize?, resolve: @escaping (TextEntryTarget) -> ResolvedTextEntryInput? ) { - self.translation = translation - self.target = target.withResolvedInput(target.postFocusRead, translation: translation) + self.target = target.withResolvedInput(target.postFocusRead) lastRead = target.postFocusRead self.resolve = resolve } @@ -284,7 +317,7 @@ extension RunnerTests { return lastRead } let read = resolve(target) - target = target.withResolvedInput(read, translation: translation) + target = target.withResolvedInput(read) lastRead = read return read } @@ -507,7 +540,10 @@ extension RunnerTests { return resolveUnboundTextEntryInput(app: app, target: target) } func boundInput(_ candidate: XCUIElement) -> ResolvedTextEntryInput? { - guard let input = ResolvedTextEntryInput(reading: candidate), input.identity == boundIdentity else { return nil } + guard let input = ResolvedTextEntryInput( + reading: candidate, + contentTranslation: target.contentTranslation + ), input.identity == boundIdentity else { return nil } return input } if target.prefersFocusedElement, let focused = focusedTextInput(app: app), let input = boundInput(focused) { @@ -516,12 +552,17 @@ extension RunnerTests { if let element = target.element, let input = boundInput(element) { return input } - if let input = target.inputAtRefreshPoint, let read = textInputIfStillResolves(input) { + if let input = target.inputAtRefreshPoint, + let read = textInputIfStillResolves(input, contentTranslation: target.contentTranslation) { return read } if target.inputAtRefreshPoint == nil, let refreshPoint = target.refreshPoint, - case .matches(let candidates) = probeTextInputs(app: app, point: refreshPoint), + case .matches(let candidates) = probeTextInputs( + app: app, + point: refreshPoint, + contentTranslation: target.contentTranslation + ), let match = candidates.first(where: { $0.identity == boundIdentity }) { return match } @@ -556,7 +597,11 @@ extension RunnerTests { guard let refreshPoint = target.refreshPoint else { return true } - switch probeTextInputs(app: app, point: refreshPoint) { + switch probeTextInputs( + app: app, + point: refreshPoint, + contentTranslation: target.contentTranslation + ) { case .absent: return true case .matches(let candidates): @@ -590,17 +635,25 @@ extension RunnerTests { /// Resolution for a target not bound to an input yet: the first element that exists. private func resolveUnboundTextEntryInput(app: XCUIApplication, target: TextEntryTarget) -> ResolvedTextEntryInput? { - if target.prefersFocusedElement, let focused = focusedTextInput(app: app).flatMap(ResolvedTextEntryInput.init(reading:)) { + let translation = target.contentTranslation + if target.prefersFocusedElement, + let focused = focusedTextInput(app: app).flatMap({ ResolvedTextEntryInput(reading: $0, contentTranslation: translation) }) { return focused } - if let element = target.element, let input = ResolvedTextEntryInput(reading: element) { + if let element = target.element, + let input = ResolvedTextEntryInput(reading: element, contentTranslation: translation) { return input } if let refreshPoint = target.refreshPoint, - let refreshed = textInputAt(app: app, x: refreshPoint.x, y: refreshPoint.y) { + let refreshed = textInputAt( + app: app, + x: refreshPoint.x, + y: refreshPoint.y, + contentTranslation: translation + ) { return refreshed } - return focusedTextInput(app: app).flatMap(ResolvedTextEntryInput.init(reading:)) + return focusedTextInput(app: app).flatMap { ResolvedTextEntryInput(reading: $0, contentTranslation: translation) } } private func moveCaretToEnd(app: XCUIApplication, input: ResolvedTextEntryInput) { @@ -619,12 +672,14 @@ extension RunnerTests { // zero-size one (how a departed handle reads), there is no point to dispatch — an edge-tap // from a frame the field moved after records a failure or navigates away, the shape // `clearTextInput`'s comment refuses — so degrade to the caller's point-free delete burst. - // The snapshot frame names the window's own content space on a placed window, so it enters - // `tapAt` translated (#3899). - guard let read = input.snapshot?.frame, !read.isEmpty else { + // The read's snapshot frame names the window's own content space on a placed window, so the + // point enters `tapAt` through `frameOnScreen`, which carries the read's content vector (#3899). + // A read with no snapshot gets no point at all: `frameOnScreen`'s fallback is the live + // `element.frame` read this function exists to avoid. + guard input.snapshot?.frame.isEmpty == false else { return } - let frame = snapshotFrameOnScreen(read, translation: liveContentTranslation(app: app)) + let frame = input.frameOnScreen _ = tapAt( app: app, x: frame.minX + max(2, frame.width - 4), diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift index 1ac9056733..5d4140d917 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift @@ -61,7 +61,8 @@ extension RunnerTests { func stabilizeTextInputBeforeTyping( app: XCUIApplication, target: XCUIElement?, - keyboardVisibleBeforeTap: Bool? = nil + keyboardVisibleBeforeTap: Bool? = nil, + contentTranslation: CGSize? = nil ) -> TextEntryStabilization { #if os(tvOS) return TextEntryStabilization(element: target, evidence: .focusEngine, polls: 0) @@ -78,8 +79,9 @@ extension RunnerTests { // The tapped input's own read comes before the keyboard. While a software keyboard is // presented, the keyboard query blocks and then finds none. An input that reports keyboard // focus is stronger evidence than the keyboard's arrival, which shows only that some input - // took focus. - let read = target.flatMap(ResolvedTextEntryInput.init(reading:)) + // took focus. The read carries the focus step's content vector (#3899): the hand-on read + // becomes the entry's first read, and its frames must enter screen space. + let read = target.flatMap { ResolvedTextEntryInput(reading: $0, contentTranslation: contentTranslation) } if let read, let snapshot = read.snapshot, snapshotHasKeyboardFocus(snapshot) { return TextEntryStabilization(element: read.element, evidence: .inputReportsKeyboardFocus(read), polls: polls) } @@ -130,16 +132,23 @@ extension RunnerTests { return TextEntryTarget(element: focused, refreshPoint: nil, prefersFocusedElement: true) } + // The one content-vector read for this command (#3899): every target this focus step builds + // carries it, every read those targets produce converts through it, and the retap and reader + // downstream reuse it. On a device whose app fills the screen it is nil and every frame stays + // exactly as read. + let contentTranslation = liveContentTranslation(app: app) let keyboardVisibleBeforeTap = isKeyboardVisible(app: app) - let input = textInputAt(app: app, x: x, y: y) + let input = textInputAt(app: app, x: x, y: y, contentTranslation: contentTranslation) let target = input?.element let requestedPoint = CGPoint(x: x, y: y) - // The input's frame arrives from a snapshot, which reports a placed window's content in the - // window's own space; the tap below is a screen point (#3899). An input read live (no - // snapshot) already names screen space, so only the snapshot frame is translated here. - if let snapshotFrame = input?.snapshot?.frame, !snapshotFrame.isEmpty { - let frame = snapshotFrameOnScreen(snapshotFrame, translation: liveContentTranslation(app: app)) - _ = tapAt(app: app, x: frame.midX, y: frame.midY) + // The focus tap goes to the input's own center, and the input's frame arrives from a snapshot, + // which reports a placed window's content in the window's own space while `tapAt` dispatches a + // screen point (#3899). An input read live (no snapshot) already names screen space. + let inputCenter = input.map { $0.frameOnScreen }.flatMap { + $0.isEmpty ? nil : CGPoint(x: $0.midX, y: $0.midY) + } + if let inputCenter { + _ = tapAt(app: app, x: inputCenter.x, y: inputCenter.y) } else { _ = tapAt(app: app, x: x, y: y) } @@ -150,13 +159,14 @@ extension RunnerTests { return focusedTextEntryTarget( target, requestedPoint: requestedPoint, - translation: liveContentTranslation(app: app) + contentTranslation: contentTranslation ) } let stabilized = stabilizeTextInputBeforeTyping( app: app, target: target, - keyboardVisibleBeforeTap: keyboardVisibleBeforeTap + keyboardVisibleBeforeTap: keyboardVisibleBeforeTap, + contentTranslation: contentTranslation ) NSLog( "%@%@ polls=%d", @@ -169,7 +179,7 @@ extension RunnerTests { read.element, read: read, requestedPoint: requestedPoint, - translation: liveContentTranslation(app: app) + contentTranslation: contentTranslation ) } let readyTarget = TextEntryTarget( @@ -183,7 +193,7 @@ extension RunnerTests { return focusedTextEntryTarget( element, requestedPoint: requestedPoint, - translation: liveContentTranslation(app: app) + contentTranslation: contentTranslation ) } @@ -191,16 +201,23 @@ extension RunnerTests { private func focusedTextEntryTarget( _ element: XCUIElement?, requestedPoint: CGPoint, - translation: CGSize? + contentTranslation: CGSize? ) -> TextEntryTarget { guard let element else { - return TextEntryTarget(element: nil, refreshPoint: requestedPoint, prefersFocusedElement: false) + // No element: the target keeps its refresh point, and the reads a later resolve makes through + // that point carry this command's vector (#3899). + return TextEntryTarget( + element: nil, + refreshPoint: requestedPoint, + prefersFocusedElement: false, + contentTranslation: contentTranslation + ) } return focusedTextEntryTarget( element, - read: ResolvedTextEntryInput(reading: element), + read: ResolvedTextEntryInput(reading: element, contentTranslation: contentTranslation), requestedPoint: requestedPoint, - translation: translation + contentTranslation: contentTranslation ) } @@ -209,16 +226,16 @@ extension RunnerTests { _ element: XCUIElement, read: ResolvedTextEntryInput?, requestedPoint: CGPoint, - translation: CGSize? + contentTranslation: CGSize? ) -> TextEntryTarget { return TextEntryTarget( element: element, refreshPoint: textEntryRefreshPoint( - frame: read?.snapshot?.frame, - fallbackTo: requestedPoint, - translation: translation + read: read, + fallbackTo: requestedPoint ), prefersFocusedElement: false, + contentTranslation: contentTranslation, postFocusRead: read?.snapshot == nil ? nil : read ) } @@ -231,21 +248,22 @@ extension RunnerTests { /// the target gets NO refresh point rather than the pre-focus one: this route's first resolve is /// not yet bound to an identity, and a field that went unqueryable on focus may have moved the /// layout so the old point names a different input, so fail closed with a typed reason instead of - /// typing into that other field. + /// typing into that other field. The frame is the read's snapshot frame plus its content vector + /// (#3899) — never the live `element.frame` fallback `frameOnScreen` uses for a snapshotless read. private func textEntryRefreshPoint( - frame: CGRect?, - fallbackTo requestedPoint: CGPoint, - translation: CGSize? + read: ResolvedTextEntryInput?, + fallbackTo requestedPoint: CGPoint ) -> CGPoint? { - guard let frame else { + guard let snapshot = read?.snapshot else { return nil } - guard !frame.isEmpty else { + guard !snapshot.frame.isEmpty else { return requestedPoint } - // The read's frame names the window's content space on a placed window; a refresh point is - // compared against screen-space probes, so the center enters the target translated (#3899). - let onScreen = snapshotFrameOnScreen(frame, translation: translation) - return CGPoint(x: onScreen.midX, y: onScreen.midY) + let frame = snapshot.frame.offsetBy( + dx: read?.contentTranslation?.width ?? 0, + dy: read?.contentTranslation?.height ?? 0 + ) + return CGPoint(x: frame.midX, y: frame.midY) } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift index f4c49a5a67..52bd98268d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift @@ -85,22 +85,52 @@ extension RunnerTests { /// `ResolvedTextEntryInput` already reads elements this way. It is read again after the /// wait rather than reused, because focusing a field moves the layout: the repeat aims at where the /// field is then, which is the moment the read it replaced ran at. + /// + /// The read enters `retapAt` as a `ResolvedTextEntryInput`, so the repeat taps + /// `frameOnScreen` — the snapshot frame plus the content vector of a placed window (#3899). The base + /// read carries no vector; the re-read carries the live one, read only on this branch, so a tap that + /// confirms focus dispatches no window lookup. func waitForTextEntryReadinessAfterTap(app: XCUIApplication, element: XCUIElement) { #if os(iOS) if waitForTextEntryFocus(app: app, element: element, timeout: TextEntryTiming.readinessTimeout) { return } - guard let settled = safely("TEXT_ENTRY_READINESS_FRAME", { try? element.snapshot() }) else { - return + // The re-read enters as a `ResolvedTextEntryInput` carrying the content vector of a placed + // window (#3899), read only on this branch, so a tap that confirms focus dispatches no window + // lookup. The window read is contained like the probe lookups: a transient window that records + // an XCTest issue costs the vector, not the command. + let settled: ResolvedTextEntryInput? + switch containingTextInputProbeIssues(fallback: nil as ResolvedTextEntryInput?, { _ in + ResolvedTextEntryInput(reading: element, contentTranslation: liveContentTranslation(app: app)) + }) { + case .success(let read): + settled = read + case .failure: + settled = nil } - let frame = settled.frame - if !frame.isEmpty { - _ = tapAt(app: app, x: frame.midX, y: frame.midY) - _ = waitForTextEntryFocus(app: app, element: element, timeout: TextEntryTiming.readinessTimeout) + guard let settled else { + return } + retapAt(read: settled, app: app, element: element) #endif } + /// The readiness retap, split out so the unit lane can drive it and assert where the point lands. + /// `tapAt` takes a screen point, so the point is `frameOnScreen`'s center. A read with no snapshot + /// taps nothing: `frameOnScreen`'s fallback is the live `element.frame` read — the one that records + /// an XCTest failure on a handle that stopped answering its query (#3060). + func retapAt(read: ResolvedTextEntryInput, app: XCUIApplication, element: XCUIElement) { + guard read.snapshot != nil else { + return + } + let frame = read.frameOnScreen + guard !frame.isEmpty else { + return + } + _ = tapAt(app: app, x: frame.midX, y: frame.midY) + _ = waitForTextEntryFocus(app: app, element: element, timeout: TextEntryTiming.readinessTimeout) + } + private func waitForTextEntryFocus(app: XCUIApplication, element: XCUIElement, timeout: TimeInterval) -> Bool { let deadline = Date().addingTimeInterval(timeout) while Date() < deadline { @@ -129,6 +159,8 @@ extension RunnerTests { /// fallback at all. func keyboardFocusConfirmed(app: XCUIApplication, element: XCUIElement) -> Bool { #if os(iOS) + // The read needs no content vector: this corroborates identity and position raw-frame against + // raw-frame, and never turns a frame into a dispatched point (#3899's rule). return ResolvedTextEntryInput(reading: element).map { keyboardFocusConfirmed(app: app, read: $0) } ?? false #else return false diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift index 9a449a51c9..39f0e8b6d6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift @@ -18,18 +18,32 @@ enum TextInputProbeOutcome { } extension RunnerTests { - func textInputAt(app: XCUIApplication, x: Double, y: Double) -> ResolvedTextEntryInput? { - textInputCandidatesAt(app: app, point: CGPoint(x: x, y: y)).first + func textInputAt( + app: XCUIApplication, + x: Double, + y: Double, + contentTranslation: CGSize? + ) -> ResolvedTextEntryInput? { + textInputCandidatesAt(app: app, point: CGPoint(x: x, y: y), contentTranslation: contentTranslation).first } - func textInputCandidatesAt(app: XCUIApplication, point: CGPoint) -> [ResolvedTextEntryInput] { + func textInputCandidatesAt( + app: XCUIApplication, + point: CGPoint, + contentTranslation: CGSize? + ) -> [ResolvedTextEntryInput] { safely("TEXT_INPUT_AT_POINT", []) { - queryTextInputs(app: app, point: point, shouldStop: { false }) + queryTextInputs(app: app, point: point, contentTranslation: contentTranslation, shouldStop: { false }) } } - func coordinateTapTextInputAt(app: XCUIApplication, x: Double, y: Double) -> XCUIElement? { - switch probeTextInputs(app: app, point: CGPoint(x: x, y: y)) { + func coordinateTapTextInputAt( + app: XCUIApplication, + x: Double, + y: Double, + contentTranslation: CGSize? + ) -> XCUIElement? { + switch probeTextInputs(app: app, point: CGPoint(x: x, y: y), contentTranslation: contentTranslation) { case .matches(let inputs): return inputs.first?.element case .absent: @@ -54,11 +68,17 @@ extension RunnerTests { func coordinateTapTextInputIdentityAt( app: XCUIApplication, x: Double, - y: Double + y: Double, + contentTranslation: CGSize? ) -> CoordinateTextInputResolution { let probed = containingTextInputProbeIssues(fallback: CoordinateTextInputResolution.unavailable) { shouldStop -> CoordinateTextInputResolution in - guard let input = queryTextInputs(app: app, point: CGPoint(x: x, y: y), shouldStop: shouldStop).first else { + guard let input = queryTextInputs( + app: app, + point: CGPoint(x: x, y: y), + contentTranslation: contentTranslation, + shouldStop: shouldStop + ).first else { return .absent } guard let identity = input.identity else { @@ -76,9 +96,13 @@ extension RunnerTests { /// `input`'s handle, read again, when it still resolves to the input it named when found, read /// inside the same issue containment. A read that cannot answer counts as no. - func textInputIfStillResolves(_ input: TextInputAtPoint) -> ResolvedTextEntryInput? { + func textInputIfStillResolves( + _ input: TextInputAtPoint, + contentTranslation: CGSize? + ) -> ResolvedTextEntryInput? { let probed = containingTextInputProbeIssues(fallback: nil as ResolvedTextEntryInput?) { _ in - ResolvedTextEntryInput(reading: input.element).flatMap { $0.identity == input.identity ? $0 : nil } + ResolvedTextEntryInput(reading: input.element, contentTranslation: contentTranslation) + .flatMap { $0.identity == input.identity ? $0 : nil } } guard case .success(let read) = probed else { return nil } return read @@ -94,9 +118,13 @@ extension RunnerTests { return read } - func probeTextInputs(app: XCUIApplication, point: CGPoint) -> TextInputProbeOutcome { + func probeTextInputs( + app: XCUIApplication, + point: CGPoint, + contentTranslation: CGSize? + ) -> TextInputProbeOutcome { switch containingTextInputProbeIssues(fallback: [], { shouldStop in - queryTextInputs(app: app, point: point, shouldStop: shouldStop) + queryTextInputs(app: app, point: point, contentTranslation: contentTranslation, shouldStop: shouldStop) }) { case .success(let inputs): return inputs.isEmpty ? .absent : .matches(inputs) @@ -106,8 +134,9 @@ extension RunnerTests { } /// Runs an optional probe whose XCTest issues are contained instead of recorded: an issue makes - /// the probe unavailable rather than failing the command and invalidating the runner. - private func containingTextInputProbeIssues( + /// the probe unavailable rather than failing the command and invalidating the runner. Visible to + /// the readiness retap, whose window re-read needs the same containment (#3899). + func containingTextInputProbeIssues( fallback: T, _ probe: (_ shouldStop: () -> Bool) -> T ) -> Result { @@ -151,10 +180,10 @@ extension RunnerTests { private func queryTextInputs( app: XCUIApplication, point: CGPoint, + contentTranslation: CGSize?, shouldStop: () -> Bool ) -> [ResolvedTextEntryInput] { var inputs: [ResolvedTextEntryInput] = [] - let translation = liveContentTranslation(app: app) for query in [app.textFields, app.secureTextFields, app.searchFields, app.textViews] { if shouldStop() { break } let candidates = query.allElementsBoundByIndex @@ -168,18 +197,15 @@ extension RunnerTests { // cannot be snapshotted is read live, so its failure still reaches the caller's issue handling. for element in candidates { if shouldStop() { break } - guard let candidate = ResolvedTextEntryInput(readingListed: element) else { continue } - let attributes = candidate.attributes - // Only the snapshot names the window's content space; a candidate read live (no snapshot) - // already reports screen space, and translating it again would move the candidate off the - // point it is being matched against (#3899). - let frame: CGRect - if let snapshot = candidate.snapshot { - frame = snapshotFrameOnScreen(snapshot.frame, translation: translation) - } else { - frame = attributes.frame - } - if isCoordinateTextInputCandidate(enabled: attributes.isEnabled, frame: frame, point: point) { + guard let candidate = ResolvedTextEntryInput( + readingListed: element, + contentTranslation: contentTranslation + ) else { continue } + if isCoordinateTextInputCandidate( + enabled: candidate.attributes.isEnabled, + frame: candidate.frameOnScreen, + point: point + ) { inputs.append(candidate) } } @@ -189,7 +215,12 @@ extension RunnerTests { } /// The vector that carries this session's snapshot-read app-content frames to their place on the - /// screen, or nil when the app fills the screen (#3899). + /// screen, or nil when the app fills the screen (#3899). Read once at the start of each command + /// that dispatches points derived from snapshot frames — the focus step, the coordinate fill route, + /// the readiness retap, a coordinate text read — and carried on the types the command builds from + /// it (`TextEntryTarget.contentTranslation`, `ResolvedTextEntryInput.contentTranslation`). No read + /// inside a loop re-reads it: the whole-app queries below cost about 90 ms each on heavy React + /// Native screens, and a per-read re-query would undo the fill-read work of #3881, #3906 and #3909. /// /// The live `XCUIElement.frame` property already reports a placed window's content in screen space, /// but `XCUIElementSnapshot.frame` reports it in the window's own content space, so the same probe @@ -199,15 +230,17 @@ extension RunnerTests { /// place on the screen. Any other pair of boxes declares no vector, so full-screen, iPhone, and /// each foldable panel keep their frames exactly as read. func liveContentTranslation(app: XCUIApplication) -> CGSize? { - safely("LIVE_CONTENT_TRANSLATION", nil) { + // The window read is contained like the probe lookups it replaces: a transient window that + // records an XCTest issue costs the vector, not the runner session. + let read = containingTextInputProbeIssues(fallback: nil as CGSize?) { _ in guard let resolved = resolveRunnerWindowElement(app: app) else { return nil } return SnapshotContentTranslation.offset(reportedBox: app.frame, viewportBox: resolved.frame) } + switch read { + case .success(let translation): + return translation + case .failure: + return nil + } } } - -/// `frame` as the screen names it, given `liveContentTranslation`'s vector. -func snapshotFrameOnScreen(_ frame: CGRect, translation: CGSize?) -> CGRect { - guard let translation else { return frame } - return frame.offsetBy(dx: translation.width, dy: translation.height) -} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift index 75aaffeece..142f9c4415 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift @@ -16,13 +16,7 @@ extension RunnerTests { ) -> TextEntryResult { let totalStartedAt = Date() let resolve = { (target: TextEntryTarget) in self.resolveTextEntryInput(app: app, target: target) } - // Reads inside the typing loop answer from snapshots, which report a placed window's content in - // the window's own space; the reader's refresh points are screen points (#3899). - var reader = TextEntryReader( - target: target, - translation: liveContentTranslation(app: app), - resolve: resolve - ) + var reader = TextEntryReader(target: target, resolve: resolve) guard !text.isEmpty else { // Replacing a field WITH the empty string is the clear-field primitive (#2063): the typing // is vacuous but the clear is not. Returning here unconditionally left the old value in @@ -80,27 +74,14 @@ extension RunnerTests { case .completed(let result): return result case .fallback(let fallbackTarget): - reader = TextEntryReader( - target: fallbackTarget, - translation: liveContentTranslation(app: app), - resolve: resolve - ) + reader = TextEntryReader(target: fallbackTarget, resolve: resolve) } } - // The keyboard evidence is decided once, at the command's start. Measured on iPad Pro 11-inch - // (M5, iOS 27.0) with Windowed Apps (#3899): `app.keyboards` answered `false` 50 ms after - // answering `true` for the same on-screen keyboard, while the app's own AX reads kept failing - // with `kAXErrorServerNotFound` under a synthesized post burst. A per-post re-query therefore - // refused a receiver mid-burst and reported `TEXT_INPUT_NOT_FOCUSED` for a focused field, one - // character per call. An app that loses the keyboard mid-command keeps the start-of-command - // answer: the extra posts land on the digitizer and do nothing, the outcome a tap into empty - // space already has. - let softwareKeyboardVisibleAtStart = isKeyboardVisible(app: app) let shouldUseSynthesizedFirstResponderType = Self.shouldUseSynthesizedFirstResponderType( repairMode: repairMode, text: text, fromTapWitness: reader.target.fromTapWitness, - softwareKeyboardVisible: softwareKeyboardVisibleAtStart + softwareKeyboardVisible: isKeyboardVisible(app: app) ) let initialResolveStartedAt = Date() let initialTarget = reader.input() @@ -119,7 +100,7 @@ extension RunnerTests { // Dispatching text through XCTest without a resolved target or evidence of a focused // responder records a test failure. That tears down the long-lived runner and turns a // single invalid request into a restart cascade, so fail before entering that channel. - guard initialTarget != nil || (reader.target.prefersFocusedElement && softwareKeyboardVisibleAtStart) else { + guard initialTarget != nil || (reader.target.prefersFocusedElement && isKeyboardVisible(app: app)) else { logTextEntryPhase( commandId: commandId, phase: "total", @@ -214,8 +195,7 @@ extension RunnerTests { textEntryRoute = "xctest-element" reader.dispatch { currentTarget.element.typeText(value) } return nil - } else if reader.target.boundIdentity == nil && reader.target.prefersFocusedElement - && softwareKeyboardVisibleAtStart { + } else if reader.target.boundIdentity == nil && reader.target.prefersFocusedElement && isKeyboardVisible(app: app) { #if os(iOS) // Two ways this post leaves the synthesized channel, and both hand the text to // application-wide typing: the command's own budget refused it, or XCTest's private synthesis diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift index acbb62ef54..bea8d91e60 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift @@ -27,12 +27,15 @@ extension RunnerTests { hasY: command.y != nil, xCTestChannelPenalized: xCTestChannelPenalized ), let x = command.x, let y = command.y { + // The one content-vector read for this command (#3899): the target built here carries it, and + // every read that target produces converts through it, as the focus route's targets do. + let contentTranslation = liveContentTranslation(app: activeApp) // Resolved before the tap: the focus it causes can move the field away from this point. // A replacement with no input at the point has nothing to bind to: its clear and its typed // text would reach the app-level first responder, so an empty point and a lookup that cannot // answer both refuse here, each under its own reason. let inputAtPoint: TextInputAtPoint - switch coordinateTapTextInputIdentityAt(app: activeApp, x: x, y: y) { + switch coordinateTapTextInputIdentityAt(app: activeApp, x: x, y: y, contentTranslation: contentTranslation) { case .input(let input): inputAtPoint = input case .absent: @@ -54,6 +57,7 @@ extension RunnerTests { element: nil, refreshPoint: CGPoint(x: x, y: y), prefersFocusedElement: false, + contentTranslation: contentTranslation, inputAtRefreshPoint: inputAtPoint ) case .xctestFallback: diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift index a4a83c3df1..60979694f1 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift @@ -6,7 +6,7 @@ extension RunnerTests { /// cannot answer. A fixture that does not expose its own field is a broken lane, not a refusal. @MainActor func requireTextInputAtPoint(app: XCUIApplication, x: Double, y: Double) throws -> TextInputAtPoint { - switch coordinateTapTextInputIdentityAt(app: app, x: x, y: y) { + switch coordinateTapTextInputIdentityAt(app: app, x: x, y: y, contentTranslation: nil) { case .input(let input): return input case .absent: diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift index f70b57c907..9d27049ec2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift @@ -83,7 +83,7 @@ extension RunnerTests { } let target = TextEntryTarget(element: nil, refreshPoint: point, prefersFocusedElement: false) - let (lookup, lookupReads) = try countingReads { textInputAt(app: app, x: point.x, y: point.y) } + let (lookup, lookupReads) = try countingReads { textInputAt(app: app, x: point.x, y: point.y, contentTranslation: nil) } let (input, resolutionReads) = try countingReads { resolveTextEntryInput(app: app, target: target) } XCTAssertEqual(lookup?.snapshot?.identifier, "agent-device-hardware-keyboard-input") @@ -109,7 +109,12 @@ extension RunnerTests { "software keyboard is down: this simulator cannot show the keyboard a focus tap raises" ) - let stabilized = stabilizeTextInputBeforeTyping(app: app, target: field, keyboardVisibleBeforeTap: false) + let stabilized = stabilizeTextInputBeforeTyping( + app: app, + target: field, + keyboardVisibleBeforeTap: false, + contentTranslation: nil + ) guard case .inputReportsKeyboardFocus(let read) = stabilized.evidence else { return XCTFail("focus confirmed by \(stabilized.evidence?.name ?? "nothing"), not by the tapped input") diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryReadinessTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryReadinessTests.swift new file mode 100644 index 0000000000..415a55a4b7 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryReadinessTests.swift @@ -0,0 +1,57 @@ +import XCTest + +extension RunnerTests { +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) + /// The readiness retap dispatches `ResolvedTextEntryInput.frameOnScreen`'s center, so the snapshot + /// frame of a placed window passes through the read's content vector before it becomes a screen + /// point (#3899). Before that rule the retap tapped the bare `snapshot().frame` center, which on + /// a windowed iPad lands outside the window, and the fill after it failed with + /// TEXT_INPUT_NOT_FOCUSED — the issue's own failure shape. + /// + /// The fixture cannot be placed in a window, so the read is driven directly: the vector moves the + /// field's real snapshot frame exactly onto the press-counting witness button below it. A point + /// dispatched at `frameOnScreen`'s center presses that button; a point dispatched at the raw + /// snapshot center taps the field and moves no counter. The same read with no vector is the + /// closest negative: a full-screen app retaps the field itself and the counter stays put. + @MainActor + func testReadinessRetapDispatchesTheReadsScreenSpaceCenter() throws { + app.launchArguments = ["--agent-device-text-entry-regression", "--agent-device-text-entry-tap-witness"] + app.launch() + defer { invalidateCachedTarget(reason: "unit_test_cleanup"); app.terminate() } + XCTAssertTrue(app.waitForFixtureExistence(timeout: appExistenceTimeout)) + let field = app.textFields["agent-device-hardware-keyboard-input"] + XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout)) + let witness = app.buttons["agent-device-tap-witness"] + XCTAssertTrue(witness.waitForFixtureExistence(timeout: appExistenceTimeout)) + let presses = app.staticTexts["agent-device-tap-witness-presses"] + XCTAssertTrue(presses.waitForFixtureExistence(timeout: appExistenceTimeout)) + let fieldFrame = field.frame + let witnessFrame = witness.frame + XCTAssertFalse(fieldFrame.isEmpty) + XCTAssertFalse(witnessFrame.isEmpty) + + // Closest negative: a read with no vector — a full-screen app — retaps the field's own center, + // which is not the button's point. + let untranslated = try XCTUnwrap(ResolvedTextEntryInput(reading: field, contentTranslation: nil)) + retapAt(read: untranslated, app: app, element: field) + XCTAssertEqual(presses.label, "presses=0", "a vectorless read dispatched a point off the field") + + // The read a probe makes on a placed window: the snapshot reports the field's box and the read + // carries the window's vector. Here the vector is chosen so the translated box is the button. + let vector = CGSize( + width: witnessFrame.midX - fieldFrame.midX, + height: witnessFrame.midY - fieldFrame.midY + ) + let translated = try XCTUnwrap(ResolvedTextEntryInput(reading: field, contentTranslation: vector)) + XCTAssertEqual(translated.frameOnScreen.midX, witnessFrame.midX, accuracy: 0.5) + XCTAssertEqual(translated.frameOnScreen.midY, witnessFrame.midY, accuracy: 0.5) + + retapAt(read: translated, app: app, element: field) + XCTAssertEqual( + presses.label, + "presses=1", + "the retap dispatched the raw snapshot center, not the read's screen-space center (#3899)" + ) + } +#endif +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift index c9b306758a..6c7b54b49a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift @@ -15,7 +15,7 @@ extension RunnerTests { let completed = expectation(description: "optional probe completed inside runner wait") DispatchQueue.main.async { self.textInputProbeIssueForTesting = XCTIssue(type: .assertionFailure, compactDescription: "Optional probe issue during runner wait") - _ = self.probeTextInputs(app: self.app, point: point) + _ = self.probeTextInputs(app: self.app, point: point, contentTranslation: nil) completed.fulfill() } guard XCTWaiter.wait(for: [completed], timeout: 5) == .completed else { @@ -174,8 +174,8 @@ extension RunnerTests { } XCTExpectFailure("Required read and later issue belong to their caller", options: options) { textInputProbeIssueForTesting = expected - _ = textInputAt(app: app, x: point.x, y: point.y) - _ = probeTextInputs(app: app, point: point) + _ = textInputAt(app: app, x: point.x, y: point.y, contentTranslation: nil) + _ = probeTextInputs(app: app, point: point, contentTranslation: nil) record(expected) } XCTAssertEqual(observed, 2) @@ -192,7 +192,7 @@ extension RunnerTests { XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout)) let frame = field.frame textInputProbeIssueForTesting = XCTIssue(type: .assertionFailure, compactDescription: "Failed to get matching snapshot: kAXErrorIllegalArgument") - let outcome = probeTextInputs(app: app, point: CGPoint(x: frame.midX, y: frame.midY)) + let outcome = probeTextInputs(app: app, point: CGPoint(x: frame.midX, y: frame.midY), contentTranslation: nil) guard case .unavailable = outcome else { return XCTFail("A suppressed AX issue must discard the matching candidate") } diff --git a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift index 7718fbabc1..95396fc3a4 100644 --- a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift +++ b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift @@ -200,6 +200,11 @@ public enum SnapshotGeometrySpace: Equatable { ) { return .windowContent(offset: offset) } + // A host that declares no space of its own publishes what the platform reported. A system + // surface under a placed window — the keyboard at (0,925,834,282), a SpringBoard overlay at + // (0,0,834,1210) — reports the screen's own box and is already in screen space; the two + // `windowContentCases` rows for those windows pin that inheriting the carrier's vector would + // move every key by the window origin (#3899's own failure shape). return .appOrientation } diff --git a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift index ceb6f35cdb..9b80b197d2 100644 --- a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift +++ b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift @@ -616,6 +616,9 @@ final class CoordinateSpaceTests: XCTestCase { /// The dispatch direction for a windowed app: the caller's point is the point touched, so /// `native(point:)` turns a screen-space point's axes without moving it toward the window origin /// (#3899). Under portrait — the only orientation measured for a windowed app — it is the identity. + /// The landscape rows for a placed window stay the documented unmeasured gap (ADR 0004): the turn + /// here is asserted about the window's side lengths, while the digitizer turns about the screen's, + /// and no windowed landscape capture has been measured to say which basis is right. func testNativePointForAWindowedAppKeepsTheCallersScreenPoint() { let windowOnScreen = CGRect(x: 224, y: 207, width: 386, height: 682) for orientation in [ @@ -631,15 +634,6 @@ final class CoordinateSpaceTests: XCTestCase { "interfaceOrientation \(orientation)" ) } - // A turn moves axes only: the same point under a landscape window keeps its own origin story. - XCTAssertEqual( - CoordinateSpaceRotation.native( - point: CGPoint(x: 578, y: 239), - in: windowOnScreen, - interfaceOrientation: RunnerInterfaceOrientation.landscapeRight - ), - CGPoint(x: 682 - 239, y: 578) - ) } func testNormalizedOfAnEmptyArrayIsEmpty() { diff --git a/contracts/fixtures/window-coordinate-space.json b/contracts/fixtures/window-coordinate-space.json index 1211632d64..dbd6bfd551 100644 --- a/contracts/fixtures/window-coordinate-space.json +++ b/contracts/fixtures/window-coordinate-space.json @@ -218,7 +218,7 @@ "offset": null }, { - "name": "a half-tolerance origin miss is float representation, not a different placement", + "name": "a half-tolerance side-length miss is float representation, not a different box", "reportedBox": { "x": 0, "y": 0, "width": 385.5, "height": 682 }, "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, "offset": { "width": 224, "height": 207 } diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index 11f0799d22..db819ae872 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -660,9 +660,11 @@ Translation is the one rule that reads an origin, so it is the only place the fi Capture is half of the promise; dispatch is the other half. The measured dispatch spaces for a placed window are both screen-anchored: the synthesized path lands a point at its screen position, and XCTest's own `XCUICoordinate` on the app anchors at the screen's origin too. `interactionCoordinate` therefore passes the caller's screen point through, and `resolveRunnerWindow` keeps naming the window a foldable's tap resolves its display on (ADR 0025). -The live `XCUIElement.frame` property reports screen space on a placed window, as the selector-tap path proved on the device: it measured the `Add` button at (641,321) for a window at (324,307). +The live `XCUIElement.frame` property reports screen space on a placed window, as the selector-tap path proved on the device: it measured the `Add` button at (659,321) for a window at (324,307). The one live read that does not is `XCUIElementSnapshot.frame`, which reports the window's content space like the capture channel does. So the snapshot capture converts in `SnapshotGeometrySpace.normalized`, and the text-input probe converts its snapshot reads through the same `SnapshotContentTranslation` rule read from the two live boxes (`app.frame` against the resolved window's frame); every other live read needs no conversion. +The probe-side owner is `ResolvedTextEntryInput.frameOnScreen`. +The vector is read once per command and stored on the target and each read the target produces, so the probe matching, the focus tap, the refresh point, the caret tap, and the readiness retap all dispatch through that one property. Where a probe read has no snapshot and falls back to the live element, its frame is already screen space and is not translated. `app.frame` unions transformed subtrees, so content pushed off a placed window (a closed drawer at negative x) can stop the two live boxes matching and read no vector; the capture still converts from the snapshot root box, so a probe may then read window-local frames where the capture published screen space. A mid-command window move has the same effect on a vector read at command start. Both are reads of a moving target, not a second space: a re-opened command reads the current vector. diff --git a/packages/platform-apple/src/snapshot-route.test.ts b/packages/platform-apple/src/snapshot-route.test.ts index ca1a92920c..afd4dadacd 100644 --- a/packages/platform-apple/src/snapshot-route.test.ts +++ b/packages/platform-apple/src/snapshot-route.test.ts @@ -407,7 +407,7 @@ test.for([ ['foreground-owner-changed', 'while another surface owned the foreground'], [ 'app-window-not-full-screen', - 'while the app window did not fill the screen; XCTest geometry for a windowed app can also be offset from the screen', + 'while the app window did not fill the screen; the runner republishes a windowed capture in screen space', ], ] as const)('a %s refusal sends only that capture to the runner', async ([code, consequence]) => { const source = sourceReturning({ stage: 'failed', failure: { kind: 'unsupported', code } }); diff --git a/packages/platform-apple/src/snapshot-route.ts b/packages/platform-apple/src/snapshot-route.ts index 5fcee4fca7..c977be2baf 100644 --- a/packages/platform-apple/src/snapshot-route.ts +++ b/packages/platform-apple/src/snapshot-route.ts @@ -512,7 +512,7 @@ function captureScopedFallbackWarning(code: string, consequence: CaptureScopedCo case 'foreground-ownership': return `${thisCapture}, while another surface owned the foreground.`; case 'window-not-full-screen': - return `${thisCapture}, while the app window did not fill the screen; XCTest geometry for a windowed app can also be offset from the screen.`; + return `${thisCapture}, while the app window did not fill the screen; the runner republishes a windowed capture in screen space.`; case 'coordinate-space-unresolved': return `${thisCapture}, which reports captured geometry in the app's own orientation space.`; } diff --git a/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts b/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts index 4ce6887031..b181eef498 100644 --- a/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts +++ b/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts @@ -75,12 +75,6 @@ export function isQuarterTurnedWindowFrame(windowFrame: Rect, appFrame: Rect): b } /* - * A window that does not fill the screen (#3899) is translated by the capture that owns it, and this - * file is not that capture. The bridge guest refuses a windowed tree (`app-window-not-full-screen`) - * before this decoder ever sees one, so no host tree arrives in the window's content space and there - * is nothing here to translate. The rule lives with the producer that converts — - * `SnapshotContentTranslation` beside the Swift capture — and `windowContentCases` in - * `contracts/fixtures/window-coordinate-space.json` is replayed there alone. The quarter-turn - * detection above stays in both languages because this decoder does read it: it is what refuses a - * turned tree the reader cannot resolve. + * The #3899 window-content translation belongs to the Swift capture's `SnapshotContentTranslation`, + * replayed from `windowContentCases` there alone. */ From b4ec02750f2e5908322cbc852254f8e3d1e488ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 24 Oct 2026 10:34:58 +0200 Subject: [PATCH 08/15] docs(ios): name the probe-side owner and the screen-space dispatch basis (#3899) Amends ADR 0004 to name ResolvedTextEntryInput as the probe-side owner of the content vector, cites the measured Add-button rect at (659,321), records that the runner republishes a windowed capture in screen space where the bridge-refusal warning said it was offset, and names screen space plus the app.frame origin assumption in the interactionCoordinate comment. --- .../RunnerTests+TextEntryFocus.swift | 3 ++- docs/adr/0004-ios-snapshot-backend-strategy.md | 7 +++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift index 5d4140d917..cd1ef382b2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift @@ -185,7 +185,8 @@ extension RunnerTests { let readyTarget = TextEntryTarget( element: stabilized.element ?? target, refreshPoint: requestedPoint, - prefersFocusedElement: false + prefersFocusedElement: false, + contentTranslation: contentTranslation ) let element = stabilized.focusConfirmed ? readyTarget.element diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index db819ae872..a1dfccf286 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -768,10 +768,9 @@ If the frames differ, the guest refuses the capture (`app-window-not-full-screen This refusal provides evidence about the screen arrangement, not the app, so it has capture scope. If the runtime cannot report either frame, the guest keeps the capture. -The XCTest fallback is not correct for a windowed app either. -The runner clips window-relative element frames against the screen-relative app frame. -That is a runner gap. -It is not a reason to publish a bridge tree that is known to be wrong. +The XCTest fallback is the correct capture for a windowed app, after the amendment above. +Before it, the runner clipped window-relative element frames against the screen-relative app frame, and that gap is what made this refusal protect a real failure. +The refusal stays: this channel reports window-relative frames and no channel here converts them. Rejected alternatives: From 70f6faa55235d69fa33f504c901013168e3dcff1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 24 Oct 2026 10:54:43 +0200 Subject: [PATCH 09/15] refactor(ios): cut the screen-space comments and the duplicated frame offset (#3899) The vector on the target replaces the offset arithmetic in textEntryRefreshPoint, which read the snapshot frame and added the vector itself. One owner now: ResolvedTextEntryInput.frameOnScreen. The comments that restated the capture's rule at every call site are cut to the constraint the code cannot show. SnapshotCoordinateSpace.swift ships in the npm package without comment stripping, so its prose lives in ADR 0004 and the golden table instead; the same rule text is removed from the TypeScript decoder's foot note. --- .../RunnerTests+CommandExecution.swift | 3 +- .../RunnerTests+Interaction.swift | 14 +++-- .../RunnerTests+TextEntry.swift | 52 +++++++------------ .../RunnerTests+TextEntryFocus.swift | 29 ++++------- .../RunnerTests+TextEntryReadiness.swift | 20 +++---- .../RunnerTests+TextInputProbe.swift | 27 ++++------ .../RunnerTests+TypeExecution.swift | 3 +- .../SnapshotCoordinateSpace.swift | 52 +++++++------------ .../adr/0004-ios-snapshot-backend-strategy.md | 2 +- .../window-coordinate-space.ts | 7 +-- 10 files changed, 74 insertions(+), 135 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index ea6a445abf..adcab6215c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -155,8 +155,7 @@ extension RunnerTests { ) let textInput: XCUIElement? if !xCTestTextInputProbeSkipped { - // The lookup compares the point, which is a screen point, with candidate frames; on a - // placed window the snapshot frames need the content vector (#3899). + // The lookup compares candidate frames with this screen point (#3899). textInput = coordinateTapTextInputAt( app: activeApp, x: x, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 4b134a059d..704b2bed10 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -352,8 +352,7 @@ extension RunnerTests { func readTextAt(app: XCUIApplication, x: Double, y: Double) -> String? { let point = CGPoint(x: x, y: y) - // The point is a screen point; candidate snapshot frames of a placed window need the content - // vector to be compared with it (#3899). + // The lookup compares candidate frames with this screen point (#3899). let textInputCandidates = textInputCandidatesAt( app: app, point: point, @@ -623,15 +622,14 @@ extension RunnerTests { #if !os(tvOS) private func interactionCoordinate(app: XCUIApplication, x: Double, y: Double) -> XCUICoordinate { #if os(iOS) - // A caller's point is a screen point: coordinate taps, double taps, long presses, and drags - // anchor at the application's origin, not the resolved window, so the point is dispatched as - // given. That holds because `app.coordinate(withNormalizedOffset: .zero)` sits at the origin of - // the space `app.frame` names, and for a windowed app the measured `app.frame` starts at the - // screen's origin. The first qualifying window is not always the one under the finger — a + // A caller's point is a screen point: these gestures anchor at the app's origin, not the + // resolved window, so it is dispatched as given. The anchor sits at the origin of the space + // `app.frame` names, and for a windowed app the measured `app.frame` starts at the screen's + // origin (#3899). The first qualifying window is not always the one under the finger — a // SpringBoard `alert accept` can live in an alert window while the wallpaper or status-bar // window qualifies first, so a window-relative anchor misses the button. Reference frames and // the synthesized display ID still resolve through `resolveRunnerWindow`, so a foldable tap - // keeps its panel and stays on its display (#3899, ADR 0025). + // keeps its panel and stays on its display (ADR 0025). let origin = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0)) return origin.withOffset(CGVector(dx: x, dy: y)) #else diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index 097213aac1..c198ea2534 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -149,10 +149,9 @@ extension RunnerTests { let element: XCUIElement? let refreshPoint: CGPoint? let prefersFocusedElement: Bool - /// The live snapshot-frame vector for the reading session, read once when this target was built - /// (#3899). Every snapshot frame the target's reads produce is compared with screen points, so - /// every such read carries this vector (`ResolvedTextEntryInput.frameOnScreen`). Nil on a device - /// whose app fills the screen, where a snapshot frame is already a screen frame. + /// The content vector of a placed window, read once for the command that built this target and + /// handed to every read it produces, which turns its snapshot frames into screen points (#3899). + /// Nil when the app fills the screen. let contentTranslation: CGSize? let fromTapWitness: Bool /// The input the first resolved element was. Once bound, resolution refuses any other input. @@ -187,25 +186,18 @@ extension RunnerTests { self.postFocusRead = postFocusRead } - /// The target moved to the input a resolution read: its element and identity, and — when the - /// read carries a snapshot — a refresh point at that frame's center on the screen (`frameOnScreen` - /// plus this target's content vector, #3899). A read with no snapshot keeps the old refresh - /// point: its center would come from the live `element.frame`, the read a post-focus handle - /// cannot answer without recording a failure (#3060). + /// The target moved to the input a resolution read: its element, a refresh point at its frame's + /// center on the screen, and its identity while the target is unbound. A read with no snapshot has + /// no frame here, so its center cannot come from the live `element.frame` — the read a post-focus + /// handle cannot answer without recording a failure (#3060). func withResolvedInput(_ input: ResolvedTextEntryInput?) -> TextEntryTarget { guard let input else { return self } - let nextRefreshPoint: CGPoint? - if let snapshot = input.snapshot { - let frame = input.frameOnScreen - nextRefreshPoint = frame.isEmpty ? refreshPoint : CGPoint(x: frame.midX, y: frame.midY) - } else { - nextRefreshPoint = refreshPoint - } + let frame = input.snapshot == nil ? .zero : input.frameOnScreen return TextEntryTarget( element: input.element, - refreshPoint: nextRefreshPoint, + refreshPoint: frame.isEmpty ? refreshPoint : CGPoint(x: frame.midX, y: frame.midY), prefersFocusedElement: prefersFocusedElement, contentTranslation: contentTranslation, fromTapWitness: fromTapWitness, @@ -222,18 +214,14 @@ extension RunnerTests { /// removed input or a focused Flutter password field (#3060), records an XCTest failure that ends /// the runner session, the snapshot only throws. /// - /// This type owns the probe-side space of #3899 (ADR 0004). `XCUIElementSnapshot.frame` reports a - /// placed window's content in the window's own space, while every point this read feeds — a refresh - /// point, a focus tap, a probe match, a readiness retap — is a screen point. So the read carries - /// the content vector it was taken under, and `frameOnScreen` is the only frame consumers use: a - /// snapshot frame plus the vector, or the live frame when there is no snapshot. Raw frames - /// (`snapshot.frame`) are compared only with each other. + /// This type owns the probe-side space of #3899 (ADR 0004): a snapshot frame names a placed + /// window's content space while every point this read feeds is a screen point, so consumers take + /// `frameOnScreen` and compare raw frames only with each other. struct ResolvedTextEntryInput { let element: XCUIElement /// Nil when the element exists but could not be snapshotted. let snapshot: XCUIElementSnapshot? - /// The live snapshot-frame vector read once for the command that made this read (#3899). Nil - /// when the app fills the screen, where a snapshot frame is already a screen frame. + /// The vector of the command that made this read (#3899); nil when the app fills the screen. let contentTranslation: CGSize? private init(element: XCUIElement, snapshot: XCUIElementSnapshot?, contentTranslation: CGSize?) { @@ -278,9 +266,9 @@ extension RunnerTests { return element } - /// The input's box in screen space: the snapshot frame carried by `contentTranslation` to its - /// place on the screen, or the live frame when there is no snapshot — the live - /// `XCUIElement.frame` already reports a placed window's content in screen space (#3899). + /// The input's box in screen space: the snapshot frame plus `contentTranslation`, or the live + /// frame when there is no snapshot — that property already reports a placed window's content in + /// screen space (#3899). var frameOnScreen: CGRect { guard let snapshot else { return element.frame @@ -296,8 +284,6 @@ extension RunnerTests { /// `dispatch` runs, and resolves the target again after that, so a bound target refuses an input /// that took its place. Each resolution moves the target to the input it read. Every post and every /// pause between reads goes through `dispatch`; the pause a typing plan takes itself follows a post. - /// The reader hands the target's content vector to every read it resolves, so one vector rules the - /// command (#3899). struct TextEntryReader { private(set) var target: TextEntryTarget private var lastRead: ResolvedTextEntryInput? @@ -672,10 +658,8 @@ extension RunnerTests { // zero-size one (how a departed handle reads), there is no point to dispatch — an edge-tap // from a frame the field moved after records a failure or navigates away, the shape // `clearTextInput`'s comment refuses — so degrade to the caller's point-free delete burst. - // The read's snapshot frame names the window's own content space on a placed window, so the - // point enters `tapAt` through `frameOnScreen`, which carries the read's content vector (#3899). - // A read with no snapshot gets no point at all: `frameOnScreen`'s fallback is the live - // `element.frame` read this function exists to avoid. + // A snapshotless read gets no point at all: `frameOnScreen`'s fallback is the live + // `element.frame` read this function exists to avoid (#3060). guard input.snapshot?.frame.isEmpty == false else { return } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift index cd1ef382b2..fb0c4592f8 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift @@ -79,8 +79,7 @@ extension RunnerTests { // The tapped input's own read comes before the keyboard. While a software keyboard is // presented, the keyboard query blocks and then finds none. An input that reports keyboard // focus is stronger evidence than the keyboard's arrival, which shows only that some input - // took focus. The read carries the focus step's content vector (#3899): the hand-on read - // becomes the entry's first read, and its frames must enter screen space. + // took focus. let read = target.flatMap { ResolvedTextEntryInput(reading: $0, contentTranslation: contentTranslation) } if let read, let snapshot = read.snapshot, snapshotHasKeyboardFocus(snapshot) { return TextEntryStabilization(element: read.element, evidence: .inputReportsKeyboardFocus(read), polls: polls) @@ -132,18 +131,15 @@ extension RunnerTests { return TextEntryTarget(element: focused, refreshPoint: nil, prefersFocusedElement: true) } - // The one content-vector read for this command (#3899): every target this focus step builds - // carries it, every read those targets produce converts through it, and the retap and reader - // downstream reuse it. On a device whose app fills the screen it is nil and every frame stays - // exactly as read. + // The one content-vector read for this command (#3899): every target built here carries it and + // every read those targets produce converts through it. let contentTranslation = liveContentTranslation(app: app) let keyboardVisibleBeforeTap = isKeyboardVisible(app: app) let input = textInputAt(app: app, x: x, y: y, contentTranslation: contentTranslation) let target = input?.element let requestedPoint = CGPoint(x: x, y: y) - // The focus tap goes to the input's own center, and the input's frame arrives from a snapshot, - // which reports a placed window's content in the window's own space while `tapAt` dispatches a - // screen point (#3899). An input read live (no snapshot) already names screen space. + // The tap goes to the input's own center, and `tapAt` dispatches a screen point, so the point + // comes from `frameOnScreen` (#3899). let inputCenter = input.map { $0.frameOnScreen }.flatMap { $0.isEmpty ? nil : CGPoint(x: $0.midX, y: $0.midY) } @@ -205,8 +201,6 @@ extension RunnerTests { contentTranslation: CGSize? ) -> TextEntryTarget { guard let element else { - // No element: the target keeps its refresh point, and the reads a later resolve makes through - // that point carry this command's vector (#3899). return TextEntryTarget( element: nil, refreshPoint: requestedPoint, @@ -249,22 +243,19 @@ extension RunnerTests { /// the target gets NO refresh point rather than the pre-focus one: this route's first resolve is /// not yet bound to an identity, and a field that went unqueryable on focus may have moved the /// layout so the old point names a different input, so fail closed with a typed reason instead of - /// typing into that other field. The frame is the read's snapshot frame plus its content vector - /// (#3899) — never the live `element.frame` fallback `frameOnScreen` uses for a snapshotless read. + /// typing into that other field. The frame is `read`'s snapshot frame plus its content vector, + /// never the live `element.frame` (#3899, #3060). private func textEntryRefreshPoint( read: ResolvedTextEntryInput?, fallbackTo requestedPoint: CGPoint ) -> CGPoint? { - guard let snapshot = read?.snapshot else { + guard let read, read.snapshot != nil else { return nil } - guard !snapshot.frame.isEmpty else { + let frame = read.frameOnScreen + guard !frame.isEmpty else { return requestedPoint } - let frame = snapshot.frame.offsetBy( - dx: read?.contentTranslation?.width ?? 0, - dy: read?.contentTranslation?.height ?? 0 - ) return CGPoint(x: frame.midX, y: frame.midY) } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift index 52bd98268d..a8f19d0d3a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift @@ -86,19 +86,15 @@ extension RunnerTests { /// wait rather than reused, because focusing a field moves the layout: the repeat aims at where the /// field is then, which is the moment the read it replaced ran at. /// - /// The read enters `retapAt` as a `ResolvedTextEntryInput`, so the repeat taps - /// `frameOnScreen` — the snapshot frame plus the content vector of a placed window (#3899). The base - /// read carries no vector; the re-read carries the live one, read only on this branch, so a tap that - /// confirms focus dispatches no window lookup. + /// The re-read carries the live content vector, so the repeat taps a screen point (#3899). It is + /// read only on this branch: a tap that confirms focus dispatches no window lookup. func waitForTextEntryReadinessAfterTap(app: XCUIApplication, element: XCUIElement) { #if os(iOS) if waitForTextEntryFocus(app: app, element: element, timeout: TextEntryTiming.readinessTimeout) { return } - // The re-read enters as a `ResolvedTextEntryInput` carrying the content vector of a placed - // window (#3899), read only on this branch, so a tap that confirms focus dispatches no window - // lookup. The window read is contained like the probe lookups: a transient window that records - // an XCTest issue costs the vector, not the command. + // Contained like the probe lookups: a transient window that records an XCTest issue costs the + // vector, not the command. let settled: ResolvedTextEntryInput? switch containingTextInputProbeIssues(fallback: nil as ResolvedTextEntryInput?, { _ in ResolvedTextEntryInput(reading: element, contentTranslation: liveContentTranslation(app: app)) @@ -116,9 +112,8 @@ extension RunnerTests { } /// The readiness retap, split out so the unit lane can drive it and assert where the point lands. - /// `tapAt` takes a screen point, so the point is `frameOnScreen`'s center. A read with no snapshot - /// taps nothing: `frameOnScreen`'s fallback is the live `element.frame` read — the one that records - /// an XCTest failure on a handle that stopped answering its query (#3060). + /// A read with no snapshot taps nothing: its `frameOnScreen` is the live `element.frame`, the read + /// a handle that stopped answering its query cannot make without recording a failure (#3060). func retapAt(read: ResolvedTextEntryInput, app: XCUIApplication, element: XCUIElement) { guard read.snapshot != nil else { return @@ -159,8 +154,7 @@ extension RunnerTests { /// fallback at all. func keyboardFocusConfirmed(app: XCUIApplication, element: XCUIElement) -> Bool { #if os(iOS) - // The read needs no content vector: this corroborates identity and position raw-frame against - // raw-frame, and never turns a frame into a dispatched point (#3899's rule). + // No content vector: this corroborates raw frame against raw frame and dispatches no point. return ResolvedTextEntryInput(reading: element).map { keyboardFocusConfirmed(app: app, read: $0) } ?? false #else return false diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift index 39f0e8b6d6..40346eef1e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift @@ -24,7 +24,8 @@ extension RunnerTests { y: Double, contentTranslation: CGSize? ) -> ResolvedTextEntryInput? { - textInputCandidatesAt(app: app, point: CGPoint(x: x, y: y), contentTranslation: contentTranslation).first + let point = CGPoint(x: x, y: y) + return textInputCandidatesAt(app: app, point: point, contentTranslation: contentTranslation).first } func textInputCandidatesAt( @@ -134,8 +135,7 @@ extension RunnerTests { } /// Runs an optional probe whose XCTest issues are contained instead of recorded: an issue makes - /// the probe unavailable rather than failing the command and invalidating the runner. Visible to - /// the readiness retap, whose window re-read needs the same containment (#3899). + /// the probe unavailable rather than failing the command and invalidating the runner. func containingTextInputProbeIssues( fallback: T, _ probe: (_ shouldStop: () -> Bool) -> T @@ -215,20 +215,15 @@ extension RunnerTests { } /// The vector that carries this session's snapshot-read app-content frames to their place on the - /// screen, or nil when the app fills the screen (#3899). Read once at the start of each command - /// that dispatches points derived from snapshot frames — the focus step, the coordinate fill route, - /// the readiness retap, a coordinate text read — and carried on the types the command builds from - /// it (`TextEntryTarget.contentTranslation`, `ResolvedTextEntryInput.contentTranslation`). No read - /// inside a loop re-reads it: the whole-app queries below cost about 90 ms each on heavy React - /// Native screens, and a per-read re-query would undo the fill-read work of #3881, #3906 and #3909. + /// screen, or nil when the app fills the screen (#3899). Each command that dispatches points from + /// snapshot frames reads it once, at its own start, and hands it down; a read inside a typing loop + /// never calls this, because the two live box reads below cost about 90 ms each on a heavy React + /// Native screen. /// - /// The live `XCUIElement.frame` property already reports a placed window's content in screen space, - /// but `XCUIElementSnapshot.frame` reports it in the window's own content space, so the same probe - /// sees two spaces on an iPad Windowed Apps window. The signature and the vector are the capture's - /// rule (`SnapshotContentTranslation`, ADR 0004) read from the two live boxes that name it: the - /// application's `frame` is a `(0,0)`-anchored copy of the box the resolved window reports at its - /// place on the screen. Any other pair of boxes declares no vector, so full-screen, iPhone, and - /// each foldable panel keep their frames exactly as read. + /// The signature and the vector are the capture's rule (`SnapshotContentTranslation`, ADR 0004) + /// read from the two live boxes that name it: the application's `frame` is a `(0,0)`-anchored copy + /// of the box the resolved window reports at its place on the screen. Any other pair of boxes + /// declares no vector, so full-screen, iPhone, and each foldable panel keep frames as read. func liveContentTranslation(app: XCUIApplication) -> CGSize? { // The window read is contained like the probe lookups it replaces: a transient window that // records an XCTest issue costs the vector, not the runner session. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift index bea8d91e60..8e49b4a3e7 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift @@ -27,8 +27,7 @@ extension RunnerTests { hasY: command.y != nil, xCTestChannelPenalized: xCTestChannelPenalized ), let x = command.x, let y = command.y { - // The one content-vector read for this command (#3899): the target built here carries it, and - // every read that target produces converts through it, as the focus route's targets do. + // The one content-vector read for this command (#3899); the target built here carries it. let contentTranslation = liveContentTranslation(app: activeApp) // Resolved before the tap: the focus it causes can move the field away from this point. // A replacement with no input at the point has nothing to bind to: its clear and its typed diff --git a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift index 95396fc3a4..90cd0d5488 100644 --- a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift +++ b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift @@ -5,9 +5,8 @@ import CoreGraphics // synthesized dispatch (forward) and capture (inverse). Rationale and measurements: ADR 0004 // "the coordinate space of a captured subtree" and contracts/fixtures/window-coordinate-space.json. // -// Rotation moves axes only. It never moves an origin: a placed window's offset from the screen -// belongs to `SnapshotContentTranslation` (#3899), which is the one rule that turns a reported box -// into its screen-space box. Keeping the two apart is what lets a capture publish one space. +// Rotation moves axes only and never an origin; a placed window's offset from the screen belongs +// to `SnapshotContentTranslation` (#3899). public enum RunnerInterfaceOrientation { public static let unknown = 0 @@ -18,11 +17,9 @@ public enum RunnerInterfaceOrientation { } public enum CoordinateSpaceRotation { - // Dispatch turns a point's axes and never moves its origin: the turn says which axis a box runs - // along, not where that box sits (#3899). The digitizer's origin is the display's own, which is - // where a windowed app's screen-space points are already measured from, so a placed window's - // offset from the screen belongs to `SnapshotContentTranslation` — the one rule that reads an - // offset — and not to this function. The `frame` argument names the side lengths the turn swaps. + // Dispatch turns a point's axes and never moves its origin: the digitizer's origin is the + // display's own, which is where a windowed app's screen-space points are already measured from + // (#3899). `frame` names the side lengths the turn swaps, nothing more. public static func native( point: CGPoint, in frame: CGRect, @@ -107,25 +104,16 @@ public enum CoordinateSpaceRotation { } } -/// Whether a carrier's own box declares that its subtree reports in the carrier's content space -/// rather than in the space the viewport names (#3899). +/// The vector from a carrier's reported box to the screen space it is placed at (#3899). /// -/// An iPad window that does not fill the screen (Windowed Apps, Stage Manager) reports its own frame -/// at its place on the screen while its subtree arrives anchored at its own top-left: measured on -/// iPad Pro 11-inch (M5, iOS 27.0) with a Calendar window whose `frame` is `(224,207,386,682)`, the -/// application root reports `(0,0,386,682)` and its Add button `(335,14,37,36)` at screen -/// `(559,221)`. A full-screen window, an iPhone, and each foldable panel report origins at zero on -/// both sides, so the rule reads zero translation there and every published rect is unchanged. -/// -/// The signature is the carrier's box as a `(0,0)`-anchored copy of the viewport box: the side -/// lengths match within the tolerance and the viewport — the app's screen-anchored box — starts -/// somewhere else on the screen. Any other carrier box names no space: it is not the app's own -/// window, so its numbers say nothing about where the app's subtree starts. +/// The signature is a `(0,0)`-anchored copy of the viewport box whose viewport starts elsewhere on +/// the screen; any other carrier box is not the app's own window, so its numbers say nothing about +/// where its subtree starts. Measured rows and the full rationale: ADR 0004 and the +/// `windowContentCases` table this replays against. public enum SnapshotContentTranslation { - /// The vector from a carrier's reported box to the screen space it is placed at, or `nil` when the - /// carrier's box declares no translation. One box offset from one viewport box, decided from the - /// same tolerance the quarter-turn rule reads; replayed against - /// `contracts/fixtures/window-coordinate-space.json` in `windowContentCases`. + /// `nil` when the carrier's box declares no translation. A full-screen app, an iPhone, and each + /// foldable panel report origins at zero on both sides, so the rule reads none there and every + /// published rect stays as the platform reported it. public static func offset(reportedBox: CGRect, viewportBox: CGRect) -> CGSize? { guard SnapshotGeometry.isPositiveFinite(reportedBox), SnapshotGeometry.isPositiveFinite(viewportBox) @@ -176,10 +164,8 @@ public enum SnapshotGeometrySpace: Equatable { } } - /// A carrier declares its own subtree's space from its own box. A `.reported` viewport is the - /// screen-anchored fact both declarations read against: a quarter turn needs its orientation, and - /// a window-content offset needs its origin. A `.derived` viewport carries no screen anchor, so a - /// subtree under it publishes what the platform reported rather than a guessed space. + /// A `.reported` viewport is the screen-anchored fact both declarations read against; a `.derived` + /// viewport carries no screen anchor, so a subtree under it publishes what the platform reported. public static func space( reportedBySurfaceHost isSurfaceHost: Bool, reportedFrame: CGRect, @@ -200,11 +186,9 @@ public enum SnapshotGeometrySpace: Equatable { ) { return .windowContent(offset: offset) } - // A host that declares no space of its own publishes what the platform reported. A system - // surface under a placed window — the keyboard at (0,925,834,282), a SpringBoard overlay at - // (0,0,834,1210) — reports the screen's own box and is already in screen space; the two - // `windowContentCases` rows for those windows pin that inheriting the carrier's vector would - // move every key by the window origin (#3899's own failure shape). + // A system surface under a placed window — the keyboard, a SpringBoard overlay — reports the + // screen's own box and is already in screen space; `windowContentCases` pins that inheriting + // the carrier's vector would move every key by the window origin. return .appOrientation } diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index a1dfccf286..67d1a05ad8 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -651,7 +651,7 @@ Any other carrier box names no space, so the rule leaves its numbers alone: the A full-screen window, an iPhone, and each foldable panel of ADR 0025 report origins at zero on both sides, so the rule reads no vector there and every rect is unchanged. `SnapshotContentTranslation.offset(reportedBox:viewportBox:)` is that rule, replayed from `contracts/fixtures/window-coordinate-space.json` in the capture's language alone: the host bridge refuses a windowed tree (`app-window-not-full-screen`) before its decoder can read one, so the translation has one producer-side owner and the host's parity replay covers only the rules that decoder reads. -Rotation and translation stay separate rules, and the split is the point. +Rotation and translation stay separate rules. `CoordinateSpaceRotation` turns a box's axes and never moves its origin: the digitizer's origin is the display's own, which is where a placed window's screen-space points are already measured from. Before this amendment, that function localized a non-zero frame origin away and never added it back. Every configuration measured before this one has its app frame at the screen origin, so the localization was dead code there and the dispatch bug here. diff --git a/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts b/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts index b181eef498..ee97982866 100644 --- a/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts +++ b/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts @@ -23,7 +23,7 @@ import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; * reported, that tree mixes two spaces, clips its own content, and moves every tap by the window * origin (#3899). That translation belongs to the capture that performs it — the runner's Swift * `SnapshotContentTranslation` — not to this decoder, because the guest refuses a windowed tree - * before this file can read one; see the note at the foot of this file. + * before this file can read one. * * A rule that reads a box reads what the box can say. The quarter turn is a statement about which * axis a box runs along, so it compares side lengths and nothing else. A content translation is a @@ -73,8 +73,3 @@ export function isQuarterTurnedWindowFrame(windowFrame: Rect, appFrame: Rect): b Math.abs(windowFrame.height - appFrame.width) <= WINDOW_QUARTER_TURN_TOLERANCE ); } - -/* - * The #3899 window-content translation belongs to the Swift capture's `SnapshotContentTranslation`, - * replayed from `windowContentCases` there alone. - */ From de2778e6f373eb8ab7e22c3d85c13c8f7baccfc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 25 Oct 2026 10:51:52 +0100 Subject: [PATCH 10/15] refactor(runner): make the text-entry content vector a non-defaulted tri-state frameOnScreen becomes optional and carries the #3060 rule: it names no box without a snapshot or under an unknown content space, so the four consumer guards collapse into one nil check. liveContentTranslation returns TextEntryContentTranslation (known/none/unknown) instead of CGSize?, so a failed window read can no longer look like full screen and send the readiness retap to a raw window-local point. The vector now rides on TextInputAtPoint, cutting one parameter from the probe signatures. retapAt skips the point on unknown; probe matching and the focus tap fall back to element.frame. Adds the unknown-space retap unit test. --- .../RunnerTests+TextEntry.swift | 80 ++++++++++++------- .../RunnerTests+TextEntryFocus.swift | 51 +++++++----- .../RunnerTests+TextEntryReadiness.swift | 25 +++--- .../RunnerTests+TextInputProbe.swift | 61 ++++++++------ ...RunnerTests+CoordinateTextEntryTests.swift | 2 +- ...unnerTests+SynthesizedTextEntryTests.swift | 15 +++- .../RunnerTests+TextEntryLatencyTests.swift | 8 +- .../RunnerTests+TextEntryPolicyTests.swift | 8 +- .../RunnerTests+TextEntryReadinessTests.swift | 30 +++++-- .../RunnerTests+TextInputProbeTests.swift | 8 +- .../RunnerTests+TextTypingTests.swift | 2 + .../adr/0004-ios-snapshot-backend-strategy.md | 7 +- 12 files changed, 191 insertions(+), 106 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index c198ea2534..ce8756c135 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -137,22 +137,37 @@ extension RunnerTests { var isDistinguishable: Bool { !identifier.isEmpty } } - /// A text input found under a point, and the identity it carried when found. The element is an - /// index-bound query handle: an input inserted or reordered ahead of it later re-binds the handle - /// to that input, and the identity is what notices. + /// A text input found under a point, the identity it carried when found, and the content vector of + /// the command that found it. The element is an index-bound query handle: an input inserted or + /// reordered ahead of it later re-binds the handle to that input, and the identity is what notices. struct TextInputAtPoint { let element: XCUIElement let identity: TextEntryInputIdentity + let contentTranslation: TextEntryContentTranslation + } + + /// The vector that carries a snapshot-read frame to its place on the screen (#3899), read once per + /// command by `liveContentTranslation`. Non-defaulted everywhere it travels: a vector nobody read + /// must never read as full screen, because on a placed window that silently dispatches window-local + /// points as screen points — the #3899 failure. + enum TextEntryContentTranslation { + /// The vector of a placed window: its content box's origin on the screen. + case known(CGSize) + /// The app fills the screen, so a snapshot frame is already a screen frame. + case none + /// The window read could not answer, so the space of a snapshot frame is unknown. Consumers + /// refuse it as a screen frame: `frameOnScreen` reads nil and nothing dispatches a point from it. + case unknown } struct TextEntryTarget { let element: XCUIElement? let refreshPoint: CGPoint? let prefersFocusedElement: Bool - /// The content vector of a placed window, read once for the command that built this target and - /// handed to every read it produces, which turns its snapshot frames into screen points (#3899). - /// Nil when the app fills the screen. - let contentTranslation: CGSize? + /// The content vector of the command that built this target, handed to every read it produces, + /// which turns their snapshot frames into screen points (#3899). Non-defaulted: the route that + /// reads no window carries `.unknown`, not a vector that would read as full screen. + let contentTranslation: TextEntryContentTranslation let fromTapWitness: Bool /// The input the first resolved element was. Once bound, resolution refuses any other input. let boundIdentity: TextEntryInputIdentity? @@ -170,7 +185,7 @@ extension RunnerTests { element: XCUIElement?, refreshPoint: CGPoint?, prefersFocusedElement: Bool, - contentTranslation: CGSize? = nil, + contentTranslation: TextEntryContentTranslation, fromTapWitness: Bool = false, boundIdentity: TextEntryInputIdentity? = nil, inputAtRefreshPoint: TextInputAtPoint? = nil, @@ -187,14 +202,14 @@ extension RunnerTests { } /// The target moved to the input a resolution read: its element, a refresh point at its frame's - /// center on the screen, and its identity while the target is unbound. A read with no snapshot has - /// no frame here, so its center cannot come from the live `element.frame` — the read a post-focus - /// handle cannot answer without recording a failure (#3060). + /// center on the screen, and its identity while the target is unbound. A read that cannot name a + /// screen box — no snapshot, or an unknown content space — keeps the target's current refresh + /// point rather than a center it cannot trust. func withResolvedInput(_ input: ResolvedTextEntryInput?) -> TextEntryTarget { guard let input else { return self } - let frame = input.snapshot == nil ? .zero : input.frameOnScreen + let frame = input.frameOnScreen ?? .zero return TextEntryTarget( element: input.element, refreshPoint: frame.isEmpty ? refreshPoint : CGPoint(x: frame.midX, y: frame.midY), @@ -221,10 +236,10 @@ extension RunnerTests { let element: XCUIElement /// Nil when the element exists but could not be snapshotted. let snapshot: XCUIElementSnapshot? - /// The vector of the command that made this read (#3899); nil when the app fills the screen. - let contentTranslation: CGSize? + /// The vector state of the command that made this read (#3899). + let contentTranslation: TextEntryContentTranslation - private init(element: XCUIElement, snapshot: XCUIElementSnapshot?, contentTranslation: CGSize?) { + private init(element: XCUIElement, snapshot: XCUIElementSnapshot?, contentTranslation: TextEntryContentTranslation) { self.element = element self.snapshot = snapshot self.contentTranslation = contentTranslation @@ -233,7 +248,7 @@ extension RunnerTests { /// Reads a handle that may name an input the app has removed; nil when it names none or the read /// raised. The snapshot of a missing element throws only after XCTest waits about two seconds for /// a match, so existence is read first. - init?(reading element: XCUIElement, contentTranslation: CGSize? = nil) { + init?(reading element: XCUIElement, contentTranslation: TextEntryContentTranslation) { var exists = false var snapshot: XCUIElementSnapshot? if let exception = RunnerObjCExceptionCatcher.catchException({ @@ -249,7 +264,7 @@ extension RunnerTests { /// Reads a handle its query has just listed, inside the caller's exception containment, so the /// snapshot is read first and existence only when the snapshot fails; nil when it names no element. - init?(readingListed element: XCUIElement, contentTranslation: CGSize? = nil) { + init?(readingListed element: XCUIElement, contentTranslation: TextEntryContentTranslation) { let snapshot = try? element.snapshot() guard snapshot != nil || element.exists else { return nil } self.init(element: element, snapshot: snapshot, contentTranslation: contentTranslation) @@ -266,17 +281,25 @@ extension RunnerTests { return element } - /// The input's box in screen space: the snapshot frame plus `contentTranslation`, or the live - /// frame when there is no snapshot — that property already reports a placed window's content in - /// screen space (#3899). - var frameOnScreen: CGRect { + /// The input's box in screen space, or nil when this read cannot name one: no snapshot, or a + /// window read that could not answer, so the snapshot frame's space is unknown. A nil here is the + /// one signal every consumer uses to refuse a dispatched point: the live `element.frame` is a + /// read a handle that stopped answering its query cannot make without recording a failure + /// (#3060), and an unknown space must never be dispatched as a screen point (#3899). Callers + /// that need a box for a route which dispatches nothing fall back to `element.frame`, which + /// already reports a placed window's content in screen space. + var frameOnScreen: CGRect? { guard let snapshot else { - return element.frame + return nil } - guard let contentTranslation else { + switch contentTranslation { + case .known(let offset): + return snapshot.frame.offsetBy(dx: offset.width, dy: offset.height) + case .none: return snapshot.frame + case .unknown: + return nil } - return snapshot.frame.offsetBy(dx: contentTranslation.width, dy: contentTranslation.height) } } @@ -539,7 +562,7 @@ extension RunnerTests { return input } if let input = target.inputAtRefreshPoint, - let read = textInputIfStillResolves(input, contentTranslation: target.contentTranslation) { + let read = textInputIfStillResolves(input) { return read } if target.inputAtRefreshPoint == nil, @@ -658,12 +681,11 @@ extension RunnerTests { // zero-size one (how a departed handle reads), there is no point to dispatch — an edge-tap // from a frame the field moved after records a failure or navigates away, the shape // `clearTextInput`'s comment refuses — so degrade to the caller's point-free delete burst. - // A snapshotless read gets no point at all: `frameOnScreen`'s fallback is the live - // `element.frame` read this function exists to avoid (#3060). - guard input.snapshot?.frame.isEmpty == false else { + // A snapshotless read gets no point at all: `frameOnScreen` is nil without a snapshot, so the + // live `element.frame` read this function exists to avoid is never reached (#3060). + guard let frame = input.frameOnScreen, !frame.isEmpty else { return } - let frame = input.frameOnScreen _ = tapAt( app: app, x: frame.minX + max(2, frame.width - 4), diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift index fb0c4592f8..55d852968f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift @@ -26,7 +26,9 @@ extension RunnerTests { } @MainActor - private func rememberedTextEntryTarget() -> TextEntryTarget? { + private func rememberedTextEntryTarget( + contentTranslation: TextEntryContentTranslation + ) -> TextEntryTarget? { guard let witness = textEntryTapWitness else { return nil } @@ -54,6 +56,7 @@ extension RunnerTests { element: element, refreshPoint: nil, prefersFocusedElement: false, + contentTranslation: contentTranslation, fromTapWitness: true ) } @@ -61,8 +64,8 @@ extension RunnerTests { func stabilizeTextInputBeforeTyping( app: XCUIApplication, target: XCUIElement?, - keyboardVisibleBeforeTap: Bool? = nil, - contentTranslation: CGSize? = nil + contentTranslation: TextEntryContentTranslation, + keyboardVisibleBeforeTap: Bool? = nil ) -> TextEntryStabilization { #if os(tvOS) return TextEntryStabilization(element: target, evidence: .focusEngine, polls: 0) @@ -104,9 +107,14 @@ extension RunnerTests { @MainActor func focusTextInputForTextEntry(app: XCUIApplication, x: Double?, y: Double?) -> TextEntryTarget { + // The one content-vector read for this command (#3899): every target built here carries it and + // every read those targets produce converts through it. + let contentTranslation = liveContentTranslation(app: app) guard let x, let y else { let softwareKeyboardVisible = isKeyboardVisible(app: app) - if !softwareKeyboardVisible, let rememberedTarget = rememberedTextEntryTarget() { + if !softwareKeyboardVisible, let rememberedTarget = rememberedTextEntryTarget( + contentTranslation: contentTranslation + ) { return rememberedTarget } // Bare `type` targets the current first responder. On iOS we intentionally do not let the @@ -117,7 +125,8 @@ extension RunnerTests { return TextEntryTarget( element: focusedTextInput(app: app), refreshPoint: nil, - prefersFocusedElement: true + prefersFocusedElement: true, + contentTranslation: contentTranslation ) } let focused = waitForTextEntryReadiness( @@ -125,22 +134,26 @@ extension RunnerTests { target: TextEntryTarget( element: focusedTextInput(app: app), refreshPoint: nil, - prefersFocusedElement: true + prefersFocusedElement: true, + contentTranslation: contentTranslation ) ) - return TextEntryTarget(element: focused, refreshPoint: nil, prefersFocusedElement: true) + return TextEntryTarget( + element: focused, + refreshPoint: nil, + prefersFocusedElement: true, + contentTranslation: contentTranslation + ) } - // The one content-vector read for this command (#3899): every target built here carries it and - // every read those targets produce converts through it. - let contentTranslation = liveContentTranslation(app: app) let keyboardVisibleBeforeTap = isKeyboardVisible(app: app) let input = textInputAt(app: app, x: x, y: y, contentTranslation: contentTranslation) let target = input?.element let requestedPoint = CGPoint(x: x, y: y) // The tap goes to the input's own center, and `tapAt` dispatches a screen point, so the point - // comes from `frameOnScreen` (#3899). - let inputCenter = input.map { $0.frameOnScreen }.flatMap { + // comes from `frameOnScreen`; the live frame is the box left when the command's window read + // could not answer (#3899). + let inputCenter = (input?.frameOnScreen ?? input?.element.frame).flatMap { $0.isEmpty ? nil : CGPoint(x: $0.midX, y: $0.midY) } if let inputCenter { @@ -161,8 +174,8 @@ extension RunnerTests { let stabilized = stabilizeTextInputBeforeTyping( app: app, target: target, - keyboardVisibleBeforeTap: keyboardVisibleBeforeTap, - contentTranslation: contentTranslation + contentTranslation: contentTranslation, + keyboardVisibleBeforeTap: keyboardVisibleBeforeTap ) NSLog( "%@%@ polls=%d", @@ -198,7 +211,7 @@ extension RunnerTests { private func focusedTextEntryTarget( _ element: XCUIElement?, requestedPoint: CGPoint, - contentTranslation: CGSize? + contentTranslation: TextEntryContentTranslation ) -> TextEntryTarget { guard let element else { return TextEntryTarget( @@ -221,7 +234,7 @@ extension RunnerTests { _ element: XCUIElement, read: ResolvedTextEntryInput?, requestedPoint: CGPoint, - contentTranslation: CGSize? + contentTranslation: TextEntryContentTranslation ) -> TextEntryTarget { return TextEntryTarget( element: element, @@ -244,15 +257,15 @@ extension RunnerTests { /// not yet bound to an identity, and a field that went unqueryable on focus may have moved the /// layout so the old point names a different input, so fail closed with a typed reason instead of /// typing into that other field. The frame is `read`'s snapshot frame plus its content vector, - /// never the live `element.frame` (#3899, #3060). + /// never the live `element.frame`, and `frameOnScreen` is nil for a read that cannot name a screen + /// box (#3899, #3060). private func textEntryRefreshPoint( read: ResolvedTextEntryInput?, fallbackTo requestedPoint: CGPoint ) -> CGPoint? { - guard let read, read.snapshot != nil else { + guard let frame = read?.frameOnScreen else { return nil } - let frame = read.frameOnScreen guard !frame.isEmpty else { return requestedPoint } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift index a8f19d0d3a..fb29f558c3 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift @@ -86,15 +86,16 @@ extension RunnerTests { /// wait rather than reused, because focusing a field moves the layout: the repeat aims at where the /// field is then, which is the moment the read it replaced ran at. /// - /// The re-read carries the live content vector, so the repeat taps a screen point (#3899). It is - /// read only on this branch: a tap that confirms focus dispatches no window lookup. + /// The re-read carries a fresh content-vector read, so the repeat taps a screen point (#3899). It + /// is read only on this branch: a tap that confirms focus dispatches no window lookup. func waitForTextEntryReadinessAfterTap(app: XCUIApplication, element: XCUIElement) { #if os(iOS) if waitForTextEntryFocus(app: app, element: element, timeout: TextEntryTiming.readinessTimeout) { return } - // Contained like the probe lookups: a transient window that records an XCTest issue costs the - // vector, not the command. + // Contained like the probe lookups: a transient window or a snapshot that records an XCTest + // issue costs the retap, not the command. A failed window read leaves the read's vector + // `.unknown`, which names no screen box, so `retapAt` taps nothing (#3899). let settled: ResolvedTextEntryInput? switch containingTextInputProbeIssues(fallback: nil as ResolvedTextEntryInput?, { _ in ResolvedTextEntryInput(reading: element, contentTranslation: liveContentTranslation(app: app)) @@ -112,14 +113,13 @@ extension RunnerTests { } /// The readiness retap, split out so the unit lane can drive it and assert where the point lands. - /// A read with no snapshot taps nothing: its `frameOnScreen` is the live `element.frame`, the read - /// a handle that stopped answering its query cannot make without recording a failure (#3060). + /// The retap taps nothing when the read cannot name a screen box: with no snapshot that box is the + /// live `element.frame`, the read a handle that stopped answering its query cannot make without + /// recording a failure (#3060); with an unknown content space a snapshot frame is not known to be + /// a screen box, and a raw window-local point is exactly the #3899 failure. The probe-side rule + /// `clearTextInput` and `moveCaretToEnd` settled on: skip the point, never guess through it. func retapAt(read: ResolvedTextEntryInput, app: XCUIApplication, element: XCUIElement) { - guard read.snapshot != nil else { - return - } - let frame = read.frameOnScreen - guard !frame.isEmpty else { + guard let frame = read.frameOnScreen, !frame.isEmpty else { return } _ = tapAt(app: app, x: frame.midX, y: frame.midY) @@ -155,7 +155,8 @@ extension RunnerTests { func keyboardFocusConfirmed(app: XCUIApplication, element: XCUIElement) -> Bool { #if os(iOS) // No content vector: this corroborates raw frame against raw frame and dispatches no point. - return ResolvedTextEntryInput(reading: element).map { keyboardFocusConfirmed(app: app, read: $0) } ?? false + return ResolvedTextEntryInput(reading: element, contentTranslation: .unknown) + .map { keyboardFocusConfirmed(app: app, read: $0) } ?? false #else return false #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift index 40346eef1e..6e6b2a5db2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift @@ -22,7 +22,7 @@ extension RunnerTests { app: XCUIApplication, x: Double, y: Double, - contentTranslation: CGSize? + contentTranslation: TextEntryContentTranslation ) -> ResolvedTextEntryInput? { let point = CGPoint(x: x, y: y) return textInputCandidatesAt(app: app, point: point, contentTranslation: contentTranslation).first @@ -31,7 +31,7 @@ extension RunnerTests { func textInputCandidatesAt( app: XCUIApplication, point: CGPoint, - contentTranslation: CGSize? + contentTranslation: TextEntryContentTranslation ) -> [ResolvedTextEntryInput] { safely("TEXT_INPUT_AT_POINT", []) { queryTextInputs(app: app, point: point, contentTranslation: contentTranslation, shouldStop: { false }) @@ -42,7 +42,7 @@ extension RunnerTests { app: XCUIApplication, x: Double, y: Double, - contentTranslation: CGSize? + contentTranslation: TextEntryContentTranslation ) -> XCUIElement? { switch probeTextInputs(app: app, point: CGPoint(x: x, y: y), contentTranslation: contentTranslation) { case .matches(let inputs): @@ -70,7 +70,7 @@ extension RunnerTests { app: XCUIApplication, x: Double, y: Double, - contentTranslation: CGSize? + contentTranslation: TextEntryContentTranslation ) -> CoordinateTextInputResolution { let probed = containingTextInputProbeIssues(fallback: CoordinateTextInputResolution.unavailable) { shouldStop -> CoordinateTextInputResolution in @@ -85,7 +85,11 @@ extension RunnerTests { guard let identity = input.identity else { return .unavailable } - return .input(TextInputAtPoint(element: input.element, identity: identity)) + return .input(TextInputAtPoint( + element: input.element, + identity: identity, + contentTranslation: contentTranslation + )) } switch probed { case .success(let resolution): @@ -96,13 +100,13 @@ extension RunnerTests { } /// `input`'s handle, read again, when it still resolves to the input it named when found, read - /// inside the same issue containment. A read that cannot answer counts as no. + /// inside the same issue containment, carrying the vector the finding command read. A read that + /// cannot answer counts as no. func textInputIfStillResolves( - _ input: TextInputAtPoint, - contentTranslation: CGSize? + _ input: TextInputAtPoint ) -> ResolvedTextEntryInput? { let probed = containingTextInputProbeIssues(fallback: nil as ResolvedTextEntryInput?) { _ in - ResolvedTextEntryInput(reading: input.element, contentTranslation: contentTranslation) + ResolvedTextEntryInput(reading: input.element, contentTranslation: input.contentTranslation) .flatMap { $0.identity == input.identity ? $0 : nil } } guard case .success(let read) = probed else { return nil } @@ -122,7 +126,7 @@ extension RunnerTests { func probeTextInputs( app: XCUIApplication, point: CGPoint, - contentTranslation: CGSize? + contentTranslation: TextEntryContentTranslation ) -> TextInputProbeOutcome { switch containingTextInputProbeIssues(fallback: [], { shouldStop in queryTextInputs(app: app, point: point, contentTranslation: contentTranslation, shouldStop: shouldStop) @@ -180,7 +184,7 @@ extension RunnerTests { private func queryTextInputs( app: XCUIApplication, point: CGPoint, - contentTranslation: CGSize?, + contentTranslation: TextEntryContentTranslation, shouldStop: () -> Bool ) -> [ResolvedTextEntryInput] { var inputs: [ResolvedTextEntryInput] = [] @@ -203,7 +207,10 @@ extension RunnerTests { ) else { continue } if isCoordinateTextInputCandidate( enabled: candidate.attributes.isEnabled, - frame: candidate.frameOnScreen, + // A candidate with no snapshot is matched on its live frame, which is already screen space + // and whose failure reaches the caller's issue handling. `frameOnScreen` is nil only when + // the command's window read could not answer, so the live frame is the only box left. + frame: candidate.frameOnScreen ?? candidate.element.frame, point: point ) { inputs.append(candidate) @@ -214,28 +221,36 @@ extension RunnerTests { return inputs.sorted { smallestElementFirst($0.attributes, $1.attributes) } } - /// The vector that carries this session's snapshot-read app-content frames to their place on the - /// screen, or nil when the app fills the screen (#3899). Each command that dispatches points from - /// snapshot frames reads it once, at its own start, and hands it down; a read inside a typing loop - /// never calls this, because the two live box reads below cost about 90 ms each on a heavy React - /// Native screen. + /// The vector state that carries this session's snapshot-read app-content frames to their place on + /// the screen (#3899). Each command that dispatches points from snapshot frames reads it once, at + /// its own start, and hands it down; a read inside a typing loop never calls this, because the two + /// live box reads below cost about 90 ms each on a heavy React Native screen. /// /// The signature and the vector are the capture's rule (`SnapshotContentTranslation`, ADR 0004) /// read from the two live boxes that name it: the application's `frame` is a `(0,0)`-anchored copy /// of the box the resolved window reports at its place on the screen. Any other pair of boxes /// declares no vector, so full-screen, iPhone, and each foldable panel keep frames as read. - func liveContentTranslation(app: XCUIApplication) -> CGSize? { + /// + /// A read that cannot answer returns `.unknown`, never `.none`: on a placed window, a failed read + /// that looked like full screen would dispatch raw window-local points as screen points — the + /// #3899 failure — so consumers refuse `.unknown` instead of guessing through it. + func liveContentTranslation(app: XCUIApplication) -> TextEntryContentTranslation { // The window read is contained like the probe lookups it replaces: a transient window that // records an XCTest issue costs the vector, not the runner session. - let read = containingTextInputProbeIssues(fallback: nil as CGSize?) { _ in + let read = containingTextInputProbeIssues(fallback: nil as TextEntryContentTranslation?) { _ in guard let resolved = resolveRunnerWindowElement(app: app) else { return nil } - return SnapshotContentTranslation.offset(reportedBox: app.frame, viewportBox: resolved.frame) + guard let translation = SnapshotContentTranslation.offset(reportedBox: app.frame, viewportBox: resolved.frame) else { + // Explicit type: `.none` alone means `Optional.none` to the compiler here. + return TextEntryContentTranslation.none + } + return .known(translation) } switch read { - case .success(let translation): - return translation + case .success(let state): + guard let state else { return .unknown } + return state case .failure: - return nil + return .unknown } } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift index 60979694f1..342bfc3e80 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift @@ -6,7 +6,7 @@ extension RunnerTests { /// cannot answer. A fixture that does not expose its own field is a broken lane, not a refusal. @MainActor func requireTextInputAtPoint(app: XCUIApplication, x: Double, y: Double) throws -> TextInputAtPoint { - switch coordinateTapTextInputIdentityAt(app: app, x: x, y: y, contentTranslation: nil) { + switch coordinateTapTextInputIdentityAt(app: app, x: x, y: y, contentTranslation: .none) { case .input(let input): return input case .absent: diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift index 5e78b2ad30..3b7174b672 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift @@ -232,6 +232,7 @@ extension RunnerTests { element: nil, refreshPoint: CGPoint(x: frame.midX, y: frame.midY), prefersFocusedElement: false, + contentTranslation: .none, inputAtRefreshPoint: found ), text: "fresh", @@ -315,10 +316,11 @@ extension RunnerTests { XCTAssertEqual(rebound.identifier, "agent-device-text-entry-neighbour") let neighbourFrame = neighbour.frame let found = [ - "re-bound": TextInputAtPoint(element: rebound, identity: TextEntryInputIdentity(elementType: .textField, identifier: "")), + "re-bound": TextInputAtPoint(element: rebound, identity: TextEntryInputIdentity(elementType: .textField, identifier: ""), contentTranslation: .none), "missing": TextInputAtPoint( element: app.textFields["agent-device-gone"], - identity: TextEntryInputIdentity(elementType: .textField, identifier: "agent-device-gone") + identity: TextEntryInputIdentity(elementType: .textField, identifier: "agent-device-gone"), + contentTranslation: .none ), ] @@ -329,6 +331,7 @@ extension RunnerTests { element: nil, refreshPoint: CGPoint(x: neighbourFrame.midX, y: neighbourFrame.midY), prefersFocusedElement: false, + contentTranslation: .none, inputAtRefreshPoint: inputAtRefreshPoint ), text: "", @@ -361,9 +364,11 @@ extension RunnerTests { element: nil, refreshPoint: CGPoint(x: neighbourFrame.midX, y: neighbourFrame.midY), prefersFocusedElement: false, + contentTranslation: .none, inputAtRefreshPoint: TextInputAtPoint( element: rebound, - identity: TextEntryInputIdentity(elementType: .textField, identifier: field.identifier) + identity: TextEntryInputIdentity(elementType: .textField, identifier: field.identifier), + contentTranslation: .none ) ), text: "", @@ -395,9 +400,11 @@ extension RunnerTests { element: nil, refreshPoint: CGPoint(x: neighbourFrame.midX, y: neighbourFrame.midY), prefersFocusedElement: false, + contentTranslation: .none, inputAtRefreshPoint: TextInputAtPoint( element: app.textFields.element, - identity: TextEntryInputIdentity(elementType: .textField, identifier: "") + identity: TextEntryInputIdentity(elementType: .textField, identifier: ""), + contentTranslation: .none ) ), text: "", diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift index 9d27049ec2..a329e86a27 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift @@ -81,9 +81,9 @@ extension RunnerTests { let value = body() return (value, try XCTUnwrap(Int(valueReads.label)) - before) } - let target = TextEntryTarget(element: nil, refreshPoint: point, prefersFocusedElement: false) + let target = TextEntryTarget(element: nil, refreshPoint: point, prefersFocusedElement: false, contentTranslation: .none) - let (lookup, lookupReads) = try countingReads { textInputAt(app: app, x: point.x, y: point.y, contentTranslation: nil) } + let (lookup, lookupReads) = try countingReads { textInputAt(app: app, x: point.x, y: point.y, contentTranslation: .none) } let (input, resolutionReads) = try countingReads { resolveTextEntryInput(app: app, target: target) } XCTAssertEqual(lookup?.snapshot?.identifier, "agent-device-hardware-keyboard-input") @@ -112,8 +112,8 @@ extension RunnerTests { let stabilized = stabilizeTextInputBeforeTyping( app: app, target: field, - keyboardVisibleBeforeTap: false, - contentTranslation: nil + contentTranslation: .none, + keyboardVisibleBeforeTap: false ) guard case .inputReportsKeyboardFocus(let read) = stabilized.evidence else { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift index 02e1978022..20f62e56b9 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift @@ -739,6 +739,7 @@ extension RunnerTests { element: nil, refreshPoint: CGPoint(x: frame.midX, y: frame.midY), prefersFocusedElement: false, + contentTranslation: .none, inputAtRefreshPoint: try requireTextInputAtPoint(app: app, x: frame.midX, y: frame.midY) ), text: "abc", @@ -785,7 +786,8 @@ extension RunnerTests { target: TextEntryTarget( element: nil, refreshPoint: CGPoint(x: 10, y: 20), - prefersFocusedElement: false + prefersFocusedElement: false, + contentTranslation: .none ), text: "abc", delaySeconds: 0, @@ -810,7 +812,7 @@ extension RunnerTests { func testEmptyReplacementWithoutResolvableTargetFailsClosed() { let result = typeTextReliably( app: springboard, - target: TextEntryTarget(element: nil, refreshPoint: nil, prefersFocusedElement: false), + target: TextEntryTarget(element: nil, refreshPoint: nil, prefersFocusedElement: false, contentTranslation: .none), text: "", delaySeconds: 0, repairMode: .replacement, @@ -830,7 +832,7 @@ extension RunnerTests { XCTAssertEqual( awaitSynthesizedReplacementCommit( app: XCUIApplication(), - target: TextEntryTarget(element: nil, refreshPoint: CGPoint(x: 10, y: 20), prefersFocusedElement: false), + target: TextEntryTarget(element: nil, refreshPoint: CGPoint(x: 10, y: 20), prefersFocusedElement: false, contentTranslation: .none), expectedText: expectedText ), .unobservable diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryReadinessTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryReadinessTests.swift index 415a55a4b7..54940d964b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryReadinessTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryReadinessTests.swift @@ -30,9 +30,9 @@ extension RunnerTests { XCTAssertFalse(fieldFrame.isEmpty) XCTAssertFalse(witnessFrame.isEmpty) - // Closest negative: a read with no vector — a full-screen app — retaps the field's own center, + // Closest negative: a full-screen app — a vector of `.none` — retaps the field's own center, // which is not the button's point. - let untranslated = try XCTUnwrap(ResolvedTextEntryInput(reading: field, contentTranslation: nil)) + let untranslated = try XCTUnwrap(ResolvedTextEntryInput(reading: field, contentTranslation: .none)) retapAt(read: untranslated, app: app, element: field) XCTAssertEqual(presses.label, "presses=0", "a vectorless read dispatched a point off the field") @@ -42,9 +42,9 @@ extension RunnerTests { width: witnessFrame.midX - fieldFrame.midX, height: witnessFrame.midY - fieldFrame.midY ) - let translated = try XCTUnwrap(ResolvedTextEntryInput(reading: field, contentTranslation: vector)) - XCTAssertEqual(translated.frameOnScreen.midX, witnessFrame.midX, accuracy: 0.5) - XCTAssertEqual(translated.frameOnScreen.midY, witnessFrame.midY, accuracy: 0.5) + let translated = try XCTUnwrap(ResolvedTextEntryInput(reading: field, contentTranslation: .known(vector))) + XCTAssertEqual(try XCTUnwrap(translated.frameOnScreen).midX, witnessFrame.midX, accuracy: 0.5) + XCTAssertEqual(try XCTUnwrap(translated.frameOnScreen).midY, witnessFrame.midY, accuracy: 0.5) retapAt(read: translated, app: app, element: field) XCTAssertEqual( @@ -53,5 +53,25 @@ extension RunnerTests { "the retap dispatched the raw snapshot center, not the read's screen-space center (#3899)" ) } + + /// A vector read that could not answer is `.unknown`, and `frameOnScreen` names no box through it: + /// a snapshot frame of a placed window is not known to be a screen box, so the readiness retap + /// dispatches nothing rather than a raw window-local point — the #3899 failure shape. A transient + /// window issue used to look like full screen here and move the retap outside the window. + @MainActor + func testReadinessRetapTapsNothingWhenTheContentSpaceIsUnknown() throws { + app.launchArguments = ["--agent-device-text-entry-regression", "--agent-device-text-entry-tap-witness"] + app.launch() + defer { invalidateCachedTarget(reason: "unit_test_cleanup"); app.terminate() } + XCTAssertTrue(app.waitForFixtureExistence(timeout: appExistenceTimeout)) + let field = app.textFields["agent-device-hardware-keyboard-input"] + XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout)) + let presses = app.staticTexts["agent-device-tap-witness-presses"] + XCTAssertTrue(presses.waitForFixtureExistence(timeout: appExistenceTimeout)) + let read = try XCTUnwrap(ResolvedTextEntryInput(reading: field, contentTranslation: .unknown)) + XCTAssertNil(read.frameOnScreen) + retapAt(read: read, app: app, element: field) + XCTAssertEqual(presses.label, "presses=0", "an unknown content space dispatched a point (#3899)") + } #endif } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift index 6c7b54b49a..5d1fe3a468 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift @@ -15,7 +15,7 @@ extension RunnerTests { let completed = expectation(description: "optional probe completed inside runner wait") DispatchQueue.main.async { self.textInputProbeIssueForTesting = XCTIssue(type: .assertionFailure, compactDescription: "Optional probe issue during runner wait") - _ = self.probeTextInputs(app: self.app, point: point, contentTranslation: nil) + _ = self.probeTextInputs(app: self.app, point: point, contentTranslation: .none) completed.fulfill() } guard XCTWaiter.wait(for: [completed], timeout: 5) == .completed else { @@ -174,8 +174,8 @@ extension RunnerTests { } XCTExpectFailure("Required read and later issue belong to their caller", options: options) { textInputProbeIssueForTesting = expected - _ = textInputAt(app: app, x: point.x, y: point.y, contentTranslation: nil) - _ = probeTextInputs(app: app, point: point, contentTranslation: nil) + _ = textInputAt(app: app, x: point.x, y: point.y, contentTranslation: .none) + _ = probeTextInputs(app: app, point: point, contentTranslation: .none) record(expected) } XCTAssertEqual(observed, 2) @@ -192,7 +192,7 @@ extension RunnerTests { XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout)) let frame = field.frame textInputProbeIssueForTesting = XCTIssue(type: .assertionFailure, compactDescription: "Failed to get matching snapshot: kAXErrorIllegalArgument") - let outcome = probeTextInputs(app: app, point: CGPoint(x: frame.midX, y: frame.midY), contentTranslation: nil) + let outcome = probeTextInputs(app: app, point: CGPoint(x: frame.midX, y: frame.midY), contentTranslation: .none) guard case .unavailable = outcome else { return XCTFail("A suppressed AX issue must discard the matching candidate") } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift index 2eb822e0f1..5adac13051 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift @@ -108,6 +108,7 @@ extension RunnerTests { element: textField, refreshPoint: nil, prefersFocusedElement: false, + contentTranslation: .none, fromTapWitness: true ), text: "\n", @@ -513,6 +514,7 @@ extension RunnerTests { element: nil, refreshPoint: nil, prefersFocusedElement: true, + contentTranslation: .none, fromTapWitness: true ), text: text, diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index 67d1a05ad8..b8b5b3ef30 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -664,8 +664,11 @@ The live `XCUIElement.frame` property reports screen space on a placed window, a The one live read that does not is `XCUIElementSnapshot.frame`, which reports the window's content space like the capture channel does. So the snapshot capture converts in `SnapshotGeometrySpace.normalized`, and the text-input probe converts its snapshot reads through the same `SnapshotContentTranslation` rule read from the two live boxes (`app.frame` against the resolved window's frame); every other live read needs no conversion. The probe-side owner is `ResolvedTextEntryInput.frameOnScreen`. -The vector is read once per command and stored on the target and each read the target produces, so the probe matching, the focus tap, the refresh point, the caret tap, and the readiness retap all dispatch through that one property. -Where a probe read has no snapshot and falls back to the live element, its frame is already screen space and is not translated. +The vector is read once per command as `TextEntryContentTranslation` — `known(offset)`, `none` for a full-screen app, or `unknown` when the window read could not answer — and it has no default anywhere it travels, so an omitted read can never read as full screen. +It is stored on the target and each read the target produces, so the probe matching, the focus tap, the refresh point, the caret tap, and the readiness retap all dispatch through that one property. +`frameOnScreen` is optional: it names no box when the read has no snapshot or the vector is `unknown`, and every dispatched-point consumer refuses a nil rather than re-adding its own snapshot guard. +Probe matching and the focus tap compare `element.frame` when `frameOnScreen` is nil, the only box a read that cannot translate still has. +The readiness retap taps nothing on `unknown`: dispatching a raw window-local point is the failure this amendment fixes, and skipping one retap costs the caller's next read, not the session. `app.frame` unions transformed subtrees, so content pushed off a placed window (a closed drawer at negative x) can stop the two live boxes matching and read no vector; the capture still converts from the snapshot root box, so a probe may then read window-local frames where the capture published screen space. A mid-command window move has the same effect on a vector read at command start. Both are reads of a moving target, not a second space: a re-opened command reads the current vector. The host bridge keeps refusing a windowed capture (`app-window-not-full-screen`) rather than converting one, so the TypeScript twin of the rule runs today only in the parity replay. From a956ec4f4951573f86828dbb4e839d1ddeaf995e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 25 Oct 2026 10:57:05 +0100 Subject: [PATCH 11/15] feat(runner): log the readiness retap point and its skips --- .../RunnerTests+TextEntryReadiness.swift | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift index fb29f558c3..b67d4b6349 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift @@ -119,9 +119,19 @@ extension RunnerTests { /// a screen box, and a raw window-local point is exactly the #3899 failure. The probe-side rule /// `clearTextInput` and `moveCaretToEnd` settled on: skip the point, never guess through it. func retapAt(read: ResolvedTextEntryInput, app: XCUIApplication, element: XCUIElement) { - guard let frame = read.frameOnScreen, !frame.isEmpty else { + guard let frame = read.frameOnScreen else { + NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_RETAP skipped reason=no-screen-frame") return } + guard !frame.isEmpty else { + NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_RETAP skipped reason=empty-frame") + return + } + NSLog( + "AGENT_DEVICE_RUNNER_TEXT_ENTRY_RETAP x=%.1f y=%.1f", + frame.midX, + frame.midY + ) _ = tapAt(app: app, x: frame.midX, y: frame.midY) _ = waitForTextEntryFocus(app: app, element: element, timeout: TextEntryTiming.readinessTimeout) } From 257cc7c2272649a36fec7bc1604fcfb2376ca3f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 25 Oct 2026 11:51:21 +0100 Subject: [PATCH 12/15] fix(runner): take the probe vector from the resolved window box alone Reply to review 5642729478: - liveContentTranslation builds the zero-origin reported box from the resolved window's size, so an app.frame that unions transformed subtrees cannot silently cost a placed window its vector (P1 thread 1). - Synthesized dispatch turns about the display's oriented box, never a placed window's smaller box; landscape under Windowed Apps stays unmeasured and the ADR says so (P1 thread 2). - requireTextInputAtPoint passes the live vector (P2 thread 3). - ADR 0004 distinguishes the refusing bridge from the converting runner and states the focus-tap/retap refusal rule (P2 threads 4-5). - The initial focus tap dispatches the caller's requested point when the read names no screen box; the live element.frame is never dispatched, and focusTapPoint is pinned by a unit test (P2 thread 7). --- .../RunnerTests+SynthesizedInteraction.swift | 48 +++++++++++++++++-- .../RunnerTests+TextEntry.swift | 15 +++--- .../RunnerTests+TextEntryFocus.swift | 22 ++++----- .../RunnerTests+TextInputProbe.swift | 29 ++++++----- ...RunnerTests+CoordinateTextEntryTests.swift | 25 +++++++++- ...unnerTests+SynthesizedTextEntryTests.swift | 4 +- .../CoordinateSpaceTests.swift | 37 ++++++++++++-- .../adr/0004-ios-snapshot-backend-strategy.md | 15 +++--- 8 files changed, 144 insertions(+), 51 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index cde5f867c0..70f3d2bc09 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -31,14 +31,15 @@ extension RunnerTests { ) } let frame = context.referenceFrame + let rotationFrame = synthesizedRotationFrame(app: app, fallback: frame, interfaceOrientation: orientation) let start = CoordinateSpaceRotation.native( point: CGPoint(x: x, y: y), - in: frame, + in: rotationFrame, interfaceOrientation: orientation ) let end = CoordinateSpaceRotation.native( point: CGPoint(x: x2, y: y2), - in: frame, + in: rotationFrame, interfaceOrientation: orientation ) logSynthesizedDispatch( @@ -117,7 +118,11 @@ extension RunnerTests { } let point = CoordinateSpaceRotation.native( point: CGPoint(x: x, y: y), - in: context.referenceFrame, + in: synthesizedRotationFrame( + app: app, + fallback: context.referenceFrame, + interfaceOrientation: orientation + ), interfaceOrientation: orientation ) logSynthesizedDispatch( @@ -191,7 +196,11 @@ extension RunnerTests { let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) let point = CoordinateSpaceRotation.native( point: CGPoint(x: x, y: y), - in: context.referenceFrame, + in: synthesizedRotationFrame( + app: app, + fallback: context.referenceFrame, + interfaceOrientation: orientation + ), interfaceOrientation: orientation ) logSynthesizedDispatch( @@ -543,6 +552,31 @@ extension RunnerTests { ) return frame } + + /// The box whose side lengths a dispatch turn swaps. The digitizer turns about the display's + /// own axes, so a screen-space point must turn about the display's side lengths even when the + /// app runs in a placed window, whose box is smaller (#3899). Landscape under Windowed Apps is + /// unmeasured (ADR 0004); the rule follows the digitizer there anyway. A multi-screen host — an + /// ADR 0025 foldable — resolves its window to the panel that hosts the app, so there the + /// caller's window box already names the display and stays the basis. A full-screen app's + /// window box is its display's box, so this changes no measured route. + func synthesizedRotationFrame( + app: XCUIApplication, + fallback: CGRect, + interfaceOrientation: Int + ) -> CGRect { + guard XCUIScreen.screens.count == 1 else { return fallback } + let screen = UIScreen.main + let scale = screen.nativeScale + guard scale > 0 else { return fallback } + return Self.soleScreenReferenceFrame( + nativePortraitSize: CGSize( + width: screen.nativeBounds.width / scale, + height: screen.nativeBounds.height / scale + ), + interfaceOrientation: interfaceOrientation + ) ?? fallback + } #endif /// The screen in the app's interface orientation, anchored at the origin like a window frame. An @@ -736,7 +770,11 @@ extension RunnerTests { pointer.samples.map { sample in let point = CoordinateSpaceRotation.native( point: CGPoint(x: sample.point.x, y: sample.point.y), - in: frame, + in: synthesizedRotationFrame( + app: app, + fallback: frame, + interfaceOrientation: orientation + ), interfaceOrientation: orientation ) return [ diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index ce8756c135..eed5425e82 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -137,9 +137,10 @@ extension RunnerTests { var isDistinguishable: Bool { !identifier.isEmpty } } - /// A text input found under a point, the identity it carried when found, and the content vector of - /// the command that found it. The element is an index-bound query handle: an input inserted or - /// reordered ahead of it later re-binds the handle to that input, and the identity is what notices. + /// A text input found under a point: the handle, the identity it carried when found, and the + /// vector that matched it — a later read of the handle must convert through the same vector the + /// lookup used. The element is an index-bound query handle: an input inserted or reordered ahead + /// of it later re-binds the handle to that input, and the identity is what notices. struct TextInputAtPoint { let element: XCUIElement let identity: TextEntryInputIdentity @@ -148,15 +149,15 @@ extension RunnerTests { /// The vector that carries a snapshot-read frame to its place on the screen (#3899), read once per /// command by `liveContentTranslation`. Non-defaulted everywhere it travels: a vector nobody read - /// must never read as full screen, because on a placed window that silently dispatches window-local - /// points as screen points — the #3899 failure. + /// must never read as full screen, because on a placed window that dispatches window-local points + /// as screen points — the #3899 failure. enum TextEntryContentTranslation { /// The vector of a placed window: its content box's origin on the screen. case known(CGSize) /// The app fills the screen, so a snapshot frame is already a screen frame. case none - /// The window read could not answer, so the space of a snapshot frame is unknown. Consumers - /// refuse it as a screen frame: `frameOnScreen` reads nil and nothing dispatches a point from it. + /// The window read could not answer. `frameOnScreen` reads nil, so no point dispatches from a + /// snapshot frame. case unknown } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift index 55d852968f..2309a3be97 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift @@ -105,6 +105,15 @@ extension RunnerTests { #endif } + /// The point the initial focus tap dispatches: the center of the input's screen box when the + /// read can name one, and the caller's requested point otherwise. The requested point is + /// already a screen point, so the live `element.frame` is never dispatched: on `.unknown` it is + /// a window-local box, and on a missing snapshot it is the read #3060 forbids (#3899). + static func focusTapPoint(frameOnScreen: CGRect?, requestedPoint: CGPoint) -> CGPoint { + guard let frameOnScreen, !frameOnScreen.isEmpty else { return requestedPoint } + return .init(x: frameOnScreen.midX, y: frameOnScreen.midY) + } + @MainActor func focusTextInputForTextEntry(app: XCUIApplication, x: Double?, y: Double?) -> TextEntryTarget { // The one content-vector read for this command (#3899): every target built here carries it and @@ -150,17 +159,8 @@ extension RunnerTests { let input = textInputAt(app: app, x: x, y: y, contentTranslation: contentTranslation) let target = input?.element let requestedPoint = CGPoint(x: x, y: y) - // The tap goes to the input's own center, and `tapAt` dispatches a screen point, so the point - // comes from `frameOnScreen`; the live frame is the box left when the command's window read - // could not answer (#3899). - let inputCenter = (input?.frameOnScreen ?? input?.element.frame).flatMap { - $0.isEmpty ? nil : CGPoint(x: $0.midX, y: $0.midY) - } - if let inputCenter { - _ = tapAt(app: app, x: inputCenter.x, y: inputCenter.y) - } else { - _ = tapAt(app: app, x: x, y: y) - } + let tapPoint = Self.focusTapPoint(frameOnScreen: input?.frameOnScreen, requestedPoint: requestedPoint) + _ = tapAt(app: app, x: tapPoint.x, y: tapPoint.y) // A visible keyboard is not enough evidence for app.typeText, because focus may still // belong to a previous field. With a concrete target we type through XCUIElement.typeText, // so after tapping it the iOS readiness timeout cannot prove anything stronger. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift index 6e6b2a5db2..700946ad94 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift @@ -222,32 +222,31 @@ extension RunnerTests { } /// The vector state that carries this session's snapshot-read app-content frames to their place on - /// the screen (#3899). Each command that dispatches points from snapshot frames reads it once, at - /// its own start, and hands it down; a read inside a typing loop never calls this, because the two - /// live box reads below cost about 90 ms each on a heavy React Native screen. - /// - /// The signature and the vector are the capture's rule (`SnapshotContentTranslation`, ADR 0004) - /// read from the two live boxes that name it: the application's `frame` is a `(0,0)`-anchored copy - /// of the box the resolved window reports at its place on the screen. Any other pair of boxes - /// declares no vector, so full-screen, iPhone, and each foldable panel keep frames as read. + /// the screen (#3899), read once per command at its start. A placed window's snapshot frames are + /// content-local (ADR 0004), so the window's own origin on the screen is the vector; a window + /// anchored at the origin — full screen, iPhone, and each ADR 0025 foldable panel — declares none. /// /// A read that cannot answer returns `.unknown`, never `.none`: on a placed window, a failed read /// that looked like full screen would dispatch raw window-local points as screen points — the /// #3899 failure — so consumers refuse `.unknown` instead of guessing through it. func liveContentTranslation(app: XCUIApplication) -> TextEntryContentTranslation { - // The window read is contained like the probe lookups it replaces: a transient window that - // records an XCTest issue costs the vector, not the runner session. - let read = containingTextInputProbeIssues(fallback: nil as TextEntryContentTranslation?) { _ in - guard let resolved = resolveRunnerWindowElement(app: app) else { return nil } - guard let translation = SnapshotContentTranslation.offset(reportedBox: app.frame, viewportBox: resolved.frame) else { + // Contained like the probe lookups it wraps: a transient window that records an XCTest issue + // costs the vector, not the runner session. This is the one window query a command makes; the + // resolved window's box is enough because `SnapshotContentTranslation.offset` reads only the + // origins, and a placed window's size is its own. + let read = containingTextInputProbeIssues(fallback: TextEntryContentTranslation.unknown) { _ in + guard let resolved = resolveRunnerWindowElement(app: app) else { return .unknown } + guard let offset = SnapshotContentTranslation.offset( + reportedBox: CGRect(origin: .zero, size: resolved.frame.size), + viewportBox: resolved.frame + ) else { // Explicit type: `.none` alone means `Optional.none` to the compiler here. return TextEntryContentTranslation.none } - return .known(translation) + return .known(offset) } switch read { case .success(let state): - guard let state else { return .unknown } return state case .failure: return .unknown diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift index 342bfc3e80..4bce84d095 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift @@ -6,7 +6,12 @@ extension RunnerTests { /// cannot answer. A fixture that does not expose its own field is a broken lane, not a refusal. @MainActor func requireTextInputAtPoint(app: XCUIApplication, x: Double, y: Double) throws -> TextInputAtPoint { - switch coordinateTapTextInputIdentityAt(app: app, x: x, y: y, contentTranslation: .none) { + switch coordinateTapTextInputIdentityAt( + app: app, + x: x, + y: y, + contentTranslation: liveContentTranslation(app: app) + ) { case .input(let input): return input case .absent: @@ -111,5 +116,23 @@ extension RunnerTests { "the refusal dispatched its focus tap: the point's control took the press" ) } + + /// The initial focus tap dispatches the input's screen box only when the read can name one. + /// With no box — `.unknown` or a missing snapshot — it dispatches the caller's requested point, + /// never a live `element.frame` center (#3899). + func testFocusTapPointNeverDispatchesALiveFrame() { + let requested = CGPoint(x: 120, y: 640) + XCTAssertEqual( + Self.focusTapPoint(frameOnScreen: CGRect(x: 300, y: 600, width: 200, height: 40), requestedPoint: requested), + CGPoint(x: 400, y: 620), + "a known screen box moves the tap to its center" + ) + XCTAssertEqual(Self.focusTapPoint(frameOnScreen: nil, requestedPoint: requested), requested) + XCTAssertEqual( + Self.focusTapPoint(frameOnScreen: .null, requestedPoint: requested), + requested, + "an empty box names no center" + ) + } #endif } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift index 3b7174b672..4b22ac1329 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift @@ -232,7 +232,7 @@ extension RunnerTests { element: nil, refreshPoint: CGPoint(x: frame.midX, y: frame.midY), prefersFocusedElement: false, - contentTranslation: .none, + contentTranslation: .none, inputAtRefreshPoint: found ), text: "fresh", @@ -331,7 +331,7 @@ extension RunnerTests { element: nil, refreshPoint: CGPoint(x: neighbourFrame.midX, y: neighbourFrame.midY), prefersFocusedElement: false, - contentTranslation: .none, + contentTranslation: .none, inputAtRefreshPoint: inputAtRefreshPoint ), text: "", diff --git a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift index 9b80b197d2..817429ad22 100644 --- a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift +++ b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift @@ -616,9 +616,9 @@ final class CoordinateSpaceTests: XCTestCase { /// The dispatch direction for a windowed app: the caller's point is the point touched, so /// `native(point:)` turns a screen-space point's axes without moving it toward the window origin /// (#3899). Under portrait — the only orientation measured for a windowed app — it is the identity. - /// The landscape rows for a placed window stay the documented unmeasured gap (ADR 0004): the turn - /// here is asserted about the window's side lengths, while the digitizer turns about the screen's, - /// and no windowed landscape capture has been measured to say which basis is right. + /// The basis row below pins the rule the dispatch follows in landscape: the digitizer turns about + /// the display's side lengths, so a windowed app's point turns about the display's box, not the + /// placed window's. Landscape under Windowed Apps is an unmeasured gap (ADR 0004). func testNativePointForAWindowedAppKeepsTheCallersScreenPoint() { let windowOnScreen = CGRect(x: 224, y: 207, width: 386, height: 682) for orientation in [ @@ -636,6 +636,37 @@ final class CoordinateSpaceTests: XCTestCase { } } + /// A windowed landscape point turns about the display's box: the turn stays inside the display's + /// native portrait box, and a placed window's smaller box would move the point instead. The + /// runner passes the display's box as the basis (`synthesizedRotationFrame`). + func testNativePointInWindowedLandscapeTurnsAboutTheDisplayBox() { + let displayLandscapeBox = CGRect(x: 0, y: 0, width: 1210, height: 834) + let windowOnScreen = CGRect(x: 224, y: 207, width: 386, height: 682) + let screenPoint = CGPoint(x: 600, y: 300) + XCTAssertEqual( + CoordinateSpaceRotation.native( + point: screenPoint, + in: displayLandscapeBox, + interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + ), + CGPoint(x: 534, y: 600), + "the display basis lands inside the display's native portrait box (834 x 1210)" + ) + XCTAssertNotEqual( + CoordinateSpaceRotation.native( + point: screenPoint, + in: windowOnScreen, + interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + ), + CoordinateSpaceRotation.native( + point: screenPoint, + in: displayLandscapeBox, + interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + ), + "the placed window's side lengths are not the digitizer's basis" + ) + } + func testNormalizedOfAnEmptyArrayIsEmpty() { XCTAssertEqual( SnapshotGeometrySpace.normalized( diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index b8b5b3ef30..1ed09cbf51 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -662,17 +662,18 @@ The measured dispatch spaces for a placed window are both screen-anchored: the s `interactionCoordinate` therefore passes the caller's screen point through, and `resolveRunnerWindow` keeps naming the window a foldable's tap resolves its display on (ADR 0025). The live `XCUIElement.frame` property reports screen space on a placed window, as the selector-tap path proved on the device: it measured the `Add` button at (659,321) for a window at (324,307). The one live read that does not is `XCUIElementSnapshot.frame`, which reports the window's content space like the capture channel does. -So the snapshot capture converts in `SnapshotGeometrySpace.normalized`, and the text-input probe converts its snapshot reads through the same `SnapshotContentTranslation` rule read from the two live boxes (`app.frame` against the resolved window's frame); every other live read needs no conversion. +So the snapshot capture converts in `SnapshotGeometrySpace.normalized`, and the text-input probe converts its snapshot reads through the same `SnapshotContentTranslation` rule read from the resolved window's live box (its origin, against a zero-origin box of its own size); every other live read needs no conversion. The probe-side owner is `ResolvedTextEntryInput.frameOnScreen`. The vector is read once per command as `TextEntryContentTranslation` — `known(offset)`, `none` for a full-screen app, or `unknown` when the window read could not answer — and it has no default anywhere it travels, so an omitted read can never read as full screen. It is stored on the target and each read the target produces, so the probe matching, the focus tap, the refresh point, the caret tap, and the readiness retap all dispatch through that one property. -`frameOnScreen` is optional: it names no box when the read has no snapshot or the vector is `unknown`, and every dispatched-point consumer refuses a nil rather than re-adding its own snapshot guard. -Probe matching and the focus tap compare `element.frame` when `frameOnScreen` is nil, the only box a read that cannot translate still has. +`frameOnScreen` is optional: it names no box when the read has no snapshot or the vector is `unknown`. +No point dispatch reads a live `element.frame`: the focus tap falls back to the caller's requested point, which is already a screen point, and the retap and the caret tap dispatch nothing on a nil. +Probe matching compares `element.frame` when `frameOnScreen` is nil — a comparison needs the only box a read that cannot translate still has, and a comparison dispatches nothing. The readiness retap taps nothing on `unknown`: dispatching a raw window-local point is the failure this amendment fixes, and skipping one retap costs the caller's next read, not the session. -`app.frame` unions transformed subtrees, so content pushed off a placed window (a closed drawer at negative x) can stop the two live boxes matching and read no vector; the capture still converts from the snapshot root box, so a probe may then read window-local frames where the capture published screen space. A mid-command window move has the same effect on a vector read at command start. -Both are reads of a moving target, not a second space: a re-opened command reads the current vector. -The host bridge keeps refusing a windowed capture (`app-window-not-full-screen`) rather than converting one, so the TypeScript twin of the rule runs today only in the parity replay. -Portrait is the only orientation measured for a placed window; the quarter-turn table's windowed row pins the arithmetic (a frame origin never travels through the turn), not a device measurement, and landscape under Windowed Apps stays unverified until measured. +The probe trusts that selector's naming — the same first non-empty window every synthesized dispatch already books as its reference box — and a window that is moved between the command's read and its dispatch drifts, like every live read: a re-opened command reads the current vector. +The host bridge still refuses a windowed capture (`app-window-not-full-screen`) and converts nothing, while the XCTest runner — the fallback the bridge's refusal routes to — converts through the rule above, so the bridge's refusal stays the reason the TypeScript twin of the rule runs today only in the parity replay. +Portrait is the only orientation measured for a placed window; the quarter-turn table's windowed row pins the arithmetic (a frame origin never travels through the turn), not a device measurement. +Dispatch rotation turns about the display's side lengths, as the digitizer does, never about a placed window's smaller box; landscape under Windowed Apps stays unmeasured until measured, and the rotation basis follows the display there on the digitizer's rule, not on a measurement. ## Amendment: modal containment in the presentation cut From f8b3d990d7c5b9080d374848d418d06d2a713208 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 25 Oct 2026 12:35:38 +0100 Subject: [PATCH 13/15] refactor(runner): shrink the windowed-iPad screen-space change to its rule Keep the accepted design (one SnapshotContentTranslation rule in normalized, ResolvedTextEntryInput.frameOnScreen, a non-defaulted known/none/unknown vector read once per command, rotation about the sole display) and cut what did not serve it: cosmetic renames and reformatting, comments that restated code, the one-caller focusTapPoint helper, TextInputAtPoint's copy of the target's vector, the duplicated sole-display math, duplicate unit tests and fixture rows, and most of the ADR 0004 amendment prose. Co-Authored-By: Claude Opus 5.5 --- .../RunnerTests+CommandExecution.swift | 8 +- .../RunnerTests+Interaction.swift | 23 +-- .../RunnerTests+SynthesizedInteraction.swift | 58 ++---- .../RunnerTests+TextEntry.swift | 101 +++------- .../RunnerTests+TextEntryFocus.swift | 76 ++------ .../RunnerTests+TextEntryReadiness.swift | 41 +--- .../RunnerTests+TextInputProbe.swift | 103 +++------- .../RunnerTests+TypeExecution.swift | 3 +- ...RunnerTests+CoordinateTextEntryTests.swift | 25 +-- ...unnerTests+SnapshotPresentationTests.swift | 127 ------------ ...nerTests+SynthesizedInteractionTests.swift | 15 ++ ...unnerTests+SynthesizedTextEntryTests.swift | 15 +- .../RunnerTests+TextEntryLatencyTests.swift | 7 +- .../RunnerTests+TextEntryReadinessTests.swift | 72 ++----- .../SnapshotCoordinateSpace.swift | 86 +++------ .../CoordinateSpaceTests.swift | 182 +++--------------- .../fixtures/window-coordinate-space.json | 60 +----- .../adr/0004-ios-snapshot-backend-strategy.md | 72 +++---- .../window-coordinate-space.test.ts | 36 +--- .../window-coordinate-space.ts | 36 ++-- 20 files changed, 239 insertions(+), 907 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index adcab6215c..c8c622cfe6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -155,13 +155,7 @@ extension RunnerTests { ) let textInput: XCUIElement? if !xCTestTextInputProbeSkipped { - // The lookup compares candidate frames with this screen point (#3899). - textInput = coordinateTapTextInputAt( - app: activeApp, - x: x, - y: y, - contentTranslation: liveContentTranslation(app: activeApp) - ) + textInput = coordinateTapTextInputAt(app: activeApp, x: x, y: y) } else { textInput = nil // Clear before dispatch so a throwing gesture cannot retain the preceding field's witness. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 704b2bed10..3684adfee4 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -352,12 +352,8 @@ extension RunnerTests { func readTextAt(app: XCUIApplication, x: Double, y: Double) -> String? { let point = CGPoint(x: x, y: y) - // The lookup compares candidate frames with this screen point (#3899). - let textInputCandidates = textInputCandidatesAt( - app: app, - point: point, - contentTranslation: liveContentTranslation(app: app) - ).map(\.element) + let textInputCandidates = textInputCandidatesAt(app: app, point: point, contentTranslation: liveContentTranslation(app: app)) + .map(\.element) for element in textInputCandidates where prefersExpandedTextRead(element) { if let text = readableText(for: element) { return text @@ -622,14 +618,13 @@ extension RunnerTests { #if !os(tvOS) private func interactionCoordinate(app: XCUIApplication, x: Double, y: Double) -> XCUICoordinate { #if os(iOS) - // A caller's point is a screen point: these gestures anchor at the app's origin, not the - // resolved window, so it is dispatched as given. The anchor sits at the origin of the space - // `app.frame` names, and for a windowed app the measured `app.frame` starts at the screen's - // origin (#3899). The first qualifying window is not always the one under the finger — a - // SpringBoard `alert accept` can live in an alert window while the wallpaper or status-bar - // window qualifies first, so a window-relative anchor misses the button. Reference frames and - // the synthesized display ID still resolve through `resolveRunnerWindow`, so a foldable tap - // keeps its panel and stays on its display (ADR 0025). + // Coordinate taps, double taps, long presses, and drags anchor at the app origin, not the + // resolved window: callers pass screen points, and `app.frame` starts at the screen origin even + // for a placed iPad window (#3899). The first qualifying + // window is not always the one under the finger — a SpringBoard `alert accept` can live in an + // alert window while the wallpaper or status-bar window qualifies first, so a window-relative + // anchor misses the button. Reference frames and the synthesized display ID still resolve + // through `resolveRunnerWindow`, so a foldable tap keeps its panel and stays on its display. let origin = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0)) return origin.withOffset(CGVector(dx: x, dy: y)) #else diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index 70f3d2bc09..d6d6d9e9b4 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -31,7 +31,7 @@ extension RunnerTests { ) } let frame = context.referenceFrame - let rotationFrame = synthesizedRotationFrame(app: app, fallback: frame, interfaceOrientation: orientation) + let rotationFrame = synthesizedRotationFrame(fallback: frame, interfaceOrientation: orientation) let start = CoordinateSpaceRotation.native( point: CGPoint(x: x, y: y), in: rotationFrame, @@ -118,11 +118,7 @@ extension RunnerTests { } let point = CoordinateSpaceRotation.native( point: CGPoint(x: x, y: y), - in: synthesizedRotationFrame( - app: app, - fallback: context.referenceFrame, - interfaceOrientation: orientation - ), + in: synthesizedRotationFrame(fallback: context.referenceFrame, interfaceOrientation: orientation), interfaceOrientation: orientation ) logSynthesizedDispatch( @@ -196,11 +192,7 @@ extension RunnerTests { let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) let point = CoordinateSpaceRotation.native( point: CGPoint(x: x, y: y), - in: synthesizedRotationFrame( - app: app, - fallback: context.referenceFrame, - interfaceOrientation: orientation - ), + in: synthesizedRotationFrame(fallback: context.referenceFrame, interfaceOrientation: orientation), interfaceOrientation: orientation ) logSynthesizedDispatch( @@ -533,17 +525,10 @@ extension RunnerTests { /// that can show an app in part of a screen (iPad, more than one screen) keeps requiring a window. @MainActor func soleScreenReferenceFrame(app: XCUIApplication) -> CGRect? { - guard UIDevice.current.userInterfaceIdiom == .phone, XCUIScreen.screens.count == 1 else { + guard UIDevice.current.userInterfaceIdiom == .phone else { return nil } - let screen = UIScreen.main - let scale = screen.nativeScale - guard scale > 0 else { return nil } - let frame = Self.soleScreenReferenceFrame( - nativePortraitSize: CGSize( - width: screen.nativeBounds.width / scale, - height: screen.nativeBounds.height / scale - ), + let frame = soleDisplayFrame( interfaceOrientation: Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) ) NSLog( @@ -553,29 +538,25 @@ extension RunnerTests { return frame } - /// The box whose side lengths a dispatch turn swaps. The digitizer turns about the display's - /// own axes, so a screen-space point must turn about the display's side lengths even when the - /// app runs in a placed window, whose box is smaller (#3899). Landscape under Windowed Apps is - /// unmeasured (ADR 0004); the rule follows the digitizer there anyway. A multi-screen host — an - /// ADR 0025 foldable — resolves its window to the panel that hosts the app, so there the - /// caller's window box already names the display and stays the basis. A full-screen app's - /// window box is its display's box, so this changes no measured route. - func synthesizedRotationFrame( - app: XCUIApplication, - fallback: CGRect, - interfaceOrientation: Int - ) -> CGRect { - guard XCUIScreen.screens.count == 1 else { return fallback } + /// The box a synthesized turn swaps side lengths about. The digitizer turns about its display, so a + /// placed iPad window's smaller box is not the basis (#3899); a multi-screen host keeps the + /// resolved window, which names its panel (ADR 0025). + func synthesizedRotationFrame(fallback: CGRect, interfaceOrientation: Int) -> CGRect { + soleDisplayFrame(interfaceOrientation: interfaceOrientation) ?? fallback + } + + private func soleDisplayFrame(interfaceOrientation: Int) -> CGRect? { + guard XCUIScreen.screens.count == 1 else { return nil } let screen = UIScreen.main let scale = screen.nativeScale - guard scale > 0 else { return fallback } + guard scale > 0 else { return nil } return Self.soleScreenReferenceFrame( nativePortraitSize: CGSize( width: screen.nativeBounds.width / scale, height: screen.nativeBounds.height / scale ), interfaceOrientation: interfaceOrientation - ) ?? fallback + ) } #endif @@ -766,15 +747,12 @@ extension RunnerTests { width: plan.viewport.width, height: plan.viewport.height ) + let rotationFrame = synthesizedRotationFrame(fallback: frame, interfaceOrientation: orientation) let pointerSamples: [[[String: NSNumber]]] = plan.pointers.map { pointer in pointer.samples.map { sample in let point = CoordinateSpaceRotation.native( point: CGPoint(x: sample.point.x, y: sample.point.y), - in: synthesizedRotationFrame( - app: app, - fallback: frame, - interfaceOrientation: orientation - ), + in: rotationFrame, interfaceOrientation: orientation ) return [ diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index eed5425e82..c088401e55 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -137,27 +137,21 @@ extension RunnerTests { var isDistinguishable: Bool { !identifier.isEmpty } } - /// A text input found under a point: the handle, the identity it carried when found, and the - /// vector that matched it — a later read of the handle must convert through the same vector the - /// lookup used. The element is an index-bound query handle: an input inserted or reordered ahead - /// of it later re-binds the handle to that input, and the identity is what notices. + /// A text input found under a point, and the identity it carried when found. The element is an + /// index-bound query handle: an input inserted or reordered ahead of it later re-binds the handle + /// to that input, and the identity is what notices. struct TextInputAtPoint { let element: XCUIElement let identity: TextEntryInputIdentity - let contentTranslation: TextEntryContentTranslation } - /// The vector that carries a snapshot-read frame to its place on the screen (#3899), read once per - /// command by `liveContentTranslation`. Non-defaulted everywhere it travels: a vector nobody read - /// must never read as full screen, because on a placed window that dispatches window-local points - /// as screen points — the #3899 failure. + /// The vector from a snapshot frame to the screen (#3899, ADR 0004), read once per command by + /// `liveContentTranslation` and never defaulted: a vector nobody read must not pass for full screen. enum TextEntryContentTranslation { - /// The vector of a placed window: its content box's origin on the screen. case known(CGSize) /// The app fills the screen, so a snapshot frame is already a screen frame. case none - /// The window read could not answer. `frameOnScreen` reads nil, so no point dispatches from a - /// snapshot frame. + /// The window read could not answer, so no snapshot frame names a screen box. case unknown } @@ -165,9 +159,6 @@ extension RunnerTests { let element: XCUIElement? let refreshPoint: CGPoint? let prefersFocusedElement: Bool - /// The content vector of the command that built this target, handed to every read it produces, - /// which turns their snapshot frames into screen points (#3899). Non-defaulted: the route that - /// reads no window carries `.unknown`, not a vector that would read as full screen. let contentTranslation: TextEntryContentTranslation let fromTapWitness: Bool /// The input the first resolved element was. Once bound, resolution refuses any other input. @@ -202,10 +193,8 @@ extension RunnerTests { self.postFocusRead = postFocusRead } - /// The target moved to the input a resolution read: its element, a refresh point at its frame's - /// center on the screen, and its identity while the target is unbound. A read that cannot name a - /// screen box — no snapshot, or an unknown content space — keeps the target's current refresh - /// point rather than a center it cannot trust. + /// The target moved to the input a resolution read: its element, a refresh point at its center, + /// and its identity while the target is unbound. func withResolvedInput(_ input: ResolvedTextEntryInput?) -> TextEntryTarget { guard let input else { return self @@ -229,15 +218,10 @@ extension RunnerTests { /// Pro, iOS 27.0). Where a live read of an element that stopped answering its query, such as a /// removed input or a focused Flutter password field (#3060), records an XCTest failure that ends /// the runner session, the snapshot only throws. - /// - /// This type owns the probe-side space of #3899 (ADR 0004): a snapshot frame names a placed - /// window's content space while every point this read feeds is a screen point, so consumers take - /// `frameOnScreen` and compare raw frames only with each other. struct ResolvedTextEntryInput { let element: XCUIElement /// Nil when the element exists but could not be snapshotted. let snapshot: XCUIElementSnapshot? - /// The vector state of the command that made this read (#3899). let contentTranslation: TextEntryContentTranslation private init(element: XCUIElement, snapshot: XCUIElementSnapshot?, contentTranslation: TextEntryContentTranslation) { @@ -282,24 +266,15 @@ extension RunnerTests { return element } - /// The input's box in screen space, or nil when this read cannot name one: no snapshot, or a - /// window read that could not answer, so the snapshot frame's space is unknown. A nil here is the - /// one signal every consumer uses to refuse a dispatched point: the live `element.frame` is a - /// read a handle that stopped answering its query cannot make without recording a failure - /// (#3060), and an unknown space must never be dispatched as a screen point (#3899). Callers - /// that need a box for a route which dispatches nothing fall back to `element.frame`, which - /// already reports a placed window's content in screen space. + /// The input's screen box. Nil without a snapshot, because the live `element.frame` of a handle + /// that stopped answering records a failure (#3060), and nil for an `.unknown` vector, because a + /// window-local point must not dispatch as a screen point (#3899). Point dispatch refuses nil. var frameOnScreen: CGRect? { - guard let snapshot else { - return nil - } + guard let frame = snapshot?.frame else { return nil } switch contentTranslation { - case .known(let offset): - return snapshot.frame.offsetBy(dx: offset.width, dy: offset.height) - case .none: - return snapshot.frame - case .unknown: - return nil + case .known(let offset): return frame.offsetBy(dx: offset.width, dy: offset.height) + case .none: return frame + case .unknown: return nil } } } @@ -313,10 +288,7 @@ extension RunnerTests { private var lastRead: ResolvedTextEntryInput? private let resolve: (TextEntryTarget) -> ResolvedTextEntryInput? - init( - target: TextEntryTarget, - resolve: @escaping (TextEntryTarget) -> ResolvedTextEntryInput? - ) { + init(target: TextEntryTarget, resolve: @escaping (TextEntryTarget) -> ResolvedTextEntryInput?) { self.target = target.withResolvedInput(target.postFocusRead) lastRead = target.postFocusRead self.resolve = resolve @@ -550,10 +522,8 @@ extension RunnerTests { return resolveUnboundTextEntryInput(app: app, target: target) } func boundInput(_ candidate: XCUIElement) -> ResolvedTextEntryInput? { - guard let input = ResolvedTextEntryInput( - reading: candidate, - contentTranslation: target.contentTranslation - ), input.identity == boundIdentity else { return nil } + guard let input = ResolvedTextEntryInput(reading: candidate, contentTranslation: target.contentTranslation), + input.identity == boundIdentity else { return nil } return input } if target.prefersFocusedElement, let focused = focusedTextInput(app: app), let input = boundInput(focused) { @@ -563,16 +533,12 @@ extension RunnerTests { return input } if let input = target.inputAtRefreshPoint, - let read = textInputIfStillResolves(input) { + let read = textInputIfStillResolves(input, contentTranslation: target.contentTranslation) { return read } if target.inputAtRefreshPoint == nil, let refreshPoint = target.refreshPoint, - case .matches(let candidates) = probeTextInputs( - app: app, - point: refreshPoint, - contentTranslation: target.contentTranslation - ), + case .matches(let candidates) = probeTextInputs(app: app, point: refreshPoint, contentTranslation: target.contentTranslation), let match = candidates.first(where: { $0.identity == boundIdentity }) { return match } @@ -607,11 +573,7 @@ extension RunnerTests { guard let refreshPoint = target.refreshPoint else { return true } - switch probeTextInputs( - app: app, - point: refreshPoint, - contentTranslation: target.contentTranslation - ) { + switch probeTextInputs(app: app, point: refreshPoint, contentTranslation: target.contentTranslation) { case .absent: return true case .matches(let candidates): @@ -645,25 +607,20 @@ extension RunnerTests { /// Resolution for a target not bound to an input yet: the first element that exists. private func resolveUnboundTextEntryInput(app: XCUIApplication, target: TextEntryTarget) -> ResolvedTextEntryInput? { - let translation = target.contentTranslation - if target.prefersFocusedElement, - let focused = focusedTextInput(app: app).flatMap({ ResolvedTextEntryInput(reading: $0, contentTranslation: translation) }) { + let read = { (element: XCUIElement) in + ResolvedTextEntryInput(reading: element, contentTranslation: target.contentTranslation) + } + if target.prefersFocusedElement, let focused = focusedTextInput(app: app).flatMap(read) { return focused } - if let element = target.element, - let input = ResolvedTextEntryInput(reading: element, contentTranslation: translation) { + if let element = target.element, let input = read(element) { return input } if let refreshPoint = target.refreshPoint, - let refreshed = textInputAt( - app: app, - x: refreshPoint.x, - y: refreshPoint.y, - contentTranslation: translation - ) { + let refreshed = textInputAt(app: app, x: refreshPoint.x, y: refreshPoint.y, contentTranslation: target.contentTranslation) { return refreshed } - return focusedTextInput(app: app).flatMap { ResolvedTextEntryInput(reading: $0, contentTranslation: translation) } + return focusedTextInput(app: app).flatMap(read) } private func moveCaretToEnd(app: XCUIApplication, input: ResolvedTextEntryInput) { @@ -682,8 +639,6 @@ extension RunnerTests { // zero-size one (how a departed handle reads), there is no point to dispatch — an edge-tap // from a frame the field moved after records a failure or navigates away, the shape // `clearTextInput`'s comment refuses — so degrade to the caller's point-free delete burst. - // A snapshotless read gets no point at all: `frameOnScreen` is nil without a snapshot, so the - // live `element.frame` read this function exists to avoid is never reached (#3060). guard let frame = input.frameOnScreen, !frame.isEmpty else { return } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift index 2309a3be97..0f598bcb68 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift @@ -26,9 +26,7 @@ extension RunnerTests { } @MainActor - private func rememberedTextEntryTarget( - contentTranslation: TextEntryContentTranslation - ) -> TextEntryTarget? { + private func rememberedTextEntryTarget(contentTranslation: TextEntryContentTranslation) -> TextEntryTarget? { guard let witness = textEntryTapWitness else { return nil } @@ -105,25 +103,12 @@ extension RunnerTests { #endif } - /// The point the initial focus tap dispatches: the center of the input's screen box when the - /// read can name one, and the caller's requested point otherwise. The requested point is - /// already a screen point, so the live `element.frame` is never dispatched: on `.unknown` it is - /// a window-local box, and on a missing snapshot it is the read #3060 forbids (#3899). - static func focusTapPoint(frameOnScreen: CGRect?, requestedPoint: CGPoint) -> CGPoint { - guard let frameOnScreen, !frameOnScreen.isEmpty else { return requestedPoint } - return .init(x: frameOnScreen.midX, y: frameOnScreen.midY) - } - @MainActor func focusTextInputForTextEntry(app: XCUIApplication, x: Double?, y: Double?) -> TextEntryTarget { - // The one content-vector read for this command (#3899): every target built here carries it and - // every read those targets produce converts through it. let contentTranslation = liveContentTranslation(app: app) guard let x, let y else { let softwareKeyboardVisible = isKeyboardVisible(app: app) - if !softwareKeyboardVisible, let rememberedTarget = rememberedTextEntryTarget( - contentTranslation: contentTranslation - ) { + if !softwareKeyboardVisible, let rememberedTarget = rememberedTextEntryTarget(contentTranslation: contentTranslation) { return rememberedTarget } // Bare `type` targets the current first responder. On iOS we intentionally do not let the @@ -147,29 +132,21 @@ extension RunnerTests { contentTranslation: contentTranslation ) ) - return TextEntryTarget( - element: focused, - refreshPoint: nil, - prefersFocusedElement: true, - contentTranslation: contentTranslation - ) + return TextEntryTarget(element: focused, refreshPoint: nil, prefersFocusedElement: true, contentTranslation: contentTranslation) } let keyboardVisibleBeforeTap = isKeyboardVisible(app: app) let input = textInputAt(app: app, x: x, y: y, contentTranslation: contentTranslation) let target = input?.element let requestedPoint = CGPoint(x: x, y: y) - let tapPoint = Self.focusTapPoint(frameOnScreen: input?.frameOnScreen, requestedPoint: requestedPoint) + // `tapAt` takes a screen point: with no screen box, tap the caller's own point (#3899, #3060). + let tapPoint = input?.frameOnScreen.flatMap { $0.isEmpty ? nil : CGPoint(x: $0.midX, y: $0.midY) } ?? requestedPoint _ = tapAt(app: app, x: tapPoint.x, y: tapPoint.y) // A visible keyboard is not enough evidence for app.typeText, because focus may still // belong to a previous field. With a concrete target we type through XCUIElement.typeText, // so after tapping it the iOS readiness timeout cannot prove anything stronger. if keyboardVisibleBeforeTap, let target { - return focusedTextEntryTarget( - target, - requestedPoint: requestedPoint, - contentTranslation: contentTranslation - ) + return focusedTextEntryTarget(target, requestedPoint: requestedPoint, contentTranslation: contentTranslation) } let stabilized = stabilizeTextInputBeforeTyping( app: app, @@ -184,12 +161,7 @@ extension RunnerTests { stabilized.polls ) if case .inputReportsKeyboardFocus(let read) = stabilized.evidence { - return focusedTextEntryTarget( - read.element, - read: read, - requestedPoint: requestedPoint, - contentTranslation: contentTranslation - ) + return focusedTextEntryTarget(read.element, read: read, requestedPoint: requestedPoint, contentTranslation: contentTranslation) } let readyTarget = TextEntryTarget( element: stabilized.element ?? target, @@ -200,11 +172,7 @@ extension RunnerTests { let element = stabilized.focusConfirmed ? readyTarget.element : (waitForTextEntryReadiness(app: app, target: readyTarget) ?? readyTarget.element) - return focusedTextEntryTarget( - element, - requestedPoint: requestedPoint, - contentTranslation: contentTranslation - ) + return focusedTextEntryTarget(element, requestedPoint: requestedPoint, contentTranslation: contentTranslation) } /// The target the focus step hands to text entry, with the read its refresh point came from. @@ -214,19 +182,10 @@ extension RunnerTests { contentTranslation: TextEntryContentTranslation ) -> TextEntryTarget { guard let element else { - return TextEntryTarget( - element: nil, - refreshPoint: requestedPoint, - prefersFocusedElement: false, - contentTranslation: contentTranslation - ) + return TextEntryTarget(element: nil, refreshPoint: requestedPoint, prefersFocusedElement: false, contentTranslation: contentTranslation) } - return focusedTextEntryTarget( - element, - read: ResolvedTextEntryInput(reading: element, contentTranslation: contentTranslation), - requestedPoint: requestedPoint, - contentTranslation: contentTranslation - ) + let read = ResolvedTextEntryInput(reading: element, contentTranslation: contentTranslation) + return focusedTextEntryTarget(element, read: read, requestedPoint: requestedPoint, contentTranslation: contentTranslation) } /// `read` is `element` read after the focus tap, or nil when the read found no element. @@ -238,10 +197,7 @@ extension RunnerTests { ) -> TextEntryTarget { return TextEntryTarget( element: element, - refreshPoint: textEntryRefreshPoint( - read: read, - fallbackTo: requestedPoint - ), + refreshPoint: textEntryRefreshPoint(frame: read?.frameOnScreen, fallbackTo: requestedPoint), prefersFocusedElement: false, contentTranslation: contentTranslation, postFocusRead: read?.snapshot == nil ? nil : read @@ -256,14 +212,12 @@ extension RunnerTests { /// the target gets NO refresh point rather than the pre-focus one: this route's first resolve is /// not yet bound to an identity, and a field that went unqueryable on focus may have moved the /// layout so the old point names a different input, so fail closed with a typed reason instead of - /// typing into that other field. The frame is `read`'s snapshot frame plus its content vector, - /// never the live `element.frame`, and `frameOnScreen` is nil for a read that cannot name a screen - /// box (#3899, #3060). + /// typing into that other field. private func textEntryRefreshPoint( - read: ResolvedTextEntryInput?, + frame: CGRect?, fallbackTo requestedPoint: CGPoint ) -> CGPoint? { - guard let frame = read?.frameOnScreen else { + guard let frame else { return nil } guard !frame.isEmpty else { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift index b67d4b6349..2f26b95688 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift @@ -85,53 +85,24 @@ extension RunnerTests { /// `ResolvedTextEntryInput` already reads elements this way. It is read again after the /// wait rather than reused, because focusing a field moves the layout: the repeat aims at where the /// field is then, which is the moment the read it replaced ran at. - /// - /// The re-read carries a fresh content-vector read, so the repeat taps a screen point (#3899). It - /// is read only on this branch: a tap that confirms focus dispatches no window lookup. func waitForTextEntryReadinessAfterTap(app: XCUIApplication, element: XCUIElement) { #if os(iOS) if waitForTextEntryFocus(app: app, element: element, timeout: TextEntryTiming.readinessTimeout) { return } - // Contained like the probe lookups: a transient window or a snapshot that records an XCTest - // issue costs the retap, not the command. A failed window read leaves the read's vector - // `.unknown`, which names no screen box, so `retapAt` taps nothing (#3899). - let settled: ResolvedTextEntryInput? - switch containingTextInputProbeIssues(fallback: nil as ResolvedTextEntryInput?, { _ in + guard let settled = safely("TEXT_ENTRY_READINESS_FRAME", { ResolvedTextEntryInput(reading: element, contentTranslation: liveContentTranslation(app: app)) - }) { - case .success(let read): - settled = read - case .failure: - settled = nil - } - guard let settled else { + }) else { return } retapAt(read: settled, app: app, element: element) #endif } - /// The readiness retap, split out so the unit lane can drive it and assert where the point lands. - /// The retap taps nothing when the read cannot name a screen box: with no snapshot that box is the - /// live `element.frame`, the read a handle that stopped answering its query cannot make without - /// recording a failure (#3060); with an unknown content space a snapshot frame is not known to be - /// a screen box, and a raw window-local point is exactly the #3899 failure. The probe-side rule - /// `clearTextInput` and `moveCaretToEnd` settled on: skip the point, never guess through it. + /// The readiness repeat, at the read's screen box; a read that names none taps nothing (#3899). func retapAt(read: ResolvedTextEntryInput, app: XCUIApplication, element: XCUIElement) { - guard let frame = read.frameOnScreen else { - NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_RETAP skipped reason=no-screen-frame") - return - } - guard !frame.isEmpty else { - NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_RETAP skipped reason=empty-frame") - return - } - NSLog( - "AGENT_DEVICE_RUNNER_TEXT_ENTRY_RETAP x=%.1f y=%.1f", - frame.midX, - frame.midY - ) + guard let frame = read.frameOnScreen, !frame.isEmpty else { return } + NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_RETAP x=%.1f y=%.1f", frame.midX, frame.midY) _ = tapAt(app: app, x: frame.midX, y: frame.midY) _ = waitForTextEntryFocus(app: app, element: element, timeout: TextEntryTiming.readinessTimeout) } @@ -164,7 +135,7 @@ extension RunnerTests { /// fallback at all. func keyboardFocusConfirmed(app: XCUIApplication, element: XCUIElement) -> Bool { #if os(iOS) - // No content vector: this corroborates raw frame against raw frame and dispatches no point. + // `.unknown`: this compares raw frames with each other and dispatches no point. return ResolvedTextEntryInput(reading: element, contentTranslation: .unknown) .map { keyboardFocusConfirmed(app: app, read: $0) } ?? false #else diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift index 700946ad94..78dc4c6708 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift @@ -18,33 +18,18 @@ enum TextInputProbeOutcome { } extension RunnerTests { - func textInputAt( - app: XCUIApplication, - x: Double, - y: Double, - contentTranslation: TextEntryContentTranslation - ) -> ResolvedTextEntryInput? { - let point = CGPoint(x: x, y: y) - return textInputCandidatesAt(app: app, point: point, contentTranslation: contentTranslation).first + func textInputAt(app: XCUIApplication, x: Double, y: Double, contentTranslation: TextEntryContentTranslation) -> ResolvedTextEntryInput? { + textInputCandidatesAt(app: app, point: CGPoint(x: x, y: y), contentTranslation: contentTranslation).first } - func textInputCandidatesAt( - app: XCUIApplication, - point: CGPoint, - contentTranslation: TextEntryContentTranslation - ) -> [ResolvedTextEntryInput] { + func textInputCandidatesAt(app: XCUIApplication, point: CGPoint, contentTranslation: TextEntryContentTranslation) -> [ResolvedTextEntryInput] { safely("TEXT_INPUT_AT_POINT", []) { queryTextInputs(app: app, point: point, contentTranslation: contentTranslation, shouldStop: { false }) } } - func coordinateTapTextInputAt( - app: XCUIApplication, - x: Double, - y: Double, - contentTranslation: TextEntryContentTranslation - ) -> XCUIElement? { - switch probeTextInputs(app: app, point: CGPoint(x: x, y: y), contentTranslation: contentTranslation) { + func coordinateTapTextInputAt(app: XCUIApplication, x: Double, y: Double) -> XCUIElement? { + switch probeTextInputs(app: app, point: CGPoint(x: x, y: y), contentTranslation: liveContentTranslation(app: app)) { case .matches(let inputs): return inputs.first?.element case .absent: @@ -75,21 +60,14 @@ extension RunnerTests { let probed = containingTextInputProbeIssues(fallback: CoordinateTextInputResolution.unavailable) { shouldStop -> CoordinateTextInputResolution in guard let input = queryTextInputs( - app: app, - point: CGPoint(x: x, y: y), - contentTranslation: contentTranslation, - shouldStop: shouldStop + app: app, point: CGPoint(x: x, y: y), contentTranslation: contentTranslation, shouldStop: shouldStop ).first else { return .absent } guard let identity = input.identity else { return .unavailable } - return .input(TextInputAtPoint( - element: input.element, - identity: identity, - contentTranslation: contentTranslation - )) + return .input(TextInputAtPoint(element: input.element, identity: identity)) } switch probed { case .success(let resolution): @@ -100,14 +78,13 @@ extension RunnerTests { } /// `input`'s handle, read again, when it still resolves to the input it named when found, read - /// inside the same issue containment, carrying the vector the finding command read. A read that - /// cannot answer counts as no. + /// inside the same issue containment. A read that cannot answer counts as no. func textInputIfStillResolves( - _ input: TextInputAtPoint + _ input: TextInputAtPoint, + contentTranslation: TextEntryContentTranslation ) -> ResolvedTextEntryInput? { let probed = containingTextInputProbeIssues(fallback: nil as ResolvedTextEntryInput?) { _ in - ResolvedTextEntryInput(reading: input.element, contentTranslation: input.contentTranslation) - .flatMap { $0.identity == input.identity ? $0 : nil } + ResolvedTextEntryInput(reading: input.element, contentTranslation: contentTranslation).flatMap { $0.identity == input.identity ? $0 : nil } } guard case .success(let read) = probed else { return nil } return read @@ -123,11 +100,7 @@ extension RunnerTests { return read } - func probeTextInputs( - app: XCUIApplication, - point: CGPoint, - contentTranslation: TextEntryContentTranslation - ) -> TextInputProbeOutcome { + func probeTextInputs(app: XCUIApplication, point: CGPoint, contentTranslation: TextEntryContentTranslation) -> TextInputProbeOutcome { switch containingTextInputProbeIssues(fallback: [], { shouldStop in queryTextInputs(app: app, point: point, contentTranslation: contentTranslation, shouldStop: shouldStop) }) { @@ -140,7 +113,7 @@ extension RunnerTests { /// Runs an optional probe whose XCTest issues are contained instead of recorded: an issue makes /// the probe unavailable rather than failing the command and invalidating the runner. - func containingTextInputProbeIssues( + private func containingTextInputProbeIssues( fallback: T, _ probe: (_ shouldStop: () -> Bool) -> T ) -> Result { @@ -201,18 +174,10 @@ extension RunnerTests { // cannot be snapshotted is read live, so its failure still reaches the caller's issue handling. for element in candidates { if shouldStop() { break } - guard let candidate = ResolvedTextEntryInput( - readingListed: element, - contentTranslation: contentTranslation - ) else { continue } - if isCoordinateTextInputCandidate( - enabled: candidate.attributes.isEnabled, - // A candidate with no snapshot is matched on its live frame, which is already screen space - // and whose failure reaches the caller's issue handling. `frameOnScreen` is nil only when - // the command's window read could not answer, so the live frame is the only box left. - frame: candidate.frameOnScreen ?? candidate.element.frame, - point: point - ) { + guard let candidate = ResolvedTextEntryInput(readingListed: element, contentTranslation: contentTranslation) else { continue } + // Matching dispatches nothing, so a read with no screen box compares its live frame. + let frame = candidate.frameOnScreen ?? candidate.element.frame + if isCoordinateTextInputCandidate(enabled: candidate.attributes.isEnabled, frame: frame, point: point) { inputs.append(candidate) } } @@ -221,35 +186,15 @@ extension RunnerTests { return inputs.sorted { smallestElementFirst($0.attributes, $1.attributes) } } - /// The vector state that carries this session's snapshot-read app-content frames to their place on - /// the screen (#3899), read once per command at its start. A placed window's snapshot frames are - /// content-local (ADR 0004), so the window's own origin on the screen is the vector; a window - /// anchored at the origin — full screen, iPhone, and each ADR 0025 foldable panel — declares none. - /// - /// A read that cannot answer returns `.unknown`, never `.none`: on a placed window, a failed read - /// that looked like full screen would dispatch raw window-local points as screen points — the - /// #3899 failure — so consumers refuse `.unknown` instead of guessing through it. + /// The command's one content-vector read: the resolved window's screen origin when it is placed off + /// the screen origin (#3899). A read that cannot answer is `.unknown`, never `.none`. func liveContentTranslation(app: XCUIApplication) -> TextEntryContentTranslation { - // Contained like the probe lookups it wraps: a transient window that records an XCTest issue - // costs the vector, not the runner session. This is the one window query a command makes; the - // resolved window's box is enough because `SnapshotContentTranslation.offset` reads only the - // origins, and a placed window's size is its own. let read = containingTextInputProbeIssues(fallback: TextEntryContentTranslation.unknown) { _ in - guard let resolved = resolveRunnerWindowElement(app: app) else { return .unknown } - guard let offset = SnapshotContentTranslation.offset( - reportedBox: CGRect(origin: .zero, size: resolved.frame.size), - viewportBox: resolved.frame - ) else { - // Explicit type: `.none` alone means `Optional.none` to the compiler here. - return TextEntryContentTranslation.none - } - return .known(offset) - } - switch read { - case .success(let state): - return state - case .failure: - return .unknown + guard let window = resolveRunnerWindowElement(app: app)?.frame else { return .unknown } + return SnapshotContentTranslation.offset(reportedBox: CGRect(origin: .zero, size: window.size), viewportBox: window) + .map { .known($0) } ?? TextEntryContentTranslation.none } + guard case .success(let translation) = read else { return .unknown } + return translation } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift index 8e49b4a3e7..6017473904 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift @@ -27,12 +27,11 @@ extension RunnerTests { hasY: command.y != nil, xCTestChannelPenalized: xCTestChannelPenalized ), let x = command.x, let y = command.y { - // The one content-vector read for this command (#3899); the target built here carries it. - let contentTranslation = liveContentTranslation(app: activeApp) // Resolved before the tap: the focus it causes can move the field away from this point. // A replacement with no input at the point has nothing to bind to: its clear and its typed // text would reach the app-level first responder, so an empty point and a lookup that cannot // answer both refuse here, each under its own reason. + let contentTranslation = liveContentTranslation(app: activeApp) let inputAtPoint: TextInputAtPoint switch coordinateTapTextInputIdentityAt(app: activeApp, x: x, y: y, contentTranslation: contentTranslation) { case .input(let input): diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift index 4bce84d095..a91f188fee 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift @@ -6,12 +6,7 @@ extension RunnerTests { /// cannot answer. A fixture that does not expose its own field is a broken lane, not a refusal. @MainActor func requireTextInputAtPoint(app: XCUIApplication, x: Double, y: Double) throws -> TextInputAtPoint { - switch coordinateTapTextInputIdentityAt( - app: app, - x: x, - y: y, - contentTranslation: liveContentTranslation(app: app) - ) { + switch coordinateTapTextInputIdentityAt(app: app, x: x, y: y, contentTranslation: liveContentTranslation(app: app)) { case .input(let input): return input case .absent: @@ -116,23 +111,5 @@ extension RunnerTests { "the refusal dispatched its focus tap: the point's control took the press" ) } - - /// The initial focus tap dispatches the input's screen box only when the read can name one. - /// With no box — `.unknown` or a missing snapshot — it dispatches the caller's requested point, - /// never a live `element.frame` center (#3899). - func testFocusTapPointNeverDispatchesALiveFrame() { - let requested = CGPoint(x: 120, y: 640) - XCTAssertEqual( - Self.focusTapPoint(frameOnScreen: CGRect(x: 300, y: 600, width: 200, height: 40), requestedPoint: requested), - CGPoint(x: 400, y: 620), - "a known screen box moves the tap to its center" - ) - XCTAssertEqual(Self.focusTapPoint(frameOnScreen: nil, requestedPoint: requested), requested) - XCTAssertEqual( - Self.focusTapPoint(frameOnScreen: .null, requestedPoint: requested), - requested, - "an empty box names no center" - ) - } #endif } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift index 1510dce82a..8138eb41fb 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift @@ -662,133 +662,6 @@ extension RunnerTests { XCTAssertNotNil(keyQ, "`q` key must be presented under --depth \(options.depth ?? -1)") } - /// #3899: an iPad window that does not fill the screen publishes screen space. The measured shape is - /// iPad Pro 11-inch (M5, iOS 27.0) with Windowed Apps on and a Calendar window at (224,207,386x682): - /// the app's own window arrives as a (0,0)-anchored copy of that box and its subtree with it, so the - /// Add button reports (335,14,37,36) for a control drawn at (559,221). The one normalization pass - /// reads the declaration and publishes the subtree at its place on the screen, which is what lets the - /// fold keep every node instead of clipping each one to (w − x) × (h − y) and refusing the capture. - func testAWindowedSubtreeIsPublishedInScreenSpaceAndKeepsEveryNode() throws { - let windowOnScreen = CGRect(x: 224, y: 207, width: 386, height: 682) - func node( - _ index: Int, _ type: String, _ label: String, _ rect: CGRect, _ parent: Int?, _ depth: Int - ) -> RawAXNode { - RawAXNode( - index: index, type: type, label: label, identifier: nil, value: nil, - rect: SnapshotRect(x: rect.minX, y: rect.minY, width: rect.width, height: rect.height), - enabled: true, focused: nil, selected: nil, hittable: false, - depth: depth, parentIndex: parent, hiddenContentAbove: nil, hiddenContentBelow: nil - ) - } - let reportedRoot = CGRect(x: 0, y: 0, width: 386, height: 682) - let acquired = [ - node(0, "Application", "Calendar", reportedRoot, nil, 0), - node(1, "Window", "Window", reportedRoot, 0, 1), - node(2, "Other", "Toolbar", reportedRoot, 1, 2), - node(3, "Button", "Add", CGRect(x: 335, y: 14, width: 37, height: 36), 2, 3), - node(4, "Button", "Today", CGRect(x: 8, y: 583, width: 90, height: 38), 2, 3), - // A SpringBoard overlay under the app root reports the screen's own box: its subtree is already - // in screen space and must not move. - node(5, "Window", "Alert", CGRect(x: 0, y: 0, width: 834, height: 1210), 0, 1), - node(6, "Button", "Discard Changes", CGRect(x: 300, y: 500, width: 240, height: 44), 5, 2), - ] - let normalized = SnapshotGeometrySpace.normalized( - nodes: acquired, - viewport: .reported( - box: windowOnScreen, - interfaceOrientation: RunnerInterfaceOrientation.portrait - ) - ) - let options = PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false) - let capture = try XCTUnwrap(try SnapshotPresentation.present( - SnapshotAcquisition( - hint: SnapshotPresentation.captureHint(for: options), - nodes: normalized, - truncated: false, - effectiveDepth: nil, - viewport: .reported(box: windowOnScreen) - ), - options: options - )) - - // One space: the root answers with the window's screen box, and the button with the position the - // screenshot draws. - XCTAssertEqual(capture.nodes.first?.rect, SnapshotRect(windowOnScreen)) - let add = try XCTUnwrap(capture.nodes.first { $0.label == "Add" }) - XCTAssertEqual(add.rect, SnapshotRect(x: 559, y: 221, width: 37, height: 36)) - XCTAssertEqual(add.hittable, true) - // Nothing is dropped for being "outside the clip": the whole acquired tree is still there. The one - // clipped node is the system overlay that really is larger than the app's window, which is the - // fold's normal answer to oversized content — not the mixed-space clip that used to publish - // (w − x) × (h − y) boxes and refuse this capture outright. - XCTAssertEqual(Set(capture.nodes.map(\.label)), Set(acquired.map(\.label))) - let clipNote = capture.qualityNotes.first { $0.code == .regularGeometryClipped } - XCTAssertEqual(clipNote?.droppedNodeCount, 0, "no node is dropped for the window's origin") - XCTAssertEqual(clipNote?.clippedNodeCount, 1, "only the overlay larger than the window is clipped") - XCTAssertEqual( - capture.nodes.first { $0.label == "Alert" }?.rect, - SnapshotRect(x: 224, y: 207, width: 386, height: 682), - "the overlay is clipped to the window's visible box, not to (w − x) × (h − y) of it" - ) - // The system window and its button stay where the platform reported them. - let discard = try XCTUnwrap(capture.nodes.first { $0.label == "Discard Changes" }) - XCTAssertEqual(discard.rect, SnapshotRect(x: 300, y: 500, width: 240, height: 44)) - } - - /// #3899 keeps a full-screen capture byte-identical: the same tree reported from the screen origin is - /// published exactly as reported, with no translation read from its zero-origin viewport. - func testAFullScreenCapturePublishesWhatThePlatformReported() throws { - let screen = CGRect(x: 0, y: 0, width: 834, height: 1210) - func node( - _ index: Int, _ type: String, _ label: String, _ rect: CGRect, _ parent: Int?, _ depth: Int - ) -> RawAXNode { - RawAXNode( - index: index, type: type, label: label, identifier: nil, value: nil, - rect: SnapshotRect(x: rect.minX, y: rect.minY, width: rect.width, height: rect.height), - enabled: true, focused: nil, selected: nil, hittable: false, - depth: depth, parentIndex: parent, hiddenContentAbove: nil, hiddenContentBelow: nil - ) - } - let acquired = [ - node(0, "Application", "Calendar", screen, nil, 0), - node(1, "Window", "Window", screen, 0, 1), - node(2, "Button", "Add", CGRect(x: 335, y: 14, width: 37, height: 36), 1, 2), - ] - let normalized = SnapshotGeometrySpace.normalized( - nodes: acquired, - viewport: .reported( - box: screen, - interfaceOrientation: RunnerInterfaceOrientation.portrait - ) - ) - XCTAssertEqual(normalized.map(\.rect), acquired.map(\.rect)) - XCTAssertEqual( - SnapshotContentTranslation.offset(reportedBox: screen, viewportBox: screen), - nil, - "a viewport at the screen origin carries no translation" - ) - } - - /// #3899 keeps ADR 0025 foldable panel taps: a panel reports its own box from its own display's - /// origin, so no translation is read and the panel's geometry stays exactly as measured. - func testAFoldablePanelReportsNoContentTranslation() { - let panel = CGRect(x: 0, y: 0, width: 951, height: 669) - XCTAssertNil( - SnapshotContentTranslation.offset(reportedBox: panel, viewportBox: panel), - "a panel box anchored at its display's origin names no translation" - ) - // A digitizer point on the panel is passed through, so the fold's display selection stays the only - // window-dependent choice the tap path makes. - XCTAssertEqual( - CoordinateSpaceRotation.native( - point: CGPoint(x: 476, y: 226), - in: panel, - interfaceOrientation: RunnerInterfaceOrientation.portrait - ), - CGPoint(x: 476, y: 226) - ) - } - /// #2891: a capture whose viewport read failed publishes no `hittable` for a node whose answer is /// containment, while a root and a disabled node stay a declared `false`. Pinned on the /// encoded objects rather than on `capture.nodes`, because the promise is about the wire: a test diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift index c056141e8c..3507d57076 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift @@ -167,6 +167,21 @@ extension RunnerTests { XCTAssertEqual(displayID, 0) } } + + /// #3899: a synthesized turn swaps the display's side lengths, never a placed window's smaller box. + func testSynthesizedRotationFrameIsTheDisplayNotAPlacedWindow() throws { + try XCTSkipUnless(XCUIScreen.screens.count == 1, "a multi-screen host keeps its panel's window box") + let placedWindow = CGRect(x: 224, y: 207, width: 386, height: 682) + let screen = UIScreen.main.bounds.size + XCTAssertEqual( + synthesizedRotationFrame(fallback: placedWindow, interfaceOrientation: RunnerInterfaceOrientation.portrait), + CGRect(origin: .zero, size: screen) + ) + XCTAssertEqual( + synthesizedRotationFrame(fallback: placedWindow, interfaceOrientation: RunnerInterfaceOrientation.landscapeRight), + CGRect(x: 0, y: 0, width: screen.height, height: screen.width) + ) + } #endif func testSoleScreenReferenceFrameRefusesAnUnknownOrientationOrAnEmptyScreen() { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift index 4b22ac1329..e5411bfc5d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift @@ -316,11 +316,10 @@ extension RunnerTests { XCTAssertEqual(rebound.identifier, "agent-device-text-entry-neighbour") let neighbourFrame = neighbour.frame let found = [ - "re-bound": TextInputAtPoint(element: rebound, identity: TextEntryInputIdentity(elementType: .textField, identifier: ""), contentTranslation: .none), + "re-bound": TextInputAtPoint(element: rebound, identity: TextEntryInputIdentity(elementType: .textField, identifier: "")), "missing": TextInputAtPoint( element: app.textFields["agent-device-gone"], - identity: TextEntryInputIdentity(elementType: .textField, identifier: "agent-device-gone"), - contentTranslation: .none + identity: TextEntryInputIdentity(elementType: .textField, identifier: "agent-device-gone") ), ] @@ -364,11 +363,10 @@ extension RunnerTests { element: nil, refreshPoint: CGPoint(x: neighbourFrame.midX, y: neighbourFrame.midY), prefersFocusedElement: false, - contentTranslation: .none, + contentTranslation: .none, inputAtRefreshPoint: TextInputAtPoint( element: rebound, - identity: TextEntryInputIdentity(elementType: .textField, identifier: field.identifier), - contentTranslation: .none + identity: TextEntryInputIdentity(elementType: .textField, identifier: field.identifier) ) ), text: "", @@ -400,11 +398,10 @@ extension RunnerTests { element: nil, refreshPoint: CGPoint(x: neighbourFrame.midX, y: neighbourFrame.midY), prefersFocusedElement: false, - contentTranslation: .none, + contentTranslation: .none, inputAtRefreshPoint: TextInputAtPoint( element: app.textFields.element, - identity: TextEntryInputIdentity(elementType: .textField, identifier: ""), - contentTranslation: .none + identity: TextEntryInputIdentity(elementType: .textField, identifier: "") ) ), text: "", diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift index a329e86a27..9b94a9df90 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift @@ -109,12 +109,7 @@ extension RunnerTests { "software keyboard is down: this simulator cannot show the keyboard a focus tap raises" ) - let stabilized = stabilizeTextInputBeforeTyping( - app: app, - target: field, - contentTranslation: .none, - keyboardVisibleBeforeTap: false - ) + let stabilized = stabilizeTextInputBeforeTyping(app: app, target: field, contentTranslation: .none, keyboardVisibleBeforeTap: false) guard case .inputReportsKeyboardFocus(let read) = stabilized.evidence else { return XCTFail("focus confirmed by \(stabilized.evidence?.name ?? "nothing"), not by the tapped input") diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryReadinessTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryReadinessTests.swift index 54940d964b..f42559ae14 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryReadinessTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryReadinessTests.swift @@ -2,17 +2,10 @@ import XCTest extension RunnerTests { #if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) - /// The readiness retap dispatches `ResolvedTextEntryInput.frameOnScreen`'s center, so the snapshot - /// frame of a placed window passes through the read's content vector before it becomes a screen - /// point (#3899). Before that rule the retap tapped the bare `snapshot().frame` center, which on - /// a windowed iPad lands outside the window, and the fill after it failed with - /// TEXT_INPUT_NOT_FOCUSED — the issue's own failure shape. - /// - /// The fixture cannot be placed in a window, so the read is driven directly: the vector moves the - /// field's real snapshot frame exactly onto the press-counting witness button below it. A point - /// dispatched at `frameOnScreen`'s center presses that button; a point dispatched at the raw - /// snapshot center taps the field and moves no counter. The same read with no vector is the - /// closest negative: a full-screen app retaps the field itself and the counter stays put. + /// #3899: the readiness retap dispatches the read's screen box, not its raw snapshot frame. The + /// fixture cannot be placed in a window, so a `.known` vector moves the field's snapshot frame onto + /// the press-counting witness button: only a retap through `frameOnScreen` presses it. An + /// `.unknown` vector names no box and taps nothing; `.none` retaps the field itself. @MainActor func testReadinessRetapDispatchesTheReadsScreenSpaceCenter() throws { app.launchArguments = ["--agent-device-text-entry-regression", "--agent-device-text-entry-tap-witness"] @@ -20,58 +13,25 @@ extension RunnerTests { defer { invalidateCachedTarget(reason: "unit_test_cleanup"); app.terminate() } XCTAssertTrue(app.waitForFixtureExistence(timeout: appExistenceTimeout)) let field = app.textFields["agent-device-hardware-keyboard-input"] - XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout)) let witness = app.buttons["agent-device-tap-witness"] - XCTAssertTrue(witness.waitForFixtureExistence(timeout: appExistenceTimeout)) let presses = app.staticTexts["agent-device-tap-witness-presses"] - XCTAssertTrue(presses.waitForFixtureExistence(timeout: appExistenceTimeout)) + for element in [field, witness, presses] { + XCTAssertTrue(element.waitForFixtureExistence(timeout: appExistenceTimeout)) + } let fieldFrame = field.frame let witnessFrame = witness.frame - XCTAssertFalse(fieldFrame.isEmpty) - XCTAssertFalse(witnessFrame.isEmpty) - // Closest negative: a full-screen app — a vector of `.none` — retaps the field's own center, - // which is not the button's point. - let untranslated = try XCTUnwrap(ResolvedTextEntryInput(reading: field, contentTranslation: .none)) - retapAt(read: untranslated, app: app, element: field) - XCTAssertEqual(presses.label, "presses=0", "a vectorless read dispatched a point off the field") + let unknown = try XCTUnwrap(ResolvedTextEntryInput(reading: field, contentTranslation: .unknown)) + XCTAssertNil(unknown.frameOnScreen) + retapAt(read: unknown, app: app, element: field) + XCTAssertEqual(presses.label, "presses=0", "an unknown content space dispatched a point") - // The read a probe makes on a placed window: the snapshot reports the field's box and the read - // carries the window's vector. Here the vector is chosen so the translated box is the button. - let vector = CGSize( - width: witnessFrame.midX - fieldFrame.midX, - height: witnessFrame.midY - fieldFrame.midY - ) - let translated = try XCTUnwrap(ResolvedTextEntryInput(reading: field, contentTranslation: .known(vector))) - XCTAssertEqual(try XCTUnwrap(translated.frameOnScreen).midX, witnessFrame.midX, accuracy: 0.5) - XCTAssertEqual(try XCTUnwrap(translated.frameOnScreen).midY, witnessFrame.midY, accuracy: 0.5) + retapAt(read: try XCTUnwrap(ResolvedTextEntryInput(reading: field, contentTranslation: .none)), app: app, element: field) + XCTAssertEqual(presses.label, "presses=0", "a full-screen read dispatched a point off the field") - retapAt(read: translated, app: app, element: field) - XCTAssertEqual( - presses.label, - "presses=1", - "the retap dispatched the raw snapshot center, not the read's screen-space center (#3899)" - ) - } - - /// A vector read that could not answer is `.unknown`, and `frameOnScreen` names no box through it: - /// a snapshot frame of a placed window is not known to be a screen box, so the readiness retap - /// dispatches nothing rather than a raw window-local point — the #3899 failure shape. A transient - /// window issue used to look like full screen here and move the retap outside the window. - @MainActor - func testReadinessRetapTapsNothingWhenTheContentSpaceIsUnknown() throws { - app.launchArguments = ["--agent-device-text-entry-regression", "--agent-device-text-entry-tap-witness"] - app.launch() - defer { invalidateCachedTarget(reason: "unit_test_cleanup"); app.terminate() } - XCTAssertTrue(app.waitForFixtureExistence(timeout: appExistenceTimeout)) - let field = app.textFields["agent-device-hardware-keyboard-input"] - XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout)) - let presses = app.staticTexts["agent-device-tap-witness-presses"] - XCTAssertTrue(presses.waitForFixtureExistence(timeout: appExistenceTimeout)) - let read = try XCTUnwrap(ResolvedTextEntryInput(reading: field, contentTranslation: .unknown)) - XCTAssertNil(read.frameOnScreen) - retapAt(read: read, app: app, element: field) - XCTAssertEqual(presses.label, "presses=0", "an unknown content space dispatched a point (#3899)") + let vector = CGSize(width: witnessFrame.midX - fieldFrame.midX, height: witnessFrame.midY - fieldFrame.midY) + retapAt(read: try XCTUnwrap(ResolvedTextEntryInput(reading: field, contentTranslation: .known(vector))), app: app, element: field) + XCTAssertEqual(presses.label, "presses=1", "the retap dispatched the raw snapshot center, not the screen-space one") } #endif } diff --git a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift index 90cd0d5488..8d441911ce 100644 --- a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift +++ b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift @@ -4,9 +4,6 @@ import CoreGraphics // Rotation between an app's interface space and the device's native (portrait-up) space, shared by // synthesized dispatch (forward) and capture (inverse). Rationale and measurements: ADR 0004 // "the coordinate space of a captured subtree" and contracts/fixtures/window-coordinate-space.json. -// -// Rotation moves axes only and never an origin; a placed window's offset from the screen belongs -// to `SnapshotContentTranslation` (#3899). public enum RunnerInterfaceOrientation { public static let unknown = 0 @@ -18,8 +15,7 @@ public enum RunnerInterfaceOrientation { public enum CoordinateSpaceRotation { // Dispatch turns a point's axes and never moves its origin: the digitizer's origin is the - // display's own, which is where a windowed app's screen-space points are already measured from - // (#3899). `frame` names the side lengths the turn swaps, nothing more. + // display's own, where a placed window's screen points are already measured from (#3899). public static func native( point: CGPoint, in frame: CGRect, @@ -61,23 +57,23 @@ public enum CoordinateSpaceRotation { ) -> CGPoint { let width = Double(frame.width) let height = Double(frame.height) - let x: Double - let y: Double + let localX: Double + let localY: Double switch interfaceOrientation { case RunnerInterfaceOrientation.landscapeRight: - x = Double(point.y) - y = height - Double(point.x) + localX = Double(point.y) + localY = height - Double(point.x) case RunnerInterfaceOrientation.landscapeLeft: - x = width - Double(point.y) - y = Double(point.x) + localX = width - Double(point.y) + localY = Double(point.x) case RunnerInterfaceOrientation.portraitUpsideDown: - x = width - Double(point.x) - y = height - Double(point.y) + localX = width - Double(point.x) + localY = height - Double(point.y) default: - x = Double(point.x) - y = Double(point.y) + localX = Double(point.x) + localY = Double(point.y) } - return CGPoint(x: x, y: y) + return CGPoint(x: localX, y: localY) } public static func oriented( @@ -104,34 +100,16 @@ public enum CoordinateSpaceRotation { } } -/// The vector from a carrier's reported box to the screen space it is placed at (#3899). -/// -/// The signature is a `(0,0)`-anchored copy of the viewport box whose viewport starts elsewhere on -/// the screen; any other carrier box is not the app's own window, so its numbers say nothing about -/// where its subtree starts. Measured rows and the full rationale: ADR 0004 and the -/// `windowContentCases` table this replays against. +/// A placed iPad window reports its subtree from its own top-left (#3899, ADR 0004). Its carrier is a +/// `(0,0)`-anchored copy of a viewport box that starts elsewhere; the viewport origin is the offset. public enum SnapshotContentTranslation { - /// `nil` when the carrier's box declares no translation. A full-screen app, an iPhone, and each - /// foldable panel report origins at zero on both sides, so the rule reads none there and every - /// published rect stays as the platform reported it. public static func offset(reportedBox: CGRect, viewportBox: CGRect) -> CGSize? { - guard SnapshotGeometry.isPositiveFinite(reportedBox), - SnapshotGeometry.isPositiveFinite(viewportBox) - else { - return nil - } - guard abs(reportedBox.minX) <= SnapshotGeometrySpace.quarterTurnTolerance, - abs(reportedBox.minY) <= SnapshotGeometrySpace.quarterTurnTolerance - else { - return nil - } - guard abs(viewportBox.minX) > SnapshotGeometrySpace.quarterTurnTolerance - || abs(viewportBox.minY) > SnapshotGeometrySpace.quarterTurnTolerance - else { - return nil - } - guard abs(reportedBox.width - viewportBox.width) <= SnapshotGeometrySpace.quarterTurnTolerance, - abs(reportedBox.height - viewportBox.height) <= SnapshotGeometrySpace.quarterTurnTolerance + let tolerance = SnapshotGeometrySpace.quarterTurnTolerance + func near(_ a: CGFloat, _ b: CGFloat) -> Bool { abs(a - b) <= tolerance } + guard SnapshotGeometry.isPositiveFinite(reportedBox), SnapshotGeometry.isPositiveFinite(viewportBox), + near(reportedBox.minX, 0), near(reportedBox.minY, 0), + !near(viewportBox.minX, 0) || !near(viewportBox.minY, 0), + near(reportedBox.width, viewportBox.width), near(reportedBox.height, viewportBox.height) else { return nil } @@ -142,9 +120,6 @@ public enum SnapshotContentTranslation { public enum SnapshotGeometrySpace: Equatable { case appOrientation case deviceNative(appFrame: SnapshotViewport.Box, interfaceOrientation: Int) - /// The subtree was reported in a placed window's own content space and is published translated - /// into the viewport's screen space (`SnapshotContentTranslation`, #3899). The offset is the - /// vector from the window's reported box to the viewport box it sits at on screen. case windowContent(offset: CGSize) public static let quarterTurnTolerance: Double = 1 @@ -164,8 +139,8 @@ public enum SnapshotGeometrySpace: Equatable { } } - /// A `.reported` viewport is the screen-anchored fact both declarations read against; a `.derived` - /// viewport carries no screen anchor, so a subtree under it publishes what the platform reported. + /// Only a `.reported` viewport is anchored on the screen, so only it can anchor a rotation or a + /// translation. public static func space( reportedBySurfaceHost isSurfaceHost: Bool, reportedFrame: CGRect, @@ -176,20 +151,13 @@ public enum SnapshotGeometrySpace: Equatable { guard case .reported(let appFrame, let interfaceOrientation) = viewport else { return .appOrientation } - if namesQuarterTurn(interfaceOrientation), - isQuarterTurned(reportedFrame, relativeTo: appFrame.rect) { + if namesQuarterTurn(interfaceOrientation), isQuarterTurned(reportedFrame, relativeTo: appFrame.rect) { return .deviceNative(appFrame: appFrame, interfaceOrientation: interfaceOrientation) } - if let offset = SnapshotContentTranslation.offset( - reportedBox: reportedFrame, - viewportBox: appFrame.rect - ) { - return .windowContent(offset: offset) - } - // A system surface under a placed window — the keyboard, a SpringBoard overlay — reports the - // screen's own box and is already in screen space; `windowContentCases` pins that inheriting - // the carrier's vector would move every key by the window origin. - return .appOrientation + // A system window under a placed one (the keyboard, a SpringBoard overlay) reports the screen's + // own box, so it names no offset and keeps its frames. + return SnapshotContentTranslation.offset(reportedBox: reportedFrame, viewportBox: appFrame.rect) + .map { .windowContent(offset: $0) } ?? .appOrientation } public static func namesQuarterTurn(_ interfaceOrientation: Int) -> Bool { diff --git a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift index 817429ad22..897bf4fe28 100644 --- a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift +++ b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift @@ -29,17 +29,13 @@ final class CoordinateSpaceTests: XCTestCase { } struct WindowContentCase: Decodable { + struct Offset: Decodable { let width, height: Double } let name: String let reportedBox: Frame let viewportBox: Frame let offset: Offset? } - struct Offset: Decodable { - let width: Double - let height: Double - } - let constants: Constants let quarterTurnCases: [QuarterTurnCase] let rotationCases: [RotationCase] @@ -316,34 +312,18 @@ final class CoordinateSpaceTests: XCTestCase { ) } - /// Golden parity table (#2612, #3899): every case in contracts/fixtures/window-coordinate-space.json - /// must agree here, and the quarter-turn rows with the vitest twin - /// (packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts). The - /// window-content rows replay here alone: the host channel's guest refuses a windowed tree - /// before its decoder can read one, so this capture is the rule's only production owner. Add - /// cases to the table, never fork the rule. Detection is replayed under both landscape turns, because + /// Golden parity table (#2612): every case in contracts/fixtures/window-coordinate-space.json + /// must agree with the vitest twin + /// (packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts). Add + /// cases there, never fork the rule. Detection is replayed under both landscape turns, because /// which window declares the native space cannot depend on which way the app is turned; the - /// rotation rows carry their own orientation. + /// rotation rows carry their own orientation. The window-content rows (#3899) replay here alone: + /// the bridge guest refuses a windowed tree before the vitest twin's decoder can read one. func testWindowCoordinateSpaceMatchesGoldenParityTable() throws { let fixture = try loadWindowCoordinateSpaceFixture() XCTAssertFalse(fixture.quarterTurnCases.isEmpty, "parity table must not be empty") XCTAssertFalse(fixture.rotationCases.isEmpty, "parity table must not be empty") XCTAssertFalse(fixture.windowContentCases.isEmpty, "parity table must not be empty") - XCTAssertEqual( - Set(fixture.quarterTurnCases.map(\.name)).count, - fixture.quarterTurnCases.count, - "parity table case names must be unique" - ) - XCTAssertEqual( - Set(fixture.rotationCases.map(\.name)).count, - fixture.rotationCases.count, - "parity table case names must be unique" - ) - XCTAssertEqual( - Set(fixture.windowContentCases.map(\.name)).count, - fixture.windowContentCases.count, - "parity table case names must be unique" - ) XCTAssertEqual( fixture.constants.quarterTurnTolerance, SnapshotGeometrySpace.quarterTurnTolerance, @@ -381,13 +361,9 @@ final class CoordinateSpaceTests: XCTestCase { ) } for testCase in fixture.windowContentCases { - let expected = testCase.offset.map { CGSize(width: $0.width, height: $0.height) } XCTAssertEqual( - SnapshotContentTranslation.offset( - reportedBox: testCase.reportedBox.cgRect, - viewportBox: testCase.viewportBox.cgRect - ), - expected, + SnapshotContentTranslation.offset(reportedBox: testCase.reportedBox.cgRect, viewportBox: testCase.viewportBox.cgRect), + testCase.offset.map { CGSize(width: $0.width, height: $0.height) }, testCase.name ) } @@ -528,71 +504,14 @@ final class CoordinateSpaceTests: XCTestCase { ) } - /// The measured windowed iPad Calendar window (#3899) driven through the pass: the carrier reports - /// a `(0,0)`-anchored copy of the viewport box, declares its subtree window-relative, and every - /// rect under it comes back at its place on the screen — the root at the window's screen box, the - /// Add button at (559,221) where the screenshot draws it, and a full-screen-capture tree untouched. - func testNormalizedTranslatesAWindowedSubtreeIntoScreenSpace() { + /// #3899: a placed window's subtree publishes at its screen position, while the system windows + /// under it and a window at the screen origin keep what the platform reported. + func testNormalizedTranslatesOnlyAPlacedWindowsSubtree() { let windowOnScreen = CGRect(x: 224, y: 207, width: 386, height: 682) let reportedRoot = CGRect(x: 0, y: 0, width: 386, height: 682) - let reportedAddButton = CGRect(x: 335, y: 14, width: 37, height: 36) - let acquired = [ - coordinateNode(0, "Application", reportedRoot, nil, 0, label: "Calendar"), - coordinateNode(1, "Window", reportedRoot, 0, 1), - coordinateNode(2, "Other", reportedRoot, 1, 2), - coordinateNode(3, "Button", reportedAddButton, 2, 3, label: "Add"), - ] - let normalized = SnapshotGeometrySpace.normalized( - nodes: acquired, - viewport: .reported(box: windowOnScreen, interfaceOrientation: RunnerInterfaceOrientation.portrait) - ) - XCTAssertEqual(normalized[0].rect.cgRect, windowOnScreen) - XCTAssertEqual(normalized[1].rect.cgRect, windowOnScreen) - XCTAssertEqual(normalized[2].rect.cgRect, windowOnScreen) - XCTAssertEqual(normalized[3].rect.cgRect, CGRect(x: 559, y: 221, width: 37, height: 36)) - XCTAssertEqual(normalized[3].hittable, true) - - // The same window moved: the subtree follows the new origin. - let moved = SnapshotGeometrySpace.normalized( - nodes: acquired, - viewport: .reported( - box: CGRect(x: 120, y: 90, width: 386, height: 682), - interfaceOrientation: RunnerInterfaceOrientation.portrait - ) - ) - XCTAssertEqual(moved[3].rect.cgRect, CGRect(x: 455, y: 104, width: 37, height: 36)) - } - - /// What must not move: a full-screen capture, and any subtree under a carrier that is not the app's - /// own placed window. Those trees keep exactly the rect the platform reported (#3899 keeps - /// full-screen and ADR 0025 panel behavior identical). - func testNormalizedTranslatesNothingForFullScreenAndForeignCarriers() { - // Full-screen iPad: reported boxes already start at the viewport. - let fullScreen = CGRect(x: 0, y: 0, width: 834, height: 1210) let acquired = [ - coordinateNode(0, "Application", fullScreen, nil, 0), - coordinateNode(1, "Window", fullScreen, 0, 1), - coordinateNode(2, "Button", CGRect(x: 335, y: 14, width: 37, height: 36), 1, 2), - ] - XCTAssertEqual( - SnapshotGeometrySpace.normalized( - nodes: acquired, - viewport: .reported( - box: fullScreen, - interfaceOrientation: RunnerInterfaceOrientation.portrait - ) - ).map(\.rect), - acquired.map(\.rect) - ) - - // A windowed app, measured shape: the app's own window declares the translation and carries the - // subtree; the keyboard window and the SpringBoard overlay report the screen's own box, so their - // subtrees stay where the platform put them. - let windowOnScreen = CGRect(x: 224, y: 207, width: 386, height: 682) - let reportedRoot = CGRect(x: 0, y: 0, width: 386, height: 682) - let mixed = [ coordinateNode(0, "Application", reportedRoot, nil, 0), - coordinateNode(1, "Window", reportedRoot, 0, 1, label: "appWindow"), + coordinateNode(1, "Window", reportedRoot, 0, 1), coordinateNode(2, "Other", reportedRoot, 1, 2), coordinateNode(3, "Button", CGRect(x: 335, y: 14, width: 37, height: 36), 2, 3, label: "Add"), coordinateNode(4, "Window", CGRect(x: 0, y: 925, width: 834, height: 282), 0, 1, label: "keyboard"), @@ -600,71 +519,18 @@ final class CoordinateSpaceTests: XCTestCase { coordinateNode(6, "Window", CGRect(x: 0, y: 0, width: 834, height: 1210), 0, 1, label: "springboard"), coordinateNode(7, "Alert", CGRect(x: 300, y: 500, width: 240, height: 160), 6, 2), ] - let normalized = SnapshotGeometrySpace.normalized( - nodes: mixed, - viewport: .reported(box: windowOnScreen, interfaceOrientation: RunnerInterfaceOrientation.portrait) - ) - XCTAssertEqual(normalized[1].rect.cgRect, windowOnScreen, "appWindow") - XCTAssertEqual(normalized[2].rect.cgRect, windowOnScreen) - XCTAssertEqual(normalized[3].rect.cgRect, CGRect(x: 559, y: 221, width: 37, height: 36)) - XCTAssertEqual(normalized[4].rect, mixed[4].rect, "keyboard") - XCTAssertEqual(normalized[5].rect, mixed[5].rect) - XCTAssertEqual(normalized[6].rect, mixed[6].rect, "springboard") - XCTAssertEqual(normalized[7].rect, mixed[7].rect) - } - - /// The dispatch direction for a windowed app: the caller's point is the point touched, so - /// `native(point:)` turns a screen-space point's axes without moving it toward the window origin - /// (#3899). Under portrait — the only orientation measured for a windowed app — it is the identity. - /// The basis row below pins the rule the dispatch follows in landscape: the digitizer turns about - /// the display's side lengths, so a windowed app's point turns about the display's box, not the - /// placed window's. Landscape under Windowed Apps is an unmeasured gap (ADR 0004). - func testNativePointForAWindowedAppKeepsTheCallersScreenPoint() { - let windowOnScreen = CGRect(x: 224, y: 207, width: 386, height: 682) - for orientation in [ - RunnerInterfaceOrientation.portrait, RunnerInterfaceOrientation.unknown - ] { - XCTAssertEqual( - CoordinateSpaceRotation.native( - point: CGPoint(x: 578, y: 239), - in: windowOnScreen, - interfaceOrientation: orientation - ), - CGPoint(x: 578, y: 239), - "interfaceOrientation \(orientation)" - ) + func normalizedRects(viewport: CGRect) -> [CGRect] { + SnapshotGeometrySpace.normalized( + nodes: acquired, + viewport: .reported(box: viewport, interfaceOrientation: RunnerInterfaceOrientation.portrait) + ).map(\.rect.cgRect) } - } - - /// A windowed landscape point turns about the display's box: the turn stays inside the display's - /// native portrait box, and a placed window's smaller box would move the point instead. The - /// runner passes the display's box as the basis (`synthesizedRotationFrame`). - func testNativePointInWindowedLandscapeTurnsAboutTheDisplayBox() { - let displayLandscapeBox = CGRect(x: 0, y: 0, width: 1210, height: 834) - let windowOnScreen = CGRect(x: 224, y: 207, width: 386, height: 682) - let screenPoint = CGPoint(x: 600, y: 300) - XCTAssertEqual( - CoordinateSpaceRotation.native( - point: screenPoint, - in: displayLandscapeBox, - interfaceOrientation: RunnerInterfaceOrientation.landscapeRight - ), - CGPoint(x: 534, y: 600), - "the display basis lands inside the display's native portrait box (834 x 1210)" - ) - XCTAssertNotEqual( - CoordinateSpaceRotation.native( - point: screenPoint, - in: windowOnScreen, - interfaceOrientation: RunnerInterfaceOrientation.landscapeRight - ), - CoordinateSpaceRotation.native( - point: screenPoint, - in: displayLandscapeBox, - interfaceOrientation: RunnerInterfaceOrientation.landscapeRight - ), - "the placed window's side lengths are not the digitizer's basis" - ) + let reported = acquired.map(\.rect.cgRect) + let placed = normalizedRects(viewport: windowOnScreen) + XCTAssertEqual(Array(placed[0...2]), [windowOnScreen, windowOnScreen, windowOnScreen]) + XCTAssertEqual(placed[3], CGRect(x: 559, y: 221, width: 37, height: 36), "Add draws at (559,221)") + XCTAssertEqual(Array(placed[4...]), Array(reported[4...]), "system windows already report screen space") + XCTAssertEqual(normalizedRects(viewport: reportedRoot), reported, "a window at the screen origin is unchanged") } func testNormalizedOfAnEmptyArrayIsEmpty() { diff --git a/contracts/fixtures/window-coordinate-space.json b/contracts/fixtures/window-coordinate-space.json index dbd6bfd551..0ba25479c7 100644 --- a/contracts/fixtures/window-coordinate-space.json +++ b/contracts/fixtures/window-coordinate-space.json @@ -1,5 +1,5 @@ { - "description": "Golden table for the coordinate space a captured window's own box declares (#2612), the twin of tap-keyboard-occlusion-policy.json on the geometry that guard reads. iOS hosts some system surfaces in the device's native (portrait-up) space even while the app is rotated, so their whole subtree arrives quarter-turned: measured on iPhone 17 Pro (iOS 26.2) with the system keyboard up over a landscape app frame (0,0,874,402), UIRemoteKeyboardWindow reports its own box as (0,0,402,874) — the app's box with the two side lengths swapped — while the app's own UIWindow, UITextEffectsWindow and a dev-menu overlay window all report (0,0,874,402); in portrait the same keyboard window reports (0,0,402,874) against a (0,0,402,874) app frame, i.e. nothing is turned. RULE 1 (detection) is enforced twice, in packages/contracts/src/window-coordinate-space.ts and in SnapshotGeometrySpace inside apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift, and decides from side lengths alone: a window is quarter-turned relative to an app frame when the app frame is not square (abs(app.width - app.height) > quarterTurnTolerance), both frames are finite, positive and non-empty, and the window's side lengths match the app's on the swapped axes (abs(window.width - app.height) <= quarterTurnTolerance && abs(window.height - app.width) <= quarterTurnTolerance). Origins are deliberately never compared: the turn is a statement about which axis a box runs along, and a window may legitimately be placed elsewhere. A square app frame cannot be told from its own quarter turn, so it is left alone rather than guessed at. The tolerance is 1 because measured captures match the swap to the point and this absorbs float representation only. RULE 2 (rotation) is the capture's way back for the runner's own UIInterfaceOrientation raw values — 1 portrait, 2 portraitUpsideDown, 3 landscapeRight, 4 landscapeLeft, and 0 unknown, where portrait and unknown publish what the platform reported rather than inventing a turn — implemented by CoordinateSpaceRotation.oriented(rect:in:interfaceOrientation:) in that same Swift file, whose rect helper turns opposite corners of the reported box and re-orders them so the result keeps a top-left origin. `rotationCases` are native -> oriented rows: the landscape ones measured on the device above, the 180 degree and identity ones derived from that table. Capture turns back with the exact inverse of what dispatch turns forward, so each row also has to come back to its own `native` box under the forward mapping; that is why one measured-looking landscape-left pairing whose native and oriented boxes are identical is absent — the table computes no such row. Infinity has no JSON spelling, so the unusable frame a platform hands back when it could not resolve a box is named {\"infinite\": true} and decodes to CGRect.infinite on the Swift side and to an infinite rect on the TypeScript side. RULE 3 (content translation) is the other space a captured subtree arrives in, and the one rule in this file that reads an origin. An iPad window that does not fill the screen (Windowed Apps, Stage Manager) reports its own screen box while the subtree under it arrives anchored at its own top-left: measured on iPad Pro 11-inch (M5, iOS 27.0) with a Calendar window at screen (224,207,386x682), the application root reports (0,0,386,682) and its Add button (335,14,37,36) for a control drawn at screen (559,221). Published as reported, that tree mixes two spaces, clips its own content, and moves every tap by the window origin (#3899). SnapshotContentTranslation.offset(reportedBox:viewportBox:) reads the carrier box against the viewport box the platform reported, and names the viewport origin as the offset only when the carrier is the app's own window: a (0,0)-anchored box whose side lengths match the viewport's within the same tolerance, while the viewport starts somewhere else on the screen. A carrier that is not the app's own window names no space, so its numbers are left alone: the keyboard window and a SpringBoard overlay under a windowed app report the screen's own box and are already in screen space. A full-screen window, an iPhone, and each foldable panel of ADR 0025 report origins at zero on both sides, so the rule reads no offset there and every published rect stays exactly as it was measured before. `windowContentCases` are reportedBox + viewportBox to offset rows, replayed against that Swift function alone; a null offset means the rule declares no translation. The host decoder (packages/platform-apple/src/snapshot-source/window-coordinate-space.ts) replays only the quarter-turn rows: its channel's guest refuses a windowed tree (app-window-not-full-screen, #3899) before that decoder can read one, so the translation has exactly one producer-side owner. Because a translation is the one thing a rotation must not also do, `rotationCases` pin the turn as axes-only: `native` and `oriented` never move a box's origin, and a row whose app frame is offset from the screen comes back at the same place as its own zero-origin twin.", + "description": "Golden table for the coordinate space a captured window's own box declares (#2612), the twin of tap-keyboard-occlusion-policy.json on the geometry that guard reads. iOS hosts some system surfaces in the device's native (portrait-up) space even while the app is rotated, so their whole subtree arrives quarter-turned: measured on iPhone 17 Pro (iOS 26.2) with the system keyboard up over a landscape app frame (0,0,874,402), UIRemoteKeyboardWindow reports its own box as (0,0,402,874) — the app's box with the two side lengths swapped — while the app's own UIWindow, UITextEffectsWindow and a dev-menu overlay window all report (0,0,874,402); in portrait the same keyboard window reports (0,0,402,874) against a (0,0,402,874) app frame, i.e. nothing is turned. RULE 1 (detection) is enforced twice, in packages/contracts/src/window-coordinate-space.ts and in SnapshotGeometrySpace inside apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift, and decides from side lengths alone: a window is quarter-turned relative to an app frame when the app frame is not square (abs(app.width - app.height) > quarterTurnTolerance), both frames are finite, positive and non-empty, and the window's side lengths match the app's on the swapped axes (abs(window.width - app.height) <= quarterTurnTolerance && abs(window.height - app.width) <= quarterTurnTolerance). Origins are deliberately never compared: the turn is a statement about which axis a box runs along, and a window may legitimately be placed elsewhere. A square app frame cannot be told from its own quarter turn, so it is left alone rather than guessed at. The tolerance is 1 because measured captures match the swap to the point and this absorbs float representation only. RULE 2 (rotation) is the capture's way back for the runner's own UIInterfaceOrientation raw values — 1 portrait, 2 portraitUpsideDown, 3 landscapeRight, 4 landscapeLeft, and 0 unknown, where portrait and unknown publish what the platform reported rather than inventing a turn — implemented by CoordinateSpaceRotation.oriented(rect:in:interfaceOrientation:) in that same Swift file, whose rect helper turns opposite corners of the reported box and re-orders them so the result keeps a top-left origin. `rotationCases` are native -> oriented rows: the landscape ones measured on the device above, the 180 degree and identity ones derived from that table. Capture turns back with the exact inverse of what dispatch turns forward, so each row also has to come back to its own `native` box under the forward mapping; that is why one measured-looking landscape-left pairing whose native and oriented boxes are identical is absent — the table computes no such row. Infinity has no JSON spelling, so the unusable frame a platform hands back when it could not resolve a box is named {\"infinite\": true} and decodes to CGRect.infinite on the Swift side and to an infinite rect on the TypeScript side. RULE 3 (content translation, #3899) is the one rule here that reads an origin: an iPad window placed off the screen origin (Windowed Apps, Stage Manager) reports its subtree from its own top-left — measured on iPad Pro 11-inch (M5, iOS 27.0), a Calendar window at screen (224,207,386x682) reports its root as (0,0,386,682) and its Add button as (335,14,37,36) for a control drawn at (559,221). SnapshotContentTranslation.offset(reportedBox:viewportBox:) names the viewport origin as the offset only when the carrier is a (0,0)-anchored box whose side lengths match the viewport's within the tolerance and the viewport starts elsewhere; any other carrier, such as the keyboard window or a SpringBoard overlay that report the screen's own box, and every window at the screen origin name none (null). `windowContentCases` replay against that Swift function alone, because the bridge guest refuses a windowed tree before the host decoder can read one. `rotationCases` pin the turn as axes-only: a non-zero app frame origin never travels through it.", "constants": { "quarterTurnTolerance": 1 }, @@ -170,17 +170,11 @@ ], "windowContentCases": [ { - "name": "RULE 3 (content translation): a Calendar window placed at (224,207) reports its own screen box and its subtree from its own top-left", + "name": "a Calendar window placed at (224,207) reports its subtree from its own top-left", "reportedBox": { "x": 0, "y": 0, "width": 386, "height": 682 }, "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, "offset": { "width": 224, "height": 207 } }, - { - "name": "the same window moved elsewhere on the screen carries its subtree by the new origin", - "reportedBox": { "x": 0, "y": 0, "width": 386, "height": 682 }, - "viewportBox": { "x": 120, "y": 90, "width": 386, "height": 682 }, - "offset": { "width": 120, "height": 90 } - }, { "name": "the app's own window box reports the screen box it is placed at and translates nothing", "reportedBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, @@ -188,35 +182,11 @@ "offset": null }, { - "name": "a full-screen iPad window fills the screen from (0,0), so its subtree starts where the viewport starts", + "name": "a full-screen window — iPad, iPhone, or an ADR 0025 foldable panel — starts at (0,0) and translates nothing", "reportedBox": { "x": 0, "y": 0, "width": 834, "height": 1210 }, "viewportBox": { "x": 0, "y": 0, "width": 834, "height": 1210 }, "offset": null }, - { - "name": "a full-screen iPhone window is the same no-op on the smaller screen", - "reportedBox": { "x": 0, "y": 0, "width": 402, "height": 874 }, - "viewportBox": { "x": 0, "y": 0, "width": 402, "height": 874 }, - "offset": null - }, - { - "name": "a Duo panel reports its own box from the origin of its own display, so no translation is read (ADR 0025)", - "reportedBox": { "x": 0, "y": 0, "width": 951, "height": 669 }, - "viewportBox": { "x": 0, "y": 0, "width": 951, "height": 669 }, - "offset": null - }, - { - "name": "the keyboard window under a windowed app reports the screen's box and stays in screen space", - "reportedBox": { "x": 0, "y": 925, "width": 834, "height": 282 }, - "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, - "offset": null - }, - { - "name": "a SpringBoard overlay under a windowed app reports the whole screen and translates nothing", - "reportedBox": { "x": 0, "y": 0, "width": 834, "height": 1210 }, - "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, - "offset": null - }, { "name": "a half-tolerance side-length miss is float representation, not a different box", "reportedBox": { "x": 0, "y": 0, "width": 385.5, "height": 682 }, @@ -228,30 +198,6 @@ "reportedBox": { "x": 0, "y": 0, "width": 384, "height": 682 }, "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, "offset": null - }, - { - "name": "a carrier anchored inside the screen names some content box, not the app's window", - "reportedBox": { "x": 40, "y": 12, "width": 386, "height": 682 }, - "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, - "offset": null - }, - { - "name": "a viewport the capture derived from its own root carries no screen anchor to translate toward", - "reportedBox": { "x": 0, "y": 0, "width": 386, "height": 682 }, - "viewportBox": { "x": 0, "y": 0, "width": 386, "height": 682 }, - "offset": null - }, - { - "name": "a zero-width carrier carries no geometry to read a space from", - "reportedBox": { "x": 0, "y": 0, "width": 0, "height": 682 }, - "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, - "offset": null - }, - { - "name": "a non-finite viewport declares nothing", - "reportedBox": { "x": 0, "y": 0, "width": 386, "height": 682 }, - "viewportBox": { "infinite": true }, - "offset": null } ] } diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index 1ed09cbf51..539c16afb4 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -628,52 +628,28 @@ A band taller than it is wide is what the output of a producer that did not norm ## Amendment: a window that does not fill the screen (issue #3899) -An iPad window that does not fill the screen splits a capture in two. -The measurement used iPad Pro 11-inch (M5, iOS 27.0) with Windowed Apps on and a Calendar window at screen `(224,207,386x682)`: - -- The app's root and its subtree report `(0,0)`-anchored frames. The window's `Add` button reads `(335,14,37,36)` and draws at screen `(559,221)`. -- The runner's live `XCUIApplication.windows` read reports the window's screen box `(224,207,386,682)`. -- A system window under the app root — the keyboard at `(0,925,834,282)`, a SpringBoard overlay at `(0,0,834,1210)` — reports the screen's own box. - -The viewport is built from the live read, so the capture published a screen-space viewport around window-relative rects. -That mix produced three failures from one cause: the fold clipped each node to `(w - x) x (h - y)` and the regular capture died with `regular-node-outside-cumulative-clip`; a published rect named no position in the screenshot; and a coordinate tap landed at the point minus the window origin. -The same drift appears in any app the user has placed, so Stage Manager behaves the same way. - -Decision. A window that does not fill the screen publishes screen space in points. -One capture publishes one space, as the #2612 amendment decided, and the space a placed window names is the screen's — because that is the space its viewport, its screenshot, and its digitizer already use. -The conversion runs where the platform's frame is still the platform's frame: inside the same `SnapshotGeometrySpace.normalized` pass that turns a quarter-turned subtree. -No per-command patch reads a window origin. - -A window declares its subtree's space from its own box, and this is the one rule in the table that reads an origin. -The carrier's box must be a `(0,0)`-anchored copy of the viewport box the platform reported, and the viewport must start somewhere else on the screen. -Then the subtree was reported in the carrier's content space, and the viewport's origin is the one vector that carries it to the screen. -Any other carrier box names no space, so the rule leaves its numbers alone: the keyboard window and the SpringBoard overlay above report the screen's own box and are already in screen space. -A full-screen window, an iPhone, and each foldable panel of ADR 0025 report origins at zero on both sides, so the rule reads no vector there and every rect is unchanged. -`SnapshotContentTranslation.offset(reportedBox:viewportBox:)` is that rule, replayed from `contracts/fixtures/window-coordinate-space.json` in the capture's language alone: the host bridge refuses a windowed tree (`app-window-not-full-screen`) before its decoder can read one, so the translation has one producer-side owner and the host's parity replay covers only the rules that decoder reads. - -Rotation and translation stay separate rules. -`CoordinateSpaceRotation` turns a box's axes and never moves its origin: the digitizer's origin is the display's own, which is where a placed window's screen-space points are already measured from. -Before this amendment, that function localized a non-zero frame origin away and never added it back. -Every configuration measured before this one has its app frame at the screen origin, so the localization was dead code there and the dispatch bug here. -Translation is the one rule that reads an origin, so it is the only place the fix can live. - -Capture is half of the promise; dispatch is the other half. -The measured dispatch spaces for a placed window are both screen-anchored: the synthesized path lands a point at its screen position, and XCTest's own `XCUICoordinate` on the app anchors at the screen's origin too. -`interactionCoordinate` therefore passes the caller's screen point through, and `resolveRunnerWindow` keeps naming the window a foldable's tap resolves its display on (ADR 0025). -The live `XCUIElement.frame` property reports screen space on a placed window, as the selector-tap path proved on the device: it measured the `Add` button at (659,321) for a window at (324,307). -The one live read that does not is `XCUIElementSnapshot.frame`, which reports the window's content space like the capture channel does. -So the snapshot capture converts in `SnapshotGeometrySpace.normalized`, and the text-input probe converts its snapshot reads through the same `SnapshotContentTranslation` rule read from the resolved window's live box (its origin, against a zero-origin box of its own size); every other live read needs no conversion. -The probe-side owner is `ResolvedTextEntryInput.frameOnScreen`. -The vector is read once per command as `TextEntryContentTranslation` — `known(offset)`, `none` for a full-screen app, or `unknown` when the window read could not answer — and it has no default anywhere it travels, so an omitted read can never read as full screen. -It is stored on the target and each read the target produces, so the probe matching, the focus tap, the refresh point, the caret tap, and the readiness retap all dispatch through that one property. -`frameOnScreen` is optional: it names no box when the read has no snapshot or the vector is `unknown`. -No point dispatch reads a live `element.frame`: the focus tap falls back to the caller's requested point, which is already a screen point, and the retap and the caret tap dispatch nothing on a nil. -Probe matching compares `element.frame` when `frameOnScreen` is nil — a comparison needs the only box a read that cannot translate still has, and a comparison dispatches nothing. -The readiness retap taps nothing on `unknown`: dispatching a raw window-local point is the failure this amendment fixes, and skipping one retap costs the caller's next read, not the session. -The probe trusts that selector's naming — the same first non-empty window every synthesized dispatch already books as its reference box — and a window that is moved between the command's read and its dispatch drifts, like every live read: a re-opened command reads the current vector. -The host bridge still refuses a windowed capture (`app-window-not-full-screen`) and converts nothing, while the XCTest runner — the fallback the bridge's refusal routes to — converts through the rule above, so the bridge's refusal stays the reason the TypeScript twin of the rule runs today only in the parity replay. -Portrait is the only orientation measured for a placed window; the quarter-turn table's windowed row pins the arithmetic (a frame origin never travels through the turn), not a device measurement. -Dispatch rotation turns about the display's side lengths, as the digitizer does, never about a placed window's smaller box; landscape under Windowed Apps stays unmeasured until measured, and the rotation basis follows the display there on the digitizer's rule, not on a measurement. +An iPad window that does not fill the screen (Windowed Apps, Stage Manager) reports its subtree from its own top-left. +Measured on iPad Pro 11-inch (M5, iOS 27.0) with a Calendar window at screen `(224,207,386x682)`: the root reads `(0,0,386,682)` and `Add` reads `(335,14,37,36)` but draws at `(559,221)`. +The runner's live window read, the keyboard window, and a SpringBoard overlay report screen boxes. +Published as reported, the capture mixed two spaces: the fold clipped nodes (`regular-node-outside-cumulative-clip`), rects named no screenshot position, and taps landed at the point minus the window origin. + +Decision. A placed window publishes screen space in points, because its viewport, its screenshot, and the digitizer already use that space. +`SnapshotContentTranslation.offset` is the one rule, applied in the same `SnapshotGeometrySpace.normalized` pass that turns a quarter-turned subtree. +A carrier declares an offset only when its box is a `(0,0)`-anchored copy of the reported viewport box and the viewport starts elsewhere; the offset is the viewport origin. +Any other carrier keeps its frames, so the screen-space system windows above do not move. +A full-screen window, an iPhone, and each ADR 0025 foldable panel start at `(0,0)`, so they read no offset and publish unchanged. +`contracts/fixtures/window-coordinate-space.json` replays the rule in Swift only, because the bridge guest refuses a windowed tree (`app-window-not-full-screen`) before the host decoder reads one. + +Rotation and translation stay separate. `CoordinateSpaceRotation` turns axes and never moves an origin: the old localization of a frame origin was dead code at every measured full-screen origin and the dispatch bug here. +A synthesized turn swaps the sole display's side lengths, as the digitizer does, not a placed window's; a multi-screen host keeps the window box of its panel. + +Dispatch takes screen points. `XCUIElement.frame` and the app-anchored `XCUICoordinate` are screen space on a placed window; `XCUIElementSnapshot.frame` is window content space. +The text-entry probe therefore owns one screen box, `ResolvedTextEntryInput.frameOnScreen`. +Each command reads the vector once from the resolved window's box as `TextEntryContentTranslation` (`known`, `none`, or `unknown` when the read failed), with no default, so a failed read never passes for full screen. +The target and each read it produces carry that vector. `frameOnScreen` is nil without a snapshot (#3060) or with `unknown`. +On nil, the focus tap dispatches the caller's point, the refresh point, the caret tap, and the readiness retap dispatch nothing, and probe matching compares the live `element.frame`, because a comparison dispatches nothing. + +Open gaps: landscape under Windowed Apps, Stage Manager, and a window partly off screen are unmeasured; their rows pin arithmetic only. A window moved between the command's vector read and its dispatch drifts like any live read. ## Amendment: modal containment in the presentation cut @@ -772,9 +748,7 @@ If the frames differ, the guest refuses the capture (`app-window-not-full-screen This refusal provides evidence about the screen arrangement, not the app, so it has capture scope. If the runtime cannot report either frame, the guest keeps the capture. -The XCTest fallback is the correct capture for a windowed app, after the amendment above. -Before it, the runner clipped window-relative element frames against the screen-relative app frame, and that gap is what made this refusal protect a real failure. -The refusal stays: this channel reports window-relative frames and no channel here converts them. +The bridge still refuses and converts nothing; the XCTest fallback publishes a windowed app in screen space (#3899 amendment above). Rejected alternatives: diff --git a/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts b/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts index a95d4908c8..9c26b5df36 100644 --- a/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts +++ b/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts @@ -38,18 +38,10 @@ type RotationCase = { oriented: FixtureFrame; }; -type WindowContentCase = { - name: string; - reportedBox: FixtureFrame; - viewportBox: FixtureFrame; - offset: { width: number; height: number } | null; -}; - type Fixture = { constants: { quarterTurnTolerance: number }; quarterTurnCases: QuarterTurnCase[]; rotationCases: RotationCase[]; - windowContentCases: WindowContentCase[]; }; const TABLE_PATH = path.resolve( @@ -90,11 +82,8 @@ test('the quarter-turn window rule agrees with every golden parity table case', const table = loadTable(); assert.ok(table.quarterTurnCases.length > 0, 'parity table must not be empty'); assert.ok(table.rotationCases.length > 0, 'parity table must not be empty'); - assert.ok(table.windowContentCases.length > 0, 'parity table must not be empty'); assertUniqueNames( - [...table.quarterTurnCases, ...table.rotationCases, ...table.windowContentCases].map( - (fixture) => fixture.name, - ), + [...table.quarterTurnCases, ...table.rotationCases].map((fixture) => fixture.name), 'parity table', ); for (const fixture of table.quarterTurnCases) { @@ -106,11 +95,6 @@ test('the quarter-turn window rule agrees with every golden parity table case', } }); -// `windowContentCases` is replayed in the Swift capture alone (CoordinateSpaceTests): this decoder -// never receives a windowed tree — the bridge guest refuses it first (#3899) — so a second replay -// here would pin arithmetic no production reader runs. The table still loads here, and its rows are -// read for the name-uniqueness guard below. - test('the quarter-turn tolerance belongs to the table, not this file', () => { assert.equal(loadTable().constants.quarterTurnTolerance, WINDOW_QUARTER_TURN_TOLERANCE); }); @@ -120,12 +104,8 @@ const LANDSCAPE_RIGHT = 3; const LANDSCAPE_LEFT = 4; /** - * The dispatch direction — the point a caller passes, in the native space a synthesized event is - * performed in — as the runner's own `CoordinateSpaceRotation.native(point:in:interfaceOrientation:)` - * computes it. - * - * The turn moves a point's axes and never its origin (#3899): the digitizer's origin is the display's - * own, which is where a placed window's screen-space points are already measured from. + * The dispatch direction — the caller's screen point in the native space a synthesized event is performed in + * — as the runner's own `CoordinateSpaceRotation.native(point:in:interfaceOrientation:)` computes it. * * The capture's inverse is NOT implemented here: it lives in Swift alone, and a TypeScript copy of it * would prove nothing about the code that runs. What this proves is that the table cannot be edited @@ -135,15 +115,17 @@ const LANDSCAPE_LEFT = 4; * forward map it ships with. */ function nativePoint(point: Point, app: Rect, interfaceOrientation: number): Point { + // The turn moves axes and never an origin (#3899). + const { x: localX, y: localY } = point; switch (interfaceOrientation) { case LANDSCAPE_RIGHT: - return { x: app.height - point.y, y: point.x }; + return { x: app.height - localY, y: localX }; case LANDSCAPE_LEFT: - return { x: point.y, y: app.width - point.x }; + return { x: localY, y: app.width - localX }; case PORTRAIT_UPSIDE_DOWN: - return { x: app.width - point.x, y: app.height - point.y }; + return { x: app.width - localX, y: app.height - localY }; default: - return { x: point.x, y: point.y }; + return { x: localX, y: localY }; } } diff --git a/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts b/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts index ee97982866..eef984c2ae 100644 --- a/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts +++ b/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts @@ -15,31 +15,19 @@ import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; * two spaces coincide and nothing is turned. Rules that read the reported numbers refused app content * the keyboard was nowhere near and let a tap through into a key (#2612). * - * A window that does not fill the screen — iPad Windowed Apps or Stage Manager — declares the other - * space. Measured on iPad Pro 11-inch (M5, iOS 27.0) with a Calendar window whose screen box is - * `(224,207,386,682)`, the window itself reports that screen box while the subtree under it arrives - * anchored at its own top-left: the application root reads `(0,0,386,682)` and its `Add` button - * `(335,14,37,36)` — the button's screen position `(559,221)` minus the window origin. Published as - * reported, that tree mixes two spaces, clips its own content, and moves every tap by the window - * origin (#3899). That translation belongs to the capture that performs it — the runner's Swift - * `SnapshotContentTranslation` — not to this decoder, because the guest refuses a windowed tree - * before this file can read one. + * The box is the only evidence, so the rule reads side lengths and nothing else. Origins are not + * compared: a quarter turn says which axis a box runs along, not where it sits, and a window may + * legitimately be placed elsewhere. A square app frame cannot be told from its own quarter turn, so it + * is left alone rather than guessed at — rotating geometry that did not need it is the worse failure. * - * A rule that reads a box reads what the box can say. The quarter turn is a statement about which - * axis a box runs along, so it compares side lengths and nothing else. A content translation is a - * statement about where a subtree starts, so it reads all four numbers of the carrier's box against - * the viewport box the platform reported: the carrier must be the app's own window — a `(0,0)`-anchored - * copy of the viewport's side lengths — and the viewport must start somewhere else on the screen. - * A full-screen window, an iPhone, and each foldable panel report origins at zero on both sides, so - * the rule reads no translation there and every published rect stays as the platform reported it. - * - * RULE 1 is enforced in both languages, because both channels read it. RULE 3 (the content - * translation) and RULE 2 (the rotation back, `CoordinateSpaceRotation.oriented(rect:in:interfaceOrientation:)`) - * live with the runner's capture alone. One table proves the shared rules are one rule: - * `contracts/fixtures/window-coordinate-space.json` is replayed here and against - * `SnapshotGeometrySpace`, `SnapshotContentTranslation`, and `CoordinateSpaceRotation` in - * `apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift`; - * change any rule only through that table. + * The same rule is enforced in Swift for the capture that produces these trees, and one table proves + * they are one rule: `contracts/fixtures/window-coordinate-space.json` is replayed here and against + * `SnapshotGeometrySpace` in + * `apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift`. The table also + * pins the way back (`CoordinateSpaceRotation.oriented(rect:in:interfaceOrientation:)`, Swift-only + * today); change either rule only through that table. A placed iPad window's content translation + * (#3899) is the Swift capture's alone (`SnapshotContentTranslation`): the guest refuses a windowed + * tree before this decoder can read one. * * Why this is a detector and not a repair: the bridge guest cannot read the app's interface * orientation. Measured in the #2659 spike (verdict on the issue, write-up in the diff of #2667): From 07343adc90d727e3f0062854bbe0ed91056fcf63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 25 Oct 2026 13:21:04 +0100 Subject: [PATCH 14/15] test(snapshot): pin translated hittability and the offset rule's empty-box guards Review follow-ups to the shrink: assert a translated node's hittability, add the two fixture rows that discriminate the positive-finite guards, keep windowContentCases names unique, read the display size from the fixed coordinate space, and correct ADR 0004's vector-read and gap wording. Co-Authored-By: Claude Opus 5.5 --- ...nerTests+SynthesizedInteractionTests.swift | 2 +- .../CoordinateSpaceTests.swift | 10 ++++---- .../fixtures/window-coordinate-space.json | 12 ++++++++++ .../adr/0004-ios-snapshot-backend-strategy.md | 24 ++++++++++++------- .../window-coordinate-space.test.ts | 5 +++- 5 files changed, 38 insertions(+), 15 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift index 3507d57076..6d848a006f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift @@ -172,7 +172,7 @@ extension RunnerTests { func testSynthesizedRotationFrameIsTheDisplayNotAPlacedWindow() throws { try XCTSkipUnless(XCUIScreen.screens.count == 1, "a multi-screen host keeps its panel's window box") let placedWindow = CGRect(x: 224, y: 207, width: 386, height: 682) - let screen = UIScreen.main.bounds.size + let screen = UIScreen.main.fixedCoordinateSpace.bounds.size XCTAssertEqual( synthesizedRotationFrame(fallback: placedWindow, interfaceOrientation: RunnerInterfaceOrientation.portrait), CGRect(origin: .zero, size: screen) diff --git a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift index 897bf4fe28..d0412a4433 100644 --- a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift +++ b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift @@ -519,18 +519,20 @@ final class CoordinateSpaceTests: XCTestCase { coordinateNode(6, "Window", CGRect(x: 0, y: 0, width: 834, height: 1210), 0, 1, label: "springboard"), coordinateNode(7, "Alert", CGRect(x: 300, y: 500, width: 240, height: 160), 6, 2), ] - func normalizedRects(viewport: CGRect) -> [CGRect] { + func normalized(viewport: CGRect) -> [RawAXNode] { SnapshotGeometrySpace.normalized( nodes: acquired, viewport: .reported(box: viewport, interfaceOrientation: RunnerInterfaceOrientation.portrait) - ).map(\.rect.cgRect) + ) } let reported = acquired.map(\.rect.cgRect) - let placed = normalizedRects(viewport: windowOnScreen) + let placedNodes = normalized(viewport: windowOnScreen) + let placed = placedNodes.map(\.rect.cgRect) XCTAssertEqual(Array(placed[0...2]), [windowOnScreen, windowOnScreen, windowOnScreen]) XCTAssertEqual(placed[3], CGRect(x: 559, y: 221, width: 37, height: 36), "Add draws at (559,221)") + XCTAssertEqual(placedNodes[3].hittable, true, "hittability reads the translated frame") XCTAssertEqual(Array(placed[4...]), Array(reported[4...]), "system windows already report screen space") - XCTAssertEqual(normalizedRects(viewport: reportedRoot), reported, "a window at the screen origin is unchanged") + XCTAssertEqual(normalized(viewport: reportedRoot).map(\.rect.cgRect), reported, "a window at the screen origin is unchanged") } func testNormalizedOfAnEmptyArrayIsEmpty() { diff --git a/contracts/fixtures/window-coordinate-space.json b/contracts/fixtures/window-coordinate-space.json index 0ba25479c7..b1fa727e5f 100644 --- a/contracts/fixtures/window-coordinate-space.json +++ b/contracts/fixtures/window-coordinate-space.json @@ -198,6 +198,18 @@ "reportedBox": { "x": 0, "y": 0, "width": 384, "height": 682 }, "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, "offset": null + }, + { + "name": "an empty carrier names no box to translate", + "reportedBox": { "x": 0, "y": 0, "width": 0, "height": 0 }, + "viewportBox": { "x": 224, "y": 207, "width": 0.5, "height": 0.5 }, + "offset": null + }, + { + "name": "an empty viewport names no screen box to translate toward", + "reportedBox": { "x": 0, "y": 0, "width": 0.5, "height": 0.5 }, + "viewportBox": { "x": 224, "y": 207, "width": 0, "height": 0 }, + "offset": null } ] } diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index 539c16afb4..cc65bfac0d 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -628,28 +628,34 @@ A band taller than it is wide is what the output of a producer that did not norm ## Amendment: a window that does not fill the screen (issue #3899) -An iPad window that does not fill the screen (Windowed Apps, Stage Manager) reports its subtree from its own top-left. +An iPad window that does not fill the screen (Windowed Apps; Stage Manager is unmeasured) reports its subtree from its own top-left. Measured on iPad Pro 11-inch (M5, iOS 27.0) with a Calendar window at screen `(224,207,386x682)`: the root reads `(0,0,386,682)` and `Add` reads `(335,14,37,36)` but draws at `(559,221)`. The runner's live window read, the keyboard window, and a SpringBoard overlay report screen boxes. Published as reported, the capture mixed two spaces: the fold clipped nodes (`regular-node-outside-cumulative-clip`), rects named no screenshot position, and taps landed at the point minus the window origin. -Decision. A placed window publishes screen space in points, because its viewport, its screenshot, and the digitizer already use that space. +Decision: a placed window publishes screen space in points, because its viewport, its screenshot, and the digitizer already use that space. `SnapshotContentTranslation.offset` is the one rule, applied in the same `SnapshotGeometrySpace.normalized` pass that turns a quarter-turned subtree. A carrier declares an offset only when its box is a `(0,0)`-anchored copy of the reported viewport box and the viewport starts elsewhere; the offset is the viewport origin. Any other carrier keeps its frames, so the screen-space system windows above do not move. A full-screen window, an iPhone, and each ADR 0025 foldable panel start at `(0,0)`, so they read no offset and publish unchanged. `contracts/fixtures/window-coordinate-space.json` replays the rule in Swift only, because the bridge guest refuses a windowed tree (`app-window-not-full-screen`) before the host decoder reads one. -Rotation and translation stay separate. `CoordinateSpaceRotation` turns axes and never moves an origin: the old localization of a frame origin was dead code at every measured full-screen origin and the dispatch bug here. +Rotation and translation stay separate. +`CoordinateSpaceRotation` turns axes and never moves an origin: the old localization of a frame origin was dead code at every measured full-screen origin and the dispatch bug here. A synthesized turn swaps the sole display's side lengths, as the digitizer does, not a placed window's; a multi-screen host keeps the window box of its panel. -Dispatch takes screen points. `XCUIElement.frame` and the app-anchored `XCUICoordinate` are screen space on a placed window; `XCUIElementSnapshot.frame` is window content space. +Dispatch takes screen points. +`XCUIElement.frame` and the app-anchored `XCUICoordinate` are screen space on a placed window; `XCUIElementSnapshot.frame` is window content space. The text-entry probe therefore owns one screen box, `ResolvedTextEntryInput.frameOnScreen`. -Each command reads the vector once from the resolved window's box as `TextEntryContentTranslation` (`known`, `none`, or `unknown` when the read failed), with no default, so a failed read never passes for full screen. -The target and each read it produces carry that vector. `frameOnScreen` is nil without a snapshot (#3060) or with `unknown`. -On nil, the focus tap dispatches the caller's point, the refresh point, the caret tap, and the readiness retap dispatch nothing, and probe matching compares the live `element.frame`, because a comparison dispatches nothing. - -Open gaps: landscape under Windowed Apps, Stage Manager, and a window partly off screen are unmeasured; their rows pin arithmetic only. A window moved between the command's vector read and its dispatch drifts like any live read. +A command reads the vector from the resolved window's box once, where it builds its target, and the readiness retap reads it again on its own branch. +The vector is `TextEntryContentTranslation` (`known`, `none`, or `unknown` when the read failed), with no default, so a failed read never passes for full screen. +The target and each read it produces carry that vector. +`frameOnScreen` is nil without a snapshot (#3060) or with `unknown`. +On nil, the focus tap dispatches the caller's own point, and the refresh point, the caret tap, and the readiness retap dispatch nothing. +Probe matching then compares the live `element.frame`, because a comparison dispatches nothing. + +Open gaps: landscape under Windowed Apps (fixture arithmetic only), Stage Manager, and a window partly off screen are unmeasured. +A window moved between the command's vector read and its dispatch drifts like any live read. ## Amendment: modal containment in the presentation cut diff --git a/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts b/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts index 9c26b5df36..2edd1a74fb 100644 --- a/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts +++ b/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts @@ -42,6 +42,7 @@ type Fixture = { constants: { quarterTurnTolerance: number }; quarterTurnCases: QuarterTurnCase[]; rotationCases: RotationCase[]; + windowContentCases: { name: string }[]; }; const TABLE_PATH = path.resolve( @@ -83,7 +84,9 @@ test('the quarter-turn window rule agrees with every golden parity table case', assert.ok(table.quarterTurnCases.length > 0, 'parity table must not be empty'); assert.ok(table.rotationCases.length > 0, 'parity table must not be empty'); assertUniqueNames( - [...table.quarterTurnCases, ...table.rotationCases].map((fixture) => fixture.name), + [...table.quarterTurnCases, ...table.rotationCases, ...table.windowContentCases].map( + (fixture) => fixture.name, + ), 'parity table', ); for (const fixture of table.quarterTurnCases) { From dd067b89327ab0a0441471e41be9eff4d167a6ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 25 Oct 2026 13:36:45 +0100 Subject: [PATCH 15/15] fix(runner): restore the discriminating window-content rows and the retap skip reason Put back the moved-window, keyboard, and SpringBoard rows in windowContentCases, and log one skip line carrying its reason when the readiness retap names no screen point. Co-Authored-By: Claude Opus 5.5 --- .../RunnerTests+TextEntryReadiness.swift | 5 ++++- .../fixtures/window-coordinate-space.json | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift index 2f26b95688..b30d584165 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift @@ -101,7 +101,10 @@ extension RunnerTests { /// The readiness repeat, at the read's screen box; a read that names none taps nothing (#3899). func retapAt(read: ResolvedTextEntryInput, app: XCUIApplication, element: XCUIElement) { - guard let frame = read.frameOnScreen, !frame.isEmpty else { return } + guard let frame = read.frameOnScreen, !frame.isEmpty else { + NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_RETAP skipped reason=%@", read.frameOnScreen == nil ? "no-screen-frame" : "empty-frame") + return + } NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_RETAP x=%.1f y=%.1f", frame.midX, frame.midY) _ = tapAt(app: app, x: frame.midX, y: frame.midY) _ = waitForTextEntryFocus(app: app, element: element, timeout: TextEntryTiming.readinessTimeout) diff --git a/contracts/fixtures/window-coordinate-space.json b/contracts/fixtures/window-coordinate-space.json index b1fa727e5f..a0c5c3a278 100644 --- a/contracts/fixtures/window-coordinate-space.json +++ b/contracts/fixtures/window-coordinate-space.json @@ -175,6 +175,24 @@ "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, "offset": { "width": 224, "height": 207 } }, + { + "name": "the same window moved elsewhere on the screen carries its subtree by the new origin", + "reportedBox": { "x": 0, "y": 0, "width": 386, "height": 682 }, + "viewportBox": { "x": 120, "y": 90, "width": 386, "height": 682 }, + "offset": { "width": 120, "height": 90 } + }, + { + "name": "the keyboard window under a windowed app reports the screen's box and stays in screen space", + "reportedBox": { "x": 0, "y": 925, "width": 834, "height": 282 }, + "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "offset": null + }, + { + "name": "a SpringBoard overlay under a windowed app reports the whole screen and translates nothing", + "reportedBox": { "x": 0, "y": 0, "width": 834, "height": 1210 }, + "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "offset": null + }, { "name": "the app's own window box reports the screen box it is placed at and translates nothing", "reportedBox": { "x": 224, "y": 207, "width": 386, "height": 682 },