diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index ae1407d1d0..3684adfee4 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -352,7 +352,8 @@ extension RunnerTests { func readTextAt(app: XCUIApplication, x: Double, y: Double) -> String? { let point = CGPoint(x: x, y: y) - let textInputCandidates = textInputCandidatesAt(app: app, point: point).map(\.element) + let textInputCandidates = textInputCandidatesAt(app: app, point: point, contentTranslation: liveContentTranslation(app: app)) + .map(\.element) for element in textInputCandidates where prefersExpandedTextRead(element) { if let text = readableText(for: element) { return text @@ -618,7 +619,8 @@ extension RunnerTests { private func interactionCoordinate(app: XCUIApplication, x: Double, y: Double) -> XCUICoordinate { #if os(iOS) // Coordinate taps, double taps, long presses, and drags anchor at the app origin, not the - // resolved window: on iOS the snapshot hands back app-space points, and the first qualifying + // resolved window: callers pass screen points, and `app.frame` starts at the screen origin even + // for a placed iPad window (#3899). The first qualifying // window is not always the one under the finger — a SpringBoard `alert accept` can live in an // alert window while the wallpaper or status-bar window qualifies first, so a window-relative // anchor misses the button. Reference frames and the synthesized display ID still resolve diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index cde5f867c0..d6d6d9e9b4 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -31,14 +31,15 @@ extension RunnerTests { ) } let frame = context.referenceFrame + let rotationFrame = synthesizedRotationFrame(fallback: frame, interfaceOrientation: orientation) let start = CoordinateSpaceRotation.native( point: CGPoint(x: x, y: y), - in: frame, + in: rotationFrame, interfaceOrientation: orientation ) let end = CoordinateSpaceRotation.native( point: CGPoint(x: x2, y: y2), - in: frame, + in: rotationFrame, interfaceOrientation: orientation ) logSynthesizedDispatch( @@ -117,7 +118,7 @@ extension RunnerTests { } let point = CoordinateSpaceRotation.native( point: CGPoint(x: x, y: y), - in: context.referenceFrame, + in: synthesizedRotationFrame(fallback: context.referenceFrame, interfaceOrientation: orientation), interfaceOrientation: orientation ) logSynthesizedDispatch( @@ -191,7 +192,7 @@ extension RunnerTests { let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) let point = CoordinateSpaceRotation.native( point: CGPoint(x: x, y: y), - in: context.referenceFrame, + in: synthesizedRotationFrame(fallback: context.referenceFrame, interfaceOrientation: orientation), interfaceOrientation: orientation ) logSynthesizedDispatch( @@ -524,24 +525,38 @@ extension RunnerTests { /// that can show an app in part of a screen (iPad, more than one screen) keeps requiring a window. @MainActor func soleScreenReferenceFrame(app: XCUIApplication) -> CGRect? { - guard UIDevice.current.userInterfaceIdiom == .phone, XCUIScreen.screens.count == 1 else { + guard UIDevice.current.userInterfaceIdiom == .phone else { return nil } + let frame = soleDisplayFrame( + interfaceOrientation: Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) + ) + NSLog( + "AGENT_DEVICE_RUNNER_SYNTHESIZED_SOLE_SCREEN_ROUTE resolved=%@", + frame.map { NSCoder.string(for: $0) } ?? "none" + ) + return frame + } + + /// The box a synthesized turn swaps side lengths about. The digitizer turns about its display, so a + /// placed iPad window's smaller box is not the basis (#3899); a multi-screen host keeps the + /// resolved window, which names its panel (ADR 0025). + func synthesizedRotationFrame(fallback: CGRect, interfaceOrientation: Int) -> CGRect { + soleDisplayFrame(interfaceOrientation: interfaceOrientation) ?? fallback + } + + private func soleDisplayFrame(interfaceOrientation: Int) -> CGRect? { + guard XCUIScreen.screens.count == 1 else { return nil } let screen = UIScreen.main let scale = screen.nativeScale guard scale > 0 else { return nil } - let frame = Self.soleScreenReferenceFrame( + return Self.soleScreenReferenceFrame( nativePortraitSize: CGSize( width: screen.nativeBounds.width / scale, height: screen.nativeBounds.height / scale ), - interfaceOrientation: Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) - ) - NSLog( - "AGENT_DEVICE_RUNNER_SYNTHESIZED_SOLE_SCREEN_ROUTE resolved=%@", - frame.map { NSCoder.string(for: $0) } ?? "none" + interfaceOrientation: interfaceOrientation ) - return frame } #endif @@ -732,11 +747,12 @@ extension RunnerTests { width: plan.viewport.width, height: plan.viewport.height ) + let rotationFrame = synthesizedRotationFrame(fallback: frame, interfaceOrientation: orientation) let pointerSamples: [[[String: NSNumber]]] = plan.pointers.map { pointer in pointer.samples.map { sample in let point = CoordinateSpaceRotation.native( point: CGPoint(x: sample.point.x, y: sample.point.y), - in: frame, + in: rotationFrame, interfaceOrientation: orientation ) return [ diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index 5ac510f125..c088401e55 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -145,10 +145,21 @@ extension RunnerTests { let identity: TextEntryInputIdentity } + /// The vector from a snapshot frame to the screen (#3899, ADR 0004), read once per command by + /// `liveContentTranslation` and never defaulted: a vector nobody read must not pass for full screen. + enum TextEntryContentTranslation { + case known(CGSize) + /// The app fills the screen, so a snapshot frame is already a screen frame. + case none + /// The window read could not answer, so no snapshot frame names a screen box. + case unknown + } + struct TextEntryTarget { let element: XCUIElement? let refreshPoint: CGPoint? let prefersFocusedElement: Bool + let contentTranslation: TextEntryContentTranslation let fromTapWitness: Bool /// The input the first resolved element was. Once bound, resolution refuses any other input. let boundIdentity: TextEntryInputIdentity? @@ -166,6 +177,7 @@ extension RunnerTests { element: XCUIElement?, refreshPoint: CGPoint?, prefersFocusedElement: Bool, + contentTranslation: TextEntryContentTranslation, fromTapWitness: Bool = false, boundIdentity: TextEntryInputIdentity? = nil, inputAtRefreshPoint: TextInputAtPoint? = nil, @@ -174,6 +186,7 @@ extension RunnerTests { self.element = element self.refreshPoint = refreshPoint self.prefersFocusedElement = prefersFocusedElement + self.contentTranslation = contentTranslation self.fromTapWitness = fromTapWitness self.boundIdentity = boundIdentity ?? inputAtRefreshPoint?.identity self.inputAtRefreshPoint = inputAtRefreshPoint @@ -186,11 +199,12 @@ extension RunnerTests { guard let input else { return self } - let frame = input.snapshot?.frame ?? .zero + let frame = input.frameOnScreen ?? .zero return TextEntryTarget( element: input.element, refreshPoint: frame.isEmpty ? refreshPoint : CGPoint(x: frame.midX, y: frame.midY), prefersFocusedElement: prefersFocusedElement, + contentTranslation: contentTranslation, fromTapWitness: fromTapWitness, boundIdentity: boundIdentity ?? input.identity, inputAtRefreshPoint: inputAtRefreshPoint @@ -208,16 +222,18 @@ extension RunnerTests { let element: XCUIElement /// Nil when the element exists but could not be snapshotted. let snapshot: XCUIElementSnapshot? + let contentTranslation: TextEntryContentTranslation - private init(element: XCUIElement, snapshot: XCUIElementSnapshot?) { + private init(element: XCUIElement, snapshot: XCUIElementSnapshot?, contentTranslation: TextEntryContentTranslation) { self.element = element self.snapshot = snapshot + self.contentTranslation = contentTranslation } /// Reads a handle that may name an input the app has removed; nil when it names none or the read /// raised. The snapshot of a missing element throws only after XCTest waits about two seconds for /// a match, so existence is read first. - init?(reading element: XCUIElement) { + init?(reading element: XCUIElement, contentTranslation: TextEntryContentTranslation) { var exists = false var snapshot: XCUIElementSnapshot? if let exception = RunnerObjCExceptionCatcher.catchException({ @@ -228,15 +244,15 @@ extension RunnerTests { return nil } guard exists else { return nil } - self.init(element: element, snapshot: snapshot) + self.init(element: element, snapshot: snapshot, contentTranslation: contentTranslation) } /// Reads a handle its query has just listed, inside the caller's exception containment, so the /// snapshot is read first and existence only when the snapshot fails; nil when it names no element. - init?(readingListed element: XCUIElement) { + init?(readingListed element: XCUIElement, contentTranslation: TextEntryContentTranslation) { let snapshot = try? element.snapshot() guard snapshot != nil || element.exists else { return nil } - self.init(element: element, snapshot: snapshot) + self.init(element: element, snapshot: snapshot, contentTranslation: contentTranslation) } var identity: TextEntryInputIdentity? { @@ -249,6 +265,18 @@ extension RunnerTests { } return element } + + /// The input's screen box. Nil without a snapshot, because the live `element.frame` of a handle + /// that stopped answering records a failure (#3060), and nil for an `.unknown` vector, because a + /// window-local point must not dispatch as a screen point (#3899). Point dispatch refuses nil. + var frameOnScreen: CGRect? { + guard let frame = snapshot?.frame else { return nil } + switch contentTranslation { + case .known(let offset): return frame.offsetBy(dx: offset.width, dy: offset.height) + case .none: return frame + case .unknown: return nil + } + } } /// A text entry target and its input as last read. `input()` hands the last read out again until @@ -494,7 +522,8 @@ extension RunnerTests { return resolveUnboundTextEntryInput(app: app, target: target) } func boundInput(_ candidate: XCUIElement) -> ResolvedTextEntryInput? { - guard let input = ResolvedTextEntryInput(reading: candidate), input.identity == boundIdentity else { return nil } + guard let input = ResolvedTextEntryInput(reading: candidate, contentTranslation: target.contentTranslation), + input.identity == boundIdentity else { return nil } return input } if target.prefersFocusedElement, let focused = focusedTextInput(app: app), let input = boundInput(focused) { @@ -503,12 +532,13 @@ extension RunnerTests { if let element = target.element, let input = boundInput(element) { return input } - if let input = target.inputAtRefreshPoint, let read = textInputIfStillResolves(input) { + if let input = target.inputAtRefreshPoint, + let read = textInputIfStillResolves(input, contentTranslation: target.contentTranslation) { return read } if target.inputAtRefreshPoint == nil, let refreshPoint = target.refreshPoint, - case .matches(let candidates) = probeTextInputs(app: app, point: refreshPoint), + case .matches(let candidates) = probeTextInputs(app: app, point: refreshPoint, contentTranslation: target.contentTranslation), let match = candidates.first(where: { $0.identity == boundIdentity }) { return match } @@ -543,7 +573,7 @@ extension RunnerTests { guard let refreshPoint = target.refreshPoint else { return true } - switch probeTextInputs(app: app, point: refreshPoint) { + switch probeTextInputs(app: app, point: refreshPoint, contentTranslation: target.contentTranslation) { case .absent: return true case .matches(let candidates): @@ -577,17 +607,20 @@ extension RunnerTests { /// Resolution for a target not bound to an input yet: the first element that exists. private func resolveUnboundTextEntryInput(app: XCUIApplication, target: TextEntryTarget) -> ResolvedTextEntryInput? { - if target.prefersFocusedElement, let focused = focusedTextInput(app: app).flatMap(ResolvedTextEntryInput.init(reading:)) { + let read = { (element: XCUIElement) in + ResolvedTextEntryInput(reading: element, contentTranslation: target.contentTranslation) + } + if target.prefersFocusedElement, let focused = focusedTextInput(app: app).flatMap(read) { return focused } - if let element = target.element, let input = ResolvedTextEntryInput(reading: element) { + if let element = target.element, let input = read(element) { return input } if let refreshPoint = target.refreshPoint, - let refreshed = textInputAt(app: app, x: refreshPoint.x, y: refreshPoint.y) { + let refreshed = textInputAt(app: app, x: refreshPoint.x, y: refreshPoint.y, contentTranslation: target.contentTranslation) { return refreshed } - return focusedTextInput(app: app).flatMap(ResolvedTextEntryInput.init(reading:)) + return focusedTextInput(app: app).flatMap(read) } private func moveCaretToEnd(app: XCUIApplication, input: ResolvedTextEntryInput) { @@ -606,7 +639,7 @@ extension RunnerTests { // zero-size one (how a departed handle reads), there is no point to dispatch — an edge-tap // from a frame the field moved after records a failure or navigates away, the shape // `clearTextInput`'s comment refuses — so degrade to the caller's point-free delete burst. - guard let frame = input.snapshot?.frame, !frame.isEmpty else { + guard let frame = input.frameOnScreen, !frame.isEmpty else { return } _ = tapAt( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift index 34f289cccb..0f598bcb68 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift @@ -26,7 +26,7 @@ extension RunnerTests { } @MainActor - private func rememberedTextEntryTarget() -> TextEntryTarget? { + private func rememberedTextEntryTarget(contentTranslation: TextEntryContentTranslation) -> TextEntryTarget? { guard let witness = textEntryTapWitness else { return nil } @@ -54,6 +54,7 @@ extension RunnerTests { element: element, refreshPoint: nil, prefersFocusedElement: false, + contentTranslation: contentTranslation, fromTapWitness: true ) } @@ -61,6 +62,7 @@ extension RunnerTests { func stabilizeTextInputBeforeTyping( app: XCUIApplication, target: XCUIElement?, + contentTranslation: TextEntryContentTranslation, keyboardVisibleBeforeTap: Bool? = nil ) -> TextEntryStabilization { #if os(tvOS) @@ -79,7 +81,7 @@ extension RunnerTests { // presented, the keyboard query blocks and then finds none. An input that reports keyboard // focus is stronger evidence than the keyboard's arrival, which shows only that some input // took focus. - let read = target.flatMap(ResolvedTextEntryInput.init(reading:)) + let read = target.flatMap { ResolvedTextEntryInput(reading: $0, contentTranslation: contentTranslation) } if let read, let snapshot = read.snapshot, snapshotHasKeyboardFocus(snapshot) { return TextEntryStabilization(element: read.element, evidence: .inputReportsKeyboardFocus(read), polls: polls) } @@ -103,9 +105,10 @@ extension RunnerTests { @MainActor func focusTextInputForTextEntry(app: XCUIApplication, x: Double?, y: Double?) -> TextEntryTarget { + let contentTranslation = liveContentTranslation(app: app) guard let x, let y else { let softwareKeyboardVisible = isKeyboardVisible(app: app) - if !softwareKeyboardVisible, let rememberedTarget = rememberedTextEntryTarget() { + if !softwareKeyboardVisible, let rememberedTarget = rememberedTextEntryTarget(contentTranslation: contentTranslation) { return rememberedTarget } // Bare `type` targets the current first responder. On iOS we intentionally do not let the @@ -116,7 +119,8 @@ extension RunnerTests { return TextEntryTarget( element: focusedTextInput(app: app), refreshPoint: nil, - prefersFocusedElement: true + prefersFocusedElement: true, + contentTranslation: contentTranslation ) } let focused = waitForTextEntryReadiness( @@ -124,30 +128,30 @@ extension RunnerTests { target: TextEntryTarget( element: focusedTextInput(app: app), refreshPoint: nil, - prefersFocusedElement: true + prefersFocusedElement: true, + contentTranslation: contentTranslation ) ) - return TextEntryTarget(element: focused, refreshPoint: nil, prefersFocusedElement: true) + return TextEntryTarget(element: focused, refreshPoint: nil, prefersFocusedElement: true, contentTranslation: contentTranslation) } let keyboardVisibleBeforeTap = isKeyboardVisible(app: app) - let input = textInputAt(app: app, x: x, y: y) + let input = textInputAt(app: app, x: x, y: y, contentTranslation: contentTranslation) let target = input?.element let requestedPoint = CGPoint(x: x, y: y) - if let frame = input?.attributes.frame, !frame.isEmpty { - _ = tapAt(app: app, x: frame.midX, y: frame.midY) - } else { - _ = tapAt(app: app, x: x, y: y) - } + // `tapAt` takes a screen point: with no screen box, tap the caller's own point (#3899, #3060). + let tapPoint = input?.frameOnScreen.flatMap { $0.isEmpty ? nil : CGPoint(x: $0.midX, y: $0.midY) } ?? requestedPoint + _ = tapAt(app: app, x: tapPoint.x, y: tapPoint.y) // A visible keyboard is not enough evidence for app.typeText, because focus may still // belong to a previous field. With a concrete target we type through XCUIElement.typeText, // so after tapping it the iOS readiness timeout cannot prove anything stronger. if keyboardVisibleBeforeTap, let target { - return focusedTextEntryTarget(target, requestedPoint: requestedPoint) + return focusedTextEntryTarget(target, requestedPoint: requestedPoint, contentTranslation: contentTranslation) } let stabilized = stabilizeTextInputBeforeTyping( app: app, target: target, + contentTranslation: contentTranslation, keyboardVisibleBeforeTap: keyboardVisibleBeforeTap ) NSLog( @@ -157,37 +161,45 @@ extension RunnerTests { stabilized.polls ) if case .inputReportsKeyboardFocus(let read) = stabilized.evidence { - return focusedTextEntryTarget(read.element, read: read, requestedPoint: requestedPoint) + return focusedTextEntryTarget(read.element, read: read, requestedPoint: requestedPoint, contentTranslation: contentTranslation) } let readyTarget = TextEntryTarget( element: stabilized.element ?? target, refreshPoint: requestedPoint, - prefersFocusedElement: false + prefersFocusedElement: false, + contentTranslation: contentTranslation ) let element = stabilized.focusConfirmed ? readyTarget.element : (waitForTextEntryReadiness(app: app, target: readyTarget) ?? readyTarget.element) - return focusedTextEntryTarget(element, requestedPoint: requestedPoint) + return focusedTextEntryTarget(element, requestedPoint: requestedPoint, contentTranslation: contentTranslation) } /// The target the focus step hands to text entry, with the read its refresh point came from. - private func focusedTextEntryTarget(_ element: XCUIElement?, requestedPoint: CGPoint) -> TextEntryTarget { + private func focusedTextEntryTarget( + _ element: XCUIElement?, + requestedPoint: CGPoint, + contentTranslation: TextEntryContentTranslation + ) -> TextEntryTarget { guard let element else { - return TextEntryTarget(element: nil, refreshPoint: requestedPoint, prefersFocusedElement: false) + return TextEntryTarget(element: nil, refreshPoint: requestedPoint, prefersFocusedElement: false, contentTranslation: contentTranslation) } - return focusedTextEntryTarget(element, read: ResolvedTextEntryInput(reading: element), requestedPoint: requestedPoint) + let read = ResolvedTextEntryInput(reading: element, contentTranslation: contentTranslation) + return focusedTextEntryTarget(element, read: read, requestedPoint: requestedPoint, contentTranslation: contentTranslation) } /// `read` is `element` read after the focus tap, or nil when the read found no element. private func focusedTextEntryTarget( _ element: XCUIElement, read: ResolvedTextEntryInput?, - requestedPoint: CGPoint + requestedPoint: CGPoint, + contentTranslation: TextEntryContentTranslation ) -> TextEntryTarget { return TextEntryTarget( element: element, - refreshPoint: textEntryRefreshPoint(frame: read?.snapshot?.frame, fallbackTo: requestedPoint), + refreshPoint: textEntryRefreshPoint(frame: read?.frameOnScreen, fallbackTo: requestedPoint), prefersFocusedElement: false, + contentTranslation: contentTranslation, postFocusRead: read?.snapshot == nil ? nil : read ) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift index f4c49a5a67..b30d584165 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift @@ -90,17 +90,26 @@ extension RunnerTests { if waitForTextEntryFocus(app: app, element: element, timeout: TextEntryTiming.readinessTimeout) { return } - guard let settled = safely("TEXT_ENTRY_READINESS_FRAME", { try? element.snapshot() }) else { + guard let settled = safely("TEXT_ENTRY_READINESS_FRAME", { + ResolvedTextEntryInput(reading: element, contentTranslation: liveContentTranslation(app: app)) + }) else { return } - let frame = settled.frame - if !frame.isEmpty { - _ = tapAt(app: app, x: frame.midX, y: frame.midY) - _ = waitForTextEntryFocus(app: app, element: element, timeout: TextEntryTiming.readinessTimeout) - } + retapAt(read: settled, app: app, element: element) #endif } + /// The readiness repeat, at the read's screen box; a read that names none taps nothing (#3899). + func retapAt(read: ResolvedTextEntryInput, app: XCUIApplication, element: XCUIElement) { + guard let frame = read.frameOnScreen, !frame.isEmpty else { + NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_RETAP skipped reason=%@", read.frameOnScreen == nil ? "no-screen-frame" : "empty-frame") + return + } + NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_RETAP x=%.1f y=%.1f", frame.midX, frame.midY) + _ = tapAt(app: app, x: frame.midX, y: frame.midY) + _ = waitForTextEntryFocus(app: app, element: element, timeout: TextEntryTiming.readinessTimeout) + } + private func waitForTextEntryFocus(app: XCUIApplication, element: XCUIElement, timeout: TimeInterval) -> Bool { let deadline = Date().addingTimeInterval(timeout) while Date() < deadline { @@ -129,7 +138,9 @@ extension RunnerTests { /// fallback at all. func keyboardFocusConfirmed(app: XCUIApplication, element: XCUIElement) -> Bool { #if os(iOS) - return ResolvedTextEntryInput(reading: element).map { keyboardFocusConfirmed(app: app, read: $0) } ?? false + // `.unknown`: this compares raw frames with each other and dispatches no point. + return ResolvedTextEntryInput(reading: element, contentTranslation: .unknown) + .map { keyboardFocusConfirmed(app: app, read: $0) } ?? false #else return false #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift index 642e9db8f6..78dc4c6708 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift @@ -1,4 +1,5 @@ import XCTest +import AgentDeviceSnapshotPresentation final class TextInputProbeIssues { let thread = Thread.current @@ -17,18 +18,18 @@ enum TextInputProbeOutcome { } extension RunnerTests { - func textInputAt(app: XCUIApplication, x: Double, y: Double) -> ResolvedTextEntryInput? { - textInputCandidatesAt(app: app, point: CGPoint(x: x, y: y)).first + func textInputAt(app: XCUIApplication, x: Double, y: Double, contentTranslation: TextEntryContentTranslation) -> ResolvedTextEntryInput? { + textInputCandidatesAt(app: app, point: CGPoint(x: x, y: y), contentTranslation: contentTranslation).first } - func textInputCandidatesAt(app: XCUIApplication, point: CGPoint) -> [ResolvedTextEntryInput] { + func textInputCandidatesAt(app: XCUIApplication, point: CGPoint, contentTranslation: TextEntryContentTranslation) -> [ResolvedTextEntryInput] { safely("TEXT_INPUT_AT_POINT", []) { - queryTextInputs(app: app, point: point, shouldStop: { false }) + queryTextInputs(app: app, point: point, contentTranslation: contentTranslation, shouldStop: { false }) } } func coordinateTapTextInputAt(app: XCUIApplication, x: Double, y: Double) -> XCUIElement? { - switch probeTextInputs(app: app, point: CGPoint(x: x, y: y)) { + switch probeTextInputs(app: app, point: CGPoint(x: x, y: y), contentTranslation: liveContentTranslation(app: app)) { case .matches(let inputs): return inputs.first?.element case .absent: @@ -53,11 +54,14 @@ extension RunnerTests { func coordinateTapTextInputIdentityAt( app: XCUIApplication, x: Double, - y: Double + y: Double, + contentTranslation: TextEntryContentTranslation ) -> CoordinateTextInputResolution { let probed = containingTextInputProbeIssues(fallback: CoordinateTextInputResolution.unavailable) { shouldStop -> CoordinateTextInputResolution in - guard let input = queryTextInputs(app: app, point: CGPoint(x: x, y: y), shouldStop: shouldStop).first else { + guard let input = queryTextInputs( + app: app, point: CGPoint(x: x, y: y), contentTranslation: contentTranslation, shouldStop: shouldStop + ).first else { return .absent } guard let identity = input.identity else { @@ -75,9 +79,12 @@ extension RunnerTests { /// `input`'s handle, read again, when it still resolves to the input it named when found, read /// inside the same issue containment. A read that cannot answer counts as no. - func textInputIfStillResolves(_ input: TextInputAtPoint) -> ResolvedTextEntryInput? { + func textInputIfStillResolves( + _ input: TextInputAtPoint, + contentTranslation: TextEntryContentTranslation + ) -> ResolvedTextEntryInput? { let probed = containingTextInputProbeIssues(fallback: nil as ResolvedTextEntryInput?) { _ in - ResolvedTextEntryInput(reading: input.element).flatMap { $0.identity == input.identity ? $0 : nil } + ResolvedTextEntryInput(reading: input.element, contentTranslation: contentTranslation).flatMap { $0.identity == input.identity ? $0 : nil } } guard case .success(let read) = probed else { return nil } return read @@ -93,9 +100,9 @@ extension RunnerTests { return read } - func probeTextInputs(app: XCUIApplication, point: CGPoint) -> TextInputProbeOutcome { + func probeTextInputs(app: XCUIApplication, point: CGPoint, contentTranslation: TextEntryContentTranslation) -> TextInputProbeOutcome { switch containingTextInputProbeIssues(fallback: [], { shouldStop in - queryTextInputs(app: app, point: point, shouldStop: shouldStop) + queryTextInputs(app: app, point: point, contentTranslation: contentTranslation, shouldStop: shouldStop) }) { case .success(let inputs): return inputs.isEmpty ? .absent : .matches(inputs) @@ -150,6 +157,7 @@ extension RunnerTests { private func queryTextInputs( app: XCUIApplication, point: CGPoint, + contentTranslation: TextEntryContentTranslation, shouldStop: () -> Bool ) -> [ResolvedTextEntryInput] { var inputs: [ResolvedTextEntryInput] = [] @@ -166,9 +174,10 @@ extension RunnerTests { // cannot be snapshotted is read live, so its failure still reaches the caller's issue handling. for element in candidates { if shouldStop() { break } - guard let candidate = ResolvedTextEntryInput(readingListed: element) else { continue } - let attributes = candidate.attributes - if isCoordinateTextInputCandidate(enabled: attributes.isEnabled, frame: attributes.frame, point: point) { + guard let candidate = ResolvedTextEntryInput(readingListed: element, contentTranslation: contentTranslation) else { continue } + // Matching dispatches nothing, so a read with no screen box compares its live frame. + let frame = candidate.frameOnScreen ?? candidate.element.frame + if isCoordinateTextInputCandidate(enabled: candidate.attributes.isEnabled, frame: frame, point: point) { inputs.append(candidate) } } @@ -176,4 +185,16 @@ extension RunnerTests { guard !shouldStop() else { return [] } return inputs.sorted { smallestElementFirst($0.attributes, $1.attributes) } } + + /// The command's one content-vector read: the resolved window's screen origin when it is placed off + /// the screen origin (#3899). A read that cannot answer is `.unknown`, never `.none`. + func liveContentTranslation(app: XCUIApplication) -> TextEntryContentTranslation { + let read = containingTextInputProbeIssues(fallback: TextEntryContentTranslation.unknown) { _ in + guard let window = resolveRunnerWindowElement(app: app)?.frame else { return .unknown } + return SnapshotContentTranslation.offset(reportedBox: CGRect(origin: .zero, size: window.size), viewportBox: window) + .map { .known($0) } ?? TextEntryContentTranslation.none + } + guard case .success(let translation) = read else { return .unknown } + return translation + } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift index acbb62ef54..6017473904 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift @@ -31,8 +31,9 @@ extension RunnerTests { // A replacement with no input at the point has nothing to bind to: its clear and its typed // text would reach the app-level first responder, so an empty point and a lookup that cannot // answer both refuse here, each under its own reason. + let contentTranslation = liveContentTranslation(app: activeApp) let inputAtPoint: TextInputAtPoint - switch coordinateTapTextInputIdentityAt(app: activeApp, x: x, y: y) { + switch coordinateTapTextInputIdentityAt(app: activeApp, x: x, y: y, contentTranslation: contentTranslation) { case .input(let input): inputAtPoint = input case .absent: @@ -54,6 +55,7 @@ extension RunnerTests { element: nil, refreshPoint: CGPoint(x: x, y: y), prefersFocusedElement: false, + contentTranslation: contentTranslation, inputAtRefreshPoint: inputAtPoint ) case .xctestFallback: diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift index a4a83c3df1..a91f188fee 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift @@ -6,7 +6,7 @@ extension RunnerTests { /// cannot answer. A fixture that does not expose its own field is a broken lane, not a refusal. @MainActor func requireTextInputAtPoint(app: XCUIApplication, x: Double, y: Double) throws -> TextInputAtPoint { - switch coordinateTapTextInputIdentityAt(app: app, x: x, y: y) { + switch coordinateTapTextInputIdentityAt(app: app, x: x, y: y, contentTranslation: liveContentTranslation(app: app)) { case .input(let input): return input case .absent: diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift index c056141e8c..6d848a006f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift @@ -167,6 +167,21 @@ extension RunnerTests { XCTAssertEqual(displayID, 0) } } + + /// #3899: a synthesized turn swaps the display's side lengths, never a placed window's smaller box. + func testSynthesizedRotationFrameIsTheDisplayNotAPlacedWindow() throws { + try XCTSkipUnless(XCUIScreen.screens.count == 1, "a multi-screen host keeps its panel's window box") + let placedWindow = CGRect(x: 224, y: 207, width: 386, height: 682) + let screen = UIScreen.main.fixedCoordinateSpace.bounds.size + XCTAssertEqual( + synthesizedRotationFrame(fallback: placedWindow, interfaceOrientation: RunnerInterfaceOrientation.portrait), + CGRect(origin: .zero, size: screen) + ) + XCTAssertEqual( + synthesizedRotationFrame(fallback: placedWindow, interfaceOrientation: RunnerInterfaceOrientation.landscapeRight), + CGRect(x: 0, y: 0, width: screen.height, height: screen.width) + ) + } #endif func testSoleScreenReferenceFrameRefusesAnUnknownOrientationOrAnEmptyScreen() { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift index 5e78b2ad30..e5411bfc5d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift @@ -232,6 +232,7 @@ extension RunnerTests { element: nil, refreshPoint: CGPoint(x: frame.midX, y: frame.midY), prefersFocusedElement: false, + contentTranslation: .none, inputAtRefreshPoint: found ), text: "fresh", @@ -329,6 +330,7 @@ extension RunnerTests { element: nil, refreshPoint: CGPoint(x: neighbourFrame.midX, y: neighbourFrame.midY), prefersFocusedElement: false, + contentTranslation: .none, inputAtRefreshPoint: inputAtRefreshPoint ), text: "", @@ -361,6 +363,7 @@ extension RunnerTests { element: nil, refreshPoint: CGPoint(x: neighbourFrame.midX, y: neighbourFrame.midY), prefersFocusedElement: false, + contentTranslation: .none, inputAtRefreshPoint: TextInputAtPoint( element: rebound, identity: TextEntryInputIdentity(elementType: .textField, identifier: field.identifier) @@ -395,6 +398,7 @@ extension RunnerTests { element: nil, refreshPoint: CGPoint(x: neighbourFrame.midX, y: neighbourFrame.midY), prefersFocusedElement: false, + contentTranslation: .none, inputAtRefreshPoint: TextInputAtPoint( element: app.textFields.element, identity: TextEntryInputIdentity(elementType: .textField, identifier: "") diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift index f70b57c907..9b94a9df90 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift @@ -81,9 +81,9 @@ extension RunnerTests { let value = body() return (value, try XCTUnwrap(Int(valueReads.label)) - before) } - let target = TextEntryTarget(element: nil, refreshPoint: point, prefersFocusedElement: false) + let target = TextEntryTarget(element: nil, refreshPoint: point, prefersFocusedElement: false, contentTranslation: .none) - let (lookup, lookupReads) = try countingReads { textInputAt(app: app, x: point.x, y: point.y) } + let (lookup, lookupReads) = try countingReads { textInputAt(app: app, x: point.x, y: point.y, contentTranslation: .none) } let (input, resolutionReads) = try countingReads { resolveTextEntryInput(app: app, target: target) } XCTAssertEqual(lookup?.snapshot?.identifier, "agent-device-hardware-keyboard-input") @@ -109,7 +109,7 @@ extension RunnerTests { "software keyboard is down: this simulator cannot show the keyboard a focus tap raises" ) - let stabilized = stabilizeTextInputBeforeTyping(app: app, target: field, keyboardVisibleBeforeTap: false) + let stabilized = stabilizeTextInputBeforeTyping(app: app, target: field, contentTranslation: .none, keyboardVisibleBeforeTap: false) guard case .inputReportsKeyboardFocus(let read) = stabilized.evidence else { return XCTFail("focus confirmed by \(stabilized.evidence?.name ?? "nothing"), not by the tapped input") diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift index 02e1978022..20f62e56b9 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryPolicyTests.swift @@ -739,6 +739,7 @@ extension RunnerTests { element: nil, refreshPoint: CGPoint(x: frame.midX, y: frame.midY), prefersFocusedElement: false, + contentTranslation: .none, inputAtRefreshPoint: try requireTextInputAtPoint(app: app, x: frame.midX, y: frame.midY) ), text: "abc", @@ -785,7 +786,8 @@ extension RunnerTests { target: TextEntryTarget( element: nil, refreshPoint: CGPoint(x: 10, y: 20), - prefersFocusedElement: false + prefersFocusedElement: false, + contentTranslation: .none ), text: "abc", delaySeconds: 0, @@ -810,7 +812,7 @@ extension RunnerTests { func testEmptyReplacementWithoutResolvableTargetFailsClosed() { let result = typeTextReliably( app: springboard, - target: TextEntryTarget(element: nil, refreshPoint: nil, prefersFocusedElement: false), + target: TextEntryTarget(element: nil, refreshPoint: nil, prefersFocusedElement: false, contentTranslation: .none), text: "", delaySeconds: 0, repairMode: .replacement, @@ -830,7 +832,7 @@ extension RunnerTests { XCTAssertEqual( awaitSynthesizedReplacementCommit( app: XCUIApplication(), - target: TextEntryTarget(element: nil, refreshPoint: CGPoint(x: 10, y: 20), prefersFocusedElement: false), + target: TextEntryTarget(element: nil, refreshPoint: CGPoint(x: 10, y: 20), prefersFocusedElement: false, contentTranslation: .none), expectedText: expectedText ), .unobservable diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryReadinessTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryReadinessTests.swift new file mode 100644 index 0000000000..f42559ae14 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryReadinessTests.swift @@ -0,0 +1,37 @@ +import XCTest + +extension RunnerTests { +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) + /// #3899: the readiness retap dispatches the read's screen box, not its raw snapshot frame. The + /// fixture cannot be placed in a window, so a `.known` vector moves the field's snapshot frame onto + /// the press-counting witness button: only a retap through `frameOnScreen` presses it. An + /// `.unknown` vector names no box and taps nothing; `.none` retaps the field itself. + @MainActor + func testReadinessRetapDispatchesTheReadsScreenSpaceCenter() throws { + app.launchArguments = ["--agent-device-text-entry-regression", "--agent-device-text-entry-tap-witness"] + app.launch() + defer { invalidateCachedTarget(reason: "unit_test_cleanup"); app.terminate() } + XCTAssertTrue(app.waitForFixtureExistence(timeout: appExistenceTimeout)) + let field = app.textFields["agent-device-hardware-keyboard-input"] + let witness = app.buttons["agent-device-tap-witness"] + let presses = app.staticTexts["agent-device-tap-witness-presses"] + for element in [field, witness, presses] { + XCTAssertTrue(element.waitForFixtureExistence(timeout: appExistenceTimeout)) + } + let fieldFrame = field.frame + let witnessFrame = witness.frame + + let unknown = try XCTUnwrap(ResolvedTextEntryInput(reading: field, contentTranslation: .unknown)) + XCTAssertNil(unknown.frameOnScreen) + retapAt(read: unknown, app: app, element: field) + XCTAssertEqual(presses.label, "presses=0", "an unknown content space dispatched a point") + + retapAt(read: try XCTUnwrap(ResolvedTextEntryInput(reading: field, contentTranslation: .none)), app: app, element: field) + XCTAssertEqual(presses.label, "presses=0", "a full-screen read dispatched a point off the field") + + let vector = CGSize(width: witnessFrame.midX - fieldFrame.midX, height: witnessFrame.midY - fieldFrame.midY) + retapAt(read: try XCTUnwrap(ResolvedTextEntryInput(reading: field, contentTranslation: .known(vector))), app: app, element: field) + XCTAssertEqual(presses.label, "presses=1", "the retap dispatched the raw snapshot center, not the screen-space one") + } +#endif +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift index c9b306758a..5d1fe3a468 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift @@ -15,7 +15,7 @@ extension RunnerTests { let completed = expectation(description: "optional probe completed inside runner wait") DispatchQueue.main.async { self.textInputProbeIssueForTesting = XCTIssue(type: .assertionFailure, compactDescription: "Optional probe issue during runner wait") - _ = self.probeTextInputs(app: self.app, point: point) + _ = self.probeTextInputs(app: self.app, point: point, contentTranslation: .none) completed.fulfill() } guard XCTWaiter.wait(for: [completed], timeout: 5) == .completed else { @@ -174,8 +174,8 @@ extension RunnerTests { } XCTExpectFailure("Required read and later issue belong to their caller", options: options) { textInputProbeIssueForTesting = expected - _ = textInputAt(app: app, x: point.x, y: point.y) - _ = probeTextInputs(app: app, point: point) + _ = textInputAt(app: app, x: point.x, y: point.y, contentTranslation: .none) + _ = probeTextInputs(app: app, point: point, contentTranslation: .none) record(expected) } XCTAssertEqual(observed, 2) @@ -192,7 +192,7 @@ extension RunnerTests { XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout)) let frame = field.frame textInputProbeIssueForTesting = XCTIssue(type: .assertionFailure, compactDescription: "Failed to get matching snapshot: kAXErrorIllegalArgument") - let outcome = probeTextInputs(app: app, point: CGPoint(x: frame.midX, y: frame.midY)) + let outcome = probeTextInputs(app: app, point: CGPoint(x: frame.midX, y: frame.midY), contentTranslation: .none) guard case .unavailable = outcome else { return XCTFail("A suppressed AX issue must discard the matching candidate") } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift index 2eb822e0f1..5adac13051 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift @@ -108,6 +108,7 @@ extension RunnerTests { element: textField, refreshPoint: nil, prefersFocusedElement: false, + contentTranslation: .none, fromTapWitness: true ), text: "\n", @@ -513,6 +514,7 @@ extension RunnerTests { element: nil, refreshPoint: nil, prefersFocusedElement: true, + contentTranslation: .none, fromTapWitness: true ), text: text, diff --git a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift index 300b8c910e..8d441911ce 100644 --- a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift +++ b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift @@ -14,18 +14,15 @@ public enum RunnerInterfaceOrientation { } public enum CoordinateSpaceRotation { - // Dispatch rotates a point about the reference frame's top-left. The origin is localized away - // and never added back: an XCUIApplication session observes only its own scene surface, so its - // windows report (0,0)-anchored frames (measured on iPhone landscape both ways and on every - // system window in window-coordinate-space.json), and displacement between panels or panes is - // carried by the record's displayID, not by an origin. + // Dispatch turns a point's axes and never moves its origin: the digitizer's origin is the + // display's own, where a placed window's screen points are already measured from (#3899). public static func native( point: CGPoint, in frame: CGRect, interfaceOrientation: Int ) -> CGPoint { - let localX = Double(point.x) - Double(frame.minX) - let localY = Double(point.y) - Double(frame.minY) + let localX = Double(point.x) + let localY = Double(point.y) let width = Double(frame.width) let height = Double(frame.height) switch interfaceOrientation { @@ -76,7 +73,7 @@ public enum CoordinateSpaceRotation { localX = Double(point.x) localY = Double(point.y) } - return CGPoint(x: localX + Double(frame.minX), y: localY + Double(frame.minY)) + return CGPoint(x: localX, y: localY) } public static func oriented( @@ -103,9 +100,27 @@ public enum CoordinateSpaceRotation { } } +/// A placed iPad window reports its subtree from its own top-left (#3899, ADR 0004). Its carrier is a +/// `(0,0)`-anchored copy of a viewport box that starts elsewhere; the viewport origin is the offset. +public enum SnapshotContentTranslation { + public static func offset(reportedBox: CGRect, viewportBox: CGRect) -> CGSize? { + let tolerance = SnapshotGeometrySpace.quarterTurnTolerance + func near(_ a: CGFloat, _ b: CGFloat) -> Bool { abs(a - b) <= tolerance } + guard SnapshotGeometry.isPositiveFinite(reportedBox), SnapshotGeometry.isPositiveFinite(viewportBox), + near(reportedBox.minX, 0), near(reportedBox.minY, 0), + !near(viewportBox.minX, 0) || !near(viewportBox.minY, 0), + near(reportedBox.width, viewportBox.width), near(reportedBox.height, viewportBox.height) + else { + return nil + } + return CGSize(width: viewportBox.minX, height: viewportBox.minY) + } +} + public enum SnapshotGeometrySpace: Equatable { case appOrientation case deviceNative(appFrame: SnapshotViewport.Box, interfaceOrientation: Int) + case windowContent(offset: CGSize) public static let quarterTurnTolerance: Double = 1 @@ -119,10 +134,13 @@ public enum SnapshotGeometrySpace: Equatable { in: appFrame.rect, interfaceOrientation: interfaceOrientation ) + case .windowContent(let offset): + return reportedFrame.offsetBy(dx: offset.width, dy: offset.height) } } - /// Only a `.reported` viewport carries an orientation, so only it can anchor a rotation. + /// Only a `.reported` viewport is anchored on the screen, so only it can anchor a rotation or a + /// translation. public static func space( reportedBySurfaceHost isSurfaceHost: Bool, reportedFrame: CGRect, @@ -130,13 +148,16 @@ public enum SnapshotGeometrySpace: Equatable { viewport: SnapshotViewport ) -> SnapshotGeometrySpace { guard isSurfaceHost else { return inherited } - guard case .reported(let appFrame, let interfaceOrientation) = viewport, - namesQuarterTurn(interfaceOrientation), - isQuarterTurned(reportedFrame, relativeTo: appFrame.rect) - else { + guard case .reported(let appFrame, let interfaceOrientation) = viewport else { return .appOrientation } - return .deviceNative(appFrame: appFrame, interfaceOrientation: interfaceOrientation) + if namesQuarterTurn(interfaceOrientation), isQuarterTurned(reportedFrame, relativeTo: appFrame.rect) { + return .deviceNative(appFrame: appFrame, interfaceOrientation: interfaceOrientation) + } + // A system window under a placed one (the keyboard, a SpringBoard overlay) reports the screen's + // own box, so it names no offset and keeps its frames. + return SnapshotContentTranslation.offset(reportedBox: reportedFrame, viewportBox: appFrame.rect) + .map { .windowContent(offset: $0) } ?? .appOrientation } public static func namesQuarterTurn(_ interfaceOrientation: Int) -> Bool { diff --git a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift index c5df2bc22c..d0412a4433 100644 --- a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift +++ b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift @@ -28,9 +28,18 @@ final class CoordinateSpaceTests: XCTestCase { let oriented: Frame } + struct WindowContentCase: Decodable { + struct Offset: Decodable { let width, height: Double } + let name: String + let reportedBox: Frame + let viewportBox: Frame + let offset: Offset? + } + let constants: Constants let quarterTurnCases: [QuarterTurnCase] let rotationCases: [RotationCase] + let windowContentCases: [WindowContentCase] } /// Identity in portrait/unknown, 90° per landscape, 180° upside-down. @@ -63,8 +72,8 @@ final class CoordinateSpaceTests: XCTestCase { in: offsetLandscape, interfaceOrientation: RunnerInterfaceOrientation.landscapeLeft ), - CGPoint(x: 268, y: 1040), - "non-zero frame origin is localized before rotation" + CGPoint(x: 288, y: 1030), + "a frame offset from the screen never moves the turned point (#3899)" ) } @@ -308,11 +317,13 @@ final class CoordinateSpaceTests: XCTestCase { /// (packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts). Add /// cases there, never fork the rule. Detection is replayed under both landscape turns, because /// which window declares the native space cannot depend on which way the app is turned; the - /// rotation rows carry their own orientation. + /// rotation rows carry their own orientation. The window-content rows (#3899) replay here alone: + /// the bridge guest refuses a windowed tree before the vitest twin's decoder can read one. func testWindowCoordinateSpaceMatchesGoldenParityTable() throws { let fixture = try loadWindowCoordinateSpaceFixture() XCTAssertFalse(fixture.quarterTurnCases.isEmpty, "parity table must not be empty") XCTAssertFalse(fixture.rotationCases.isEmpty, "parity table must not be empty") + XCTAssertFalse(fixture.windowContentCases.isEmpty, "parity table must not be empty") XCTAssertEqual( fixture.constants.quarterTurnTolerance, SnapshotGeometrySpace.quarterTurnTolerance, @@ -349,6 +360,13 @@ final class CoordinateSpaceTests: XCTestCase { testCase.name ) } + for testCase in fixture.windowContentCases { + XCTAssertEqual( + SnapshotContentTranslation.offset(reportedBox: testCase.reportedBox.cgRect, viewportBox: testCase.viewportBox.cgRect), + testCase.offset.map { CGSize(width: $0.width, height: $0.height) }, + testCase.name + ) + } } private func deviceNative(_ appFrame: CGRect, _ interfaceOrientation: Int) -> SnapshotGeometrySpace { @@ -486,6 +504,37 @@ final class CoordinateSpaceTests: XCTestCase { ) } + /// #3899: a placed window's subtree publishes at its screen position, while the system windows + /// under it and a window at the screen origin keep what the platform reported. + func testNormalizedTranslatesOnlyAPlacedWindowsSubtree() { + let windowOnScreen = CGRect(x: 224, y: 207, width: 386, height: 682) + let reportedRoot = CGRect(x: 0, y: 0, width: 386, height: 682) + let acquired = [ + coordinateNode(0, "Application", reportedRoot, nil, 0), + coordinateNode(1, "Window", reportedRoot, 0, 1), + coordinateNode(2, "Other", reportedRoot, 1, 2), + coordinateNode(3, "Button", CGRect(x: 335, y: 14, width: 37, height: 36), 2, 3, label: "Add"), + coordinateNode(4, "Window", CGRect(x: 0, y: 925, width: 834, height: 282), 0, 1, label: "keyboard"), + coordinateNode(5, "Key", CGRect(x: 300, y: 1100, width: 60, height: 60), 4, 2), + coordinateNode(6, "Window", CGRect(x: 0, y: 0, width: 834, height: 1210), 0, 1, label: "springboard"), + coordinateNode(7, "Alert", CGRect(x: 300, y: 500, width: 240, height: 160), 6, 2), + ] + func normalized(viewport: CGRect) -> [RawAXNode] { + SnapshotGeometrySpace.normalized( + nodes: acquired, + viewport: .reported(box: viewport, interfaceOrientation: RunnerInterfaceOrientation.portrait) + ) + } + let reported = acquired.map(\.rect.cgRect) + let placedNodes = normalized(viewport: windowOnScreen) + let placed = placedNodes.map(\.rect.cgRect) + XCTAssertEqual(Array(placed[0...2]), [windowOnScreen, windowOnScreen, windowOnScreen]) + XCTAssertEqual(placed[3], CGRect(x: 559, y: 221, width: 37, height: 36), "Add draws at (559,221)") + XCTAssertEqual(placedNodes[3].hittable, true, "hittability reads the translated frame") + XCTAssertEqual(Array(placed[4...]), Array(reported[4...]), "system windows already report screen space") + XCTAssertEqual(normalized(viewport: reportedRoot).map(\.rect.cgRect), reported, "a window at the screen origin is unchanged") + } + func testNormalizedOfAnEmptyArrayIsEmpty() { XCTAssertEqual( SnapshotGeometrySpace.normalized( diff --git a/contracts/fixtures/window-coordinate-space.json b/contracts/fixtures/window-coordinate-space.json index ca92a01a71..a0c5c3a278 100644 --- a/contracts/fixtures/window-coordinate-space.json +++ b/contracts/fixtures/window-coordinate-space.json @@ -1,5 +1,5 @@ { - "description": "Golden table for the coordinate space a captured window's own box declares (#2612), the twin of tap-keyboard-occlusion-policy.json on the geometry that guard reads. iOS hosts some system surfaces in the device's native (portrait-up) space even while the app is rotated, so their whole subtree arrives quarter-turned: measured on iPhone 17 Pro (iOS 26.2) with the system keyboard up over a landscape app frame (0,0,874,402), UIRemoteKeyboardWindow reports its own box as (0,0,402,874) — the app's box with the two side lengths swapped — while the app's own UIWindow, UITextEffectsWindow and a dev-menu overlay window all report (0,0,874,402); in portrait the same keyboard window reports (0,0,402,874) against a (0,0,402,874) app frame, i.e. nothing is turned. RULE 1 (detection) is enforced twice, in packages/contracts/src/window-coordinate-space.ts and in SnapshotGeometrySpace inside apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift, and decides from side lengths alone: a window is quarter-turned relative to an app frame when the app frame is not square (abs(app.width - app.height) > quarterTurnTolerance), both frames are finite, positive and non-empty, and the window's side lengths match the app's on the swapped axes (abs(window.width - app.height) <= quarterTurnTolerance && abs(window.height - app.width) <= quarterTurnTolerance). Origins are deliberately never compared: the turn is a statement about which axis a box runs along, and a window may legitimately be placed elsewhere. A square app frame cannot be told from its own quarter turn, so it is left alone rather than guessed at. The tolerance is 1 because measured captures match the swap to the point and this absorbs float representation only. RULE 2 (rotation) is the capture's way back for the runner's own UIInterfaceOrientation raw values — 1 portrait, 2 portraitUpsideDown, 3 landscapeRight, 4 landscapeLeft, and 0 unknown, where portrait and unknown publish what the platform reported rather than inventing a turn — implemented by CoordinateSpaceRotation.oriented(rect:in:interfaceOrientation:) in that same Swift file, whose rect helper turns opposite corners of the reported box and re-orders them so the result keeps a top-left origin. `rotationCases` are native -> oriented rows: the landscape ones measured on the device above, the 180 degree and identity ones derived from that table. Capture turns back with the exact inverse of what dispatch turns forward, so each row also has to come back to its own `native` box under the forward mapping; that is why one measured-looking landscape-left pairing whose native and oriented boxes are identical is absent — the table computes no such row. Infinity has no JSON spelling, so the unusable frame a platform hands back when it could not resolve a box is named {\"infinite\": true} and decodes to CGRect.infinite on the Swift side and to an infinite rect on the TypeScript side.", + "description": "Golden table for the coordinate space a captured window's own box declares (#2612), the twin of tap-keyboard-occlusion-policy.json on the geometry that guard reads. iOS hosts some system surfaces in the device's native (portrait-up) space even while the app is rotated, so their whole subtree arrives quarter-turned: measured on iPhone 17 Pro (iOS 26.2) with the system keyboard up over a landscape app frame (0,0,874,402), UIRemoteKeyboardWindow reports its own box as (0,0,402,874) — the app's box with the two side lengths swapped — while the app's own UIWindow, UITextEffectsWindow and a dev-menu overlay window all report (0,0,874,402); in portrait the same keyboard window reports (0,0,402,874) against a (0,0,402,874) app frame, i.e. nothing is turned. RULE 1 (detection) is enforced twice, in packages/contracts/src/window-coordinate-space.ts and in SnapshotGeometrySpace inside apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift, and decides from side lengths alone: a window is quarter-turned relative to an app frame when the app frame is not square (abs(app.width - app.height) > quarterTurnTolerance), both frames are finite, positive and non-empty, and the window's side lengths match the app's on the swapped axes (abs(window.width - app.height) <= quarterTurnTolerance && abs(window.height - app.width) <= quarterTurnTolerance). Origins are deliberately never compared: the turn is a statement about which axis a box runs along, and a window may legitimately be placed elsewhere. A square app frame cannot be told from its own quarter turn, so it is left alone rather than guessed at. The tolerance is 1 because measured captures match the swap to the point and this absorbs float representation only. RULE 2 (rotation) is the capture's way back for the runner's own UIInterfaceOrientation raw values — 1 portrait, 2 portraitUpsideDown, 3 landscapeRight, 4 landscapeLeft, and 0 unknown, where portrait and unknown publish what the platform reported rather than inventing a turn — implemented by CoordinateSpaceRotation.oriented(rect:in:interfaceOrientation:) in that same Swift file, whose rect helper turns opposite corners of the reported box and re-orders them so the result keeps a top-left origin. `rotationCases` are native -> oriented rows: the landscape ones measured on the device above, the 180 degree and identity ones derived from that table. Capture turns back with the exact inverse of what dispatch turns forward, so each row also has to come back to its own `native` box under the forward mapping; that is why one measured-looking landscape-left pairing whose native and oriented boxes are identical is absent — the table computes no such row. Infinity has no JSON spelling, so the unusable frame a platform hands back when it could not resolve a box is named {\"infinite\": true} and decodes to CGRect.infinite on the Swift side and to an infinite rect on the TypeScript side. RULE 3 (content translation, #3899) is the one rule here that reads an origin: an iPad window placed off the screen origin (Windowed Apps, Stage Manager) reports its subtree from its own top-left — measured on iPad Pro 11-inch (M5, iOS 27.0), a Calendar window at screen (224,207,386x682) reports its root as (0,0,386,682) and its Add button as (335,14,37,36) for a control drawn at (559,221). SnapshotContentTranslation.offset(reportedBox:viewportBox:) names the viewport origin as the offset only when the carrier is a (0,0)-anchored box whose side lengths match the viewport's within the tolerance and the viewport starts elsewhere; any other carrier, such as the keyboard window or a SpringBoard overlay that report the screen's own box, and every window at the screen origin name none (null). `windowContentCases` replay against that Swift function alone, because the bridge guest refuses a windowed tree before the host decoder can read one. `rotationCases` pin the turn as axes-only: a non-zero app frame origin never travels through it.", "constants": { "quarterTurnTolerance": 1 }, @@ -126,11 +126,11 @@ "oriented": { "x": 75, "y": 198, "width": 0, "height": 0 } }, { - "name": "landscape right localizes a non-zero app frame origin before the turn and restores it after", + "name": "a non-zero app frame origin does not travel through the turn: the same native box comes back at the same place (#3899)", "interfaceOrientation": 3, "app": { "x": 12, "y": 24, "width": 874, "height": 402 }, "native": { "x": 2, "y": 75, "width": 202, "height": 724 }, - "oriented": { "x": 87, "y": 222, "width": 724, "height": 202 } + "oriented": { "x": 75, "y": 198, "width": 724, "height": 202 } }, { "name": "landscape left turns the same plane the other way, which is where its own native column is", @@ -167,5 +167,67 @@ "native": { "x": 226, "y": 475, "width": 0, "height": 0 }, "oriented": { "x": 476, "y": 226, "width": 0, "height": 0 } } + ], + "windowContentCases": [ + { + "name": "a Calendar window placed at (224,207) reports its subtree from its own top-left", + "reportedBox": { "x": 0, "y": 0, "width": 386, "height": 682 }, + "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "offset": { "width": 224, "height": 207 } + }, + { + "name": "the same window moved elsewhere on the screen carries its subtree by the new origin", + "reportedBox": { "x": 0, "y": 0, "width": 386, "height": 682 }, + "viewportBox": { "x": 120, "y": 90, "width": 386, "height": 682 }, + "offset": { "width": 120, "height": 90 } + }, + { + "name": "the keyboard window under a windowed app reports the screen's box and stays in screen space", + "reportedBox": { "x": 0, "y": 925, "width": 834, "height": 282 }, + "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "offset": null + }, + { + "name": "a SpringBoard overlay under a windowed app reports the whole screen and translates nothing", + "reportedBox": { "x": 0, "y": 0, "width": 834, "height": 1210 }, + "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "offset": null + }, + { + "name": "the app's own window box reports the screen box it is placed at and translates nothing", + "reportedBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "offset": null + }, + { + "name": "a full-screen window — iPad, iPhone, or an ADR 0025 foldable panel — starts at (0,0) and translates nothing", + "reportedBox": { "x": 0, "y": 0, "width": 834, "height": 1210 }, + "viewportBox": { "x": 0, "y": 0, "width": 834, "height": 1210 }, + "offset": null + }, + { + "name": "a half-tolerance side-length miss is float representation, not a different box", + "reportedBox": { "x": 0, "y": 0, "width": 385.5, "height": 682 }, + "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "offset": { "width": 224, "height": 207 } + }, + { + "name": "a carrier two points narrower than the viewport is some other box and declares no space", + "reportedBox": { "x": 0, "y": 0, "width": 384, "height": 682 }, + "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "offset": null + }, + { + "name": "an empty carrier names no box to translate", + "reportedBox": { "x": 0, "y": 0, "width": 0, "height": 0 }, + "viewportBox": { "x": 224, "y": 207, "width": 0.5, "height": 0.5 }, + "offset": null + }, + { + "name": "an empty viewport names no screen box to translate toward", + "reportedBox": { "x": 0, "y": 0, "width": 0.5, "height": 0.5 }, + "viewportBox": { "x": 224, "y": 207, "width": 0, "height": 0 }, + "offset": null + } ] } diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index 0d7d1c78c3..cc65bfac0d 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -626,6 +626,37 @@ The rule detects un-normalized arrival, not a standing fact about iOS. The producers that the earlier sections of this ADR describe do normalize. A band taller than it is wide is what the output of a producer that did not normalize looks like. +## Amendment: a window that does not fill the screen (issue #3899) + +An iPad window that does not fill the screen (Windowed Apps; Stage Manager is unmeasured) reports its subtree from its own top-left. +Measured on iPad Pro 11-inch (M5, iOS 27.0) with a Calendar window at screen `(224,207,386x682)`: the root reads `(0,0,386,682)` and `Add` reads `(335,14,37,36)` but draws at `(559,221)`. +The runner's live window read, the keyboard window, and a SpringBoard overlay report screen boxes. +Published as reported, the capture mixed two spaces: the fold clipped nodes (`regular-node-outside-cumulative-clip`), rects named no screenshot position, and taps landed at the point minus the window origin. + +Decision: a placed window publishes screen space in points, because its viewport, its screenshot, and the digitizer already use that space. +`SnapshotContentTranslation.offset` is the one rule, applied in the same `SnapshotGeometrySpace.normalized` pass that turns a quarter-turned subtree. +A carrier declares an offset only when its box is a `(0,0)`-anchored copy of the reported viewport box and the viewport starts elsewhere; the offset is the viewport origin. +Any other carrier keeps its frames, so the screen-space system windows above do not move. +A full-screen window, an iPhone, and each ADR 0025 foldable panel start at `(0,0)`, so they read no offset and publish unchanged. +`contracts/fixtures/window-coordinate-space.json` replays the rule in Swift only, because the bridge guest refuses a windowed tree (`app-window-not-full-screen`) before the host decoder reads one. + +Rotation and translation stay separate. +`CoordinateSpaceRotation` turns axes and never moves an origin: the old localization of a frame origin was dead code at every measured full-screen origin and the dispatch bug here. +A synthesized turn swaps the sole display's side lengths, as the digitizer does, not a placed window's; a multi-screen host keeps the window box of its panel. + +Dispatch takes screen points. +`XCUIElement.frame` and the app-anchored `XCUICoordinate` are screen space on a placed window; `XCUIElementSnapshot.frame` is window content space. +The text-entry probe therefore owns one screen box, `ResolvedTextEntryInput.frameOnScreen`. +A command reads the vector from the resolved window's box once, where it builds its target, and the readiness retap reads it again on its own branch. +The vector is `TextEntryContentTranslation` (`known`, `none`, or `unknown` when the read failed), with no default, so a failed read never passes for full screen. +The target and each read it produces carry that vector. +`frameOnScreen` is nil without a snapshot (#3060) or with `unknown`. +On nil, the focus tap dispatches the caller's own point, and the refresh point, the caret tap, and the readiness retap dispatch nothing. +Probe matching then compares the live `element.frame`, because a comparison dispatches nothing. + +Open gaps: landscape under Windowed Apps (fixture arithmetic only), Stage Manager, and a window partly off screen are unmeasured. +A window moved between the command's vector read and its dispatch drifts like any live read. + ## Amendment: modal containment in the presentation cut The bridge reads one window and reports every container attached to it. @@ -723,10 +754,7 @@ If the frames differ, the guest refuses the capture (`app-window-not-full-screen This refusal provides evidence about the screen arrangement, not the app, so it has capture scope. If the runtime cannot report either frame, the guest keeps the capture. -The XCTest fallback is not correct for a windowed app either. -The runner clips window-relative element frames against the screen-relative app frame. -That is a runner gap. -It is not a reason to publish a bridge tree that is known to be wrong. +The bridge still refuses and converts nothing; the XCTest fallback publishes a windowed app in screen space (#3899 amendment above). Rejected alternatives: diff --git a/packages/platform-apple/src/snapshot-route.test.ts b/packages/platform-apple/src/snapshot-route.test.ts index ca1a92920c..afd4dadacd 100644 --- a/packages/platform-apple/src/snapshot-route.test.ts +++ b/packages/platform-apple/src/snapshot-route.test.ts @@ -407,7 +407,7 @@ test.for([ ['foreground-owner-changed', 'while another surface owned the foreground'], [ 'app-window-not-full-screen', - 'while the app window did not fill the screen; XCTest geometry for a windowed app can also be offset from the screen', + 'while the app window did not fill the screen; the runner republishes a windowed capture in screen space', ], ] as const)('a %s refusal sends only that capture to the runner', async ([code, consequence]) => { const source = sourceReturning({ stage: 'failed', failure: { kind: 'unsupported', code } }); diff --git a/packages/platform-apple/src/snapshot-route.ts b/packages/platform-apple/src/snapshot-route.ts index 5fcee4fca7..c977be2baf 100644 --- a/packages/platform-apple/src/snapshot-route.ts +++ b/packages/platform-apple/src/snapshot-route.ts @@ -512,7 +512,7 @@ function captureScopedFallbackWarning(code: string, consequence: CaptureScopedCo case 'foreground-ownership': return `${thisCapture}, while another surface owned the foreground.`; case 'window-not-full-screen': - return `${thisCapture}, while the app window did not fill the screen; XCTest geometry for a windowed app can also be offset from the screen.`; + return `${thisCapture}, while the app window did not fill the screen; the runner republishes a windowed capture in screen space.`; case 'coordinate-space-unresolved': return `${thisCapture}, which reports captured geometry in the app's own orientation space.`; } diff --git a/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts b/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts index 8abe052783..2edd1a74fb 100644 --- a/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts +++ b/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts @@ -42,6 +42,7 @@ type Fixture = { constants: { quarterTurnTolerance: number }; quarterTurnCases: QuarterTurnCase[]; rotationCases: RotationCase[]; + windowContentCases: { name: string }[]; }; const TABLE_PATH = path.resolve( @@ -83,7 +84,9 @@ test('the quarter-turn window rule agrees with every golden parity table case', assert.ok(table.quarterTurnCases.length > 0, 'parity table must not be empty'); assert.ok(table.rotationCases.length > 0, 'parity table must not be empty'); assertUniqueNames( - [...table.quarterTurnCases, ...table.rotationCases].map((fixture) => fixture.name), + [...table.quarterTurnCases, ...table.rotationCases, ...table.windowContentCases].map( + (fixture) => fixture.name, + ), 'parity table', ); for (const fixture of table.quarterTurnCases) { @@ -104,7 +107,7 @@ const LANDSCAPE_RIGHT = 3; const LANDSCAPE_LEFT = 4; /** - * The dispatch direction — an app-space point in the native space a synthesized event is performed in + * The dispatch direction — the caller's screen point in the native space a synthesized event is performed in * — as the runner's own `CoordinateSpaceRotation.native(point:in:interfaceOrientation:)` computes it. * * The capture's inverse is NOT implemented here: it lives in Swift alone, and a TypeScript copy of it @@ -115,8 +118,8 @@ const LANDSCAPE_LEFT = 4; * forward map it ships with. */ function nativePoint(point: Point, app: Rect, interfaceOrientation: number): Point { - const localX = point.x - app.x; - const localY = point.y - app.y; + // The turn moves axes and never an origin (#3899). + const { x: localX, y: localY } = point; switch (interfaceOrientation) { case LANDSCAPE_RIGHT: return { x: app.height - localY, y: localX }; diff --git a/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts b/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts index 13c58fa59e..eef984c2ae 100644 --- a/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts +++ b/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts @@ -25,7 +25,9 @@ import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; * `SnapshotGeometrySpace` in * `apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift`. The table also * pins the way back (`CoordinateSpaceRotation.oriented(rect:in:interfaceOrientation:)`, Swift-only - * today); change either rule only through that table. + * today); change either rule only through that table. A placed iPad window's content translation + * (#3899) is the Swift capture's alone (`SnapshotContentTranslation`): the guest refuses a windowed + * tree before this decoder can read one. * * Why this is a detector and not a repair: the bridge guest cannot read the app's interface * orientation. Measured in the #2659 spike (verdict on the issue, write-up in the diff of #2667):