diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m index 420e806ae7..c9b6d61bd7 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m @@ -848,6 +848,31 @@ - (void)viewDidLoad { } } + // Overlapping sibling buttons must not become children of an ordinary slider. + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-wide-slider"]) { + UISlider *slider = [[UISlider alloc] initWithFrame:CGRectMake(20, 20, 360, 31)]; + slider.accessibilityLabel = @"Volume"; + [self.view addSubview:slider]; + NSArray *titles = @[ @"Left", @"Right" ]; + for (NSUInteger i = 0; i < titles.count; i++) { + UIButton *button = [UIButton buttonWithType:UIButtonTypeSystem]; + [button setTitle:titles[i] forState:UIControlStateNormal]; + button.frame = CGRectMake(30 + i * 170, 20, 100, 30); + [self.view addSubview:button]; + } + } + + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-collapsed-tab-bar"]) { + UITabBar *tabBar = [[UITabBar alloc] initWithFrame:CGRectMake(20, 20, 360, 60)]; + tabBar.items = @[ + [[UITabBarItem alloc] initWithTitle:@"First tab" image:nil tag:0], + [[UITabBarItem alloc] initWithTitle:@"Second tab" image:nil tag:1], + ]; + tabBar.isAccessibilityElement = YES; + tabBar.accessibilityLabel = @"Collapsed tabs"; + [self.view addSubview:tabBar]; + } + BOOL markedButton = [NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-dismiss-popup"]; if (markedButton || [NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-dismiss-popup-window"]) { UIButton *dismissRegion = [UIButton buttonWithType:UIButtonTypeSystem]; diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index 0e0670aecb..b3c6bf0812 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -21,6 +21,8 @@ extension RunnerTests { struct SnapshotTraversalContext { let queryRoot: XCUIElement let rootSnapshot: XCUIElementSnapshot + /// How long reading `rootSnapshot` took: what one more query over the whole app is expected to cost. + let rootReadDuration: TimeInterval /** Carries which way the app's interface is turned from the device's native space (#2612). */ let viewport: SnapshotViewport /** @@ -144,9 +146,11 @@ extension RunnerTests { min(startedAt.addingTimeInterval(flatInteractiveFallbackBudget), planDeadline) } - /// Whether a sweep query started at `now` can still finish before the slice `deadline`. - static func querySweepCanStartQuery(deadline: Date, now: Date) -> Bool { - deadline.timeIntervalSince(now) >= flatInteractiveQueryBudget + /// Whether a query started at `now` and expected to take `expectedCost` can still finish before + /// `deadline`. XCTest cannot cancel a query once it starts; the least admitted cost is a sweep + /// query's. + static func xctestQueryCanStart(deadline: Date, now: Date, expectedCost: TimeInterval = 0) -> Bool { + deadline.timeIntervalSince(now) >= max(flatInteractiveQueryBudget, expectedCost) } /// What one capture may spend reading the keyboard band before it gives up on the fact and lets the @@ -181,18 +185,19 @@ extension RunnerTests { @MainActor func recursiveTreeSnapshotAcquisition( context: SnapshotTraversalContext, - hint: CaptureHint - ) throws -> SnapshotAcquisition { - var cachedDescendantElements: [XCUIElement]? - func collapsedTabDescendants() -> [XCUIElement] { - if let cachedDescendantElements { - return cachedDescendantElements - } - let result = snapshotElementsQuery { - context.queryRoot.descendants(matching: .any).allElementsBoundByIndex + hint: CaptureHint, + processingDeadline: Date + ) throws -> (acquisition: SnapshotAcquisition, outcome: SnapshotTierOutcome) { + var outcome = SnapshotTierOutcome.completed + var cachedCollapsedTabCandidates: [CollapsedTabCandidate]? + func collapsedTabCandidatesOnce() -> [CollapsedTabCandidate] { + if let cachedCollapsedTabCandidates { + return cachedCollapsedTabCandidates } - cachedDescendantElements = result.elements - return result.elements + let candidates = collapsedTabCandidates(context: context, processingDeadline: processingDeadline) + if candidates == nil { outcome = .deadlineExhausted } + cachedCollapsedTabCandidates = candidates ?? [] + return candidates ?? [] } // Acquisition serializes reported frames only; no coordinate space or fold decision is carried @@ -212,7 +217,7 @@ extension RunnerTests { appendCollapsedTabFallbackNodes( to: &nodes, containerSnapshot: context.rootSnapshot, - resolveElements: collapsedTabDescendants, + resolveCandidates: collapsedTabCandidatesOnce, depth: 1, parentIndex: 0 ) @@ -269,19 +274,22 @@ extension RunnerTests { appendCollapsedTabFallbackNodes( to: &nodes, containerSnapshot: snapshot, - resolveElements: collapsedTabDescendants, + resolveCandidates: collapsedTabCandidatesOnce, depth: depth + 1, parentIndex: node.index ) } } - return SnapshotAcquisition( - hint: hint, - nodes: nodes, - truncated: false, - effectiveDepth: nil, - viewport: context.viewport + return ( + SnapshotAcquisition( + hint: hint, + nodes: nodes, + truncated: outcome == .deadlineExhausted, + effectiveDepth: nil, + viewport: context.viewport + ), + outcome ) } @@ -469,7 +477,7 @@ extension RunnerTests { let flatElements = flatInteractiveElements(app: app, deadline: deadline) var outcome = flatElements.outcome for element in flatElements.elements { - if !Self.querySweepCanStartQuery(deadline: deadline, now: Date()) { + if !Self.xctestQueryCanStart(deadline: deadline, now: Date()) { NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_FLAT_FALLBACK_DEADLINE") outcome = .deadlineExhausted break diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift index 420b55e7ba..c3288efb66 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift @@ -40,9 +40,11 @@ extension RunnerTests { ) throws -> SnapshotTraversalContext? { let treeSliceBudget = treeCaptureSliceBudgetOverride ?? treeCaptureSliceBudget let slice = min(treeSliceBudget, max(0.5, captureDeadline.timeIntervalSinceNow)) + let rootReadStartedAt = Date() guard let rootSnapshot = try captureSnapshotRootBounded(app, sliceSeconds: slice) else { return nil } + let rootReadDuration = Date().timeIntervalSince(rootReadStartedAt) // The viewport and the interface orientation are one hop: geometry that arrives in the device's // native space can only be placed relative to the app's own frame and rotation, and asking for @@ -64,6 +66,7 @@ extension RunnerTests { return SnapshotTraversalContext( queryRoot: app, rootSnapshot: rootSnapshot, + rootReadDuration: rootReadDuration, viewport: viewport, keyboardBand: keyboardBand ) @@ -184,7 +187,7 @@ extension RunnerTests { func evaluateSnapshot(_ snapshot: XCUIElementSnapshot) -> SnapshotEvaluation { let label = aggregatedLabel(for: snapshot) ?? snapshot.label.trimmingCharacters(in: .whitespacesAndNewlines) let identifier = snapshot.identifier.trimmingCharacters(in: .whitespacesAndNewlines) - let valueText = snapshotValueText(snapshot) + let valueText = Self.valueText(snapshot.value) return SnapshotEvaluation( label: label, identifier: identifier, @@ -231,8 +234,9 @@ extension RunnerTests { } } - private func snapshotValueText(_ snapshot: XCUIElementSnapshot) -> String? { - guard let value = snapshot.value else { return nil } + /// An element's value as the node publishes it; blank reads as no value. + static func valueText(_ value: Any?) -> String? { + guard let value else { return nil } let text = String(describing: value).trimmingCharacters(in: .whitespacesAndNewlines) return text.isEmpty ? nil : text } @@ -282,7 +286,7 @@ extension RunnerTests { if depth > 4 { return nil } let text = snapshot.label.trimmingCharacters(in: .whitespacesAndNewlines) if !text.isEmpty { return text } - if let valueText = snapshotValueText(snapshot) { return valueText } + if let valueText = Self.valueText(snapshot.value) { return valueText } for child in snapshot.children { if let childLabel = aggregatedLabel(for: child, depth: depth + 1) { return childLabel @@ -294,13 +298,13 @@ extension RunnerTests { func appendCollapsedTabFallbackNodes( to nodes: inout [RawAXNode], containerSnapshot: XCUIElementSnapshot, - resolveElements: () -> [XCUIElement], + resolveCandidates: () -> [CollapsedTabCandidate], depth: Int, parentIndex: Int ) { let fallbackNodes = collapsedTabFallbackNodes( for: containerSnapshot, - resolveElements: resolveElements, + resolveCandidates: resolveCandidates, startingIndex: nodes.count, depth: depth, parentIndex: parentIndex @@ -308,28 +312,134 @@ extension RunnerTests { nodes.append(contentsOf: fallbackNodes) } + /// Every descendant of `root` in its captured snapshot, without `root` itself. + static func snapshotDescendants(of root: XCUIElementSnapshot) -> [XCUIElementSnapshot] { + var descendants: [XCUIElementSnapshot] = [] + var stack = Array(root.children.reversed()) + while let snapshot = stack.popLast() { + descendants.append(snapshot) + stack.append(contentsOf: snapshot.children.reversed()) + } + return descendants + } + + /// One live element a collapsed tab container may adopt, as the matching query's predicate read it. + struct CollapsedTabCandidate { + let type: XCUIElement.ElementType + let frame: CGRect + let label: String + let identifier: String + let valueText: String? + let enabled: Bool + let focused: Bool + let selected: Bool + } + + /// The live elements any collapsed tab container in this capture may adopt, read in one XCTest + /// query: `collapsedTabCandidatePredicate` copies each match's attributes while XCTest evaluates + /// it, so no match costs a further round trip. The query starts only while the time left before + /// `processingDeadline` covers what this capture's root tree read took; nil means it did not start. + @MainActor + func collapsedTabCandidates( + context: SnapshotTraversalContext, + processingDeadline: Date + ) -> [CollapsedTabCandidate]? { + let containerFrames = Self.snapshotDescendants(of: context.rootSnapshot) + .filter(isCollapsedTabContainer) + .map(\.frame) + var candidates: [CollapsedTabCandidate] = [] + let predicate = Self.collapsedTabCandidatePredicate(containerFrames: containerFrames) { attributes in + candidates.append(self.collapsedTabCandidate(attributes)) + } + guard + Self.xctestQueryCanStart( + deadline: processingDeadline, + now: Date(), + expectedCost: context.rootReadDuration + ) + else { + NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_TAB_FALLBACK_QUERY_NOT_STARTED") + return nil + } + let matches = snapshotElementsQuery { + context.queryRoot.descendants(matching: .any).matching(predicate).allElementsBoundByIndex + }.elements + return matches.isEmpty ? [] : candidates + } + + private func collapsedTabCandidate(_ attributes: XCUIElementAttributes) -> CollapsedTabCandidate { + CollapsedTabCandidate( + type: attributes.elementType, + frame: attributes.frame, + label: attributes.label.trimmingCharacters(in: .whitespacesAndNewlines), + identifier: attributes.identifier.trimmingCharacters(in: .whitespacesAndNewlines), + valueText: Self.valueText(attributes.value), + enabled: attributes.isEnabled, + focused: focusBool(attributes as! NSObject), + selected: attributes.isSelected + ) + } + + /// Matches an element some container in `containerFrames` may adopt: it has content + /// (`hasCollapsedTabContent`) and the type and geometry `isCollapsedTabCandidate` accepts. XCTest + /// evaluates it against each element's attributes while it resolves the query, and `matched` + /// receives the attributes of each element it accepts. + static func collapsedTabCandidatePredicate( + containerFrames: [CGRect], + matched: @escaping (XCUIElementAttributes) -> Void + ) -> NSPredicate { + NSPredicate { object, _ in + guard let attributes = object as? XCUIElementAttributes, hasCollapsedTabContent(attributes), + containerFrames.contains(where: { + isCollapsedTabCandidate(type: attributes.elementType, frame: attributes.frame, in: $0) + }) + else { + return false + } + matched(attributes) + return true + } + } + + /// The geometry and type a collapsed tab container's item has: a candidate type, smaller than the + /// container, and centered inside it. + static func isCollapsedTabCandidate( + type: XCUIElement.ElementType, + frame: CGRect, + in containerFrame: CGRect + ) -> Bool { + guard collapsedTabCandidateTypes.contains(type), !frame.isNull, !frame.isEmpty else { + return false + } + let area = max(CGFloat(1), frame.width * frame.height) + let containerArea = max(CGFloat(1), containerFrame.width * containerFrame.height) + return area < containerArea * 0.9 && containerFrame.contains(CGPoint(x: frame.midX, y: frame.midY)) + } + + /// Whether an element names something an agent can address: a label, identifier or value that is + /// not blank. Candidate types carry no placeholder. + static func hasCollapsedTabContent(_ attributes: XCUIElementAttributes) -> Bool { + [attributes.label, attributes.identifier] + .contains { !$0.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty } + || valueText(attributes.value) != nil + } + private func collapsedTabFallbackNodes( for containerSnapshot: XCUIElementSnapshot, - resolveElements: () -> [XCUIElement], + resolveCandidates: () -> [CollapsedTabCandidate], startingIndex: Int, depth: Int, parentIndex: Int ) -> [RawAXNode] { - if !containerSnapshot.children.isEmpty { return [] } - // This fallback reads live element frames, which XCTest reports in the app's own space, and a - // collapsed tab container sits under the app's own window. `SnapshotGeometrySpace.normalized` - // keys rotation on ancestry, so these nodes never enter a declared native space and need no - // special case here; the containment and area rules below compare reported frames to each other. - guard shouldExpandCollapsedTabContainer(containerSnapshot) else { return [] } + guard isCollapsedTabContainer(containerSnapshot) else { return [] } + // Candidates are read from live elements, which XCTest reports in the app's own space, and a + // collapsed tab container sits under the app's own window. `SnapshotGeometrySpace.normalized` keys + // rotation on ancestry, so these nodes never enter a declared native space and need no special + // case here; `isCollapsedTabCandidate` compares reported frames to each other. let containerFrame = containerSnapshot.frame - if containerFrame.isNull || containerFrame.isEmpty { return [] } - - // Collapsed tab containers should be rare, so a full descendant scan is acceptable once per - // snapshot as a fallback for XCTest omitting the tab children from the snapshot tree. - let elements = resolveElements() - let candidates = elements.compactMap { element in + let candidates = resolveCandidates().compactMap { candidate in collapsedTabCandidateNode( - element: element, + candidate: candidate, containerSnapshot: containerSnapshot, containerFrame: containerFrame ) @@ -362,67 +472,41 @@ extension RunnerTests { } private func collapsedTabCandidateNode( - element: XCUIElement, + candidate: CollapsedTabCandidate, containerSnapshot: XCUIElementSnapshot, containerFrame: CGRect ) -> RawAXNode? { - var node: RawAXNode? - let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ - if !element.exists { return } - let elementType = element.elementType - if !Self.collapsedTabCandidateTypes.contains(elementType) { return } - let frame = element.frame - if frame.isNull || frame.isEmpty { return } - if frame.equalTo(containerFrame) { return } - let area = max(CGFloat(1), frame.width * frame.height) - let containerArea = max(CGFloat(1), containerFrame.width * containerFrame.height) - if area >= containerArea * 0.9 { return } - let center = CGPoint(x: frame.midX, y: frame.midY) - if !containerFrame.contains(center) { return } - - let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines) - let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines) - let valueText = snapshotValueText(element) - let placeholder = elementPlaceholderText(element, type: elementType) - let hasContent = !label.isEmpty || !identifier.isEmpty || valueText != nil || placeholder != nil - if !hasContent { return } - if sameSemanticElement( - containerSnapshot: containerSnapshot, - elementType: elementType, - label: label, - identifier: identifier - ) { - return - } - - // The containment and area rules above compared reported frames with each other; the node - // joins the tree carrying the frame the platform reported, and normalization places it. - node = RawAXNode( - index: 0, - type: elementTypeName(elementType), - label: label.isEmpty ? nil : label, - identifier: identifier.isEmpty ? nil : identifier, - value: valueText, - placeholder: placeholder, - rect: SnapshotRect(frame), - enabled: element.isEnabled, - focused: elementHasFocus(element) ? true : nil, - selected: element.isSelected ? true : nil, - hittable: false, - depth: 0, - parentIndex: nil, - hiddenContentAbove: nil, - hiddenContentBelow: nil - ) - }) - if let exceptionMessage { - NSLog( - "AGENT_DEVICE_RUNNER_SNAPSHOT_TAB_FALLBACK_IGNORED_EXCEPTION=%@", - exceptionMessage - ) + if !Self.isCollapsedTabCandidate(type: candidate.type, frame: candidate.frame, in: containerFrame) { return nil } - return node + if sameSemanticElement( + containerSnapshot: containerSnapshot, + elementType: candidate.type, + label: candidate.label, + identifier: candidate.identifier + ) { + return nil + } + + // The containment and area rules above compared reported frames with each other; the node + // joins the tree carrying the frame the platform reported, and normalization places it. + return RawAXNode( + index: 0, + type: elementTypeName(candidate.type), + label: candidate.label.isEmpty ? nil : candidate.label, + identifier: candidate.identifier.isEmpty ? nil : candidate.identifier, + value: candidate.valueText, + placeholder: nil, + rect: SnapshotRect(candidate.frame), + enabled: candidate.enabled, + focused: candidate.focused ? true : nil, + selected: candidate.selected ? true : nil, + hittable: false, + depth: 0, + parentIndex: nil, + hiddenContentAbove: nil, + hiddenContentBelow: nil + ) } func snapshotHasFocus(_ snapshot: XCUIElementSnapshot) -> Bool { @@ -453,24 +537,20 @@ extension RunnerTests { return snapshot.isSelected } - private func shouldExpandCollapsedTabContainer(_ snapshot: XCUIElementSnapshot) -> Bool { + /// A wide tab bar or segmented control that XCTest reports without children. + func isCollapsedTabContainer(_ snapshot: XCUIElementSnapshot) -> Bool { + if !snapshot.children.isEmpty { return false } let frame = snapshot.frame if frame.isNull || frame.isEmpty { return false } if frame.width < max(CGFloat(160), frame.height * 1.75) { return false } switch snapshot.elementType { - case .tabBar, .segmentedControl, .slider: + case .tabBar, .segmentedControl: return true default: return false } } - private func snapshotValueText(_ element: XCUIElement) -> String? { - let text = String(describing: element.value ?? "") - .trimmingCharacters(in: .whitespacesAndNewlines) - return text.isEmpty ? nil : text - } - private func sameSemanticElement( containerSnapshot: XCUIElementSnapshot, elementType: XCUIElement.ElementType, @@ -519,7 +599,7 @@ extension RunnerTests { } /// Runs sweep queries in order until one reports AX unavailable, or until the next one could not - /// finish before `deadline` (`querySweepCanStartQuery`). A sweep stopped by the deadline reports + /// finish before `deadline` (`xctestQueryCanStart`). A sweep stopped by the deadline reports /// `.deadlineExhausted` rather than a truncation flag: what it collected is a partial tree, and /// only the caller that owns the tier decides whether that counts as an answer (#2781). static func runFlatInteractiveQueries( @@ -530,7 +610,7 @@ extension RunnerTests { ) -> (elements: [Element], outcome: SnapshotTierOutcome) { var elements: [Element] = [] for query in queries { - if !querySweepCanStartQuery(deadline: deadline, now: now()) { + if !xctestQueryCanStart(deadline: deadline, now: now()) { NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_FLAT_FALLBACK_DEADLINE") return (elements, .deadlineExhausted) } @@ -573,7 +653,7 @@ extension RunnerTests { if frame.isNull || frame.isEmpty { return } let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines) let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines) - let valueText = snapshotValueText(element) + let valueText = Self.valueText(element.value) let elementType = element.elementType let enabled = element.isEnabled diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index 596c9a39ca..70574b4e92 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -505,16 +505,22 @@ extension RunnerTests { return (nil, .completed) } keyboardBand = context.keyboardBand + let processingSlice = min(self.treeCaptureSliceBudget, max(0.5, deadline.timeIntervalSinceNow)) + let processingDeadline = Date().addingTimeInterval(processingSlice) let tree = try self.runMainThreadWork( "tree_processing", - timeout: min(self.treeCaptureSliceBudget, max(0.5, deadline.timeIntervalSinceNow)), + timeout: processingSlice, timeoutError: self.snapshotMainThreadTimeoutError("processing tree snapshot") ) { hint.isRaw - ? try self.rawTreeSnapshotAcquisition(context: context, hint: hint) - : try self.recursiveTreeSnapshotAcquisition(context: context, hint: hint) + ? (try self.rawTreeSnapshotAcquisition(context: context, hint: hint), .completed) + : try self.recursiveTreeSnapshotAcquisition( + context: context, + hint: hint, + processingDeadline: processingDeadline + ) } - return (tree, .completed) + return (tree.acquisition, tree.outcome) case .querySweep: let sliceDeadline = Self.querySweepSliceDeadline(startedAt: Date(), planDeadline: deadline) let sweep = try self.runMainThreadWork( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotAcquisitionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotAcquisitionTests.swift new file mode 100644 index 0000000000..a4db44a49c --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotAcquisitionTests.swift @@ -0,0 +1,68 @@ +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +import XCTest +import AgentDeviceSnapshotPresentation + +/// An element's attributes as XCTest hands them to a query predicate. +@MainActor +private final class ElementAttributesFixture: NSObject, XCUIElementSnapshot { + let elementType: XCUIElement.ElementType + let frame: CGRect + let label: String + + init(_ elementType: XCUIElement.ElementType, _ frame: CGRect, label: String = "") { + self.elementType = elementType + self.frame = frame + self.label = label + } + + var identifier: String { "" } + var value: Any? { nil } + var title: String { "" } + var isEnabled: Bool { true } + var horizontalSizeClass: XCUIElement.SizeClass { .unspecified } + var verticalSizeClass: XCUIElement.SizeClass { .unspecified } + var placeholderValue: String? { nil } + var isSelected: Bool { false } +#if !os(macOS) + var hasFocus: Bool { false } +#endif + var children: [XCUIElementSnapshot] { [] } + var dictionaryRepresentation: [XCUIElement.AttributeName: Any] { [:] } +} + +extension RunnerTests { + private static let containerFrame = CGRect(x: 20, y: 800, width: 360, height: 31) + + @MainActor + func testCollapsedTabCandidatePredicateMatchesLabeledCandidatesCenteredInAContainer() { + var matched: [String] = [] + let predicate = Self.collapsedTabCandidatePredicate(containerFrames: [ + CGRect(x: 0, y: 0, width: 400, height: 50), + Self.containerFrame, + ]) { matched.append($0.label) } + + XCTAssertTrue(predicate.evaluate(with: ElementAttributesFixture(.button, CGRect(x: 30, y: 800, width: 100, height: 30), label: "Left"))) + XCTAssertTrue(predicate.evaluate(with: ElementAttributesFixture(.staticText, CGRect(x: 10, y: 10, width: 80, height: 20), label: "Home"))) + XCTAssertEqual(matched, ["Left", "Home"], "every accepted element's attributes reach the caller") + } + + @MainActor + func testCollapsedTabCandidatePredicateRefusesTheContainerOtherTypesAndOutsiders() { + // Closest negatives, each labeled unless it tests content: the container's own frame and a + // wrapper centered on it, both too large; an unlabeled view over it; a non-candidate type over + // it; and a candidate overlapping it with its center outside. + var matched: [String] = [] + let predicate = Self.collapsedTabCandidatePredicate(containerFrames: [Self.containerFrame]) { + matched.append($0.label) + } + let item = CGRect(x: 30, y: 800, width: 100, height: 30) + + XCTAssertFalse(predicate.evaluate(with: ElementAttributesFixture(.other, Self.containerFrame, label: "Volume"))) + XCTAssertFalse(predicate.evaluate(with: ElementAttributesFixture(.other, CGRect(x: 0, y: 515, width: 400, height: 600), label: "Sheet"))) + XCTAssertFalse(predicate.evaluate(with: ElementAttributesFixture(.other, item, label: " "))) + XCTAssertFalse(predicate.evaluate(with: ElementAttributesFixture(.image, item, label: "Left"))) + XCTAssertFalse(predicate.evaluate(with: ElementAttributesFixture(.button, CGRect(x: 30, y: 780, width: 100, height: 30), label: "Left"))) + XCTAssertEqual(matched, [], "a refused element's attributes are not recorded") + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift index f3e9c5c234..f181db2ae8 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift @@ -70,6 +70,37 @@ private final class RunnerSlowSweepQueryStub: NSObject { } } +/// Per-run state for `RunnerSlowRootSnapshotStub`: the swizzled IMP cannot capture the original +/// implementation or a test-local flag. +private enum RunnerSlowRootSnapshotGate { + /// Longer than half the plan deadline the test passes, so whatever slice is left after the root + /// read is shorter than the read itself, while the real read still has most of `delay` to finish. + static let delay: TimeInterval = 3 + static var original: IMP? + static var delayed = false +} + +/// Stands in for `-[XCUIElement snapshotWithError:]` so the capture's root read, the plan's first +/// tree snapshot, takes at least `RunnerSlowRootSnapshotGate.delay` and then reads the real tree. +private final class RunnerSlowRootSnapshotStub: NSObject { + @objc(snapshotWithError:) + func snapshot() throws -> XCUIElementSnapshot { + if !RunnerSlowRootSnapshotGate.delayed { + RunnerSlowRootSnapshotGate.delayed = true + Thread.sleep(forTimeInterval: RunnerSlowRootSnapshotGate.delay) + } + typealias Read = @convention(c) ( + AnyObject, Selector, AutoreleasingUnsafeMutablePointer? + ) -> XCUIElementSnapshot? + let read = unsafeBitCast(RunnerSlowRootSnapshotGate.original, to: Read.self) + var error: NSError? + guard let snapshot = read(self, #selector(XCUIElement.snapshot), &error) else { + throw error ?? NSError(domain: "AgentDeviceRunner.tests", code: 2) + } + return snapshot + } +} + extension RunnerTests { /// The Bluesky feed shape: the tree XPC grinds past its slice. The plan must recover through /// private AX without queueing the query sweep behind the abandoned XPC, and a fresh process's @@ -445,6 +476,60 @@ extension RunnerTests { XCTAssertEqual(quality.reasonCode, "budget") } + /// A childless tab bar makes the tree tier run the collapsed tab query, and a slow root read leaves too + /// little of the processing slice for it to start. The plan must treat that tree as a tier that + /// ran out of time and move on, not accept the truncated tree as a healthy capture. + func testPlanMovesPastATreeTierWhoseCollapsedTabQueryCannotStart() throws { + guard + let snapshotMethod = class_getInstanceMethod( + XCUIApplication.self, + #selector(XCUIElement.snapshot) + ), + let stubMethod = class_getInstanceMethod( + RunnerSlowRootSnapshotStub.self, + #selector(RunnerSlowRootSnapshotStub.snapshot) + ) + else { + XCTFail("unable to install the slow root snapshot stub") + return + } + app.launchArguments = ["--agent-device-crowded-screen", "--agent-device-collapsed-tab-bar"] + app.launch() + XCTAssertTrue(app.staticTexts["agent-device-crowded-row-499"].waitForExistence(timeout: appExistenceTimeout)) + let captureTarget = MainActor.assumeIsolated { + mainOwned.app = app + mainOwned.bundleId = "com.callstack.agentdevice.runner.collapsed-tab-query-test" + return takeSnapshotCaptureTarget(app: app) + } + snapshotXCTestPenaltyWarmupExemption.isPending = false + clearSnapshotXCTestChannelPenalty(reason: "test-setup") + RunnerSlowRootSnapshotGate.delayed = false + RunnerSlowRootSnapshotGate.original = method_setImplementation( + snapshotMethod, + method_getImplementation(stubMethod) + ) + defer { + if let original = RunnerSlowRootSnapshotGate.original { + method_setImplementation(snapshotMethod, original) + } + clearSnapshotXCTestChannelPenalty(reason: "test-cleanup") + MainActor.assumeIsolated { + invalidateCachedTarget(reason: "unit_test_cleanup") + } + app.terminate() + } + + let payload = try runDeferredPlanOffMain( + target: captureTarget, + deadline: Date().addingTimeInterval(2 * RunnerSlowRootSnapshotGate.delay - 0.2) + ) + + XCTAssertTrue(RunnerSlowRootSnapshotGate.delayed, "the plan read the root tree through the stub") + let quality = try XCTUnwrap(payload.snapshotQuality) + XCTAssertEqual(quality.reasonCode, "budget", "the tree tier ran out of its processing slice") + XCTAssertNotEqual(quality.state, .healthy, "a tree without its collapsed tab items is not healthy") + } + private func launchPenalizedPrivateAXBlindTarget(bundleId: String) throws -> SnapshotCaptureTarget { app.launchArguments = ["--agent-device-selector-read-regression"] app.launch() @@ -475,6 +560,7 @@ extension RunnerTests { /// returns its payload; `afterPlan` runs on that queue once the plan has answered. private func runDeferredPlanOffMain( target: SnapshotCaptureTarget, + deadline: Date = Date().addingTimeInterval(20), afterPlan: @escaping () -> Void = {} ) throws -> DataPayload { final class ResultBox { @@ -490,7 +576,7 @@ extension RunnerTests { target: target, options: PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false), terminal: .sparseWithFatalOnAXFailure, - deadline: Date().addingTimeInterval(20) + deadline: deadline ) } catch { box.error = error diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift index 93cb5ee709..5e3323cb29 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTests.swift @@ -190,12 +190,22 @@ extension RunnerTests { XCTAssertLessThanOrEqual(clock, sliceDeadline, "no started query may outlive the tier slice") XCTAssertTrue( - Self.querySweepCanStartQuery(deadline: sliceDeadline, now: sliceDeadline.addingTimeInterval(-0.2)) + Self.xctestQueryCanStart(deadline: sliceDeadline, now: sliceDeadline.addingTimeInterval(-0.2)) ) XCTAssertFalse( - Self.querySweepCanStartQuery(deadline: sliceDeadline, now: sliceDeadline.addingTimeInterval(-0.05)) + Self.xctestQueryCanStart(deadline: sliceDeadline, now: sliceDeadline.addingTimeInterval(-0.05)) + ) + XCTAssertFalse(Self.xctestQueryCanStart(deadline: sliceDeadline, now: sliceDeadline)) + } + + func testCollapsedTabQueryStartsOnlyWhileTheSliceCoversItsExpectedCost() { + let now = Date(timeIntervalSinceReferenceDate: 1_000) + + XCTAssertTrue(Self.xctestQueryCanStart(deadline: now.addingTimeInterval(2), now: now, expectedCost: 2)) + XCTAssertFalse( + Self.xctestQueryCanStart(deadline: now.addingTimeInterval(1.9), now: now, expectedCost: 2), + "a query expected to outlast the slice must not start" ) - XCTAssertFalse(Self.querySweepCanStartQuery(deadline: sliceDeadline, now: sliceDeadline)) } func testQuerySweepRunsEveryQueryThatFitsAndStopsOnAXUnavailable() { @@ -362,6 +372,75 @@ extension RunnerTests { try self.snapshotRaw(target: target, options: options) } } + + @MainActor + func testCrowdedScreenWithAWideSliderPreservesItsSiblingButtons() throws { + app.launchArguments = ["--agent-device-crowded-screen", "--agent-device-wide-slider"] + app.launch() + defer { + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + XCTAssertTrue(app.staticTexts["agent-device-crowded-row-499"].waitForExistence(timeout: appExistenceTimeout)) + let hint = CaptureHint( + projection: .regular, + depth: nil, + regularPresentedDepth: nil, + interactiveOnly: false, + customActions: false + ) + let context = try XCTUnwrap(makeSnapshotTraversalContext(app: app, hint: hint)) + let startedAt = Date() + let capture = try recursiveTreeSnapshotAcquisition( + context: context, + hint: hint, + processingDeadline: startedAt.addingTimeInterval(treeCaptureSliceBudget) + ) + let elapsed = Date().timeIntervalSince(startedAt) + let nodes = capture.acquisition.nodes + XCTAssertEqual(capture.outcome, .completed) + XCTAssertFalse(capture.acquisition.truncated) + + XCTAssertLessThan(elapsed, treeCaptureSliceBudget, "the tree walk must fit its processing slice") + let slider = try XCTUnwrap(nodes.first { $0.type == "Slider" && $0.label == "Volume" }) + XCTAssertTrue(nodes.filter { $0.parentIndex == slider.index }.isEmpty) + for label in ["Left", "Right"] { + let buttons = nodes.filter { $0.type == "Button" && $0.label == label } + XCTAssertEqual(buttons.count, 1, "each sibling button appears once") + XCTAssertEqual(buttons.first?.parentIndex, slider.parentIndex) + } + } + + @MainActor + func testCollapsedTabQueryReportsWhenItsProcessingSliceCannotStartIt() throws { + app.launchArguments = ["--agent-device-crowded-screen", "--agent-device-collapsed-tab-bar"] + app.launch() + defer { + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + XCTAssertTrue(app.staticTexts["agent-device-crowded-row-499"].waitForExistence(timeout: appExistenceTimeout)) + let hint = CaptureHint( + projection: .regular, + depth: nil, + regularPresentedDepth: nil, + interactiveOnly: false, + customActions: false + ) + let context = try XCTUnwrap(makeSnapshotTraversalContext(app: app, hint: hint)) + let tabBar = try XCTUnwrap(Self.snapshotDescendants(of: context.rootSnapshot).first { $0.elementType == .tabBar }) + XCTAssertTrue(tabBar.children.isEmpty, "the fixture must exercise collapsed-tab recovery") + let unstarted = try recursiveTreeSnapshotAcquisition(context: context, hint: hint, processingDeadline: Date()) + XCTAssertEqual(unstarted.outcome, .deadlineExhausted) + XCTAssertTrue(unstarted.acquisition.truncated) + let capture = try recursiveTreeSnapshotAcquisition( + context: context, + hint: hint, + processingDeadline: Date().addingTimeInterval(treeCaptureSliceBudget) + ) + XCTAssertEqual(capture.outcome, .completed) + XCTAssertFalse(capture.acquisition.truncated) + } #endif } #endif