diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift index df933a24ab..d212491567 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift @@ -132,7 +132,7 @@ extension RunnerTests { privateAXAcceptedDepthBundleId = bundleId privateAXAcceptedDepthProcessIdentifier = processIdentifier privateAXAcceptedDepth = depth - privateAXAcceptedDepthUntil = Date().addingTimeInterval(snapshotXCTestChannelPenaltyDuration) + privateAXAcceptedDepthUntil = Date().addingTimeInterval(SnapshotXCTestChannelPenalty.duration) privateAXAcceptedDepthLock.unlock() NSLog("AGENT_DEVICE_RUNNER_PRIVATE_AX_DEPTH_REMEMBERED depth=%ld bundle=%@", depth, bundleId ?? "") } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index a0be484d79..da71bebe7b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -47,6 +47,45 @@ enum SnapshotXCTestChannelPlanState: Equatable { case boundedXCTestProbe } +/// The XCTest accessibility channel penalty (#1105/#1156). While it is active, regular plans for its +/// bundle defer XCTest tiers. Each recovery interval, one deferred plan runs the bounded tree probe +/// first, inside that capture, so a screen that is light again can clear the penalty before it +/// expires (#3844). +struct SnapshotXCTestChannelPenalty: Equatable { + static let duration: TimeInterval = 120 + static let recoveryProbeInterval: TimeInterval = 30 + + private(set) var bundleId: String? + private(set) var until = Date.distantPast + private(set) var probeDueAt = Date.distantFuture + + /// A penalty armed without a bundle id applies to whatever target is current. + func isActive(for bundleId: String?, at now: Date) -> Bool { + guard now < until else { return false } + guard let penalized = self.bundleId else { return true } + return penalized == bundleId + } + + func isProbeDue(for bundleId: String?, at now: Date) -> Bool { + isActive(for: bundleId, at: now) && now >= probeDueAt + } + + mutating func arm(bundleId: String?, at now: Date) { + self.bundleId = bundleId + until = now.addingTimeInterval(Self.duration) + probeDueAt = now.addingTimeInterval(Self.recoveryProbeInterval) + } + + /// A started probe moves the next one a full interval out, whatever the probe finds. + mutating func claimProbe(at now: Date) { + probeDueAt = now.addingTimeInterval(Self.recoveryProbeInterval) + } + + mutating func clear() { + self = SnapshotXCTestChannelPenalty() + } +} + /// How one tier's bounded work ended. `deadlineExhausted` is a tier timeout: the tier stopped /// starting work it could not finish inside its own slice, so what it returns is a partial result /// it never completed collecting. The plan keeps that result only as the fallback and lets the next @@ -64,6 +103,12 @@ struct EffectiveSnapshotCapturePlan { /// Non-nil only when the plan was narrowed by an explicit internal backend preference. This /// keeps the quality marker tied to the plan decision rather than to an untrusted request field. let preferredBackend: SnapshotBackendKind? + + /// A deferred plan that starts with an XCTest tier is running the due recovery probe. + var leadsWithXCTestRecoveryProbe: Bool { + xCTestChannelState == .deferredToIndependentBackend + && plan.first?.usesXCTestAccessibilityChannel == true + } } /// What the plan runner does when every backend failed or stayed sparse. @@ -144,8 +189,7 @@ extension RunnerTests { func penalizeSnapshotXCTestChannel(bundleId: String?, reason: String) { snapshotXCTestChannelPenaltyLock.lock() - snapshotXCTestChannelPenaltyBundleId = bundleId - snapshotXCTestChannelPenaltyUntil = Date().addingTimeInterval(snapshotXCTestChannelPenaltyDuration) + snapshotXCTestChannelPenalty.arm(bundleId: bundleId, at: Date()) snapshotXCTestChannelPenaltyLock.unlock() NSLog( "AGENT_DEVICE_RUNNER_SNAPSHOT_XCTEST_CHANNEL_PENALIZED bundle=%@ reason=%@", @@ -156,9 +200,8 @@ extension RunnerTests { func clearSnapshotXCTestChannelPenalty(reason: String) { snapshotXCTestChannelPenaltyLock.lock() - let hadActivePenalty = Date() < snapshotXCTestChannelPenaltyUntil - snapshotXCTestChannelPenaltyBundleId = nil - snapshotXCTestChannelPenaltyUntil = Date.distantPast + let hadActivePenalty = Date() < snapshotXCTestChannelPenalty.until + snapshotXCTestChannelPenalty.clear() snapshotXCTestChannelPenaltyLock.unlock() if hadActivePenalty { NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_XCTEST_CHANNEL_PENALTY_CLEARED reason=%@", reason) @@ -168,10 +211,20 @@ extension RunnerTests { func isSnapshotXCTestChannelPenalized(bundleId: String?) -> Bool { snapshotXCTestChannelPenaltyLock.lock() defer { snapshotXCTestChannelPenaltyLock.unlock() } - guard Date() < snapshotXCTestChannelPenaltyUntil else { return false } - // A penalty recorded without a bundle id applies to whatever target is current. - guard let penalized = snapshotXCTestChannelPenaltyBundleId else { return true } - return penalized == bundleId + return snapshotXCTestChannelPenalty.isActive(for: bundleId, at: Date()) + } + + func isSnapshotXCTestRecoveryProbeDue(bundleId: String?) -> Bool { + snapshotXCTestChannelPenaltyLock.lock() + defer { snapshotXCTestChannelPenaltyLock.unlock() } + return snapshotXCTestChannelPenalty.isProbeDue(for: bundleId, at: Date()) + } + + func claimSnapshotXCTestRecoveryProbe(bundleId: String?) { + snapshotXCTestChannelPenaltyLock.lock() + snapshotXCTestChannelPenalty.claimProbe(at: Date()) + snapshotXCTestChannelPenaltyLock.unlock() + NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_XCTEST_RECOVERY_PROBE bundle=%@", bundleId ?? "") } /// The pre-seeded first-failure a penalized plan stamps into its verdict. The deferred case @@ -244,11 +297,13 @@ extension RunnerTests { /// Pure plan-reorder rule: an internal preferred backend pins the regular plan to that backend; /// this is the only force seam used by same-backend evidence and conformance captures. A /// penalized XCTest accessibility channel uses independent backends when the platform has one, - /// otherwise it keeps XCTest work on a short probe. The raw diagnostic plan keeps tree-first - /// errors, and unknown plans are left untouched. + /// otherwise it keeps XCTest work on a short probe. A due recovery probe puts that short tree + /// ahead of the independent backend. The raw diagnostic plan keeps tree-first errors, and unknown + /// plans are left untouched. static func effectiveSnapshotCapturePlan( _ plan: [SnapshotBackendKind], xCTestChannelPenalized: Bool, + xCTestRecoveryProbeDue: Bool = false, availableBackends: Set = Set(SnapshotBackendKind.allCases), preferredBackend: String? = nil ) -> EffectiveSnapshotCapturePlan { @@ -278,10 +333,12 @@ extension RunnerTests { if !recoveryPlan.isEmpty { // The independent backend reads only the app it can match as the active AX application, so // an out-of-process surface over the app (the Save Password sheet) leaves it empty for the - // whole penalty. The tree, on the bounded probe's short slice, stays behind it; the query - // sweep does not, since its grind is what the penalty keeps off the main thread. + // whole penalty. The tree, on the bounded probe's short slice, stays behind it, or leads when + // the recovery probe is due; the query sweep does not, since its grind is what the penalty + // keeps off the main thread. + let boundedTree = availablePlan.filter { $0 == .recursiveTree } return EffectiveSnapshotCapturePlan( - plan: recoveryPlan + availablePlan.filter { $0 == .recursiveTree }, + plan: xCTestRecoveryProbeDue ? boundedTree + recoveryPlan : recoveryPlan + boundedTree, xCTestChannelState: .deferredToIndependentBackend, treeCaptureSliceBudgetOverride: Self.penalizedXCTestProbeTreeSliceBudget, preferredBackend: nil @@ -324,8 +381,10 @@ extension RunnerTests { // deliberately, so no degradation warning should render for it. var xCTestChannelPenalized = false var xCTestChannelPenalizedByBreaker = false + var xCTestRecoveryProbeDue = false #if os(iOS) xCTestChannelPenalizedByBreaker = isSnapshotXCTestChannelPenalized(bundleId: target.bundleId) + xCTestRecoveryProbeDue = isSnapshotXCTestRecoveryProbeDue(bundleId: target.bundleId) xCTestChannelPenalized = Self.snapshotXCTestChannelTreatedAsPenalized( penalized: xCTestChannelPenalizedByBreaker, preferredBackend: options.preferredBackend @@ -334,6 +393,7 @@ extension RunnerTests { let effective = Self.effectiveSnapshotCapturePlan( plan, xCTestChannelPenalized: xCTestChannelPenalized, + xCTestRecoveryProbeDue: xCTestRecoveryProbeDue, availableBackends: Set(SnapshotBackendKind.allCases.filter(\.isAvailableOnCurrentPlatform)), preferredBackend: options.preferredBackend ) @@ -374,6 +434,9 @@ extension RunnerTests { } continue } + if kind == effectivePlan.first && effective.leadsWithXCTestRecoveryProbe { + claimSnapshotXCTestRecoveryProbe(bundleId: target.bundleId) + } if kind.usesXCTestAccessibilityChannel { xCTestTierRan = true } let attempt = try captureWithBackend( kind, @@ -386,7 +449,8 @@ extension RunnerTests { kind, attempt: attempt, bundleId: target.bundleId, - penaltySuppressed: suppressXCTestPenalty + penaltySuppressed: suppressXCTestPenalty, + channelState: effective.xCTestChannelState ) if case let .failed(failure, phase: _) = attempt.outcome { if Self.isAxSnapshotFailure(failure) { axFailure = failure } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift index 3a3cfa03a6..86b8454926 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift @@ -119,9 +119,10 @@ extension RunnerTests { static func snapshotXCTestPenaltyReason( kind: SnapshotBackendKind, attempt: SnapshotBackendAttempt, - slowThresholdMs: Double + slowThresholdMs: Double, + channelState: SnapshotXCTestChannelPlanState = .normal ) -> String? { - guard kind.usesXCTestAccessibilityChannel else { return nil } + guard channelState == .normal, kind.usesXCTestAccessibilityChannel else { return nil } if case let .failed(failure, phase: .acquisition) = attempt.outcome, failure.code == Self.xCTestSnapshotTimeoutCode { @@ -134,18 +135,46 @@ extension RunnerTests { return "slow_\(kind.rawValue)_capture_\(Int(attempt.timing.acquisitionMs))ms" } + /// Only an accepted, fast tree proves that a bounded recovery probe restored the channel. + static func snapshotXCTestRecoveryProbeSucceeded( + kind: SnapshotBackendKind, + attempt: SnapshotBackendAttempt, + channelState: SnapshotXCTestChannelPlanState + ) -> Bool { + guard channelState != .normal, + kind == .recursiveTree, + attempt.timing.acquisitionMs <= Self.penalizedXCTestProbeTreeSliceBudget * 1_000, + case let .captured(capture) = attempt.outcome + else { return false } + return Self.snapshotTierRejectionReason( + outcome: attempt.tierOutcome, + kind: kind, + payload: capture.qualityPayload ?? capture.payload + ) == nil + } + func recordXCTestSnapshotBackendAttemptIfNeeded( _ kind: SnapshotBackendKind, attempt: SnapshotBackendAttempt, bundleId: String?, - penaltySuppressed: Bool + penaltySuppressed: Bool, + channelState: SnapshotXCTestChannelPlanState ) { guard !penaltySuppressed else { return } + if Self.snapshotXCTestRecoveryProbeSucceeded( + kind: kind, + attempt: attempt, + channelState: channelState + ) { + clearSnapshotXCTestChannelPenalty(reason: "healthy_tree_recovery_probe") + return + } guard let reason = Self.snapshotXCTestPenaltyReason( kind: kind, attempt: attempt, - slowThresholdMs: snapshotXCTestSlowCaptureThreshold * 1_000 + slowThresholdMs: snapshotXCTestSlowCaptureThreshold * 1_000, + channelState: channelState ) else { return } penalizeSnapshotXCTestChannel( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index b334d7301d..4c7d5516ba 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -125,9 +125,7 @@ final class RunnerTests: XCTestCase { // channel when an independent recovery backend exists, or use a bounded XCTest probe when it // does not, for the same screen class (#1105/#1156). let snapshotXCTestChannelPenaltyLock = NSLock() - var snapshotXCTestChannelPenaltyBundleId: String? - var snapshotXCTestChannelPenaltyUntil = Date.distantPast - let snapshotXCTestChannelPenaltyDuration: TimeInterval = 120 + var snapshotXCTestChannelPenalty = SnapshotXCTestChannelPenalty() let snapshotXCTestPenaltyWarmupExemption = SnapshotXCTestPenaltyWarmupExemption() // Sticky per-bundle hint for the private AX depth ladder: deep RN screens reject the default // depth with kAXErrorIllegalArgument on EVERY capture, so once a shallower rung is accepted diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift index 55f3c0416f..2e2cc45490 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanOccupancyTests.swift @@ -430,6 +430,14 @@ extension RunnerTests { XCTAssertEqual(quality.state, .recovered) XCTAssertEqual(quality.backend, SnapshotBackendKind.recursiveTree.rawValue) XCTAssertGreaterThan(payload.nodes?.count ?? 0, 1, "the bounded tree answers with a real tree") + let timing = try XCTUnwrap(quality.timing) + guard timing.acquisitionMs <= Self.penalizedXCTestProbeTreeSliceBudget * 1_000 else { + throw XCTSkip("the accepted tree's total acquisition exceeded the fast-probe threshold on this host") + } + XCTAssertFalse( + isSnapshotXCTestChannelPenalized(bundleId: captureTarget.bundleId), + "an accepted fast tree probe must clear the existing penalty early" + ) } /// When the bounded tree also grinds past its slice, the plan ends sparse, and the verdict still @@ -465,9 +473,15 @@ extension RunnerTests { tearDownPenalizedPrivateAXBlindTarget() } + let originalPenaltyUntil = snapshotXCTestChannelPenalty.until let payload = try runDeferredPlanOffMain(target: captureTarget) { RunnerBlockingSnapshotGate.release.signal() } + XCTAssertEqual( + snapshotXCTestChannelPenalty.until, + originalPenaltyUntil, + "the bounded recovery probe must not renew its own penalty when its tree work is abandoned" + ) let quality = try XCTUnwrap(payload.snapshotQuality) XCTAssertEqual(quality.state, .sparse) @@ -537,6 +551,64 @@ extension RunnerTests { XCTAssertNotEqual(quality.state, .healthy, "a tree without its collapsed tab items is not healthy") } + /// A due recovery probe runs inside the capture, ahead of private AX. Its abandoned tree work is + /// charged to that capture: the plan answers while the work is outstanding, so that response's + /// occupancy stamp reports it. The probe cannot renew the penalty, and the next plan inside the + /// interval defers without entering the tree. + func testDueRecoveryProbeRunsInsideTheCaptureAndChargesItsAbandonedWorkToIt() throws { + let captureTarget = try launchPenalizedPrivateAXBlindTarget( + bundleId: "com.callstack.agentdevice.runner.due-probe-test" + ) + defer { tearDownPenalizedPrivateAXBlindTarget() } + let options = PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false) + let acquisition = try XCTUnwrap(privateAXSnapshotAcquisition( + target: captureTarget, hint: SnapshotPresentation.captureHint(for: options), + deadline: Date().addingTimeInterval(10) + )) + privateAXAcquisitionOverrideForTesting = { acquisition } + let snapshotMethod = try XCTUnwrap(class_getInstanceMethod(XCUIApplication.self, #selector(XCUIElement.snapshot))) + let stubMethod = try XCTUnwrap(class_getInstanceMethod(RunnerBlockingSnapshotStub.self, #selector(RunnerBlockingSnapshotStub.snapshot))) + let original = method_getImplementation(snapshotMethod) + RunnerBlockingSnapshotGate.release = DispatchSemaphore(value: 0) + RunnerBlockingSnapshotGate.entered = DispatchSemaphore(value: 0) + method_setImplementation(snapshotMethod, method_getImplementation(stubMethod)) + defer { + RunnerBlockingSnapshotGate.release.signal() + method_setImplementation(snapshotMethod, original) + waitForAbandonedMainThreadWorkToDrain() + } + snapshotXCTestChannelPenalty.claimProbe( + at: Date(timeIntervalSinceNow: -SnapshotXCTestChannelPenalty.recoveryProbeInterval) + ) + let expiry = snapshotXCTestChannelPenalty.until + final class Occupancy { var atAnswer = false } + let occupancy = Occupancy() + + let probed = try runDeferredPlanOffMain(target: captureTarget) { + occupancy.atAnswer = self.hasAbandonedMainThreadWork() + RunnerBlockingSnapshotGate.release.signal() + } + + XCTAssertEqual(RunnerBlockingSnapshotGate.entered.wait(timeout: .now()), .success, "the probe ran before the answer") + XCTAssertTrue(occupancy.atAnswer, "the capture that ran the probe answers as busy") + XCTAssertEqual(probed.snapshotQuality?.backend, SnapshotBackendKind.privateAX.rawValue) + XCTAssertEqual(probed.snapshotQuality?.reasonCode, "deferred") + XCTAssertEqual(snapshotXCTestChannelPenalty.until, expiry, "a failed probe must not renew the penalty") + XCTAssertFalse(isSnapshotXCTestRecoveryProbeDue(bundleId: captureTarget.bundleId)) + waitForAbandonedMainThreadWorkToDrain() + + let deferred = try runDeferredPlanOffMain(target: captureTarget) + XCTAssertEqual(deferred.snapshotQuality?.backend, SnapshotBackendKind.privateAX.rawValue) + XCTAssertEqual(RunnerBlockingSnapshotGate.entered.wait(timeout: .now()), .timedOut, "no probe inside the interval") + } + + private func waitForAbandonedMainThreadWorkToDrain() { + let deadline = Date().addingTimeInterval(3) + while hasAbandonedMainThreadWork(), Date() < deadline { + RunLoop.current.run(until: Date().addingTimeInterval(0.01)) + } + } + private func launchPenalizedPrivateAXBlindTarget(bundleId: String) throws -> SnapshotCaptureTarget { app.launchArguments = ["--agent-device-selector-read-regression"] app.launch() diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift index d8ce8065fb..4d88f3c9d7 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotCapturePlanTests.swift @@ -538,10 +538,7 @@ extension RunnerTests { } func testSnapshotXCTestChannelPenaltyMatchesBundleAndExpires() { - defer { - snapshotXCTestChannelPenaltyBundleId = nil - snapshotXCTestChannelPenaltyUntil = .distantPast - } + defer { snapshotXCTestChannelPenalty.clear() } penalizeSnapshotXCTestChannel(bundleId: "xyz.blueskyweb.app", reason: "test") XCTAssertTrue(isSnapshotXCTestChannelPenalized(bundleId: "xyz.blueskyweb.app")) @@ -552,7 +549,9 @@ extension RunnerTests { XCTAssertTrue(isSnapshotXCTestChannelPenalized(bundleId: "com.other.app")) // Expired penalties stop applying. - snapshotXCTestChannelPenaltyUntil = Date(timeIntervalSinceNow: -1) + snapshotXCTestChannelPenalty.arm( + bundleId: nil, at: Date(timeIntervalSinceNow: -SnapshotXCTestChannelPenalty.duration - 1) + ) XCTAssertFalse(isSnapshotXCTestChannelPenalized(bundleId: "com.other.app")) } @@ -612,5 +611,113 @@ extension RunnerTests { XCTAssertLessThanOrEqual(nodes.count, unscoped.count) } #endif + + func testOnlyFastAcceptedTreeProbeClearsPenalty() { + let payload = DataPayload(nodes: [ + planTestNode(index: 0, type: "Application", label: "App"), + planTestNode(index: 1, type: "Button", label: "Open", hittable: true, parentIndex: 0), + ], truncated: false) + let healthy = SnapshotBackendCapture(payload: payload, effectiveDepth: nil) + let fast = SnapshotBackendAttempt( + outcome: .captured(healthy), + timing: SnapshotCaptureTiming(acquisitionMs: 100, presentationMs: 10) + ) + for state in [SnapshotXCTestChannelPlanState.deferredToIndependentBackend, .boundedXCTestProbe] { + XCTAssertTrue(Self.snapshotXCTestRecoveryProbeSucceeded( + kind: .recursiveTree, attempt: fast, channelState: state + )) + for kind in [SnapshotBackendKind.querySweep, .privateAX] { + XCTAssertFalse(Self.snapshotXCTestRecoveryProbeSucceeded( + kind: kind, attempt: fast, channelState: state + )) + } + XCTAssertTrue(Self.snapshotXCTestRecoveryProbeSucceeded( + kind: .recursiveTree, + attempt: SnapshotBackendAttempt(outcome: .captured(healthy), timing: SnapshotCaptureTiming(acquisitionMs: 1_000, presentationMs: 10)), + channelState: state + )) + let rejected = [ + SnapshotBackendAttempt(outcome: .captured(healthy), timing: fast.timing, tierOutcome: .deadlineExhausted), + SnapshotBackendAttempt(outcome: .noCapture, timing: fast.timing), + SnapshotBackendAttempt(outcome: .captured(healthy), timing: SnapshotCaptureTiming(acquisitionMs: 1_001, presentationMs: 0)), + SnapshotBackendAttempt(outcome: .captured(SnapshotBackendCapture( + payload: payload, effectiveDepth: nil, qualityPayload: DataPayload(nodes: [], truncated: false) + )), timing: fast.timing), + ] + for attempt in rejected { + XCTAssertFalse(Self.snapshotXCTestRecoveryProbeSucceeded( + kind: .recursiveTree, attempt: attempt, channelState: state + )) + } + } + XCTAssertFalse(Self.snapshotXCTestRecoveryProbeSucceeded( + kind: .recursiveTree, attempt: fast, channelState: .normal + )) + } + + func testDueRecoveryProbeLeadsOnlyTheDeferredRegularPlanAndNeverAddsTheSweep() { + let due = Self.effectiveSnapshotCapturePlan( + Self.regularVisiblePlan, xCTestChannelPenalized: true, xCTestRecoveryProbeDue: true + ) + XCTAssertEqual(due.plan, [.recursiveTree, .privateAX]) + XCTAssertEqual(due.xCTestChannelState, .deferredToIndependentBackend) + XCTAssertEqual(due.treeCaptureSliceBudgetOverride, Self.penalizedXCTestProbeTreeSliceBudget) + XCTAssertTrue(due.leadsWithXCTestRecoveryProbe) + + let notDue = Self.effectiveSnapshotCapturePlan(Self.regularVisiblePlan, xCTestChannelPenalized: true) + XCTAssertEqual(notDue.plan, [.privateAX, .recursiveTree]) + XCTAssertFalse(notDue.leadsWithXCTestRecoveryProbe) + + let physicalDevice = Self.effectiveSnapshotCapturePlan( + Self.regularVisiblePlan, xCTestChannelPenalized: true, xCTestRecoveryProbeDue: true, + availableBackends: [.recursiveTree, .querySweep] + ) + XCTAssertEqual(physicalDevice.plan, [.recursiveTree, .querySweep]) + XCTAssertFalse(physicalDevice.leadsWithXCTestRecoveryProbe, "the bounded plan probes on every capture") + for pinned in ["private-ax", "tree"] { + let plan = Self.effectiveSnapshotCapturePlan( + Self.regularVisiblePlan, xCTestChannelPenalized: true, xCTestRecoveryProbeDue: true, + preferredBackend: pinned + ) + XCTAssertEqual(plan.plan.count, 1) + XCTAssertFalse(plan.leadsWithXCTestRecoveryProbe) + } + let raw = Self.effectiveSnapshotCapturePlan( + Self.rawDiagnosticPlan, xCTestChannelPenalized: true, xCTestRecoveryProbeDue: true + ) + XCTAssertEqual(raw.plan, Self.rawDiagnosticPlan) + XCTAssertFalse(raw.leadsWithXCTestRecoveryProbe) + } + + func testChannelPenaltyProbeIsDueEachIntervalUntilItExpiresOrClears() { + let armedAt = Date(timeIntervalSinceReferenceDate: 1_000) + var penalty = SnapshotXCTestChannelPenalty() + XCTAssertFalse(penalty.isActive(for: "app", at: armedAt)) + XCTAssertFalse(penalty.isProbeDue(for: "app", at: armedAt)) + + penalty.arm(bundleId: "app", at: armedAt) + XCTAssertTrue(penalty.isActive(for: "app", at: armedAt)) + XCTAssertFalse(penalty.isActive(for: "other", at: armedAt)) + XCTAssertFalse(penalty.isProbeDue(for: "app", at: armedAt.addingTimeInterval(29.9))) + XCTAssertTrue(penalty.isProbeDue(for: "app", at: armedAt.addingTimeInterval(30))) + XCTAssertFalse(penalty.isProbeDue(for: "other", at: armedAt.addingTimeInterval(30))) + + let probedAt = armedAt.addingTimeInterval(45) + penalty.claimProbe(at: probedAt) + XCTAssertTrue(penalty.isActive(for: "app", at: probedAt)) + XCTAssertFalse(penalty.isProbeDue(for: "app", at: probedAt.addingTimeInterval(29.9))) + XCTAssertTrue(penalty.isProbeDue(for: "app", at: probedAt.addingTimeInterval(30))) + XCTAssertFalse( + penalty.isProbeDue(for: "app", at: armedAt.addingTimeInterval(SnapshotXCTestChannelPenalty.duration)), + "an expired penalty has no probe to run" + ) + + penalty.clear() + XCTAssertFalse(penalty.isActive(for: "app", at: probedAt)) + XCTAssertFalse(penalty.isProbeDue(for: "app", at: .distantFuture)) + + penalty.arm(bundleId: nil, at: armedAt) + XCTAssertTrue(penalty.isActive(for: "any", at: armedAt), "a bundle-less penalty applies to any target") + } } #endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTimingTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTimingTests.swift index 822f8dbab9..9a5033934b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTimingTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotTimingTests.swift @@ -122,5 +122,76 @@ extension RunnerTests { XCTAssertEqual(timer.timing.acquisitionMs, 2_000, accuracy: 0.001) XCTAssertEqual(timer.timing.presentationMs, 5_000, accuracy: 0.001) } + + func testNormalPlanSweepTimeoutPenalizesRegardlessOfItsShortSlice() { + let timeout = SnapshotCaptureFailure( + code: Self.xCTestSnapshotTimeoutCode, message: "same timeout text", hint: "test" + ) + for outcome in [SnapshotBackendAttempt.Outcome.failed(timeout, phase: .acquisition), .noCapture] { + let short = SnapshotBackendAttempt( + outcome: outcome, + timing: SnapshotCaptureTiming(acquisitionMs: 1_000, presentationMs: 10), + tierOutcome: .deadlineExhausted + ) + XCTAssertEqual(Self.snapshotXCTestPenaltyReason( + kind: .querySweep, attempt: short, slowThresholdMs: 3_000 + ), "queries_backend_timeout") + XCTAssertEqual(Self.snapshotXCTestPenaltyReason( + kind: .recursiveTree, attempt: short, slowThresholdMs: 3_000 + ), "tree_backend_timeout") + let slow = SnapshotBackendAttempt( + outcome: outcome, + timing: SnapshotCaptureTiming(acquisitionMs: 3_001, presentationMs: 10), + tierOutcome: .deadlineExhausted + ) + XCTAssertEqual(Self.snapshotXCTestPenaltyReason( + kind: .querySweep, attempt: slow, slowThresholdMs: 3_000 + ), "queries_backend_timeout") + } + let untyped = SnapshotBackendAttempt( + outcome: .failed(SnapshotCaptureFailure( + code: "OTHER_FAILURE", message: "same timeout text", hint: "test" + ), phase: .acquisition), + timing: SnapshotCaptureTiming(acquisitionMs: 100, presentationMs: 0) + ) + XCTAssertNil(Self.snapshotXCTestPenaltyReason( + kind: .recursiveTree, attempt: untyped, slowThresholdMs: 3_000 + )) + } + + func testBoundedRecoveryAttemptsCannotRenewPenaltyOnEitherPlatformPlan() { + let timeout = SnapshotCaptureFailure( + code: Self.xCTestSnapshotTimeoutCode, message: "test timeout", hint: "test" + ) + let attempts = [ + SnapshotBackendAttempt( + outcome: .failed(timeout, phase: .acquisition), + timing: SnapshotCaptureTiming(acquisitionMs: 1_000, presentationMs: 0) + ), + SnapshotBackendAttempt( + outcome: .noCapture, + timing: SnapshotCaptureTiming(acquisitionMs: 1_000, presentationMs: 0), + tierOutcome: .deadlineExhausted + ), + SnapshotBackendAttempt( + outcome: .noCapture, + timing: SnapshotCaptureTiming(acquisitionMs: 3_001, presentationMs: 0) + ), + ] + for state in [SnapshotXCTestChannelPlanState.deferredToIndependentBackend, .boundedXCTestProbe] { + for kind in [SnapshotBackendKind.recursiveTree, .querySweep] { + for attempt in attempts { + XCTAssertNil(Self.snapshotXCTestPenaltyReason( + kind: kind, attempt: attempt, slowThresholdMs: 3_000, channelState: state + )) + } + } + } + for attempt in attempts { + XCTAssertNotNil(Self.snapshotXCTestPenaltyReason( + kind: .recursiveTree, attempt: attempt, slowThresholdMs: 3_000 + ), "ordinary tree timeouts and slow acquisitions must still penalize") + } + } } #endif diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index b7b20790e6..0d7d1c78c3 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -234,13 +234,29 @@ On heavy or animating screens, the recursive tree capture can work for many seco This pushes the chained plan past the runner's main-thread watchdog and fills the main queue with retries. The plan now carries its umbrella deadline into the query-sweep and private-AX tiers. When the budget is spent, later ladder rungs stop. -A slow, timed-out, or watchdog-abandoned XCTest-backed capture penalizes the XCTest accessibility channel for that bundle for a bounded window. +A slow XCTest acquisition, a timed-out XCTest tier, or watchdog-abandoned system-modal work penalizes the XCTest accessibility channel for that bundle for a bounded window. Subsequent regular plans derive the next step from backend traits (`effectiveSnapshotCapturePlan`). If a runnable non-XCTest backend exists, they defer to that independent tier. If it does not exist, as on physical iOS devices today, they run a short XCTest probe instead of the full tree slice. Thus, healthy screens can recover without another long capture of the hostile screen. The raw diagnostic plan is exempt and keeps tree-first error propagation. +Issue #3844 showed that the bounded recovery probe could renew its own penalty. +A bounded recovery probe now leaves the penalty deadline unchanged when it fails. +Its short slice tests recovery under a known penalty, so its timeout supplies no new evidence about the channel. +A completed tree probe clears the penalty early only when its acquisition fits the probe slice and its quality is accepted. +Thirty seconds after the penalty is armed, and thirty seconds after each probe starts, a recovery probe is due. +A regular plan with a due probe runs the bounded tree before the independent backend, in the same capture. +The runner starts no XCTest main-thread work after it answers a command. +Probe work that outlives its slice is charged to the capture that started it, so that response reports the main thread as busy. +Physical-device plans probe on every capture already. Explicit backend pins and the raw plan keep their order. +After the iPad amendment below, a Simulator capture meets this policy only when the bridge sends it to XCTest, for example for a windowed app or a presented system surface. +The interval limits hostile-screen probe cost while giving healthy screens a recovery opportunity before penalty expiry. +A normal-plan query-sweep timeout rejects the partial capture and penalizes the channel. +Its short slice is evidence of a hostile tree, and deferral prevents repeated slow sweeps on that tree. +A normal-plan XCTest acquisition above the existing three-second threshold also remains penalty evidence. +The plan deadline and abandoned-main-thread-work gate remain in force on every attempt, including physical-device probes (#1105, #1156, #2621). + A third shape followed on the same app class after the plan recovered reliably. On that app class, each of the query-sweep tier's 19 `allElementsBoundByIndex` reads fails with `kAXErrorIllegalArgument`. XCTest records every one as a test failure worded `Failed to resolve query: ...`.