From 7713e759fd9e63b1d6e7042e780e2b9e032fc8ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 15 Oct 2026 21:50:42 +0200 Subject: [PATCH 1/8] fix(ios): make alert dismiss pick the decline button, never an affirmative one chooseAlertButton answered 'dismiss' with the last button whenever no known decline label matched. The App Tracking Transparency prompt lists 'Allow' last and words its decline in a label the list never carried, so 'alert dismiss' reported 'dismissed' while iOS recorded an authorization (#3627). Move the accept/dismiss choice into RunnerAlertButtonChoice so the rule is unit-testable without a screen, and make dismiss answer only a button the accept rule does not claim: the known decline label when present, otherwise the last button no accept label matches, which covers declines worded in a language or phrasing the list does not carry. A prompt whose every button reads affirmative now fails with 'alert dismiss button not found' and taps nothing. Recognize 'Ask App Not to Track' as a decline label. The runner fixture app gains an ATT-shaped prompt with an answer canary so the iOS lane asserts which button the app really took. --- .../AgentDeviceRunner/AgentDeviceRunnerApp.m | 40 ++++++++ .../RunnerAlertButtonChoice.swift | 58 +++++++++++ .../RunnerTests+Alert.swift | 40 ++------ .../RunnerTests+AlertButtonChoiceTests.swift | 97 +++++++++++++++++++ .../UnitTests/RunnerTests+AlertTests.swift | 5 - website/docs/docs/commands.md | 1 + 6 files changed, 202 insertions(+), 39 deletions(-) create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertButtonChoice.swift create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m index 109c610949..baff9e82c6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m @@ -87,6 +87,7 @@ @interface AgentDeviceRunnerViewController : UIViewController @property(nonatomic, assign) NSUInteger maskedTogglePresses; @property(nonatomic, strong) UILabel *alertActionStatus; @property(nonatomic, strong) UILabel *alertActivationBusyAnswer; +@property(nonatomic, strong) UILabel *alertDeclineAnswer; @property(nonatomic, assign) NSUInteger firstAlertActions; @property(nonatomic, assign) NSUInteger replacementAlertActions; @property(nonatomic, strong) UILabel *textEntryWriteBackStatus; @@ -392,8 +393,34 @@ - (void)presentAlertFixtureReplacement:(BOOL)replacement { [self presentViewController:alert animated:NO completion:nil]; } +// The App Tracking Transparency prompt's button order, which is what `alert dismiss` got wrong: +// the affirmative button is last, so a fallback to it answers "Allow" (#3627). The answer label +// says which button the app really took, so a test does not trust the command's own report. +- (void)presentAlertDeclineChoiceFixture { + UIAlertController *alert = [UIAlertController + alertControllerWithTitle:@"“Runner” Would Like to Track Your Activity" + message:@"Track your activity across other companies' apps and websites." + preferredStyle:UIAlertControllerStyleAlert]; + NSArray *buttons = @[ + @[@"Ask App Not to Track", @"Declined"], + @[@"Allow", @"Authorized"], + ]; + for (NSArray *button in buttons) { + [alert addAction:[UIAlertAction actionWithTitle:button[0] + style:UIAlertActionStyleDefault + handler:^(UIAlertAction *action) { + (void)action; + self.alertDeclineAnswer.text = button[1]; + }]]; + } + [self presentViewController:alert animated:NO completion:nil]; +} + - (void)viewDidAppear:(BOOL)animated { [super viewDidAppear:animated]; + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-decline-choice"]) { + [self presentAlertDeclineChoiceFixture]; + } if (!self.alertFixtureStarted && [NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-replacement-regression"]) { self.alertFixtureStarted = YES; @@ -575,6 +602,19 @@ - (void)viewDidLoad { [self updateAlertActionStatus]; } + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-decline-choice"]) { + UILabel *declineAnswer = [[UILabel alloc] init]; + declineAnswer.text = @"Unanswered"; + declineAnswer.accessibilityIdentifier = @"agent-device-alert-decline-answer"; + declineAnswer.translatesAutoresizingMaskIntoConstraints = NO; + [self.view addSubview:declineAnswer]; + [NSLayoutConstraint activateConstraints:@[ + [declineAnswer.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor], + [declineAnswer.topAnchor constraintEqualToAnchor:label.bottomAnchor constant:24], + ]]; + self.alertDeclineAnswer = declineAnswer; + } + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-activation-busy"]) { UILabel *busyAnswer = [[UILabel alloc] init]; busyAnswer.text = @"Unanswered"; diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertButtonChoice.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertButtonChoice.swift new file mode 100644 index 0000000000..e95e8bd9d9 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertButtonChoice.swift @@ -0,0 +1,58 @@ +import Foundation + +/// Which of an alert's buttons an `accept` or `dismiss` activates, decided on the button labels +/// alone, so the rule is checkable without a screen. +/// +/// `dismiss` never answers a button `isAcceptButton` recognizes. iOS words the App Tracking +/// Transparency prompt's decline in every language differently, so no decline list carries it, +/// and a fallback to the last button answered "Allow" while reporting `dismissed` (#3627). With +/// no known decline label, the only candidate that could decline is a label the accept list does +/// not claim, so `dismiss` takes the last such button; when every button reads affirmative there +/// is no decline to name and the command fails without a tap. +enum RunnerAlertButtonChoice { + static func buttonIndex(for labels: [String], action: String) -> Int? { + if action == "accept" { + if let accept = labels.firstIndex(where: { isAcceptButton($0) }) { + return accept + } + // A lone button is the only way out of the prompt, so accept takes it unless it declines. + return labels.count == 1 && !isDismissButton(labels[0]) ? 0 : nil + } + if let decline = labels.firstIndex(where: { isDismissButton($0) }) { + return decline + } + return labels.lastIndex { !isAcceptButton($0) } + } + + static func isAcceptButton(_ label: String) -> Bool { + let normalized = normalize(label) + return [ + "ok", + "allow", + "yes", + "continue", + "done", + "open", + "open settings" + ].contains(normalized) || normalized.hasPrefix("confirm") + } + + static func isDismissButton(_ label: String) -> Bool { + [ + "cancel", + "close", + "dismiss", + "don't allow", + "don’t allow", + "not now", + "no", + "keep browsing", + "later", + "ask app not to track" + ].contains(normalize(label)) + } + + private static func normalize(_ label: String) -> String { + label.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + } +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift index fc5d205bbd..cfb78714e7 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift @@ -247,41 +247,13 @@ extension RunnerTests { private static let dismissPopupMarkerText = NSPredicate(format: "SELF MATCHES[c] %@", dismissPopupMarkerPattern) private func chooseAlertButton(_ buttons: [XCUIElement], action: String) -> XCUIElement? { - if action == "accept" { - if let accept = buttons.first(where: { isAcceptButton($0.label) }) { - return accept - } - return buttons.count == 1 && !isDismissButton(buttons[0].label) ? buttons[0] : nil + guard let index = RunnerAlertButtonChoice.buttonIndex( + for: buttons.map(\.label), + action: action + ) else { + return nil } - - return buttons.first(where: { isDismissButton($0.label) }) ?? buttons.last - } - - func isAcceptButton(_ label: String) -> Bool { - let normalized = label.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() - return [ - "ok", - "allow", - "yes", - "continue", - "done", - "open", - "open settings" - ].contains(normalized) || normalized.hasPrefix("confirm") - } - - private func isDismissButton(_ label: String) -> Bool { - [ - "cancel", - "close", - "dismiss", - "don't allow", - "don’t allow", - "not now", - "no", - "keep browsing", - "later" - ].contains(label.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()) + return buttons[index] } private func preferredAlertTitle(_ element: XCUIElement, buttons: [XCUIElement]) -> String { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift new file mode 100644 index 0000000000..66354d4016 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift @@ -0,0 +1,97 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +extension RunnerTests { + func testAlertAcceptTreatsOpenAsAffirmative() { + XCTAssertTrue(RunnerAlertButtonChoice.isAcceptButton("Open")) + } + + func testAlertDismissPicksTheTrackingPromptDeclineLabel() { + // The App Tracking Transparency prompt, as iOS words it in English (#3627). + XCTAssertEqual( + RunnerAlertButtonChoice.buttonIndex(for: ["Ask App Not to Track", "Allow"], action: "dismiss"), + 0 + ) + XCTAssertEqual( + RunnerAlertButtonChoice.buttonIndex(for: ["Ask App Not to Track", "Allow"], action: "accept"), + 1 + ) + } + + func testAlertDismissKeepsAnUnrecognizedDeclineLabelInsteadOfReachingAnAffirmativeOne() { + // A localization the dismiss list cannot carry: the label the accept list does not claim is + // the only candidate that could decline, so it wins over the trailing "Allow". + XCTAssertEqual( + RunnerAlertButtonChoice.buttonIndex(for: ["App nicht verfolgen lassen", "Allow"], action: "dismiss"), + 0 + ) + XCTAssertEqual( + RunnerAlertButtonChoice.buttonIndex(for: ["More info", "Allow", "OK"], action: "dismiss"), + 0 + ) + } + + func testAlertDismissRefusesAPromptWhoseEveryButtonIsAffirmative() { + for labels in [["Allow"], ["Allow", "OK"], ["Yes", "Continue", "Done"]] { + XCTAssertNil( + RunnerAlertButtonChoice.buttonIndex(for: labels, action: "dismiss"), + "dismiss must fail without a tap rather than authorize through \(labels)" + ) + } + } + + func testAlertDismissStillPicksTheKnownCancelLabelAndAcceptStillTakesOK() { + XCTAssertEqual( + RunnerAlertButtonChoice.buttonIndex(for: ["Cancel", "OK"], action: "dismiss"), + 0 + ) + XCTAssertEqual( + RunnerAlertButtonChoice.buttonIndex(for: ["Cancel", "OK"], action: "accept"), + 1 + ) + } + + func testAlertAcceptKeepsItsGuardsOnTheDeclineLabelsItMustNotTake() { + // A lone declining button is not an acceptance, and with a choice, accept takes the first + // label the accept list claims. + XCTAssertNil(RunnerAlertButtonChoice.buttonIndex(for: ["Ask App Not to Track"], action: "accept")) + XCTAssertNil(RunnerAlertButtonChoice.buttonIndex(for: ["Cancel"], action: "accept")) + XCTAssertEqual( + RunnerAlertButtonChoice.buttonIndex(for: ["Not now", "Confirm purchase"], action: "accept"), + 1 + ) + } +} +#endif + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) +extension RunnerTests { + /// The reporter's scenario end to end: `alert dismiss` on a prompt whose decline label the list + /// did not carry, and whose affirmative button was the last one. The command's own report was + /// `dismissed` either way, so the canary — which button the app actually answered — is the + /// assertion that matters (#3627). + @MainActor + func testAlertDismissAnswersTheTrackingPromptDeclineAndNeverAuthorizes() throws { + app.launchArguments = ["--agent-device-alert-decline-choice"] + app.launch() + defer { + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + XCTAssertTrue(app.alerts.firstMatch.waitForExistence(timeout: appExistenceTimeout)) + let alert = try XCTUnwrap( + resolveAlert(app: app, deadline: Date().addingTimeInterval(RunnerTests.defaultAlertCommandTimeout)) + ) + XCTAssertEqual(alert.buttons.map(\.label).last, "Allow", "the fixture keeps the affirmative button last"); + + let response = handleAlert( + alert, + action: "dismiss", + deadline: Date().addingTimeInterval(RunnerTests.alertActivationDeadline) + ) + + XCTAssertTrue(response.ok, String(describing: response.error)) + XCTAssertEqual(app.staticTexts["agent-device-alert-decline-answer"].label, "Declined"); + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift index 42f706b2c9..c8f3a24207 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift @@ -8,13 +8,8 @@ extension RunnerTests { XCTAssertEqual(app.state, .notRunning) XCTAssertNil(resolveAlert(app: app, deadline: Date().addingTimeInterval(1))) } - - func testAlertAcceptTreatsOpenAsAffirmative() { - XCTAssertTrue(isAcceptButton("Open")) - } } #endif - #if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) extension RunnerTests { @MainActor diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index aaf2069bda..8409323d50 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -387,6 +387,7 @@ agent-device alert dismiss - `alert` (same as `alert get`) inspects or handles system alerts on iOS simulator, macOS desktop, and Android permission dialogs. On iOS it also covers app-owned modal popups with native blocking markers. - Use `alert get` for a quick check, and `alert wait ` only when a prompt may appear after async work. Don't use `settings permission` to answer a dialog already on screen; it is for setup before a flow. - `accept` and `dismiss` are sent once on every platform and never replayed, even if the response is lost. On iOS the runner activates the button once, then only observes until the alert disappears, changes, or the deadline expires. A changed alert does not prove a permission was granted; verify the app outcome. +- On iOS, `dismiss` never activates a button the accept labels recognize. It prefers a known declining label, and with none it takes the last button no accept label matches, so a prompt whose decline is worded in a phrasing or language the list does not carry still declines rather than answers "Allow". A prompt whose every button reads affirmative fails with `alert dismiss button not found` and taps nothing (#3627). - An unreadable or ambiguous capture after the action fails with `error.details.runnerErrorCode: ALERT_CONFIRMATION_UNAVAILABLE`; an expired runner deadline fails with `ALERT_DEADLINE_EXCEEDED`. Neither proves the alert is gone or that nothing happened. Run `alert get` before acting again. - iOS runner refusals carry `error.details.reason`: `runner_busy` means the runner ran nothing (`dispatched: no`); `runner_main_thread_timeout` means the action may still land (`dispatched: unknown`). - If a sheet or permission prompt is visible but `alert` reports none, treat it as app UI: run `snapshot -i -s ""`, then `press @ref`. Android alert support is snapshot-derived, and not every iOS simulator permission surface is a native alert. From ad57d2cbc1494506a56000826420c5f5a17c30ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 15 Oct 2026 23:24:17 +0200 Subject: [PATCH 2/8] fix(ios): disclose the alert button refusal and tighten its contract Cold-review follow-ups on the decline-choice rule: - The new 'dismiss has no button to name' refusal now answers with the runner code ALERT_BUTTON_NOT_FOUND and a hint, bound in RUNNER_ERROR_CODE_DISPATCH as dispatched: no like every other reply that ran nothing, with a row and driver in the dispatch-disclosure golden table. Before, the codeless refusal read as dispatched: unknown and taught callers to key on the message text. - The decline-choice fixture carries the one-shot guard its sibling uses and an affirmative-only variant, so the refusal's response shape is asserted end to end: code ALERT_BUTTON_NOT_FOUND, canary Unanswered. - The alert command description and the commands.md note state the refusal by its typed code, name the one-button 'OK' case, and stop promising that the fallback finds a decline. --- .../AgentDeviceRunner/AgentDeviceRunnerApp.m | 19 ++++++++---- .../RunnerTests+Alert.swift | 8 ++++- .../RunnerTests+AlertButtonChoiceTests.swift | 31 +++++++++++++++++-- contracts/fixtures/dispatch-disclosure.json | 6 ++++ packages/contracts/src/alert-contract.ts | 8 +++++ .../index-dispatch-disclosure.test.ts | 1 + .../src/runner/runner-contract.ts | 6 +++- src/commands/capture/alert.ts | 2 +- website/docs/docs/commands.md | 2 +- 9 files changed, 71 insertions(+), 12 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m index baff9e82c6..4f4881fde6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m @@ -396,15 +396,18 @@ - (void)presentAlertFixtureReplacement:(BOOL)replacement { // The App Tracking Transparency prompt's button order, which is what `alert dismiss` got wrong: // the affirmative button is last, so a fallback to it answers "Allow" (#3627). The answer label // says which button the app really took, so a test does not trust the command's own report. +// `--agent-device-alert-affirmative-only` shows the same alert with no decline button at all, +// where `dismiss` must refuse without activating anything. - (void)presentAlertDeclineChoiceFixture { + BOOL affirmativeOnly = + [NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-affirmative-only"]; UIAlertController *alert = [UIAlertController alertControllerWithTitle:@"“Runner” Would Like to Track Your Activity" message:@"Track your activity across other companies' apps and websites." preferredStyle:UIAlertControllerStyleAlert]; - NSArray *buttons = @[ - @[@"Ask App Not to Track", @"Declined"], - @[@"Allow", @"Authorized"], - ]; + NSArray *buttons = affirmativeOnly + ? @[ @[@"Allow", @"Authorized"], @[@"OK", @"Authorized"] ] + : @[ @[@"Ask App Not to Track", @"Declined"], @[@"Allow", @"Authorized"] ]; for (NSArray *button in buttons) { [alert addAction:[UIAlertAction actionWithTitle:button[0] style:UIAlertActionStyleDefault @@ -418,7 +421,10 @@ - (void)presentAlertDeclineChoiceFixture { - (void)viewDidAppear:(BOOL)animated { [super viewDidAppear:animated]; - if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-decline-choice"]) { + if (!self.alertFixtureStarted && + ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-decline-choice"] || + [NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-affirmative-only"])) { + self.alertFixtureStarted = YES; [self presentAlertDeclineChoiceFixture]; } if (!self.alertFixtureStarted && @@ -602,7 +608,8 @@ - (void)viewDidLoad { [self updateAlertActionStatus]; } - if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-decline-choice"]) { + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-decline-choice"] || + [NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-affirmative-only"]) { UILabel *declineAnswer = [[UILabel alloc] init]; declineAnswer.text = @"Unanswered"; declineAnswer.accessibilityIdentifier = @"agent-device-alert-decline-answer"; diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift index cfb78714e7..c054827424 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift @@ -59,7 +59,13 @@ extension RunnerTests { func handleAlert(_ alert: RunnerAlert, action: String, deadline: Date) -> Response { if action == "accept" || action == "dismiss" { guard let button = chooseAlertButton(alert.buttons, action: action) else { - return Response(ok: false, error: ErrorPayload(message: "alert \(action) button not found")) + // The choice refused before any activation, so this discloses dispatched: no like the + // runner's other pre-run refusals; naming an affirmative button here is what #3627 is about. + return Response(ok: false, error: ErrorPayload( + code: "ALERT_BUTTON_NOT_FOUND", + message: "alert \(action) button not found", + hint: "No alert button was activated. Inspect the current alert before deciding on an action." + )) } guard Date() < deadline else { return alertVerificationResponse(.timedOut, action: action, activated: false) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift index 66354d4016..f277406107 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift @@ -82,7 +82,7 @@ extension RunnerTests { let alert = try XCTUnwrap( resolveAlert(app: app, deadline: Date().addingTimeInterval(RunnerTests.defaultAlertCommandTimeout)) ) - XCTAssertEqual(alert.buttons.map(\.label).last, "Allow", "the fixture keeps the affirmative button last"); + XCTAssertEqual(alert.buttons.map(\.label).last, "Allow", "the fixture keeps the affirmative button last") let response = handleAlert( alert, @@ -91,7 +91,34 @@ extension RunnerTests { ) XCTAssertTrue(response.ok, String(describing: response.error)) - XCTAssertEqual(app.staticTexts["agent-device-alert-decline-answer"].label, "Declined"); + XCTAssertEqual(app.staticTexts["agent-device-alert-decline-answer"].label, "Declined") + } + + /// A prompt whose every button the accept rule claims has no decline to name: `dismiss` must + /// refuse with the typed code and tap nothing, and the canary proves the prompt stayed + /// unanswered (#3627). + @MainActor + func testAlertDismissRefusesAnAffirmativeOnlyPromptWithoutActivatingIt() throws { + app.launchArguments = ["--agent-device-alert-decline-choice", "--agent-device-alert-affirmative-only"] + app.launch() + defer { + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + XCTAssertTrue(app.alerts.firstMatch.waitForExistence(timeout: appExistenceTimeout)) + let alert = try XCTUnwrap( + resolveAlert(app: app, deadline: Date().addingTimeInterval(RunnerTests.defaultAlertCommandTimeout)) + ) + + let response = handleAlert( + alert, + action: "dismiss", + deadline: Date().addingTimeInterval(RunnerTests.alertActivationDeadline) + ) + + XCTAssertFalse(response.ok) + XCTAssertEqual(response.error?.code, "ALERT_BUTTON_NOT_FOUND") + XCTAssertEqual(app.staticTexts["agent-device-alert-decline-answer"].label, "Unanswered") } } #endif diff --git a/contracts/fixtures/dispatch-disclosure.json b/contracts/fixtures/dispatch-disclosure.json index 4fc748c1fd..7bf37cbdb3 100644 --- a/contracts/fixtures/dispatch-disclosure.json +++ b/contracts/fixtures/dispatch-disclosure.json @@ -205,6 +205,12 @@ "trigger": "structured runner reply with code TEXT_INPUT_PROBE_UNAVAILABLE: a coordinate fill refused before its focus tap because the pre-tap input lookup could not answer", "dispatched": "no" }, + { + "id": "ios-runner.reply.ALERT_BUTTON_NOT_FOUND", + "producer": "ios-runner", + "trigger": "structured runner reply with code ALERT_BUTTON_NOT_FOUND: the accept/dismiss button choice refused before any activation (#3627)", + "dispatched": "no" + }, { "id": "ios-runner.reply.APP_SCREEN_WINDOW_UNRESOLVED", "producer": "ios-runner", diff --git a/packages/contracts/src/alert-contract.ts b/packages/contracts/src/alert-contract.ts index 018d5ceb3c..14cf3dd238 100644 --- a/packages/contracts/src/alert-contract.ts +++ b/packages/contracts/src/alert-contract.ts @@ -12,6 +12,14 @@ export const ALERT_ACTION_RETRY_MS = 2_000; export const ALERT_NOT_FOUND_RUNNER_CODE = 'ALERT_NOT_FOUND'; export const ALERT_NOT_FOUND_REASON = 'alert-not-found'; +/** + * The iOS runner refused an `accept`/`dismiss` because no button on the alert fits the action: + * `dismiss` found no button the accept rule does not claim, and naming one would answer an + * affirmative button (#3627). The refusal is made in the button choice, before any activation, + * so it discloses `dispatched: no` like the other replies that ran nothing. + */ +export const ALERT_BUTTON_NOT_FOUND_RUNNER_CODE = 'ALERT_BUTTON_NOT_FOUND'; + export const ALERT_ACTIONS = ['get', 'accept', 'dismiss', 'wait'] as const; export type AlertAction = (typeof ALERT_ACTIONS)[number]; diff --git a/packages/platform-apple/src/runner/__tests__/index-dispatch-disclosure.test.ts b/packages/platform-apple/src/runner/__tests__/index-dispatch-disclosure.test.ts index 7e66b4dfb9..3db43476a1 100644 --- a/packages/platform-apple/src/runner/__tests__/index-dispatch-disclosure.test.ts +++ b/packages/platform-apple/src/runner/__tests__/index-dispatch-disclosure.test.ts @@ -171,6 +171,7 @@ const DRIVERS: Record Promise> = { 'ios-runner.reply.NO_TEXT_INPUT_AT_POINT': () => replyFailure('NO_TEXT_INPUT_AT_POINT'), 'ios-runner.reply.TEXT_INPUT_PROBE_UNAVAILABLE': () => replyFailure('TEXT_INPUT_PROBE_UNAVAILABLE'), + 'ios-runner.reply.ALERT_BUTTON_NOT_FOUND': () => replyFailure('ALERT_BUTTON_NOT_FOUND'), 'ios-runner.reply.APP_SCREEN_WINDOW_UNRESOLVED': () => replyFailure('APP_SCREEN_WINDOW_UNRESOLVED'), 'ios-runner.reply.APP_SCREEN_UNRESOLVED': () => replyFailure('APP_SCREEN_UNRESOLVED'), diff --git a/packages/platform-apple/src/runner/runner-contract.ts b/packages/platform-apple/src/runner/runner-contract.ts index f29118d7ee..98769aed7c 100644 --- a/packages/platform-apple/src/runner/runner-contract.ts +++ b/packages/platform-apple/src/runner/runner-contract.ts @@ -5,7 +5,10 @@ import { type DispatchDisclosure, } from '@agent-device/kernel/errors'; import crypto from 'node:crypto'; -import { ALERT_NOT_FOUND_RUNNER_CODE } from '@agent-device/contracts/alert-contract'; +import { + ALERT_BUTTON_NOT_FOUND_RUNNER_CODE, + ALERT_NOT_FOUND_RUNNER_CODE, +} from '@agent-device/contracts/alert-contract'; import type { DeviceRotation } from '@agent-device/contracts/device'; import type { SnapshotPreferredBackend } from '@agent-device/kernel/snapshot'; import type { ClickButton } from '@agent-device/contracts/click-button'; @@ -319,6 +322,7 @@ const RUNNER_ERROR_CODE_DISPATCH: ReadonlyMap = new [ALERT_NOT_FOUND_RUNNER_CODE, 'no'], [NO_TEXT_INPUT_AT_POINT_RUNNER_CODE, 'no'], [TEXT_INPUT_PROBE_UNAVAILABLE_RUNNER_CODE, 'no'], + [ALERT_BUTTON_NOT_FOUND_RUNNER_CODE, 'no'], ...[...RUNNER_SCREEN_CAPTURE_REFUSAL_RUNNER_CODES].map((code) => [code, 'no'] as const), ]); diff --git a/src/commands/capture/alert.ts b/src/commands/capture/alert.ts index f07dc964a0..3c8f774195 100644 --- a/src/commands/capture/alert.ts +++ b/src/commands/capture/alert.ts @@ -18,7 +18,7 @@ import { AppError } from '@agent-device/kernel/errors'; const ALERT_COMMAND_NAME = 'alert'; const alertCommandDescription = - 'Inspect, wait for, accept, or dismiss a platform alert. Use get before acting when the alert content matters; accept and dismiss change the active alert state. Each iOS XCTest execution activates once, then only observes. Inspect again after an unconfirmed action; never assume a timeout means nothing happened.'; + 'Inspect, wait for, accept, or dismiss a platform alert. Use get before acting when the alert content matters; accept and dismiss change the active alert state. Each iOS XCTest execution activates once, then only observes. On iOS, dismiss taps only a button no accept label claims; a prompt whose every button reads affirmative fails with ALERT_BUTTON_NOT_FOUND and taps nothing. Inspect again after an unconfirmed action; never assume a timeout means nothing happened.'; const alertCommandMetadata = defineFieldCommandMetadata( ALERT_COMMAND_NAME, diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 8409323d50..67e1db54a2 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -387,7 +387,7 @@ agent-device alert dismiss - `alert` (same as `alert get`) inspects or handles system alerts on iOS simulator, macOS desktop, and Android permission dialogs. On iOS it also covers app-owned modal popups with native blocking markers. - Use `alert get` for a quick check, and `alert wait ` only when a prompt may appear after async work. Don't use `settings permission` to answer a dialog already on screen; it is for setup before a flow. - `accept` and `dismiss` are sent once on every platform and never replayed, even if the response is lost. On iOS the runner activates the button once, then only observes until the alert disappears, changes, or the deadline expires. A changed alert does not prove a permission was granted; verify the app outcome. -- On iOS, `dismiss` never activates a button the accept labels recognize. It prefers a known declining label, and with none it takes the last button no accept label matches, so a prompt whose decline is worded in a phrasing or language the list does not carry still declines rather than answers "Allow". A prompt whose every button reads affirmative fails with `alert dismiss button not found` and taps nothing (#3627). +- On iOS, `dismiss` activates only a button the accept labels do not claim: a known declining label first, otherwise the last button no accept label matches, so a decline worded in a phrasing or language the list does not carry — App Tracking Transparency's "Ask App Not to Track" — is still taken ahead of a trailing "Allow". A prompt whose every button reads affirmative, a one-button "OK" alert included, fails with `error.code: ALERT_BUTTON_NOT_FOUND` and `details.dispatched: no`, tapping nothing ([#3627](https://github.com/callstack/agent-device/issues/3627)). - An unreadable or ambiguous capture after the action fails with `error.details.runnerErrorCode: ALERT_CONFIRMATION_UNAVAILABLE`; an expired runner deadline fails with `ALERT_DEADLINE_EXCEEDED`. Neither proves the alert is gone or that nothing happened. Run `alert get` before acting again. - iOS runner refusals carry `error.details.reason`: `runner_busy` means the runner ran nothing (`dispatched: no`); `runner_main_thread_timeout` means the action may still land (`dispatched: unknown`). - If a sheet or permission prompt is visible but `alert` reports none, treat it as app UI: run `snapshot -i -s ""`, then `press @ref`. Android alert support is snapshot-derived, and not every iOS simulator permission surface is a native alert. From 287cf8028095705afdceac47d81622cd0e5acfe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 03:54:59 +0200 Subject: [PATCH 3/8] fix(ios): refuse an alert dismiss fallback that cannot name a decline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cold-review follow-up on the decline-choice rule: - iOS localizes the affirmative button too, so a fully localized prompt leaves every label unclaimed and the last-unclaimed fallback answered the localized 'Allow' — the same authorize-on-guess failure #3627 is about, one locale over. The fallback now fires only when exactly one button is unclaimed; when more than one is, the choice refuses with ALERT_BUTTON_NOT_FOUND and taps nothing, like an all-affirmative prompt. The localization unit vector carries both buttons in the locale, not a German decline paired with an English 'Allow'. - 'ask app not to track' in the decline list is pinned by the shape only it decides: ['Ask App Not to Track', 'Learn More']. - The no-button-activated hint is named once and shared by the refusal and the unactivated verification branches so the two cannot drift. - The command description and commands.md state the scoped guarantee and the runner boundary instead of promising any unclaimed label is a decline, and the two behavior pins are annotated as pins. --- .../RunnerAlertButtonChoice.swift | 9 +++-- .../RunnerTests+Alert.swift | 7 +++- .../RunnerTests+AlertObservation.swift | 2 +- .../RunnerTests+AlertButtonChoiceTests.swift | 37 +++++++++++++++---- src/commands/capture/alert.ts | 2 +- website/docs/docs/commands.md | 2 +- 6 files changed, 45 insertions(+), 14 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertButtonChoice.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertButtonChoice.swift index e95e8bd9d9..c6ff7fb681 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertButtonChoice.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertButtonChoice.swift @@ -7,8 +7,10 @@ import Foundation /// Transparency prompt's decline in every language differently, so no decline list carries it, /// and a fallback to the last button answered "Allow" while reporting `dismissed` (#3627). With /// no known decline label, the only candidate that could decline is a label the accept list does -/// not claim, so `dismiss` takes the last such button; when every button reads affirmative there -/// is no decline to name and the command fails without a tap. +/// not claim — but iOS localizes the affirmative too, so a fully localized prompt leaves every +/// label unclaimed and no candidate is identifiable. So `dismiss` takes a fallback only when +/// exactly one button is unclaimed; when more than one is, or every button reads affirmative, +/// there is no decline to name and the command fails without a tap. enum RunnerAlertButtonChoice { static func buttonIndex(for labels: [String], action: String) -> Int? { if action == "accept" { @@ -21,7 +23,8 @@ enum RunnerAlertButtonChoice { if let decline = labels.firstIndex(where: { isDismissButton($0) }) { return decline } - return labels.lastIndex { !isAcceptButton($0) } + let unclaimed = labels.enumerated().filter { !isAcceptButton($0.element) } + return unclaimed.count == 1 ? unclaimed.first?.offset : nil } static func isAcceptButton(_ label: String) -> Bool { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift index c054827424..72b980f11e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift @@ -16,6 +16,11 @@ extension RunnerTests { static let defaultAlertCommandTimeout: TimeInterval = 10 + /// One recovery sentence for every alert failure that activated nothing, so the refusal in + /// `handleAlert` and the unactivated `alertVerificationResponse` branches cannot drift apart. + static let alertNoButtonActivatedHint = + "No alert button was activated. Inspect the current alert before deciding on an action." + static func alertCommandTimeout(timeoutMs: Double?) -> TimeInterval { guard let timeoutMs, timeoutMs.isFinite else { return defaultAlertCommandTimeout } return max(0.001, timeoutMs / 1000) @@ -64,7 +69,7 @@ extension RunnerTests { return Response(ok: false, error: ErrorPayload( code: "ALERT_BUTTON_NOT_FOUND", message: "alert \(action) button not found", - hint: "No alert button was activated. Inspect the current alert before deciding on an action." + hint: RunnerTests.alertNoButtonActivatedHint )) } guard Date() < deadline else { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AlertObservation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AlertObservation.swift index 6ef66572c9..a010810149 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AlertObservation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AlertObservation.swift @@ -117,7 +117,7 @@ extension RunnerTests { message: message, hint: activated ? "The button was activated once. Inspect the current alert before deciding on another action." - : "No alert button was activated. Inspect the current alert before deciding on an action." + : RunnerTests.alertNoButtonActivatedHint )) } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift index f277406107..01e05b7ea8 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift @@ -16,13 +16,19 @@ extension RunnerTests { RunnerAlertButtonChoice.buttonIndex(for: ["Ask App Not to Track", "Allow"], action: "accept"), 1 ) + // The list entry alone decides this shape: with only "Ask App Not to Track" claimed as a + // decline, both labels would otherwise be unclaimed and the choice must refuse. + XCTAssertEqual( + RunnerAlertButtonChoice.buttonIndex(for: ["Ask App Not to Track", "Learn More"], action: "dismiss"), + 0 + ) } func testAlertDismissKeepsAnUnrecognizedDeclineLabelInsteadOfReachingAnAffirmativeOne() { - // A localization the dismiss list cannot carry: the label the accept list does not claim is - // the only candidate that could decline, so it wins over the trailing "Allow". + // Copy the accept list cannot claim: it is the sole unclaimed label, the only candidate that + // could decline, so it wins over the trailing "Allow". XCTAssertEqual( - RunnerAlertButtonChoice.buttonIndex(for: ["App nicht verfolgen lassen", "Allow"], action: "dismiss"), + RunnerAlertButtonChoice.buttonIndex(for: ["Keep my activity private", "Allow"], action: "dismiss"), 0 ) XCTAssertEqual( @@ -31,6 +37,20 @@ extension RunnerTests { ) } + func testAlertDismissRefusesAPromptWhoseLabelsNeitherListRecognizes() { + // A fully localized prompt: iOS localizes the affirmative too, so no label is claimed by + // either list, no position identifies the decline, and a guess could authorize (#3627). + for labels in [ + ["Demander à l’app de ne pas suivre", "Autoriser"], + ["Not this time", "Allow Tracking"] + ] { + XCTAssertNil( + RunnerAlertButtonChoice.buttonIndex(for: labels, action: "dismiss"), + "dismiss must fail without a tap rather than guess through \(labels)" + ) + } + } + func testAlertDismissRefusesAPromptWhoseEveryButtonIsAffirmative() { for labels in [["Allow"], ["Allow", "OK"], ["Yes", "Continue", "Done"]] { XCTAssertNil( @@ -40,6 +60,8 @@ extension RunnerTests { } } + /// Pins, not evidence of this change: both branches predate it and hold under the old rule + /// too, so they fail only if the accept path or the known-label branches regress. func testAlertDismissStillPicksTheKnownCancelLabelAndAcceptStillTakesOK() { XCTAssertEqual( RunnerAlertButtonChoice.buttonIndex(for: ["Cancel", "OK"], action: "dismiss"), @@ -51,6 +73,7 @@ extension RunnerTests { ) } + /// Pin, not evidence of this change: the accept path is untouched by the dismiss rule. func testAlertAcceptKeepsItsGuardsOnTheDeclineLabelsItMustNotTake() { // A lone declining button is not an acceptance, and with a choice, accept takes the first // label the accept list claims. @@ -66,10 +89,10 @@ extension RunnerTests { #if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) extension RunnerTests { - /// The reporter's scenario end to end: `alert dismiss` on a prompt whose decline label the list - /// did not carry, and whose affirmative button was the last one. The command's own report was - /// `dismissed` either way, so the canary — which button the app actually answered — is the - /// assertion that matters (#3627). + /// The reporter's scenario end to end: `alert dismiss` on the App Tracking Transparency prompt, + /// whose affirmative button is the last one. The command's own report was `dismissed` either + /// way, so the canary — which button the app actually answered — is the assertion that matters + /// (#3627). @MainActor func testAlertDismissAnswersTheTrackingPromptDeclineAndNeverAuthorizes() throws { app.launchArguments = ["--agent-device-alert-decline-choice"] diff --git a/src/commands/capture/alert.ts b/src/commands/capture/alert.ts index 3c8f774195..b443ba4d16 100644 --- a/src/commands/capture/alert.ts +++ b/src/commands/capture/alert.ts @@ -18,7 +18,7 @@ import { AppError } from '@agent-device/kernel/errors'; const ALERT_COMMAND_NAME = 'alert'; const alertCommandDescription = - 'Inspect, wait for, accept, or dismiss a platform alert. Use get before acting when the alert content matters; accept and dismiss change the active alert state. Each iOS XCTest execution activates once, then only observes. On iOS, dismiss taps only a button no accept label claims; a prompt whose every button reads affirmative fails with ALERT_BUTTON_NOT_FOUND and taps nothing. Inspect again after an unconfirmed action; never assume a timeout means nothing happened.'; + 'Inspect, wait for, accept, or dismiss a platform alert. Use get before acting when the alert content matters; accept and dismiss change the active alert state. Each iOS XCTest execution activates once, then only observes. On the Apple runner, dismiss taps only a button no accept label claims and refuses ALERT_BUTTON_NOT_FOUND rather than guess; a prompt whose every button reads affirmative taps nothing. Inspect again after an unconfirmed action; never assume a timeout means nothing happened.'; const alertCommandMetadata = defineFieldCommandMetadata( ALERT_COMMAND_NAME, diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 67e1db54a2..b91e44a23f 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -387,7 +387,7 @@ agent-device alert dismiss - `alert` (same as `alert get`) inspects or handles system alerts on iOS simulator, macOS desktop, and Android permission dialogs. On iOS it also covers app-owned modal popups with native blocking markers. - Use `alert get` for a quick check, and `alert wait ` only when a prompt may appear after async work. Don't use `settings permission` to answer a dialog already on screen; it is for setup before a flow. - `accept` and `dismiss` are sent once on every platform and never replayed, even if the response is lost. On iOS the runner activates the button once, then only observes until the alert disappears, changes, or the deadline expires. A changed alert does not prove a permission was granted; verify the app outcome. -- On iOS, `dismiss` activates only a button the accept labels do not claim: a known declining label first, otherwise the last button no accept label matches, so a decline worded in a phrasing or language the list does not carry — App Tracking Transparency's "Ask App Not to Track" — is still taken ahead of a trailing "Allow". A prompt whose every button reads affirmative, a one-button "OK" alert included, fails with `error.code: ALERT_BUTTON_NOT_FOUND` and `details.dispatched: no`, tapping nothing ([#3627](https://github.com/callstack/agent-device/issues/3627)). +- On the Apple XCUITest runner, `dismiss` activates only a button the accept labels do not claim: a known declining label first, otherwise the sole unclaimed button, so an English decline the lists never carried — App Tracking Transparency's "Ask App Not to Track" — is still taken ahead of a trailing "Allow". When more than one button is unclaimed, which is what a fully localized prompt looks like because iOS localizes the affirmative too, or when every button reads affirmative, a one-button "OK" alert included, `dismiss` fails with `error.code: ALERT_BUTTON_NOT_FOUND` and `details.dispatched: no`, tapping nothing; an `accept` that finds no button it may name fails the same way ([#3627](https://github.com/callstack/agent-device/issues/3627)). - An unreadable or ambiguous capture after the action fails with `error.details.runnerErrorCode: ALERT_CONFIRMATION_UNAVAILABLE`; an expired runner deadline fails with `ALERT_DEADLINE_EXCEEDED`. Neither proves the alert is gone or that nothing happened. Run `alert get` before acting again. - iOS runner refusals carry `error.details.reason`: `runner_busy` means the runner ran nothing (`dispatched: no`); `runner_main_thread_timeout` means the action may still land (`dispatched: unknown`). - If a sheet or permission prompt is visible but `alert` reports none, treat it as app UI: run `snapshot -i -s ""`, then `press @ref`. Android alert support is snapshot-derived, and not every iOS simulator permission surface is a native alert. From afcad63d192a28db0b671c52d3128f29d399a8ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 06:46:39 +0200 Subject: [PATCH 4/8] fix(ios): claim consent labels the dismiss fallback must never reach MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second cold-review round on the decline-choice rule: - 'Accept', 'I Agree', 'Got it', 'Allow Tracking' and 'Use App' were outside isAcceptButton, so the sole-unclaimed fallback activated a one-button consent prompt of that copy and reported 'dismissed' — the incident's shape one label away from the list. The accept list now claims them, with pins. - The two canary fixtures resolve under the sibling lane's resolution allowance instead of the 10s command timeout: resolution is a pure read that a contended hosted simulator has spent 40s on, and timing it out would read as a red fix. - commands.md and the CLI description state the guarantee as the runner's English label lists actually deliver, attribute the ATT decline to its list entry, and stop promising 'every button reads affirmative' beyond what the lists recognize. - Pin comments state the invariant they hold rather than this PR's review history. --- .../RunnerAlertButtonChoice.swift | 10 +++++ .../RunnerTests+AlertButtonChoiceTests.swift | 45 +++++++++++++------ src/commands/capture/alert.ts | 2 +- website/docs/docs/commands.md | 2 +- 4 files changed, 43 insertions(+), 16 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertButtonChoice.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertButtonChoice.swift index c6ff7fb681..05ef772921 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertButtonChoice.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertButtonChoice.swift @@ -31,7 +31,17 @@ enum RunnerAlertButtonChoice { let normalized = normalize(label) return [ "ok", + "accept", + "agree", + "i agree", + "got it", + "i understand", "allow", + "allow all", + "allow all requests", + "allow tracking", + "start tracking", + "use app", "yes", "continue", "done", diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift index 01e05b7ea8..ee09e6e965 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift @@ -6,6 +6,18 @@ extension RunnerTests { XCTAssertTrue(RunnerAlertButtonChoice.isAcceptButton("Open")) } + func testAlertDismissRefusesPlainAffirmativeLabelsOutsideTheOriginalList() { + // The sole-unclaimed fallback may only ever reach a label `isAcceptButton` does not read as + // affirmative, so consent copy that plainly reads affirmative must be claimed here too — + // otherwise dismissing a one-button "Accept" prompt would activate it (#3627). + for labels in [["Accept"], ["I Agree"], ["Got it"], ["Allow Tracking"], ["Use App"]] { + XCTAssertNil( + RunnerAlertButtonChoice.buttonIndex(for: labels, action: "dismiss"), + "dismiss must fail without a tap rather than consent through \(labels)" + ) + } + } + func testAlertDismissPicksTheTrackingPromptDeclineLabel() { // The App Tracking Transparency prompt, as iOS words it in English (#3627). XCTAssertEqual( @@ -35,20 +47,22 @@ extension RunnerTests { RunnerAlertButtonChoice.buttonIndex(for: ["More info", "Allow", "OK"], action: "dismiss"), 0 ) + XCTAssertEqual( + RunnerAlertButtonChoice.buttonIndex(for: ["Not this time", "Allow Tracking"], action: "dismiss"), + 0 + ) } func testAlertDismissRefusesAPromptWhoseLabelsNeitherListRecognizes() { // A fully localized prompt: iOS localizes the affirmative too, so no label is claimed by // either list, no position identifies the decline, and a guess could authorize (#3627). - for labels in [ - ["Demander à l’app de ne pas suivre", "Autoriser"], - ["Not this time", "Allow Tracking"] - ] { - XCTAssertNil( - RunnerAlertButtonChoice.buttonIndex(for: labels, action: "dismiss"), - "dismiss must fail without a tap rather than guess through \(labels)" - ) - } + XCTAssertNil( + RunnerAlertButtonChoice.buttonIndex( + for: ["Demander à l’app de ne pas suivre", "Autoriser le suivi"], + action: "dismiss" + ), + "dismiss must fail without a tap rather than guess through a localized prompt" + ) } func testAlertDismissRefusesAPromptWhoseEveryButtonIsAffirmative() { @@ -60,8 +74,8 @@ extension RunnerTests { } } - /// Pins, not evidence of this change: both branches predate it and hold under the old rule - /// too, so they fail only if the accept path or the known-label branches regress. + /// A known decline label wins ahead of any unclaimed label, and accept takes the label its own + /// list claims even when a decline is present. func testAlertDismissStillPicksTheKnownCancelLabelAndAcceptStillTakesOK() { XCTAssertEqual( RunnerAlertButtonChoice.buttonIndex(for: ["Cancel", "OK"], action: "dismiss"), @@ -73,7 +87,8 @@ extension RunnerTests { ) } - /// Pin, not evidence of this change: the accept path is untouched by the dismiss rule. + /// accept never activates a listed decline: alone it is no acceptance, and with a choice it + /// takes only a label its own list claims. func testAlertAcceptKeepsItsGuardsOnTheDeclineLabelsItMustNotTake() { // A lone declining button is not an acceptance, and with a choice, accept takes the first // label the accept list claims. @@ -102,8 +117,10 @@ extension RunnerTests { app.terminate() } XCTAssertTrue(app.alerts.firstMatch.waitForExistence(timeout: appExistenceTimeout)) + // Resolution is a pure read; a contended hosted simulator can spend far more than the 10 s + // command timeout on it, so it runs under the sibling fixtures' resolution allowance. let alert = try XCTUnwrap( - resolveAlert(app: app, deadline: Date().addingTimeInterval(RunnerTests.defaultAlertCommandTimeout)) + resolveAlert(app: app, deadline: Date().addingTimeInterval(RunnerTests.alertResolutionAllowance)) ) XCTAssertEqual(alert.buttons.map(\.label).last, "Allow", "the fixture keeps the affirmative button last") @@ -130,7 +147,7 @@ extension RunnerTests { } XCTAssertTrue(app.alerts.firstMatch.waitForExistence(timeout: appExistenceTimeout)) let alert = try XCTUnwrap( - resolveAlert(app: app, deadline: Date().addingTimeInterval(RunnerTests.defaultAlertCommandTimeout)) + resolveAlert(app: app, deadline: Date().addingTimeInterval(RunnerTests.alertResolutionAllowance)) ) let response = handleAlert( diff --git a/src/commands/capture/alert.ts b/src/commands/capture/alert.ts index b443ba4d16..5d02853292 100644 --- a/src/commands/capture/alert.ts +++ b/src/commands/capture/alert.ts @@ -18,7 +18,7 @@ import { AppError } from '@agent-device/kernel/errors'; const ALERT_COMMAND_NAME = 'alert'; const alertCommandDescription = - 'Inspect, wait for, accept, or dismiss a platform alert. Use get before acting when the alert content matters; accept and dismiss change the active alert state. Each iOS XCTest execution activates once, then only observes. On the Apple runner, dismiss taps only a button no accept label claims and refuses ALERT_BUTTON_NOT_FOUND rather than guess; a prompt whose every button reads affirmative taps nothing. Inspect again after an unconfirmed action; never assume a timeout means nothing happened.'; + 'Inspect, wait for, accept, or dismiss a platform alert. Use get before acting when the alert content matters; accept and dismiss change the active alert state. Each iOS XCTest execution activates once, then only observes. On the Apple runner, dismiss taps a known decline or the sole button no accept label recognizes, and refuses ALERT_BUTTON_NOT_FOUND rather than guess; a prompt whose every button the accept labels recognize taps nothing. Inspect again after an unconfirmed action; never assume a timeout means nothing happened.'; const alertCommandMetadata = defineFieldCommandMetadata( ALERT_COMMAND_NAME, diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index b91e44a23f..fd8eab0c48 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -387,7 +387,7 @@ agent-device alert dismiss - `alert` (same as `alert get`) inspects or handles system alerts on iOS simulator, macOS desktop, and Android permission dialogs. On iOS it also covers app-owned modal popups with native blocking markers. - Use `alert get` for a quick check, and `alert wait ` only when a prompt may appear after async work. Don't use `settings permission` to answer a dialog already on screen; it is for setup before a flow. - `accept` and `dismiss` are sent once on every platform and never replayed, even if the response is lost. On iOS the runner activates the button once, then only observes until the alert disappears, changes, or the deadline expires. A changed alert does not prove a permission was granted; verify the app outcome. -- On the Apple XCUITest runner, `dismiss` activates only a button the accept labels do not claim: a known declining label first, otherwise the sole unclaimed button, so an English decline the lists never carried — App Tracking Transparency's "Ask App Not to Track" — is still taken ahead of a trailing "Allow". When more than one button is unclaimed, which is what a fully localized prompt looks like because iOS localizes the affirmative too, or when every button reads affirmative, a one-button "OK" alert included, `dismiss` fails with `error.code: ALERT_BUTTON_NOT_FOUND` and `details.dispatched: no`, tapping nothing; an `accept` that finds no button it may name fails the same way ([#3627](https://github.com/callstack/agent-device/issues/3627)). +- On the Apple XCUITest runner, `dismiss` activates a known declining label, or — when the labels carry none — the sole button the accept labels do not read as affirmative, so unusual decline copy is still taken ahead of a trailing "Allow". When every button reads affirmative, or more than one is unclaimed (a fully localized prompt, because iOS localizes the affirmative too), `dismiss` fails with `error.code: ALERT_BUTTON_NOT_FOUND` and `details.dispatched: no`, tapping nothing; an `accept` that finds no button it may name fails the same way ([#3627](https://github.com/callstack/agent-device/issues/3627)). Both label lists are English, so "reads affirmative" means "the accept list recognizes it". - An unreadable or ambiguous capture after the action fails with `error.details.runnerErrorCode: ALERT_CONFIRMATION_UNAVAILABLE`; an expired runner deadline fails with `ALERT_DEADLINE_EXCEEDED`. Neither proves the alert is gone or that nothing happened. Run `alert get` before acting again. - iOS runner refusals carry `error.details.reason`: `runner_busy` means the runner ran nothing (`dispatched: no`); `runner_main_thread_timeout` means the action may still land (`dispatched: unknown`). - If a sheet or permission prompt is visible but `alert` reports none, treat it as app UI: run `snapshot -i -s ""`, then `press @ref`. Android alert support is snapshot-derived, and not every iOS simulator permission surface is a native alert. From f173f21cdaf3f0adbc7ea02a3d3fbf161423bc57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 08:37:16 +0200 Subject: [PATCH 5/8] fix(ios): dismiss only a listed decline, never a guessed one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Parent-evidenced vectors showed the sole-unclaimed fallback still reached consent copy the accept list had not been taught yet: 'Accept all', 'Agree to Terms', 'I accept', 'Accept terms' were each tapped and reported 'dismissed'. Growing the accept list cannot close an open-ended copy space, so the rule stops guessing: - dismiss resolves only a label isDismissButton recognizes; any other prompt — unknown wording, localization, consent copy — answers ALERT_BUTTON_NOT_FOUND and taps nothing, the issue's stated alternative to picking the decline. Unknown buttons stay reachable by name through 'alert get' plus snapshot + press. - With no fallback, the accept list needs no consent-label arms race and returns to its base membership. - The pure tests pin the refusal set with the parent's exact shapes, keep the known-decline positives including the 'Ask App Not to Track' entry pin, and keep accept's guards unchanged. - commands.md, the CLI description, and the code contract constant's doc say the list-scoped rule and name the press-by-label path. --- .../RunnerAlertButtonChoice.swift | 32 +++------ .../RunnerTests+AlertButtonChoiceTests.swift | 65 +++++++++---------- packages/contracts/src/alert-contract.ts | 8 +-- src/commands/capture/alert.ts | 2 +- website/docs/docs/commands.md | 2 +- 5 files changed, 45 insertions(+), 64 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertButtonChoice.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertButtonChoice.swift index 05ef772921..14ac41ed61 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertButtonChoice.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertButtonChoice.swift @@ -3,14 +3,14 @@ import Foundation /// Which of an alert's buttons an `accept` or `dismiss` activates, decided on the button labels /// alone, so the rule is checkable without a screen. /// -/// `dismiss` never answers a button `isAcceptButton` recognizes. iOS words the App Tracking -/// Transparency prompt's decline in every language differently, so no decline list carries it, -/// and a fallback to the last button answered "Allow" while reporting `dismissed` (#3627). With -/// no known decline label, the only candidate that could decline is a label the accept list does -/// not claim — but iOS localizes the affirmative too, so a fully localized prompt leaves every -/// label unclaimed and no candidate is identifiable. So `dismiss` takes a fallback only when -/// exactly one button is unclaimed; when more than one is, or every button reads affirmative, -/// there is no decline to name and the command fails without a tap. +/// `dismiss` activates only a button `isDismissButton` recognizes. Any label-based guess at an +/// unrecognized decline can only ever land on a button the accept rule does not recognize either +/// — iOS localizes the App Tracking Transparency prompt's "Allow", consent copy is open-ended, +/// and no list closes either set — and answering such a button while reporting `dismissed` is +/// exactly the failure #3627 reports. So an unrecognized decline is refused with +/// `ALERT_BUTTON_NOT_FOUND` and no tap rather than guessed at. Unknown wording stays reachable: +/// `alert get` shows the labels, and `snapshot` + `press @ref` names the button instead of +/// inferring it. enum RunnerAlertButtonChoice { static func buttonIndex(for labels: [String], action: String) -> Int? { if action == "accept" { @@ -20,28 +20,14 @@ enum RunnerAlertButtonChoice { // A lone button is the only way out of the prompt, so accept takes it unless it declines. return labels.count == 1 && !isDismissButton(labels[0]) ? 0 : nil } - if let decline = labels.firstIndex(where: { isDismissButton($0) }) { - return decline - } - let unclaimed = labels.enumerated().filter { !isAcceptButton($0.element) } - return unclaimed.count == 1 ? unclaimed.first?.offset : nil + return labels.firstIndex(where: { isDismissButton($0) }) } static func isAcceptButton(_ label: String) -> Bool { let normalized = normalize(label) return [ "ok", - "accept", - "agree", - "i agree", - "got it", - "i understand", "allow", - "allow all", - "allow all requests", - "allow tracking", - "start tracking", - "use app", "yes", "continue", "done", diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift index ee09e6e965..900ebe39cf 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift @@ -6,18 +6,6 @@ extension RunnerTests { XCTAssertTrue(RunnerAlertButtonChoice.isAcceptButton("Open")) } - func testAlertDismissRefusesPlainAffirmativeLabelsOutsideTheOriginalList() { - // The sole-unclaimed fallback may only ever reach a label `isAcceptButton` does not read as - // affirmative, so consent copy that plainly reads affirmative must be claimed here too — - // otherwise dismissing a one-button "Accept" prompt would activate it (#3627). - for labels in [["Accept"], ["I Agree"], ["Got it"], ["Allow Tracking"], ["Use App"]] { - XCTAssertNil( - RunnerAlertButtonChoice.buttonIndex(for: labels, action: "dismiss"), - "dismiss must fail without a tap rather than consent through \(labels)" - ) - } - } - func testAlertDismissPicksTheTrackingPromptDeclineLabel() { // The App Tracking Transparency prompt, as iOS words it in English (#3627). XCTAssertEqual( @@ -28,34 +16,37 @@ extension RunnerTests { RunnerAlertButtonChoice.buttonIndex(for: ["Ask App Not to Track", "Allow"], action: "accept"), 1 ) - // The list entry alone decides this shape: with only "Ask App Not to Track" claimed as a - // decline, both labels would otherwise be unclaimed and the choice must refuse. + // The list entry alone decides this shape: without it neither label is a recognized decline + // and the choice must refuse rather than guess. XCTAssertEqual( RunnerAlertButtonChoice.buttonIndex(for: ["Ask App Not to Track", "Learn More"], action: "dismiss"), 0 ) } - func testAlertDismissKeepsAnUnrecognizedDeclineLabelInsteadOfReachingAnAffirmativeOne() { - // Copy the accept list cannot claim: it is the sole unclaimed label, the only candidate that - // could decline, so it wins over the trailing "Allow". - XCTAssertEqual( - RunnerAlertButtonChoice.buttonIndex(for: ["Keep my activity private", "Allow"], action: "dismiss"), - 0 - ) - XCTAssertEqual( - RunnerAlertButtonChoice.buttonIndex(for: ["More info", "Allow", "OK"], action: "dismiss"), - 0 - ) - XCTAssertEqual( - RunnerAlertButtonChoice.buttonIndex(for: ["Not this time", "Allow Tracking"], action: "dismiss"), - 0 - ) + func testAlertDismissRefusesUnrecognizedDeclineCopyInsteadOfGuessing() { + // No fallback position is safe: every label-based guess at an unknown decline can land on a + // localized or reworded affirmative, which is the #3627 failure. Unknown wording is reached + // by `alert get` plus `snapshot`/`press`, which name the button instead of inferring it. + for labels in [ + ["Keep my activity private", "Allow"], + ["Not this time", "Allow Tracking"], + ["More info", "Allow", "OK"], + ["Accept all", "OK"], + ["Agree to Terms", "OK"], + ["I accept", "OK"], + ["Accept terms", "Continue"] + ] { + XCTAssertNil( + RunnerAlertButtonChoice.buttonIndex(for: labels, action: "dismiss"), + "dismiss must fail without a tap rather than guess through \(labels)" + ) + } } func testAlertDismissRefusesAPromptWhoseLabelsNeitherListRecognizes() { - // A fully localized prompt: iOS localizes the affirmative too, so no label is claimed by - // either list, no position identifies the decline, and a guess could authorize (#3627). + // A fully localized prompt: iOS localizes both buttons, so no label is claimed by either + // list, no position identifies the decline, and a guess could authorize (#3627). XCTAssertNil( RunnerAlertButtonChoice.buttonIndex( for: ["Demander à l’app de ne pas suivre", "Autoriser le suivi"], @@ -74,13 +65,17 @@ extension RunnerTests { } } - /// A known decline label wins ahead of any unclaimed label, and accept takes the label its own - /// list claims even when a decline is present. + /// The known-decline branches stay exact: a listed decline resolves by position first, and + /// accept takes the label its own list claims even when a decline is present. func testAlertDismissStillPicksTheKnownCancelLabelAndAcceptStillTakesOK() { XCTAssertEqual( RunnerAlertButtonChoice.buttonIndex(for: ["Cancel", "OK"], action: "dismiss"), 0 ) + XCTAssertEqual( + RunnerAlertButtonChoice.buttonIndex(for: ["Don’t Allow", "Allow"], action: "dismiss"), + 0 + ) XCTAssertEqual( RunnerAlertButtonChoice.buttonIndex(for: ["Cancel", "OK"], action: "accept"), 1 @@ -134,8 +129,8 @@ extension RunnerTests { XCTAssertEqual(app.staticTexts["agent-device-alert-decline-answer"].label, "Declined") } - /// A prompt whose every button the accept rule claims has no decline to name: `dismiss` must - /// refuse with the typed code and tap nothing, and the canary proves the prompt stayed + /// A prompt carrying no button the decline list recognizes has no decline to name: `dismiss` + /// must refuse with the typed code and tap nothing, and the canary proves the prompt stayed /// unanswered (#3627). @MainActor func testAlertDismissRefusesAnAffirmativeOnlyPromptWithoutActivatingIt() throws { diff --git a/packages/contracts/src/alert-contract.ts b/packages/contracts/src/alert-contract.ts index 14cf3dd238..8784b57017 100644 --- a/packages/contracts/src/alert-contract.ts +++ b/packages/contracts/src/alert-contract.ts @@ -13,10 +13,10 @@ export const ALERT_NOT_FOUND_RUNNER_CODE = 'ALERT_NOT_FOUND'; export const ALERT_NOT_FOUND_REASON = 'alert-not-found'; /** - * The iOS runner refused an `accept`/`dismiss` because no button on the alert fits the action: - * `dismiss` found no button the accept rule does not claim, and naming one would answer an - * affirmative button (#3627). The refusal is made in the button choice, before any activation, - * so it discloses `dispatched: no` like the other replies that ran nothing. + * The Apple runner refused an `accept`/`dismiss` because no button on the alert fits the action: + * `dismiss` found no label on its decline list, and tapping anything else could answer an + * affirmative (#3627). The refusal is made in the button choice, before any activation, so it + * discloses `dispatched: no` like the other replies that ran nothing. */ export const ALERT_BUTTON_NOT_FOUND_RUNNER_CODE = 'ALERT_BUTTON_NOT_FOUND'; diff --git a/src/commands/capture/alert.ts b/src/commands/capture/alert.ts index 5d02853292..c3560b23e7 100644 --- a/src/commands/capture/alert.ts +++ b/src/commands/capture/alert.ts @@ -18,7 +18,7 @@ import { AppError } from '@agent-device/kernel/errors'; const ALERT_COMMAND_NAME = 'alert'; const alertCommandDescription = - 'Inspect, wait for, accept, or dismiss a platform alert. Use get before acting when the alert content matters; accept and dismiss change the active alert state. Each iOS XCTest execution activates once, then only observes. On the Apple runner, dismiss taps a known decline or the sole button no accept label recognizes, and refuses ALERT_BUTTON_NOT_FOUND rather than guess; a prompt whose every button the accept labels recognize taps nothing. Inspect again after an unconfirmed action; never assume a timeout means nothing happened.'; + 'Inspect, wait for, accept, or dismiss a platform alert. Use get before acting when the alert content matters; accept and dismiss change the active alert state. Each iOS XCTest execution activates once, then only observes. On the Apple runner, dismiss taps only a button on its decline list and refuses ALERT_BUTTON_NOT_FOUND rather than guess at unknown wording; press the button by label when a prompt uses it. Inspect again after an unconfirmed action; never assume a timeout means nothing happened.'; const alertCommandMetadata = defineFieldCommandMetadata( ALERT_COMMAND_NAME, diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index fd8eab0c48..a27ff03b29 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -387,7 +387,7 @@ agent-device alert dismiss - `alert` (same as `alert get`) inspects or handles system alerts on iOS simulator, macOS desktop, and Android permission dialogs. On iOS it also covers app-owned modal popups with native blocking markers. - Use `alert get` for a quick check, and `alert wait ` only when a prompt may appear after async work. Don't use `settings permission` to answer a dialog already on screen; it is for setup before a flow. - `accept` and `dismiss` are sent once on every platform and never replayed, even if the response is lost. On iOS the runner activates the button once, then only observes until the alert disappears, changes, or the deadline expires. A changed alert does not prove a permission was granted; verify the app outcome. -- On the Apple XCUITest runner, `dismiss` activates a known declining label, or — when the labels carry none — the sole button the accept labels do not read as affirmative, so unusual decline copy is still taken ahead of a trailing "Allow". When every button reads affirmative, or more than one is unclaimed (a fully localized prompt, because iOS localizes the affirmative too), `dismiss` fails with `error.code: ALERT_BUTTON_NOT_FOUND` and `details.dispatched: no`, tapping nothing; an `accept` that finds no button it may name fails the same way ([#3627](https://github.com/callstack/agent-device/issues/3627)). Both label lists are English, so "reads affirmative" means "the accept list recognizes it". +- On the Apple XCUITest runner, `dismiss` activates only a button its decline list recognizes (`Cancel`, `Don't Allow`, `Ask App Not to Track`, and similar English labels). It never taps a button that list does not recognize: unknown, reworded, or fully localized decline copy fails with `error.code: ALERT_BUTTON_NOT_FOUND` and `details.dispatched: no`, tapping nothing, rather than guessing at an affirmative the way a last-button fallback once did ([#3627](https://github.com/callstack/agent-device/issues/3627)). A button choice that finds no button to name answers with that code and `dispatched: no` for `accept` too. To answer a prompt with unrecognized wording, read the labels with `alert get`, then press the button by label through `snapshot` + `press @ref`. - An unreadable or ambiguous capture after the action fails with `error.details.runnerErrorCode: ALERT_CONFIRMATION_UNAVAILABLE`; an expired runner deadline fails with `ALERT_DEADLINE_EXCEEDED`. Neither proves the alert is gone or that nothing happened. Run `alert get` before acting again. - iOS runner refusals carry `error.details.reason`: `runner_busy` means the runner ran nothing (`dispatched: no`); `runner_main_thread_timeout` means the action may still land (`dispatched: unknown`). - If a sheet or permission prompt is visible but `alert` reports none, treat it as app UI: run `snapshot -i -s ""`, then `press @ref`. Android alert support is snapshot-derived, and not every iOS simulator permission surface is a native alert. From fcbf6ece2922104f43c02bccea6eacd7188c2324 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 09:10:53 +0200 Subject: [PATCH 6/8] fix(ios): keep the dismiss-marker popup surface dismissable through its marker alert dismiss on an app-owned popover resolved through the native "dismiss popup" marker regressed: that surface's close control ("Close popover") is on no decline list, and base reached it through the positional fallback. Restore it as a property of the surface, not a position: only a RunnerAlert whose source is the marker window may take its last actionable element, so consent prompts (never marker-marked) keep the #3627 refusal. Pin the surface through chooseAlertButton and the label rule in the pure table. Also from review: scope the decline-list claim to the Apple XCUITest runner in the CLI description and wire-code contract, declare ALERT_BUTTON_NOT_FOUND in RunnerWireErrorCode where the protocol names wire codes, and drop the migration narration from the user doc. --- .../RunnerAlertButtonChoice.swift | 12 +++++++++-- .../RunnerTests+Alert.swift | 15 ++++++++------ .../RunnerTests.swift | 3 +++ .../RunnerTests+AlertButtonChoiceTests.swift | 20 +++++++++++++++++++ .../UnitTests/RunnerTests+AlertTests.swift | 15 ++++++++++++++ packages/contracts/src/alert-contract.ts | 8 ++++---- src/commands/capture/alert.ts | 2 +- website/docs/docs/commands.md | 2 +- 8 files changed, 63 insertions(+), 14 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertButtonChoice.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertButtonChoice.swift index 14ac41ed61..95c9088a90 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertButtonChoice.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertButtonChoice.swift @@ -11,8 +11,13 @@ import Foundation /// `ALERT_BUTTON_NOT_FOUND` and no tap rather than guessed at. Unknown wording stays reachable: /// `alert get` shows the labels, and `snapshot` + `press @ref` names the button instead of /// inferring it. +/// +/// The one recognized decline outside the list is a dismiss-popup surface itself: its native +/// accessibility marker is the app's own declaration that the surface closes the popover, so +/// `popupMarkerSurface` lets `dismiss` take the last actionable element of that window, as +/// before the #3627 rule existed. A consent prompt can never carry that marker. enum RunnerAlertButtonChoice { - static func buttonIndex(for labels: [String], action: String) -> Int? { + static func buttonIndex(for labels: [String], action: String, popupMarkerSurface: Bool = false) -> Int? { if action == "accept" { if let accept = labels.firstIndex(where: { isAcceptButton($0) }) { return accept @@ -20,7 +25,10 @@ enum RunnerAlertButtonChoice { // A lone button is the only way out of the prompt, so accept takes it unless it declines. return labels.count == 1 && !isDismissButton(labels[0]) ? 0 : nil } - return labels.firstIndex(where: { isDismissButton($0) }) + if let decline = labels.firstIndex(where: { isDismissButton($0) }) { + return decline + } + return popupMarkerSurface ? labels.indices.last : nil } static func isAcceptButton(_ label: String) -> Bool { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift index 72b980f11e..04cb621c03 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift @@ -63,11 +63,11 @@ extension RunnerTests { @MainActor func handleAlert(_ alert: RunnerAlert, action: String, deadline: Date) -> Response { if action == "accept" || action == "dismiss" { - guard let button = chooseAlertButton(alert.buttons, action: action) else { + guard let button = chooseAlertButton(alert, action: action) else { // The choice refused before any activation, so this discloses dispatched: no like the // runner's other pre-run refusals; naming an affirmative button here is what #3627 is about. return Response(ok: false, error: ErrorPayload( - code: "ALERT_BUTTON_NOT_FOUND", + code: RunnerWireErrorCode.alertButtonNotFound, message: "alert \(action) button not found", hint: RunnerTests.alertNoButtonActivatedHint )) @@ -257,14 +257,17 @@ extension RunnerTests { ) private static let dismissPopupMarkerText = NSPredicate(format: "SELF MATCHES[c] %@", dismissPopupMarkerPattern) - private func chooseAlertButton(_ buttons: [XCUIElement], action: String) -> XCUIElement? { + // Internal, not private: the dismiss-popup wiring is pinned by + // RunnerTests+AlertTests, and the label rule itself by RunnerAlertButtonChoice. + func chooseAlertButton(_ alert: RunnerAlert, action: String) -> XCUIElement? { guard let index = RunnerAlertButtonChoice.buttonIndex( - for: buttons.map(\.label), - action: action + for: alert.buttons.map(\.label), + action: action, + popupMarkerSurface: alert.source == .dismissPopup ) else { return nil } - return buttons[index] + return alert.buttons[index] } private func preferredAlertTitle(_ element: XCUIElement, buttons: [XCUIElement]) -> String { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index fcb0d942cc..d2f63502d6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -36,11 +36,14 @@ final class RunnerTests: XCTestCase { /// `NO_TEXT_INPUT_AT_POINT` refuses a coordinate fill whose point identified no text input. /// `TEXT_INPUT_PROBE_UNAVAILABLE` refuses one whose input lookup could not answer on the degraded /// channel, so the runner cannot tell whether an input is there. + /// `ALERT_BUTTON_NOT_FOUND` is the alert button choice refusing before any activation, so it + /// discloses dispatched: no. enum RunnerWireErrorCode { static let mainThreadTimeout = "MAIN_THREAD_TIMEOUT" static let appNotRunning = "APP_NOT_RUNNING" static let noTextInputAtPoint = "NO_TEXT_INPUT_AT_POINT" static let textInputProbeUnavailable = "TEXT_INPUT_PROBE_UNAVAILABLE" + static let alertButtonNotFound = "ALERT_BUTTON_NOT_FOUND" } static let springboardBundleId = "com.apple.springboard" diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift index 900ebe39cf..f8862aa6d5 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertButtonChoiceTests.swift @@ -56,6 +56,26 @@ extension RunnerTests { ) } + func testAlertDismissClosesOnlyAPopupMarkerSurfaceThroughItsLastElement() { + // The marker is the app's own declaration that the surface closes the popover, so that + // surface keeps base's close-by-last-element behavior. No consent prompt can carry it, so + // nothing else gains a positional tap. + XCTAssertEqual( + RunnerAlertButtonChoice.buttonIndex(for: ["Close popover"], action: "dismiss", popupMarkerSurface: true), + 0 + ) + XCTAssertEqual( + RunnerAlertButtonChoice.buttonIndex( + for: ["Details", "Close popover"], action: "dismiss", popupMarkerSurface: true + ), + 1 + ) + XCTAssertNil( + RunnerAlertButtonChoice.buttonIndex(for: ["Close popover"], action: "dismiss", popupMarkerSurface: false), + "an unlabeled-by-the-list prompt without the marker still refuses" + ) + } + func testAlertDismissRefusesAPromptWhoseEveryButtonIsAffirmative() { for labels in [["Allow"], ["Allow", "OK"], ["Yes", "Continue", "Done"]] { XCTAssertNil( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift index c8f3a24207..00419f9847 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AlertTests.swift @@ -50,6 +50,21 @@ extension RunnerTests { XCTAssertTrue(alert.buttons.contains { $0.identifier == "agent-device-close-popover" }) } + /// `alert dismiss` on the marker window must still name its close button: the marker is the + /// app's own decline declaration, so the choice reaches it even though "Close popover" is on + /// no list (#3627 left this surface at base behavior). + @MainActor + func testAlertDismissOnADismissPopupWindowChoosesThePopoverCloseButton() throws { + launchCrowdedScreen(extraArguments: ["--agent-device-dismiss-popup-window"]) + defer { terminateCrowdedScreen() } + + let alert = try XCTUnwrap( + resolveAlert(app: app, deadline: Date().addingTimeInterval(RunnerTests.defaultAlertCommandTimeout)) + ) + let button = try XCTUnwrap(chooseAlertButton(alert, action: "dismiss")) + XCTAssertEqual(button.identifier, "agent-device-close-popover") + } + private func launchCrowdedScreen(extraArguments: [String]) { app.launchArguments = ["--agent-device-crowded-screen"] + extraArguments app.launch() diff --git a/packages/contracts/src/alert-contract.ts b/packages/contracts/src/alert-contract.ts index 8784b57017..b7af81b93a 100644 --- a/packages/contracts/src/alert-contract.ts +++ b/packages/contracts/src/alert-contract.ts @@ -13,10 +13,10 @@ export const ALERT_NOT_FOUND_RUNNER_CODE = 'ALERT_NOT_FOUND'; export const ALERT_NOT_FOUND_REASON = 'alert-not-found'; /** - * The Apple runner refused an `accept`/`dismiss` because no button on the alert fits the action: - * `dismiss` found no label on its decline list, and tapping anything else could answer an - * affirmative (#3627). The refusal is made in the button choice, before any activation, so it - * discloses `dispatched: no` like the other replies that ran nothing. + * The Apple XCUITest runner refused an `accept`/`dismiss` because no button on the alert fits + * the action: `dismiss` found no label on its decline list, and tapping anything else could + * answer an affirmative (#3627). The refusal is made in the button choice, before any + * activation, so it discloses `dispatched: no` like the other replies that ran nothing. */ export const ALERT_BUTTON_NOT_FOUND_RUNNER_CODE = 'ALERT_BUTTON_NOT_FOUND'; diff --git a/src/commands/capture/alert.ts b/src/commands/capture/alert.ts index c3560b23e7..10d725dab6 100644 --- a/src/commands/capture/alert.ts +++ b/src/commands/capture/alert.ts @@ -18,7 +18,7 @@ import { AppError } from '@agent-device/kernel/errors'; const ALERT_COMMAND_NAME = 'alert'; const alertCommandDescription = - 'Inspect, wait for, accept, or dismiss a platform alert. Use get before acting when the alert content matters; accept and dismiss change the active alert state. Each iOS XCTest execution activates once, then only observes. On the Apple runner, dismiss taps only a button on its decline list and refuses ALERT_BUTTON_NOT_FOUND rather than guess at unknown wording; press the button by label when a prompt uses it. Inspect again after an unconfirmed action; never assume a timeout means nothing happened.'; + 'Inspect, wait for, accept, or dismiss a platform alert. Use get before acting when the alert content matters; accept and dismiss change the active alert state. Each iOS XCTest execution activates once, then only observes. On the Apple XCUITest runner, dismiss taps only a button on its decline list (plus a popover marked dismissable) and refuses ALERT_BUTTON_NOT_FOUND rather than guess at unknown wording; press the button by label when a prompt uses it. Inspect again after an unconfirmed action; never assume a timeout means nothing happened.'; const alertCommandMetadata = defineFieldCommandMetadata( ALERT_COMMAND_NAME, diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index a27ff03b29..713bacf655 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -387,7 +387,7 @@ agent-device alert dismiss - `alert` (same as `alert get`) inspects or handles system alerts on iOS simulator, macOS desktop, and Android permission dialogs. On iOS it also covers app-owned modal popups with native blocking markers. - Use `alert get` for a quick check, and `alert wait ` only when a prompt may appear after async work. Don't use `settings permission` to answer a dialog already on screen; it is for setup before a flow. - `accept` and `dismiss` are sent once on every platform and never replayed, even if the response is lost. On iOS the runner activates the button once, then only observes until the alert disappears, changes, or the deadline expires. A changed alert does not prove a permission was granted; verify the app outcome. -- On the Apple XCUITest runner, `dismiss` activates only a button its decline list recognizes (`Cancel`, `Don't Allow`, `Ask App Not to Track`, and similar English labels). It never taps a button that list does not recognize: unknown, reworded, or fully localized decline copy fails with `error.code: ALERT_BUTTON_NOT_FOUND` and `details.dispatched: no`, tapping nothing, rather than guessing at an affirmative the way a last-button fallback once did ([#3627](https://github.com/callstack/agent-device/issues/3627)). A button choice that finds no button to name answers with that code and `dispatched: no` for `accept` too. To answer a prompt with unrecognized wording, read the labels with `alert get`, then press the button by label through `snapshot` + `press @ref`. +- On the Apple XCUITest runner, `dismiss` activates only a button its decline list recognizes (`Cancel`, `Don't Allow`, `Ask App Not to Track`, and similar English labels), or the close button of a popover whose native marker reads "dismiss popup". It never taps anything else: unknown, reworded, or fully localized decline copy fails with `error.code: ALERT_BUTTON_NOT_FOUND` and `details.dispatched: no`, tapping nothing, rather than guessing at an affirmative ([#3627](https://github.com/callstack/agent-device/issues/3627)). A button choice that finds no button to name answers with that code and `dispatched: no` for `accept` too. To answer a prompt with unrecognized wording, read the labels with `alert get`, then press the button by label through `snapshot` + `press @ref`. The macOS helper's `dismiss` keeps its own first/last-button behavior. - An unreadable or ambiguous capture after the action fails with `error.details.runnerErrorCode: ALERT_CONFIRMATION_UNAVAILABLE`; an expired runner deadline fails with `ALERT_DEADLINE_EXCEEDED`. Neither proves the alert is gone or that nothing happened. Run `alert get` before acting again. - iOS runner refusals carry `error.details.reason`: `runner_busy` means the runner ran nothing (`dispatched: no`); `runner_main_thread_timeout` means the action may still land (`dispatched: unknown`). - If a sheet or permission prompt is visible but `alert` reports none, treat it as app UI: run `snapshot -i -s ""`, then `press @ref`. Android alert support is snapshot-derived, and not every iOS simulator permission surface is a native alert. From 007a313e813adb99b8f089b34458c84c82bf359b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 14:44:45 +0200 Subject: [PATCH 7/8] docs(ios): state the macOS helper dismiss rules accurately and keep narration out of the alert fixtures --- .../AgentDeviceRunner/AgentDeviceRunnerApp.m | 9 ++++----- .../RunnerAlertButtonChoice.swift | 4 +--- .../AgentDeviceRunnerUITests/RunnerTests+Alert.swift | 2 +- website/docs/docs/commands.md | 2 +- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m index 4f4881fde6..420e806ae7 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m @@ -393,11 +393,10 @@ - (void)presentAlertFixtureReplacement:(BOOL)replacement { [self presentViewController:alert animated:NO completion:nil]; } -// The App Tracking Transparency prompt's button order, which is what `alert dismiss` got wrong: -// the affirmative button is last, so a fallback to it answers "Allow" (#3627). The answer label -// says which button the app really took, so a test does not trust the command's own report. -// `--agent-device-alert-affirmative-only` shows the same alert with no decline button at all, -// where `dismiss` must refuse without activating anything. +// Mirrors the App Tracking Transparency prompt's real button order: the affirmative button is +// last (#3627). The answer label records which button the app actually took, so a test asserts +// the app's answer instead of the command's own report. `--agent-device-alert-affirmative-only` +// shows the same alert with no decline button at all. - (void)presentAlertDeclineChoiceFixture { BOOL affirmativeOnly = [NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-alert-affirmative-only"]; diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertButtonChoice.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertButtonChoice.swift index 95c9088a90..32346dd818 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertButtonChoice.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertButtonChoice.swift @@ -8,9 +8,7 @@ import Foundation /// — iOS localizes the App Tracking Transparency prompt's "Allow", consent copy is open-ended, /// and no list closes either set — and answering such a button while reporting `dismissed` is /// exactly the failure #3627 reports. So an unrecognized decline is refused with -/// `ALERT_BUTTON_NOT_FOUND` and no tap rather than guessed at. Unknown wording stays reachable: -/// `alert get` shows the labels, and `snapshot` + `press @ref` names the button instead of -/// inferring it. +/// `ALERT_BUTTON_NOT_FOUND` and no tap rather than guessed at. /// /// The one recognized decline outside the list is a dismiss-popup surface itself: its native /// accessibility marker is the app's own declaration that the surface closes the popover, so diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift index 04cb621c03..af23abf8c5 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift @@ -65,7 +65,7 @@ extension RunnerTests { if action == "accept" || action == "dismiss" { guard let button = chooseAlertButton(alert, action: action) else { // The choice refused before any activation, so this discloses dispatched: no like the - // runner's other pre-run refusals; naming an affirmative button here is what #3627 is about. + // runner's other pre-run refusals. return Response(ok: false, error: ErrorPayload( code: RunnerWireErrorCode.alertButtonNotFound, message: "alert \(action) button not found", diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 713bacf655..c7260dcb4b 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -387,7 +387,7 @@ agent-device alert dismiss - `alert` (same as `alert get`) inspects or handles system alerts on iOS simulator, macOS desktop, and Android permission dialogs. On iOS it also covers app-owned modal popups with native blocking markers. - Use `alert get` for a quick check, and `alert wait ` only when a prompt may appear after async work. Don't use `settings permission` to answer a dialog already on screen; it is for setup before a flow. - `accept` and `dismiss` are sent once on every platform and never replayed, even if the response is lost. On iOS the runner activates the button once, then only observes until the alert disappears, changes, or the deadline expires. A changed alert does not prove a permission was granted; verify the app outcome. -- On the Apple XCUITest runner, `dismiss` activates only a button its decline list recognizes (`Cancel`, `Don't Allow`, `Ask App Not to Track`, and similar English labels), or the close button of a popover whose native marker reads "dismiss popup". It never taps anything else: unknown, reworded, or fully localized decline copy fails with `error.code: ALERT_BUTTON_NOT_FOUND` and `details.dispatched: no`, tapping nothing, rather than guessing at an affirmative ([#3627](https://github.com/callstack/agent-device/issues/3627)). A button choice that finds no button to name answers with that code and `dispatched: no` for `accept` too. To answer a prompt with unrecognized wording, read the labels with `alert get`, then press the button by label through `snapshot` + `press @ref`. The macOS helper's `dismiss` keeps its own first/last-button behavior. +- On the Apple XCUITest runner, `dismiss` activates only a button its decline list recognizes (`Cancel`, `Don't Allow`, `Ask App Not to Track`, and similar English labels), or the close button of a popover whose native marker reads "dismiss popup". It never taps anything else: unknown, reworded, or fully localized decline copy fails with `error.code: ALERT_BUTTON_NOT_FOUND` and `details.dispatched: no`, tapping nothing, rather than guessing at an affirmative ([#3627](https://github.com/callstack/agent-device/issues/3627)). A button choice that finds no button to name answers with that code and `dispatched: no` for `accept` too. To answer a prompt with unrecognized wording, read the labels with `alert get`, then press the button by label through `snapshot` + `press @ref`. The macOS helper's `dismiss` keeps its own rules (the app's cancel-button attribute, its label list, and a last-button fallback). - An unreadable or ambiguous capture after the action fails with `error.details.runnerErrorCode: ALERT_CONFIRMATION_UNAVAILABLE`; an expired runner deadline fails with `ALERT_DEADLINE_EXCEEDED`. Neither proves the alert is gone or that nothing happened. Run `alert get` before acting again. - iOS runner refusals carry `error.details.reason`: `runner_busy` means the runner ran nothing (`dispatched: no`); `runner_main_thread_timeout` means the action may still land (`dispatched: unknown`). - If a sheet or permission prompt is visible but `alert` reports none, treat it as app UI: run `snapshot -i -s ""`, then `press @ref`. Android alert support is snapshot-derived, and not every iOS simulator permission surface is a native alert. From 9203a7a4066fec99f54ef32710350bfa6462f9d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 15:18:47 +0200 Subject: [PATCH 8/8] docs(ios): state the alert button-choice constraint and help refusals precisely --- .../RunnerAlertButtonChoice.swift | 19 +++++++++---------- src/commands/capture/alert.ts | 2 +- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertButtonChoice.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertButtonChoice.swift index 32346dd818..c32f22918e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertButtonChoice.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAlertButtonChoice.swift @@ -3,17 +3,16 @@ import Foundation /// Which of an alert's buttons an `accept` or `dismiss` activates, decided on the button labels /// alone, so the rule is checkable without a screen. /// -/// `dismiss` activates only a button `isDismissButton` recognizes. Any label-based guess at an -/// unrecognized decline can only ever land on a button the accept rule does not recognize either -/// — iOS localizes the App Tracking Transparency prompt's "Allow", consent copy is open-ended, -/// and no list closes either set — and answering such a button while reporting `dismissed` is -/// exactly the failure #3627 reports. So an unrecognized decline is refused with -/// `ALERT_BUTTON_NOT_FOUND` and no tap rather than guessed at. +/// `dismiss` activates only a label `isDismissButton` recognizes. An unknown decline cannot be +/// identified from labels or button position: iOS localizes prompts unevenly, so one prompt can +/// pair a localized decline with a still-English affirmative, and a positional guess would tap +/// that affirmative and record an authorization (#3627). An unrecognized decline is therefore +/// refused with `ALERT_BUTTON_NOT_FOUND` and no tap. /// -/// The one recognized decline outside the list is a dismiss-popup surface itself: its native -/// accessibility marker is the app's own declaration that the surface closes the popover, so -/// `popupMarkerSurface` lets `dismiss` take the last actionable element of that window, as -/// before the #3627 rule existed. A consent prompt can never carry that marker. +/// The one exception is a dismiss-popup surface itself: its native accessibility marker is the +/// app's own declaration that the surface closes the popover, so `popupMarkerSurface` lets +/// `dismiss` take the last actionable element of that window. A consent prompt can never carry +/// that marker. enum RunnerAlertButtonChoice { static func buttonIndex(for labels: [String], action: String, popupMarkerSurface: Bool = false) -> Int? { if action == "accept" { diff --git a/src/commands/capture/alert.ts b/src/commands/capture/alert.ts index 10d725dab6..2ffffb133f 100644 --- a/src/commands/capture/alert.ts +++ b/src/commands/capture/alert.ts @@ -18,7 +18,7 @@ import { AppError } from '@agent-device/kernel/errors'; const ALERT_COMMAND_NAME = 'alert'; const alertCommandDescription = - 'Inspect, wait for, accept, or dismiss a platform alert. Use get before acting when the alert content matters; accept and dismiss change the active alert state. Each iOS XCTest execution activates once, then only observes. On the Apple XCUITest runner, dismiss taps only a button on its decline list (plus a popover marked dismissable) and refuses ALERT_BUTTON_NOT_FOUND rather than guess at unknown wording; press the button by label when a prompt uses it. Inspect again after an unconfirmed action; never assume a timeout means nothing happened.'; + 'Inspect, wait for, accept, or dismiss a platform alert. Use get before acting when the alert content matters; accept and dismiss change the active alert state. Each iOS XCTest execution activates once, then only observes. On the Apple XCUITest runner, dismiss taps only a recognized decline label (Cancel, Don\'t Allow, Ask App Not to Track, and similar) or the close element of a popover whose accessibility marker reads exactly "dismiss popup"; when no button fits the action, accept and dismiss answer ALERT_BUTTON_NOT_FOUND having tapped nothing — read labels with alert get, then press the button by label. Inspect again after an unconfirmed action; never assume a timeout means nothing happened.'; const alertCommandMetadata = defineFieldCommandMetadata( ALERT_COMMAND_NAME,