From b59aac43880c4dc50c8fb1f2d37b760ce430454c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 15 Oct 2026 21:56:19 +0200 Subject: [PATCH 01/12] feat(limrun): serve iOS alert get/wait/accept/dismiss from tree capture Limrun iOS direct sessions refused all four alert legs with unsupported-provider-mode even though the session's element-tree capture already shows the system alert and its touch seam can press its buttons. Issue #3349. Detect the alert from the capture: a system alert arrives as its own blank-label AXApplication root hosting text and pressable buttons, while the app's own UIAlertController rides the app's named root, so app-owned dialogs stay app UI. Accept and dismiss press exactly the button whose label states the asked-for choice and confirm the alert is gone; an ambiguous choice is reported, never guessed, and dismiss never authorizes tracking. The four legs bind through the same catalog and facts the Android tree-based legs already use. --- .../src/alert-detection.fixtures.ts | 246 ++++++++++++++++++ .../src/alert-detection.test.ts | 174 +++++++++++++ .../provider-limrun/src/alert-detection.ts | 173 ++++++++++++ packages/provider-limrun/src/alert.ts | 184 +++++++++++++ .../src/interaction-operations.test.ts | 17 +- .../src/interaction-operations.ts | 13 +- .../src/ios-interactor-alert.test.ts | 180 +++++++++++++ packages/provider-limrun/src/ios.ts | 26 ++ 8 files changed, 997 insertions(+), 16 deletions(-) create mode 100644 packages/provider-limrun/src/alert-detection.fixtures.ts create mode 100644 packages/provider-limrun/src/alert-detection.test.ts create mode 100644 packages/provider-limrun/src/alert-detection.ts create mode 100644 packages/provider-limrun/src/alert.ts create mode 100644 packages/provider-limrun/src/ios-interactor-alert.test.ts diff --git a/packages/provider-limrun/src/alert-detection.fixtures.ts b/packages/provider-limrun/src/alert-detection.fixtures.ts new file mode 100644 index 0000000000..2f0cf4c356 --- /dev/null +++ b/packages/provider-limrun/src/alert-detection.fixtures.ts @@ -0,0 +1,246 @@ +import type { IosTreeNode } from './snapshot.ts'; + +/** + * Trees shaped from live Limrun iOS instances (issue #3349 evidence). The shapes are what the + * detection rules are actually tested against, so each keeps the provider's real fields: + * `role`/`type` pairs, `AXLabel`, per-node `pid`, and a `frame` dictionary. + * + * The captured discriminator: a system alert arrives as its own `AXApplication` root whose + * `AXLabel` is blank and whose `pid` (and every child's `pid`) is the hosting process, not the + * app's. The app's own `UIAlertController` arrives inside the app's root with the app's `pid`. + */ + +/** The app under test, as the app's own root reports it. */ +export const LIMRUN_APP_PID = 59_012; +/** The process hosting system alerts (SpringBoard's alert agent on the instance). */ +export const LIMRUN_ALERT_HOST_PID = 24_261; +/** SpringBoard, which contributes the status-bar nodes that ride along on app screens. */ +export const LIMRUN_SPRINGBOARD_PID = 33_887; + +export const LIMRUN_ALERT_SCREEN: Readonly<{ width: number; height: number }> = Object.freeze({ + width: 402, + height: 874, +}); + +function alertRoot(children: IosTreeNode[]): IosTreeNode { + return { + type: 'Application', + role: 'AXApplication', + AXLabel: ' ', + pid: LIMRUN_ALERT_HOST_PID, + frame: { x: 0, y: 0, ...LIMRUN_ALERT_SCREEN }, + traits: ['None'], + children, + }; +} + +/** Positive control: the live ATT prompt tree — its own root, host pid, two texts, two buttons. */ +export function limrunAttAlertTree(): IosTreeNode { + return alertRoot([ + { + type: 'StaticText', + role: 'AXStaticText', + AXLabel: 'Allow “ATTProbe” to track your activity across other companies’ apps and websites?', + pid: LIMRUN_ALERT_HOST_PID, + frame: { x: 71, y: 381.33, width: 260, height: 64.33 }, + traits: ['Scrollable', 'StaticText'], + }, + { + type: 'StaticText', + role: 'AXStaticText', + AXLabel: 'This build asks for tracking authorization once, for automation testing.', + pid: LIMRUN_ALERT_HOST_PID, + frame: { x: 71, y: 453.33, width: 260, height: 38 }, + traits: ['Scrollable', 'StaticText'], + }, + { + type: 'Button', + role: 'AXButton', + AXLabel: 'Ask App Not to Track', + pid: LIMRUN_ALERT_HOST_PID, + enabled: true, + frame: { x: 57, y: 511.67, width: 288, height: 48 }, + traits: ['Button', 'Scrollable'], + }, + { + type: 'Button', + role: 'AXButton', + AXLabel: 'Allow', + pid: LIMRUN_ALERT_HOST_PID, + enabled: true, + frame: { x: 57, y: 567.67, width: 288, height: 48 }, + traits: ['Button', 'Scrollable'], + }, + ]); +} + +/** The same alert with its buttons removed: a text sheet is not an actionable alert. */ +export function limrunAlertTreeWithoutButtons(): IosTreeNode { + const root = limrunAttAlertTree(); + return { ...root, children: (root.children ?? []).filter((n) => n.role !== 'AXButton') }; +} + +/** An alert whose only choices are ambiguous: no label classifies as accept or dismiss. */ +export function limrunAlertAmbiguousButtonsTree(): IosTreeNode { + const root = limrunAttAlertTree(); + return { + ...root, + children: [ + ...(root.children ?? []).slice(0, 2), + { + type: 'Button', + role: 'AXButton', + AXLabel: 'Maybe Later', + pid: LIMRUN_ALERT_HOST_PID, + enabled: true, + frame: { x: 57, y: 511.67, width: 288, height: 48 }, + traits: ['Button', 'Scrollable'], + }, + { + type: 'Button', + role: 'AXButton', + AXLabel: 'More Info', + pid: LIMRUN_ALERT_HOST_PID, + enabled: true, + frame: { x: 57, y: 567.67, width: 288, height: 48 }, + traits: ['Button', 'Scrollable'], + }, + ], + }; +} + +/** The live app home screen: app root with the app pid, plus SpringBoard status-bar nodes. */ +export function limrunAppScreenTree(): IosTreeNode { + return { + type: 'Application', + role: 'AXApplication', + AXLabel: 'ATTProbe', + pid: LIMRUN_APP_PID, + frame: { x: 0, y: 0, ...LIMRUN_ALERT_SCREEN }, + traits: ['None'], + children: [ + { + type: 'StaticText', + role: 'AXStaticText', + AXLabel: 'ATT Probe', + pid: LIMRUN_APP_PID, + frame: { x: 161, y: 418.5, width: 80, height: 20.33 }, + }, + { + type: 'Button', + role: 'AXButton', + AXLabel: 'Request tracking', + AXUniqueId: 'request-tracking', + pid: LIMRUN_APP_PID, + enabled: true, + frame: { x: 136.67, y: 441, width: 129, height: 20.33 }, + }, + { + type: 'Button', + role: 'AXButton', + AXLabel: 'Present app alert', + AXUniqueId: 'present-app-alert', + pid: LIMRUN_APP_PID, + enabled: true, + frame: { x: 136, y: 463.33, width: 130.33, height: 20.33 }, + }, + { + type: 'StaticText', + role: 'AXStaticText', + AXLabel: '21:15', + pid: LIMRUN_SPRINGBOARD_PID, + frame: { x: 53, y: 22.33, width: 42, height: 20.33 }, + }, + { + type: 'GenericElement', + role: 'AXGenericElement', + AXLabel: 'Cellular', + pid: LIMRUN_SPRINGBOARD_PID, + frame: { x: 288.33, y: 26.33, width: 19.33, height: 12.33 }, + }, + { + type: 'GenericElement', + role: 'AXGenericElement', + AXLabel: '100% battery power', + pid: LIMRUN_SPRINGBOARD_PID, + frame: { x: 339.33, y: 26, width: 27.33, height: 13 }, + }, + ], + }; +} + +/** + * Negative control: the live app-owned `UIAlertController` ("Unsaved changes" / Cancel / + * Discard). It shares the flat text+button shape of a system alert but rides the app's root + * with the app's pid, so the alert legs must refuse to treat it as one. + */ +export function limrunAppAlertDialogTree(): IosTreeNode { + const home = limrunAppScreenTree(); + return { + ...home, + children: [ + { + type: 'StaticText', + role: 'AXStaticText', + AXLabel: 'Unsaved changes', + pid: LIMRUN_APP_PID, + frame: { x: 82, y: 408.67, width: 238, height: 20.33 }, + }, + { + type: 'StaticText', + role: 'AXStaticText', + AXLabel: 'Leave without saving?', + pid: LIMRUN_APP_PID, + frame: { x: 82, y: 432.67, width: 238, height: 15.67 }, + }, + { + type: 'Button', + role: 'AXButton', + AXLabel: 'Cancel', + pid: LIMRUN_APP_PID, + enabled: true, + frame: { x: 66, y: 469.33, width: 135, height: 44 }, + traits: ['Button', 'Scrollable'], + }, + { + type: 'Button', + role: 'AXButton', + AXLabel: 'Discard', + pid: LIMRUN_APP_PID, + enabled: true, + frame: { x: 201.33, y: 469.33, width: 134.67, height: 44 }, + traits: ['Button', 'Scrollable'], + }, + ...(home.children ?? []).slice(3), + ], + }; +} + +/** SpringBoard's home screen: a blank-label root, but full of launch icons rather than a dialog. */ +export function limrunHomeScreenTree(): IosTreeNode { + return alertRoot([ + { + type: 'Button', + role: 'AXButton', + AXLabel: 'Maps', + pid: LIMRUN_SPRINGBOARD_PID, + enabled: true, + frame: { x: 24.33, y: 88, width: 168.33, height: 191 }, + traits: ['Scrollable', 'UpdatesFrequently', 'LaunchIcon'], + }, + { + type: 'Button', + role: 'AXButton', + AXLabel: 'Calendar', + pid: LIMRUN_SPRINGBOARD_PID, + enabled: true, + frame: { x: 209.67, y: 88, width: 168.33, height: 191 }, + traits: ['Scrollable', 'UpdatesFrequently', 'LaunchIcon'], + }, + ]); +} + +/** A capture with no alert: the app screen with the ATT alert grafted on as a sibling root. */ +export function limrunAlertOverlayOnAppScreenTree(): IosTreeNode[] { + return [limrunAppScreenTree(), limrunAttAlertTree()]; +} diff --git a/packages/provider-limrun/src/alert-detection.test.ts b/packages/provider-limrun/src/alert-detection.test.ts new file mode 100644 index 0000000000..a67dd88ddc --- /dev/null +++ b/packages/provider-limrun/src/alert-detection.test.ts @@ -0,0 +1,174 @@ +import { describe, expect, it } from 'vitest'; +import { + chooseLimrunAlertButton, + findLimrunAlertCandidate, + isLimrunAlertHostRoot, +} from './alert-detection.ts'; +import { + limrunAlertAmbiguousButtonsTree, + limrunAlertOverlayOnAppScreenTree, + limrunAlertTreeWithoutButtons, + limrunAppAlertDialogTree, + limrunAppScreenTree, + limrunAttAlertTree, + limrunHomeScreenTree, +} from './alert-detection.fixtures.ts'; + +function alertTreeWithButtons(...labels: string[]) { + const root = limrunAttAlertTree(); + return { + ...root, + children: [ + ...(root.children ?? []).slice(0, 2), + ...labels.map((label, index) => ({ + type: 'Button', + role: 'AXButton', + AXLabel: label, + pid: root.pid, + enabled: true, + frame: { x: 57, y: 511.67 + index * 56, width: 288, height: 48 }, + traits: ['Button', 'Scrollable'], + })), + ], + }; +} + +describe('isLimrunAlertHostRoot', () => { + it('names the blank-label application root a system alert arrives on', () => { + expect(isLimrunAlertHostRoot(limrunAttAlertTree())).toBe(true); + }); + + it('refuses an app-owned root: the app names its own application', () => { + expect(isLimrunAlertHostRoot(limrunAppScreenTree())).toBe(false); + expect(isLimrunAlertHostRoot(limrunAppAlertDialogTree())).toBe(false); + }); + + it('refuses roots that are not applications', () => { + expect(isLimrunAlertHostRoot({ role: 'AXGroup', AXLabel: ' ' })).toBe(false); + expect(isLimrunAlertHostRoot({ role: 'AXButton', AXLabel: ' ' })).toBe(false); + }); +}); + +describe('findLimrunAlertCandidate', () => { + it('reads the live ATT alert: title, message, and both buttons in tree order', () => { + const candidate = findLimrunAlertCandidate(limrunAttAlertTree()); + expect(candidate).not.toBeNull(); + expect(candidate?.alert.title).toBe( + 'Allow “ATTProbe” to track your activity across other companies’ apps and websites?', + ); + expect(candidate?.alert.message).toBe( + 'This build asks for tracking authorization once, for automation testing.', + ); + expect(candidate?.alert.buttons).toEqual(['Ask App Not to Track', 'Allow']); + expect(candidate?.alert.platform).toBe('ios'); + expect(candidate?.alert.source).toBe('permission'); + expect(candidate?.buttons.map((button) => button.role)).toEqual(['dismiss', 'accept']); + // The tap point is the live button frame's center, rounded the way `centerOfRect` rounds. + expect(candidate?.buttons[1]).toEqual({ + label: 'Allow', + x: 201, + y: 592, + role: 'accept', + }); + }); + + it('finds the alert when the app screen is the first root of the capture', () => { + const candidate = findLimrunAlertCandidate(limrunAlertOverlayOnAppScreenTree()); + expect(candidate?.alert.buttons).toEqual(['Ask App Not to Track', 'Allow']); + }); + + it('refuses the live app-owned UIAlertController: a named root is app UI, not a host sheet', () => { + expect(findLimrunAlertCandidate(limrunAppAlertDialogTree())).toBeNull(); + }); + + it('refuses a plain app screen', () => { + expect(findLimrunAlertCandidate(limrunAppScreenTree())).toBeNull(); + }); + + it("refuses SpringBoard's home screen: launch icons are not a dialog", () => { + expect(findLimrunAlertCandidate(limrunHomeScreenTree())).toBeNull(); + }); + + it('refuses an alert-shaped sheet with no actionable button', () => { + expect(findLimrunAlertCandidate(limrunAlertTreeWithoutButtons())).toBeNull(); + }); + + it('reports a permission alert from its button vocabulary and a plain one as a system dialog', () => { + expect(findLimrunAlertCandidate(limrunAttAlertTree())?.alert.source).toBe('permission'); + expect(findLimrunAlertCandidate(alertTreeWithButtons('No', 'Yes'))?.alert.source).toBe( + 'system-dialog', + ); + }); +}); + +describe('chooseLimrunAlertButton', () => { + it('accepts the affirmative and dismisses the denying choice of the ATT prompt', () => { + const buttons = findLimrunAlertCandidate(limrunAttAlertTree())!.buttons; + expect(chooseLimrunAlertButton(buttons, 'accept')?.label).toBe('Allow'); + expect(chooseLimrunAlertButton(buttons, 'dismiss')?.label).toBe('Ask App Not to Track'); + }); + + it('fails an ambiguous accept closed when no button classifies as affirmative', () => { + const buttons = findLimrunAlertCandidate(limrunAlertAmbiguousButtonsTree())!.buttons; + expect(chooseLimrunAlertButton(buttons, 'accept')).toBeNull(); + }); + + it('fails an ambiguous dismiss closed when no button classifies as denying', () => { + const buttons = findLimrunAlertCandidate(limrunAlertAmbiguousButtonsTree())!.buttons; + expect(chooseLimrunAlertButton(buttons, 'dismiss')).toBeNull(); + }); + + it('refuses an action that would press the wrong choice of a one-button alert', () => { + const allowOnly = findLimrunAlertCandidate(alertTreeWithButtons('Allow'))!.buttons; + const denyOnly = findLimrunAlertCandidate( + alertTreeWithButtons('Ask App Not to Track'), + )!.buttons; + expect(chooseLimrunAlertButton(allowOnly, 'accept')?.label).toBe('Allow'); + expect(chooseLimrunAlertButton(allowOnly, 'dismiss')).toBeNull(); + expect(chooseLimrunAlertButton(denyOnly, 'dismiss')?.label).toBe('Ask App Not to Track'); + expect(chooseLimrunAlertButton(denyOnly, 'accept')).toBeNull(); + }); + + it('classifies the classic iOS pair, the curly-apostrophe denial, and an open-settings prompt', () => { + const classic = findLimrunAlertCandidate(alertTreeWithButtons("Don't Allow", 'OK'))!.buttons; + expect(chooseLimrunAlertButton(classic, 'accept')?.label).toBe('OK'); + expect(chooseLimrunAlertButton(classic, 'dismiss')?.label).toBe("Don't Allow"); + const curly = findLimrunAlertCandidate(alertTreeWithButtons('Don’t Allow', 'Allow'))!.buttons; + expect(chooseLimrunAlertButton(curly, 'dismiss')?.label).toBe('Don’t Allow'); + const settings = findLimrunAlertCandidate( + alertTreeWithButtons('Not Now', 'Open Settings'), + )!.buttons; + expect(chooseLimrunAlertButton(settings, 'accept')?.label).toBe('Open Settings'); + expect(chooseLimrunAlertButton(settings, 'dismiss')?.label).toBe('Not Now'); + }); + + it('refuses the candidate when no button is enabled with a frame to tap', () => { + const root = limrunAttAlertTree(); + const crippled = { + ...root, + children: [ + ...(root.children ?? []).slice(0, 2), + { + type: 'Button', + role: 'AXButton', + AXLabel: 'Allow', + pid: root.pid, + enabled: false, + frame: { x: 57, y: 511.67, width: 288, height: 48 }, + traits: ['Button'], + }, + { + type: 'Button', + role: 'AXButton', + AXLabel: 'Ask App Not to Track', + pid: root.pid, + enabled: true, + traits: ['Button'], + }, + ], + }; + // The only button that could deny has no frame to tap, and the framed one is disabled: + // nothing is actionable, so the whole candidate is refused rather than half-served. + expect(findLimrunAlertCandidate(crippled)).toBeNull(); + }); +}); diff --git a/packages/provider-limrun/src/alert-detection.ts b/packages/provider-limrun/src/alert-detection.ts new file mode 100644 index 0000000000..7b6b8061c6 --- /dev/null +++ b/packages/provider-limrun/src/alert-detection.ts @@ -0,0 +1,173 @@ +import type { AlertInfo, AlertSource } from '@agent-device/contracts/alert-contract'; +import { centerOfRect, type Rect } from '@agent-device/kernel/snapshot'; +import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; +import { readIosNodeChildren, readIosNodeRect, type IosTreeNode } from './snapshot.ts'; + +/** + * System-alert detection over a Limrun iOS element tree (issue #3349). A Limrun iOS direct + * session has no XCUITest runner, but its element tree does carry the system alert: on a live + * instance the ATT prompt arrives as its own `AXApplication` root whose `AXLabel` is blank and + * whose content nodes carry the hosting process's `pid` — not the app's. The app's own + * `UIAlertController`, by contrast, rides inside the app's root, labeled with the app name and + * carrying the app's pid. The rules below encode exactly that observed split; they are + * conservative in both directions and every shape here is pinned to the live captures. + */ + +type LimrunAlertButtonRole = 'accept' | 'dismiss' | 'neutral'; + +export type LimrunAlertButton = Readonly<{ + label: string; + x: number; + y: number; + role: LimrunAlertButtonRole; +}>; + +/** Only the two host-sheet sources exist on this surface: no node names a package to attribute. */ +type LimrunAlertSource = Extract; + +export type LimrunAlertInfo = AlertInfo & { + buttons: string[]; + platform: 'ios'; + source: LimrunAlertSource; +}; + +export type LimrunAlertCandidate = Readonly<{ + alert: LimrunAlertInfo; + buttons: LimrunAlertButton[]; +}>; + +const ACCEPT_LABEL_PATTERN = + /^(?:ok|allow|allow all|always allow|yes|continue|save|confirm|turn on|open settings)$/i; +const DISMISS_LABEL_PATTERN = + /^(?:cancel|deny|don.?t allow|don’t allow|don.t allow|not now|no|dismiss|close|later|skip|ask app not to track)$/i; +/** The subset that states a grant/deny choice; a plain Yes/No dialog is not a permission prompt. */ +const PERMISSION_LABEL_PATTERN = + /^(?:allow.*|always allow|deny|don.?t allow|don’t allow|don.t allow|ask app not to track|turn on|open settings)$/i; + +function nodeLabel(node: IosTreeNode): string { + const label = node.AXLabel ?? node.label; + return typeof label === 'string' ? label.trim() : ''; +} + +/** + * The application root a system alert arrives on: the blank-label `AXApplication`. An app names + * its own root, so a labeled application is the app's scene — including when it hosts an + * in-app `UIAlertController` — and never a system host. SpringBoard also reports a blank label; + * its screens are refused downstream because they carry no dialog-shaped content. + */ +export function isLimrunAlertHostRoot(node: IosTreeNode): boolean { + if (node.role !== 'AXApplication' && node.type !== 'Application') return false; + return nodeLabel(node).length === 0; +} + +/** + * The actionable alert on this capture, or null. A host root qualifies only when its content is + * dialog-shaped: a button that can actually be pressed (enabled, with a complete rect) and at + * least one piece of text or another button beside it. That refusals both an app dialog (wrong + * root) and SpringBoard's home screen (blank root, launch icons instead of a dialog). + */ +export function findLimrunAlertCandidate( + tree: IosTreeNode | IosTreeNode[], +): LimrunAlertCandidate | null { + for (const root of Array.isArray(tree) ? tree : [tree]) { + if (!isLimrunAlertHostRoot(root)) continue; + const candidate = readAlertCandidateFromHostRoot(root); + if (candidate) return candidate; + } + return null; +} + +function readAlertCandidateFromHostRoot(root: IosTreeNode): LimrunAlertCandidate | null { + const descendants = collectDescendants(root); + const buttons = findAlertButtons(descendants); + if (buttons.length === 0) return null; + const texts = descendants + .filter((node) => isStaticTextNode(node) && !isButtonNode(node)) + .map(nodeLabel) + .filter((text) => text.length > 0); + // A dialog says something: SpringBoard's blank-label home screen offers launch icons, not a + // message with a choice, and stays unreported. + if (texts.length === 0) return null; + const [title, ...rest] = texts; + const alert: LimrunAlertInfo = { + ...(title ? { title } : {}), + ...(rest.length > 0 ? { message: [...new Set(rest)].join('\n') } : {}), + buttons: buttons.map((button) => button.label), + platform: 'ios', + source: classifyAlertSource(buttons), + }; + return { alert, buttons }; +} + +function collectDescendants(root: IosTreeNode): IosTreeNode[] { + const nodes: IosTreeNode[] = []; + const visit = (node: IosTreeNode) => { + for (const child of readIosNodeChildren(node)) { + nodes.push(child); + visit(child); + } + }; + visit(root); + return nodes; +} + +function isButtonNode(node: IosTreeNode): boolean { + return node.role === 'AXButton' || node.type === 'Button'; +} + +function isStaticTextNode(node: IosTreeNode): boolean { + return node.role === 'AXStaticText' || node.type === 'StaticText'; +} + +/** + * Pressable buttons only: a disabled button is not a choice the alert offers right now, and a + * button without a complete rect has no point the touch seam could aim at. + */ +function findAlertButtons(nodes: IosTreeNode[]): LimrunAlertButton[] { + const seen = new Set(); + const buttons: LimrunAlertButton[] = []; + for (const node of nodes) { + if (!isButtonNode(node) || node.enabled === false) continue; + const label = nodeLabel(node); + if (!label || seen.has(label.toLowerCase())) continue; + const rect = readIosNodeRect(node); + if (!isPressableRect(rect)) continue; + seen.add(label.toLowerCase()); + const point = centerOfRect(rect); + buttons.push({ label, x: point.x, y: point.y, role: classifyAlertButton(label) }); + } + return buttons; +} + +function isPressableRect(rect: Rect | undefined): rect is Rect { + return isPositiveFiniteRect(rect); +} + +function classifyAlertButton(label: string): LimrunAlertButtonRole { + const trimmed = label.trim(); + if (ACCEPT_LABEL_PATTERN.test(trimmed)) return 'accept'; + if (DISMISS_LABEL_PATTERN.test(trimmed)) return 'dismiss'; + return 'neutral'; +} + +/** A button vocabulary naming a grant/deny choice makes this a permission prompt. */ +function classifyAlertSource(buttons: readonly LimrunAlertButton[]): LimrunAlertSource { + return buttons.some((button) => PERMISSION_LABEL_PATTERN.test(button.label.trim())) + ? 'permission' + : 'system-dialog'; +} + +/** + * The button one action may press, or null when pressing any visible choice would guess. + * Deliberately stricter than Android's: with no `android:id/button1`-style identity to fall back + * on, only a button whose own label states the asked-for choice may be pressed — an accept must + * never press a denial, and an alert whose choices are all neutral is refused for both actions + * rather than resolved by position. + */ +export function chooseLimrunAlertButton( + buttons: readonly LimrunAlertButton[], + action: 'accept' | 'dismiss', +): LimrunAlertButton | null { + const wanted = action === 'accept' ? 'accept' : 'dismiss'; + return buttons.find((button) => button.role === wanted) ?? null; +} diff --git a/packages/provider-limrun/src/alert.ts b/packages/provider-limrun/src/alert.ts new file mode 100644 index 0000000000..7d133f9fcd --- /dev/null +++ b/packages/provider-limrun/src/alert.ts @@ -0,0 +1,184 @@ +import { + ALERT_ACTION_RETRY_MS, + ALERT_POLL_INTERVAL_MS, + DEFAULT_ALERT_TIMEOUT_MS, +} from '@agent-device/contracts/alert-contract'; +import { AppError } from '@agent-device/kernel/errors'; +import { successText } from '@agent-device/kernel/success-text'; +import { setTimeout as sleep } from 'node:timers/promises'; +import { + chooseLimrunAlertButton, + findLimrunAlertCandidate, + type LimrunAlertCandidate, + type LimrunAlertInfo, +} from './alert-detection.ts'; +import type { IosTreeNode } from './snapshot.ts'; + +/** + * The four alert legs on a Limrun iOS direct session (issue #3349). There is no XCUITest runner + * to ask, so the legs observe through the session's own element-tree capture and act through the + * same point-touch seam every other touch operation rides. The response records match the + * Android tree-based legs' `alertStatus`/`alertWait`/`alertHandled` kinds, which is the shape the + * daemon composes CLI and SDK output around. + */ + +/** The client methods the legs use; declared here so the legs are testable without a live instance. */ +export type LimrunIosAlertClient = Readonly<{ + elementTree: () => Promise; + tap: (x: number, y: number) => Promise; +}>; + +export type LimrunIosAlertResult = + | { + kind: 'alertStatus'; + platform: 'ios'; + action: 'get'; + alert: LimrunAlertInfo | null; + message?: string; + } + | { + kind: 'alertWait'; + platform: 'ios'; + action: 'wait'; + alert: LimrunAlertInfo; + waitedMs: number; + message?: string; + } + | { + kind: 'alertHandled'; + platform: 'ios'; + action: 'accept' | 'dismiss'; + handled: true; + alert: LimrunAlertInfo; + button: string; + coordinates: { x: number; y: number }; + message?: string; + }; + +export type LimrunIosAlertAction = 'get' | 'wait' | 'accept' | 'dismiss'; + +const ALERT_NOT_FOUND_HINT = + 'If the sheet is visible in snapshot but alert reports no alert, it is likely app-owned UI. Use snapshot -i and press the visible label/ref.'; + +export async function handleLimrunIosAlert( + client: LimrunIosAlertClient, + action: LimrunIosAlertAction, + options?: Readonly<{ timeoutMs?: number }>, +): Promise { + if (action === 'get') { + const candidate = await readLimrunAlertCandidate(client); + return { + kind: 'alertStatus', + platform: 'ios', + action: 'get', + alert: candidate?.alert ?? null, + ...(candidate ? successText('Alert visible') : successText('No alert visible')), + }; + } + if (action === 'wait') { + const start = Date.now(); + const candidate = await pollLimrunAlertCandidate( + client, + options?.timeoutMs ?? DEFAULT_ALERT_TIMEOUT_MS, + ); + if (!candidate) { + throw new AppError('COMMAND_FAILED', 'alert wait timed out'); + } + return { + kind: 'alertWait', + platform: 'ios', + action: 'wait', + alert: candidate.alert, + waitedMs: Date.now() - start, + ...successText('Alert visible'), + }; + } + return await actOnLimrunIosAlert(client, action); +} + +/** + * Accept and dismiss share the Android legs' contract: look within the action budget, press + * exactly the one button the action's label vocabulary names, then confirm the alert is gone. + * `dismiss` never authorizes anything — an alert whose denying choice cannot be named is + * reported as the ambiguity it is instead of pressing something. + */ +async function actOnLimrunIosAlert( + client: LimrunIosAlertClient, + action: 'accept' | 'dismiss', +): Promise { + const candidate = await pollLimrunAlertCandidate(client, ALERT_ACTION_RETRY_MS); + if (!candidate) { + throw new AppError('COMMAND_FAILED', 'alert not found', { hint: ALERT_NOT_FOUND_HINT }); + } + const button = chooseLimrunAlertButton(candidate.buttons, action); + if (!button) { + throw new AppError( + 'COMMAND_FAILED', + `alert ${action} found an alert but no button that clearly matches the action`, + { + alert: candidate.alert, + hint: 'Inspect alert get --json for the visible buttons, then press the choice you want by label/ref.', + }, + ); + } + await client.tap(button.x, button.y); + await confirmLimrunAlertDismissed(client, candidate, action); + return { + kind: 'alertHandled', + platform: 'ios', + action, + handled: true, + alert: candidate.alert, + button: button.label, + coordinates: { x: button.x, y: button.y }, + ...successText(`Alert ${action}ed`), + }; +} + +/** + * `alert accept|dismiss` means the dialog is gone, not that a tap was sent: the next command + * reads the app, and an alert still up would be read as the app's state. A different alert + * taking its place counts as dismissed, the same as the Android legs accept. + */ +async function confirmLimrunAlertDismissed( + client: LimrunIosAlertClient, + dismissed: LimrunAlertCandidate, + action: 'accept' | 'dismiss', +): Promise { + const start = Date.now(); + for (;;) { + const current = await readLimrunAlertCandidate(client); + if (!current || !sameLimrunAlert(current.alert, dismissed.alert)) return; + if (Date.now() - start >= ALERT_ACTION_RETRY_MS) { + throw new AppError('COMMAND_FAILED', `alert ${action} did not dismiss the visible alert`, { + alert: dismissed.alert, + hint: 'The alert button was pressed but the dialog is still visible. Inspect alert get --json, then press the visible button by label/ref or retry.', + }); + } + await sleep(ALERT_POLL_INTERVAL_MS); + } +} + +function sameLimrunAlert(left: LimrunAlertInfo, right: LimrunAlertInfo): boolean { + return left.title === right.title && left.buttons.join('\u0000') === right.buttons.join('\u0000'); +} + +async function pollLimrunAlertCandidate( + client: LimrunIosAlertClient, + timeoutMs: number, +): Promise { + const start = Date.now(); + for (;;) { + const candidate = await readLimrunAlertCandidate(client); + if (candidate) return candidate; + if (Date.now() - start >= timeoutMs) return null; + await sleep(ALERT_POLL_INTERVAL_MS); + } +} + +async function readLimrunAlertCandidate( + client: LimrunIosAlertClient, +): Promise { + const tree = (await client.elementTree()) as IosTreeNode | IosTreeNode[]; + return findLimrunAlertCandidate(tree); +} diff --git a/packages/provider-limrun/src/interaction-operations.test.ts b/packages/provider-limrun/src/interaction-operations.test.ts index 02b387bb65..5f88785bea 100644 --- a/packages/provider-limrun/src/interaction-operations.test.ts +++ b/packages/provider-limrun/src/interaction-operations.test.ts @@ -125,19 +125,20 @@ test('the iOS leg serves the simctl settings only and reads nothing back', () => ).toEqual(liveSessionUnavailable); }); -// R59: same split again — the Android leg reuses the local family's interactor, and the iOS -// direct session has no XCUITest runner to read a sheet from. -test('alert legs ride the Android interactor and are refused on the iOS leg', () => { +// R59 split, then #3349 closed the iOS half: the Android leg reuses the local family's +// interactor, and the iOS direct session reads system alerts from its element-tree capture and +// presses their buttons through its touch seam, so a live session serves all four legs on both +// platforms. +test('alert legs are available on both legs of a live session and refused without one', () => { for (const leg of ['readAlert', 'awaitAlert', 'acceptAlert', 'dismissAlert'] as const) { expect(limrunAlertOperationFacts(androidMobileDevice)[leg]).toEqual({ available: true }); - expect(limrunAlertOperationFacts(iosDevice)[leg]).toEqual({ - available: false, - reason: 'unsupported-provider-mode', - hint: 'Limrun iOS direct sessions do not expose alert inspection yet.', - }); + expect(limrunAlertOperationFacts(iosDevice)[leg]).toEqual({ available: true }); expect(limrunAlertOperationFacts(androidMobileDevice, liveSessionUnavailable)[leg]).toEqual( liveSessionUnavailable, ); + expect(limrunAlertOperationFacts(iosDevice, liveSessionUnavailable)[leg]).toEqual( + liveSessionUnavailable, + ); } }); diff --git a/packages/provider-limrun/src/interaction-operations.ts b/packages/provider-limrun/src/interaction-operations.ts index 48b5c66519..17867bd811 100644 --- a/packages/provider-limrun/src/interaction-operations.ts +++ b/packages/provider-limrun/src/interaction-operations.ts @@ -268,22 +268,19 @@ export function limrunClipboardOperationFacts( }); } -const alertUnavailableIos = unavailableFact( - 'unsupported-provider-mode', - 'Limrun iOS direct sessions do not expose alert inspection yet.', -); - /** * `alert` splits like every other interaction leaf: the Android leg rides the local family's own * interactor factory, which reads the same accessibility dump it always did; the iOS direct - * session has no XCUITest runner to read a sheet from, so all four legs refuse together. + * session reads system alerts out of the same element-tree capture its snapshot rides and presses + * their buttons through the same touch seam (issue #3349), so all four legs are available on + * both legs of a live session. */ export function limrunAlertOperationFacts( device: DeviceInfo, liveSessionUnavailable?: RuntimeOperationUnavailability, ) { - const cell = - liveSessionUnavailable ?? (device.platform === 'android' ? available : alertUnavailableIos); + void device; + const cell = liveSessionUnavailable ?? available; return Object.freeze({ ...alertRuntimeOperationFacts({ read: cell, wait: cell, accept: cell, dismiss: cell }), }); diff --git a/packages/provider-limrun/src/ios-interactor-alert.test.ts b/packages/provider-limrun/src/ios-interactor-alert.test.ts new file mode 100644 index 0000000000..649a286271 --- /dev/null +++ b/packages/provider-limrun/src/ios-interactor-alert.test.ts @@ -0,0 +1,180 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + limrunAlertTreeWithoutButtons, + limrunAppAlertDialogTree, + limrunAppScreenTree, + limrunAttAlertTree, +} from './alert-detection.fixtures.ts'; +import { createLimrunIosInteractor, type LimrunIosSession } from './ios.ts'; +import type { IosTreeNode } from './snapshot.ts'; + +/** + * The four legs as the interactor exposes them: observation is one `elementTree()` read per + * probe, action is a `tap()` at the chosen button's center. Trees are the live Limrun captures. + */ +function sessionWithTrees(trees: Array) { + const elementTree = vi.fn(async () => { + const next = trees.length > 1 ? trees.shift() : trees[0]; + return [next]; + }); + const client = { elementTree, tap: vi.fn(async () => {}) }; + const session = { + platform: 'ios', + instanceId: 'limrun-alert-instance', + client, + } as unknown as LimrunIosSession; + return { interactor: createLimrunIosInteractor(session), client }; +} + +describe('readAlert', () => { + it('reports the live ATT alert with its buttons', async () => { + const { interactor } = sessionWithTrees([limrunAttAlertTree()]); + + const result = await interactor.readAlert!(); + + expect(result.kind).toBe('alertStatus'); + expect(result.alert).toMatchObject({ + platform: 'ios', + source: 'permission', + buttons: ['Ask App Not to Track', 'Allow'], + }); + }); + + it('reports no alert for the live app-owned dialog rather than claiming it', async () => { + const { interactor } = sessionWithTrees([limrunAppAlertDialogTree()]); + + const result = await interactor.readAlert!(); + + expect(result.kind).toBe('alertStatus'); + expect(result.alert).toBeNull(); + }); +}); + +describe('awaitAlert', () => { + it('returns as soon as a capture shows the alert', async () => { + const { interactor } = sessionWithTrees([limrunAppScreenTree(), limrunAttAlertTree()]); + + const result = await interactor.awaitAlert!({ timeoutMs: 5_000 }); + + expect(result.kind).toBe('alertWait'); + expect(result.alert).toMatchObject({ buttons: ['Ask App Not to Track', 'Allow'] }); + }); + + it('times out while no capture ever shows an alert', async () => { + const { interactor } = sessionWithTrees([limrunAppScreenTree()]); + + await expect(interactor.awaitAlert!({ timeoutMs: 1 })).rejects.toThrow('alert wait timed out'); + }); +}); + +describe('acceptAlert', () => { + it('taps the affirmative choice and reports it, then re-reads to confirm', async () => { + const { interactor, client } = sessionWithTrees([ + limrunAttAlertTree(), + limrunAppScreenTree(), + limrunAppScreenTree(), + ]); + + const result = await interactor.acceptAlert!(); + + expect(result).toMatchObject({ + kind: 'alertHandled', + action: 'accept', + handled: true, + button: 'Allow', + // `centerOfRect` rounds: the live Allow frame centers at (201, 592). + coordinates: { x: 201, y: 592 }, + }); + expect(client.tap).toHaveBeenCalledTimes(1); + expect(client.tap).toHaveBeenCalledWith(201, 592); + }); + + it('refuses an alert whose buttons name no affirmative choice and taps nothing', async () => { + const root = limrunAttAlertTree(); + const ambiguous = { + ...root, + children: [ + ...(root.children ?? []).slice(0, 2), + { + type: 'Button', + role: 'AXButton', + AXLabel: 'Maybe Later', + pid: root.pid, + enabled: true, + frame: { x: 57, y: 511.67, width: 288, height: 48 }, + }, + { + type: 'Button', + role: 'AXButton', + AXLabel: 'More Info', + pid: root.pid, + enabled: true, + frame: { x: 57, y: 567.67, width: 288, height: 48 }, + }, + ], + }; + const { interactor, client } = sessionWithTrees([ambiguous]); + + await expect(interactor.acceptAlert!()).rejects.toThrow( + 'alert accept found an alert but no button that clearly matches the action', + ); + expect(client.tap).not.toHaveBeenCalled(); + }); + + it('refuses an app-owned dialog as no alert and taps nothing', async () => { + const { interactor, client } = sessionWithTrees([limrunAppAlertDialogTree()]); + + await expect(interactor.acceptAlert!()).rejects.toThrow('alert not found'); + expect(client.tap).not.toHaveBeenCalled(); + }); +}); + +describe('dismissAlert', () => { + it('taps the denying choice, never the affirmative one', async () => { + const { interactor, client } = sessionWithTrees([ + limrunAttAlertTree(), + limrunAppScreenTree(), + limrunAppScreenTree(), + ]); + + const result = await interactor.dismissAlert!(); + + expect(result).toMatchObject({ + kind: 'alertHandled', + action: 'dismiss', + button: 'Ask App Not to Track', + }); + expect(client.tap).toHaveBeenCalledWith(201, 536); + }); + + it('reports a press that left the same alert visible', async () => { + const stuck = [limrunAttAlertTree(), limrunAttAlertTree()]; + const { interactor } = sessionWithTrees(stuck); + + await expect(interactor.dismissAlert!()).rejects.toThrow( + 'alert dismiss did not dismiss the visible alert', + ); + }); + + it('accepts a replacement alert as a dismissal of the first', async () => { + const replacement = { + ...limrunAttAlertTree(), + children: (limrunAttAlertTree().children ?? []).map((child, index) => + index >= 2 ? { ...child, AXLabel: index === 2 ? 'No' : 'Yes' } : child, + ), + }; + const { interactor } = sessionWithTrees([limrunAttAlertTree(), replacement, replacement]); + + await expect(interactor.dismissAlert!()).resolves.toMatchObject({ + kind: 'alertHandled', + button: 'Ask App Not to Track', + }); + }); + + it('refuses an alert-shaped sheet with no pressable button as no alert', async () => { + const { interactor, client } = sessionWithTrees([limrunAlertTreeWithoutButtons()]); + + await expect(interactor.dismissAlert!()).rejects.toThrow('alert not found'); + expect(client.tap).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/provider-limrun/src/ios.ts b/packages/provider-limrun/src/ios.ts index de9b69cbd9..0c8a425ffc 100644 --- a/packages/provider-limrun/src/ios.ts +++ b/packages/provider-limrun/src/ios.ts @@ -406,6 +406,32 @@ class LimrunIosInteractor implements Interactor { const { setLimrunIosSetting } = await import('./ios-settings.ts'); return await setLimrunIosSetting(this.session, setting, state, appId, options); } + + /** + * The four alert legs observe through the same element-tree capture the snapshot rides and act + * through the same point tap every touch operation rides; `alert.ts` holds the host-root rule + * that separates a system alert from the app's own dialog. Loaded on the alert path like the + * other seam modules to keep this provider's declared import-time closure budget. + */ + async readAlert(): Promise> { + const { handleLimrunIosAlert } = await import('./alert.ts'); + return await handleLimrunIosAlert(this.session.client, 'get'); + } + + async awaitAlert(options?: { timeoutMs?: number }): Promise> { + const { handleLimrunIosAlert } = await import('./alert.ts'); + return await handleLimrunIosAlert(this.session.client, 'wait', options); + } + + async acceptAlert(): Promise> { + const { handleLimrunIosAlert } = await import('./alert.ts'); + return await handleLimrunIosAlert(this.session.client, 'accept'); + } + + async dismissAlert(): Promise> { + const { handleLimrunIosAlert } = await import('./alert.ts'); + return await handleLimrunIosAlert(this.session.client, 'dismiss'); + } } async function prepareLimrunIosAsset( From 791519ae1a3f3edde3e685f7377d1cc0a093b6bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 15 Oct 2026 23:10:39 +0200 Subject: [PATCH 02/12] fix(limrun): harden iOS alert detection against ambiguous sheets Review follow-up on the alert legs: a sheet must offer at least two pressable choices, launch-icon buttons never qualify, a capture whose host sheets disagree is refused instead of shadowing one with the other, and an action fires only when exactly one button states its choice. The legs' poll/confirm budgets now spend through an injected sleep seam so their tests drive the clock, a malformed timeout fails as a bounded default instead of an unbound loop, and the grant vocabulary covers the iOS variants (Allow Once / While Using App / All the Time) and Close App. User docs and CHANGELOG updated. --- CHANGELOG.md | 1 + .../src/alert-detection.test.ts | 78 +++++++--- .../provider-limrun/src/alert-detection.ts | 63 +++++--- packages/provider-limrun/src/alert.test.ts | 136 ++++++++++++++++++ packages/provider-limrun/src/alert.ts | 46 ++++-- .../src/ios-interactor-alert.test.ts | 81 ++++------- website/docs/docs/limrun.md | 2 +- 7 files changed, 301 insertions(+), 106 deletions(-) create mode 100644 packages/provider-limrun/src/alert.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 89500c54b4..1112e96624 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Add `settings mobile-data on|off` for adb-backed Android targets. It switches mobile data through `svc data` and reports the `mobileData` state stored in the global `mobile_data` row; multi-SIM read-back is not covered yet. Every other target refuses it before anything runs, and `--app` is refused because the switch is device-wide. - On iOS, a `fill` that runs under a degraded XCTest snapshot channel now aims at a point, including ref, selector and replay fills. When the runner identifies no text input under that point, it refuses with `NO_TEXT_INPUT_AT_POINT` (`error.details.reason: no_text_input_at_point`, `dispatched: no`) before dispatching its focus tap, and no field changes; when the input lookup itself cannot answer, it refuses with `TEXT_INPUT_PROBE_UNAVAILABLE` (`error.details.reason: text_input_probe_unavailable`). Previously such a fill cleared and typed through the app-level first responder, so it could rewrite a field the caller never named, or report a clear failure after emptying it. A ref or selector fill whose rect centre misses the real field (an outer container, a non-input node) now needs a point inside the input's own rect. - Opening a macOS app by bundle identifier now waits briefly when the app is already running but not yet registered with LaunchServices — the case when a supervisor or dev script started it directly — and if registration arrives within that bounded window the open attaches to the running copy, so the session drives the app the caller started, with its arguments. A copy still unlisted when the window closes gets the plain `open` as before, which can still duplicate; the wait only bounds the race, it does not close it. Launching an app that is not running costs one extra LaunchServices lookup and one process-table read, plus one bundle-identifier read when app-shaped processes exist. +- Limrun iOS sessions now serve `alert get/wait/accept/dismiss` instead of refusing with `unsupported-provider-mode`. System alerts are read from the same element-tree capture snapshots ride — a blank-label application root hosting a text-and-buttons sheet — and accepted or dismissed by tapping the button whose label states the asked-for choice; an alert whose choices leave the action undecided is reported, never guessed, and app-owned dialogs stay app UI. - Limrun iOS snapshots no longer fail when the instance reports a numeric or boolean accessibility value (the system photo picker's tabs report `1`/`0`): the provider seam normalizes variant scalars to text, the way the Apple tree does. Limrun nodes now also publish `selected: true` for nodes carrying the `Selected` trait, so snapshot output and `selected:` selectors can tell a picked photo from an unpicked one. - `settings` refuses a setting the target does not serve before anything changes, with `UNSUPPORTED_OPERATION`, `settings is not supported on this device`, and the target's `error.details.reason` (such as `unsupported-platform-leaf`). This replaces the macOS `INVALID_ARGS` refusal and the Apple `setting-unsupported-on-leaf` reason, and now also covers settings that used to reach the device first, such as Android `reset-keychain` or Apple `animations`. - `settings` refuses, before touching the device, a state its setting does not accept (`Invalid state: …`), a permission action or mode its target does not take (such as `permission deny accessibility`), and a stray extra argument (such as `clear-app-state `). The CLI and the daemon accept the same state spellings, in any casing or surrounding whitespace. diff --git a/packages/provider-limrun/src/alert-detection.test.ts b/packages/provider-limrun/src/alert-detection.test.ts index a67dd88ddc..325543e329 100644 --- a/packages/provider-limrun/src/alert-detection.test.ts +++ b/packages/provider-limrun/src/alert-detection.test.ts @@ -64,12 +64,7 @@ describe('findLimrunAlertCandidate', () => { expect(candidate?.alert.source).toBe('permission'); expect(candidate?.buttons.map((button) => button.role)).toEqual(['dismiss', 'accept']); // The tap point is the live button frame's center, rounded the way `centerOfRect` rounds. - expect(candidate?.buttons[1]).toEqual({ - label: 'Allow', - x: 201, - y: 592, - role: 'accept', - }); + expect(candidate?.buttons[1]).toEqual({ label: 'Allow', x: 201, y: 592, role: 'accept' }); }); it('finds the alert when the app screen is the first root of the capture', () => { @@ -89,10 +84,38 @@ describe('findLimrunAlertCandidate', () => { expect(findLimrunAlertCandidate(limrunHomeScreenTree())).toBeNull(); }); + it('refuses a widget-bearing home screen: LaunchIcon buttons never qualify', () => { + const home = limrunHomeScreenTree(); + const withWidgets = { + ...home, + children: [ + { + type: 'StaticText', + role: 'AXStaticText', + AXLabel: 'Mostly Sunny', + pid: home.pid, + frame: { x: 24, y: 300, width: 160, height: 20 }, + }, + ...(home.children ?? []), + ], + }; + expect(findLimrunAlertCandidate(withWidgets)).toBeNull(); + }); + it('refuses an alert-shaped sheet with no actionable button', () => { expect(findLimrunAlertCandidate(limrunAlertTreeWithoutButtons())).toBeNull(); }); + it('refuses a one-choice sheet: no action can tell its single option from the asked-for one', () => { + expect(findLimrunAlertCandidate(alertTreeWithButtons('Allow'))).toBeNull(); + expect(findLimrunAlertCandidate(alertTreeWithButtons('OK'))).toBeNull(); + }); + + it('refuses a capture whose host sheets disagree, rather than shadowing one with the other', () => { + const second = { ...limrunAttAlertTree(), pid: 99 }; + expect(findLimrunAlertCandidate([limrunAttAlertTree(), second])).toBeNull(); + }); + it('reports a permission alert from its button vocabulary and a plain one as a system dialog', () => { expect(findLimrunAlertCandidate(limrunAttAlertTree())?.alert.source).toBe('permission'); expect(findLimrunAlertCandidate(alertTreeWithButtons('No', 'Yes'))?.alert.source).toBe( @@ -118,18 +141,21 @@ describe('chooseLimrunAlertButton', () => { expect(chooseLimrunAlertButton(buttons, 'dismiss')).toBeNull(); }); - it('refuses an action that would press the wrong choice of a one-button alert', () => { - const allowOnly = findLimrunAlertCandidate(alertTreeWithButtons('Allow'))!.buttons; - const denyOnly = findLimrunAlertCandidate( - alertTreeWithButtons('Ask App Not to Track'), - )!.buttons; + it('never picks a choice of the wrong role, whatever else the alert offers', () => { + const allowOnly = findLimrunAlertCandidate(alertTreeWithButtons('Deny', 'Allow'))!.buttons; expect(chooseLimrunAlertButton(allowOnly, 'accept')?.label).toBe('Allow'); - expect(chooseLimrunAlertButton(allowOnly, 'dismiss')).toBeNull(); - expect(chooseLimrunAlertButton(denyOnly, 'dismiss')?.label).toBe('Ask App Not to Track'); - expect(chooseLimrunAlertButton(denyOnly, 'accept')).toBeNull(); + expect(chooseLimrunAlertButton(allowOnly, 'dismiss')?.label).toBe('Deny'); + const noAccept = findLimrunAlertCandidate( + alertTreeWithButtons('Cancel', 'Maybe Later'), + )!.buttons; + expect(chooseLimrunAlertButton(noAccept, 'accept')).toBeNull(); + const noDismiss = findLimrunAlertCandidate( + alertTreeWithButtons('Open Settings', 'Continue'), + )!.buttons; + expect(chooseLimrunAlertButton(noDismiss, 'dismiss')).toBeNull(); }); - it('classifies the classic iOS pair, the curly-apostrophe denial, and an open-settings prompt', () => { + it('classifies the classic iOS pair, the typographic apostrophe, and iOS variant grants', () => { const classic = findLimrunAlertCandidate(alertTreeWithButtons("Don't Allow", 'OK'))!.buttons; expect(chooseLimrunAlertButton(classic, 'accept')?.label).toBe('OK'); expect(chooseLimrunAlertButton(classic, 'dismiss')?.label).toBe("Don't Allow"); @@ -140,9 +166,15 @@ describe('chooseLimrunAlertButton', () => { )!.buttons; expect(chooseLimrunAlertButton(settings, 'accept')?.label).toBe('Open Settings'); expect(chooseLimrunAlertButton(settings, 'dismiss')?.label).toBe('Not Now'); + for (const label of ['Allow Once', 'Allow All the Time', 'Allow While Using App']) { + const granted = findLimrunAlertCandidate( + alertTreeWithButtons('Ask App Not to Track', label), + )!.buttons; + expect(chooseLimrunAlertButton(granted, 'accept')?.label).toBe(label); + } }); - it('refuses the candidate when no button is enabled with a frame to tap', () => { + it('refuses the candidate when no button is pressable', () => { const root = limrunAttAlertTree(); const crippled = { ...root, @@ -163,12 +195,22 @@ describe('chooseLimrunAlertButton', () => { AXLabel: 'Ask App Not to Track', pid: root.pid, enabled: true, + hittable: false, + frame: { x: 57, y: 567.67, width: 288, height: 48 }, + traits: ['Button'], + }, + { + type: 'Button', + role: 'AXButton', + AXLabel: 'Later', + pid: root.pid, + enabled: true, traits: ['Button'], }, ], }; - // The only button that could deny has no frame to tap, and the framed one is disabled: - // nothing is actionable, so the whole candidate is refused rather than half-served. + // Disabled, unhittable, and frameless buttons are all unpressable; with nothing pressable + // left, the whole candidate is refused rather than half-served. expect(findLimrunAlertCandidate(crippled)).toBeNull(); }); }); diff --git a/packages/provider-limrun/src/alert-detection.ts b/packages/provider-limrun/src/alert-detection.ts index 7b6b8061c6..409c3fbb13 100644 --- a/packages/provider-limrun/src/alert-detection.ts +++ b/packages/provider-limrun/src/alert-detection.ts @@ -6,11 +6,14 @@ import { readIosNodeChildren, readIosNodeRect, type IosTreeNode } from './snapsh /** * System-alert detection over a Limrun iOS element tree (issue #3349). A Limrun iOS direct * session has no XCUITest runner, but its element tree does carry the system alert: on a live - * instance the ATT prompt arrives as its own `AXApplication` root whose `AXLabel` is blank and - * whose content nodes carry the hosting process's `pid` — not the app's. The app's own - * `UIAlertController`, by contrast, rides inside the app's root, labeled with the app name and - * carrying the app's pid. The rules below encode exactly that observed split; they are - * conservative in both directions and every shape here is pinned to the live captures. + * instance the ATT prompt arrives as its own `AXApplication` root whose `AXLabel` is blank, with + * its text and buttons as direct children all sharing the hosting process's `pid` — never the + * app's. The app's own `UIAlertController`, by contrast, rides inside the app's root, labeled + * with the app name and carrying the app's pid. The rules below encode that observed split and + * refuse everything it does not vouch for: a named root is app UI, launch-icon buttons are a + * home screen, and a button without a frame or a live `pid` match across visible roots is not + * pressable. What one live capture cannot exclude — an app that left its own root's label blank + * while showing a dialog — is why the acting legs press only label-unambiguous buttons below. */ type LimrunAlertButtonRole = 'accept' | 'dismiss' | 'neutral'; @@ -37,12 +40,15 @@ export type LimrunAlertCandidate = Readonly<{ }>; const ACCEPT_LABEL_PATTERN = - /^(?:ok|allow|allow all|always allow|yes|continue|save|confirm|turn on|open settings)$/i; + /^(?:ok|allow|allow all|allow once|allow all the time|allow while using (?:the )?app|allow notifications|always allow|yes|continue|save|confirm|turn on|open settings|while using the app|only this time)$/i; +// `don.?t allow` covers the straight, missing, and typographic apostrophe in one alternate. const DISMISS_LABEL_PATTERN = - /^(?:cancel|deny|don.?t allow|don’t allow|don.t allow|not now|no|dismiss|close|later|skip|ask app not to track)$/i; + /^(?:cancel|deny|don.?t allow|not now|no|dismiss|close|close app|later|skip|ask app not to track)$/i; /** The subset that states a grant/deny choice; a plain Yes/No dialog is not a permission prompt. */ const PERMISSION_LABEL_PATTERN = - /^(?:allow.*|always allow|deny|don.?t allow|don’t allow|don.t allow|ask app not to track|turn on|open settings)$/i; + /^(?:allow.*|deny|don.?t allow|ask app not to track|turn on|open settings)$/i; +/** SpringBoard marks its home-screen icons with this trait; an alert never carries it. */ +const LAUNCH_ICON_TRAIT = 'LaunchIcon'; function nodeLabel(node: IosTreeNode): string { const label = node.AXLabel ?? node.label; @@ -62,25 +68,36 @@ export function isLimrunAlertHostRoot(node: IosTreeNode): boolean { /** * The actionable alert on this capture, or null. A host root qualifies only when its content is - * dialog-shaped: a button that can actually be pressed (enabled, with a complete rect) and at - * least one piece of text or another button beside it. That refusals both an app dialog (wrong - * root) and SpringBoard's home screen (blank root, launch icons instead of a dialog). + * dialog-shaped: at least two pressable choices (enabled, hittable, with a complete rect), at + * least one piece of text, and no SpringBoard launch icons. One live capture cannot exclude an + * app that leaves its own root's label blank while showing a dialog — the provider reports no + * capture-level fact that separates it from the alert host — which is why the acting legs press + * only buttons whose label states the asked-for choice, and an alert-shaped sheet offering a + * single choice is refused entirely rather than acted on by position. + * + * The button vocabulary is English, matching the untranslated simulator locale these instances + * run; a localized alert classifies as undecided and the acting legs report the ambiguity rather + * than guessing. */ export function findLimrunAlertCandidate( tree: IosTreeNode | IosTreeNode[], ): LimrunAlertCandidate | null { + const candidates: LimrunAlertCandidate[] = []; for (const root of Array.isArray(tree) ? tree : [tree]) { if (!isLimrunAlertHostRoot(root)) continue; const candidate = readAlertCandidateFromHostRoot(root); - if (candidate) return candidate; + if (candidate) candidates.push(candidate); } - return null; + // More than one alert-shaped host sheet is a shape this reader does not understand; pressing + // into it would guess, and the observation would silently shadow one with the other. + return candidates.length === 1 ? candidates[0]! : null; } function readAlertCandidateFromHostRoot(root: IosTreeNode): LimrunAlertCandidate | null { const descendants = collectDescendants(root); const buttons = findAlertButtons(descendants); - if (buttons.length === 0) return null; + // A lone choice says nothing about which action the caller asked for; refuse the sheet. + if (buttons.length < 2) return null; const texts = descendants .filter((node) => isStaticTextNode(node) && !isButtonNode(node)) .map(nodeLabel) @@ -120,14 +137,17 @@ function isStaticTextNode(node: IosTreeNode): boolean { } /** - * Pressable buttons only: a disabled button is not a choice the alert offers right now, and a - * button without a complete rect has no point the touch seam could aim at. + * Pressable alert choices only: a disabled button is not a choice the alert offers right now, a + * button without a complete rect has no point the touch seam could aim at, a button the provider + * reports as unhittable cannot be pressed, and a `LaunchIcon` is a home-screen app icon rather + * than a dialog button. */ function findAlertButtons(nodes: IosTreeNode[]): LimrunAlertButton[] { const seen = new Set(); const buttons: LimrunAlertButton[] = []; for (const node of nodes) { - if (!isButtonNode(node) || node.enabled === false) continue; + if (!isButtonNode(node) || node.enabled === false || node.hittable === false) continue; + if (node.traits?.includes(LAUNCH_ICON_TRAIT)) continue; const label = nodeLabel(node); if (!label || seen.has(label.toLowerCase())) continue; const rect = readIosNodeRect(node); @@ -160,14 +180,15 @@ function classifyAlertSource(buttons: readonly LimrunAlertButton[]): LimrunAlert /** * The button one action may press, or null when pressing any visible choice would guess. * Deliberately stricter than Android's: with no `android:id/button1`-style identity to fall back - * on, only a button whose own label states the asked-for choice may be pressed — an accept must - * never press a denial, and an alert whose choices are all neutral is refused for both actions - * rather than resolved by position. + * on, only a button whose own label states the asked-for choice may be pressed, and even then + * only when exactly one choice does — an accept must never press a denial, and an alert whose + * choices leave the action undecided is refused rather than resolved by tree position. */ export function chooseLimrunAlertButton( buttons: readonly LimrunAlertButton[], action: 'accept' | 'dismiss', ): LimrunAlertButton | null { const wanted = action === 'accept' ? 'accept' : 'dismiss'; - return buttons.find((button) => button.role === wanted) ?? null; + const matching = buttons.filter((button) => button.role === wanted); + return matching.length === 1 ? matching[0]! : null; } diff --git a/packages/provider-limrun/src/alert.test.ts b/packages/provider-limrun/src/alert.test.ts new file mode 100644 index 0000000000..954e953d45 --- /dev/null +++ b/packages/provider-limrun/src/alert.test.ts @@ -0,0 +1,136 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + limrunAlertAmbiguousButtonsTree, + limrunAppScreenTree, + limrunAttAlertTree, +} from './alert-detection.fixtures.ts'; +import { handleLimrunIosAlert, type LimrunIosAlertClient } from './alert.ts'; +import type { IosTreeNode } from './snapshot.ts'; + +// The clock is the assertion, not the wait (`docs/agents/testing.md`): only `Date` is faked, and +// the legs' sleep seam advances it, so budget loops terminate without real time. +const advanceClock = async (ms: number) => { + vi.advanceTimersByTime(ms); +}; + +function fakeClock() { + vi.useFakeTimers({ toFake: ['Date'] }); +} + +function clientWithTrees(trees: Array): LimrunIosAlertClient { + const elementTree = vi.fn(async () => { + const next = trees.length > 1 ? trees.shift() : trees[0]; + return [next] as unknown as ReturnType; + }); + return { elementTree, tap: vi.fn(async () => {}) }; +} + +afterEach(() => { + vi.useRealTimers(); +}); + +describe('wait budget', () => { + it('spends exactly the caller window through the sleep seam and reports the timeout', async () => { + fakeClock(); + const client = clientWithTrees([limrunAppScreenTree()]); + const sleep = vi.fn(advanceClock); + + const settled = handleLimrunIosAlert(client, 'wait', { timeoutMs: 5_000, sleep }).then( + () => undefined, + (error: unknown) => error, + ); + const error = await settled; + + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toBe('alert wait timed out'); + // One poll per interval over the whole window: never fewer (gave up early) nor real seconds. + expect(sleep).toHaveBeenCalledTimes(Math.ceil(5_000 / 300)); + }); + + it('treats a malformed window as the bounded default rather than an unbound loop', async () => { + fakeClock(); + const client = clientWithTrees([limrunAppScreenTree()]); + const sleep = vi.fn(advanceClock); + + await expect( + handleLimrunIosAlert(client, 'wait', { timeoutMs: Number.NaN, sleep }), + ).rejects.toThrow('alert wait timed out'); + // The fallback is the contract default, spent on the driven clock — this returned, so the + // loop had a bound, which a NaN comparison would not have given it. + expect(sleep).toHaveBeenCalledTimes(Math.ceil(10_000 / 300)); + }); + + it('surfaces a failed capture as itself, never as a timeout', async () => { + const elementTree = vi + .fn() + .mockRejectedValue( + new Error('websocket closed'), + ) as unknown as LimrunIosAlertClient['elementTree']; + const client = { elementTree, tap: vi.fn(async () => {}) } satisfies LimrunIosAlertClient; + + await expect(handleLimrunIosAlert(client, 'wait', { sleep: async () => {} })).rejects.toThrow( + 'websocket closed', + ); + }); +}); + +describe('acting budgets', () => { + it('spends the confirm budget through the sleep seam and reports a stuck alert', async () => { + fakeClock(); + const client = clientWithTrees([limrunAttAlertTree(), limrunAttAlertTree()]); + const sleep = vi.fn(advanceClock); + + await expect(handleLimrunIosAlert(client, 'dismiss', { sleep })).rejects.toThrow( + 'alert dismiss did not dismiss the visible alert', + ); + expect(client.tap).toHaveBeenCalledTimes(1); + expect(sleep).toHaveBeenCalled(); + }); + + it('refuses an ambiguous alert without tapping', async () => { + const client = clientWithTrees([limrunAlertAmbiguousButtonsTree()]); + + await expect(handleLimrunIosAlert(client, 'accept', { sleep: async () => {} })).rejects.toThrow( + 'no button that clearly matches the action', + ); + expect(client.tap).not.toHaveBeenCalled(); + }); +}); + +describe('button vocabulary', () => { + function alertWith(left: string, right: string): IosTreeNode { + const root = limrunAttAlertTree(); + return { + ...root, + children: [ + ...(root.children ?? []).slice(0, 2), + ...[left, right].map((label, index) => ({ + type: 'Button', + role: 'AXButton', + AXLabel: label, + pid: root.pid, + enabled: true, + frame: { x: 57, y: 511.67 + index * 56, width: 288, height: 48 }, + traits: ['Button', 'Scrollable'], + })), + ], + }; + } + + it('accepts the iOS variant grants, not just the bare Allow', async () => { + for (const label of ['Allow While Using App', 'Allow Once', 'Allow All the Time']) { + const client = clientWithTrees([ + alertWith('Ask App Not to Track', label), + limrunAppScreenTree(), + ]); + const result = await handleLimrunIosAlert(client, 'accept', { sleep: async () => {} }); + expect(result.kind === 'alertHandled' && result.button).toBe(label); + } + }); + + it('dismisses Close App, the ANRB denial', async () => { + const client = clientWithTrees([alertWith('Close App', 'Wait'), limrunAppScreenTree()]); + const result = await handleLimrunIosAlert(client, 'dismiss', { sleep: async () => {} }); + expect(result.kind === 'alertHandled' && result.button).toBe('Close App'); + }); +}); diff --git a/packages/provider-limrun/src/alert.ts b/packages/provider-limrun/src/alert.ts index 7d133f9fcd..35526f2dd7 100644 --- a/packages/provider-limrun/src/alert.ts +++ b/packages/provider-limrun/src/alert.ts @@ -5,6 +5,7 @@ import { } from '@agent-device/contracts/alert-contract'; import { AppError } from '@agent-device/kernel/errors'; import { successText } from '@agent-device/kernel/success-text'; +import type { InstanceClient as LimrunIosClient } from '@limrun/api/ios-client'; import { setTimeout as sleep } from 'node:timers/promises'; import { chooseLimrunAlertButton, @@ -12,7 +13,6 @@ import { type LimrunAlertCandidate, type LimrunAlertInfo, } from './alert-detection.ts'; -import type { IosTreeNode } from './snapshot.ts'; /** * The four alert legs on a Limrun iOS direct session (issue #3349). There is no XCUITest runner @@ -24,10 +24,17 @@ import type { IosTreeNode } from './snapshot.ts'; /** The client methods the legs use; declared here so the legs are testable without a live instance. */ export type LimrunIosAlertClient = Readonly<{ - elementTree: () => Promise; - tap: (x: number, y: number) => Promise; + elementTree: LimrunIosClient['elementTree']; + tap: LimrunIosClient['tap']; }>; +/** Tests pass a zero-wait sleep; production rides the timer. */ +export type LimrunAlertSleep = (milliseconds: number) => Promise; + +const realSleep: LimrunAlertSleep = async (milliseconds) => { + await sleep(milliseconds); +}; + export type LimrunIosAlertResult = | { kind: 'alertStatus'; @@ -60,11 +67,23 @@ export type LimrunIosAlertAction = 'get' | 'wait' | 'accept' | 'dismiss'; const ALERT_NOT_FOUND_HINT = 'If the sheet is visible in snapshot but alert reports no alert, it is likely app-owned UI. Use snapshot -i and press the visible label/ref.'; +/** A malformed window must fail as an immediate timeout, never as a loop whose bound is never reached. */ +function alertTimeoutMs(timeoutMs: number | undefined, fallback: number): number { + return typeof timeoutMs === 'number' && Number.isFinite(timeoutMs) && timeoutMs >= 0 + ? timeoutMs + : fallback; +} + export async function handleLimrunIosAlert( client: LimrunIosAlertClient, action: LimrunIosAlertAction, - options?: Readonly<{ timeoutMs?: number }>, + options?: Readonly<{ + timeoutMs?: number; + /** Tests advance polls without waiting production time (`docs/agents/testing.md`). */ + sleep?: LimrunAlertSleep; + }>, ): Promise { + const nap = options?.sleep ?? realSleep; if (action === 'get') { const candidate = await readLimrunAlertCandidate(client); return { @@ -79,7 +98,8 @@ export async function handleLimrunIosAlert( const start = Date.now(); const candidate = await pollLimrunAlertCandidate( client, - options?.timeoutMs ?? DEFAULT_ALERT_TIMEOUT_MS, + alertTimeoutMs(options?.timeoutMs, DEFAULT_ALERT_TIMEOUT_MS), + nap, ); if (!candidate) { throw new AppError('COMMAND_FAILED', 'alert wait timed out'); @@ -93,7 +113,7 @@ export async function handleLimrunIosAlert( ...successText('Alert visible'), }; } - return await actOnLimrunIosAlert(client, action); + return await actOnLimrunIosAlert(client, action, nap); } /** @@ -105,8 +125,9 @@ export async function handleLimrunIosAlert( async function actOnLimrunIosAlert( client: LimrunIosAlertClient, action: 'accept' | 'dismiss', + nap: LimrunAlertSleep, ): Promise { - const candidate = await pollLimrunAlertCandidate(client, ALERT_ACTION_RETRY_MS); + const candidate = await pollLimrunAlertCandidate(client, ALERT_ACTION_RETRY_MS, nap); if (!candidate) { throw new AppError('COMMAND_FAILED', 'alert not found', { hint: ALERT_NOT_FOUND_HINT }); } @@ -122,7 +143,7 @@ async function actOnLimrunIosAlert( ); } await client.tap(button.x, button.y); - await confirmLimrunAlertDismissed(client, candidate, action); + await confirmLimrunAlertDismissed(client, candidate, action, nap); return { kind: 'alertHandled', platform: 'ios', @@ -144,6 +165,7 @@ async function confirmLimrunAlertDismissed( client: LimrunIosAlertClient, dismissed: LimrunAlertCandidate, action: 'accept' | 'dismiss', + nap: LimrunAlertSleep, ): Promise { const start = Date.now(); for (;;) { @@ -155,7 +177,7 @@ async function confirmLimrunAlertDismissed( hint: 'The alert button was pressed but the dialog is still visible. Inspect alert get --json, then press the visible button by label/ref or retry.', }); } - await sleep(ALERT_POLL_INTERVAL_MS); + await nap(ALERT_POLL_INTERVAL_MS); } } @@ -166,19 +188,19 @@ function sameLimrunAlert(left: LimrunAlertInfo, right: LimrunAlertInfo): boolean async function pollLimrunAlertCandidate( client: LimrunIosAlertClient, timeoutMs: number, + nap: LimrunAlertSleep, ): Promise { const start = Date.now(); for (;;) { const candidate = await readLimrunAlertCandidate(client); if (candidate) return candidate; if (Date.now() - start >= timeoutMs) return null; - await sleep(ALERT_POLL_INTERVAL_MS); + await nap(ALERT_POLL_INTERVAL_MS); } } async function readLimrunAlertCandidate( client: LimrunIosAlertClient, ): Promise { - const tree = (await client.elementTree()) as IosTreeNode | IosTreeNode[]; - return findLimrunAlertCandidate(tree); + return findLimrunAlertCandidate(await client.elementTree()); } diff --git a/packages/provider-limrun/src/ios-interactor-alert.test.ts b/packages/provider-limrun/src/ios-interactor-alert.test.ts index 649a286271..4c9d6523a3 100644 --- a/packages/provider-limrun/src/ios-interactor-alert.test.ts +++ b/packages/provider-limrun/src/ios-interactor-alert.test.ts @@ -11,6 +11,8 @@ import type { IosTreeNode } from './snapshot.ts'; /** * The four legs as the interactor exposes them: observation is one `elementTree()` read per * probe, action is a `tap()` at the chosen button's center. Trees are the live Limrun captures. + * These cases resolve on the first probe each; budget loops are pinned through the legs' sleep + * seam in `alert.test.ts`, so nothing here waits production time. */ function sessionWithTrees(trees: Array) { const elementTree = vi.fn(async () => { @@ -52,19 +54,13 @@ describe('readAlert', () => { describe('awaitAlert', () => { it('returns as soon as a capture shows the alert', async () => { - const { interactor } = sessionWithTrees([limrunAppScreenTree(), limrunAttAlertTree()]); + const { interactor } = sessionWithTrees([limrunAttAlertTree(), limrunAttAlertTree()]); const result = await interactor.awaitAlert!({ timeoutMs: 5_000 }); expect(result.kind).toBe('alertWait'); expect(result.alert).toMatchObject({ buttons: ['Ask App Not to Track', 'Allow'] }); }); - - it('times out while no capture ever shows an alert', async () => { - const { interactor } = sessionWithTrees([limrunAppScreenTree()]); - - await expect(interactor.awaitAlert!({ timeoutMs: 1 })).rejects.toThrow('alert wait timed out'); - }); }); describe('acceptAlert', () => { @@ -90,43 +86,36 @@ describe('acceptAlert', () => { }); it('refuses an alert whose buttons name no affirmative choice and taps nothing', async () => { - const root = limrunAttAlertTree(); - const ambiguous = { - ...root, - children: [ - ...(root.children ?? []).slice(0, 2), - { - type: 'Button', - role: 'AXButton', - AXLabel: 'Maybe Later', - pid: root.pid, - enabled: true, - frame: { x: 57, y: 511.67, width: 288, height: 48 }, - }, - { - type: 'Button', - role: 'AXButton', - AXLabel: 'More Info', - pid: root.pid, - enabled: true, - frame: { x: 57, y: 567.67, width: 288, height: 48 }, - }, - ], - }; - const { interactor, client } = sessionWithTrees([ambiguous]); + const { interactor, client } = sessionWithTrees([ + { + ...limrunAlertTreeWithoutButtons(), + children: [ + ...(limrunAlertTreeWithoutButtons().children ?? []), + { + type: 'Button', + role: 'AXButton', + AXLabel: 'Maybe Later', + pid: limrunAlertTreeWithoutButtons().pid, + enabled: true, + frame: { x: 57, y: 511.67, width: 288, height: 48 }, + }, + { + type: 'Button', + role: 'AXButton', + AXLabel: 'More Info', + pid: limrunAlertTreeWithoutButtons().pid, + enabled: true, + frame: { x: 57, y: 567.67, width: 288, height: 48 }, + }, + ], + }, + ]); await expect(interactor.acceptAlert!()).rejects.toThrow( 'alert accept found an alert but no button that clearly matches the action', ); expect(client.tap).not.toHaveBeenCalled(); }); - - it('refuses an app-owned dialog as no alert and taps nothing', async () => { - const { interactor, client } = sessionWithTrees([limrunAppAlertDialogTree()]); - - await expect(interactor.acceptAlert!()).rejects.toThrow('alert not found'); - expect(client.tap).not.toHaveBeenCalled(); - }); }); describe('dismissAlert', () => { @@ -147,15 +136,6 @@ describe('dismissAlert', () => { expect(client.tap).toHaveBeenCalledWith(201, 536); }); - it('reports a press that left the same alert visible', async () => { - const stuck = [limrunAttAlertTree(), limrunAttAlertTree()]; - const { interactor } = sessionWithTrees(stuck); - - await expect(interactor.dismissAlert!()).rejects.toThrow( - 'alert dismiss did not dismiss the visible alert', - ); - }); - it('accepts a replacement alert as a dismissal of the first', async () => { const replacement = { ...limrunAttAlertTree(), @@ -170,11 +150,4 @@ describe('dismissAlert', () => { button: 'Ask App Not to Track', }); }); - - it('refuses an alert-shaped sheet with no pressable button as no alert', async () => { - const { interactor, client } = sessionWithTrees([limrunAlertTreeWithoutButtons()]); - - await expect(interactor.dismissAlert!()).rejects.toThrow('alert not found'); - expect(client.tap).not.toHaveBeenCalled(); - }); }); diff --git a/website/docs/docs/limrun.md b/website/docs/docs/limrun.md index d95345efa1..00cd8d4790 100644 --- a/website/docs/docs/limrun.md +++ b/website/docs/docs/limrun.md @@ -71,7 +71,7 @@ agent-device disconnect On Android, agent-device connects to Limrun over an ADB tunnel. Snapshots use the Android snapshot helper, installs upload the app through Limrun asset storage, and port reverse runs over ADB, including the usual reverse setup for a local Metro server. -On iOS, agent-device supports app lifecycle commands, snapshots, screenshots, taps, text input, scrolling, the home button, clipboard read and write, and app installation. `open --launch-args` relaunches the app with those arguments through simctl. iOS cannot reverse a remote device port to a local host port, so for Metro use a publicly reachable HTTPS endpoint or bridge URL instead of a local-only address. +On iOS, agent-device supports app lifecycle commands, snapshots, screenshots, taps, text input, scrolling, the home button, clipboard read and write, app installation, and system alerts. `alert get/wait/accept/dismiss` read the alert from the same element tree as snapshots and press its buttons through taps; app-owned dialogs are not system alerts there, so use `snapshot -i` and press their visible labels. `open --launch-args` relaunches the app with those arguments through simctl. iOS cannot reverse a remote device port to a local host port, so for Metro or React DevTools use a publicly reachable HTTPS endpoint or bridge URL instead of a local-only address. On iOS, `settings` supports `appearance`, `permission`, `location`, and `clear-app-state`, with the same arguments as a local simulator. `clear-app-state` resets the app's data container through Limrun's soft reset. That reset relaunches the app once before agent-device stops it, so the app may run once after its data is cleared. `reset-keychain`, `wifi`, `airplane`, `mobile-data`, `faceid`, `touchid`, and `text-size` are unsupported, and you cannot read back any iOS setting. On Android, `settings mobile-data on|off` runs over the ADB tunnel like on a local device. From 85ca5ca68e6dda81d841cca45c9f1fa4ae032059 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 07:47:32 +0200 Subject: [PATCH 03/12] refactor(alert): drive every tree-based alert backend through one shared driver PR #3699 review: the Limrun iOS alert legs duplicated the Android tree-alert driver (poll, confirm, same-alert, response building) and its label vocabulary. Both backends now reach those steps through one backend-neutral driver, runTreeAlertAction in platform-android (exported via the mechanics facet provider-limrun already depends on), and classify button roles through one English vocabulary that lives in contracts/alert-contract with the alert budgets and typed reasons (contracts owns vocabulary only; the driver loop owns the AppError/timer mechanics R18 keeps out of contracts). Each backend supplies only its own seams: Android wraps its dump reader and keeps its platform-id roles, neutral/Back fallbacks, and tap/Back presses; Limrun supplies the blank-root process-exclusive host reader, the label-strict chooser, and client.tap. The confirm-on-ambiguous-capture and duplicate-label threads settle once in the shared code: the three-way alert|none|undecided observation gates confirmation in the driver, and selectUniqueAlertButtonByRole refuses duplicated roles at the shared choice boundary. The deadline order resolves to read-first: over a positive window both predecessor loops already observed before waiting. The alert-not-found and undecided refusals now carry the same typed details.reason and dispatched: 'no' on both backends. --- CHANGELOG.md | 3 +- packages/contracts/src/alert-contract.ts | 68 ++- .../src/__tests__/alert-detection.test.ts | 18 + .../platform-android/src/alert-detection.ts | 34 +- packages/platform-android/src/alert.ts | 420 ++++++++++++------ packages/platform-android/src/mechanics.ts | 13 +- .../src/alert-detection.fixtures.ts | 107 ++++- .../src/alert-detection.test.ts | 203 +++++---- .../provider-limrun/src/alert-detection.ts | 173 +++++--- packages/provider-limrun/src/alert.test.ts | 93 +++- packages/provider-limrun/src/alert.ts | 215 ++------- .../src/ios-interactor-alert.test.ts | 20 +- website/docs/docs/limrun.md | 2 +- 13 files changed, 871 insertions(+), 498 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1112e96624..422482b2c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,8 @@ - Add `settings mobile-data on|off` for adb-backed Android targets. It switches mobile data through `svc data` and reports the `mobileData` state stored in the global `mobile_data` row; multi-SIM read-back is not covered yet. Every other target refuses it before anything runs, and `--app` is refused because the switch is device-wide. - On iOS, a `fill` that runs under a degraded XCTest snapshot channel now aims at a point, including ref, selector and replay fills. When the runner identifies no text input under that point, it refuses with `NO_TEXT_INPUT_AT_POINT` (`error.details.reason: no_text_input_at_point`, `dispatched: no`) before dispatching its focus tap, and no field changes; when the input lookup itself cannot answer, it refuses with `TEXT_INPUT_PROBE_UNAVAILABLE` (`error.details.reason: text_input_probe_unavailable`). Previously such a fill cleared and typed through the app-level first responder, so it could rewrite a field the caller never named, or report a clear failure after emptying it. A ref or selector fill whose rect centre misses the real field (an outer container, a non-input node) now needs a point inside the input's own rect. - Opening a macOS app by bundle identifier now waits briefly when the app is already running but not yet registered with LaunchServices — the case when a supervisor or dev script started it directly — and if registration arrives within that bounded window the open attaches to the running copy, so the session drives the app the caller started, with its arguments. A copy still unlisted when the window closes gets the plain `open` as before, which can still duplicate; the wait only bounds the race, it does not close it. Launching an app that is not running costs one extra LaunchServices lookup and one process-table read, plus one bundle-identifier read when app-shaped processes exist. -- Limrun iOS sessions now serve `alert get/wait/accept/dismiss` instead of refusing with `unsupported-provider-mode`. System alerts are read from the same element-tree capture snapshots ride — a blank-label application root hosting a text-and-buttons sheet — and accepted or dismissed by tapping the button whose label states the asked-for choice; an alert whose choices leave the action undecided is reported, never guessed, and app-owned dialogs stay app UI. +- Limrun iOS sessions now serve `alert get/wait/accept/dismiss` instead of refusing with `unsupported-provider-mode`. System alerts are read from the same element-tree capture snapshots ride — a blank-label, process-exclusive application root hosting a text-and-buttons sheet — and accepted or dismissed by tapping the button whose English label states the asked-for action; a capture that is alert-shaped but unresolvable (degraded buttons, competing sheets, an unverifiable owner) is reported as undecided and never pressed or counted as dismissed. The one shape no capture fact separates is an app dialog under a blanked root shared by no other process; the label-only acting legs answer it with a refusal, never a guess. +- The tree-based alert backends (Android and Limrun iOS) now share one alert driver and one English button-label vocabulary: `alert accept/dismiss` classify the same label the same way on both platforms, so Android now recognizes iOS variant grants (`Allow Once`, `Allow All the Time`, `Allow While Using App`, `Allow Notifications`) and `Ask App Not to Track` as acting choices, and two buttons sharing one role no longer resolve by tree order — the shared choice boundary refuses, and each backend's own policy answers (Android's neutral-button/Back fallback, Limrun's typed refusal). `alert not found` and the no-matching-button refusal now carry the same typed `details.reason` on both platforms. - Limrun iOS snapshots no longer fail when the instance reports a numeric or boolean accessibility value (the system photo picker's tabs report `1`/`0`): the provider seam normalizes variant scalars to text, the way the Apple tree does. Limrun nodes now also publish `selected: true` for nodes carrying the `Selected` trait, so snapshot output and `selected:` selectors can tell a picked photo from an unpicked one. - `settings` refuses a setting the target does not serve before anything changes, with `UNSUPPORTED_OPERATION`, `settings is not supported on this device`, and the target's `error.details.reason` (such as `unsupported-platform-leaf`). This replaces the macOS `INVALID_ARGS` refusal and the Apple `setting-unsupported-on-leaf` reason, and now also covers settings that used to reach the device first, such as Android `reset-keychain` or Apple `animations`. - `settings` refuses, before touching the device, a state its setting does not accept (`Invalid state: …`), a permission action or mode its target does not take (such as `permission deny accessibility`), and a stray extra argument (such as `clear-app-state `). The CLI and the daemon accept the same state spellings, in any casing or surrounding whitespace. diff --git a/packages/contracts/src/alert-contract.ts b/packages/contracts/src/alert-contract.ts index b7af81b93a..93e34299d7 100644 --- a/packages/contracts/src/alert-contract.ts +++ b/packages/contracts/src/alert-contract.ts @@ -6,8 +6,10 @@ export const ALERT_ACTION_RETRY_MS = 2_000; * The one alert failure the family retries on: the backend looked and there was no alert *yet*. * Both Apple backends state it in typed form — the XCTest runner as this `ErrorPayload.code` * (surfacing as `details.runnerErrorCode`, like `RUNNER_BUSY`, without changing the wire error - * code), the macOS helper as `details.reason`. Retry and the fallback hint key on these, never on - * the message text: a transport, runner or helper failure must never read as an absent alert. + * code), the macOS helper as details.reason — and so do the tree-based acting legs (Android and + * Limrun iOS, through the shared driver), as this `details.reason` paired with `dispatched: + * 'no'`. Retry and the fallback hint key on these, never on the message text: a transport, + * runner or helper failure must never read as an absent alert. */ export const ALERT_NOT_FOUND_RUNNER_CODE = 'ALERT_NOT_FOUND'; export const ALERT_NOT_FOUND_REASON = 'alert-not-found'; @@ -20,6 +22,68 @@ export const ALERT_NOT_FOUND_REASON = 'alert-not-found'; */ export const ALERT_BUTTON_NOT_FOUND_RUNNER_CODE = 'ALERT_BUTTON_NOT_FOUND'; +export type AlertButtonRole = 'accept' | 'dismiss' | 'neutral'; + +/** + * What one capture from a tree-based alert backend says about alerts. `none` states absence; + * `undecided` states the capture is alert-shaped yet unresolvable (a degraded or shared root, + * competing sheets) and proves nothing — confirming a dismissal may accept `none`, never + * `undecided`. Backends whose capture cannot reach `undecided` report a quiet capture as `none`. + */ +export type AlertCapture = + | { kind: 'alert'; candidate: TCandidate } + | { kind: 'none' } + | { kind: 'undecided' }; + +/** + * The one button label vocabulary every tree-based alert backend classifies with. English-only, + * matching the untranslated device locale the alert lanes are validated against; a localized + * button classifies `neutral`, so acting legs refuse on the button-undecided reason rather than + * guessing at a translation. + */ +const ACCEPT_LABEL_PATTERN = + /^(?:ok|allow|allow all|allow once|allow all the time|allow while using (?:the )?app|allow notifications|always allow|yes|continue|save|confirm|turn on|open settings|while using the app|only this time)$/i; +// `don.?t allow` covers the straight, missing, and typographic apostrophe in one alternate. +const DISMISS_LABEL_PATTERN = + /^(?:cancel|deny|don.?t allow|not now|no|dismiss|close|close app|later|skip|ask app not to track)$/i; + +/** One button's action role under the shared vocabulary; anything unnamed is neutral. */ +export function classifyAlertButtonRole(label: string): AlertButtonRole { + const trimmed = label.trim(); + if (ACCEPT_LABEL_PATTERN.test(trimmed)) return 'accept'; + if (DISMISS_LABEL_PATTERN.test(trimmed)) return 'dismiss'; + return 'neutral'; +} + +/** + * The one choice boundary for a shared choice: a role exactly one visible button carries, or + * null when none does and when two or more do — pressing one of a duplicated choice would pick + * a side the capture does not name, so duplicates refuse rather than resolve by tree order. + */ +export function selectUniqueAlertButtonByRole>( + buttons: readonly TButton[], + role: 'accept' | 'dismiss', +): TButton | null { + const matching = buttons.filter((button) => button.role === role); + return matching.length === 1 ? (matching[0] ?? null) : null; +} + +/** + * The reason a tree-based acting leg (`alert accept|dismiss`) refuses after seeing an alert: the + * visible buttons name no single choice for the asked-for action. Callers branch on + * `error.details.reason` instead of the prose; `dispatched: 'no'` states the refusal ran before + * any press. + */ +export const ALERT_BUTTON_UNDECIDED_REASON = 'alert_button_undecided'; + +/** + * The reason a tree-based acting leg refuses before it has a candidate at all: the capture was + * alert-shaped but unresolvable (degraded buttons, competing sheets, an unverifiable owner). + * This is not absence — reporting `alert-not-found` over it would state more than the capture + * supports — and confirming a dismissal must never accept such a capture as proof of absence. + */ +export const ALERT_CAPTURE_UNDECIDED_REASON = 'alert_capture_undecided'; + export const ALERT_ACTIONS = ['get', 'accept', 'dismiss', 'wait'] as const; export type AlertAction = (typeof ALERT_ACTIONS)[number]; diff --git a/packages/platform-android/src/__tests__/alert-detection.test.ts b/packages/platform-android/src/__tests__/alert-detection.test.ts index 126cdc8a8d..1c56a0560f 100644 --- a/packages/platform-android/src/__tests__/alert-detection.test.ts +++ b/packages/platform-android/src/__tests__/alert-detection.test.ts @@ -58,3 +58,21 @@ test('chooseAndroidAlertButton accepts a single neutral button', () => { assert.equal(chooseAndroidAlertButton(candidate?.buttons ?? [], 'accept')?.label, 'Later'); }); + +test('the shared vocabulary classifies iOS variant grants on Android trees too', () => { + // One label vocabulary: the same label states the same role on every tree backend, so + // `alert accept` can act on the same button on both platforms. Non-button ids keep the + // classification on the labels rather than the platform id map. + const candidate = findAndroidAlertCandidate([ + node(0, 'android.app.AlertDialog'), + text(1, 'Location access', 'android:id/alertTitle'), + button(2, "Don't allow", 'com.example:id/deny', { x: 52, y: 612 }), + button(3, 'Allow while using the app', 'com.example:id/grant', { x: 210, y: 612 }), + ]); + + assert.equal( + chooseAndroidAlertButton(candidate?.buttons ?? [], 'accept')?.label, + 'Allow while using the app', + ); + assert.equal(chooseAndroidAlertButton(candidate?.buttons ?? [], 'dismiss')?.label, "Don't allow"); +}); diff --git a/packages/platform-android/src/alert-detection.ts b/packages/platform-android/src/alert-detection.ts index 89f66a3925..83906cabd6 100644 --- a/packages/platform-android/src/alert-detection.ts +++ b/packages/platform-android/src/alert-detection.ts @@ -1,14 +1,18 @@ import { centerOfRect, type RawSnapshotNode } from '@agent-device/kernel/snapshot'; -import type { AlertInfo, AlertSource } from '@agent-device/contracts/alert-contract'; +import { + type AlertButtonRole, + classifyAlertButtonRole, + type AlertInfo, + type AlertSource, + selectUniqueAlertButtonByRole, +} from '@agent-device/contracts/alert-contract'; import { isSnapshotNodeInteractionBlocked } from '@agent-device/capture-kit/snapshot-occlusion'; -type AndroidAlertButtonRole = 'accept' | 'dismiss' | 'neutral'; - export type AndroidAlertButton = { label: string; x: number; y: number; - role: AndroidAlertButtonRole; + role: AlertButtonRole; }; type AndroidAlertSource = Extract; @@ -43,10 +47,6 @@ const ANDROID_ALERT_BUTTON_ID_PATTERN = /^android:id\/button[123]$/i; const ANDROID_PERMISSION_ID_PATTERN = /(?:^|:)id\/permission_/i; const ANDROID_BLOCKING_DIALOG_PATTERN = /\b(?:is(?:n't| not) responding|keeps stopping|has stopped|close app|app info)\b/i; -const ACCEPT_LABEL_PATTERN = - /^(?:ok|allow|allow all|while using the app|only this time|yes|continue|save|confirm|turn on|open settings)$/i; -const DISMISS_LABEL_PATTERN = - /^(?:cancel|deny|don.t allow|don’t allow|not now|no|dismiss|close|close app|later|skip)$/i; export function classifyAndroidAlertIdentifier( identifier: string | null, @@ -85,12 +85,18 @@ function isReachableAlertNode(node: RawSnapshotNode): boolean { return !isSnapshotNodeInteractionBlocked(node); } +/** + * Android's acting policy on the shared choice boundary: the unique button whose role the + * action names; else, for dismiss, the first neutral button (Android dialogs commonly name the + * leaving path only neutrally); else, for accept, a single-button dialog's only affirmative + * path. Two buttons sharing a role refuse at the shared boundary rather than resolving by tree + * order — the driver turns that refusal into a Back key for dismiss, or a typed refusal. + */ export function chooseAndroidAlertButton( buttons: AndroidAlertButton[], action: 'accept' | 'dismiss', ): AndroidAlertButton | null { - const role = action === 'accept' ? 'accept' : 'dismiss'; - const exact = buttons.find((button) => button.role === role); + const exact = selectUniqueAlertButtonByRole(buttons, action); if (exact) return exact; if (action === 'dismiss') { return buttons.find((button) => button.role === 'neutral') ?? null; @@ -172,16 +178,14 @@ function findAlertButtons(nodes: RawSnapshotNode[]): AndroidAlertButton[] { return buttons; } -function classifyAndroidAlertButton(node: RawSnapshotNode, label: string): AndroidAlertButtonRole { +function classifyAndroidAlertButton(node: RawSnapshotNode, label: string): AlertButtonRole { const identifier = node.identifier ?? ''; const roleFromId = classifyAndroidAlertButtonById(identifier); if (roleFromId) return roleFromId; - if (ACCEPT_LABEL_PATTERN.test(label.trim())) return 'accept'; - if (DISMISS_LABEL_PATTERN.test(label.trim())) return 'dismiss'; - return 'neutral'; + return classifyAlertButtonRole(label); } -function classifyAndroidAlertButtonById(identifier: string): AndroidAlertButtonRole | null { +function classifyAndroidAlertButtonById(identifier: string): AlertButtonRole | null { if (/(?:^|:)id\/button1$/i.test(identifier)) return 'accept'; if (/(?:^|:)id\/button2$/i.test(identifier)) return 'dismiss'; if (/(?:^|:)id\/button3$/i.test(identifier)) return 'neutral'; diff --git a/packages/platform-android/src/alert.ts b/packages/platform-android/src/alert.ts index 711ba999be..a5ef0ec909 100644 --- a/packages/platform-android/src/alert.ts +++ b/packages/platform-android/src/alert.ts @@ -1,12 +1,18 @@ import { type AlertAction, ALERT_ACTION_RETRY_MS, + ALERT_BUTTON_UNDECIDED_REASON, + ALERT_CAPTURE_UNDECIDED_REASON, + type AlertCapture, + type AlertInfo, + type AlertPlatform, + ALERT_NOT_FOUND_REASON, ALERT_POLL_INTERVAL_MS, DEFAULT_ALERT_TIMEOUT_MS, } from '@agent-device/contracts/alert-contract'; import { AppError } from '@agent-device/kernel/errors'; import { withDiagnosticTimer } from '@agent-device/host-kit/diagnostics'; -import { Deadline, pollUntilSettled } from '@agent-device/host-kit/retry'; +import { sleep } from '@agent-device/host-kit/retry'; import { successText } from '@agent-device/kernel/success-text'; import type { DeviceInfo } from '@agent-device/kernel/device'; @@ -14,189 +20,353 @@ import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; import { chooseAndroidAlertButton, findAndroidAlertCandidate, + type AndroidAlertButton, type AndroidAlertCandidate, type AndroidAlertInfo, } from './alert-detection.ts'; import { backAndroid, pressAndroid } from './input-actions.ts'; -type AndroidAlertOptions = { +/** + * THE tree-based alert driver (PR #3699 review): every backend that reads alerts out of a + * capture tree — the Android ADB legs here and the Limrun iOS legs in `provider-limrun` — + * reaches poll, confirm, same-alert, refusal, and response building through this one module, + * and its button roles through the shared label vocabulary in `contracts/alert-contract`. A + * backend supplies only its own observation (`capture` + `findCandidate`), its platform's + * acting policy (`choose`), and its press seam (`press`). + */ + +/** The point a press seam aims at: a button's center in the backend's own coordinate space. */ +export type TreeAlertButton = Readonly<{ label: string; x: number; y: number }>; + +/** + * What one capture says about alerts; the one three-way observation lives in the alert + * contract next to the shared vocabulary, and every tree backend's reader answers in it. + */ +export type TreeAlertObservation = AlertCapture; + +/** One acting choice: press a named button, or press a platform fallback key (Android Back). */ +export type TreeAlertPress = + | { kind: 'press'; button: TButton } + | { kind: 'fallback'; label: string }; + +export type TreeAlertCandidate< + TAlert extends AlertInfo & { buttons: string[] }, + TButton extends TreeAlertButton, +> = Readonly<{ + alert: TAlert; + buttons: TButton[]; +}>; + +export type TreeAlertSleep = (milliseconds: number) => Promise; + +/** + * The seams one tree-based backend owns. `timeoutMs` bounds `wait`; the acting legs ride the + * contract action budget. The sleep seam exists so tests advance polls on a driven clock + * (`docs/agents/testing.md`) instead of production time. + */ +export type TreeAlertDriver< + TCapture, + TAlert extends AlertInfo & { buttons: string[] }, + TButton extends TreeAlertButton, + TCandidate extends TreeAlertCandidate, + TPlatform extends AlertPlatform, +> = Readonly<{ + platform: TPlatform; timeoutMs?: number; - captureNodes: () => Promise; -}; + sleep?: TreeAlertSleep; + capture: () => Promise; + findCandidate: (capture: TCapture) => TreeAlertObservation; + choose: (candidate: TCandidate, action: 'accept' | 'dismiss') => TreeAlertPress | null; + press: (choice: TreeAlertPress) => Promise; +}>; -export type AndroidAlertResult = +/** The records the daemon composes CLI and SDK output around, identical across tree backends. */ +export type TreeAlertResult< + TAlert extends AlertInfo & { buttons: string[] }, + TPlatform extends AlertPlatform, +> = | { kind: 'alertStatus'; - platform: 'android'; + platform: TPlatform; action: 'get'; - alert: AndroidAlertInfo | null; + alert: TAlert | null; message?: string; } | { kind: 'alertWait'; - platform: 'android'; + platform: TPlatform; action: 'wait'; - alert: AndroidAlertInfo; + alert: TAlert; waitedMs: number; message?: string; } | { kind: 'alertHandled'; - platform: 'android'; + platform: TPlatform; action: 'accept' | 'dismiss'; handled: true; - alert: AndroidAlertInfo; + alert: TAlert; button: string; coordinates?: { x: number; y: number }; message?: string; }; -export async function handleAndroidAlert( - device: DeviceInfo, +/** One observation of the backend's own capture, bound by the driver from its two seams. */ +type TreeAlertObserver = () => Promise>; + +const ALERT_NOT_FOUND_HINT = + 'If a sheet is visible in snapshot but alert reports no alert, it is likely app-owned UI. Use snapshot -i and press the visible label/ref.'; + +/** + * Runs one alert leg. The deadline order is read-first on every budget: over a positive window + * both predecessor loops took their first immediate observation before any waiting, and one + * immediate observation is the honest answer shape for a zero-length `wait` window too. A + * malformed `timeoutMs` is the bounded default, never a loop whose bound is never reached. + */ +export async function runTreeAlertAction< + TCapture, + TAlert extends AlertInfo & { buttons: string[] }, + TButton extends TreeAlertButton, + TCandidate extends TreeAlertCandidate, + TPlatform extends AlertPlatform, +>( action: AlertAction, - options: AndroidAlertOptions, -): Promise { - if (action === 'wait') { - return await waitForAndroidAlert( - options.captureNodes, - options.timeoutMs ?? DEFAULT_ALERT_TIMEOUT_MS, - ); - } + driver: TreeAlertDriver, +): Promise> { + const nap = driver.sleep ?? sleep; + const observe: TreeAlertObserver = async () => + driver.findCandidate(await driver.capture()); if (action === 'get') { - const candidate = await readAndroidAlertCandidate(options.captureNodes); - return buildAndroidAlertStatusResponse(candidate?.alert ?? null); + return buildTreeAlertStatusResponse(driver.platform, await observe()); + } + if (action === 'wait') { + const start = Date.now(); + const polled = await pollTreeAlert(observe, alertWindowMs(driver.timeoutMs), nap); + if (polled.kind !== 'alert') { + // Both a quiet capture and one that stayed alert-shaped yet unresolvable mean no + // actionable alert ever resolved within the window; the timeout phrasing is the honest + // answer for either. + throw new AppError('COMMAND_FAILED', 'alert wait timed out'); + } + return { + kind: 'alertWait', + platform: driver.platform, + action: 'wait', + alert: polled.candidate.alert, + waitedMs: Date.now() - start, + ...successText('Alert visible'), + }; } - return await handleAndroidAlertAction(device, action, options.captureNodes); + return await actOnTreeAlert(action, driver, observe, nap); } -async function waitForAndroidAlert( - captureNodes: AndroidAlertOptions['captureNodes'], +function alertWindowMs(timeoutMs: number | undefined): number { + return typeof timeoutMs === 'number' && Number.isFinite(timeoutMs) && timeoutMs >= 0 + ? timeoutMs + : DEFAULT_ALERT_TIMEOUT_MS; +} + +/** + * Polls for an actionable candidate within the budget. The terminal answer distinguishes a + * window that stayed quiet (`none`) from one that stayed alert-shaped yet unresolvable + * (`undecided`): a degraded sheet that never resolved is not an absence, and callers must not + * report the two alike. + */ +async function pollTreeAlert( + observe: TreeAlertObserver, timeoutMs: number, -): Promise { + nap: TreeAlertSleep, +): Promise> { const start = Date.now(); - const candidate = await pollAndroidAlertCandidate(captureNodes, timeoutMs); - if (!candidate) { - throw new AppError('COMMAND_FAILED', 'alert wait timed out'); + let sawUndecided = false; + for (;;) { + const observation = await observe(); + if (observation.kind === 'alert') return observation; + if (observation.kind === 'undecided') sawUndecided = true; + if (Date.now() - start >= timeoutMs) { + return sawUndecided ? { kind: 'undecided' } : { kind: 'none' }; + } + await nap(ALERT_POLL_INTERVAL_MS); } - return { - kind: 'alertWait', - platform: 'android', - action: 'wait', - alert: candidate.alert, - waitedMs: Date.now() - start, - ...successText('Alert visible'), - }; } -async function handleAndroidAlertAction( - device: DeviceInfo, +/** + * Accept and dismiss share one contract on every tree backend: look within the action budget, + * press exactly the one choice the action's role names, then confirm the alert is gone. + * `dismiss` never authorizes anything — an alert whose denying choice cannot be named is + * reported as the ambiguity it is instead of pressing something. + */ +async function actOnTreeAlert< + TCapture, + TAlert extends AlertInfo & { buttons: string[] }, + TButton extends TreeAlertButton, + TCandidate extends TreeAlertCandidate, + TPlatform extends AlertPlatform, +>( action: 'accept' | 'dismiss', - captureNodes: AndroidAlertOptions['captureNodes'], -): Promise { - const candidate = await pollAndroidAlertCandidate(captureNodes, ALERT_ACTION_RETRY_MS); - if (!candidate) { - throw new AppError('COMMAND_FAILED', 'alert not found', { - hint: 'If a sheet is visible in snapshot but alert reports no alert, it is likely app-owned UI. Use snapshot -i and press the visible label/ref.', - }); + driver: TreeAlertDriver, + observe: TreeAlertObserver, + nap: TreeAlertSleep, +): Promise> { + const polled = await pollTreeAlert(observe, ALERT_ACTION_RETRY_MS, nap); + if (polled.kind === 'undecided') { + throw new AppError( + 'COMMAND_FAILED', + `alert ${action} refused: the capture is alert-shaped but cannot be resolved to one alert`, + { + reason: ALERT_CAPTURE_UNDECIDED_REASON, + dispatched: 'no', + hint: 'Inspect alert get --json and snapshot -i, then press the visible choice by label/ref.', + }, + ); } - - const button = chooseAndroidAlertButton(candidate.buttons, action); - if (button) { - await pressAndroid(device, button.x, button.y); - await confirmAndroidAlertDismissed(candidate.alert, action, captureNodes); - return buildAndroidAlertHandledResponse(action, candidate.alert, button.label, { - x: button.x, - y: button.y, + if (polled.kind === 'none') { + throw new AppError('COMMAND_FAILED', 'alert not found', { + reason: ALERT_NOT_FOUND_REASON, + dispatched: 'no', + hint: ALERT_NOT_FOUND_HINT, }); } - - if (action === 'dismiss') { - await backAndroid(device); - await confirmAndroidAlertDismissed(candidate.alert, action, captureNodes); - return buildAndroidAlertHandledResponse(action, candidate.alert, 'Back'); + const candidate = polled.candidate; + const choice = driver.choose(candidate, action); + if (!choice) { + throw new AppError( + 'COMMAND_FAILED', + `alert ${action} found an alert but no button that clearly matches the action`, + { + reason: ALERT_BUTTON_UNDECIDED_REASON, + dispatched: 'no', + alert: candidate.alert, + hint: 'Inspect alert get --json for the visible buttons, then press the choice you want by label/ref.', + }, + ); } - - throw new AppError('COMMAND_FAILED', 'alert accept found an alert but no accept button', { + await driver.press(choice); + await confirmTreeAlertDismissed(observe, candidate.alert, action, nap); + return { + kind: 'alertHandled', + platform: driver.platform, + action, + handled: true, alert: candidate.alert, - hint: 'Inspect alert get --json for visible buttons, then use press by visible label/ref if needed.', - }); + button: choice.kind === 'press' ? choice.button.label : choice.label, + ...(choice.kind === 'press' ? { coordinates: { x: choice.button.x, y: choice.button.y } } : {}), + ...successText(`Alert ${action}ed`), + }; } /** - * `alert accept|dismiss` means the dialog is gone, not that a button was pressed: the next - * command reads the app, and a capture that lands while the dialog window is still up sees - * only the dialog. A different alert taking its place counts as dismissed. Bounded by the - * same budget the pre-press lookup uses; iOS's runner applies the same re-check. + * `alert accept|dismiss` means the dialog is gone, not that a press was sent: the next command + * reads the app, and a capture that lands while the dialog window is still up sees only the + * dialog. Only two answers confirm that: observed absence (`none`), or a different alert taking + * the first's place. An undecided post-press capture proves nothing about absence, so it is + * retried within the budget and reported as a failure at its end rather than returned as + * handled. Bounded by the same budget the pre-press lookup uses; iOS's runner applies the same + * re-check. */ -async function confirmAndroidAlertDismissed( - dismissed: AndroidAlertInfo, +async function confirmTreeAlertDismissed< + TAlert extends AlertInfo & { buttons: string[] }, + TCandidate extends TreeAlertCandidate, +>( + observe: TreeAlertObserver, + dismissed: TAlert, action: 'accept' | 'dismiss', - captureNodes: AndroidAlertOptions['captureNodes'], + nap: TreeAlertSleep, ): Promise { - const { settled } = await pollUntilSettled({ - probe: () => readAndroidAlertCandidate(captureNodes), - settled: (current) => !current || !sameAndroidAlert(current.alert, dismissed), - deadline: Deadline.fromTimeoutMs(ALERT_ACTION_RETRY_MS), - intervalMs: ALERT_POLL_INTERVAL_MS, - }); - if (settled) return; - throw new AppError('COMMAND_FAILED', `alert ${action} did not dismiss the visible alert`, { - alert: dismissed, - hint: 'The alert button was pressed but the dialog is still visible. Inspect alert get --json, then press the visible button by label/ref or retry.', - }); + const start = Date.now(); + for (;;) { + const observation = await observe(); + if (observation.kind === 'none') return; + if (observation.kind === 'alert' && !sameTreeAlert(observation.candidate.alert, dismissed)) { + return; + } + if (Date.now() - start >= ALERT_ACTION_RETRY_MS) { + throw new AppError('COMMAND_FAILED', `alert ${action} did not dismiss the visible alert`, { + alert: dismissed, + hint: 'The alert button was pressed but the dialog is still visible. Inspect alert get --json, then press the visible button by label/ref or retry.', + }); + } + await nap(ALERT_POLL_INTERVAL_MS); + } } -function sameAndroidAlert(left: AndroidAlertInfo, right: AndroidAlertInfo): boolean { +/** Alert identity across a dismissal re-check: the same title and the same button row, in order. */ +function sameTreeAlert( + left: AlertInfo & { buttons: string[] }, + right: AlertInfo & { buttons: string[] }, +): boolean { return left.title === right.title && left.buttons.join('\u0000') === right.buttons.join('\u0000'); } -async function pollAndroidAlertCandidate( - captureNodes: AndroidAlertOptions['captureNodes'], - timeoutMs: number, -): Promise { - const { value } = await pollUntilSettled({ - probe: () => readAndroidAlertCandidate(captureNodes), - settled: (candidate) => candidate !== null, - deadline: Deadline.fromTimeoutMs(timeoutMs), - intervalMs: ALERT_POLL_INTERVAL_MS, - }); - return value; -} - -async function readAndroidAlertCandidate( - captureNodes: AndroidAlertOptions['captureNodes'], -): Promise { - const result = await withDiagnosticTimer('snapshot_capture', captureNodes, { - backend: 'android', - purpose: 'alert', - }); - return findAndroidAlertCandidate(result); -} - -function buildAndroidAlertStatusResponse(alert: AndroidAlertInfo | null): AndroidAlertResult { +function buildTreeAlertStatusResponse< + TAlert extends AlertInfo & { buttons: string[] }, + TButton extends TreeAlertButton, + TCandidate extends TreeAlertCandidate, + TPlatform extends AlertPlatform, +>( + platform: TPlatform, + observation: TreeAlertObservation, +): TreeAlertResult { return { kind: 'alertStatus', - platform: 'android', + platform, action: 'get', - alert, - ...(alert ? successText('Alert visible') : successText('No alert visible')), + alert: observation.kind === 'alert' ? observation.candidate.alert : null, + ...(observation.kind === 'alert' + ? successText('Alert visible') + : observation.kind === 'undecided' + ? successText('Cannot decide the alert: the capture is alert-shaped but unresolvable') + : successText('No alert visible')), }; } -function buildAndroidAlertHandledResponse( - action: 'accept' | 'dismiss', - alert: AndroidAlertInfo, - button: string, - coordinates?: { x: number; y: number }, -): AndroidAlertResult { - return { - kind: 'alertHandled', +// --- the Android legs: the driver above plus this platform's own observation and acting policy --- + +type AndroidAlertOptions = { + timeoutMs?: number; + captureNodes: () => Promise; +}; + +export type AndroidAlertResult = TreeAlertResult; + +export async function handleAndroidAlert( + device: DeviceInfo, + action: AlertAction, + options: AndroidAlertOptions, +): Promise { + return await runTreeAlertAction< + RawSnapshotNode[], + AndroidAlertInfo, + AndroidAlertButton, + AndroidAlertCandidate, + 'android' + >(action, { platform: 'android', - action, - handled: true, - alert, - button, - ...(coordinates ? { coordinates } : {}), - ...successText(`Alert ${action}ed`), - }; + timeoutMs: options.timeoutMs, + capture: () => + withDiagnosticTimer('snapshot_capture', options.captureNodes, { + backend: 'android', + purpose: 'alert', + }), + // Android's dump either names a candidate or names none; it has no capture state that can + // reach `undecided`. + findCandidate: (nodes) => { + const candidate = findAndroidAlertCandidate(nodes); + return candidate ? { kind: 'alert', candidate } : { kind: 'none' }; + }, + choose: (candidate, act) => { + const button = chooseAndroidAlertButton(candidate.buttons, act); + if (button) return { kind: 'press', button }; + // No button states any leaving choice: dismissing an Android dialog is what the Back + // key means. The refusal message and typed reason come from the driver, not from here. + return act === 'dismiss' ? { kind: 'fallback', label: 'Back' } : null; + }, + press: async (choice) => { + if (choice.kind === 'press') { + await pressAndroid(device, choice.button.x, choice.button.y); + return; + } + await backAndroid(device); + }, + }); } diff --git a/packages/platform-android/src/mechanics.ts b/packages/platform-android/src/mechanics.ts index a175cb35e8..d2097ca41c 100644 --- a/packages/platform-android/src/mechanics.ts +++ b/packages/platform-android/src/mechanics.ts @@ -48,7 +48,18 @@ export { resetAndroidAdbShellProtocolProbes, } from './adb-shell-protocol.ts'; export { runAndroidAdb, runAndroidExecOut, runAndroidShell, sleep } from './adb.ts'; -export { handleAndroidAlert, type AndroidAlertResult } from './alert.ts'; +export { + handleAndroidAlert, + runTreeAlertAction, + type AndroidAlertResult, + type TreeAlertButton, + type TreeAlertCandidate, + type TreeAlertDriver, + type TreeAlertObservation, + type TreeAlertPress, + type TreeAlertResult, + type TreeAlertSleep, +} from './alert.ts'; export { classifyAndroidAlertIdentifier, findAndroidAlertCandidate, diff --git a/packages/provider-limrun/src/alert-detection.fixtures.ts b/packages/provider-limrun/src/alert-detection.fixtures.ts index 2f0cf4c356..8e645d77f2 100644 --- a/packages/provider-limrun/src/alert-detection.fixtures.ts +++ b/packages/provider-limrun/src/alert-detection.fixtures.ts @@ -80,6 +80,49 @@ export function limrunAlertTreeWithoutButtons(): IosTreeNode { return { ...root, children: (root.children ?? []).filter((n) => n.role !== 'AXButton') }; } +/** + * The live ATT prompt with every choice degraded out of action: one disabled, one explicitly + * unhittable, one label-less and frameless. This is what a capture looks like when the tap was + * ignored or the sheet is mid-transition — the dialog is still visibly up, yet nothing on it + * can be pressed. + */ +export function limrunAlertTreeWithUnpressableButtons(): IosTreeNode { + const root = limrunAttAlertTree(); + return { + ...root, + children: [ + ...(root.children ?? []).slice(0, 2), + { + type: 'Button', + role: 'AXButton', + AXLabel: 'Allow', + pid: root.pid, + enabled: false, + frame: { x: 57, y: 511.67, width: 288, height: 48 }, + traits: ['Button'], + }, + { + type: 'Button', + role: 'AXButton', + AXLabel: 'Ask App Not to Track', + pid: root.pid, + enabled: true, + hittable: false, + frame: { x: 57, y: 567.67, width: 288, height: 48 }, + traits: ['Button'], + }, + { + type: 'Button', + role: 'AXButton', + AXLabel: 'Later', + pid: root.pid, + enabled: true, + traits: ['Button'], + }, + ], + }; +} + /** An alert whose only choices are ambiguous: no label classifies as accept or dismiss. */ export function limrunAlertAmbiguousButtonsTree(): IosTreeNode { const root = limrunAttAlertTree(); @@ -216,31 +259,51 @@ export function limrunAppAlertDialogTree(): IosTreeNode { }; } -/** SpringBoard's home screen: a blank-label root, but full of launch icons rather than a dialog. */ +/** SpringBoard's home screen: a blank-label root, but full of launch icons rather than a dialog. + * Root and icons share SpringBoard's pid, as the live home capture reports it. */ export function limrunHomeScreenTree(): IosTreeNode { - return alertRoot([ - { - type: 'Button', - role: 'AXButton', - AXLabel: 'Maps', - pid: LIMRUN_SPRINGBOARD_PID, - enabled: true, - frame: { x: 24.33, y: 88, width: 168.33, height: 191 }, - traits: ['Scrollable', 'UpdatesFrequently', 'LaunchIcon'], - }, - { - type: 'Button', - role: 'AXButton', - AXLabel: 'Calendar', - pid: LIMRUN_SPRINGBOARD_PID, - enabled: true, - frame: { x: 209.67, y: 88, width: 168.33, height: 191 }, - traits: ['Scrollable', 'UpdatesFrequently', 'LaunchIcon'], - }, - ]); + return { + ...alertRoot([ + { + type: 'Button', + role: 'AXButton', + AXLabel: 'Maps', + pid: LIMRUN_SPRINGBOARD_PID, + enabled: true, + frame: { x: 24.33, y: 88, width: 168.33, height: 191 }, + traits: ['Scrollable', 'UpdatesFrequently', 'LaunchIcon'], + }, + { + type: 'Button', + role: 'AXButton', + AXLabel: 'Calendar', + pid: LIMRUN_SPRINGBOARD_PID, + enabled: true, + frame: { x: 209.67, y: 88, width: 168.33, height: 191 }, + traits: ['Scrollable', 'UpdatesFrequently', 'LaunchIcon'], + }, + ]), + pid: LIMRUN_SPRINGBOARD_PID, + }; } -/** A capture with no alert: the app screen with the ATT alert grafted on as a sibling root. */ +/** + * The app screen and the ATT alert as sibling roots — what a capture with a live system alert + * over an app looks like. The alert is the actionable one here; the app screen is background. + */ export function limrunAlertOverlayOnAppScreenTree(): IosTreeNode[] { return [limrunAppScreenTree(), limrunAttAlertTree()]; } + +/** + * Lookalike (not a live capture): the live app dialog of {@link limrunAppAlertDialogTree} with + * its root's label blanked, content pids untouched. The captures reject it through process + * exclusivity — every live app capture carries foreign-pid status-bar nodes on the app root, + * while the system alert root is process-exclusive. The residual the detector honestly cannot + * separate is an app dialog under a blanked root in a capture with no foreign-pid node at all; + * that is what the label-strict acting legs backstop. + */ +export function limrunBlankRootAppDialogLookalikeTree(): IosTreeNode { + const dialog = limrunAppAlertDialogTree(); + return { ...dialog, AXLabel: ' ' }; +} diff --git a/packages/provider-limrun/src/alert-detection.test.ts b/packages/provider-limrun/src/alert-detection.test.ts index 325543e329..c778438dab 100644 --- a/packages/provider-limrun/src/alert-detection.test.ts +++ b/packages/provider-limrun/src/alert-detection.test.ts @@ -1,16 +1,18 @@ import { describe, expect, it } from 'vitest'; import { chooseLimrunAlertButton, - findLimrunAlertCandidate, isLimrunAlertHostRoot, + readLimrunAlertCapture, } from './alert-detection.ts'; import { limrunAlertAmbiguousButtonsTree, limrunAlertOverlayOnAppScreenTree, + limrunAlertTreeWithUnpressableButtons, limrunAlertTreeWithoutButtons, limrunAppAlertDialogTree, limrunAppScreenTree, limrunAttAlertTree, + limrunBlankRootAppDialogLookalikeTree, limrunHomeScreenTree, } from './alert-detection.fixtures.ts'; @@ -49,39 +51,48 @@ describe('isLimrunAlertHostRoot', () => { }); }); -describe('findLimrunAlertCandidate', () => { +describe('readLimrunAlertCapture', () => { it('reads the live ATT alert: title, message, and both buttons in tree order', () => { - const candidate = findLimrunAlertCandidate(limrunAttAlertTree()); - expect(candidate).not.toBeNull(); - expect(candidate?.alert.title).toBe( + const capture = readLimrunAlertCapture(limrunAttAlertTree()); + expect(capture.kind).toBe('alert'); + if (capture.kind !== 'alert') return; + expect(capture.candidate.alert.title).toBe( 'Allow “ATTProbe” to track your activity across other companies’ apps and websites?', ); - expect(candidate?.alert.message).toBe( + expect(capture.candidate.alert.message).toBe( 'This build asks for tracking authorization once, for automation testing.', ); - expect(candidate?.alert.buttons).toEqual(['Ask App Not to Track', 'Allow']); - expect(candidate?.alert.platform).toBe('ios'); - expect(candidate?.alert.source).toBe('permission'); - expect(candidate?.buttons.map((button) => button.role)).toEqual(['dismiss', 'accept']); + expect(capture.candidate.alert.buttons).toEqual(['Ask App Not to Track', 'Allow']); + expect(capture.candidate.alert.platform).toBe('ios'); + expect(capture.candidate.alert.source).toBe('permission'); + expect(capture.candidate.buttons.map((button) => button.role)).toEqual(['dismiss', 'accept']); // The tap point is the live button frame's center, rounded the way `centerOfRect` rounds. - expect(candidate?.buttons[1]).toEqual({ label: 'Allow', x: 201, y: 592, role: 'accept' }); + expect(capture.candidate.buttons[1]).toEqual({ + label: 'Allow', + x: 201, + y: 592, + role: 'accept', + }); }); it('finds the alert when the app screen is the first root of the capture', () => { - const candidate = findLimrunAlertCandidate(limrunAlertOverlayOnAppScreenTree()); - expect(candidate?.alert.buttons).toEqual(['Ask App Not to Track', 'Allow']); + const capture = readLimrunAlertCapture(limrunAlertOverlayOnAppScreenTree()); + expect(capture.kind === 'alert' && capture.candidate.alert.buttons).toEqual([ + 'Ask App Not to Track', + 'Allow', + ]); }); it('refuses the live app-owned UIAlertController: a named root is app UI, not a host sheet', () => { - expect(findLimrunAlertCandidate(limrunAppAlertDialogTree())).toBeNull(); + expect(readLimrunAlertCapture(limrunAppAlertDialogTree()).kind).toBe('none'); }); it('refuses a plain app screen', () => { - expect(findLimrunAlertCandidate(limrunAppScreenTree())).toBeNull(); + expect(readLimrunAlertCapture(limrunAppScreenTree()).kind).toBe('none'); }); it("refuses SpringBoard's home screen: launch icons are not a dialog", () => { - expect(findLimrunAlertCandidate(limrunHomeScreenTree())).toBeNull(); + expect(readLimrunAlertCapture(limrunHomeScreenTree()).kind).toBe('none'); }); it('refuses a widget-bearing home screen: LaunchIcon buttons never qualify', () => { @@ -99,118 +110,140 @@ describe('findLimrunAlertCandidate', () => { ...(home.children ?? []), ], }; - expect(findLimrunAlertCandidate(withWidgets)).toBeNull(); + expect(readLimrunAlertCapture(withWidgets).kind).toBe('none'); }); - it('refuses an alert-shaped sheet with no actionable button', () => { - expect(findLimrunAlertCandidate(limrunAlertTreeWithoutButtons())).toBeNull(); + it('reports a text-only sheet on a blank root as undecided, not absence', () => { + // Buttons can drop out between captures mid-transition; a message on a host-shaped root + // is alert-shaped content the reader cannot resolve, and absence is a stronger claim. + expect(readLimrunAlertCapture(limrunAlertTreeWithoutButtons()).kind).toBe('undecided'); }); - it('refuses a one-choice sheet: no action can tell its single option from the asked-for one', () => { - expect(findLimrunAlertCandidate(alertTreeWithButtons('Allow'))).toBeNull(); - expect(findLimrunAlertCandidate(alertTreeWithButtons('OK'))).toBeNull(); + it('reports a one-choice sheet as undecided: alert-shaped, not resolvable to an action', () => { + expect(readLimrunAlertCapture(alertTreeWithButtons('Allow')).kind).toBe('undecided'); + expect(readLimrunAlertCapture(alertTreeWithButtons('OK')).kind).toBe('undecided'); }); - it('refuses a capture whose host sheets disagree, rather than shadowing one with the other', () => { - const second = { ...limrunAttAlertTree(), pid: 99 }; - expect(findLimrunAlertCandidate([limrunAttAlertTree(), second])).toBeNull(); + it('reports a capture mixing an app screen and a text-only sheet as undecided', () => { + expect( + readLimrunAlertCapture([limrunAppScreenTree(), limrunAlertTreeWithoutButtons()]).kind, + ).toBe('undecided'); + }); + + it('reports a capture whose host sheets disagree as undecided, not as no alert', () => { + const second = { + ...limrunAttAlertTree(), + pid: 99, + children: (limrunAttAlertTree().children ?? []).map((child) => ({ ...child, pid: 99 })), + }; + expect(readLimrunAlertCapture([limrunAttAlertTree(), second]).kind).toBe('undecided'); + }); + + it('reports a blank-label root that reports no pid as undecided: unverifiable, not absent', () => { + const root = limrunAttAlertTree(); + expect(readLimrunAlertCapture({ ...root, pid: undefined }).kind).toBe('undecided'); + }); + + it('reports a blank-label root whose content belongs to another process as undecided', () => { + const root = limrunAttAlertTree(); + const foreignContent = { + ...root, + children: (root.children ?? []).map((child) => ({ ...child, pid: 4242 })), + }; + expect(readLimrunAlertCapture(foreignContent).kind).toBe('undecided'); + }); + + it('reports the blank-root app-dialog lookalike as undecided, never an actionable alert', () => { + // The live app captures ride SpringBoard status nodes (a foreign pid) on the app root; the + // system alert root is process-exclusive. Blanking the app dialog's label keeps that + // discriminator intact: the capture is alert-shaped yet never resolves to a candidate some + // action could press into. + expect(readLimrunAlertCapture(limrunBlankRootAppDialogLookalikeTree()).kind).toBe('undecided'); + }); + + it('reports a blank-label sheet sharing its root with another process as undecided', () => { + const root = limrunAttAlertTree(); + const shared = { + ...root, + children: [ + ...(root.children ?? []), + { + type: 'GenericElement', + role: 'AXGenericElement', + AXLabel: 'Cellular', + pid: 4242, + frame: { x: 288, y: 26, width: 19, height: 12 }, + }, + ], + }; + expect(readLimrunAlertCapture(shared).kind).toBe('undecided'); + }); + + it('reports a sheet whose every button is unpressable as undecided, not absence', () => { + // Disabled, explicitly unhittable, and frameless buttons press nothing; the dialog they + // belong to is still visibly up, so the capture must not claim absence. + expect(readLimrunAlertCapture(limrunAlertTreeWithUnpressableButtons()).kind).toBe('undecided'); }); it('reports a permission alert from its button vocabulary and a plain one as a system dialog', () => { - expect(findLimrunAlertCandidate(limrunAttAlertTree())?.alert.source).toBe('permission'); - expect(findLimrunAlertCandidate(alertTreeWithButtons('No', 'Yes'))?.alert.source).toBe( - 'system-dialog', - ); + const att = readLimrunAlertCapture(limrunAttAlertTree()); + expect(att.kind === 'alert' && att.candidate.alert.source).toBe('permission'); + const plain = readLimrunAlertCapture(alertTreeWithButtons('No', 'Yes')); + expect(plain.kind === 'alert' && plain.candidate.alert.source).toBe('system-dialog'); }); }); describe('chooseLimrunAlertButton', () => { + function buttonsOf(tree: Parameters[0]) { + const capture = readLimrunAlertCapture(tree); + if (capture.kind !== 'alert') throw new Error('expected an alert capture'); + return capture.candidate.buttons; + } + it('accepts the affirmative and dismisses the denying choice of the ATT prompt', () => { - const buttons = findLimrunAlertCandidate(limrunAttAlertTree())!.buttons; + const buttons = buttonsOf(limrunAttAlertTree()); expect(chooseLimrunAlertButton(buttons, 'accept')?.label).toBe('Allow'); expect(chooseLimrunAlertButton(buttons, 'dismiss')?.label).toBe('Ask App Not to Track'); }); it('fails an ambiguous accept closed when no button classifies as affirmative', () => { - const buttons = findLimrunAlertCandidate(limrunAlertAmbiguousButtonsTree())!.buttons; + const buttons = buttonsOf(limrunAlertAmbiguousButtonsTree()); expect(chooseLimrunAlertButton(buttons, 'accept')).toBeNull(); }); it('fails an ambiguous dismiss closed when no button classifies as denying', () => { - const buttons = findLimrunAlertCandidate(limrunAlertAmbiguousButtonsTree())!.buttons; + const buttons = buttonsOf(limrunAlertAmbiguousButtonsTree()); expect(chooseLimrunAlertButton(buttons, 'dismiss')).toBeNull(); }); it('never picks a choice of the wrong role, whatever else the alert offers', () => { - const allowOnly = findLimrunAlertCandidate(alertTreeWithButtons('Deny', 'Allow'))!.buttons; + const allowOnly = buttonsOf(alertTreeWithButtons('Deny', 'Allow')); expect(chooseLimrunAlertButton(allowOnly, 'accept')?.label).toBe('Allow'); expect(chooseLimrunAlertButton(allowOnly, 'dismiss')?.label).toBe('Deny'); - const noAccept = findLimrunAlertCandidate( - alertTreeWithButtons('Cancel', 'Maybe Later'), - )!.buttons; + const noAccept = buttonsOf(alertTreeWithButtons('Cancel', 'Maybe Later')); expect(chooseLimrunAlertButton(noAccept, 'accept')).toBeNull(); - const noDismiss = findLimrunAlertCandidate( - alertTreeWithButtons('Open Settings', 'Continue'), - )!.buttons; + const noDismiss = buttonsOf(alertTreeWithButtons('Open Settings', 'Continue')); expect(chooseLimrunAlertButton(noDismiss, 'dismiss')).toBeNull(); }); it('classifies the classic iOS pair, the typographic apostrophe, and iOS variant grants', () => { - const classic = findLimrunAlertCandidate(alertTreeWithButtons("Don't Allow", 'OK'))!.buttons; + const classic = buttonsOf(alertTreeWithButtons("Don't Allow", 'OK')); expect(chooseLimrunAlertButton(classic, 'accept')?.label).toBe('OK'); expect(chooseLimrunAlertButton(classic, 'dismiss')?.label).toBe("Don't Allow"); - const curly = findLimrunAlertCandidate(alertTreeWithButtons('Don’t Allow', 'Allow'))!.buttons; + const curly = buttonsOf(alertTreeWithButtons('Don’t Allow', 'Allow')); expect(chooseLimrunAlertButton(curly, 'dismiss')?.label).toBe('Don’t Allow'); - const settings = findLimrunAlertCandidate( - alertTreeWithButtons('Not Now', 'Open Settings'), - )!.buttons; + const settings = buttonsOf(alertTreeWithButtons('Not Now', 'Open Settings')); expect(chooseLimrunAlertButton(settings, 'accept')?.label).toBe('Open Settings'); expect(chooseLimrunAlertButton(settings, 'dismiss')?.label).toBe('Not Now'); for (const label of ['Allow Once', 'Allow All the Time', 'Allow While Using App']) { - const granted = findLimrunAlertCandidate( - alertTreeWithButtons('Ask App Not to Track', label), - )!.buttons; + const granted = buttonsOf(alertTreeWithButtons('Ask App Not to Track', label)); expect(chooseLimrunAlertButton(granted, 'accept')?.label).toBe(label); } }); - it('refuses the candidate when no button is pressable', () => { - const root = limrunAttAlertTree(); - const crippled = { - ...root, - children: [ - ...(root.children ?? []).slice(0, 2), - { - type: 'Button', - role: 'AXButton', - AXLabel: 'Allow', - pid: root.pid, - enabled: false, - frame: { x: 57, y: 511.67, width: 288, height: 48 }, - traits: ['Button'], - }, - { - type: 'Button', - role: 'AXButton', - AXLabel: 'Ask App Not to Track', - pid: root.pid, - enabled: true, - hittable: false, - frame: { x: 57, y: 567.67, width: 288, height: 48 }, - traits: ['Button'], - }, - { - type: 'Button', - role: 'AXButton', - AXLabel: 'Later', - pid: root.pid, - enabled: true, - traits: ['Button'], - }, - ], - }; - // Disabled, unhittable, and frameless buttons are all unpressable; with nothing pressable - // left, the whole candidate is refused rather than half-served. - expect(findLimrunAlertCandidate(crippled)).toBeNull(); + it('refuses a choice two physical buttons share: pressing one of them would guess', () => { + const doubled = buttonsOf(alertTreeWithButtons('Allow', 'Deny', 'Allow')); + expect(chooseLimrunAlertButton(doubled, 'accept')).toBeNull(); + expect(chooseLimrunAlertButton(doubled, 'dismiss')?.label).toBe('Deny'); }); }); diff --git a/packages/provider-limrun/src/alert-detection.ts b/packages/provider-limrun/src/alert-detection.ts index 409c3fbb13..837d5620f1 100644 --- a/packages/provider-limrun/src/alert-detection.ts +++ b/packages/provider-limrun/src/alert-detection.ts @@ -1,33 +1,45 @@ -import type { AlertInfo, AlertSource } from '@agent-device/contracts/alert-contract'; +import { + type AlertButtonRole, + classifyAlertButtonRole, + type AlertInfo, + type AlertSource, + selectUniqueAlertButtonByRole, +} from '@agent-device/contracts/alert-contract'; import { centerOfRect, type Rect } from '@agent-device/kernel/snapshot'; import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; +import type { TreeAlertObservation } from '@agent-device/platform-android/mechanics'; import { readIosNodeChildren, readIosNodeRect, type IosTreeNode } from './snapshot.ts'; /** * System-alert detection over a Limrun iOS element tree (issue #3349). A Limrun iOS direct * session has no XCUITest runner, but its element tree does carry the system alert: on a live - * instance the ATT prompt arrives as its own `AXApplication` root whose `AXLabel` is blank, with - * its text and buttons as direct children all sharing the hosting process's `pid` — never the - * app's. The app's own `UIAlertController`, by contrast, rides inside the app's root, labeled - * with the app name and carrying the app's pid. The rules below encode that observed split and - * refuse everything it does not vouch for: a named root is app UI, launch-icon buttons are a - * home screen, and a button without a frame or a live `pid` match across visible roots is not - * pressable. What one live capture cannot exclude — an app that left its own root's label blank - * while showing a dialog — is why the acting legs press only label-unambiguous buttons below. + * instance the ATT prompt arrived as its own `AXApplication` root whose `AXLabel` is blank, with + * its text and buttons as direct children all sharing one `pid` — never the app's. The app's own + * `UIAlertController`, by contrast, rode inside the app's root, labeled with the app name and + * carrying the app's pid, and SpringBoard's status nodes rode that app root under a third pid. + * Nothing in a capture names a package as owner, so the host rule is a heuristic over these + * observations: a host sheet qualifies only on a blank-label root that reports a `pid` and whose + * entire subtree reports that same one. A named root is app UI; launch-icon buttons are a home + * screen; a button without a frame has no point to press. The documented residual — what these + * captures cannot exclude — is an app that leaves its own root's label blank while showing a + * dialog that no other process rides; the acting legs press only label-unambiguous buttons as + * the backstop, never a guess. + * + * Button action eligibility never asserts hitability: the captures omit the `hittable` key + * entirely, so an explicit `hittable: false` refuses a button and an omitted one carries no + * signal — what admits a button is the enabled, label, and complete-rect rules. */ -type LimrunAlertButtonRole = 'accept' | 'dismiss' | 'neutral'; +/** Only the two host-sheet sources exist on this surface: no node names a package to attribute. */ +type LimrunAlertSource = Extract; export type LimrunAlertButton = Readonly<{ label: string; x: number; y: number; - role: LimrunAlertButtonRole; + role: AlertButtonRole; }>; -/** Only the two host-sheet sources exist on this surface: no node names a package to attribute. */ -type LimrunAlertSource = Extract; - export type LimrunAlertInfo = AlertInfo & { buttons: string[]; platform: 'ios'; @@ -39,11 +51,11 @@ export type LimrunAlertCandidate = Readonly<{ buttons: LimrunAlertButton[]; }>; -const ACCEPT_LABEL_PATTERN = - /^(?:ok|allow|allow all|allow once|allow all the time|allow while using (?:the )?app|allow notifications|always allow|yes|continue|save|confirm|turn on|open settings|while using the app|only this time)$/i; -// `don.?t allow` covers the straight, missing, and typographic apostrophe in one alternate. -const DISMISS_LABEL_PATTERN = - /^(?:cancel|deny|don.?t allow|not now|no|dismiss|close|close app|later|skip|ask app not to track)$/i; +// The button roles come from the shared tree-alert label vocabulary in contracts/alert-contract: +// the same label states the same role on Android and here, so `alert dismiss` cannot act +// differently on the two platforms for one label. English-only, matching the untranslated +// device locale these instances run; a localized alert resolves as a candidate whose buttons +// all classify neutral, and the acting legs refuse on the button-undecided reason. /** The subset that states a grant/deny choice; a plain Yes/No dialog is not a permission prompt. */ const PERMISSION_LABEL_PATTERN = /^(?:allow.*|deny|don.?t allow|ask app not to track|turn on|open settings)$/i; @@ -67,44 +79,74 @@ export function isLimrunAlertHostRoot(node: IosTreeNode): boolean { } /** - * The actionable alert on this capture, or null. A host root qualifies only when its content is - * dialog-shaped: at least two pressable choices (enabled, hittable, with a complete rect), at - * least one piece of text, and no SpringBoard launch icons. One live capture cannot exclude an - * app that leaves its own root's label blank while showing a dialog — the provider reports no - * capture-level fact that separates it from the alert host — which is why the acting legs press - * only buttons whose label states the asked-for choice, and an alert-shaped sheet offering a - * single choice is refused entirely rather than acted on by position. - * - * The button vocabulary is English, matching the untranslated simulator locale these instances - * run; a localized alert classifies as undecided and the acting legs report the ambiguity rather - * than guessing. + * The one capture reader: what this element tree says about system alerts, in the shared + * tree-alert observation the driver confirms against. `none` states the capture carries no + * alert-shaped content on any host root; `undecided` states it carries alert-shaped content this + * reader cannot resolve into one actionable alert — more than one alert-shaped host sheet, or a + * sheet whose choices are visible yet unactionable (a degraded or shared root). Only `alert` and + * `none` carry information; `undecided` is not absence, and confirming a dismissal may accept + * `none` but never `undecided`. */ -export function findLimrunAlertCandidate( - tree: IosTreeNode | IosTreeNode[], -): LimrunAlertCandidate | null { - const candidates: LimrunAlertCandidate[] = []; +export type LimrunAlertCapture = TreeAlertObservation; + +/** + * The actionable alert on this capture, or why the capture answers nothing. A host sheet is + * actionable only on a blank-label, process-exclusive root (it reports a `pid` and its whole + * subtree reports that same one) offering at least two pressable choices (enabled, not reported + * unhittable, with a complete rect and a label) under at least one piece of text, with no + * SpringBoard launch icons. A root that shows the same dialog-shaped content but fails one of + * those tests — a button dropping to unhittable, disabled, frameless, or label-less between + * captures, a root sharing its subtree with another pid — still shows a dialog, so the capture + * is undecided, not absent. A named root is app UI, a home screen offers launch icons, and a + * blank root with neither a message nor a pair of button-shaped nodes carries nothing a dialog + * could have degraded from: those are absence. What these captures cannot exclude is an app + * dialog under a blanked root no other process rides; the label-strict acting legs are the + * backstop named above. + */ +export function readLimrunAlertCapture(tree: IosTreeNode | IosTreeNode[]): LimrunAlertCapture { + let candidate: LimrunAlertCandidate | null = null; + let alertShapedRoots = 0; for (const root of Array.isArray(tree) ? tree : [tree]) { if (!isLimrunAlertHostRoot(root)) continue; - const candidate = readAlertCandidateFromHostRoot(root); - if (candidate) candidates.push(candidate); + const observed = readAlertHostRoot(root); + if (observed === 'absent') continue; + alertShapedRoots += 1; + if (observed !== 'undecided') candidate = observed; } - // More than one alert-shaped host sheet is a shape this reader does not understand; pressing - // into it would guess, and the observation would silently shadow one with the other. - return candidates.length === 1 ? candidates[0]! : null; + // One actionable sheet among nothing else alert-shaped is the answer. Two sheets, or one that + // is alert-shaped yet unactionable, is content this reader must not resolve by guessing — + // pressing into it would pick a side the observation does not support. + if (alertShapedRoots === 0) return { kind: 'none' }; + return alertShapedRoots === 1 && candidate ? { kind: 'alert', candidate } : { kind: 'undecided' }; } -function readAlertCandidateFromHostRoot(root: IosTreeNode): LimrunAlertCandidate | null { +type LimrunHostRootReading = LimrunAlertCandidate | 'undecided' | 'absent'; + +function readAlertHostRoot(root: IosTreeNode): LimrunHostRootReading { const descendants = collectDescendants(root); - const buttons = findAlertButtons(descendants); - // A lone choice says nothing about which action the caller asked for; refuse the sheet. - if (buttons.length < 2) return null; + // A home screen is a home screen: SpringBoard marks its icons, and widgets can carry text, + // so the trait answers absence before any dialog question is asked. + if (descendants.some(isLaunchIcon)) return 'absent'; const texts = descendants - .filter((node) => isStaticTextNode(node) && !isButtonNode(node)) + .filter(isStaticTextNode) .map(nodeLabel) .filter((text) => text.length > 0); - // A dialog says something: SpringBoard's blank-label home screen offers launch icons, not a - // message with a choice, and stays unreported. - if (texts.length === 0) return null; + const choices = descendants.filter((node) => isButtonNode(node) && !isLaunchIcon(node)); + // Absence means nothing a dialog could have degraded from: no message and no pair of + // button-shaped nodes. + if (texts.length === 0 && choices.length < 2) return 'absent'; + // Process exclusivity is the capture-side host heuristic named above: the live system alert + // root was process-exclusive (one pid on every node) while the app root rode SpringBoard's + // status nodes under a second pid. It settles actionability, never absence. + const exclusive = + typeof root.pid === 'number' && descendants.every((node) => node.pid === root.pid); + if (!exclusive) return 'undecided'; + const buttons = findAlertButtons(descendants); + // Dialog-shaped content that is not actionable — a sheet rendered mid-transition, one whose + // buttons just dropped to disabled, unhittable, or frameless, a message-less button pair — + // still shows something alert-shaped. That is undecided, never absence, so no caller can + // report it as a confirmed no-alert. + if (texts.length === 0 || buttons.length < 2) return 'undecided'; const [title, ...rest] = texts; const alert: LimrunAlertInfo = { ...(title ? { title } : {}), @@ -132,29 +174,32 @@ function isButtonNode(node: IosTreeNode): boolean { return node.role === 'AXButton' || node.type === 'Button'; } +function isLaunchIcon(node: IosTreeNode): boolean { + return node.traits?.includes(LAUNCH_ICON_TRAIT) === true; +} + function isStaticTextNode(node: IosTreeNode): boolean { return node.role === 'AXStaticText' || node.type === 'StaticText'; } /** * Pressable alert choices only: a disabled button is not a choice the alert offers right now, a - * button without a complete rect has no point the touch seam could aim at, a button the provider - * reports as unhittable cannot be pressed, and a `LaunchIcon` is a home-screen app icon rather - * than a dialog button. + * button without a complete rect has no point the touch seam could aim at, and a button the + * provider explicitly reports as `hittable: false` cannot be pressed; an omitted `hittable` + * carries no signal either way. Process membership was settled by the caller, so every physical + * node stays in the list: two buttons sharing a label are two points a tap could land on, and + * the chooser refuses rather than taking the first. */ function findAlertButtons(nodes: IosTreeNode[]): LimrunAlertButton[] { - const seen = new Set(); const buttons: LimrunAlertButton[] = []; for (const node of nodes) { if (!isButtonNode(node) || node.enabled === false || node.hittable === false) continue; - if (node.traits?.includes(LAUNCH_ICON_TRAIT)) continue; const label = nodeLabel(node); - if (!label || seen.has(label.toLowerCase())) continue; + if (!label) continue; const rect = readIosNodeRect(node); if (!isPressableRect(rect)) continue; - seen.add(label.toLowerCase()); const point = centerOfRect(rect); - buttons.push({ label, x: point.x, y: point.y, role: classifyAlertButton(label) }); + buttons.push({ label, x: point.x, y: point.y, role: classifyAlertButtonRole(label) }); } return buttons; } @@ -163,13 +208,6 @@ function isPressableRect(rect: Rect | undefined): rect is Rect { return isPositiveFiniteRect(rect); } -function classifyAlertButton(label: string): LimrunAlertButtonRole { - const trimmed = label.trim(); - if (ACCEPT_LABEL_PATTERN.test(trimmed)) return 'accept'; - if (DISMISS_LABEL_PATTERN.test(trimmed)) return 'dismiss'; - return 'neutral'; -} - /** A button vocabulary naming a grant/deny choice makes this a permission prompt. */ function classifyAlertSource(buttons: readonly LimrunAlertButton[]): LimrunAlertSource { return buttons.some((button) => PERMISSION_LABEL_PATTERN.test(button.label.trim())) @@ -180,15 +218,14 @@ function classifyAlertSource(buttons: readonly LimrunAlertButton[]): LimrunAlert /** * The button one action may press, or null when pressing any visible choice would guess. * Deliberately stricter than Android's: with no `android:id/button1`-style identity to fall back - * on, only a button whose own label states the asked-for choice may be pressed, and even then - * only when exactly one choice does — an accept must never press a denial, and an alert whose - * choices leave the action undecided is refused rather than resolved by tree position. + * on, only a button whose own label states the asked-for choice may be pressed, resolved once + * through the shared unique-choice boundary (duplicated roles refuse) — an accept must never + * press a denial, and an alert whose choices leave the action undecided is refused rather than + * resolved by tree position. */ export function chooseLimrunAlertButton( buttons: readonly LimrunAlertButton[], action: 'accept' | 'dismiss', ): LimrunAlertButton | null { - const wanted = action === 'accept' ? 'accept' : 'dismiss'; - const matching = buttons.filter((button) => button.role === wanted); - return matching.length === 1 ? matching[0]! : null; + return selectUniqueAlertButtonByRole(buttons, action); } diff --git a/packages/provider-limrun/src/alert.test.ts b/packages/provider-limrun/src/alert.test.ts index 954e953d45..54ae9803b7 100644 --- a/packages/provider-limrun/src/alert.test.ts +++ b/packages/provider-limrun/src/alert.test.ts @@ -1,6 +1,12 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + ALERT_BUTTON_UNDECIDED_REASON, + ALERT_CAPTURE_UNDECIDED_REASON, +} from '@agent-device/contracts/alert-contract'; +import { AppError } from '@agent-device/kernel/errors'; import { limrunAlertAmbiguousButtonsTree, + limrunAlertTreeWithUnpressableButtons, limrunAppScreenTree, limrunAttAlertTree, } from './alert-detection.fixtures.ts'; @@ -20,7 +26,10 @@ function fakeClock() { function clientWithTrees(trees: Array): LimrunIosAlertClient { const elementTree = vi.fn(async () => { const next = trees.length > 1 ? trees.shift() : trees[0]; - return [next] as unknown as ReturnType; + // The real client answers an array for a multi-root capture and a single node otherwise. + return (Array.isArray(next) ? next : [next]) as unknown as ReturnType< + LimrunIosAlertClient['elementTree'] + >; }); return { elementTree, tap: vi.fn(async () => {}) }; } @@ -74,6 +83,19 @@ describe('wait budget', () => { }); }); +describe('get on an undecided capture', () => { + it('says the capture cannot decide, rather than claiming no alert', async () => { + const client = clientWithTrees([[limrunAttAlertTree(), limrunAttAlertTree()]]); + + const result = await handleLimrunIosAlert(client, 'get', { sleep: async () => {} }); + + expect(result.kind === 'alertStatus' && result.alert).toBeNull(); + expect(result.kind === 'alertStatus' && result.message).toBe( + 'Cannot decide the alert: the capture is alert-shaped but unresolvable', + ); + }); +}); + describe('acting budgets', () => { it('spends the confirm budget through the sleep seam and reports a stuck alert', async () => { fakeClock(); @@ -87,12 +109,75 @@ describe('acting budgets', () => { expect(sleep).toHaveBeenCalled(); }); - it('refuses an ambiguous alert without tapping', async () => { + it('refuses an ambiguous alert without tapping, on a typed reason', async () => { const client = clientWithTrees([limrunAlertAmbiguousButtonsTree()]); - await expect(handleLimrunIosAlert(client, 'accept', { sleep: async () => {} })).rejects.toThrow( - 'no button that clearly matches the action', + const error = await handleLimrunIosAlert(client, 'accept', { + sleep: async () => {}, + }).then( + () => undefined, + (caught: unknown) => caught, + ); + expect(error).toBeInstanceOf(AppError); + const refusal = error as AppError; + expect(refusal.code).toBe('COMMAND_FAILED'); + expect(refusal.details?.reason).toBe(ALERT_BUTTON_UNDECIDED_REASON); + expect(refusal.details?.dispatched).toBe('no'); + expect(client.tap).not.toHaveBeenCalled(); + }); + + it('reports a stuck alert when the post-tap capture stays unresolvable to the end', async () => { + fakeClock(); + // The first probe resolves the single alert; every post-tap probe carries two disagreeing + // alert-shaped sheets. An unresolvable capture proves absence for nothing, so the confirm + // loop keeps its budget and fails rather than reporting the alert handled. + const client = clientWithTrees([ + limrunAttAlertTree(), + [limrunAttAlertTree(), limrunAttAlertTree()], + ]); + const sleep = vi.fn(advanceClock); + + await expect(handleLimrunIosAlert(client, 'dismiss', { sleep })).rejects.toThrow( + 'alert dismiss did not dismiss the visible alert', + ); + }); + + it('reports a stuck alert when the same root survives the tap with degraded buttons', async () => { + fakeClock(); + // The tap was ignored: the post-tap capture is the same alert root with its choices dropped + // to disabled, explicitly unhittable, and frameless. Degraded-but-present is not absence; + // the leg must fail, not report handled. + const client = clientWithTrees([limrunAttAlertTree(), limrunAlertTreeWithUnpressableButtons()]); + const sleep = vi.fn(advanceClock); + + await expect(handleLimrunIosAlert(client, 'accept', { sleep })).rejects.toThrow( + 'alert accept did not dismiss the visible alert', + ); + expect(client.tap).toHaveBeenCalledTimes(1); + }); + + it('confirms handled when the post-tap capture shows the app screen instead', async () => { + // The closest genuine no-alert negative to the case above: the same seam and budgets, and + // the app's own named root after the tap — observed absence, which does confirm. + const client = clientWithTrees([limrunAttAlertTree(), limrunAppScreenTree()]); + const result = await handleLimrunIosAlert(client, 'accept', { sleep: async () => {} }); + expect(result.kind === 'alertHandled' && result.button).toBe('Allow'); + }); + + it('refuses to act on an alert-shaped but unresolvable capture, without tapping', async () => { + fakeClock(); + const client = clientWithTrees([limrunAlertTreeWithUnpressableButtons()]); + const sleep = vi.fn(advanceClock); + + const error = await handleLimrunIosAlert(client, 'accept', { sleep }).then( + () => undefined, + (caught: unknown) => caught, ); + expect(error).toBeInstanceOf(AppError); + const refusal = error as AppError; + expect(refusal.code).toBe('COMMAND_FAILED'); + expect(refusal.details?.reason).toBe(ALERT_CAPTURE_UNDECIDED_REASON); + expect(refusal.details?.dispatched).toBe('no'); expect(client.tap).not.toHaveBeenCalled(); }); }); diff --git a/packages/provider-limrun/src/alert.ts b/packages/provider-limrun/src/alert.ts index 35526f2dd7..e34fe858d0 100644 --- a/packages/provider-limrun/src/alert.ts +++ b/packages/provider-limrun/src/alert.ts @@ -1,25 +1,27 @@ -import { - ALERT_ACTION_RETRY_MS, - ALERT_POLL_INTERVAL_MS, - DEFAULT_ALERT_TIMEOUT_MS, -} from '@agent-device/contracts/alert-contract'; -import { AppError } from '@agent-device/kernel/errors'; -import { successText } from '@agent-device/kernel/success-text'; +import type { + TreeAlertDriver, + TreeAlertObservation, + TreeAlertPress, + TreeAlertResult, + TreeAlertSleep, +} from '@agent-device/platform-android/mechanics'; +import { runTreeAlertAction } from '@agent-device/platform-android/mechanics'; import type { InstanceClient as LimrunIosClient } from '@limrun/api/ios-client'; import { setTimeout as sleep } from 'node:timers/promises'; import { chooseLimrunAlertButton, - findLimrunAlertCandidate, type LimrunAlertCandidate, type LimrunAlertInfo, + readLimrunAlertCapture, } from './alert-detection.ts'; /** * The four alert legs on a Limrun iOS direct session (issue #3349). There is no XCUITest runner - * to ask, so the legs observe through the session's own element-tree capture and act through the - * same point-touch seam every other touch operation rides. The response records match the - * Android tree-based legs' `alertStatus`/`alertWait`/`alertHandled` kinds, which is the shape the - * daemon composes CLI and SDK output around. + * to ask, so this backend supplies the shared tree-alert driver (`platform-android`'s + * `runTreeAlertAction`, the one poll/confirm/same-alert/response owner every tree-based backend + * rides) with its own observation and press seams only: observation through the session's own + * element-tree capture and its host-root reader, action through the same point-touch seam every + * other touch operation rides. */ /** The client methods the legs use; declared here so the legs are testable without a live instance. */ @@ -28,179 +30,48 @@ export type LimrunIosAlertClient = Readonly<{ tap: LimrunIosClient['tap']; }>; -/** Tests pass a zero-wait sleep; production rides the timer. */ -export type LimrunAlertSleep = (milliseconds: number) => Promise; +/** Tests advance polls on a driven clock; production rides the timer (`docs/agents/testing.md`). */ +export type LimrunAlertSleep = TreeAlertSleep; const realSleep: LimrunAlertSleep = async (milliseconds) => { await sleep(milliseconds); }; -export type LimrunIosAlertResult = - | { - kind: 'alertStatus'; - platform: 'ios'; - action: 'get'; - alert: LimrunAlertInfo | null; - message?: string; - } - | { - kind: 'alertWait'; - platform: 'ios'; - action: 'wait'; - alert: LimrunAlertInfo; - waitedMs: number; - message?: string; - } - | { - kind: 'alertHandled'; - platform: 'ios'; - action: 'accept' | 'dismiss'; - handled: true; - alert: LimrunAlertInfo; - button: string; - coordinates: { x: number; y: number }; - message?: string; - }; +export type LimrunIosAlertResult = TreeAlertResult; export type LimrunIosAlertAction = 'get' | 'wait' | 'accept' | 'dismiss'; -const ALERT_NOT_FOUND_HINT = - 'If the sheet is visible in snapshot but alert reports no alert, it is likely app-owned UI. Use snapshot -i and press the visible label/ref.'; - -/** A malformed window must fail as an immediate timeout, never as a loop whose bound is never reached. */ -function alertTimeoutMs(timeoutMs: number | undefined, fallback: number): number { - return typeof timeoutMs === 'number' && Number.isFinite(timeoutMs) && timeoutMs >= 0 - ? timeoutMs - : fallback; -} - export async function handleLimrunIosAlert( client: LimrunIosAlertClient, action: LimrunIosAlertAction, - options?: Readonly<{ - timeoutMs?: number; - /** Tests advance polls without waiting production time (`docs/agents/testing.md`). */ - sleep?: LimrunAlertSleep; - }>, + options?: Readonly<{ timeoutMs?: number; sleep?: LimrunAlertSleep }>, ): Promise { const nap = options?.sleep ?? realSleep; - if (action === 'get') { - const candidate = await readLimrunAlertCandidate(client); - return { - kind: 'alertStatus', - platform: 'ios', - action: 'get', - alert: candidate?.alert ?? null, - ...(candidate ? successText('Alert visible') : successText('No alert visible')), - }; - } - if (action === 'wait') { - const start = Date.now(); - const candidate = await pollLimrunAlertCandidate( - client, - alertTimeoutMs(options?.timeoutMs, DEFAULT_ALERT_TIMEOUT_MS), - nap, - ); - if (!candidate) { - throw new AppError('COMMAND_FAILED', 'alert wait timed out'); - } - return { - kind: 'alertWait', - platform: 'ios', - action: 'wait', - alert: candidate.alert, - waitedMs: Date.now() - start, - ...successText('Alert visible'), - }; - } - return await actOnLimrunIosAlert(client, action, nap); -} - -/** - * Accept and dismiss share the Android legs' contract: look within the action budget, press - * exactly the one button the action's label vocabulary names, then confirm the alert is gone. - * `dismiss` never authorizes anything — an alert whose denying choice cannot be named is - * reported as the ambiguity it is instead of pressing something. - */ -async function actOnLimrunIosAlert( - client: LimrunIosAlertClient, - action: 'accept' | 'dismiss', - nap: LimrunAlertSleep, -): Promise { - const candidate = await pollLimrunAlertCandidate(client, ALERT_ACTION_RETRY_MS, nap); - if (!candidate) { - throw new AppError('COMMAND_FAILED', 'alert not found', { hint: ALERT_NOT_FOUND_HINT }); - } - const button = chooseLimrunAlertButton(candidate.buttons, action); - if (!button) { - throw new AppError( - 'COMMAND_FAILED', - `alert ${action} found an alert but no button that clearly matches the action`, - { - alert: candidate.alert, - hint: 'Inspect alert get --json for the visible buttons, then press the choice you want by label/ref.', - }, - ); - } - await client.tap(button.x, button.y); - await confirmLimrunAlertDismissed(client, candidate, action, nap); - return { - kind: 'alertHandled', + const driver: TreeAlertDriver< + Awaited>, + LimrunAlertInfo, + LimrunAlertCandidate['buttons'][number], + LimrunAlertCandidate, + 'ios' + > = { platform: 'ios', - action, - handled: true, - alert: candidate.alert, - button: button.label, - coordinates: { x: button.x, y: button.y }, - ...successText(`Alert ${action}ed`), + timeoutMs: options?.timeoutMs, + sleep: nap, + capture: () => client.elementTree(), + findCandidate: (tree): TreeAlertObservation => + readLimrunAlertCapture(tree as Parameters[0]), + // Stricter than Android's by design: only the button whose own label states the asked-for + // choice may be pressed; there is no platform fallback key to invent on iOS. The shared + // driver names the refusal when this returns null. + choose: (candidate, act): TreeAlertPress | null => { + const button = chooseLimrunAlertButton(candidate.buttons, act); + return button ? { kind: 'press', button } : null; + }, + press: async (choice) => { + // `dismiss` never authorizes tracking: the chooser above presses only the one button the + // action's own label names, and a tap that falls back to a guess is never dispatched. + if (choice.kind === 'press') await client.tap(choice.button.x, choice.button.y); + }, }; -} - -/** - * `alert accept|dismiss` means the dialog is gone, not that a tap was sent: the next command - * reads the app, and an alert still up would be read as the app's state. A different alert - * taking its place counts as dismissed, the same as the Android legs accept. - */ -async function confirmLimrunAlertDismissed( - client: LimrunIosAlertClient, - dismissed: LimrunAlertCandidate, - action: 'accept' | 'dismiss', - nap: LimrunAlertSleep, -): Promise { - const start = Date.now(); - for (;;) { - const current = await readLimrunAlertCandidate(client); - if (!current || !sameLimrunAlert(current.alert, dismissed.alert)) return; - if (Date.now() - start >= ALERT_ACTION_RETRY_MS) { - throw new AppError('COMMAND_FAILED', `alert ${action} did not dismiss the visible alert`, { - alert: dismissed.alert, - hint: 'The alert button was pressed but the dialog is still visible. Inspect alert get --json, then press the visible button by label/ref or retry.', - }); - } - await nap(ALERT_POLL_INTERVAL_MS); - } -} - -function sameLimrunAlert(left: LimrunAlertInfo, right: LimrunAlertInfo): boolean { - return left.title === right.title && left.buttons.join('\u0000') === right.buttons.join('\u0000'); -} - -async function pollLimrunAlertCandidate( - client: LimrunIosAlertClient, - timeoutMs: number, - nap: LimrunAlertSleep, -): Promise { - const start = Date.now(); - for (;;) { - const candidate = await readLimrunAlertCandidate(client); - if (candidate) return candidate; - if (Date.now() - start >= timeoutMs) return null; - await nap(ALERT_POLL_INTERVAL_MS); - } -} - -async function readLimrunAlertCandidate( - client: LimrunIosAlertClient, -): Promise { - return findLimrunAlertCandidate(await client.elementTree()); + return await runTreeAlertAction(action, driver); } diff --git a/packages/provider-limrun/src/ios-interactor-alert.test.ts b/packages/provider-limrun/src/ios-interactor-alert.test.ts index 4c9d6523a3..5c8a1464c3 100644 --- a/packages/provider-limrun/src/ios-interactor-alert.test.ts +++ b/packages/provider-limrun/src/ios-interactor-alert.test.ts @@ -1,4 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; +import { ALERT_BUTTON_UNDECIDED_REASON } from '@agent-device/contracts/alert-contract'; +import { AppError } from '@agent-device/kernel/errors'; import { limrunAlertTreeWithoutButtons, limrunAppAlertDialogTree, @@ -8,6 +10,12 @@ import { import { createLimrunIosInteractor, type LimrunIosSession } from './ios.ts'; import type { IosTreeNode } from './snapshot.ts'; +// The legs load `./alert.ts` per call (ADR-0019 laziness inside the provider), and that module +// carries the shared tree-alert driver from platform-android's mechanics facet. Warm it at +// collection, like `platform-android`'s own alert test does, so the first timed leg measures +// the legs, not the module graph's first transform. +await import('./alert.ts'); + /** * The four legs as the interactor exposes them: observation is one `elementTree()` read per * probe, action is a `tap()` at the chosen button's center. Trees are the live Limrun captures. @@ -111,9 +119,15 @@ describe('acceptAlert', () => { }, ]); - await expect(interactor.acceptAlert!()).rejects.toThrow( - 'alert accept found an alert but no button that clearly matches the action', + const error = await interactor.acceptAlert!().then( + () => undefined, + (caught: unknown) => caught, ); + expect(error).toBeInstanceOf(AppError); + const refusal = error as AppError; + expect(refusal.code).toBe('COMMAND_FAILED'); + expect(refusal.details?.reason).toBe(ALERT_BUTTON_UNDECIDED_REASON); + expect(refusal.details?.dispatched).toBe('no'); expect(client.tap).not.toHaveBeenCalled(); }); }); @@ -133,6 +147,8 @@ describe('dismissAlert', () => { action: 'dismiss', button: 'Ask App Not to Track', }); + // Exactly one tap, at the denying button: proving the affirmative one was never pressed. + expect(client.tap).toHaveBeenCalledTimes(1); expect(client.tap).toHaveBeenCalledWith(201, 536); }); diff --git a/website/docs/docs/limrun.md b/website/docs/docs/limrun.md index 00cd8d4790..0df1373970 100644 --- a/website/docs/docs/limrun.md +++ b/website/docs/docs/limrun.md @@ -71,7 +71,7 @@ agent-device disconnect On Android, agent-device connects to Limrun over an ADB tunnel. Snapshots use the Android snapshot helper, installs upload the app through Limrun asset storage, and port reverse runs over ADB, including the usual reverse setup for a local Metro server. -On iOS, agent-device supports app lifecycle commands, snapshots, screenshots, taps, text input, scrolling, the home button, clipboard read and write, app installation, and system alerts. `alert get/wait/accept/dismiss` read the alert from the same element tree as snapshots and press its buttons through taps; app-owned dialogs are not system alerts there, so use `snapshot -i` and press their visible labels. `open --launch-args` relaunches the app with those arguments through simctl. iOS cannot reverse a remote device port to a local host port, so for Metro or React DevTools use a publicly reachable HTTPS endpoint or bridge URL instead of a local-only address. +On iOS, agent-device supports app lifecycle commands, snapshots, screenshots, taps, text input, scrolling, the home button, clipboard read and write, app installation, and system alerts. `alert get/wait/accept/dismiss` read the alert from the same element tree as snapshots and dispatch the selected button; a choice is pressed only when its English label states the asked-for action (`Allow`, `Don't Allow`, `Ask App Not to Track`, and similar), so a localized alert is reported but never pressed. App-owned dialogs are not system alerts there, so use `snapshot -i` and then `press @ref` or a label selector for the visible control. `open --launch-args` relaunches the app with those arguments through simctl. iOS cannot reverse a remote device port to a local host port, so for Metro or React DevTools use a publicly reachable HTTPS endpoint or bridge URL instead of a local-only address. On iOS, `settings` supports `appearance`, `permission`, `location`, and `clear-app-state`, with the same arguments as a local simulator. `clear-app-state` resets the app's data container through Limrun's soft reset. That reset relaunches the app once before agent-device stops it, so the app may run once after its data is cleared. `reset-keychain`, `wifi`, `airplane`, `mobile-data`, `faceid`, `touchid`, and `text-size` are unsupported, and you cannot read back any iOS setting. On Android, `settings mobile-data on|off` runs over the ADB tunnel like on a local device. From 2fb88ffc3182da6c47423a4521593856ad7794a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 08:27:53 +0200 Subject: [PATCH 04/12] refactor(alert): ride main's deadline primitive with no sleep seam MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit origin/main migrated the Android alert loops to pollUntilSettled/Deadline and moved timing into fake timers at the test edge. The shared driver adopts the same shape: poll/confirm run on the deadline primitive, so the driver-owned sleep seam is dead weight and goes — Limrun's tests now drive the clock the way the migrated Android tests do. --- packages/platform-android/src/mechanics.ts | 1 - packages/provider-limrun/src/alert.test.ts | 80 ++++++++----------- packages/provider-limrun/src/alert.ts | 13 +-- .../src/ios-interactor-alert.test.ts | 4 +- 4 files changed, 35 insertions(+), 63 deletions(-) diff --git a/packages/platform-android/src/mechanics.ts b/packages/platform-android/src/mechanics.ts index d2097ca41c..9bbe69cd83 100644 --- a/packages/platform-android/src/mechanics.ts +++ b/packages/platform-android/src/mechanics.ts @@ -58,7 +58,6 @@ export { type TreeAlertObservation, type TreeAlertPress, type TreeAlertResult, - type TreeAlertSleep, } from './alert.ts'; export { classifyAndroidAlertIdentifier, diff --git a/packages/provider-limrun/src/alert.test.ts b/packages/provider-limrun/src/alert.test.ts index 54ae9803b7..e6aaab0d64 100644 --- a/packages/provider-limrun/src/alert.test.ts +++ b/packages/provider-limrun/src/alert.test.ts @@ -13,14 +13,20 @@ import { import { handleLimrunIosAlert, type LimrunIosAlertClient } from './alert.ts'; import type { IosTreeNode } from './snapshot.ts'; -// The clock is the assertion, not the wait (`docs/agents/testing.md`): only `Date` is faked, and -// the legs' sleep seam advances it, so budget loops terminate without real time. -const advanceClock = async (ms: number) => { - vi.advanceTimersByTime(ms); -}; - -function fakeClock() { - vi.useFakeTimers({ toFake: ['Date'] }); +/** + * The legs share the tree-alert driver, whose polls sleep between captures on the shared + * deadline primitive; the clock is the assertion, not the wait (`docs/agents/testing.md`). + */ +async function underFakeTimers(run: () => Promise): Promise { + vi.useFakeTimers(); + try { + const outcome = run(); + outcome.catch(() => {}); + await vi.runAllTimersAsync(); + return await outcome; + } finally { + vi.useRealTimers(); + } } function clientWithTrees(trees: Array): LimrunIosAlertClient { @@ -39,34 +45,25 @@ afterEach(() => { }); describe('wait budget', () => { - it('spends exactly the caller window through the sleep seam and reports the timeout', async () => { - fakeClock(); + it('spends exactly the caller window and reports the timeout', async () => { const client = clientWithTrees([limrunAppScreenTree()]); - const sleep = vi.fn(advanceClock); - - const settled = handleLimrunIosAlert(client, 'wait', { timeoutMs: 5_000, sleep }).then( - () => undefined, - (error: unknown) => error, - ); - const error = await settled; - expect(error).toBeInstanceOf(Error); - expect((error as Error).message).toBe('alert wait timed out'); + await expect( + underFakeTimers(() => handleLimrunIosAlert(client, 'wait', { timeoutMs: 5_000 })), + ).rejects.toThrow('alert wait timed out'); // One poll per interval over the whole window: never fewer (gave up early) nor real seconds. - expect(sleep).toHaveBeenCalledTimes(Math.ceil(5_000 / 300)); + expect(client.elementTree).toHaveBeenCalledTimes(1 + Math.ceil(5_000 / 300)); }); it('treats a malformed window as the bounded default rather than an unbound loop', async () => { - fakeClock(); const client = clientWithTrees([limrunAppScreenTree()]); - const sleep = vi.fn(advanceClock); await expect( - handleLimrunIosAlert(client, 'wait', { timeoutMs: Number.NaN, sleep }), + underFakeTimers(() => handleLimrunIosAlert(client, 'wait', { timeoutMs: Number.NaN })), ).rejects.toThrow('alert wait timed out'); // The fallback is the contract default, spent on the driven clock — this returned, so the // loop had a bound, which a NaN comparison would not have given it. - expect(sleep).toHaveBeenCalledTimes(Math.ceil(10_000 / 300)); + expect(client.elementTree).toHaveBeenCalledTimes(1 + Math.ceil(10_000 / 300)); }); it('surfaces a failed capture as itself, never as a timeout', async () => { @@ -77,9 +74,7 @@ describe('wait budget', () => { ) as unknown as LimrunIosAlertClient['elementTree']; const client = { elementTree, tap: vi.fn(async () => {}) } satisfies LimrunIosAlertClient; - await expect(handleLimrunIosAlert(client, 'wait', { sleep: async () => {} })).rejects.toThrow( - 'websocket closed', - ); + await expect(handleLimrunIosAlert(client, 'wait')).rejects.toThrow('websocket closed'); }); }); @@ -87,7 +82,7 @@ describe('get on an undecided capture', () => { it('says the capture cannot decide, rather than claiming no alert', async () => { const client = clientWithTrees([[limrunAttAlertTree(), limrunAttAlertTree()]]); - const result = await handleLimrunIosAlert(client, 'get', { sleep: async () => {} }); + const result = await handleLimrunIosAlert(client, 'get'); expect(result.kind === 'alertStatus' && result.alert).toBeNull(); expect(result.kind === 'alertStatus' && result.message).toBe( @@ -97,24 +92,19 @@ describe('get on an undecided capture', () => { }); describe('acting budgets', () => { - it('spends the confirm budget through the sleep seam and reports a stuck alert', async () => { - fakeClock(); + it('spends the confirm budget and reports a stuck alert', async () => { const client = clientWithTrees([limrunAttAlertTree(), limrunAttAlertTree()]); - const sleep = vi.fn(advanceClock); - await expect(handleLimrunIosAlert(client, 'dismiss', { sleep })).rejects.toThrow( + await expect(underFakeTimers(() => handleLimrunIosAlert(client, 'dismiss'))).rejects.toThrow( 'alert dismiss did not dismiss the visible alert', ); expect(client.tap).toHaveBeenCalledTimes(1); - expect(sleep).toHaveBeenCalled(); }); it('refuses an ambiguous alert without tapping, on a typed reason', async () => { const client = clientWithTrees([limrunAlertAmbiguousButtonsTree()]); - const error = await handleLimrunIosAlert(client, 'accept', { - sleep: async () => {}, - }).then( + const error = await handleLimrunIosAlert(client, 'accept').then( () => undefined, (caught: unknown) => caught, ); @@ -127,7 +117,6 @@ describe('acting budgets', () => { }); it('reports a stuck alert when the post-tap capture stays unresolvable to the end', async () => { - fakeClock(); // The first probe resolves the single alert; every post-tap probe carries two disagreeing // alert-shaped sheets. An unresolvable capture proves absence for nothing, so the confirm // loop keeps its budget and fails rather than reporting the alert handled. @@ -135,22 +124,19 @@ describe('acting budgets', () => { limrunAttAlertTree(), [limrunAttAlertTree(), limrunAttAlertTree()], ]); - const sleep = vi.fn(advanceClock); - await expect(handleLimrunIosAlert(client, 'dismiss', { sleep })).rejects.toThrow( + await expect(underFakeTimers(() => handleLimrunIosAlert(client, 'dismiss'))).rejects.toThrow( 'alert dismiss did not dismiss the visible alert', ); }); it('reports a stuck alert when the same root survives the tap with degraded buttons', async () => { - fakeClock(); // The tap was ignored: the post-tap capture is the same alert root with its choices dropped // to disabled, explicitly unhittable, and frameless. Degraded-but-present is not absence; // the leg must fail, not report handled. const client = clientWithTrees([limrunAttAlertTree(), limrunAlertTreeWithUnpressableButtons()]); - const sleep = vi.fn(advanceClock); - await expect(handleLimrunIosAlert(client, 'accept', { sleep })).rejects.toThrow( + await expect(underFakeTimers(() => handleLimrunIosAlert(client, 'accept'))).rejects.toThrow( 'alert accept did not dismiss the visible alert', ); expect(client.tap).toHaveBeenCalledTimes(1); @@ -160,16 +146,14 @@ describe('acting budgets', () => { // The closest genuine no-alert negative to the case above: the same seam and budgets, and // the app's own named root after the tap — observed absence, which does confirm. const client = clientWithTrees([limrunAttAlertTree(), limrunAppScreenTree()]); - const result = await handleLimrunIosAlert(client, 'accept', { sleep: async () => {} }); + const result = await handleLimrunIosAlert(client, 'accept'); expect(result.kind === 'alertHandled' && result.button).toBe('Allow'); }); it('refuses to act on an alert-shaped but unresolvable capture, without tapping', async () => { - fakeClock(); const client = clientWithTrees([limrunAlertTreeWithUnpressableButtons()]); - const sleep = vi.fn(advanceClock); - const error = await handleLimrunIosAlert(client, 'accept', { sleep }).then( + const error = await underFakeTimers(() => handleLimrunIosAlert(client, 'accept')).then( () => undefined, (caught: unknown) => caught, ); @@ -208,14 +192,14 @@ describe('button vocabulary', () => { alertWith('Ask App Not to Track', label), limrunAppScreenTree(), ]); - const result = await handleLimrunIosAlert(client, 'accept', { sleep: async () => {} }); + const result = await handleLimrunIosAlert(client, 'accept'); expect(result.kind === 'alertHandled' && result.button).toBe(label); } }); it('dismisses Close App, the ANRB denial', async () => { const client = clientWithTrees([alertWith('Close App', 'Wait'), limrunAppScreenTree()]); - const result = await handleLimrunIosAlert(client, 'dismiss', { sleep: async () => {} }); + const result = await handleLimrunIosAlert(client, 'dismiss'); expect(result.kind === 'alertHandled' && result.button).toBe('Close App'); }); }); diff --git a/packages/provider-limrun/src/alert.ts b/packages/provider-limrun/src/alert.ts index e34fe858d0..d034cf094c 100644 --- a/packages/provider-limrun/src/alert.ts +++ b/packages/provider-limrun/src/alert.ts @@ -3,11 +3,9 @@ import type { TreeAlertObservation, TreeAlertPress, TreeAlertResult, - TreeAlertSleep, } from '@agent-device/platform-android/mechanics'; import { runTreeAlertAction } from '@agent-device/platform-android/mechanics'; import type { InstanceClient as LimrunIosClient } from '@limrun/api/ios-client'; -import { setTimeout as sleep } from 'node:timers/promises'; import { chooseLimrunAlertButton, type LimrunAlertCandidate, @@ -30,13 +28,6 @@ export type LimrunIosAlertClient = Readonly<{ tap: LimrunIosClient['tap']; }>; -/** Tests advance polls on a driven clock; production rides the timer (`docs/agents/testing.md`). */ -export type LimrunAlertSleep = TreeAlertSleep; - -const realSleep: LimrunAlertSleep = async (milliseconds) => { - await sleep(milliseconds); -}; - export type LimrunIosAlertResult = TreeAlertResult; export type LimrunIosAlertAction = 'get' | 'wait' | 'accept' | 'dismiss'; @@ -44,9 +35,8 @@ export type LimrunIosAlertAction = 'get' | 'wait' | 'accept' | 'dismiss'; export async function handleLimrunIosAlert( client: LimrunIosAlertClient, action: LimrunIosAlertAction, - options?: Readonly<{ timeoutMs?: number; sleep?: LimrunAlertSleep }>, + options?: Readonly<{ timeoutMs?: number }>, ): Promise { - const nap = options?.sleep ?? realSleep; const driver: TreeAlertDriver< Awaited>, LimrunAlertInfo, @@ -56,7 +46,6 @@ export async function handleLimrunIosAlert( > = { platform: 'ios', timeoutMs: options?.timeoutMs, - sleep: nap, capture: () => client.elementTree(), findCandidate: (tree): TreeAlertObservation => readLimrunAlertCapture(tree as Parameters[0]), diff --git a/packages/provider-limrun/src/ios-interactor-alert.test.ts b/packages/provider-limrun/src/ios-interactor-alert.test.ts index 5c8a1464c3..13921aec85 100644 --- a/packages/provider-limrun/src/ios-interactor-alert.test.ts +++ b/packages/provider-limrun/src/ios-interactor-alert.test.ts @@ -19,8 +19,8 @@ await import('./alert.ts'); /** * The four legs as the interactor exposes them: observation is one `elementTree()` read per * probe, action is a `tap()` at the chosen button's center. Trees are the live Limrun captures. - * These cases resolve on the first probe each; budget loops are pinned through the legs' sleep - * seam in `alert.test.ts`, so nothing here waits production time. + * These cases resolve on the first probe each; budget loops are pinned with driven clocks in + * `alert.test.ts`, so nothing here waits production time. */ function sessionWithTrees(trees: Array) { const elementTree = vi.fn(async () => { From 61a270e176f7719bdb97158cd3035a823749593a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 08:44:53 +0200 Subject: [PATCH 05/12] test(limrun): pin the accept-once/second-get-none timeline and satisfy catch-name lint Adds the maintainer's live-validated sequence at the unit seam: accept taps exactly once, and a following get on the same seam reports no alert. Renames catch params to satisfy oxlint's catch-error-name (three findings the affected gate reported). --- packages/provider-limrun/src/alert.test.ts | 30 ++++++++++++++----- .../src/ios-interactor-alert.test.ts | 2 +- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/packages/provider-limrun/src/alert.test.ts b/packages/provider-limrun/src/alert.test.ts index e6aaab0d64..bdb9baf988 100644 --- a/packages/provider-limrun/src/alert.test.ts +++ b/packages/provider-limrun/src/alert.test.ts @@ -104,12 +104,12 @@ describe('acting budgets', () => { it('refuses an ambiguous alert without tapping, on a typed reason', async () => { const client = clientWithTrees([limrunAlertAmbiguousButtonsTree()]); - const error = await handleLimrunIosAlert(client, 'accept').then( + const thrown = await handleLimrunIosAlert(client, 'accept').then( () => undefined, - (caught: unknown) => caught, + (error: unknown) => error, ); - expect(error).toBeInstanceOf(AppError); - const refusal = error as AppError; + expect(thrown).toBeInstanceOf(AppError); + const refusal = thrown as AppError; expect(refusal.code).toBe('COMMAND_FAILED'); expect(refusal.details?.reason).toBe(ALERT_BUTTON_UNDECIDED_REASON); expect(refusal.details?.dispatched).toBe('no'); @@ -150,15 +150,29 @@ describe('acting budgets', () => { expect(result.kind === 'alertHandled' && result.button).toBe('Allow'); }); + it('taps exactly once on accept, and the next get reports no alert', async () => { + // The maintainer's live-validated timeline at the unit seam: one tap for the action, the + // post-tap confirm observes the app screen, and a following read sees that same absence. + const client = clientWithTrees([limrunAttAlertTree(), limrunAppScreenTree()]); + + const handled = await handleLimrunIosAlert(client, 'accept'); + expect(handled.kind).toBe('alertHandled'); + expect(client.tap).toHaveBeenCalledTimes(1); + + const status = await handleLimrunIosAlert(client, 'get'); + expect(status.kind === 'alertStatus' && status.alert).toBeNull(); + expect(status.kind === 'alertStatus' && status.message).toBe('No alert visible'); + }); + it('refuses to act on an alert-shaped but unresolvable capture, without tapping', async () => { const client = clientWithTrees([limrunAlertTreeWithUnpressableButtons()]); - const error = await underFakeTimers(() => handleLimrunIosAlert(client, 'accept')).then( + const thrown = await underFakeTimers(() => handleLimrunIosAlert(client, 'accept')).then( () => undefined, - (caught: unknown) => caught, + (error: unknown) => error, ); - expect(error).toBeInstanceOf(AppError); - const refusal = error as AppError; + expect(thrown).toBeInstanceOf(AppError); + const refusal = thrown as AppError; expect(refusal.code).toBe('COMMAND_FAILED'); expect(refusal.details?.reason).toBe(ALERT_CAPTURE_UNDECIDED_REASON); expect(refusal.details?.dispatched).toBe('no'); diff --git a/packages/provider-limrun/src/ios-interactor-alert.test.ts b/packages/provider-limrun/src/ios-interactor-alert.test.ts index 13921aec85..109a8e13a7 100644 --- a/packages/provider-limrun/src/ios-interactor-alert.test.ts +++ b/packages/provider-limrun/src/ios-interactor-alert.test.ts @@ -121,7 +121,7 @@ describe('acceptAlert', () => { const error = await interactor.acceptAlert!().then( () => undefined, - (caught: unknown) => caught, + (error: unknown) => error, ); expect(error).toBeInstanceOf(AppError); const refusal = error as AppError; From 2a68d169f39b8d4fd87486ffa3de2860b27cf421 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 13:37:12 +0200 Subject: [PATCH 06/12] fix(alerts): resolve multi-grant sheets by a declared preference table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The unique-role choice boundary refused exactly the sheets the fix is for: Android 11+'s permission_allow_foreground_only_button and permission_allow_one_time_button both classify accept, and iOS's 'Allow Once'/'Allow While Using App' sheet refused the same way. The shared vocabulary is now a per-role preference table: distinct grants resolve by declared rank (a lasting grant before a one-time grant), and the acting legs refuse only when the capture names no single physical button — the same label at two points, or vocabulary-free multiplicity. Android keeps its neutral-button/Back fallback, Limrun keeps none. --- CHANGELOG.md | 2 +- packages/contracts/src/alert-contract.test.ts | 36 ++++++++ packages/contracts/src/alert-contract.ts | 91 +++++++++++++++---- .../src/__tests__/alert-detection.test.ts | 32 +++++++ .../src/__tests__/alert.test.ts | 33 +++++++ .../platform-android/src/alert-detection.ts | 14 +-- .../src/alert-detection.fixtures.ts | 45 +++++++++ .../provider-limrun/src/alert-detection.ts | 39 ++++---- packages/provider-limrun/src/alert.test.ts | 15 +++ website/docs/docs/limrun.md | 2 +- 10 files changed, 261 insertions(+), 48 deletions(-) create mode 100644 packages/contracts/src/alert-contract.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 422482b2c8..8afa2e20ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ - On iOS, a `fill` that runs under a degraded XCTest snapshot channel now aims at a point, including ref, selector and replay fills. When the runner identifies no text input under that point, it refuses with `NO_TEXT_INPUT_AT_POINT` (`error.details.reason: no_text_input_at_point`, `dispatched: no`) before dispatching its focus tap, and no field changes; when the input lookup itself cannot answer, it refuses with `TEXT_INPUT_PROBE_UNAVAILABLE` (`error.details.reason: text_input_probe_unavailable`). Previously such a fill cleared and typed through the app-level first responder, so it could rewrite a field the caller never named, or report a clear failure after emptying it. A ref or selector fill whose rect centre misses the real field (an outer container, a non-input node) now needs a point inside the input's own rect. - Opening a macOS app by bundle identifier now waits briefly when the app is already running but not yet registered with LaunchServices — the case when a supervisor or dev script started it directly — and if registration arrives within that bounded window the open attaches to the running copy, so the session drives the app the caller started, with its arguments. A copy still unlisted when the window closes gets the plain `open` as before, which can still duplicate; the wait only bounds the race, it does not close it. Launching an app that is not running costs one extra LaunchServices lookup and one process-table read, plus one bundle-identifier read when app-shaped processes exist. - Limrun iOS sessions now serve `alert get/wait/accept/dismiss` instead of refusing with `unsupported-provider-mode`. System alerts are read from the same element-tree capture snapshots ride — a blank-label, process-exclusive application root hosting a text-and-buttons sheet — and accepted or dismissed by tapping the button whose English label states the asked-for action; a capture that is alert-shaped but unresolvable (degraded buttons, competing sheets, an unverifiable owner) is reported as undecided and never pressed or counted as dismissed. The one shape no capture fact separates is an app dialog under a blanked root shared by no other process; the label-only acting legs answer it with a refusal, never a guess. -- The tree-based alert backends (Android and Limrun iOS) now share one alert driver and one English button-label vocabulary: `alert accept/dismiss` classify the same label the same way on both platforms, so Android now recognizes iOS variant grants (`Allow Once`, `Allow All the Time`, `Allow While Using App`, `Allow Notifications`) and `Ask App Not to Track` as acting choices, and two buttons sharing one role no longer resolve by tree order — the shared choice boundary refuses, and each backend's own policy answers (Android's neutral-button/Back fallback, Limrun's typed refusal). `alert not found` and the no-matching-button refusal now carry the same typed `details.reason` on both platforms. +- The tree-based alert backends (Android and Limrun iOS) now share one alert driver and one English button-label vocabulary declared as a per-role preference table: `alert accept/dismiss` classify the same label the same way on both platforms, so Android now recognizes iOS variant grants (`Allow Once`, `Allow All the Time`, `Allow While Using App`, `Allow Notifications`) and `Ask App Not to Track` as acting choices, and a sheet offering two grants (Android 11+'s "While using the app"/"Only this time", iOS's "Allow Once"/"Allow While Using App") resolves by the declared preference order rather than tree order or refusal. A choice refuses only when the capture names no single physical button for the action — the same label at two points — and each backend's own policy answers then (Android's neutral-button/Back fallback, Limrun's typed refusal). `alert not found` and the no-matching-button refusal now carry the same typed `details.reason` on both platforms. - Limrun iOS snapshots no longer fail when the instance reports a numeric or boolean accessibility value (the system photo picker's tabs report `1`/`0`): the provider seam normalizes variant scalars to text, the way the Apple tree does. Limrun nodes now also publish `selected: true` for nodes carrying the `Selected` trait, so snapshot output and `selected:` selectors can tell a picked photo from an unpicked one. - `settings` refuses a setting the target does not serve before anything changes, with `UNSUPPORTED_OPERATION`, `settings is not supported on this device`, and the target's `error.details.reason` (such as `unsupported-platform-leaf`). This replaces the macOS `INVALID_ARGS` refusal and the Apple `setting-unsupported-on-leaf` reason, and now also covers settings that used to reach the device first, such as Android `reset-keychain` or Apple `animations`. - `settings` refuses, before touching the device, a state its setting does not accept (`Invalid state: …`), a permission action or mode its target does not take (such as `permission deny accessibility`), and a stray extra argument (such as `clear-app-state `). The CLI and the daemon accept the same state spellings, in any casing or surrounding whitespace. diff --git a/packages/contracts/src/alert-contract.test.ts b/packages/contracts/src/alert-contract.test.ts new file mode 100644 index 0000000000..a7be5a1994 --- /dev/null +++ b/packages/contracts/src/alert-contract.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from 'vitest'; +import { classifyAlertButtonRole, selectRankedAlertButtonByRole } from './alert-contract.ts'; + +type Button = { label: string; role: ReturnType }; +const button = (label: string): Button => ({ label, role: classifyAlertButtonRole(label) }); + +describe('selectRankedAlertButtonByRole', () => { + it('resolves distinct grants by the declared preference order', () => { + const buttons = [button("Don't Allow"), button('Allow Once'), button('Allow While Using App')]; + expect(selectRankedAlertButtonByRole(buttons, 'accept')?.label).toBe('Allow While Using App'); + expect(selectRankedAlertButtonByRole(buttons, 'dismiss')?.label).toBe("Don't Allow"); + }); + + it('resolves the Android grant pair by the same table', () => { + const buttons = [button('While using the app'), button('Only this time'), button('Deny')]; + expect(selectRankedAlertButtonByRole(buttons, 'accept')?.label).toBe('While using the app'); + }); + + it('refuses on equal-rank duplicates: the capture names no single physical button', () => { + const buttons = [button('OK'), button('OK'), button('Cancel')]; + expect(selectRankedAlertButtonByRole(buttons, 'accept')).toBeNull(); + }); + + it('refuses when multiple candidates share a role the vocabulary cannot rank', () => { + // Roles can arrive from a platform id classifier; labels outside the vocabulary then tie. + const buttons: Button[] = [ + { label: 'While on the go', role: 'accept' }, + { label: 'Just this run', role: 'accept' }, + ]; + expect(selectRankedAlertButtonByRole(buttons, 'accept')).toBeNull(); + }); + + it('returns null when no button carries the role', () => { + expect(selectRankedAlertButtonByRole([button('Settings')], 'accept')).toBeNull(); + }); +}); diff --git a/packages/contracts/src/alert-contract.ts b/packages/contracts/src/alert-contract.ts index 93e34299d7..43fa824c8b 100644 --- a/packages/contracts/src/alert-contract.ts +++ b/packages/contracts/src/alert-contract.ts @@ -36,36 +36,91 @@ export type AlertCapture = | { kind: 'undecided' }; /** - * The one button label vocabulary every tree-based alert backend classifies with. English-only, - * matching the untranslated device locale the alert lanes are validated against; a localized - * button classifies `neutral`, so acting legs refuse on the button-undecided reason rather than - * guessing at a translation. + * The one button label vocabulary every tree-based alert backend classifies with, declared as a + * per-role preference table: each entry names one English label (regex only for spelling + * variants of the *same* label), earlier entries outrank later ones, so a sheet offering several + * grants resolves by declared preference ("while using the app" over "only this time") instead + * of tree order. English-only, matching the untranslated device locale the alert lanes are + * validated against; a localized button classifies `neutral`, so acting legs refuse on the + * button-undecided reason rather than guessing at a translation. */ -const ACCEPT_LABEL_PATTERN = - /^(?:ok|allow|allow all|allow once|allow all the time|allow while using (?:the )?app|allow notifications|always allow|yes|continue|save|confirm|turn on|open settings|while using the app|only this time)$/i; +const ACCEPT_LABEL_PREFERENCES: readonly RegExp[] = [ + /^allow while using (?:the )?app$/i, + /^while using the app$/i, + /^allow all$/i, + /^always allow$/i, + /^allow all the time$/i, + /^allow notifications$/i, + /^allow$/i, + /^ok$/i, + /^yes$/i, + /^continue$/i, + /^save$/i, + /^confirm$/i, + /^turn on$/i, + /^open settings$/i, + // One-time grants last: a sheet offering a lasting grant should not spend the tap here. + /^allow once$/i, + /^only this time$/i, +]; // `don.?t allow` covers the straight, missing, and typographic apostrophe in one alternate. -const DISMISS_LABEL_PATTERN = - /^(?:cancel|deny|don.?t allow|not now|no|dismiss|close|close app|later|skip|ask app not to track)$/i; +const DISMISS_LABEL_PREFERENCES: readonly RegExp[] = [ + /^cancel$/i, + /^deny$/i, + /^don.?t allow$/i, + /^not now$/i, + /^no$/i, + /^dismiss$/i, + /^close$/i, + /^close app$/i, + /^later$/i, + /^skip$/i, + /^ask app not to track$/i, +]; + +const LABEL_PREFERENCES: Record<'accept' | 'dismiss', readonly RegExp[]> = { + accept: ACCEPT_LABEL_PREFERENCES, + dismiss: DISMISS_LABEL_PREFERENCES, +}; /** One button's action role under the shared vocabulary; anything unnamed is neutral. */ export function classifyAlertButtonRole(label: string): AlertButtonRole { const trimmed = label.trim(); - if (ACCEPT_LABEL_PATTERN.test(trimmed)) return 'accept'; - if (DISMISS_LABEL_PATTERN.test(trimmed)) return 'dismiss'; + if (ACCEPT_LABEL_PREFERENCES.some((pattern) => pattern.test(trimmed))) return 'accept'; + if (DISMISS_LABEL_PREFERENCES.some((pattern) => pattern.test(trimmed))) return 'dismiss'; return 'neutral'; } /** - * The one choice boundary for a shared choice: a role exactly one visible button carries, or - * null when none does and when two or more do — pressing one of a duplicated choice would pick - * a side the capture does not name, so duplicates refuse rather than resolve by tree order. + * A role-named button's place in the declared preference table, or null when its role came from + * something other than the vocabulary (Android id classifiers, localized labels). + */ +function alertButtonPreferenceRank(label: string, role: 'accept' | 'dismiss'): number | null { + const index = LABEL_PREFERENCES[role].findIndex((pattern) => pattern.test(label.trim())); + return index === -1 ? null : index; +} + +/** + * The one choice boundary for a shared choice: pick the action's role candidates by the declared + * preference table. A single candidate wins outright; several resolve when one outscores the + * rest; a tie on the same entry (the same label at two points) or vocabulary-ranked-free + * multiplicity returns null, because the capture then names no single physical button for the + * action. Distinct labels that share a role are not a refusal: the table ranks them. */ -export function selectUniqueAlertButtonByRole>( - buttons: readonly TButton[], - role: 'accept' | 'dismiss', -): TButton | null { +export function selectRankedAlertButtonByRole< + TButton extends Readonly<{ role: AlertButtonRole; label: string }>, +>(buttons: readonly TButton[], role: 'accept' | 'dismiss'): TButton | null { const matching = buttons.filter((button) => button.role === role); - return matching.length === 1 ? (matching[0] ?? null) : null; + if (matching.length === 1) return matching[0] ?? null; + if (matching.length === 0) return null; + const ranked = matching + .map((button) => ({ button, rank: alertButtonPreferenceRank(button.label, role) })) + .filter((entry): entry is { button: TButton; rank: number } => entry.rank !== null) + .sort((a, b) => a.rank - b.rank); + if (ranked.length === 0) return null; + const [first, second] = ranked; + if (!first || (second && second.rank === first.rank)) return null; + return first.button; } /** diff --git a/packages/platform-android/src/__tests__/alert-detection.test.ts b/packages/platform-android/src/__tests__/alert-detection.test.ts index 1c56a0560f..67c0ba6b4a 100644 --- a/packages/platform-android/src/__tests__/alert-detection.test.ts +++ b/packages/platform-android/src/__tests__/alert-detection.test.ts @@ -76,3 +76,35 @@ test('the shared vocabulary classifies iOS variant grants on Android trees too', ); assert.equal(chooseAndroidAlertButton(candidate?.buttons ?? [], 'dismiss')?.label, "Don't allow"); }); + +test('the Android 11+ permission sheet resolves its two grants by the preference table', () => { + // The shipped Android 11+ shape: permission_allow_foreground_only_button and + // permission_allow_one_time_button both classify accept by id, so the shared ranked choice + // (not uniqueness) must pick the lasting grant; main pressed it too, so a refusal here + // regresses a working route. + const candidate = findAndroidAlertCandidate([ + node(0, 'android.widget.FrameLayout', { + bundleId: 'com.google.android.permissioncontroller', + }), + text( + 1, + 'Allow Probe to access location in the background?', + 'com.android.permissioncontroller:id/permission_message', + ), + button( + 2, + 'While using the app', + 'permission_allow_foreground_only_button', + { x: 32, y: 1420 }, + true, + ), + button(3, 'Only this time', 'permission_allow_one_time_button', { x: 32, y: 1490 }, true), + button(4, 'Deny', 'permission_deny_button', { x: 32, y: 1560 }, true), + ]); + + assert.equal( + chooseAndroidAlertButton(candidate?.buttons ?? [], 'accept')?.label, + 'While using the app', + ); + assert.equal(chooseAndroidAlertButton(candidate?.buttons ?? [], 'dismiss')?.label, 'Deny'); +}); diff --git a/packages/platform-android/src/__tests__/alert.test.ts b/packages/platform-android/src/__tests__/alert.test.ts index eb6a5aa95c..614005565b 100644 --- a/packages/platform-android/src/__tests__/alert.test.ts +++ b/packages/platform-android/src/__tests__/alert.test.ts @@ -88,6 +88,39 @@ test('accepting a button alert records the tapped button and its coordinates', a assert.deepEqual('coordinates' in result ? result.coordinates : undefined, { x: 116, y: 638 }); }); +test('accepting the Android 11+ permission sheet taps the preferred grant once', async () => { + runAndroidAdb.mockClear(); + // The regression the maintainer asked for at the action seam: both permission_allow_* ids + // classify accept, and the shared preference table must land one tap on the lasting grant. + const sheet = [ + node(0, 'android.widget.FrameLayout', { + bundleId: 'com.google.android.permissioncontroller', + }), + text( + 1, + 'Allow Probe to access location in the background?', + 'com.android.permissioncontroller:id/permission_message', + ), + button( + 2, + 'While using the app', + 'permission_allow_foreground_only_button', + { x: 32, y: 1420 }, + true, + ), + button(3, 'Only this time', 'permission_allow_one_time_button', { x: 32, y: 1490 }, true), + button(4, 'Deny', 'permission_deny_button', { x: 32, y: 1560 }, true), + ]; + const result = await handleAndroidAlert(device, 'accept', { + captureNodes: dialogUntilPressed(sheet), + }); + + assert.equal(result.kind, 'alertHandled'); + assert.equal(result.kind === 'alertHandled' ? result.button : undefined, 'While using the app'); + assert.deepEqual('coordinates' in result ? result.coordinates : undefined, { x: 96, y: 1446 }); + assert.equal(runAndroidAdb.mock.calls.filter(([, args]) => args.includes('tap')).length, 1); +}); + test('a fallback Back dismissal (no matching button) carries no coordinates', async () => { runAndroidAdb.mockClear(); const result = await handleAndroidAlert(device, 'dismiss', { diff --git a/packages/platform-android/src/alert-detection.ts b/packages/platform-android/src/alert-detection.ts index 83906cabd6..dcdd6e957a 100644 --- a/packages/platform-android/src/alert-detection.ts +++ b/packages/platform-android/src/alert-detection.ts @@ -4,7 +4,7 @@ import { classifyAlertButtonRole, type AlertInfo, type AlertSource, - selectUniqueAlertButtonByRole, + selectRankedAlertButtonByRole, } from '@agent-device/contracts/alert-contract'; import { isSnapshotNodeInteractionBlocked } from '@agent-device/capture-kit/snapshot-occlusion'; @@ -86,17 +86,19 @@ function isReachableAlertNode(node: RawSnapshotNode): boolean { } /** - * Android's acting policy on the shared choice boundary: the unique button whose role the - * action names; else, for dismiss, the first neutral button (Android dialogs commonly name the - * leaving path only neutrally); else, for accept, a single-button dialog's only affirmative - * path. Two buttons sharing a role refuse at the shared boundary rather than resolving by tree + * Android's acting policy on the shared choice boundary: the action's role-named button chosen by + * the declared preference table (a multi-grant permission sheet's "While using the app" outranks + * "Only this time"); else, for dismiss, the first neutral button (Android dialogs commonly name + * the leaving path only neutrally); else, for accept, a single-button dialog's only affirmative + * path. Two choices that tie on one entry (the same label at two points, or two id-classified + * grants the vocabulary cannot rank) refuse at the shared boundary rather than resolving by tree * order — the driver turns that refusal into a Back key for dismiss, or a typed refusal. */ export function chooseAndroidAlertButton( buttons: AndroidAlertButton[], action: 'accept' | 'dismiss', ): AndroidAlertButton | null { - const exact = selectUniqueAlertButtonByRole(buttons, action); + const exact = selectRankedAlertButtonByRole(buttons, action); if (exact) return exact; if (action === 'dismiss') { return buttons.find((button) => button.role === 'neutral') ?? null; diff --git a/packages/provider-limrun/src/alert-detection.fixtures.ts b/packages/provider-limrun/src/alert-detection.fixtures.ts index 8e645d77f2..ac94c09f67 100644 --- a/packages/provider-limrun/src/alert-detection.fixtures.ts +++ b/packages/provider-limrun/src/alert-detection.fixtures.ts @@ -80,6 +80,51 @@ export function limrunAlertTreeWithoutButtons(): IosTreeNode { return { ...root, children: (root.children ?? []).filter((n) => n.role !== 'AXButton') }; } +/** + * The standard iOS location prompt shape: three choices, two of which grant ("Allow Once" and + * "Allow While Using App"). The capture itself never marks a preferred grant; the shared + * preference table does, so `alert accept` presses the lasting grant rather than refusing. + */ +export function limrunLocationAlertTree(): IosTreeNode { + return alertRoot([ + { + type: 'StaticText', + role: 'AXStaticText', + AXLabel: '“Probe” Would Like to Use Your Current Location', + pid: LIMRUN_ALERT_HOST_PID, + frame: { x: 71, y: 381.33, width: 260, height: 64.33 }, + traits: ['Scrollable', 'StaticText'], + }, + { + type: 'Button', + role: 'AXButton', + AXLabel: "Don't Allow", + pid: LIMRUN_ALERT_HOST_PID, + enabled: true, + frame: { x: 57, y: 511.67, width: 288, height: 48 }, + traits: ['Button', 'Scrollable'], + }, + { + type: 'Button', + role: 'AXButton', + AXLabel: 'Allow Once', + pid: LIMRUN_ALERT_HOST_PID, + enabled: true, + frame: { x: 57, y: 567.67, width: 288, height: 48 }, + traits: ['Button', 'Scrollable'], + }, + { + type: 'Button', + role: 'AXButton', + AXLabel: 'Allow While Using App', + pid: LIMRUN_ALERT_HOST_PID, + enabled: true, + frame: { x: 57, y: 623.67, width: 288, height: 48 }, + traits: ['Button', 'Scrollable'], + }, + ]); +} + /** * The live ATT prompt with every choice degraded out of action: one disabled, one explicitly * unhittable, one label-less and frameless. This is what a capture looks like when the tap was diff --git a/packages/provider-limrun/src/alert-detection.ts b/packages/provider-limrun/src/alert-detection.ts index 837d5620f1..cdacda8ff9 100644 --- a/packages/provider-limrun/src/alert-detection.ts +++ b/packages/provider-limrun/src/alert-detection.ts @@ -3,7 +3,7 @@ import { classifyAlertButtonRole, type AlertInfo, type AlertSource, - selectUniqueAlertButtonByRole, + selectRankedAlertButtonByRole, } from '@agent-device/contracts/alert-contract'; import { centerOfRect, type Rect } from '@agent-device/kernel/snapshot'; import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; @@ -12,22 +12,16 @@ import { readIosNodeChildren, readIosNodeRect, type IosTreeNode } from './snapsh /** * System-alert detection over a Limrun iOS element tree (issue #3349). A Limrun iOS direct - * session has no XCUITest runner, but its element tree does carry the system alert: on a live - * instance the ATT prompt arrived as its own `AXApplication` root whose `AXLabel` is blank, with - * its text and buttons as direct children all sharing one `pid` — never the app's. The app's own - * `UIAlertController`, by contrast, rode inside the app's root, labeled with the app name and - * carrying the app's pid, and SpringBoard's status nodes rode that app root under a third pid. - * Nothing in a capture names a package as owner, so the host rule is a heuristic over these - * observations: a host sheet qualifies only on a blank-label root that reports a `pid` and whose - * entire subtree reports that same one. A named root is app UI; launch-icon buttons are a home - * screen; a button without a frame has no point to press. The documented residual — what these - * captures cannot exclude — is an app that leaves its own root's label blank while showing a - * dialog that no other process rides; the acting legs press only label-unambiguous buttons as - * the backstop, never a guess. - * - * Button action eligibility never asserts hitability: the captures omit the `hittable` key - * entirely, so an explicit `hittable: false` refuses a button and an omitted one carries no - * signal — what admits a button is the enabled, label, and complete-rect rules. + * session has no XCUITest runner, but its element tree carries the system alert: on a live + * instance the ATT prompt arrived as its own `AXApplication` root with a blank `AXLabel` and one + * `pid` shared by its whole subtree — never the app's. The app's own `UIAlertController` rode + * inside the app's named root under the app's pid. Nothing in a capture names an owner, so the + * host rule is a stated heuristic: a host sheet qualifies only on a blank-label root reporting a + * `pid` whose entire subtree reports that same one. The named residual these captures cannot + * exclude: an app blanking its own root label while showing an unshared dialog — the acting legs + * press only label-unambiguous buttons as the backstop. Button eligibility never asserts + * hitability: captures omit `hittable` entirely, so an explicit `false` refuses and an omission + * carries no signal; enabled, label, and complete-rect rules admit a button. */ /** Only the two host-sheet sources exist on this surface: no node names a package to attribute. */ @@ -218,14 +212,15 @@ function classifyAlertSource(buttons: readonly LimrunAlertButton[]): LimrunAlert /** * The button one action may press, or null when pressing any visible choice would guess. * Deliberately stricter than Android's: with no `android:id/button1`-style identity to fall back - * on, only a button whose own label states the asked-for choice may be pressed, resolved once - * through the shared unique-choice boundary (duplicated roles refuse) — an accept must never - * press a denial, and an alert whose choices leave the action undecided is refused rather than - * resolved by tree position. + * on, only a button whose own label states the asked-for choice may be pressed, resolved through + * the shared preference table — a multi-grant sheet ("Allow Once" / "Allow While Using App" / + * "Don't Allow") resolves to the preferred grant, while equal-rank labels or a vocabulary-free + * multiplicity refuse. An accept must never press a denial, and an undecided action is refused + * rather than resolved by tree position. */ export function chooseLimrunAlertButton( buttons: readonly LimrunAlertButton[], action: 'accept' | 'dismiss', ): LimrunAlertButton | null { - return selectUniqueAlertButtonByRole(buttons, action); + return selectRankedAlertButtonByRole(buttons, action); } diff --git a/packages/provider-limrun/src/alert.test.ts b/packages/provider-limrun/src/alert.test.ts index bdb9baf988..47e11979d2 100644 --- a/packages/provider-limrun/src/alert.test.ts +++ b/packages/provider-limrun/src/alert.test.ts @@ -9,6 +9,7 @@ import { limrunAlertTreeWithUnpressableButtons, limrunAppScreenTree, limrunAttAlertTree, + limrunLocationAlertTree, } from './alert-detection.fixtures.ts'; import { handleLimrunIosAlert, type LimrunIosAlertClient } from './alert.ts'; import type { IosTreeNode } from './snapshot.ts'; @@ -164,6 +165,20 @@ describe('acting budgets', () => { expect(status.kind === 'alertStatus' && status.message).toBe('No alert visible'); }); + it('accepts the three-button location sheet on the preferred grant, once', async () => { + // The multi-grant regression at the action seam: "Allow Once" and "Allow While Using App" + // both carry the accept role, and the shared preference table must land one tap on the + // lasting grant rather than refusing the sheet. + const client = clientWithTrees([limrunLocationAlertTree(), limrunAppScreenTree()]); + + const handled = await handleLimrunIosAlert(client, 'accept'); + expect(handled.kind).toBe('alertHandled'); + expect(handled.kind === 'alertHandled' && handled.button).toBe('Allow While Using App'); + expect(client.tap).toHaveBeenCalledTimes(1); + // Grant center: x 57 + 288/2, y 623.67 + 48/2, rounded by the press seam to whole points. + expect(client.tap).toHaveBeenCalledWith(201, 648); + }); + it('refuses to act on an alert-shaped but unresolvable capture, without tapping', async () => { const client = clientWithTrees([limrunAlertTreeWithUnpressableButtons()]); diff --git a/website/docs/docs/limrun.md b/website/docs/docs/limrun.md index 0df1373970..b5e259a400 100644 --- a/website/docs/docs/limrun.md +++ b/website/docs/docs/limrun.md @@ -71,7 +71,7 @@ agent-device disconnect On Android, agent-device connects to Limrun over an ADB tunnel. Snapshots use the Android snapshot helper, installs upload the app through Limrun asset storage, and port reverse runs over ADB, including the usual reverse setup for a local Metro server. -On iOS, agent-device supports app lifecycle commands, snapshots, screenshots, taps, text input, scrolling, the home button, clipboard read and write, app installation, and system alerts. `alert get/wait/accept/dismiss` read the alert from the same element tree as snapshots and dispatch the selected button; a choice is pressed only when its English label states the asked-for action (`Allow`, `Don't Allow`, `Ask App Not to Track`, and similar), so a localized alert is reported but never pressed. App-owned dialogs are not system alerts there, so use `snapshot -i` and then `press @ref` or a label selector for the visible control. `open --launch-args` relaunches the app with those arguments through simctl. iOS cannot reverse a remote device port to a local host port, so for Metro or React DevTools use a publicly reachable HTTPS endpoint or bridge URL instead of a local-only address. +On iOS, agent-device supports app lifecycle commands, snapshots, screenshots, taps, text input, scrolling, the home button, clipboard read and write, app installation, and system alerts. `alert get/wait/accept/dismiss` read the alert from the same element tree as snapshots and dispatch the selected button; a choice is pressed only when its English label states the asked-for action (`Allow`, `Don't Allow`, `Ask App Not to Track`, and similar), so a localized alert is reported but never pressed. A sheet offering two grants (`Allow Once` / `Allow While Using App`) presses the preferred lasting grant from the declared table, never a coin flip. App-owned dialogs are not system alerts there, so use `snapshot -i` and then `press @ref` or a label selector for the visible control. `open --launch-args` relaunches the app with those arguments through simctl. iOS cannot reverse a remote device port to a local host port, so for Metro use a publicly reachable HTTPS endpoint or bridge URL instead of a local-only address. On iOS, `settings` supports `appearance`, `permission`, `location`, and `clear-app-state`, with the same arguments as a local simulator. `clear-app-state` resets the app's data container through Limrun's soft reset. That reset relaunches the app once before agent-device stops it, so the app may run once after its data is cleared. `reset-keychain`, `wifi`, `airplane`, `mobile-data`, `faceid`, `touchid`, and `text-size` are unsupported, and you cannot read back any iOS setting. On Android, `settings mobile-data on|off` runs over the ADB tunnel like on a local device. From 9bf80e27efa1263728c89090db702b57b5c9b299 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 14:20:05 +0200 Subject: [PATCH 07/12] fix(alerts): type the undecided state and bound the seams each budget owns Three review findings on the shared tree-alert driver, plus the test that masked a polling loop: - alert get on a tree backend answered undecided exactly like absence (alert: null, message-only difference). The result now carries a typed status (visible/none/undecided) so clients switch on states, not text. - The budgets bounded only the poll sleeps: a capture or press that hung could outrun the leg. Each leg now builds a window (deadline clock plus the joined window/caller signal) that reaches the capture and press seams; the Android legs hand it to the snapshot capture and the adb input sends, which kill their child on abort. A capture that lands once its window closed is refused on a typed alert-window-expired reason and never consumed or pressed; a press cut off mid-flight says dispatched: unknown, one refused before dispatch says no. - Limrun's WebSocket transport exposes no abort on elementTree/tap (checked against the shipped typings), so the consumption rule is what keeps it honest, and the SDK gap is stated rather than papered over. - The awaitAlert first-capture test now asserts the capture count; its same-alert fixture had made it vacuous. --- CHANGELOG.md | 2 +- packages/contracts/src/alert-contract.ts | 10 + packages/contracts/src/alert-runtime.test.ts | 10 +- packages/contracts/src/alert-runtime.ts | 8 +- packages/contracts/src/interactor-types.ts | 2 + packages/platform-android/src/alert.ts | 321 +++++++++++++++--- .../platform-android/src/input-actions.ts | 28 +- packages/provider-limrun/src/alert.test.ts | 35 +- packages/provider-limrun/src/alert.ts | 13 +- .../src/ios-interactor-alert.test.ts | 7 +- packages/provider-limrun/src/ios.ts | 17 +- src/core/interactors/android.ts | 8 +- website/docs/docs/commands.md | 1 + 13 files changed, 386 insertions(+), 76 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8afa2e20ca..06ba5d0e05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ - On iOS, a `fill` that runs under a degraded XCTest snapshot channel now aims at a point, including ref, selector and replay fills. When the runner identifies no text input under that point, it refuses with `NO_TEXT_INPUT_AT_POINT` (`error.details.reason: no_text_input_at_point`, `dispatched: no`) before dispatching its focus tap, and no field changes; when the input lookup itself cannot answer, it refuses with `TEXT_INPUT_PROBE_UNAVAILABLE` (`error.details.reason: text_input_probe_unavailable`). Previously such a fill cleared and typed through the app-level first responder, so it could rewrite a field the caller never named, or report a clear failure after emptying it. A ref or selector fill whose rect centre misses the real field (an outer container, a non-input node) now needs a point inside the input's own rect. - Opening a macOS app by bundle identifier now waits briefly when the app is already running but not yet registered with LaunchServices — the case when a supervisor or dev script started it directly — and if registration arrives within that bounded window the open attaches to the running copy, so the session drives the app the caller started, with its arguments. A copy still unlisted when the window closes gets the plain `open` as before, which can still duplicate; the wait only bounds the race, it does not close it. Launching an app that is not running costs one extra LaunchServices lookup and one process-table read, plus one bundle-identifier read when app-shaped processes exist. - Limrun iOS sessions now serve `alert get/wait/accept/dismiss` instead of refusing with `unsupported-provider-mode`. System alerts are read from the same element-tree capture snapshots ride — a blank-label, process-exclusive application root hosting a text-and-buttons sheet — and accepted or dismissed by tapping the button whose English label states the asked-for action; a capture that is alert-shaped but unresolvable (degraded buttons, competing sheets, an unverifiable owner) is reported as undecided and never pressed or counted as dismissed. The one shape no capture fact separates is an app dialog under a blanked root shared by no other process; the label-only acting legs answer it with a refusal, never a guess. -- The tree-based alert backends (Android and Limrun iOS) now share one alert driver and one English button-label vocabulary declared as a per-role preference table: `alert accept/dismiss` classify the same label the same way on both platforms, so Android now recognizes iOS variant grants (`Allow Once`, `Allow All the Time`, `Allow While Using App`, `Allow Notifications`) and `Ask App Not to Track` as acting choices, and a sheet offering two grants (Android 11+'s "While using the app"/"Only this time", iOS's "Allow Once"/"Allow While Using App") resolves by the declared preference order rather than tree order or refusal. A choice refuses only when the capture names no single physical button for the action — the same label at two points — and each backend's own policy answers then (Android's neutral-button/Back fallback, Limrun's typed refusal). `alert not found` and the no-matching-button refusal now carry the same typed `details.reason` on both platforms. +- The tree-based alert backends (Android and Limrun iOS) now share one alert driver and one English button-label vocabulary declared as a per-role preference table: `alert accept/dismiss` classify the same label the same way on both platforms, so Android now recognizes iOS variant grants (`Allow Once`, `Allow All the Time`, `Allow While Using App`, `Allow Notifications`) and `Ask App Not to Track` as acting choices, and a sheet offering two grants (Android 11+'s "While using the app"/"Only this time", iOS's "Allow Once"/"Allow While Using App") resolves by the declared preference order rather than tree order or refusal. A choice refuses only when the capture names no single physical button for the action — the same label at two points — and each backend's own policy answers then (Android's neutral-button/Back fallback, Limrun's typed refusal). `alert not found` and the no-matching-button refusal now carry the same typed `details.reason` on both platforms. `alert get` reports its outcome as a typed `status` (`visible`/`none`/`undecided`) so clients switch on states instead of parsing the message, and every alert budget now bounds the capture and press inside it: Android hands the leg's cancellation to the adb executor (which kills the child), and a capture that lands after its window closed — the case Limrun's non-abortable transport can produce — is refused rather than consumed or pressed. - Limrun iOS snapshots no longer fail when the instance reports a numeric or boolean accessibility value (the system photo picker's tabs report `1`/`0`): the provider seam normalizes variant scalars to text, the way the Apple tree does. Limrun nodes now also publish `selected: true` for nodes carrying the `Selected` trait, so snapshot output and `selected:` selectors can tell a picked photo from an unpicked one. - `settings` refuses a setting the target does not serve before anything changes, with `UNSUPPORTED_OPERATION`, `settings is not supported on this device`, and the target's `error.details.reason` (such as `unsupported-platform-leaf`). This replaces the macOS `INVALID_ARGS` refusal and the Apple `setting-unsupported-on-leaf` reason, and now also covers settings that used to reach the device first, such as Android `reset-keychain` or Apple `animations`. - `settings` refuses, before touching the device, a state its setting does not accept (`Invalid state: …`), a permission action or mode its target does not take (such as `permission deny accessibility`), and a stray extra argument (such as `clear-app-state `). The CLI and the daemon accept the same state spellings, in any casing or surrounding whitespace. diff --git a/packages/contracts/src/alert-contract.ts b/packages/contracts/src/alert-contract.ts index 43fa824c8b..2b7fd1c118 100644 --- a/packages/contracts/src/alert-contract.ts +++ b/packages/contracts/src/alert-contract.ts @@ -22,6 +22,16 @@ export const ALERT_NOT_FOUND_REASON = 'alert-not-found'; */ export const ALERT_BUTTON_NOT_FOUND_RUNNER_CODE = 'ALERT_BUTTON_NOT_FOUND'; +/** + * The acting leg's window closed after the alert resolved but before the press could be sent. + * Typed so a caller can tell "the sheet was there and we refused to press late" from absence or + * an unresolvable capture — the press was never dispatched. + */ +export const ALERT_WINDOW_EXPIRED_REASON = 'alert-window-expired'; + +/** What an `alert get` capture resolved to, typed so clients switch on states, not messages. */ +export type AlertStatusKind = 'visible' | 'none' | 'undecided'; + export type AlertButtonRole = 'accept' | 'dismiss' | 'neutral'; /** diff --git a/packages/contracts/src/alert-runtime.test.ts b/packages/contracts/src/alert-runtime.test.ts index d0a5dab34d..87629b4d7e 100644 --- a/packages/contracts/src/alert-runtime.test.ts +++ b/packages/contracts/src/alert-runtime.test.ts @@ -58,6 +58,9 @@ test('each local leg forwards the window and the session target to its own owner timeoutMs: 37, appBundleId: 'com.example.app', surface: 'app', + // The request's cancellation rides along: owners whose transport cannot abort a call (the + // Limrun tree legs) still refuse to consume a capture once the request is canceled. + signal, }; expect(legs.readAlert).toHaveBeenCalledWith(expectedOptions); expect(legs.awaitAlert).toHaveBeenCalledWith(expectedOptions); @@ -76,12 +79,13 @@ test('an absent target field never reaches the owner as an explicit undefined', const readAlert = vi.fn(async (_options?: AlertInteractorOptions) => ({})); const resolveInteractor = async () => ({ readAlert }) as unknown as Interactor; const source = localInteractorSource({ device, resolveInteractor }); - const operations = bindAlertLeg('readAlert', new AbortController().signal, source); + const signal = new AbortController().signal; + const operations = bindAlertLeg('readAlert', signal, source); await operations.readAlert({ surface: 'frontmost-app' }); - expect(readAlert).toHaveBeenCalledWith({ surface: 'frontmost-app' }); - expect(Object.keys(readAlert.mock.calls[0]?.[0] ?? {})).toEqual(['surface']); + expect(readAlert).toHaveBeenCalledWith({ surface: 'frontmost-app', signal }); + expect(Object.keys(readAlert.mock.calls[0]?.[0] ?? {})).toEqual(['surface', 'signal']); }); // Facts admitted the leg, so an interactor without it is an ownership bug the caller must see diff --git a/packages/contracts/src/alert-runtime.ts b/packages/contracts/src/alert-runtime.ts index b3879cd205..c0e69cc660 100644 --- a/packages/contracts/src/alert-runtime.ts +++ b/packages/contracts/src/alert-runtime.ts @@ -94,11 +94,15 @@ async function resolveAlertInteractor( }); } -function alertInteractorOptions(input: AlertRuntimeInput): AlertInteractorOptions { +function alertInteractorOptions( + input: AlertRuntimeInput, + signal: AbortSignal | undefined, +): AlertInteractorOptions { return { ...(input.timeoutMs === undefined ? {} : { timeoutMs: input.timeoutMs }), ...(input.appBundleId === undefined ? {} : { appBundleId: input.appBundleId }), ...(input.surface === undefined ? {} : { surface: input.surface }), + ...(signal ? { signal } : {}), }; } @@ -124,7 +128,7 @@ export function bindAlertLeg( // held at its merge-base size (`eager-closure-budgets`); a static edge would grow it. const { requireInteractorMethod } = await import('./interactor-operation-binding.ts'); const method = requireInteractorMethod(interactor[leg], ALERT_LEG_LABELS[leg]); - return await method.call(interactor, alertInteractorOptions(input)); + return await method.call(interactor, alertInteractorOptions(input, signal)); }, }) as Readonly Promise>>>; } diff --git a/packages/contracts/src/interactor-types.ts b/packages/contracts/src/interactor-types.ts index c4b6381db4..d5cf0a7343 100644 --- a/packages/contracts/src/interactor-types.ts +++ b/packages/contracts/src/interactor-types.ts @@ -474,6 +474,8 @@ export type Interactor = { */ export type AlertInteractorOptions = { timeoutMs?: number; + /** The daemon request's cancellation; owners whose transport can abort thread it into the legs. */ + signal?: AbortSignal; appBundleId?: string; surface?: SessionSurface; }; diff --git a/packages/platform-android/src/alert.ts b/packages/platform-android/src/alert.ts index a5ef0ec909..b663f8bba9 100644 --- a/packages/platform-android/src/alert.ts +++ b/packages/platform-android/src/alert.ts @@ -8,11 +8,13 @@ import { type AlertPlatform, ALERT_NOT_FOUND_REASON, ALERT_POLL_INTERVAL_MS, + ALERT_WINDOW_EXPIRED_REASON, + type AlertStatusKind, DEFAULT_ALERT_TIMEOUT_MS, } from '@agent-device/contracts/alert-contract'; -import { AppError } from '@agent-device/kernel/errors'; +import { AppError, createRequestCanceledError } from '@agent-device/kernel/errors'; import { withDiagnosticTimer } from '@agent-device/host-kit/diagnostics'; -import { sleep } from '@agent-device/host-kit/retry'; +import { type DeadlineClock, Deadline, pollUntilSettled } from '@agent-device/host-kit/retry'; import { successText } from '@agent-device/kernel/success-text'; import type { DeviceInfo } from '@agent-device/kernel/device'; @@ -57,13 +59,17 @@ export type TreeAlertCandidate< buttons: TButton[]; }>; -export type TreeAlertSleep = (milliseconds: number) => Promise; - /** * The seams one tree-based backend owns. `timeoutMs` bounds `wait`; the acting legs ride the - * contract action budget. The sleep seam exists so tests advance polls on a driven clock - * (`docs/agents/testing.md`) instead of production time. + * contract action budget. Poll sleeps ride the shared `pollUntilSettled` deadline primitive, so + * tests drive the clock (`docs/agents/testing.md`) rather than the driver owning a timer seam. + * `capture` and `press` receive the live cancellation signal for their attempt — the caller's + * request signal joined with the loop's remaining window — so a backend whose transport can be + * canceled stops in-flight work, and every backend refuses to consume a capture or dispatch a + * press once the signal has fired. A transport that cannot cancel still gains both refusals. */ +export type TreeAlertSeam = Readonly<{ signal: AbortSignal }>; + export type TreeAlertDriver< TCapture, TAlert extends AlertInfo & { buttons: string[] }, @@ -73,11 +79,18 @@ export type TreeAlertDriver< > = Readonly<{ platform: TPlatform; timeoutMs?: number; - sleep?: TreeAlertSleep; - capture: () => Promise; + /** The daemon request's cancellation, when the caller rides one; joined into every seam call. */ + signal?: AbortSignal; + /** + * One capture, offered the leg's live signal. A backend whose transport can abort (Android's + * adb executor kills the child) hands it down; a backend whose transport cannot (Limrun's + * WebSocket client) ignores it, and the driver's window guard is what keeps its late captures + * unconsumed. + */ + capture: (seam: TreeAlertSeam) => Promise; findCandidate: (capture: TCapture) => TreeAlertObservation; choose: (candidate: TCandidate, action: 'accept' | 'dismiss') => TreeAlertPress | null; - press: (choice: TreeAlertPress) => Promise; + press: (choice: TreeAlertPress, seam: TreeAlertSeam) => Promise; }>; /** The records the daemon composes CLI and SDK output around, identical across tree backends. */ @@ -90,6 +103,8 @@ export type TreeAlertResult< platform: TPlatform; action: 'get'; alert: TAlert | null; + /** Typed absence vs. an unresolvable capture, so clients switch on states, not messages. */ + status: AlertStatusKind; message?: string; } | { @@ -122,6 +137,13 @@ const ALERT_NOT_FOUND_HINT = * both predecessor loops took their first immediate observation before any waiting, and one * immediate observation is the honest answer shape for a zero-length `wait` window too. A * malformed `timeoutMs` is the bounded default, never a loop whose bound is never reached. + * + * Every budget also bounds the work inside it: each polling leg carries a window signal that + * expires with its deadline and a request signal from the caller, and both reach the seams. + * Captures then stop in-flight transport work wherever the backend's transport can cancel (the + * Android legs hand the signal to the adb executor, which kills the child); backends whose + * transport cannot cancel — Limrun's WebSocket client exposes no abort — still never consume a + * capture that lands after its window closed, and no press is dispatched past the window. */ export async function runTreeAlertAction< TCapture, @@ -133,15 +155,23 @@ export async function runTreeAlertAction< action: AlertAction, driver: TreeAlertDriver, ): Promise> { - const nap = driver.sleep ?? sleep; - const observe: TreeAlertObserver = async () => - driver.findCandidate(await driver.capture()); if (action === 'get') { + // One immediate observation with no window to expire: only the caller's cancellation binds it. + const observe = treeAlertObserver(driver, undefined); return buildTreeAlertStatusResponse(driver.platform, await observe()); } if (action === 'wait') { const start = Date.now(); - const polled = await pollTreeAlert(observe, alertWindowMs(driver.timeoutMs), nap); + const window = treeAlertWindow(alertWindowMs(driver.timeoutMs), driver.signal); + const observe = treeAlertObserver(driver, window); + let polled: TreeAlertObservation; + try { + polled = await pollTreeAlert(observe, window, driver.signal); + } catch (error) { + // A capture the window cut off is, for `wait`, exactly what the window closing means. + if (isAlertWindowExpired(error)) throw new AppError('COMMAND_FAILED', 'alert wait timed out'); + throw error; + } if (polled.kind !== 'alert') { // Both a quiet capture and one that stayed alert-shaped yet unresolvable mean no // actionable alert ever resolved within the window; the timeout phrasing is the honest @@ -157,9 +187,120 @@ export async function runTreeAlertAction< ...successText('Alert visible'), }; } - return await actOnTreeAlert(action, driver, observe, nap); + return await actOnTreeAlert(action, driver); } +/** What the observer reads of a driver: the two seams it runs and the signal it joins. */ +type ObserverDriver = Readonly<{ + signal?: AbortSignal; + capture: (seam: TreeAlertSeam) => Promise; + findCandidate: (capture: TCapture) => TreeAlertObservation; +}>; + +/** + * One observation of the backend's capture, bound by the driver's window. `window` is the leg's + * deadline clock plus the joined signal (this window's expiry, the caller's cancellation) the + * seams carry. Android hands that signal to the adb executor, which kills the child process, so + * a capture in flight when the window closes is actually stopped; Limrun's WebSocket client + * exposes no abort, so the post-await check here is what keeps that backend honest — a capture + * that lands once its window has closed proves nothing, is refused on the typed window reason, + * and is never consumed. A probe that starts once the window is already spent is the polling + * loop's own closing observation: it answers normally and the loop ends on its deadline. What + * may never happen is consuming a spent-window observation as actionable, which the press-time + * window check in `actOnTreeAlert` enforces. The first observation of a leg rides only the + * caller's cancellation: the read-first contract promises one immediate honest answer even when + * the window is already spent, and a probe that starts inside the window cannot have outrun a + * deadline it has not reached yet — its own landing decides, not a race with the clock. + */ +type TreeAlertWindow = Readonly<{ deadline: DeadlineClock; signal: AbortSignal }>; + +/** + * One leg's window: a deadline clock the driver polls against and the joined signal (this + * window's expiry, the caller's cancellation) the seams carry. The clock and signal start on + * the same line, so an expired clock and an aborted signal agree on everything but the + * millisecond. + */ +function treeAlertWindow( + timeoutMs: number, + callerSignal: AbortSignal | undefined, +): TreeAlertWindow { + return { + deadline: Deadline.fromTimeoutMs(timeoutMs), + signal: AbortSignal.any([ + AbortSignal.timeout(timeoutMs), + ...(callerSignal ? [callerSignal] : []), + ]), + }; +} + +function treeAlertObserver( + driver: ObserverDriver, + window: TreeAlertWindow | undefined, +): TreeAlertObserver { + return async () => { + if (driver.signal?.aborted) throw createRequestCanceledError(); + // A window already spent at probe start is the polling loop's own closing observation: it + // answers normally (the read-first contract keeps one immediate honest answer even for a + // spent window) and the loop ends on its deadline. A probe that starts inside the window is + // guarded: its seam can be canceled, and its landing is checked against the clock. + const guarded = window !== undefined && !window.deadline.isExpired(); + const capture = await captureAlertState( + driver, + guarded ? window.signal : (driver.signal ?? neverSignal), + ); + if (guarded && window.deadline.isExpired() && !driver.signal?.aborted) { + throw alertWindowExpiredError('no'); + } + return driver.findCandidate(capture); + }; +} + +/** + * One seam capture. Inside its window the seam signal can cut the transport mid-flight; a + * capture refused by its own window signal is exactly the outrunning capture, and reports the + * typed cause. The caller's own cancellation always surfaces as the shared cancellation error. + */ +async function captureAlertState( + driver: ObserverDriver, + seamSignal: AbortSignal, +): Promise { + try { + return await driver.capture({ signal: seamSignal }); + } catch (error) { + if (driver.signal?.aborted) throw createRequestCanceledError(); + if (seamSignal.aborted) throw alertWindowExpiredError('no'); + throw error; + } +} + +/** + * The failure one seam call took when its signal cut it off: the caller canceled, or the leg's + * window closed mid-capture. `dispatched` stays 'no' for pre-press legs; the confirm leg, which + * runs after a press already landed, maps it to its own stuck-alert failure instead. + */ +function alertSeamInterrupted(canceled: boolean): AppError { + if (canceled) throw createRequestCanceledError(); + return alertWindowExpiredError('no'); +} + +function alertWindowExpiredError(dispatched: 'no'): AppError { + return new AppError('COMMAND_FAILED', 'alert window expired before the action could complete', { + reason: ALERT_WINDOW_EXPIRED_REASON, + dispatched, + hint: 'The capture outran the alert budget; nothing was pressed. Retry alert get to see the current state.', + }); +} + +function isAlertWindowExpired(error: unknown): boolean { + return ( + error instanceof AppError && + (error.details as { reason?: unknown } | undefined)?.reason === ALERT_WINDOW_EXPIRED_REASON + ); +} + +/** The seam signal for a leg with no window and no caller cancellation: nothing can cancel it. */ +const neverSignal = new AbortController().signal; + function alertWindowMs(timeoutMs: number | undefined): number { return typeof timeoutMs === 'number' && Number.isFinite(timeoutMs) && timeoutMs >= 0 ? timeoutMs @@ -167,27 +308,29 @@ function alertWindowMs(timeoutMs: number | undefined): number { } /** - * Polls for an actionable candidate within the budget. The terminal answer distinguishes a - * window that stayed quiet (`none`) from one that stayed alert-shaped yet unresolvable - * (`undecided`): a degraded sheet that never resolved is not an absence, and callers must not - * report the two alike. + * Polls for an actionable candidate within the budget, on the same deadline primitive the + * daemon's other polling loops ride. The terminal answer distinguishes a window that stayed + * quiet (`none`) from one that stayed alert-shaped yet unresolvable (`undecided`): a degraded + * sheet that never resolved is not an absence, and callers must not report the two alike. */ async function pollTreeAlert( observe: TreeAlertObserver, - timeoutMs: number, - nap: TreeAlertSleep, + window: TreeAlertWindow, + signal?: AbortSignal, ): Promise> { - const start = Date.now(); let sawUndecided = false; - for (;;) { - const observation = await observe(); - if (observation.kind === 'alert') return observation; - if (observation.kind === 'undecided') sawUndecided = true; - if (Date.now() - start >= timeoutMs) { - return sawUndecided ? { kind: 'undecided' } : { kind: 'none' }; - } - await nap(ALERT_POLL_INTERVAL_MS); - } + const { settled, value } = await pollUntilSettled({ + probe: observe, + settled: (observation) => { + if (observation.kind === 'undecided') sawUndecided = true; + return observation.kind === 'alert'; + }, + deadline: window.deadline, + intervalMs: ALERT_POLL_INTERVAL_MS, + signal, + }); + if (settled) return value; + return sawUndecided ? { kind: 'undecided' } : { kind: 'none' }; } /** @@ -205,10 +348,26 @@ async function actOnTreeAlert< >( action: 'accept' | 'dismiss', driver: TreeAlertDriver, - observe: TreeAlertObserver, - nap: TreeAlertSleep, ): Promise> { - const polled = await pollTreeAlert(observe, ALERT_ACTION_RETRY_MS, nap); + const lookupWindow = treeAlertWindow(ALERT_ACTION_RETRY_MS, driver.signal); + const observe = treeAlertObserver(driver, lookupWindow); + let polled: TreeAlertObservation; + try { + polled = await pollTreeAlert(observe, lookupWindow, driver.signal); + } catch (error) { + if (isAlertWindowExpired(error)) { + throw new AppError( + 'COMMAND_FAILED', + `alert ${action} refused: the action budget closed before an alert resolved`, + { + reason: ALERT_WINDOW_EXPIRED_REASON, + dispatched: 'no', + hint: 'Nothing was pressed. Retry alert get to see the current state.', + }, + ); + } + throw error; + } if (polled.kind === 'undecided') { throw new AppError( 'COMMAND_FAILED', @@ -241,8 +400,42 @@ async function actOnTreeAlert< }, ); } - await driver.press(choice); - await confirmTreeAlertDismissed(observe, candidate.alert, action, nap); + // The press rides the lookup's remaining window and the caller's cancellation: a press whose + // window closed before it went out has not pressed anything, and says so on a typed `no`. + const pressSignal = lookupWindow.signal; + if (pressSignal.aborted) throw alertSeamInterrupted(driver.signal?.aborted === true); + try { + await driver.press(choice, { signal: pressSignal }); + } catch (error) { + // A press cut off mid-flight cannot report what the device already received: the tap may + // or may not have been dispatched. That uncertainty is typed, never a clean 'no'. + if (pressSignal.aborted) { + if (driver.signal?.aborted) throw createRequestCanceledError(); + throw new AppError( + 'COMMAND_FAILED', + `alert ${action} was cut off while pressing; the press may or may not have landed`, + { + reason: ALERT_WINDOW_EXPIRED_REASON, + dispatched: 'unknown', + alert: candidate.alert, + hint: 'Inspect alert get --json before retrying: the dialog may already be gone.', + }, + ); + } + throw error; + } + // The confirm window is its own budget: it only starts once the press has been dispatched, + // and it must not consume the lookup's already-spent window. A capture this phase cuts off + // lands after a press, so its failure keeps the stuck-alert phrasing, never "nothing was + // pressed" — the confirm observer maps its window expiry onto that failure below. + const confirmWindow = treeAlertWindow(ALERT_ACTION_RETRY_MS, driver.signal); + await confirmTreeAlertDismissed( + treeAlertObserver(driver, confirmWindow), + confirmWindow, + candidate.alert, + action, + driver.signal, + ); return { kind: 'alertHandled', platform: driver.platform, @@ -269,25 +462,40 @@ async function confirmTreeAlertDismissed< TCandidate extends TreeAlertCandidate, >( observe: TreeAlertObserver, + window: TreeAlertWindow, dismissed: TAlert, action: 'accept' | 'dismiss', - nap: TreeAlertSleep, + signal?: AbortSignal, ): Promise { - const start = Date.now(); - for (;;) { - const observation = await observe(); - if (observation.kind === 'none') return; - if (observation.kind === 'alert' && !sameTreeAlert(observation.candidate.alert, dismissed)) { - return; - } - if (Date.now() - start >= ALERT_ACTION_RETRY_MS) { + let settled: boolean; + try { + ({ settled } = await pollUntilSettled({ + probe: observe, + settled: (observation) => + observation.kind === 'none' || + (observation.kind === 'alert' && !sameTreeAlert(observation.candidate.alert, dismissed)), + deadline: window.deadline, + intervalMs: ALERT_POLL_INTERVAL_MS, + signal, + })); + } catch (error) { + // A confirmation capture this phase cuts off says the same thing a budget spent without + // confirmation says — and it says it after a press landed, so the stuck-alert phrasing is + // the honest one here; the typed reason keeps the two causes distinguishable. + if (isAlertWindowExpired(error)) { throw new AppError('COMMAND_FAILED', `alert ${action} did not dismiss the visible alert`, { alert: dismissed, + reason: ALERT_WINDOW_EXPIRED_REASON, hint: 'The alert button was pressed but the dialog is still visible. Inspect alert get --json, then press the visible button by label/ref or retry.', }); } - await nap(ALERT_POLL_INTERVAL_MS); + throw error; } + if (settled) return; + throw new AppError('COMMAND_FAILED', `alert ${action} did not dismiss the visible alert`, { + alert: dismissed, + hint: 'The alert button was pressed but the dialog is still visible. Inspect alert get --json, then press the visible button by label/ref or retry.', + }); } /** Alert identity across a dismissal re-check: the same title and the same button row, in order. */ @@ -312,6 +520,12 @@ function buildTreeAlertStatusResponse< platform, action: 'get', alert: observation.kind === 'alert' ? observation.candidate.alert : null, + status: + observation.kind === 'alert' + ? 'visible' + : observation.kind === 'undecided' + ? 'undecided' + : 'none', ...(observation.kind === 'alert' ? successText('Alert visible') : observation.kind === 'undecided' @@ -324,7 +538,9 @@ function buildTreeAlertStatusResponse< type AndroidAlertOptions = { timeoutMs?: number; - captureNodes: () => Promise; + /** The daemon request's cancellation; the driver joins it into every capture and press. */ + signal?: AbortSignal; + captureNodes: (seam: TreeAlertSeam) => Promise; }; export type AndroidAlertResult = TreeAlertResult; @@ -343,8 +559,9 @@ export async function handleAndroidAlert( >(action, { platform: 'android', timeoutMs: options.timeoutMs, - capture: () => - withDiagnosticTimer('snapshot_capture', options.captureNodes, { + ...(options.signal ? { signal: options.signal } : {}), + capture: (seam) => + withDiagnosticTimer('snapshot_capture', () => options.captureNodes(seam), { backend: 'android', purpose: 'alert', }), @@ -361,12 +578,14 @@ export async function handleAndroidAlert( // key means. The refusal message and typed reason come from the driver, not from here. return act === 'dismiss' ? { kind: 'fallback', label: 'Back' } : null; }, - press: async (choice) => { + press: async (choice, seam) => { + // The adb executor kills the child on abort, so the press transport itself stops with the + // leg's signal; whether a canceled tap still reached the device stays typed upstream. if (choice.kind === 'press') { - await pressAndroid(device, choice.button.x, choice.button.y); + await pressAndroid(device, choice.button.x, choice.button.y, { signal: seam.signal }); return; } - await backAndroid(device); + await backAndroid(device, { signal: seam.signal }); }, }); } diff --git a/packages/platform-android/src/input-actions.ts b/packages/platform-android/src/input-actions.ts index 49fbc0fbbc..15d037bbed 100644 --- a/packages/platform-android/src/input-actions.ts +++ b/packages/platform-android/src/input-actions.ts @@ -25,23 +25,34 @@ import { AppError, discloseDispatchAfterSteps } from '@agent-device/kernel/error import type { Rect } from '@agent-device/kernel/snapshot'; import { sleep } from '@agent-device/host-kit/retry'; import { runAndroidShell } from './adb.ts'; +import type { AndroidAdbExecutorOptions } from './adb-transport.ts'; import { androidAdbResultError, discloseAdbInputDispatch } from './adb-failure.ts'; import { executeAndroidTouchPlan, readAndroidGestureViewportReading } from './touch-executor.ts'; import type { AndroidHelperSessionOptions } from './snapshot-helper-types.ts'; -export async function pressAndroid(device: DeviceInfo, x: number, y: number): Promise { +export async function pressAndroid( + device: DeviceInfo, + x: number, + y: number, + executorOptions?: AndroidAdbExecutorOptions, +): Promise { try { - await runAndroidShell(device, ['input', 'tap', x, y]); + await runAndroidShell(device, ['input', 'tap', x, y], executorOptions); } catch (error) { throw discloseAdbInputDispatch(error); } } /** Two `input tap` sends; a failure of the second follows a tap that already landed. */ -export async function doubleTapAndroid(device: DeviceInfo, x: number, y: number): Promise { - await pressAndroid(device, x, y); +export async function doubleTapAndroid( + device: DeviceInfo, + x: number, + y: number, + executorOptions?: AndroidAdbExecutorOptions, +): Promise { + await pressAndroid(device, x, y, executorOptions); try { - await pressAndroid(device, x, y); + await pressAndroid(device, x, y, executorOptions); } catch (error) { throw discloseDispatchAfterSteps(error, 1); } @@ -57,8 +68,11 @@ export async function pressAndroidTvRemote( await runAndroidShell(device, ['input', ...keyeventArgs, keyevent]); } -export async function backAndroid(device: DeviceInfo): Promise { - await runAndroidShell(device, ['input', 'keyevent', '4']); +export async function backAndroid( + device: DeviceInfo, + executorOptions?: AndroidAdbExecutorOptions, +): Promise { + await runAndroidShell(device, ['input', 'keyevent', '4'], executorOptions); } export async function homeAndroid(device: DeviceInfo): Promise { diff --git a/packages/provider-limrun/src/alert.test.ts b/packages/provider-limrun/src/alert.test.ts index 47e11979d2..1a98d5801b 100644 --- a/packages/provider-limrun/src/alert.test.ts +++ b/packages/provider-limrun/src/alert.test.ts @@ -77,6 +77,37 @@ describe('wait budget', () => { await expect(handleLimrunIosAlert(client, 'wait')).rejects.toThrow('websocket closed'); }); + + it('refuses a confirm capture that outruns its window and reports the stuck alert', async () => { + // The Limrun WebSocket client cannot abort an in-flight `elementTree`, so a stalled capture + // can outlive the confirm budget. The driver must not consume that late answer: the press + // already landed, the capture proves nothing, and the honest terminal answer is the + // stuck-alert failure on the typed window reason. + const client = { + elementTree: vi + .fn() + .mockResolvedValueOnce([limrunAttAlertTree()]) // read-first probe resolves the alert + // The confirm probe stalls 3s on the driven clock — past its 2s budget — and would + // answer "gone" if its late result were trusted. + .mockImplementationOnce( + async () => + new Promise((resolve) => { + setTimeout(() => resolve([limrunAppScreenTree()]), 3_000); + }), + ), + tap: vi.fn(async () => {}), + } satisfies LimrunIosAlertClient; + + const thrown = await underFakeTimers(() => handleLimrunIosAlert(client, 'accept')).then( + () => undefined, + (error: unknown) => error, + ); + expect(client.tap).toHaveBeenCalledTimes(1); + expect(thrown).toBeInstanceOf(AppError); + const failure = thrown as AppError; + expect(failure.message).toBe('alert accept did not dismiss the visible alert'); + expect(failure.details?.reason).toBe('alert-window-expired'); + }); }); describe('get on an undecided capture', () => { @@ -86,6 +117,9 @@ describe('get on an undecided capture', () => { const result = await handleLimrunIosAlert(client, 'get'); expect(result.kind === 'alertStatus' && result.alert).toBeNull(); + // The states are typed on the result, not just in the message: `undecided` and `none` + // arrive with the same null alert and must stay switchable apart. + expect(result.kind === 'alertStatus' && result.status).toBe('undecided'); expect(result.kind === 'alertStatus' && result.message).toBe( 'Cannot decide the alert: the capture is alert-shaped but unresolvable', ); @@ -104,7 +138,6 @@ describe('acting budgets', () => { it('refuses an ambiguous alert without tapping, on a typed reason', async () => { const client = clientWithTrees([limrunAlertAmbiguousButtonsTree()]); - const thrown = await handleLimrunIosAlert(client, 'accept').then( () => undefined, (error: unknown) => error, diff --git a/packages/provider-limrun/src/alert.ts b/packages/provider-limrun/src/alert.ts index d034cf094c..6910b9e695 100644 --- a/packages/provider-limrun/src/alert.ts +++ b/packages/provider-limrun/src/alert.ts @@ -32,10 +32,16 @@ export type LimrunIosAlertResult = TreeAlertResult; export type LimrunIosAlertAction = 'get' | 'wait' | 'accept' | 'dismiss'; +export type LimrunIosAlertOptions = Readonly<{ + timeoutMs?: number; + /** The daemon request's cancellation, refused at (never canceled inside) the transport. */ + signal?: AbortSignal; +}>; + export async function handleLimrunIosAlert( client: LimrunIosAlertClient, action: LimrunIosAlertAction, - options?: Readonly<{ timeoutMs?: number }>, + options?: LimrunIosAlertOptions, ): Promise { const driver: TreeAlertDriver< Awaited>, @@ -46,6 +52,11 @@ export async function handleLimrunIosAlert( > = { platform: 'ios', timeoutMs: options?.timeoutMs, + ...(options?.signal ? { signal: options.signal } : {}), + // The Limrun WebSocket client exposes no abort on `elementTree`/`tap`, so the seam signal + // cannot cancel its transport. The driver still enforces the window around this seam: a + // capture that lands after its window closed is refused rather than consumed, and no press + // dispatches once the window has closed. capture: () => client.elementTree(), findCandidate: (tree): TreeAlertObservation => readLimrunAlertCapture(tree as Parameters[0]), diff --git a/packages/provider-limrun/src/ios-interactor-alert.test.ts b/packages/provider-limrun/src/ios-interactor-alert.test.ts index 109a8e13a7..d3cb09b660 100644 --- a/packages/provider-limrun/src/ios-interactor-alert.test.ts +++ b/packages/provider-limrun/src/ios-interactor-alert.test.ts @@ -43,6 +43,7 @@ describe('readAlert', () => { const result = await interactor.readAlert!(); expect(result.kind).toBe('alertStatus'); + expect(result.status).toBe('visible'); expect(result.alert).toMatchObject({ platform: 'ios', source: 'permission', @@ -57,17 +58,21 @@ describe('readAlert', () => { expect(result.kind).toBe('alertStatus'); expect(result.alert).toBeNull(); + expect(result.status).toBe('none'); }); }); describe('awaitAlert', () => { it('returns as soon as a capture shows the alert', async () => { - const { interactor } = sessionWithTrees([limrunAttAlertTree(), limrunAttAlertTree()]); + // The fixture serves the same alert on every read, so the read count is the assertion that + // proves this leg returns on the FIRST capture rather than polling past it. + const { interactor, client } = sessionWithTrees([limrunAttAlertTree(), limrunAttAlertTree()]); const result = await interactor.awaitAlert!({ timeoutMs: 5_000 }); expect(result.kind).toBe('alertWait'); expect(result.alert).toMatchObject({ buttons: ['Ask App Not to Track', 'Allow'] }); + expect(client.elementTree).toHaveBeenCalledTimes(1); }); }); diff --git a/packages/provider-limrun/src/ios.ts b/packages/provider-limrun/src/ios.ts index 0c8a425ffc..9c38d6ca4d 100644 --- a/packages/provider-limrun/src/ios.ts +++ b/packages/provider-limrun/src/ios.ts @@ -413,24 +413,27 @@ class LimrunIosInteractor implements Interactor { * that separates a system alert from the app's own dialog. Loaded on the alert path like the * other seam modules to keep this provider's declared import-time closure budget. */ - async readAlert(): Promise> { + async readAlert(options?: { signal?: AbortSignal }): Promise> { const { handleLimrunIosAlert } = await import('./alert.ts'); - return await handleLimrunIosAlert(this.session.client, 'get'); + return await handleLimrunIosAlert(this.session.client, 'get', options); } - async awaitAlert(options?: { timeoutMs?: number }): Promise> { + async awaitAlert(options?: { + timeoutMs?: number; + signal?: AbortSignal; + }): Promise> { const { handleLimrunIosAlert } = await import('./alert.ts'); return await handleLimrunIosAlert(this.session.client, 'wait', options); } - async acceptAlert(): Promise> { + async acceptAlert(options?: { signal?: AbortSignal }): Promise> { const { handleLimrunIosAlert } = await import('./alert.ts'); - return await handleLimrunIosAlert(this.session.client, 'accept'); + return await handleLimrunIosAlert(this.session.client, 'accept', options); } - async dismissAlert(): Promise> { + async dismissAlert(options?: { signal?: AbortSignal }): Promise> { const { handleLimrunIosAlert } = await import('./alert.ts'); - return await handleLimrunIosAlert(this.session.client, 'dismiss'); + return await handleLimrunIosAlert(this.session.client, 'dismiss', options); } } diff --git a/src/core/interactors/android.ts b/src/core/interactors/android.ts index e61b5f94af..f9d170c72c 100644 --- a/src/core/interactors/android.ts +++ b/src/core/interactors/android.ts @@ -55,8 +55,12 @@ export function createAndroidInteractor( const helperSessionScope = androidHelperSessionScope(runnerContext?.appBundleId); const alertOptions = (timeoutMs?: number) => ({ ...(timeoutMs === undefined ? {} : { timeoutMs }), - captureNodes: async () => { - const capture = await snapshotAndroid(device, { includeHiddenContentHints: false }); + ...(runnerContext?.signal ? { signal: runnerContext.signal } : {}), + captureNodes: async (seam: { signal: AbortSignal }) => { + const capture = await snapshotAndroid(device, { + includeHiddenContentHints: false, + signal: seam.signal, + }); return buildSnapshotState(androidSnapshotPublicationInput(capture), undefined).nodes; }, }); diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index e0014faa7f..b64af5b740 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -398,6 +398,7 @@ agent-device alert dismiss - Branch on `readableCaptures`, `waitedMs`, `matches`, and `firstMatch` instead of parsing error text. `firstMatch` carries identity and text only; absence failures make no visibility or rect claim. - Polling wait timeouts, and a replayed wait refused with `wait_landmark_identity_mismatch`, also carry `captures`, `readableCaptures`, and `polls` so you can see where the budget went. Each poll has `startedMs`, `durationMs`, and an `outcome`: `readable`, `unreadable`, `retriable`, `deadline`, `runner-restart`, or `readiness`. Long waits keep the first 5 and last 25 polls. A wait that never saw a readable capture reports the error its polls hit, with that error's code and retry details, instead of a generic timeout; `wait stable` and a never-readable strict absence keep their own diagnostics. `logPath` links the full request log. - `alert` (same as `alert get`) inspects or handles system alerts on iOS simulator, macOS desktop, and Android permission dialogs. On iOS it also covers app-owned modal popups with native blocking markers. +- On the snapshot-derived backends (Android, Limrun iOS) `alert get --json` answers with a typed `status`: `visible`, `none`, or `undecided` (an alert-shaped capture that resolves to no single alert). Branch on `status`, not on the message. An action whose budget closed before it could press fails with `details.reason: alert-window-expired` and `dispatched: no`; a cutoff while pressing reports `dispatched: unknown`. - Use `alert get` for a quick check, and `alert wait ` only when a prompt may appear after async work. Don't use `settings permission` to answer a dialog already on screen; it is for setup before a flow. - `accept` and `dismiss` are sent once on every platform and never replayed, even if the response is lost. On iOS the runner activates the button once, then only observes until the alert disappears, changes, or the deadline expires. A changed alert does not prove a permission was granted; verify the app outcome. - On the Apple XCUITest runner, `dismiss` activates only a button its decline list recognizes (`Cancel`, `Don't Allow`, `Ask App Not to Track`, and similar English labels), or the close button of a popover whose native marker reads "dismiss popup". It never taps anything else: unknown, reworded, or fully localized decline copy fails with `error.code: ALERT_BUTTON_NOT_FOUND` and `details.dispatched: no`, tapping nothing, rather than guessing at an affirmative ([#3627](https://github.com/callstack/agent-device/issues/3627)). A button choice that finds no button to name answers with that code and `dispatched: no` for `accept` too. To answer a prompt with unrecognized wording, read the labels with `alert get`, then press the button by label through `snapshot` + `press @ref`. The macOS helper's `dismiss` keeps its own rules (the app's cancel-button attribute, its label list, and a last-button fallback). From 0aadaab354541b026c5590e3001f22c635c4a32b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 16:00:41 +0200 Subject: [PATCH 08/12] fix(alerts): close the window holes the review found and cap the budget MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shared tree-alert driver consumed an answer from a poll that started after its window closed (the loop tests settled before remaining, and an expired-at-start probe rode unguarded), and its first probe paid the Android helper cold start inside the action budget, aborting the very read that leg was about to press on — the executed smoke failure. The first observation is now read-first, unarmed, consumed once, and arms the window behind it; a later probe starting past the deadline never captures at all, and the press rides the armed window. - contracts: a mix of ranked and unranked role candidates refuses instead of letting the one known label win (selectRankedAlertButton- ByRole); MAX_ALERT_WINDOW_MS names the timer ceiling the daemon refuses above (INVALID_ARGS) and the driver defaults above. - android: same-label dedup keys on the physical center, so nested nodes collapse and two distinct OKs reach the shared refusal; the input dispatch option type owns cancellation only, and the tap forward picks fields so allowFailure cannot reach the transport. - docs: alert-window-expired/none/undecided reasons stated exactly per phase, Limrun's non-abortable transport described as post-capture and pre-press guards with the hung-request limit named. Prose at the driver trims to invariants. --- CHANGELOG.md | 2 +- packages/contracts/src/alert-contract.test.ts | 17 ++ packages/contracts/src/alert-contract.ts | 28 ++- .../handlers/__tests__/snapshot-alert.test.ts | 31 +++ .../src/execution/handlers/snapshot-alert.ts | 22 ++- .../src/__tests__/alert-detection.test.ts | 32 ++++ .../src/__tests__/alert.test.ts | 56 ++++++ .../src/__tests__/input-actions.test.ts | 37 ++++ .../platform-android/src/alert-detection.ts | 11 +- packages/platform-android/src/alert.ts | 178 ++++++++++-------- .../platform-android/src/input-actions.ts | 30 ++- packages/provider-limrun/src/alert.test.ts | 65 ++++++- packages/provider-limrun/src/alert.ts | 5 +- website/docs/docs/commands.md | 6 +- 14 files changed, 414 insertions(+), 106 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06ba5d0e05..adb1e76746 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ - On iOS, a `fill` that runs under a degraded XCTest snapshot channel now aims at a point, including ref, selector and replay fills. When the runner identifies no text input under that point, it refuses with `NO_TEXT_INPUT_AT_POINT` (`error.details.reason: no_text_input_at_point`, `dispatched: no`) before dispatching its focus tap, and no field changes; when the input lookup itself cannot answer, it refuses with `TEXT_INPUT_PROBE_UNAVAILABLE` (`error.details.reason: text_input_probe_unavailable`). Previously such a fill cleared and typed through the app-level first responder, so it could rewrite a field the caller never named, or report a clear failure after emptying it. A ref or selector fill whose rect centre misses the real field (an outer container, a non-input node) now needs a point inside the input's own rect. - Opening a macOS app by bundle identifier now waits briefly when the app is already running but not yet registered with LaunchServices — the case when a supervisor or dev script started it directly — and if registration arrives within that bounded window the open attaches to the running copy, so the session drives the app the caller started, with its arguments. A copy still unlisted when the window closes gets the plain `open` as before, which can still duplicate; the wait only bounds the race, it does not close it. Launching an app that is not running costs one extra LaunchServices lookup and one process-table read, plus one bundle-identifier read when app-shaped processes exist. - Limrun iOS sessions now serve `alert get/wait/accept/dismiss` instead of refusing with `unsupported-provider-mode`. System alerts are read from the same element-tree capture snapshots ride — a blank-label, process-exclusive application root hosting a text-and-buttons sheet — and accepted or dismissed by tapping the button whose English label states the asked-for action; a capture that is alert-shaped but unresolvable (degraded buttons, competing sheets, an unverifiable owner) is reported as undecided and never pressed or counted as dismissed. The one shape no capture fact separates is an app dialog under a blanked root shared by no other process; the label-only acting legs answer it with a refusal, never a guess. -- The tree-based alert backends (Android and Limrun iOS) now share one alert driver and one English button-label vocabulary declared as a per-role preference table: `alert accept/dismiss` classify the same label the same way on both platforms, so Android now recognizes iOS variant grants (`Allow Once`, `Allow All the Time`, `Allow While Using App`, `Allow Notifications`) and `Ask App Not to Track` as acting choices, and a sheet offering two grants (Android 11+'s "While using the app"/"Only this time", iOS's "Allow Once"/"Allow While Using App") resolves by the declared preference order rather than tree order or refusal. A choice refuses only when the capture names no single physical button for the action — the same label at two points — and each backend's own policy answers then (Android's neutral-button/Back fallback, Limrun's typed refusal). `alert not found` and the no-matching-button refusal now carry the same typed `details.reason` on both platforms. `alert get` reports its outcome as a typed `status` (`visible`/`none`/`undecided`) so clients switch on states instead of parsing the message, and every alert budget now bounds the capture and press inside it: Android hands the leg's cancellation to the adb executor (which kills the child), and a capture that lands after its window closed — the case Limrun's non-abortable transport can produce — is refused rather than consumed or pressed. +- The tree-based alert backends (Android and Limrun iOS) now share one alert driver and one English button-label vocabulary declared as a per-role preference table: `alert accept/dismiss` classify the same label the same way on both platforms, so Android now recognizes iOS variant grants (`Allow Once`, `Allow All the Time`, `Allow While Using App`, `Allow Notifications`) and `Ask App Not to Track` as acting choices, and a sheet offering two grants (Android 11+'s "While using the app"/"Only this time", iOS's "Allow Once"/"Allow While Using App") resolves by the declared preference order rather than tree order or refusal. A choice refuses only when the capture names no single physical button for the action — the same label at two points — and each backend's own policy answers then (Android's neutral-button/Back fallback, Limrun's typed refusal). `alert not found` and the no-matching-button refusal now carry the same typed `details.reason` on both platforms. `alert get` reports its outcome as a typed `status` (`visible`/`none`/`undecided`) so clients switch on states instead of parsing the message. Every alert budget now bounds the capture and press inside it, with per-transport honesty: Android hands the leg's cancellation to the adb executor, which kills the child in flight; Limrun's client exposes no abort, so its guards answer post-capture and pre-press — a late answer is refused and nothing is pressed, but an already-sent request runs on. An `alert wait` budget above 2147483647 ms is refused as `INVALID_ARGS` rather than misfiring a timer. - Limrun iOS snapshots no longer fail when the instance reports a numeric or boolean accessibility value (the system photo picker's tabs report `1`/`0`): the provider seam normalizes variant scalars to text, the way the Apple tree does. Limrun nodes now also publish `selected: true` for nodes carrying the `Selected` trait, so snapshot output and `selected:` selectors can tell a picked photo from an unpicked one. - `settings` refuses a setting the target does not serve before anything changes, with `UNSUPPORTED_OPERATION`, `settings is not supported on this device`, and the target's `error.details.reason` (such as `unsupported-platform-leaf`). This replaces the macOS `INVALID_ARGS` refusal and the Apple `setting-unsupported-on-leaf` reason, and now also covers settings that used to reach the device first, such as Android `reset-keychain` or Apple `animations`. - `settings` refuses, before touching the device, a state its setting does not accept (`Invalid state: …`), a permission action or mode its target does not take (such as `permission deny accessibility`), and a stray extra argument (such as `clear-app-state `). The CLI and the daemon accept the same state spellings, in any casing or surrounding whitespace. diff --git a/packages/contracts/src/alert-contract.test.ts b/packages/contracts/src/alert-contract.test.ts index a7be5a1994..0538fc7ec3 100644 --- a/packages/contracts/src/alert-contract.test.ts +++ b/packages/contracts/src/alert-contract.test.ts @@ -30,6 +30,23 @@ describe('selectRankedAlertButtonByRole', () => { expect(selectRankedAlertButtonByRole(buttons, 'accept')).toBeNull(); }); + it('refuses a mix of ranked and unranked candidates rather than preferring the known label', () => { + // The unranked one's role came from a platform id classifier, so it is a real grant the + // table cannot order. Letting the one known English label win would press an arbitrary + // grant of an unordered pair. + const buttons: Button[] = [ + { label: 'OK', role: 'accept' }, + { label: 'Unrecognized grant', role: 'accept' }, + ]; + expect(selectRankedAlertButtonByRole(buttons, 'accept')).toBeNull(); + }); + + it('keeps the declared order over mixed English variants of distinct grants', () => { + // The closest positive: both candidates are ranked, so this is a preference, not ambiguity. + const buttons = [button('Allow Once'), button('Allow')]; + expect(selectRankedAlertButtonByRole(buttons, 'accept')?.label).toBe('Allow'); + }); + it('returns null when no button carries the role', () => { expect(selectRankedAlertButtonByRole([button('Settings')], 'accept')).toBeNull(); }); diff --git a/packages/contracts/src/alert-contract.ts b/packages/contracts/src/alert-contract.ts index 2b7fd1c118..9d9835417e 100644 --- a/packages/contracts/src/alert-contract.ts +++ b/packages/contracts/src/alert-contract.ts @@ -2,6 +2,15 @@ export const ALERT_POLL_INTERVAL_MS = 300; export const DEFAULT_ALERT_TIMEOUT_MS = 10_000; export const ALERT_ACTION_RETRY_MS = 2_000; +/** + * The largest alert budget the legs accept. The window signal rides `AbortSignal.timeout`, whose + * Node ceiling is 2^31-1 ms: larger values are rejected outright there, and its underlying timer + * overflows to a 1 ms firing when merely exceeded. The daemon refuses a larger `alert wait` + * budget as `INVALID_ARGS`, and a tree driver reads one as the bounded default — never a + * silently shortened window. + */ +export const MAX_ALERT_WINDOW_MS = 2_147_483_647; + /** * The one alert failure the family retries on: the backend looked and there was no alert *yet*. * Both Apple backends state it in typed form — the XCTest runner as this `ErrorPayload.code` @@ -112,10 +121,10 @@ function alertButtonPreferenceRank(label: string, role: 'accept' | 'dismiss'): n /** * The one choice boundary for a shared choice: pick the action's role candidates by the declared - * preference table. A single candidate wins outright; several resolve when one outscores the - * rest; a tie on the same entry (the same label at two points) or vocabulary-ranked-free - * multiplicity returns null, because the capture then names no single physical button for the - * action. Distinct labels that share a role are not a refusal: the table ranks them. + * preference table. A single candidate wins outright. Several resolve only when every one of + * them carries a vocabulary rank and one rank outscores the rest: a mix of ranked and + * id-classified grants cannot be ordered by the table, so it names no single physical button and + * is refused rather than resolved by tree order or by one side's label happening to be known. */ export function selectRankedAlertButtonByRole< TButton extends Readonly<{ role: AlertButtonRole; label: string }>, @@ -123,11 +132,12 @@ export function selectRankedAlertButtonByRole< const matching = buttons.filter((button) => button.role === role); if (matching.length === 1) return matching[0] ?? null; if (matching.length === 0) return null; - const ranked = matching - .map((button) => ({ button, rank: alertButtonPreferenceRank(button.label, role) })) - .filter((entry): entry is { button: TButton; rank: number } => entry.rank !== null) - .sort((a, b) => a.rank - b.rank); - if (ranked.length === 0) return null; + const ranked = matching.map((button) => ({ + button, + rank: alertButtonPreferenceRank(button.label, role), + })); + if (ranked.some((entry) => entry.rank === null)) return null; + ranked.sort((a, b) => (a.rank ?? 0) - (b.rank ?? 0)); const [first, second] = ranked; if (!first || (second && second.rank === first.rank)) return null; return first.button; diff --git a/packages/daemon/src/execution/handlers/__tests__/snapshot-alert.test.ts b/packages/daemon/src/execution/handlers/__tests__/snapshot-alert.test.ts index f30c9788e7..8cef94a0fb 100644 --- a/packages/daemon/src/execution/handlers/__tests__/snapshot-alert.test.ts +++ b/packages/daemon/src/execution/handlers/__tests__/snapshot-alert.test.ts @@ -140,3 +140,34 @@ for (const action of ['accept', 'dismiss']) { expect(successor?.actions).toEqual([]); }); } + +test('alert wait forwards its timeout budget to the runtime leg', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'ios-sim'; + sessionStore.publish(sessionName, makeSession(sessionName)); + + await handleSnapshotCommands(sessionName, sessionStore, ['wait', '1500']); + + // The ceiling test below proves the refusal; this proves a legal budget crosses the runtime + // seam whole. (The Apple runner leg reads the state as `get` and waits on its own deadline.) + expect(mockRunnerCommand.mock.calls[0]?.[1]).toMatchObject({ + command: 'alert', + timeoutMs: 1_500, + }); +}); + +test('alert wait refuses a budget past the window ceiling instead of arming a misfired timer', async () => { + // Above 2^31-1 ms Node's timer overflows: `AbortSignal.timeout` throws outright and a bare + // setTimeout fires after 1 ms. The refusal must be the typed argument error, never either. + const sessionStore = makeSessionStore(); + const sessionName = 'ios-sim'; + sessionStore.publish(sessionName, makeSession(sessionName)); + + await expect( + handleSnapshotCommands(sessionName, sessionStore, ['wait', '4294967296']), + ).rejects.toThrow(/exceeds the maximum/); + await expect( + handleSnapshotCommands(sessionName, sessionStore, ['wait', '2147483648']), + ).rejects.toThrow(/exceeds the maximum/); + expect(mockRunnerCommand).not.toHaveBeenCalled(); +}); diff --git a/packages/daemon/src/execution/handlers/snapshot-alert.ts b/packages/daemon/src/execution/handlers/snapshot-alert.ts index 3c67413983..0e9e4f87bf 100644 --- a/packages/daemon/src/execution/handlers/snapshot-alert.ts +++ b/packages/daemon/src/execution/handlers/snapshot-alert.ts @@ -3,7 +3,9 @@ import type { AlertRuntimeInput } from '@agent-device/contracts/alert-runtime'; import { type AlertAction, DEFAULT_ALERT_TIMEOUT_MS as DEFAULT_TIMEOUT_MS, + MAX_ALERT_WINDOW_MS, } from '@agent-device/contracts/alert-contract'; +import { AppError } from '@agent-device/kernel/errors'; import type { BoundDeviceRuntime } from '@agent-device/contracts/platform-runtime'; import { alertAcceptUse, @@ -151,7 +153,7 @@ export async function handleAlertCommand( session?.trace?.outPath, ); const data = await bound.execute({ - timeoutMs: parseTimeout(req.positionals?.[1]) ?? DEFAULT_TIMEOUT_MS, + timeoutMs: alertWaitTimeoutMs(action, req.positionals?.[1]), ...alertTarget(session), execution: runtimeExecutionFromContext(context), }); @@ -179,3 +181,21 @@ function normalizeAlertAction(action: string | undefined): AlertAction { if (action === 'accept' || action === 'dismiss' || action === 'wait') return action; return 'get'; } + +/** + * The `alert wait` budget before it reaches any leg. Past the contract's window ceiling the + * budget cannot be armed as a timer at all, so the command refuses the argument instead of + * starting a wait that would misfire; the driver's own default remains the last line for + * budgets that arrive by other routes. + */ +function alertWaitTimeoutMs(action: AlertAction, rawTimeout: string | undefined): number { + const timeoutMs = parseTimeout(rawTimeout) ?? DEFAULT_TIMEOUT_MS; + if (action === 'wait' && timeoutMs > MAX_ALERT_WINDOW_MS) { + throw new AppError( + 'INVALID_ARGS', + `alert wait timeoutMs exceeds the maximum of ${MAX_ALERT_WINDOW_MS} ms`, + { hint: 'Pass a shorter timeout, or run repeated alert waits.' }, + ); + } + return timeoutMs; +} diff --git a/packages/platform-android/src/__tests__/alert-detection.test.ts b/packages/platform-android/src/__tests__/alert-detection.test.ts index 67c0ba6b4a..042bd73716 100644 --- a/packages/platform-android/src/__tests__/alert-detection.test.ts +++ b/packages/platform-android/src/__tests__/alert-detection.test.ts @@ -108,3 +108,35 @@ test('the Android 11+ permission sheet resolves its two grants by the preference ); assert.equal(chooseAndroidAlertButton(candidate?.buttons ?? [], 'dismiss')?.label, 'Deny'); }); + +test('nested nodes over one button center deduplicate to one candidate', () => { + // uiautomator commonly nests the clickable wrapper and the label node at the same center. + // They are one physical button, and collapsing them must not read as ambiguity. + const candidate = findAndroidAlertCandidate([ + node(0, 'android.app.AlertDialog'), + text(1, 'Discard draft?', 'android:id/alertTitle'), + button(2, 'OK', 'android:id/button1', { x: 52, y: 612 }), + node(3, 'android.widget.TextView', { + label: 'OK', + parentIndex: 2, + rect: { x: 52, y: 612, width: 128, height: 52 }, + }), + ]); + + assert.deepEqual(candidate?.alert.buttons, ['OK']); + assert.equal(chooseAndroidAlertButton(candidate?.buttons ?? [], 'accept')?.label, 'OK'); +}); + +test('the same label at two distinct centers keeps two candidates so the choice refuses', () => { + // Two spatially distinct OKs are two physical buttons; the capture names no single one, so + // the shared ranked boundary refuses rather than taking the tree-first candidate. + const candidate = findAndroidAlertCandidate([ + node(0, 'android.app.AlertDialog'), + text(1, 'Which OK?', 'android:id/alertTitle'), + button(2, 'OK', 'android:id/button1', { x: 52, y: 612 }), + button(3, 'OK', 'android:id/button1_copy', { x: 240, y: 612 }), + ]); + + assert.deepEqual(candidate?.alert.buttons, ['OK', 'OK']); + assert.equal(chooseAndroidAlertButton(candidate?.buttons ?? [], 'accept'), null); +}); diff --git a/packages/platform-android/src/__tests__/alert.test.ts b/packages/platform-android/src/__tests__/alert.test.ts index 614005565b..38a9215d6c 100644 --- a/packages/platform-android/src/__tests__/alert.test.ts +++ b/packages/platform-android/src/__tests__/alert.test.ts @@ -2,6 +2,8 @@ import { test, vi } from 'vitest'; import assert from 'node:assert/strict'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { deviceShellArgv, type ShellWord } from '@agent-device/kernel/device-shell'; +import { isRequestCanceledError } from '@agent-device/kernel/errors'; +import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; import { button, node, text } from './alert-fixtures.ts'; const runAndroidAdb = vi.fn(async (_device: DeviceInfo, _args: readonly string[]) => ({ @@ -194,3 +196,57 @@ test('wait with a spent budget still reports an alert that is already visible', assert.equal(result.kind, 'alertWait'); assert.equal(result.alert?.title, 'Automation confirmation'); }); + +test('the read-first capture rides the caller cancellation and dies with it', async () => { + // Android's transport is cancellable, so the invariant here is the actual child-process cut: + // the capture seam receives the leg's live AbortSignal, and aborting the caller cuts the + // in-flight capture before any press exists. + runAndroidAdb.mockClear(); + const caller = new AbortController(); + const seamSignals: AbortSignal[] = []; + let rejectCapture: ((error: Error) => void) | undefined; + const result = handleAndroidAlert(device, 'dismiss', { + signal: caller.signal, + captureNodes: (seam) => + new Promise((_resolve, reject) => { + seamSignals.push(seam.signal); + rejectCapture = reject; + seam.signal.addEventListener('abort', () => reject(new Error('adb killed')), { + once: true, + }); + }), + }); + result.catch(() => {}); + await Promise.resolve(); + caller.abort(); + rejectCapture?.(new Error('adb killed')); + + await assert.rejects(result, (error: unknown) => isRequestCanceledError(error)); + assert.equal(seamSignals.length, 1); + assert.equal(seamSignals[0]?.aborted, true); + assert.equal(runAndroidAdb.mock.calls.length, 0); +}); + +test('a slow first read is not cut off by its own action budget', async () => { + // The Android smoke timeline: the session's first capture pays the snapshot-helper cold + // start (install + session bring-up), landing well past the 2s action budget. That read is + // the leg's read-first observation — consumed whatever it costs — and the press rides the + // window it arms behind it. Without the read-first exemption this is the executed CI + // failure: `alert-window-expired, dispatched: no` with the dialog plainly on screen. + runAndroidAdb.mockClear(); + let reads = 0; + const slowWarmRead = async () => { + reads += 1; + if (reads > 1) return runAndroidAdb.mock.calls.length === 0 ? dialog : []; + // 2.5 driven seconds: past the 2s lookup window the old guarded first probe enforced. + await new Promise((resolve) => setTimeout(resolve, 2_500)); + return dialog; + }; + + const result = await underFakeTimers( + async () => await handleAndroidAlert(device, 'dismiss', { captureNodes: slowWarmRead }), + ); + + assert.equal(result.kind, 'alertHandled'); + assert.equal(runAndroidAdb.mock.calls.length, 1); +}); diff --git a/packages/platform-android/src/__tests__/input-actions.test.ts b/packages/platform-android/src/__tests__/input-actions.test.ts index 35261477c9..1d9494bfb5 100644 --- a/packages/platform-android/src/__tests__/input-actions.test.ts +++ b/packages/platform-android/src/__tests__/input-actions.test.ts @@ -4,12 +4,14 @@ import { backAndroid, homeAndroid, longPressAndroid, + pressAndroid, pressAndroidEnter, pressAndroidTvRemote, probeAndroidDisplayRotation, scrollAndroid, setAndroidOrientation, } from '../input-actions.ts'; +import { AppError } from '@agent-device/kernel/errors'; import { ANDROID_EMULATOR } from './test-utils/device-fixtures.ts'; import { withFakeAdb } from './test-utils/fake-adb.ts'; import { withAndroidAdbProvider } from '../adb-executor.ts'; @@ -371,6 +373,41 @@ test('pressAndroidEnter presses the ENTER keyevent', async () => { ); }); +test('pressAndroid surfaces a failed tap as a typed dispatch error, not a false success', async () => { + // The device route raises the failed `adb shell input tap` as its own AppError; the input + // seam must hand that through with dispatch evidence, never as a tap that happened. + await withFakeAdb( + (args) => + args[1] === 'input' + ? { stdout: '', stderr: 'INJECT_EVENTS security exception', exitCode: 1 } + : undefined, + async ({ device }) => { + await assert.rejects(pressAndroid(device, 100, 200), (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.details?.dispatched, 'unknown'); + assert.equal(error.details?.exitCode, 1); + return true; + }); + }, + ); +}); + +test('pressAndroid dispatches no failure suppression even when one is smuggled past the type', async () => { + // The option type is the interface; an untyped caller cannot widen it back. The tap still + // fails typed: `allowFailure` never reaches the transport. + await withFakeAdb( + (args) => + args[1] === 'input' ? { stdout: '', stderr: 'adb: device offline', exitCode: 1 } : undefined, + async ({ device }) => { + await assert.rejects( + pressAndroid(device, 100, 200, { allowFailure: true } as { signal?: AbortSignal }), + (error: unknown) => error instanceof AppError && error.code === 'COMMAND_FAILED', + ); + }, + ); +}); + // The orientation settle polls at its own interval; the clock is the assertion, not the wait. vi.mock('@agent-device/host-kit/retry', () => ({ sleep: async () => {} })); diff --git a/packages/platform-android/src/alert-detection.ts b/packages/platform-android/src/alert-detection.ts index dcdd6e957a..a649b820a9 100644 --- a/packages/platform-android/src/alert-detection.ts +++ b/packages/platform-android/src/alert-detection.ts @@ -161,15 +161,20 @@ function findSystemDialogNodes(nodes: RawSnapshotNode[]): RawSnapshotNode[] { } function findAlertButtons(nodes: RawSnapshotNode[]): AndroidAlertButton[] { - const seen = new Set(); + const seenPhysicalButtons = new Set(); const buttons: AndroidAlertButton[] = []; for (const node of nodes) { const label = readNodeText(node); if (!label || !node.rect || !isButtonLike(node)) continue; const normalized = label.trim().toLowerCase(); - if (!normalized || seen.has(normalized)) continue; - seen.add(normalized); + if (!normalized) continue; const point = centerOfRect(node.rect); + // Two nodes over one center are the one physical button (uiautomator commonly nests it); + // the same label at two centers is two buttons, and the shared choice boundary refuses — + // collapsing them here would hand the first one back as if the capture had named it. + const physicalButton = `${normalized}|${point.x}|${point.y}`; + if (seenPhysicalButtons.has(physicalButton)) continue; + seenPhysicalButtons.add(physicalButton); buttons.push({ label, x: point.x, diff --git a/packages/platform-android/src/alert.ts b/packages/platform-android/src/alert.ts index b663f8bba9..1535089190 100644 --- a/packages/platform-android/src/alert.ts +++ b/packages/platform-android/src/alert.ts @@ -11,6 +11,7 @@ import { ALERT_WINDOW_EXPIRED_REASON, type AlertStatusKind, DEFAULT_ALERT_TIMEOUT_MS, + MAX_ALERT_WINDOW_MS, } from '@agent-device/contracts/alert-contract'; import { AppError, createRequestCanceledError } from '@agent-device/kernel/errors'; import { withDiagnosticTimer } from '@agent-device/host-kit/diagnostics'; @@ -29,12 +30,12 @@ import { import { backAndroid, pressAndroid } from './input-actions.ts'; /** - * THE tree-based alert driver (PR #3699 review): every backend that reads alerts out of a - * capture tree — the Android ADB legs here and the Limrun iOS legs in `provider-limrun` — - * reaches poll, confirm, same-alert, refusal, and response building through this one module, - * and its button roles through the shared label vocabulary in `contracts/alert-contract`. A - * backend supplies only its own observation (`capture` + `findCandidate`), its platform's - * acting policy (`choose`), and its press seam (`press`). + * THE tree-based alert driver: every backend that reads alerts out of a capture tree — the + * Android ADB legs here and the Limrun iOS legs in `provider-limrun` — reaches poll, confirm, + * same-alert, refusal, and response building through this one module, and its button roles + * through the shared label vocabulary in `contracts/alert-contract`. A backend supplies only + * its own observation (`capture` + `findCandidate`), its platform's acting policy (`choose`), + * and its press seam (`press`). */ /** The point a press seam aims at: a button's center in the backend's own coordinate space. */ @@ -60,13 +61,9 @@ export type TreeAlertCandidate< }>; /** - * The seams one tree-based backend owns. `timeoutMs` bounds `wait`; the acting legs ride the - * contract action budget. Poll sleeps ride the shared `pollUntilSettled` deadline primitive, so - * tests drive the clock (`docs/agents/testing.md`) rather than the driver owning a timer seam. - * `capture` and `press` receive the live cancellation signal for their attempt — the caller's - * request signal joined with the loop's remaining window — so a backend whose transport can be - * canceled stops in-flight work, and every backend refuses to consume a capture or dispatch a - * press once the signal has fired. A transport that cannot cancel still gains both refusals. + * The seams one tree-based backend owns. `capture` and `press` receive the leg's live + * cancellation signal (caller cancellation joined with the leg's window): a transport that can + * cancel stops in-flight work; one that cannot still never has its late answer consumed. */ export type TreeAlertSeam = Readonly<{ signal: AbortSignal }>; @@ -81,12 +78,6 @@ export type TreeAlertDriver< timeoutMs?: number; /** The daemon request's cancellation, when the caller rides one; joined into every seam call. */ signal?: AbortSignal; - /** - * One capture, offered the leg's live signal. A backend whose transport can abort (Android's - * adb executor kills the child) hands it down; a backend whose transport cannot (Limrun's - * WebSocket client) ignores it, and the driver's window guard is what keeps its late captures - * unconsumed. - */ capture: (seam: TreeAlertSeam) => Promise; findCandidate: (capture: TCapture) => TreeAlertObservation; choose: (candidate: TCandidate, action: 'accept' | 'dismiss') => TreeAlertPress | null; @@ -126,24 +117,23 @@ export type TreeAlertResult< message?: string; }; -/** One observation of the backend's own capture, bound by the driver from its two seams. */ -type TreeAlertObserver = () => Promise>; +/** One observation of the backend's own capture. A windowed probe may answer window-closed. */ +type TreeAlertObservationOrClosed = + | TreeAlertObservation + | TreeAlertWindowClosed; + +type TreeAlertObserver = () => Promise>; const ALERT_NOT_FOUND_HINT = 'If a sheet is visible in snapshot but alert reports no alert, it is likely app-owned UI. Use snapshot -i and press the visible label/ref.'; /** - * Runs one alert leg. The deadline order is read-first on every budget: over a positive window - * both predecessor loops took their first immediate observation before any waiting, and one - * immediate observation is the honest answer shape for a zero-length `wait` window too. A - * malformed `timeoutMs` is the bounded default, never a loop whose bound is never reached. - * - * Every budget also bounds the work inside it: each polling leg carries a window signal that - * expires with its deadline and a request signal from the caller, and both reach the seams. - * Captures then stop in-flight transport work wherever the backend's transport can cancel (the - * Android legs hand the signal to the adb executor, which kills the child); backends whose - * transport cannot cancel — Limrun's WebSocket client exposes no abort — still never consume a - * capture that lands after its window closed, and no press is dispatched past the window. + * Runs one alert leg. The first observation of a polling leg is read-first: consumed whatever + * the clock says — the declared answer shape for a zero-length window and for a backend whose + * first capture is slow to start — and it arms the leg's window behind it. Later probes ride + * the window's joined signal; only Android's transport can be cut mid-flight (the adb executor + * kills the child), so on Limrun's non-abortable client the post-capture clock check is the + * guard. Nothing observed through a closed window is consumed or pressed. */ export async function runTreeAlertAction< TCapture, @@ -198,60 +188,94 @@ type ObserverDriver = Readonly<{ }>; /** - * One observation of the backend's capture, bound by the driver's window. `window` is the leg's - * deadline clock plus the joined signal (this window's expiry, the caller's cancellation) the - * seams carry. Android hands that signal to the adb executor, which kills the child process, so - * a capture in flight when the window closes is actually stopped; Limrun's WebSocket client - * exposes no abort, so the post-await check here is what keeps that backend honest — a capture - * that lands once its window has closed proves nothing, is refused on the typed window reason, - * and is never consumed. A probe that starts once the window is already spent is the polling - * loop's own closing observation: it answers normally and the loop ends on its deadline. What - * may never happen is consuming a spent-window observation as actionable, which the press-time - * window check in `actOnTreeAlert` enforces. The first observation of a leg rides only the - * caller's cancellation: the read-first contract promises one immediate honest answer even when - * the window is already spent, and a probe that starts inside the window cannot have outrun a - * deadline it has not reached yet — its own landing decides, not a race with the clock. + * A probe the window already answered: a poll that starts after its budget closed, whose answer — + * alert or absence — would be a late verdict on a question the deadline has closed. The observer + * returns this instead of capturing, and no settled predicate accepts it. */ -type TreeAlertWindow = Readonly<{ deadline: DeadlineClock; signal: AbortSignal }>; +type TreeAlertWindowClosed = Readonly<{ kind: 'window-closed' }>; /** * One leg's window: a deadline clock the driver polls against and the joined signal (this - * window's expiry, the caller's cancellation) the seams carry. The clock and signal start on - * the same line, so an expired clock and an aborted signal agree on everything but the - * millisecond. + * window's expiry, the caller's cancellation) the seams carry. `start()` arms both on the same + * line, so an expired clock and an aborted signal agree on everything but the millisecond. + * A lookup window arms when the leg's first read lands — that read is a read, not a wait, and + * spends none of the budget that bounds re-polls and the press; zero spends it instantly. */ +type TreeAlertWindow = { + readonly deadline: DeadlineClock; + readonly signal: AbortSignal; + isArmed(): boolean; + start(): void; +}; + function treeAlertWindow( timeoutMs: number, callerSignal: AbortSignal | undefined, ): TreeAlertWindow { + let armed: Readonly<{ deadline: DeadlineClock; signal: AbortSignal }> | undefined; return { - deadline: Deadline.fromTimeoutMs(timeoutMs), - signal: AbortSignal.any([ - AbortSignal.timeout(timeoutMs), - ...(callerSignal ? [callerSignal] : []), - ]), + // Stable delegating clock: the polling loop holds this reference across the arming. + deadline: { + remainingMs: () => armed?.deadline.remainingMs() ?? timeoutMs, + elapsedMs: () => armed?.deadline.elapsedMs() ?? 0, + isExpired: () => armed?.deadline.isExpired() ?? false, + }, + get signal() { + return armed?.signal ?? callerSignal ?? neverSignal; + }, + isArmed: () => armed !== undefined, + start() { + armed ??= { + deadline: Deadline.fromTimeoutMs(timeoutMs), + signal: AbortSignal.any([ + AbortSignal.timeout(timeoutMs), + ...(callerSignal ? [callerSignal] : []), + ]), + }; + }, }; } +/** + * The window rules every observation of a leg obeys. The first observation is read-first: it + * rides only the caller's cancellation, is consumed whatever it costs (the declared answer for + * a zero-length window and for a slow first capture, like Android's helper cold start), and + * arms the window behind it. Later probes ride the window signal, and a probe starting after + * the window closed never captures at all. + */ +function treeAlertObserver( + driver: ObserverDriver, + window: undefined, +): () => Promise>; +function treeAlertObserver( + driver: ObserverDriver, + window: TreeAlertWindow, +): TreeAlertObserver; function treeAlertObserver( driver: ObserverDriver, window: TreeAlertWindow | undefined, ): TreeAlertObserver { + let firstObservation = true; return async () => { if (driver.signal?.aborted) throw createRequestCanceledError(); - // A window already spent at probe start is the polling loop's own closing observation: it - // answers normally (the read-first contract keeps one immediate honest answer even for a - // spent window) and the loop ends on its deadline. A probe that starts inside the window is - // guarded: its seam can be canceled, and its landing is checked against the clock. - const guarded = window !== undefined && !window.deadline.isExpired(); + // Read-first is only the first read over an un-armed window. A confirm window is armed at + // the press, so its first capture is already a bounded probe of what follows the tap. + const readFirst = firstObservation && window?.isArmed() !== true; + firstObservation = false; + const bounded = window !== undefined && !readFirst; + if (bounded && window.deadline.isExpired()) return { kind: 'window-closed' }; const capture = await captureAlertState( driver, - guarded ? window.signal : (driver.signal ?? neverSignal), + bounded ? window.signal : (driver.signal ?? neverSignal), ); - if (guarded && window.deadline.isExpired() && !driver.signal?.aborted) { + // A bounded capture that lands once its window has closed proves nothing: refuse it typed, + // never consume it. The read-first observation is exempt — consumed whatever it costs. + if (bounded && window.deadline.isExpired() && !driver.signal?.aborted) { throw alertWindowExpiredError('no'); } - return driver.findCandidate(capture); + const observation = driver.findCandidate(capture); + window?.start(); + return observation; }; } @@ -302,7 +326,10 @@ function isAlertWindowExpired(error: unknown): boolean { const neverSignal = new AbortController().signal; function alertWindowMs(timeoutMs: number | undefined): number { - return typeof timeoutMs === 'number' && Number.isFinite(timeoutMs) && timeoutMs >= 0 + return typeof timeoutMs === 'number' && + Number.isFinite(timeoutMs) && + timeoutMs >= 0 && + timeoutMs <= MAX_ALERT_WINDOW_MS ? timeoutMs : DEFAULT_ALERT_TIMEOUT_MS; } @@ -311,7 +338,8 @@ function alertWindowMs(timeoutMs: number | undefined): number { * Polls for an actionable candidate within the budget, on the same deadline primitive the * daemon's other polling loops ride. The terminal answer distinguishes a window that stayed * quiet (`none`) from one that stayed alert-shaped yet unresolvable (`undecided`): a degraded - * sheet that never resolved is not an absence, and callers must not report the two alike. + * sheet that never resolved is not an absence, and callers must not report the two alike. A + * window-closed sentinel settles nothing: the deadline, not a late capture, ends the loop. */ async function pollTreeAlert( observe: TreeAlertObserver, @@ -329,7 +357,7 @@ async function pollTreeAlert( intervalMs: ALERT_POLL_INTERVAL_MS, signal, }); - if (settled) return value; + if (settled && value.kind === 'alert') return value; return sawUndecided ? { kind: 'undecided' } : { kind: 'none' }; } @@ -424,11 +452,12 @@ async function actOnTreeAlert< } throw error; } - // The confirm window is its own budget: it only starts once the press has been dispatched, - // and it must not consume the lookup's already-spent window. A capture this phase cuts off - // lands after a press, so its failure keeps the stuck-alert phrasing, never "nothing was - // pressed" — the confirm observer maps its window expiry onto that failure below. + // The confirm window is its own budget, armed the moment the press is dispatched: unlike the + // lookup's, it bounds its first capture too, because a post-press read that lands late proves + // nothing about what followed the tap. Its cutoff keeps the stuck-alert phrasing — a press + // already landed — which the confirm loop maps below. const confirmWindow = treeAlertWindow(ALERT_ACTION_RETRY_MS, driver.signal); + confirmWindow.start(); await confirmTreeAlertDismissed( treeAlertObserver(driver, confirmWindow), confirmWindow, @@ -449,13 +478,10 @@ async function actOnTreeAlert< } /** - * `alert accept|dismiss` means the dialog is gone, not that a press was sent: the next command - * reads the app, and a capture that lands while the dialog window is still up sees only the - * dialog. Only two answers confirm that: observed absence (`none`), or a different alert taking - * the first's place. An undecided post-press capture proves nothing about absence, so it is - * retried within the budget and reported as a failure at its end rather than returned as - * handled. Bounded by the same budget the pre-press lookup uses; iOS's runner applies the same - * re-check. + * `alert accept|dismiss` means the dialog is gone, not that a press was sent. Only two answers + * confirm that: observed absence (`none`), or a different alert taking the first's place. An + * undecided post-press capture proves nothing, so it is retried within the budget and fails at + * its end rather than returning handled. Bounded by the budget the pre-press lookup uses. */ async function confirmTreeAlertDismissed< TAlert extends AlertInfo & { buttons: string[] }, diff --git a/packages/platform-android/src/input-actions.ts b/packages/platform-android/src/input-actions.ts index 15d037bbed..b0cebe77e9 100644 --- a/packages/platform-android/src/input-actions.ts +++ b/packages/platform-android/src/input-actions.ts @@ -25,19 +25,29 @@ import { AppError, discloseDispatchAfterSteps } from '@agent-device/kernel/error import type { Rect } from '@agent-device/kernel/snapshot'; import { sleep } from '@agent-device/host-kit/retry'; import { runAndroidShell } from './adb.ts'; -import type { AndroidAdbExecutorOptions } from './adb-transport.ts'; import { androidAdbResultError, discloseAdbInputDispatch } from './adb-failure.ts'; import { executeAndroidTouchPlan, readAndroidGestureViewportReading } from './touch-executor.ts'; import type { AndroidHelperSessionOptions } from './snapshot-helper-types.ts'; +/** + * What a caller may steer on a dispatched input command: cancellation only. The wider executor + * options stay out on purpose — `allowFailure` would let a dispatch claim success over a failed + * adb command, which no input dispatch may do. + */ +export type AndroidInputDispatchOptions = Readonly<{ signal?: AbortSignal }>; + export async function pressAndroid( device: DeviceInfo, x: number, y: number, - executorOptions?: AndroidAdbExecutorOptions, + options?: AndroidInputDispatchOptions, ): Promise { try { - await runAndroidShell(device, ['input', 'tap', x, y], executorOptions); + // Picked field by field so no runtime caller can smuggle `allowFailure: true` past the type. + await runAndroidShell(device, ['input', 'tap', x, y], { + allowFailure: false, + ...(options?.signal ? { signal: options.signal } : {}), + }); } catch (error) { throw discloseAdbInputDispatch(error); } @@ -48,11 +58,11 @@ export async function doubleTapAndroid( device: DeviceInfo, x: number, y: number, - executorOptions?: AndroidAdbExecutorOptions, + options?: AndroidInputDispatchOptions, ): Promise { - await pressAndroid(device, x, y, executorOptions); + await pressAndroid(device, x, y, options); try { - await pressAndroid(device, x, y, executorOptions); + await pressAndroid(device, x, y, options); } catch (error) { throw discloseDispatchAfterSteps(error, 1); } @@ -70,9 +80,13 @@ export async function pressAndroidTvRemote( export async function backAndroid( device: DeviceInfo, - executorOptions?: AndroidAdbExecutorOptions, + options?: AndroidInputDispatchOptions, ): Promise { - await runAndroidShell(device, ['input', 'keyevent', '4'], executorOptions); + await runAndroidShell( + device, + ['input', 'keyevent', '4'], + options?.signal ? { signal: options.signal } : undefined, + ); } export async function homeAndroid(device: DeviceInfo): Promise { diff --git a/packages/provider-limrun/src/alert.test.ts b/packages/provider-limrun/src/alert.test.ts index 1a98d5801b..b53319a584 100644 --- a/packages/provider-limrun/src/alert.test.ts +++ b/packages/provider-limrun/src/alert.test.ts @@ -52,8 +52,10 @@ describe('wait budget', () => { await expect( underFakeTimers(() => handleLimrunIosAlert(client, 'wait', { timeoutMs: 5_000 })), ).rejects.toThrow('alert wait timed out'); - // One poll per interval over the whole window: never fewer (gave up early) nor real seconds. - expect(client.elementTree).toHaveBeenCalledTimes(1 + Math.ceil(5_000 / 300)); + // One poll per interval over the whole window: never fewer (gave up early) nor real + // seconds. The probe the sleep lands exactly on the deadline captures nothing — a closed + // window answers for itself — so the count is the intervals, not intervals-plus-one. + expect(client.elementTree).toHaveBeenCalledTimes(Math.ceil(5_000 / 300)); }); it('treats a malformed window as the bounded default rather than an unbound loop', async () => { @@ -64,7 +66,64 @@ describe('wait budget', () => { ).rejects.toThrow('alert wait timed out'); // The fallback is the contract default, spent on the driven clock — this returned, so the // loop had a bound, which a NaN comparison would not have given it. - expect(client.elementTree).toHaveBeenCalledTimes(1 + Math.ceil(10_000 / 300)); + expect(client.elementTree).toHaveBeenCalledTimes(Math.ceil(10_000 / 300)); + }); + + it('reads a finite budget past the timer ceiling as the bounded default, never a misfire', async () => { + // 2^32 reaches `AbortSignal.timeout` as a RangeError and 2^31 as a 1 ms timer; both must + // arrive here as the contract default, spent on the driven clock. + for (const oversized of [2 ** 31, 2 ** 32]) { + const client = clientWithTrees([limrunAppScreenTree()]); + await expect( + underFakeTimers(() => handleLimrunIosAlert(client, 'wait', { timeoutMs: oversized })), + ).rejects.toThrow('alert wait timed out'); + expect(client.elementTree).toHaveBeenCalledTimes(Math.ceil(10_000 / 300)); + } + }); + + it('never consumes a second capture that starts after the window closed', async () => { + // The closing-probe bug: a poll the sleep lands past the deadline must not even capture — + // its answer, alert or absence, is a late verdict the deadline already closed. + const client = { + elementTree: vi + .fn() + .mockResolvedValueOnce([limrunAppScreenTree()]) + // Were this answer ever consumed, `wait` would report the alert instead of timing out. + .mockResolvedValueOnce([limrunAttAlertTree()]), + tap: vi.fn(async () => {}), + } satisfies LimrunIosAlertClient; + + await expect( + underFakeTimers(() => handleLimrunIosAlert(client, 'wait', { timeoutMs: 300 })), + ).rejects.toThrow('alert wait timed out'); + expect(client.elementTree).toHaveBeenCalledTimes(1); + }); + + it('never presses on a candidate that resolves after the action window closed', async () => { + // A bounded capture stalling past its 2s lookup budget lands after the deadline. Consuming + // it would press an alert the window never resolved inside itself. + const client = { + elementTree: vi + .fn() + .mockResolvedValueOnce([limrunAppScreenTree()]) + .mockImplementationOnce( + async () => + new Promise((resolve) => { + setTimeout(() => resolve([limrunAttAlertTree()]), 3_000); + }), + ), + tap: vi.fn(async () => {}), + } satisfies LimrunIosAlertClient; + + const thrown = await underFakeTimers(() => handleLimrunIosAlert(client, 'accept')).then( + () => undefined, + (error: unknown) => error, + ); + expect(client.tap).not.toHaveBeenCalled(); + expect(thrown).toBeInstanceOf(AppError); + const failure = thrown as AppError; + expect(failure.details?.reason).toBe('alert-window-expired'); + expect(failure.details?.dispatched).toBe('no'); }); it('surfaces a failed capture as itself, never as a timeout', async () => { diff --git a/packages/provider-limrun/src/alert.ts b/packages/provider-limrun/src/alert.ts index 6910b9e695..ef0e2a6de7 100644 --- a/packages/provider-limrun/src/alert.ts +++ b/packages/provider-limrun/src/alert.ts @@ -54,9 +54,8 @@ export async function handleLimrunIosAlert( timeoutMs: options?.timeoutMs, ...(options?.signal ? { signal: options.signal } : {}), // The Limrun WebSocket client exposes no abort on `elementTree`/`tap`, so the seam signal - // cannot cancel its transport. The driver still enforces the window around this seam: a - // capture that lands after its window closed is refused rather than consumed, and no press - // dispatches once the window has closed. + // cannot cancel its transport; the shared driver's window guards are what keep this + // backend's late answers unconsumed and its presses inside the window. capture: () => client.elementTree(), findCandidate: (tree): TreeAlertObservation => readLimrunAlertCapture(tree as Parameters[0]), diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index b64af5b740..575e714566 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -397,8 +397,10 @@ agent-device alert dismiss - `wait_stable_timeout`: the UI did not settle. - Branch on `readableCaptures`, `waitedMs`, `matches`, and `firstMatch` instead of parsing error text. `firstMatch` carries identity and text only; absence failures make no visibility or rect claim. - Polling wait timeouts, and a replayed wait refused with `wait_landmark_identity_mismatch`, also carry `captures`, `readableCaptures`, and `polls` so you can see where the budget went. Each poll has `startedMs`, `durationMs`, and an `outcome`: `readable`, `unreadable`, `retriable`, `deadline`, `runner-restart`, or `readiness`. Long waits keep the first 5 and last 25 polls. A wait that never saw a readable capture reports the error its polls hit, with that error's code and retry details, instead of a generic timeout; `wait stable` and a never-readable strict absence keep their own diagnostics. `logPath` links the full request log. -- `alert` (same as `alert get`) inspects or handles system alerts on iOS simulator, macOS desktop, and Android permission dialogs. On iOS it also covers app-owned modal popups with native blocking markers. -- On the snapshot-derived backends (Android, Limrun iOS) `alert get --json` answers with a typed `status`: `visible`, `none`, or `undecided` (an alert-shaped capture that resolves to no single alert). Branch on `status`, not on the message. An action whose budget closed before it could press fails with `details.reason: alert-window-expired` and `dispatched: no`; a cutoff while pressing reports `dispatched: unknown`. +- `alert` (same as `alert get`) inspects or handles system alerts on iOS simulator, macOS desktop, and Android permission dialogs. On iOS it also covers app-owned modal popups with native blocking markers. `alert wait ` takes a budget up to 2147483647 ms (~24.8 days); larger is refused as `INVALID_ARGS` before anything runs. +- On the snapshot-derived backends (Android, Limrun iOS) `alert get --json` answers with a typed `status`: `visible`, `none`, or `undecided` (an alert-shaped capture that resolves to no single alert). Branch on `status`, not on the message. +- On those backends, when `alert accept`/`dismiss` fails, `error.details.reason` names why. Before any press, `dispatched: no` and the reason is `alert-not-found` (the action budget passed with no alert seen), `alert-window-expired` (the budget closed before a capture or the press could resolve), `alert_capture_undecided`, or `alert_button_undecided` (the sheet is too ambiguous to act on). After a press, `alert-window-expired` with "did not dismiss" wording means the tap went out and the disappearance could not be confirmed; `dispatched: unknown` means the press itself was cut off and may have landed. Run `alert get` before acting again. +- On Android, an alert capture that outlives its budget is stopped in flight. On Limrun the transport cannot cancel a request: the command refuses a late answer and presses nothing, but a hung capture can keep running after the command has already failed. If Limrun alert commands stall, restart the instance session. - Use `alert get` for a quick check, and `alert wait ` only when a prompt may appear after async work. Don't use `settings permission` to answer a dialog already on screen; it is for setup before a flow. - `accept` and `dismiss` are sent once on every platform and never replayed, even if the response is lost. On iOS the runner activates the button once, then only observes until the alert disappears, changes, or the deadline expires. A changed alert does not prove a permission was granted; verify the app outcome. - On the Apple XCUITest runner, `dismiss` activates only a button its decline list recognizes (`Cancel`, `Don't Allow`, `Ask App Not to Track`, and similar English labels), or the close button of a popover whose native marker reads "dismiss popup". It never taps anything else: unknown, reworded, or fully localized decline copy fails with `error.code: ALERT_BUTTON_NOT_FOUND` and `details.dispatched: no`, tapping nothing, rather than guessing at an affirmative ([#3627](https://github.com/callstack/agent-device/issues/3627)). A button choice that finds no button to name answers with that code and `dispatched: no` for `accept` too. To answer a prompt with unrecognized wording, read the labels with `alert get`, then press the button by label through `snapshot` + `press @ref`. The macOS helper's `dismiss` keeps its own rules (the app's cancel-button attribute, its label list, and a last-button fallback). From 2313a1a7518cc7d381043664c583d4b53ccb1d99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 16:51:31 +0200 Subject: [PATCH 09/12] fix(alerts): scope alert captures to the warm helper and split the unconfirmed press MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The executed Android smoke failure at 5b1e1c1d9: the session's alert probes ran without the app-backed helper session scope, so each probe cold-started the snapshot helper inside the window that bounds it — the request log shows the post-press confirm capture aborted at 2.3s ("operation was aborted due to timeout") while the canary proves the dialog was in fact dismissed. Alert reads now carry the same helperSessionScope as every other app-backed read. The failure also proved the wording lied: with no confirm capture ever answering, "did not dismiss the visible alert"/"dialog is still visible" accused a dialog that was already gone. Only a re-read naming the same alert earns the still-visible failure; an unconfirmed press now says exactly that, on both the cut-off and the spent-budget exits, with a hint to run alert get before retrying. The same-alert stuck-alert failure keeps its message. Pinned: interactor scoping regression (fail-before against 5b1e1c1d9) and the three Limrun confirm-phase tests moved to the truthful split; the Android still-visible pin keeps its message via the same-alert path. --- packages/platform-android/src/alert.ts | 58 +++++++++++++++++----- packages/provider-limrun/src/alert.test.ts | 32 ++++++------ src/core/interactors/android.test.ts | 16 ++++++ src/core/interactors/android.ts | 4 ++ website/docs/docs/commands.md | 2 +- 5 files changed, 84 insertions(+), 28 deletions(-) diff --git a/packages/platform-android/src/alert.ts b/packages/platform-android/src/alert.ts index 1535089190..739b4c489f 100644 --- a/packages/platform-android/src/alert.ts +++ b/packages/platform-android/src/alert.ts @@ -493,10 +493,21 @@ async function confirmTreeAlertDismissed< action: 'accept' | 'dismiss', signal?: AbortSignal, ): Promise { + // Only an observation that named this same alert on screen proves the "still visible" claim. + // Captures that were cut off or stayed unresolvable prove nothing, and the failure wording + // must not accuse a dialog that may already be gone of still standing. + let sameAlertObserved = false; + const probe: TreeAlertObserver = async () => { + const observation = await observe(); + if (observation.kind === 'alert' && sameTreeAlert(observation.candidate.alert, dismissed)) { + sameAlertObserved = true; + } + return observation; + }; let settled: boolean; try { ({ settled } = await pollUntilSettled({ - probe: observe, + probe, settled: (observation) => observation.kind === 'none' || (observation.kind === 'alert' && !sameTreeAlert(observation.candidate.alert, dismissed)), @@ -505,23 +516,44 @@ async function confirmTreeAlertDismissed< signal, })); } catch (error) { - // A confirmation capture this phase cuts off says the same thing a budget spent without - // confirmation says — and it says it after a press landed, so the stuck-alert phrasing is - // the honest one here; the typed reason keeps the two causes distinguishable. + // A confirmation capture this phase cuts off says what the press cannot say alone: whether + // the dialog is still there, or whether the check itself never answered. Only the first is + // the stuck-alert failure; the second must not claim a visible dialog it never observed. if (isAlertWindowExpired(error)) { - throw new AppError('COMMAND_FAILED', `alert ${action} did not dismiss the visible alert`, { - alert: dismissed, - reason: ALERT_WINDOW_EXPIRED_REASON, - hint: 'The alert button was pressed but the dialog is still visible. Inspect alert get --json, then press the visible button by label/ref or retry.', - }); + if (sameAlertObserved) { + throw new AppError('COMMAND_FAILED', `alert ${action} did not dismiss the visible alert`, { + alert: dismissed, + reason: ALERT_WINDOW_EXPIRED_REASON, + hint: 'The alert button was pressed but the dialog is still visible. Inspect alert get --json, then press the visible button by label/ref or retry.', + }); + } + throw new AppError( + 'COMMAND_FAILED', + `alert ${action} pressed the button but the dismissal could not be confirmed`, + { + alert: dismissed, + reason: ALERT_WINDOW_EXPIRED_REASON, + hint: 'The alert button was pressed; the post-press read never answered within the budget, so the dialog may already be gone. Run alert get --json before retrying.', + }, + ); } throw error; } if (settled) return; - throw new AppError('COMMAND_FAILED', `alert ${action} did not dismiss the visible alert`, { - alert: dismissed, - hint: 'The alert button was pressed but the dialog is still visible. Inspect alert get --json, then press the visible button by label/ref or retry.', - }); + if (sameAlertObserved) { + throw new AppError('COMMAND_FAILED', `alert ${action} did not dismiss the visible alert`, { + alert: dismissed, + hint: 'The alert button was pressed but the dialog is still visible. Inspect alert get --json, then press the visible button by label/ref or retry.', + }); + } + throw new AppError( + 'COMMAND_FAILED', + `alert ${action} pressed the button but the dismissal could not be confirmed`, + { + alert: dismissed, + hint: 'The alert button was pressed; no post-press read resolved to an answer within the budget, so the dialog may already be gone. Run alert get --json before retrying.', + }, + ); } /** Alert identity across a dismissal re-check: the same title and the same button row, in order. */ diff --git a/packages/provider-limrun/src/alert.test.ts b/packages/provider-limrun/src/alert.test.ts index b53319a584..7981853074 100644 --- a/packages/provider-limrun/src/alert.test.ts +++ b/packages/provider-limrun/src/alert.test.ts @@ -137,11 +137,11 @@ describe('wait budget', () => { await expect(handleLimrunIosAlert(client, 'wait')).rejects.toThrow('websocket closed'); }); - it('refuses a confirm capture that outruns its window and reports the stuck alert', async () => { + it('refuses a confirm capture that outruns its window without claiming a visible dialog', async () => { // The Limrun WebSocket client cannot abort an in-flight `elementTree`, so a stalled capture - // can outlive the confirm budget. The driver must not consume that late answer: the press - // already landed, the capture proves nothing, and the honest terminal answer is the - // stuck-alert failure on the typed window reason. + // can outlive the confirm budget. The driver must not consume that late answer — and because + // no post-press capture ever answered, the failure may not accuse the dialog of still being + // visible; it reports the press as unconfirmed. const client = { elementTree: vi .fn() @@ -164,7 +164,9 @@ describe('wait budget', () => { expect(client.tap).toHaveBeenCalledTimes(1); expect(thrown).toBeInstanceOf(AppError); const failure = thrown as AppError; - expect(failure.message).toBe('alert accept did not dismiss the visible alert'); + expect(failure.message).toBe( + 'alert accept pressed the button but the dismissal could not be confirmed', + ); expect(failure.details?.reason).toBe('alert-window-expired'); }); }); @@ -209,28 +211,30 @@ describe('acting budgets', () => { expect(client.tap).not.toHaveBeenCalled(); }); - it('reports a stuck alert when the post-tap capture stays unresolvable to the end', async () => { - // The first probe resolves the single alert; every post-tap probe carries two disagreeing + it('reports an unconfirmed dismissal when the post-tap capture stays unresolvable to the end', async () => { + // The first probe resolves the single alert; every post-press probe carries two disagreeing // alert-shaped sheets. An unresolvable capture proves absence for nothing, so the confirm - // loop keeps its budget and fails rather than reporting the alert handled. + // loop keeps its budget and fails — and, having never named this alert on screen again, it + // reports the press as unconfirmed rather than claiming a dialog it never re-observed. const client = clientWithTrees([ limrunAttAlertTree(), [limrunAttAlertTree(), limrunAttAlertTree()], ]); await expect(underFakeTimers(() => handleLimrunIosAlert(client, 'dismiss'))).rejects.toThrow( - 'alert dismiss did not dismiss the visible alert', + 'alert dismiss pressed the button but the dismissal could not be confirmed', ); }); - it('reports a stuck alert when the same root survives the tap with degraded buttons', async () => { - // The tap was ignored: the post-tap capture is the same alert root with its choices dropped - // to disabled, explicitly unhittable, and frameless. Degraded-but-present is not absence; - // the leg must fail, not report handled. + it('reports an unconfirmed dismissal when the same root survives with degraded buttons', async () => { + // The tap was ignored: the post-press capture is the same alert root with its choices dropped + // to disabled, explicitly unhittable, and frameless — an unresolvable shape, not a re-named + // alert. Degraded-but-present is not absence; the leg must fail, not report handled, and it + // may not claim "still visible" beyond what the undecided capture proved. const client = clientWithTrees([limrunAttAlertTree(), limrunAlertTreeWithUnpressableButtons()]); await expect(underFakeTimers(() => handleLimrunIosAlert(client, 'accept'))).rejects.toThrow( - 'alert accept did not dismiss the visible alert', + 'alert accept pressed the button but the dismissal could not be confirmed', ); expect(client.tap).toHaveBeenCalledTimes(1); }); diff --git a/src/core/interactors/android.test.ts b/src/core/interactors/android.test.ts index b403e84c29..21bb580ab6 100644 --- a/src/core/interactors/android.test.ts +++ b/src/core/interactors/android.test.ts @@ -98,6 +98,22 @@ test('a device-only session releases the helper after fill and scroll', async () }); }); +test('alert reads ride the session-scope helper, not a per-probe cold start', async () => { + // The executed Android smoke failure: an alert probe without the session scope cold-started + // the snapshot helper inside the window that bounds the probe, so the post-press confirm read + // was aborted before the device answered. Alert captures scope like every other read. + snapshotAndroidMock.mockResolvedValue(makeAndroidSnapshotCapture([])); + const interactor = createAndroidInteractor(device, undefined, { + appBundleId: 'com.example.app', + }); + + if (!interactor.readAlert) throw new Error('Android interactor must expose readAlert'); + await interactor.readAlert(); + + const options = snapshotAndroidMock.mock.lastCall?.[1]; + expect(options?.helperSessionScope).toBe('daemon-session'); +}); + test('a transient snapshot reaches the Android capture with its settle deadline', async () => { snapshotAndroidMock.mockResolvedValue(makeAndroidSnapshotCapture([])); diff --git a/src/core/interactors/android.ts b/src/core/interactors/android.ts index f9d170c72c..4d0ea7e449 100644 --- a/src/core/interactors/android.ts +++ b/src/core/interactors/android.ts @@ -60,6 +60,10 @@ export function createAndroidInteractor( const capture = await snapshotAndroid(device, { includeHiddenContentHints: false, signal: seam.signal, + // Alert probes must share the session's warm helper like every other app-backed read: + // a command-scope alert probe cold-starts the helper past the window that bounds it, + // so every re-poll and the post-press confirm would abort before the device answers. + helperSessionScope, }); return buildSnapshotState(androidSnapshotPublicationInput(capture), undefined).nodes; }, diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 575e714566..11540f829d 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -399,7 +399,7 @@ agent-device alert dismiss - Polling wait timeouts, and a replayed wait refused with `wait_landmark_identity_mismatch`, also carry `captures`, `readableCaptures`, and `polls` so you can see where the budget went. Each poll has `startedMs`, `durationMs`, and an `outcome`: `readable`, `unreadable`, `retriable`, `deadline`, `runner-restart`, or `readiness`. Long waits keep the first 5 and last 25 polls. A wait that never saw a readable capture reports the error its polls hit, with that error's code and retry details, instead of a generic timeout; `wait stable` and a never-readable strict absence keep their own diagnostics. `logPath` links the full request log. - `alert` (same as `alert get`) inspects or handles system alerts on iOS simulator, macOS desktop, and Android permission dialogs. On iOS it also covers app-owned modal popups with native blocking markers. `alert wait ` takes a budget up to 2147483647 ms (~24.8 days); larger is refused as `INVALID_ARGS` before anything runs. - On the snapshot-derived backends (Android, Limrun iOS) `alert get --json` answers with a typed `status`: `visible`, `none`, or `undecided` (an alert-shaped capture that resolves to no single alert). Branch on `status`, not on the message. -- On those backends, when `alert accept`/`dismiss` fails, `error.details.reason` names why. Before any press, `dispatched: no` and the reason is `alert-not-found` (the action budget passed with no alert seen), `alert-window-expired` (the budget closed before a capture or the press could resolve), `alert_capture_undecided`, or `alert_button_undecided` (the sheet is too ambiguous to act on). After a press, `alert-window-expired` with "did not dismiss" wording means the tap went out and the disappearance could not be confirmed; `dispatched: unknown` means the press itself was cut off and may have landed. Run `alert get` before acting again. +- On those backends, when `alert accept`/`dismiss` fails, `error.details.reason` names why. Before any press, `dispatched: no` and the reason is `alert-not-found` (the action budget passed with no alert seen), `alert-window-expired` (the budget closed before a capture or the press could resolve), `alert_capture_undecided`, or `alert_button_undecided` (the sheet is too ambiguous to act on). After a press, “did not dismiss the visible alert” means a re-read named the same dialog again; “could not be confirmed” means the press went out but no post-press read resolved in time, so the dialog may already be gone; `dispatched: unknown` means the press itself was cut off and may have landed. Run `alert get` before acting again. - On Android, an alert capture that outlives its budget is stopped in flight. On Limrun the transport cannot cancel a request: the command refuses a late answer and presses nothing, but a hung capture can keep running after the command has already failed. If Limrun alert commands stall, restart the instance session. - Use `alert get` for a quick check, and `alert wait ` only when a prompt may appear after async work. Don't use `settings permission` to answer a dialog already on screen; it is for setup before a flow. - `accept` and `dismiss` are sent once on every platform and never replayed, even if the response is lost. On iOS the runner activates the button once, then only observes until the alert disappears, changes, or the deadline expires. A changed alert does not prove a permission was granted; verify the app outcome. From 83b61291ebb5cf55a1b150fce00c6978c757be5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 17:29:44 +0200 Subject: [PATCH 10/12] fix(alerts): let only the latest answered capture claim a visible dialog Two review findings on the confirm split, one batch: - The spent-budget exits of confirmTreeAlertDismissed omitted details.reason, while commands.md promises every post-press failure branches on alert-window-expired. Both unconfirmed and still-visible failures now carry the typed reason, pinned in the Android and Limrun suites. - The same-alert flag latched on the first matching capture, so a later undecided re-read still reported the dialog as currently visible. Each probe now replaces the verdict with its own observation: the still-visible failure requires the latest capture that answered to name the same alert, and a fresher undecided read exits as an unconfirmed press. A probe whose capture never answered is not a completed observation, so the thrown cutoff keeps saying only that the press went unconfirmed, at any earlier match. Pinned fail-before at the pre-fix alert.ts: same-alert then undecided to budget-end, and same-alert then a hung capture, both expect the unconfirmed message and the typed reason. The helper-scope comment is trimmed to the non-obvious constraint per AGENTS. --- .../src/__tests__/alert.test.ts | 3 +- packages/platform-android/src/alert.ts | 31 ++++----- packages/provider-limrun/src/alert.test.ts | 63 ++++++++++++++++++- src/core/interactors/android.ts | 5 +- website/docs/docs/commands.md | 2 +- 5 files changed, 79 insertions(+), 25 deletions(-) diff --git a/packages/platform-android/src/__tests__/alert.test.ts b/packages/platform-android/src/__tests__/alert.test.ts index 38a9215d6c..5602566cbc 100644 --- a/packages/platform-android/src/__tests__/alert.test.ts +++ b/packages/platform-android/src/__tests__/alert.test.ts @@ -182,7 +182,8 @@ test('dismiss fails when the dialog is still visible after the action budget', a (error: unknown) => error instanceof Error && error.message === 'alert dismiss did not dismiss the visible alert' && - (error as { code?: string }).code === 'COMMAND_FAILED', + (error as { code?: string }).code === 'COMMAND_FAILED' && + (error as { details?: { reason?: unknown } }).details?.reason === 'alert-window-expired', ); assert.equal(runAndroidAdb.mock.calls.length, 1); }); diff --git a/packages/platform-android/src/alert.ts b/packages/platform-android/src/alert.ts index 739b4c489f..a8577dc7e4 100644 --- a/packages/platform-android/src/alert.ts +++ b/packages/platform-android/src/alert.ts @@ -493,14 +493,15 @@ async function confirmTreeAlertDismissed< action: 'accept' | 'dismiss', signal?: AbortSignal, ): Promise { - // Only an observation that named this same alert on screen proves the "still visible" claim. - // Captures that were cut off or stayed unresolvable prove nothing, and the failure wording - // must not accuse a dialog that may already be gone of still standing. - let sameAlertObserved = false; + // The "still visible" claim belongs to the latest capture that answered, never to a match a + // later undecided re-read overruled. The window-closed sentinel is the budget ending, not an + // observation, so it leaves the latest completed capture as the freshest fact. + let lastCaptureNamesAlert = false; const probe: TreeAlertObserver = async () => { const observation = await observe(); - if (observation.kind === 'alert' && sameTreeAlert(observation.candidate.alert, dismissed)) { - sameAlertObserved = true; + if (observation.kind !== 'window-closed') { + lastCaptureNamesAlert = + observation.kind === 'alert' && sameTreeAlert(observation.candidate.alert, dismissed); } return observation; }; @@ -516,33 +517,26 @@ async function confirmTreeAlertDismissed< signal, })); } catch (error) { - // A confirmation capture this phase cuts off says what the press cannot say alone: whether - // the dialog is still there, or whether the check itself never answered. Only the first is - // the stuck-alert failure; the second must not claim a visible dialog it never observed. + // The newest probe never answered, so no capture of this phase can claim where the dialog + // stands at the budget's end — even one that saw the alert a poll earlier. if (isAlertWindowExpired(error)) { - if (sameAlertObserved) { - throw new AppError('COMMAND_FAILED', `alert ${action} did not dismiss the visible alert`, { - alert: dismissed, - reason: ALERT_WINDOW_EXPIRED_REASON, - hint: 'The alert button was pressed but the dialog is still visible. Inspect alert get --json, then press the visible button by label/ref or retry.', - }); - } throw new AppError( 'COMMAND_FAILED', `alert ${action} pressed the button but the dismissal could not be confirmed`, { alert: dismissed, reason: ALERT_WINDOW_EXPIRED_REASON, - hint: 'The alert button was pressed; the post-press read never answered within the budget, so the dialog may already be gone. Run alert get --json before retrying.', + hint: 'The alert button was pressed; the latest post-press check never answered within the budget, so the dialog may already be gone. Run alert get --json before retrying.', }, ); } throw error; } if (settled) return; - if (sameAlertObserved) { + if (lastCaptureNamesAlert) { throw new AppError('COMMAND_FAILED', `alert ${action} did not dismiss the visible alert`, { alert: dismissed, + reason: ALERT_WINDOW_EXPIRED_REASON, hint: 'The alert button was pressed but the dialog is still visible. Inspect alert get --json, then press the visible button by label/ref or retry.', }); } @@ -551,6 +545,7 @@ async function confirmTreeAlertDismissed< `alert ${action} pressed the button but the dismissal could not be confirmed`, { alert: dismissed, + reason: ALERT_WINDOW_EXPIRED_REASON, hint: 'The alert button was pressed; no post-press read resolved to an answer within the budget, so the dialog may already be gone. Run alert get --json before retrying.', }, ); diff --git a/packages/provider-limrun/src/alert.test.ts b/packages/provider-limrun/src/alert.test.ts index 7981853074..e370aede8f 100644 --- a/packages/provider-limrun/src/alert.test.ts +++ b/packages/provider-limrun/src/alert.test.ts @@ -191,9 +191,68 @@ describe('acting budgets', () => { it('spends the confirm budget and reports a stuck alert', async () => { const client = clientWithTrees([limrunAttAlertTree(), limrunAttAlertTree()]); - await expect(underFakeTimers(() => handleLimrunIosAlert(client, 'dismiss'))).rejects.toThrow( - 'alert dismiss did not dismiss the visible alert', + const thrown = await underFakeTimers(() => handleLimrunIosAlert(client, 'dismiss')).then( + () => undefined, + (error: unknown) => error, + ); + expect(thrown).toBeInstanceOf(AppError); + const failure = thrown as AppError; + expect(failure.message).toBe('alert dismiss did not dismiss the visible alert'); + // Every spent-budget exit of an acting leg branches on this reason, per commands.md. + expect(failure.details?.reason).toBe('alert-window-expired'); + expect(client.tap).toHaveBeenCalledTimes(1); + }); + + it('reports an unconfirmed dismissal when an undecided re-read overrules the last match', async () => { + // Confirm probe 1 re-names the pressed alert; probe 2 is alert-shaped yet unresolvable and + // stays that way to the end of the budget. The undecided read is the fresher fact, so the + // leg may not report the dialog as standing on the older match — it reports the press as + // unconfirmed, which is all a spent budget with no settling read may say. + const client = clientWithTrees([ + limrunAttAlertTree(), + limrunAttAlertTree(), + [limrunAttAlertTree(), limrunAttAlertTree()], + ]); + + const thrown = await underFakeTimers(() => handleLimrunIosAlert(client, 'dismiss')).then( + () => undefined, + (error: unknown) => error, + ); + expect(thrown).toBeInstanceOf(AppError); + const failure = thrown as AppError; + expect(failure.message).toBe( + 'alert dismiss pressed the button but the dismissal could not be confirmed', + ); + expect(failure.details?.reason).toBe('alert-window-expired'); + }); + + it('reports an unconfirmed dismissal when a later capture hangs after one re-named the alert', async () => { + // Same alert at confirm probe 1, then a stalled transport the budget cuts off. With the + // newest read unanswered, the dialog's state at the budget's end is unknown: the leg must + // not claim it is still visible on the older match. + const client = { + elementTree: vi + .fn() + .mockResolvedValueOnce(limrunAttAlertTree()) + .mockResolvedValueOnce(limrunAttAlertTree()) + .mockImplementationOnce( + async () => + new Promise((resolve) => { + setTimeout(() => resolve([limrunAppScreenTree()]), 3_000); + }), + ), + tap: vi.fn(async () => {}), + } satisfies LimrunIosAlertClient; + + const thrown = await underFakeTimers(() => handleLimrunIosAlert(client, 'accept')).then( + () => undefined, + (error: unknown) => error, + ); + expect(thrown).toBeInstanceOf(AppError); + expect((thrown as AppError).message).toBe( + 'alert accept pressed the button but the dismissal could not be confirmed', ); + expect((thrown as AppError).details?.reason).toBe('alert-window-expired'); expect(client.tap).toHaveBeenCalledTimes(1); }); diff --git a/src/core/interactors/android.ts b/src/core/interactors/android.ts index 4d0ea7e449..b9bb6da9d1 100644 --- a/src/core/interactors/android.ts +++ b/src/core/interactors/android.ts @@ -60,9 +60,8 @@ export function createAndroidInteractor( const capture = await snapshotAndroid(device, { includeHiddenContentHints: false, signal: seam.signal, - // Alert probes must share the session's warm helper like every other app-backed read: - // a command-scope alert probe cold-starts the helper past the window that bounds it, - // so every re-poll and the post-press confirm would abort before the device answers. + // App-backed sessions keep the snapshot helper warm; a command-scoped probe would + // force-stop and cold-restart it inside the window that bounds the probe. helperSessionScope, }); return buildSnapshotState(androidSnapshotPublicationInput(capture), undefined).nodes; diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 11540f829d..9dfeb799b5 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -399,7 +399,7 @@ agent-device alert dismiss - Polling wait timeouts, and a replayed wait refused with `wait_landmark_identity_mismatch`, also carry `captures`, `readableCaptures`, and `polls` so you can see where the budget went. Each poll has `startedMs`, `durationMs`, and an `outcome`: `readable`, `unreadable`, `retriable`, `deadline`, `runner-restart`, or `readiness`. Long waits keep the first 5 and last 25 polls. A wait that never saw a readable capture reports the error its polls hit, with that error's code and retry details, instead of a generic timeout; `wait stable` and a never-readable strict absence keep their own diagnostics. `logPath` links the full request log. - `alert` (same as `alert get`) inspects or handles system alerts on iOS simulator, macOS desktop, and Android permission dialogs. On iOS it also covers app-owned modal popups with native blocking markers. `alert wait ` takes a budget up to 2147483647 ms (~24.8 days); larger is refused as `INVALID_ARGS` before anything runs. - On the snapshot-derived backends (Android, Limrun iOS) `alert get --json` answers with a typed `status`: `visible`, `none`, or `undecided` (an alert-shaped capture that resolves to no single alert). Branch on `status`, not on the message. -- On those backends, when `alert accept`/`dismiss` fails, `error.details.reason` names why. Before any press, `dispatched: no` and the reason is `alert-not-found` (the action budget passed with no alert seen), `alert-window-expired` (the budget closed before a capture or the press could resolve), `alert_capture_undecided`, or `alert_button_undecided` (the sheet is too ambiguous to act on). After a press, “did not dismiss the visible alert” means a re-read named the same dialog again; “could not be confirmed” means the press went out but no post-press read resolved in time, so the dialog may already be gone; `dispatched: unknown` means the press itself was cut off and may have landed. Run `alert get` before acting again. +- On those backends, when `alert accept`/`dismiss` fails, `error.details.reason` names why. Before any press, `dispatched: no` and the reason is `alert-not-found` (the action budget passed with no alert seen), `alert-window-expired` (the budget closed before a capture or the press could resolve), `alert_capture_undecided`, or `alert_button_undecided` (the sheet is too ambiguous to act on). After a press, `dispatched` is absent and the reason is `alert-window-expired`; “did not dismiss the visible alert” means the latest post-press capture that answered named the same dialog again, “could not be confirmed” means the press went out but the latest read was unresolvable or never answered, so the dialog may already be gone; `dispatched: unknown` means the press itself was cut off and may have landed. Run `alert get` before acting again. - On Android, an alert capture that outlives its budget is stopped in flight. On Limrun the transport cannot cancel a request: the command refuses a late answer and presses nothing, but a hung capture can keep running after the command has already failed. If Limrun alert commands stall, restart the instance session. - Use `alert get` for a quick check, and `alert wait ` only when a prompt may appear after async work. Don't use `settings permission` to answer a dialog already on screen; it is for setup before a flow. - `accept` and `dismiss` are sent once on every platform and never replayed, even if the response is lost. On iOS the runner activates the button once, then only observes until the alert disappears, changes, or the deadline expires. A changed alert does not prove a permission was granted; verify the app outcome. From 71087fb960351e890035e716715eb144df9f570f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 18:02:49 +0200 Subject: [PATCH 11/12] fix(alerts): a deadline-straddling confirm capture cannot erase an answered sighting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Coverage run at f584e8a15 (38924441176) failed the maintainer-owned provider-integration scenario 'Android alert accept fails when the dialog stays visible' — a suite the affected gate's related-test selection does not reach, so the local exact-head receipt never ran it. The driver, not the oracle, was wrong: f584's confirm catch downgraded every thrown cutoff to unconfirmed, discarding the captures that had answered before it. On CI the final confirm capture starts inside the 2s window and lands past it (the thrown path); locally it starts after the deadline (the sentinel path), which is why the same commit passed one and failed the other. The dialog had been re-named by answered reads; the error claimed the check never answered. The latest-completed-observation rule from review thread r4296070770 and from commands.md is: the latest capture that ANSWERED carries the claim. A hung probe is not an observation, so the cutoff now reports still-visible when an answered capture named the alert, and unconfirmed only when none did — the 2482e24 smoke case, with no answer at all. The undecided-overrules-match case is unchanged: undecided is an answered observation. Pin flipped to the CI timeline (fail-before at f584's catch); all 128 alert-family tests and the 197-test provider-integration project pass at this working tree. --- packages/platform-android/src/alert.ts | 16 +++++++++++++--- packages/provider-limrun/src/alert.test.ts | 14 +++++++------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/packages/platform-android/src/alert.ts b/packages/platform-android/src/alert.ts index a8577dc7e4..fbf2692c19 100644 --- a/packages/platform-android/src/alert.ts +++ b/packages/platform-android/src/alert.ts @@ -517,9 +517,19 @@ async function confirmTreeAlertDismissed< signal, })); } catch (error) { - // The newest probe never answered, so no capture of this phase can claim where the dialog - // stands at the budget's end — even one that saw the alert a poll earlier. + // The final probe's capture never answered within the budget. That does not erase the + // captures that did: the still-visible failure rides the latest completed observation, so a + // dialog re-named by a read that landed before the hang is still reported as visible. Only + // when no completed post-press read named it — the 2482e24 Android smoke case, where the + // confirm phase never got one answer — may the leg say the press went unconfirmed. if (isAlertWindowExpired(error)) { + if (lastCaptureNamesAlert) { + throw new AppError('COMMAND_FAILED', `alert ${action} did not dismiss the visible alert`, { + alert: dismissed, + reason: ALERT_WINDOW_EXPIRED_REASON, + hint: 'The alert button was pressed but the dialog was visible on the last check that answered. Inspect alert get --json, then press the visible button by label/ref or retry.', + }); + } throw new AppError( 'COMMAND_FAILED', `alert ${action} pressed the button but the dismissal could not be confirmed`, @@ -537,7 +547,7 @@ async function confirmTreeAlertDismissed< throw new AppError('COMMAND_FAILED', `alert ${action} did not dismiss the visible alert`, { alert: dismissed, reason: ALERT_WINDOW_EXPIRED_REASON, - hint: 'The alert button was pressed but the dialog is still visible. Inspect alert get --json, then press the visible button by label/ref or retry.', + hint: 'The alert button was pressed but the dialog was visible on the last check that answered. Inspect alert get --json, then press the visible button by label/ref or retry.', }); } throw new AppError( diff --git a/packages/provider-limrun/src/alert.test.ts b/packages/provider-limrun/src/alert.test.ts index e370aede8f..a360d6f82d 100644 --- a/packages/provider-limrun/src/alert.test.ts +++ b/packages/provider-limrun/src/alert.test.ts @@ -226,10 +226,12 @@ describe('acting budgets', () => { expect(failure.details?.reason).toBe('alert-window-expired'); }); - it('reports an unconfirmed dismissal when a later capture hangs after one re-named the alert', async () => { - // Same alert at confirm probe 1, then a stalled transport the budget cuts off. With the - // newest read unanswered, the dialog's state at the budget's end is unknown: the leg must - // not claim it is still visible on the older match. + it('keeps the still-visible claim when a later capture hangs after one re-named the alert', async () => { + // Same alert at confirm probe 1, then a stalled transport the budget cuts off. A hung probe + // is not an observation: the still-visible failure rides the latest capture that answered, + // which named the dialog, exactly as commands.md states. This is the executed provider + // integration timeline (Coverage run 38924441176), where the f584 catch unconditionally + // downgraded it to unconfirmed. const client = { elementTree: vi .fn() @@ -249,9 +251,7 @@ describe('acting budgets', () => { (error: unknown) => error, ); expect(thrown).toBeInstanceOf(AppError); - expect((thrown as AppError).message).toBe( - 'alert accept pressed the button but the dismissal could not be confirmed', - ); + expect((thrown as AppError).message).toBe('alert accept did not dismiss the visible alert'); expect((thrown as AppError).details?.reason).toBe('alert-window-expired'); expect(client.tap).toHaveBeenCalledTimes(1); }); From be48a285a7bf57348634d2e4b9204647737c5ba6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 18:14:29 +0200 Subject: [PATCH 12/12] test(alerts): pin the press cutoff's dispatched disclosure; review nits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The independent review of f584e8a15 returned merge-ready with three follow-up-scale findings, batched here: - The documented `dispatched: 'unknown'` press-cutoff failure had no test. Pinned in the Android suite with the real transport shape: the adb fake hangs the tap until the 2s action window closes and dies with the signal; the failure must disclose unknown, never 'no'. Mutation-checked: flipping the disclosure to 'no' fails the pin. - The `[timeout]` positional is honored by `alert wait` only — the Apple acting legs pass fixed runner budgets and the tree legs run their own lookup/confirm windows; commands.md now says the argument is wait-only per backend instead of leaving the silence to surprise callers. - `void device;` in limrunAlertOperationFacts follows the `_device` convention; the two byte-identical visible-after-press exits collapse into one helper, keeping the two hints that genuinely differ (latest read hung vs budget ended unresolved). No behavior change for any message, reason, or tap. --- .../src/__tests__/alert.test.ts | 49 ++++++++++++++++--- packages/platform-android/src/alert.ts | 31 ++++++------ .../src/interaction-operations.ts | 3 +- website/docs/docs/commands.md | 2 +- 4 files changed, 61 insertions(+), 24 deletions(-) diff --git a/packages/platform-android/src/__tests__/alert.test.ts b/packages/platform-android/src/__tests__/alert.test.ts index 5602566cbc..f4b37658fe 100644 --- a/packages/platform-android/src/__tests__/alert.test.ts +++ b/packages/platform-android/src/__tests__/alert.test.ts @@ -6,14 +6,17 @@ import { isRequestCanceledError } from '@agent-device/kernel/errors'; import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; import { button, node, text } from './alert-fixtures.ts'; -const runAndroidAdb = vi.fn(async (_device: DeviceInfo, _args: readonly string[]) => ({ - exitCode: 0, - stdout: '', - stderr: '', -})); +type FakeShellOptions = { signal?: AbortSignal }; +const runAndroidAdb = vi.fn( + async (_device: DeviceInfo, _args: readonly string[], _options?: FakeShellOptions) => ({ + exitCode: 0, + stdout: '', + stderr: '', + }), +); const runAndroidShell = vi.fn( - async (device: DeviceInfo, words: ShellWord[]) => - await runAndroidAdb(device, deviceShellArgv('adb', 'shell', words)), + async (device: DeviceInfo, words: ShellWord[], options?: FakeShellOptions) => + await runAndroidAdb(device, deviceShellArgv('adb', 'shell', words), options), ); vi.mock('../adb.ts', () => ({ runAndroidAdb, runAndroidShell })); @@ -198,6 +201,38 @@ test('wait with a spent budget still reports an alert that is already visible', assert.equal(result.alert?.title, 'Automation confirmation'); }); +test('a press cut off mid-flight discloses that it may or may not have landed', async () => { + // The one typed exit that must never collapse into 'no': the adb child is killed between + // the tap's dispatch and its rejection, so the device may already have received it. The + // press hangs until the action window's 2 s closes, then the child dies with the signal. + runAndroidAdb.mockClear(); + const tapSignals: (AbortSignal | undefined)[] = []; + runAndroidAdb.mockImplementationOnce(async (_device, args, options) => { + if (args.includes('tap')) { + tapSignals.push(options?.signal); + await new Promise((_resolve, reject) => { + options?.signal?.addEventListener('abort', () => reject(new Error('adb killed')), { + once: true, + }); + }); + } + return { exitCode: 0, stdout: '', stderr: '' }; + }); + + await assert.rejects( + handleAndroidAlert(device, 'dismiss', { captureNodes: dialogUntilPressed() }), + (error: unknown) => + error instanceof Error && + error.message === + 'alert dismiss was cut off while pressing; the press may or may not have landed' && + (error as { details?: { dispatched?: unknown; reason?: unknown } }).details?.dispatched === + 'unknown' && + (error as { details?: { reason?: unknown } }).details?.reason === 'alert-window-expired', + ); + assert.equal(tapSignals.length, 1); + assert.equal(tapSignals[0]?.aborted, true); +}); + test('the read-first capture rides the caller cancellation and dies with it', async () => { // Android's transport is cancellable, so the invariant here is the actual child-process cut: // the capture seam receives the leg's live AbortSignal, and aborting the caller cuts the diff --git a/packages/platform-android/src/alert.ts b/packages/platform-android/src/alert.ts index fbf2692c19..df7d1f07f9 100644 --- a/packages/platform-android/src/alert.ts +++ b/packages/platform-android/src/alert.ts @@ -523,13 +523,7 @@ async function confirmTreeAlertDismissed< // when no completed post-press read named it — the 2482e24 Android smoke case, where the // confirm phase never got one answer — may the leg say the press went unconfirmed. if (isAlertWindowExpired(error)) { - if (lastCaptureNamesAlert) { - throw new AppError('COMMAND_FAILED', `alert ${action} did not dismiss the visible alert`, { - alert: dismissed, - reason: ALERT_WINDOW_EXPIRED_REASON, - hint: 'The alert button was pressed but the dialog was visible on the last check that answered. Inspect alert get --json, then press the visible button by label/ref or retry.', - }); - } + if (lastCaptureNamesAlert) throw visibleAfterPressFailure(action, dismissed); throw new AppError( 'COMMAND_FAILED', `alert ${action} pressed the button but the dismissal could not be confirmed`, @@ -543,13 +537,7 @@ async function confirmTreeAlertDismissed< throw error; } if (settled) return; - if (lastCaptureNamesAlert) { - throw new AppError('COMMAND_FAILED', `alert ${action} did not dismiss the visible alert`, { - alert: dismissed, - reason: ALERT_WINDOW_EXPIRED_REASON, - hint: 'The alert button was pressed but the dialog was visible on the last check that answered. Inspect alert get --json, then press the visible button by label/ref or retry.', - }); - } + if (lastCaptureNamesAlert) throw visibleAfterPressFailure(action, dismissed); throw new AppError( 'COMMAND_FAILED', `alert ${action} pressed the button but the dismissal could not be confirmed`, @@ -561,6 +549,21 @@ async function confirmTreeAlertDismissed< ); } +/** + * The stuck-alert failure: a post-press capture that answered named the same dialog again, so + * the press did not dismiss what the latest completed read still saw on screen. + */ +function visibleAfterPressFailure( + action: 'accept' | 'dismiss', + dismissed: TAlert, +): AppError { + return new AppError('COMMAND_FAILED', `alert ${action} did not dismiss the visible alert`, { + alert: dismissed, + reason: ALERT_WINDOW_EXPIRED_REASON, + hint: 'The alert button was pressed but the dialog was visible on the last check that answered. Inspect alert get --json, then press the visible button by label/ref or retry.', + }); +} + /** Alert identity across a dismissal re-check: the same title and the same button row, in order. */ function sameTreeAlert( left: AlertInfo & { buttons: string[] }, diff --git a/packages/provider-limrun/src/interaction-operations.ts b/packages/provider-limrun/src/interaction-operations.ts index 17867bd811..0faa394905 100644 --- a/packages/provider-limrun/src/interaction-operations.ts +++ b/packages/provider-limrun/src/interaction-operations.ts @@ -276,10 +276,9 @@ export function limrunClipboardOperationFacts( * both legs of a live session. */ export function limrunAlertOperationFacts( - device: DeviceInfo, + _device: DeviceInfo, liveSessionUnavailable?: RuntimeOperationUnavailability, ) { - void device; const cell = liveSessionUnavailable ?? available; return Object.freeze({ ...alertRuntimeOperationFacts({ read: cell, wait: cell, accept: cell, dismiss: cell }), diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 9dfeb799b5..e8529577c5 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -397,7 +397,7 @@ agent-device alert dismiss - `wait_stable_timeout`: the UI did not settle. - Branch on `readableCaptures`, `waitedMs`, `matches`, and `firstMatch` instead of parsing error text. `firstMatch` carries identity and text only; absence failures make no visibility or rect claim. - Polling wait timeouts, and a replayed wait refused with `wait_landmark_identity_mismatch`, also carry `captures`, `readableCaptures`, and `polls` so you can see where the budget went. Each poll has `startedMs`, `durationMs`, and an `outcome`: `readable`, `unreadable`, `retriable`, `deadline`, `runner-restart`, or `readiness`. Long waits keep the first 5 and last 25 polls. A wait that never saw a readable capture reports the error its polls hit, with that error's code and retry details, instead of a generic timeout; `wait stable` and a never-readable strict absence keep their own diagnostics. `logPath` links the full request log. -- `alert` (same as `alert get`) inspects or handles system alerts on iOS simulator, macOS desktop, and Android permission dialogs. On iOS it also covers app-owned modal popups with native blocking markers. `alert wait ` takes a budget up to 2147483647 ms (~24.8 days); larger is refused as `INVALID_ARGS` before anything runs. +- `alert` (same as `alert get`) inspects or handles system alerts on iOS simulator, macOS desktop, and Android permission dialogs. On iOS it also covers app-owned modal popups with native blocking markers. The `[timeout]` argument applies to `alert wait` only, on every backend: `alert wait ` takes a budget up to 2147483647 ms (~24.8 days), larger is refused as `INVALID_ARGS` before anything runs, while `get`/`accept`/`dismiss` run their own fixed per-backend budgets and ignore it. - On the snapshot-derived backends (Android, Limrun iOS) `alert get --json` answers with a typed `status`: `visible`, `none`, or `undecided` (an alert-shaped capture that resolves to no single alert). Branch on `status`, not on the message. - On those backends, when `alert accept`/`dismiss` fails, `error.details.reason` names why. Before any press, `dispatched: no` and the reason is `alert-not-found` (the action budget passed with no alert seen), `alert-window-expired` (the budget closed before a capture or the press could resolve), `alert_capture_undecided`, or `alert_button_undecided` (the sheet is too ambiguous to act on). After a press, `dispatched` is absent and the reason is `alert-window-expired`; “did not dismiss the visible alert” means the latest post-press capture that answered named the same dialog again, “could not be confirmed” means the press went out but the latest read was unresolvable or never answered, so the dialog may already be gone; `dispatched: unknown` means the press itself was cut off and may have landed. Run `alert get` before acting again. - On Android, an alert capture that outlives its budget is stopped in flight. On Limrun the transport cannot cancel a request: the command refuses a late answer and presses nothing, but a hung capture can keep running after the command has already failed. If Limrun alert commands stall, restart the instance session.