diff --git a/packages/contracts/src/interaction-guarantees.ts b/packages/contracts/src/interaction-guarantees.ts index f4f7e4c9ae..2d0a6280ae 100644 --- a/packages/contracts/src/interaction-guarantees.ts +++ b/packages/contracts/src/interaction-guarantees.ts @@ -96,6 +96,7 @@ export const INTERACTION_PATH_IDS = [ 'target-drag', 'native-ref', 'coordinate', + 'find-focus-dispatch', 'maestro-direct-selector', 'maestro-non-hittable-fallback', ] as const; @@ -153,6 +154,8 @@ export type InteractionPathContract = { const GAPS_UMBRELLA_ISSUE = 'https://github.com/callstack/agent-device/issues/1081'; const TAP_OUTCOME_NOT_OBSERVED_GAP_ISSUE = 'https://github.com/callstack/agent-device/issues/3335'; +// The four guarantees this route's own seam leaves open are owned by #3734. +const FIND_FOCUS_DISPATCH_GAPS_ISSUE = 'https://github.com/callstack/agent-device/issues/3734'; /** * The moving-target gate's shared cell (#3471). Scoped to the two commands that dispatch a tap the @@ -233,7 +236,7 @@ const RUNTIME_TREE_SHARED_GUARANTEES = { // `occlusion` nor `offscreen` can reach it. keyboardOcclusion: { kind: 'runtime', - via: 'src/commands/interaction/runtime/keyboard-occlusion.ts#assertTapTargetClearOfVisibleKeyboard', + via: 'packages/selectors/src/keyboard-occlusion.ts#assertTapTargetClearOfVisibleKeyboard', }, // #1542: the base decision is the contracts-owned snapshot visibility resolver (bulk accessibility // tree), but throwIfOffscreenInteractionTarget is the actual end-to-end @@ -348,7 +351,7 @@ export const INTERACTION_DISPATCH_PATHS: Record focus`, and the focus leg of `find type`: find captures its own tree, ranks its own candidate set, runs the node stages its `findAct` row declares, then dispatches a point focus itself (`type` then dispatches the text) instead of re-entering the interaction leaf. `find click`/`find fill` are NOT this path — they hand the resolved node to `runtime-ref` as a pre-resolved target (#1654). `focus ` is the coordinate route. Every cell below classifies what the `findAct` row plus this seam actually run, not what the acting rows run.', + commands: ['find'], + guarantees: { + // find's own ranking, but under the same ambiguity contract the runtime paths use: one + // `reject-candidates` policy row (#1625), `--first`/`--last` the only narrowing, and + // `preferOnscreenMatches` as the tie-break rather than a geometry-wins rule. + disambiguation: { + kind: 'runtime', + via: 'packages/daemon/src/execution/interaction/internal/find-match-resolution.ts#resolveFindMatch', + }, + occlusion: { + kind: 'runtime', + via: 'packages/selectors/src/selector-pipeline.ts#runNodePipelineStages', + }, + // #2622: the seam dispatches the point itself, so the shared #2589 guard runs here rather + // than at the interaction leaf's door. Same refusal, same typed details, measured at the + // center the seam dispatches. + keyboardOcclusion: { + kind: 'runtime', + via: 'packages/selectors/src/keyboard-occlusion.ts#assertTapTargetClearOfVisibleKeyboard', + }, + offscreen: { + kind: 'waived', + reason: + 'gap: the `findAct` row declares `offscreen: "ignore"` because find ranks across the whole tree and prefers on-screen matches rather than refusing off-screen ones (#1625), and focus/type never re-enter the acting leaf where the runtime rows refuse. So the seam dispatches at a captured rect center the acting paths would reject as outside the root viewport — a closed drawer or an off-viewport carousel is a silent no-op focus here.', + trackingIssue: FIND_FOCUS_DISPATCH_GAPS_ISSUE, + }, + parentOwnedTouchPoint: { + kind: 'waived', + reason: + 'gap: the seam aims `centerOfRect` of the resolved node rather than the parent-owned point `resolveInteractionTouchPoint` chooses, so a match whose center is tiled by an independently interactive descendant activates that descendant instead of failing closed — and the keyboard guard above measures the same center the tap uses, which is the point actually dispatched but not the point the acting paths would aim.', + trackingIssue: FIND_FOCUS_DISPATCH_GAPS_ISSUE, + }, + // The guarantee is promotion AND annotation. This seam runs only the first half, so the + // cell cannot claim `runtime` for the whole guarantee (the same shape `native-ref` classifies + // the other way round, annotation without promotion). + nonHittable: { + kind: 'waived', + reason: + "gap: the promotion half does run — the `findAct` row declares `hittable-ancestor-below-root`, so the seam dispatches the promoted ancestor's center (regression-proven in packages/daemon/src/execution/interaction/internal/__tests__/find.test.ts) — but the annotation half never does: the response is the focus runtime's point payload, so a match still reporting `hittable: false` after promotion is accepted silently, where press/click/fill answer with `targetHittable: false` plus a hint. Closing it needs the annotation composed at this seam, which is response-shape work the `responseIdentity` waiver below also records.", + trackingIssue: FIND_FOCUS_DISPATCH_GAPS_ISSUE, + }, + // One construction site per dispatch kind, both shared with the plain commands: the focus + // payload is `executeFocusPoint`'s (the same owner `focus ` reaches), and `type`'s + // second dispatch is `executeBoundTypeText`'s. The seam hand-rolls neither. + responseConstruction: { + kind: 'runtime', + via: 'packages/daemon/src/execution/focus-runtime.ts#executeFocusPoint', + }, + responseIdentity: { + kind: 'waived', + reason: + "Intentional: the payload is the focus (or text) dispatch result, which omits `refLabel`, `selectorChain`, and `data.ref` entirely — this route issues no pinnable or diagnostic response identity, since ADR 0014 keeps mutating find's pre-action ref diagnostic-only (no `refsGeneration`) and the seam never enters the ref/selector response builder that mints the identity fields.", + }, + verifyEvidence: { + kind: 'inapplicable', + reason: + 'find exposes no --verify flag: its flag surface is snapshot depth/raw plus --first/--last, and --verify would have no post-action capture owner on a route whose response is the dispatch payload.', + }, + settleObservation: { + kind: 'inapplicable', + reason: + 'find exposes no --settle flag for the same reason; settling is built on the tree-based runtime baseline and the captures the acting leaf owns.', + }, + // Locator misses and candidate sets use find's shared diagnostics, and a pre-action guard + // refusal travels as the AppError its owner built (`coveredInteractionDetails`, + // TAP_KEYBOARD_OCCLUDES_TARGET_DETAILS) through `normalizeError`, so code, hint, and reason + // reach the wire unchanged. + errorTaxonomy: { + kind: 'runtime', + via: 'packages/daemon/src/execution/selector-match-errors.ts#buildAmbiguousMatchError', + }, + resolutionDisclosure: { + kind: 'inapplicable', + reason: + 'The additive `resolution` field discloses the provenance of an identity the response issues; this seam issues none (ADR 0014), and its ambiguity is disclosed as the bounded candidate error the `errorTaxonomy` cell names instead.', + }, + outcomeObservation: { + kind: 'waived', + reason: + 'gap: the response reports the focus (and for `type`, the text) dispatch only. Nothing observes that the field took focus or that the text landed, and unlike the tap-shaped paths this route has no opt-in --verify/--settle that could carry such evidence.', + trackingIssue: TAP_OUTCOME_NOT_OBSERVED_GAP_ISSUE, + }, + targetStability: { + kind: 'waived', + reason: + "gap: the seam dispatches a rect center captured one runner round trip earlier without the #3471 re-observation, so a focus issued right after an action that animated the surface can aim a position the target has already vacated and report success on the vacancy. The gate itself is the acting leaf's (`guardResolvedTargetAgainstMotion`), and this route does not pass through it.", + trackingIssue: FIND_FOCUS_DISPATCH_GAPS_ISSUE, + }, + }, + }, 'maestro-direct-selector': { description: 'An explicit Maestro-compatible simple-selector click completed as an XCTest element tap. This path is selected only when the runner was allowed to use the non-hittable coordinate fallback but reported that it did not use it.', diff --git a/packages/daemon/src/execution/interaction/internal/__tests__/find-keyboard-occlusion.test.ts b/packages/daemon/src/execution/interaction/internal/__tests__/find-keyboard-occlusion.test.ts new file mode 100644 index 0000000000..e86b145b48 --- /dev/null +++ b/packages/daemon/src/execution/interaction/internal/__tests__/find-keyboard-occlusion.test.ts @@ -0,0 +1,93 @@ +import { test, expect, vi, beforeEach } from 'vitest'; +import { keyboardCoveredTabBarSnapshot } from '@agent-device/test-kit'; +import type { SnapshotState } from '@agent-device/kernel/snapshot'; +import { + mockFocusPoint, + resetFindTouchRuntimeFixture, + runFindClickScenario, +} from './find-touch-runtime-fixture.ts'; +import { mockTypeText } from '../../../__tests__/interaction-get-runtime-fixture.ts'; +import { refFrameState } from '../../../../core/ref-frame.ts'; + +vi.mock('../../../../core/snapshot-interactor-capture.ts', async () => { + const fixture = await import('../../../../__tests__/support/legacy-snapshot-capture-fixture.ts'); + return { captureSnapshotWithInteractor: fixture.captureSnapshotThroughLegacyDispatchFixture }; +}); + +beforeEach(() => { + resetFindTouchRuntimeFixture(); +}); + +function keyboardTree(targetRect?: SnapshotState['nodes'][number]['rect']): SnapshotState { + return keyboardCoveredTabBarSnapshot(targetRect ? { targetRect } : {}); +} + +type Refusal = { + code?: string; + message?: string; + hint?: string; + details?: Record; +}; + +function assertKeyboardRefusal( + response: { ok: boolean; error?: Refusal }, + verb: string, +): asserts response { + expect(response.ok).toBe(false); + const error = response.error as Refusal; + expect(error.code).toBe('COMMAND_FAILED'); + expect(error.message).toBe(`Ref @e2 is behind the visible keyboard and cannot ${verb} safely`); + expect(error.details?.reason).toBe('tap_keyboard_occludes_target'); + expect(error.details?.ref).toBe('@e2'); + expect(error.details?.rect).toEqual({ x: 148, y: 791, width: 104, height: 83 }); + expect(error.details?.keyboardFrame).toEqual({ x: 0, y: 583, width: 402, height: 291 }); + expect(error.details?.dispatched).toBe('no'); + expect(error.hint).toMatch(/End editing first/); +} + +test('find focus refuses a match whose tap point sits behind the visible keyboard', async () => { + const { response, session } = await runFindClickScenario({ + positionals: ['Form', 'focus'], + nodes: keyboardTree().nodes, + }); + + assertKeyboardRefusal(response, 'be focused'); + expect(mockFocusPoint).not.toHaveBeenCalled(); + // ADR 0014: the refusal is pre-action, so the captured frame stays pinnable. + expect(refFrameState(session)).toBe('active'); +}); + +test('find type refuses before the focus or the text dispatch', async () => { + const { response, session } = await runFindClickScenario({ + positionals: ['Form', 'type', 'hello'], + nodes: keyboardTree().nodes, + }); + + assertKeyboardRefusal(response, 'be focused'); + expect(mockFocusPoint).not.toHaveBeenCalled(); + expect(mockTypeText).not.toHaveBeenCalled(); + expect(refFrameState(session)).toBe('active'); +}); + +test('find focus on a match above the keyboard band still focuses', async () => { + const { response, session } = await runFindClickScenario({ + positionals: ['Form', 'focus'], + nodes: keyboardTree({ x: 148, y: 470, width: 104, height: 110 }).nodes, + }); + + expect(response.ok).toBe(true); + expect(mockFocusPoint).toHaveBeenCalledTimes(1); + expect(refFrameState(session)).toBe('expired'); +}); + +test('find focus on a reported keyboard key still focuses it', async () => { + // The closest negative: the same guard excuses the keyboard's own control, which sits inside the + // band it would otherwise refuse. + const { response } = await runFindClickScenario({ + positionals: ['label', 'space', 'focus'], + nodes: keyboardTree().nodes, + }); + + expect(response.ok).toBe(true); + expect(mockFocusPoint).toHaveBeenCalledTimes(1); +}); diff --git a/packages/daemon/src/execution/interaction/internal/find.ts b/packages/daemon/src/execution/interaction/internal/find.ts index 6f06edcc98..b65ccfe46d 100644 --- a/packages/daemon/src/execution/interaction/internal/find.ts +++ b/packages/daemon/src/execution/interaction/internal/find.ts @@ -13,6 +13,12 @@ import { import { runNodePipelineStages } from '@agent-device/selectors/selector-pipeline'; import { SELECTOR_PIPELINE_POLICIES } from '@agent-device/selectors/selector-pipeline-policy'; import { coveredInteractionDetails } from '@agent-device/selectors/interaction-error'; +import { assertTapTargetClearOfVisibleKeyboard } from '@agent-device/selectors/keyboard-occlusion'; +import { + interactionVerb, + type InteractionAction, +} from '@agent-device/selectors/interaction-action'; +import { normalizeError } from '@agent-device/kernel/errors'; import { centerOfRect, type SnapshotAddressableNode, @@ -381,7 +387,7 @@ async function handleFindType( contextFromFlags(logPath, req.flags, session.appBundleId, session.trace?.outPath), ); recordFindAction(ctx, match, 'type'); - return { ok: true, data: response ?? { ref: match.ref } }; + return { ok: true, data: response }; } async function dispatchFocusForFindMatch( @@ -389,12 +395,26 @@ async function dispatchFocusForFindMatch( match: ResolvedMatch, ): Promise { const { req, logPath } = ctx; - const coveredResponse = rejectCoveredFindMatch(match, 'be focused'); + const coveredResponse = rejectCoveredFindMatch(match, 'focus'); if (coveredResponse) return coveredResponse; const coords = match.resolvedNode.rect ? centerOfRect(match.resolvedNode.rect) : null; if (!coords) { return errorResponse('COMMAND_FAILED', 'matched element has no bounds'); } + // The same keyboard decision the acting paths run (#2589/#2622): this seam dispatches the point + // itself instead of re-entering the interaction leaf, so the guard has to run here. + try { + assertTapTargetClearOfVisibleKeyboard({ + nodes: match.tree.nodes, + node: match.resolvedNode, + action: 'focus', + label: `Ref ${match.ref}`, + ...(match.keyboard ? { keyboard: match.keyboard } : {}), + tapPoint: coords, + }); + } catch (error) { + return { ok: false, error: normalizeError(error) }; + } // ADR 0014 side-effect seam: mutating find focus/type dispatch the device // command directly (they do not re-enter the interaction leaf), so expire the // frame here before the device op. Pre-seam guards above preserve the frame. @@ -411,15 +431,18 @@ async function dispatchFocusForFindMatch( coords, contextFromFlags(logPath, req.flags, session.appBundleId, session.trace?.outPath), ); - return { ok: true, data: response ?? { ref: match.ref } }; + return { ok: true, data: response }; } -function rejectCoveredFindMatch(match: ResolvedMatch, interaction: string): DaemonResponse | null { +function rejectCoveredFindMatch( + match: ResolvedMatch, + action: InteractionAction, +): DaemonResponse | null { const blockedNode = match.occludedNode; if (!blockedNode) return null; return errorResponse( 'COMMAND_FAILED', - `Matched element ${match.ref} is covered by another visible element and cannot ${interaction} safely`, + `Matched element ${match.ref} is covered by another visible element and cannot ${interactionVerb(action)} safely`, coveredInteractionDetails(blockedNode), ); } diff --git a/packages/platform-apple/src/runner/__tests__/snapshot-presentation.test.ts b/packages/platform-apple/src/runner/__tests__/snapshot-presentation.test.ts index 974a4e265e..a74d9620a9 100644 --- a/packages/platform-apple/src/runner/__tests__/snapshot-presentation.test.ts +++ b/packages/platform-apple/src/runner/__tests__/snapshot-presentation.test.ts @@ -251,7 +251,7 @@ test('a runner payload with the hittable bit absent presents without declaring i // what cannot be placed belongs to `readSnapshotKeyboardBandFact` in @agent-device/kernel, which owns // that table; this seam owns forwarding AND the proof that it forwards through that reader. The // same-process capture path feeds `result.keyboard` straight into the tap/click occlusion guard in -// src/commands/interaction/runtime/keyboard-occlusion.ts before any serialization, so a seam that +// packages/selectors/src/keyboard-occlusion.ts before any serialization, so a seam that // passed the raw wire value through would hand the guard an unvalidated shape with nothing red. // The malformed cases below are what prove the routing; kernel/src/record.test.ts owns the table. diff --git a/packages/selectors/package.json b/packages/selectors/package.json index 972a6c0f4a..d9bdcad640 100644 --- a/packages/selectors/package.json +++ b/packages/selectors/package.json @@ -40,6 +40,10 @@ "types": "./src/absence-observation-resolution.ts", "default": "./src/absence-observation-resolution.ts" }, + "./interaction-action": { + "types": "./src/interaction-action.ts", + "default": "./src/interaction-action.ts" + }, "./interaction-error": { "types": "./src/interaction-error.ts", "default": "./src/interaction-error.ts" @@ -56,6 +60,10 @@ "types": "./src/interaction-touch-point.ts", "default": "./src/interaction-touch-point.ts" }, + "./keyboard-occlusion": { + "types": "./src/keyboard-occlusion.ts", + "default": "./src/keyboard-occlusion.ts" + }, "./press-retarget": { "types": "./src/press-retarget.ts", "default": "./src/press-retarget.ts" diff --git a/packages/selectors/src/interaction-action.test.ts b/packages/selectors/src/interaction-action.test.ts new file mode 100644 index 0000000000..bbb959240b --- /dev/null +++ b/packages/selectors/src/interaction-action.test.ts @@ -0,0 +1,28 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { interactionVerb, type InteractionAction } from './interaction-action.ts'; + +// Exhaustive mapping over `InteractionAction` is the compiler's check. This pins the words +// themselves, so a refusal that starts describing a non-tap gesture as a tap — which is what the +// catch-all used to do for `drag` — is a deliberate change here, not an accident inside the map. +test('every action names itself in the refusal', () => { + const verbs: Array<[InteractionAction, string]> = [ + ['click', 'be tapped'], + ['press', 'be tapped'], + ['fill', 'be filled'], + ['focus', 'be focused'], + ['longPress', 'be long-pressed'], + ['hover', 'be hovered'], + ['scroll', 'be scrolled'], + ['swipe', 'be swiped'], + ['pinch', 'be pinched'], + ['pan', 'be panned'], + ['drag', 'be dragged'], + ['fling', 'be flung'], + ['rotate', 'be rotated'], + ['transform', 'be transformed'], + ]; + for (const [action, verb] of verbs) { + assert.equal(interactionVerb(action), verb, action); + } +}); diff --git a/packages/selectors/src/interaction-action.ts b/packages/selectors/src/interaction-action.ts new file mode 100644 index 0000000000..4f2b4bea43 --- /dev/null +++ b/packages/selectors/src/interaction-action.ts @@ -0,0 +1,60 @@ +/** + * The action an interaction performs, and the passive verb a pre-dispatch refusal uses for it. + * + * Living below both the command surface and the daemon's dispatch seams keeps one vocabulary for + * every guard that refuses before dispatch, so a rule shared by the acting paths and a route that + * dispatches on its own (`find focus`) cannot name the same failure two ways. + */ +export type InteractionAction = + | 'click' + | 'press' + | 'fill' + | 'focus' + | 'longPress' + | 'hover' + | 'scroll' + | 'swipe' + | 'pinch' + | 'pan' + | 'drag' + | 'fling' + | 'rotate' + | 'transform'; + +/** + * The passive verb a refusal uses for the action it declined, so every acting guard that refuses + * before dispatch describes the failure in the caller's own words. Every action is named + * explicitly and there is no `default`: a new `InteractionAction` fails to compile until it picks + * the verb its refusal will use, instead of silently being refused as a tap. + */ +export function interactionVerb(action: InteractionAction): string { + switch (action) { + case 'click': + case 'press': + return 'be tapped'; + case 'fill': + return 'be filled'; + case 'focus': + return 'be focused'; + case 'longPress': + return 'be long-pressed'; + case 'hover': + return 'be hovered'; + case 'scroll': + return 'be scrolled'; + case 'swipe': + return 'be swiped'; + case 'pinch': + return 'be pinched'; + case 'pan': + return 'be panned'; + case 'drag': + return 'be dragged'; + case 'fling': + return 'be flung'; + case 'rotate': + return 'be rotated'; + case 'transform': + return 'be transformed'; + } +} diff --git a/packages/selectors/src/keyboard-occlusion.test.ts b/packages/selectors/src/keyboard-occlusion.test.ts new file mode 100644 index 0000000000..1d9a272f33 --- /dev/null +++ b/packages/selectors/src/keyboard-occlusion.test.ts @@ -0,0 +1,159 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import type { + Point, + Rect, + SnapshotAddressableNode, + SnapshotKeyboardBandFact, + SnapshotState, +} from '@agent-device/kernel/snapshot'; +import { TAP_KEYBOARD_OCCLUDES_TARGET_DETAILS } from '@agent-device/contracts/tap-keyboard-occlusion'; +import { + KEYBOARD_COVERED_BAND_RECT, + KEYBOARD_COVERED_TAB_BAR_RECT, + keyboardCoveredTabBarSnapshot, +} from '@agent-device/test-kit'; +import { + assertTapTargetClearOfVisibleKeyboard, + describeKeyboardOccludedPointWarning, +} from './keyboard-occlusion.ts'; + +// The guard's own two contributions: the refusal it raises for a decision the classifier already +// made, and the disclosure string it composes. Which geometry decides is the classifier's suite +// (packages/contracts/src/tap-keyboard-occlusion.test.ts, golden-table-driven), and which doors run +// the guard is proven where each door lives — the interaction leaf's in +// src/commands/interaction/runtime/target-visibility-stages-keyboard-occlusion.test.ts, the +// mutating-`find` seam's in packages/daemon/src/execution/interaction/internal/__tests__/. Every one +// of those suites measures this same #2589 capture. + +function centerOf(rect: Rect): Point { + return { x: rect.x + rect.width / 2, y: rect.y + rect.height / 2 }; +} + +function addressableNode(snapshot: SnapshotState, index: number): SnapshotAddressableNode { + const node = snapshot.nodes.find((candidate) => candidate.index === index); + assert.ok(node && 'ref' in node && node.ref, `expected node ${index} to carry a ref`); + return node as SnapshotAddressableNode; +} + +/** + * The guard as a caller runs it: against the node it resolved, at the point it would tap. Omitting + * `tapPoint` taps that node's own center. + */ +function guard(params: { + snapshot: SnapshotState; + index?: number; + action?: 'click' | 'fill' | 'focus'; + label?: string; + keyboard?: SnapshotKeyboardBandFact; + tapPoint?: Point | null; +}): void { + const node = addressableNode(params.snapshot, params.index ?? 1); + assertTapTargetClearOfVisibleKeyboard({ + nodes: params.snapshot.nodes, + node, + action: params.action ?? 'click', + label: params.label ?? 'Ref @e2', + ...(params.keyboard ? { keyboard: params.keyboard } : {}), + tapPoint: + params.tapPoint === undefined && node.rect ? centerOf(node.rect) : (params.tapPoint ?? null), + }); +} + +/** Runs the guard expecting a refusal, and hands back that refusal's message and typed details. */ +function keyboardRefusalDetails(params: Parameters[0]): { + message: string; + details: Record; +} { + let message = ''; + let details: Record | undefined; + assert.throws( + () => guard(params), + (error: unknown) => { + assert.ok(error instanceof Error); + assert.equal((error as { code?: string }).code, 'COMMAND_FAILED'); + assert.match(error.message, /Ref @e2 is behind the visible keyboard/); + message = error.message; + details = (error as { details?: Record }).details; + return true; + }, + ); + assert.ok(details, 'the refusal carries details'); + return { message, details }; +} + +test('the refusal carries the shared reason, the target, and the band it measured', () => { + assert.deepEqual(keyboardRefusalDetails({ snapshot: keyboardCoveredTabBarSnapshot() }).details, { + ...TAP_KEYBOARD_OCCLUDES_TARGET_DETAILS, + ref: '@e2', + rect: KEYBOARD_COVERED_TAB_BAR_RECT, + keyboardFrame: KEYBOARD_COVERED_BAND_RECT, + dispatched: 'no', + }); +}); + +test('the refusal names the action it declined', () => { + for (const [action, verb] of [ + ['click', 'be tapped'], + ['fill', 'be filled'], + ['focus', 'be focused'], + ] as const) { + // Each iteration runs the full typed refusal — code, reason, dispatched: 'no' — so the verb + // check cannot pass on an unrelated rejection that merely mentions the action. + const { message, details } = keyboardRefusalDetails({ + snapshot: keyboardCoveredTabBarSnapshot(), + action, + }); + assert.equal(details.reason, 'tap_keyboard_occludes_target'); + assert.equal(details.dispatched, 'no'); + assert.match( + message, + new RegExp(`cannot ${verb} safely`), + `${action} refuses with the "${verb}" verb`, + ); + } +}); + +test('the refusal quotes how the caller named the target', () => { + assert.throws( + () => guard({ snapshot: keyboardCoveredTabBarSnapshot(), label: 'Selector label=Form' }), + /Selector label=Form is behind the visible keyboard/, + ); +}); + +test('a target with no measurable point cannot be refused', () => { + guard({ snapshot: keyboardCoveredTabBarSnapshot(), tapPoint: null }); +}); + +// The coordinate path discloses the same decision instead of refusing it, in one string the +// response publishes. + +function pointWarning(params: { + snapshot: SnapshotState; + point: Point; + keyboard?: SnapshotKeyboardBandFact; +}): string | undefined { + return describeKeyboardOccludedPointWarning({ + nodes: params.snapshot.nodes, + point: params.point, + viewport: params.snapshot.nodes[0]!.rect ?? null, + ...(params.keyboard ? { keyboard: params.keyboard } : {}), + }); +} + +test('a coordinate behind the keyboard discloses the point, reason, and band', () => { + const warning = pointWarning({ + snapshot: keyboardCoveredTabBarSnapshot(), + point: { x: 200, y: 810 }, + }); + assert.match(warning ?? '', /\(200, 810\) is behind the visible keyboard/); + assert.match(warning ?? '', /tap_keyboard_occludes_target/); + assert.match(warning ?? '', /keyboard frame 583\.\.874/); +}); + +test('a coordinate on a reported key discloses nothing', () => { + assert.equal( + pointWarning({ snapshot: keyboardCoveredTabBarSnapshot(), point: { x: 160, y: 760 } }), + undefined, + ); +}); diff --git a/src/commands/interaction/runtime/keyboard-occlusion.ts b/packages/selectors/src/keyboard-occlusion.ts similarity index 93% rename from src/commands/interaction/runtime/keyboard-occlusion.ts rename to packages/selectors/src/keyboard-occlusion.ts index bd45fa0bda..e28efc89aa 100644 --- a/src/commands/interaction/runtime/keyboard-occlusion.ts +++ b/packages/selectors/src/keyboard-occlusion.ts @@ -13,8 +13,7 @@ import { TAP_KEYBOARD_OCCLUDES_TARGET_DETAILS, type KeyboardSurface, } from '@agent-device/contracts/tap-keyboard-occlusion'; -import { interactionVerb } from './interaction-verb.ts'; -import type { InteractionAction } from './interaction-resolution-request.ts'; +import { interactionVerb, type InteractionAction } from './interaction-action.ts'; /** * The keyboard guard every acting path runs against the tree it already holds. @@ -24,6 +23,10 @@ import type { InteractionAction } from './interaction-resolution-request.ts'; * beside the off-screen guard rather than inside any one command keeps the fast path honest: the * native-ref preflight enters through the same door as selector and `@ref` resolution, so a backend * that "succeeds" cannot do so on a target the shared rule would refuse (ADR 0011). + * + * The rule sits below the command surface and the daemon's dispatch seams because both zones call + * it: the interaction leaf through `runInteractionPipelineStages`, and mutating `find focus`/`type`, + * which dispatch a point themselves instead of re-entering that leaf (#2622). */ /** diff --git a/packages/selectors/src/selector-pipeline-policy.ts b/packages/selectors/src/selector-pipeline-policy.ts index 75554a5f02..8233fbd627 100644 --- a/packages/selectors/src/selector-pipeline-policy.ts +++ b/packages/selectors/src/selector-pipeline-policy.ts @@ -181,7 +181,13 @@ export const SELECTOR_PIPELINE_POLICIES = { resolution: SELECTOR_RESOLUTION_POLICIES.readList, occlusion: 'ignore', }, - /** Mutating `find` (#1625). */ + /** + * Mutating `find` (#1625). `offscreen: 'ignore'` is deliberate for the ranking (`preferOnscreen + * Matches` prefers an on-screen twin rather than refusing an off-screen one), and each action then + * decides what to do with the winner: click/fill hand it to the acting leaf, while focus/type + * dispatch a point themselves, which is why ADR 0011 classifies this row's stages per action on + * the `find-focus-dispatch` path rather than reusing an acting row's cells. + */ findAct: { resolution: SELECTOR_RESOLUTION_POLICIES.findAct, occlusion: 'refuse', diff --git a/packages/test-kit/src/index.ts b/packages/test-kit/src/index.ts index b24974cd52..8794236629 100644 --- a/packages/test-kit/src/index.ts +++ b/packages/test-kit/src/index.ts @@ -62,6 +62,11 @@ export { TWO_ACTIONABLE_WRAPPER_CHAIN_NODES, UNVERIFIED_HITTABILITY_WRAPPER_CHAIN_NODES, } from './interaction-targeting.fixtures.ts'; +export { + KEYBOARD_COVERED_BAND_RECT, + KEYBOARD_COVERED_TAB_BAR_RECT, + keyboardCoveredTabBarSnapshot, +} from './keyboard-occlusion.fixtures.ts'; export { closeLoopbackServer, listenOnLoopback, diff --git a/packages/test-kit/src/keyboard-occlusion.fixtures.ts b/packages/test-kit/src/keyboard-occlusion.fixtures.ts new file mode 100644 index 0000000000..9b2cf6e741 --- /dev/null +++ b/packages/test-kit/src/keyboard-occlusion.fixtures.ts @@ -0,0 +1,98 @@ +import { makeSnapshotState } from '@agent-device/capture-kit/snapshot-state-fixtures'; +import type { Rect, SnapshotState } from '@agent-device/kernel/snapshot'; + +/** The #2589 capture's app-owned tab bar, and the band the captured keyboard docks to. */ +export const KEYBOARD_COVERED_TAB_BAR_RECT: Rect = { x: 148, y: 791, width: 104, height: 83 }; +export const KEYBOARD_COVERED_BAND_RECT: Rect = { x: 0, y: 583, width: 402, height: 291 }; + +/** + * The captured #2589 screen: a docked iPhone keyboard whose band owns the app's tab bar. + * + * The keyboard is its own system surface, so it is never a covering sibling of app content here + * (`occlusion` stays silent) and the tab bar is still inside the app's own window rect + * (`offscreen` passes) — the tap used to report success while the key ate the touch. The key rects + * are the bottom row measured on iPhone 17 Pro (26.2): a keyboard the rule will measure has to + * report one unbroken run of columns, so a reduced layout cannot stop at two keys. + * + * The variants keep every keyboard-occlusion suite on this one geometry: + * - `targetRect` moves the app-owned button, so "clear of the keyboard" is the same tree. + * - `keyboardLifted` hauls the keys above the docking budget, which stops the tree-derived rule + * measuring at all — the way to prove a refusal came from the producer's band, not the tree. + */ +export function keyboardCoveredTabBarSnapshot( + params: { targetRect?: Rect; keyboardLifted?: boolean } = {}, +): SnapshotState { + const nodes = keyboardCoveredTabBarTree().nodes.map((node) => { + if (node.index === 1 && params.targetRect) return { ...node, rect: params.targetRect }; + if (!params.keyboardLifted) return node; + return node.rect && (node.type === 'Keyboard' || node.type === 'Key') + ? { ...node, rect: { ...node.rect, y: node.rect.y - 560 } } + : node; + }); + return { ...keyboardCoveredTabBarTree(), nodes }; +} + +function keyboardCoveredTabBarTree(): SnapshotState { + return makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 402, height: 874 }, + hittable: true, + }, + { + index: 1, + depth: 2, + parentIndex: 0, + type: 'Button', + label: 'Form', + rect: KEYBOARD_COVERED_TAB_BAR_RECT, + hittable: true, + }, + { + index: 2, + depth: 1, + parentIndex: 0, + type: 'Keyboard', + rect: KEYBOARD_COVERED_BAND_RECT, + hittable: false, + }, + { + index: 3, + depth: 2, + parentIndex: 2, + type: 'Key', + label: 'globe', + rect: { x: 4.67, y: 752, width: 49.33, height: 54 }, + hittable: true, + }, + { + index: 4, + depth: 2, + parentIndex: 2, + type: 'Key', + label: '.?123', + rect: { x: 54, y: 752, width: 49.33, height: 54 }, + hittable: true, + }, + { + index: 5, + depth: 2, + parentIndex: 2, + type: 'Key', + label: 'space', + rect: { x: 103.33, y: 752, width: 197.33, height: 54 }, + hittable: true, + }, + { + index: 6, + depth: 2, + parentIndex: 2, + type: 'Key', + label: 'return', + rect: { x: 300.67, y: 752, width: 99, height: 54 }, + hittable: true, + }, + ]); +} diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index 34b03b60a2..bc3c73aac1 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -775,10 +775,12 @@ test('the real tree parses, declares, and passes R11', () => { '@agent-device/selectors/absence-observation-resolution', '@agent-device/selectors/ast', '@agent-device/selectors/engine', + '@agent-device/selectors/interaction-action', '@agent-device/selectors/interaction-error', '@agent-device/selectors/interaction-positionals', '@agent-device/selectors/interaction-targeting', '@agent-device/selectors/interaction-touch-point', + '@agent-device/selectors/keyboard-occlusion', '@agent-device/selectors/parameterized-recorded-fill', '@agent-device/selectors/press-retarget', '@agent-device/selectors/selector-pipeline', diff --git a/src/commands/interaction/runtime/interaction-resolution-request.ts b/src/commands/interaction/runtime/interaction-resolution-request.ts index 0eca2e1931..4a0ccfdc89 100644 --- a/src/commands/interaction/runtime/interaction-resolution-request.ts +++ b/src/commands/interaction/runtime/interaction-resolution-request.ts @@ -1,4 +1,5 @@ import type { ActingPipelinePolicy } from '@agent-device/selectors/selector-pipeline-policy'; +import type { InteractionAction } from '@agent-device/selectors/interaction-action'; import type { PreresolvedInteractionTarget } from '@agent-device/contracts/interaction'; import type { ReplayTargetGuardDenotation } from '@agent-device/contracts/replay'; @@ -19,22 +20,6 @@ import type { ReplayTargetGuardDenotation } from '@agent-device/contracts/replay */ export type ExpectedResolvedTarget = ReplayTargetGuardDenotation; -export type InteractionAction = - | 'click' - | 'press' - | 'fill' - | 'focus' - | 'longPress' - | 'hover' - | 'scroll' - | 'swipe' - | 'pinch' - | 'pan' - | 'drag' - | 'fling' - | 'rotate' - | 'transform'; - export type ResolveInteractionTargetParams = { action: InteractionAction; requireInteractive: boolean; diff --git a/src/commands/interaction/runtime/interaction-verb.ts b/src/commands/interaction/runtime/interaction-verb.ts deleted file mode 100644 index 31dbe7fe4a..0000000000 --- a/src/commands/interaction/runtime/interaction-verb.ts +++ /dev/null @@ -1,20 +0,0 @@ -import type { InteractionAction } from './interaction-resolution-request.ts'; - -/** - * The passive verb a refusal uses for the action it declined, so every acting guard that refuses - * before dispatch describes the failure in the caller's own words. - */ -export function interactionVerb(action: InteractionAction): string { - switch (action) { - case 'fill': - return 'be filled'; - case 'focus': - return 'be focused'; - case 'longPress': - return 'be long-pressed'; - case 'hover': - return 'be hovered'; - default: - return 'be tapped'; - } -} diff --git a/src/commands/interaction/runtime/native-ref-interaction.ts b/src/commands/interaction/runtime/native-ref-interaction.ts index 8e70ceedeb..209b62b3a1 100644 --- a/src/commands/interaction/runtime/native-ref-interaction.ts +++ b/src/commands/interaction/runtime/native-ref-interaction.ts @@ -16,7 +16,7 @@ import type { } from '@agent-device/contracts/backend'; import { toBackendContext } from '../../runtime-common.ts'; import { toBackendResult } from '../../runtime-types.ts'; -import type { InteractionAction } from './interaction-resolution-request.ts'; +import type { InteractionAction } from '@agent-device/selectors/interaction-action'; import { tryResolveRefNode } from './ref-target-resolution.ts'; import { EXACT_REF_RESOLUTION, describeNonHittableTarget } from './resolution-disclosure.ts'; import { diff --git a/src/commands/interaction/runtime/resolution-disclosure.ts b/src/commands/interaction/runtime/resolution-disclosure.ts index 6789851e5c..1d8c534903 100644 --- a/src/commands/interaction/runtime/resolution-disclosure.ts +++ b/src/commands/interaction/runtime/resolution-disclosure.ts @@ -16,7 +16,7 @@ import type { ResolutionDisclosure, SurfaceScopedNodes, } from '@agent-device/contracts/interaction'; -import type { InteractionAction } from './interaction-resolution-request.ts'; +import type { InteractionAction } from '@agent-device/selectors/interaction-action'; export type ResolvedRefNode = { ref: string; diff --git a/src/commands/interaction/runtime/resolution.ts b/src/commands/interaction/runtime/resolution.ts index 387b827caf..90931a2529 100644 --- a/src/commands/interaction/runtime/resolution.ts +++ b/src/commands/interaction/runtime/resolution.ts @@ -14,12 +14,10 @@ import type { ResolvedInteractionTarget, SurfaceScopedNodes, } from '@agent-device/contracts/interaction'; -import { describeKeyboardOccludedPointWarning } from './keyboard-occlusion.ts'; +import { describeKeyboardOccludedPointWarning } from '@agent-device/selectors/keyboard-occlusion'; +import type { InteractionAction } from '@agent-device/selectors/interaction-action'; import { assertReplayTargetResolution } from './replay-target-guard.ts'; -import type { - InteractionAction, - ResolveInteractionTargetParams, -} from './interaction-resolution-request.ts'; +import type { ResolveInteractionTargetParams } from './interaction-resolution-request.ts'; import { resolveRefInteractionTarget } from './ref-target-resolution.ts'; import { buildSelectorResolutionDisclosure, diff --git a/src/commands/interaction/runtime/selector-target-resolution.ts b/src/commands/interaction/runtime/selector-target-resolution.ts index cc7ceae36c..c9b15d541d 100644 --- a/src/commands/interaction/runtime/selector-target-resolution.ts +++ b/src/commands/interaction/runtime/selector-target-resolution.ts @@ -14,10 +14,8 @@ import { type ActionSelectorResolution, } from './selector-action-resolution.ts'; import { selectorNotFoundFailure } from './selector-read-shared.ts'; -import type { - InteractionAction, - ResolveInteractionTargetParams, -} from './interaction-resolution-request.ts'; +import type { InteractionAction } from '@agent-device/selectors/interaction-action'; +import type { ResolveInteractionTargetParams } from './interaction-resolution-request.ts'; export type SelectorResolutionAttempt = { capture: InteractionSnapshot; diff --git a/src/commands/interaction/runtime/keyboard-occlusion.test.ts b/src/commands/interaction/runtime/target-visibility-stages-keyboard-occlusion.test.ts similarity index 65% rename from src/commands/interaction/runtime/keyboard-occlusion.test.ts rename to src/commands/interaction/runtime/target-visibility-stages-keyboard-occlusion.test.ts index 324288694b..43946b0e76 100644 --- a/src/commands/interaction/runtime/keyboard-occlusion.test.ts +++ b/src/commands/interaction/runtime/target-visibility-stages-keyboard-occlusion.test.ts @@ -1,40 +1,38 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; -import type { - Point, - Rect, - SnapshotKeyboardBandFact, - SnapshotState, -} from '@agent-device/kernel/snapshot'; +import type { Point, SnapshotKeyboardBandFact, SnapshotState } from '@agent-device/kernel/snapshot'; import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; -import { keyboardCoveredTabBarSnapshot } from '../../../../test/integration/interaction-contract/fixtures.ts'; +import { + KEYBOARD_COVERED_BAND_RECT, + KEYBOARD_COVERED_TAB_BAR_RECT, + keyboardCoveredTabBarSnapshot, +} from '@agent-device/test-kit'; import { ref, selector } from './selector-read-utils.ts'; import { createInteractionDevice } from './__tests__/test-utils/index.ts'; -// #2589: the keyboard is its own system surface, so neither `occlusion` nor `offscreen` refuses a -// tap whose point belongs to it. These cover the consequences the shared classifier cannot express: -// which paths refuse, which disclose, and which stay silent. The tree itself is the contract -// fixtures' #2589 shape, so the refusal and the disclosure are measured on the same geometry the -// ADR 0011 cells claim. +// #2589 through the shared door: which acting paths refuse, which disclose, and which stay silent. +// The decision itself and the shape of its refusal are unit-proven beside the guard, in +// packages/selectors/src/keyboard-occlusion.test.ts. These run the acting paths, because the +// guarantee is that the fast path and the shared rule enter through one door — a unit test on the +// resolver alone would leave the ref path free to consult the tree again on its way to the tap. -const TAB_BAR_RECT: Rect = { x: 148, y: 791, width: 104, height: 83 }; +/** The band the #2589 capture's producer measured, in the app's orientation space. */ +const MEASURED_BAND: SnapshotKeyboardBandFact = { + kind: 'visible', + frame: KEYBOARD_COVERED_BAND_RECT, +}; -/** The contract fixture with its app-owned button moved, so the variants stay one tree, not copies. */ -function keyboardTree(params: { tabRect?: Rect } = {}): SnapshotState { - const tabRect = params.tabRect; - if (!tabRect) return keyboardCoveredTabBarSnapshot(); - return makeSnapshotState( - keyboardCoveredTabBarSnapshot().nodes.map((node) => - node.index === 1 ? { ...node, rect: tabRect } : node, - ), - ); +function tappedDevice(snapshot: SnapshotState, calls: Point[]) { + return createInteractionDevice(snapshot, { + tap: async (_context, point) => { + calls.push(point); + return { ok: true }; + }, + }); } -/** - * Keyboard-owned rects hauled above the docking budget: the tree stops describing a keyboard the - * bottom of the screen belongs to, which is what the app-drawn-keypad and aim cases measure against. - */ -function liftKeyboardOffBottomEdge(nodes: SnapshotState['nodes']): SnapshotState['nodes'] { +/** The same tree with its keyboard hoisted above the docking budget, so the tree rule stops measuring. */ +function lifted(nodes: SnapshotState['nodes']): SnapshotState['nodes'] { return nodes.map((node) => node.rect && (node.type === 'Keyboard' || node.type === 'Key') ? { ...node, rect: { ...node.rect, y: node.rect.y - 560 } } @@ -42,18 +40,19 @@ function liftKeyboardOffBottomEdge(nodes: SnapshotState['nodes']): SnapshotState ); } -function tappedDevice(snapshot: SnapshotState, calls: Point[]) { - return createInteractionDevice(snapshot, { - tap: async (_context, point) => { - calls.push(point); - return { ok: true }; - }, - }); +function captured(params: { + nodes: SnapshotState['nodes']; + keyboard?: SnapshotKeyboardBandFact; +}): SnapshotState { + return makeSnapshotState( + params.nodes, + params.keyboard ? { keyboard: params.keyboard } : undefined, + ); } test('press @ref refuses a target whose center sits behind the visible keyboard', async () => { const calls: Point[] = []; - const device = tappedDevice(keyboardTree(), calls); + const device = tappedDevice(keyboardCoveredTabBarSnapshot(), calls); await assert.rejects( () => device.interactions.click(ref('@e2'), { session: 'default' }), @@ -64,8 +63,8 @@ test('press @ref refuses a target whose center sits behind the visible keyboard' const details = (error as { details?: Record }).details; assert.equal(details?.reason, 'tap_keyboard_occludes_target'); assert.equal(details?.ref, '@e2'); - assert.deepEqual(details?.rect, TAB_BAR_RECT); - assert.deepEqual(details?.keyboardFrame, { x: 0, y: 583, width: 402, height: 291 }); + assert.deepEqual(details?.rect, KEYBOARD_COVERED_TAB_BAR_RECT); + assert.deepEqual(details?.keyboardFrame, KEYBOARD_COVERED_BAND_RECT); assert.match(String(details?.hint), /End editing first/); return true; }, @@ -74,7 +73,7 @@ test('press @ref refuses a target whose center sits behind the visible keyboard' }); test('fill refuses with the verb matching the action', async () => { - const device = tappedDevice(keyboardTree(), []); + const device = tappedDevice(keyboardCoveredTabBarSnapshot(), []); await assert.rejects( () => device.interactions.fill(ref('@e2'), 'hello', { session: 'default' }), /cannot be filled safely/, @@ -83,7 +82,7 @@ test('fill refuses with the verb matching the action', async () => { test('press selector refuses the same node the ref path refuses', async () => { const calls: Point[] = []; - const device = tappedDevice(keyboardTree(), calls); + const device = tappedDevice(keyboardCoveredTabBarSnapshot(), calls); await assert.rejects( () => device.interactions.press(selector('label=Form'), { session: 'default' }), @@ -100,7 +99,7 @@ test('press selector refuses the same node the ref path refuses', async () => { test('a keyboard key is a legitimate target: pressing one still taps', async () => { const calls: Point[] = []; - const device = tappedDevice(keyboardTree(), calls); + const device = tappedDevice(keyboardCoveredTabBarSnapshot(), calls); await device.interactions.click(ref('@e6'), { session: 'default' }); @@ -109,8 +108,10 @@ test('a keyboard key is a legitimate target: pressing one still taps', async () test('an element whose center stays above the key plane still taps', async () => { const calls: Point[] = []; - const peeking = { x: 148, y: 470, width: 104, height: 110 }; - const device = tappedDevice(keyboardTree({ tabRect: peeking }), calls); + const device = tappedDevice( + keyboardCoveredTabBarSnapshot({ targetRect: { x: 148, y: 470, width: 104, height: 110 } }), + calls, + ); await device.interactions.click(ref('@e2'), { session: 'default' }); @@ -120,7 +121,9 @@ test('an element whose center stays above the key plane still taps', async () => test('no keyboard in the tree means nothing to refuse', async () => { const calls: Point[] = []; const withoutKeyboard = makeSnapshotState( - keyboardTree().nodes.filter((node) => node.type !== 'Keyboard' && node.type !== 'Key'), + keyboardCoveredTabBarSnapshot().nodes.filter( + (node) => node.type !== 'Keyboard' && node.type !== 'Key', + ), ); const device = tappedDevice(withoutKeyboard, calls); @@ -131,8 +134,7 @@ test('no keyboard in the tree means nothing to refuse', async () => { test('an app-drawn keypad that stops short of the bottom edge is not the system keyboard', async () => { const calls: Point[] = []; - const appOwnedKeypad = makeSnapshotState(liftKeyboardOffBottomEdge(keyboardTree().nodes)); - const device = tappedDevice(appOwnedKeypad, calls); + const device = tappedDevice(keyboardCoveredTabBarSnapshot({ keyboardLifted: true }), calls); await device.interactions.click(ref('@e2'), { session: 'default' }); @@ -141,13 +143,11 @@ test('an app-drawn keypad that stops short of the bottom edge is not the system test('a coordinate behind the keyboard taps anyway and discloses the reason', async () => { const calls: Point[] = []; - const device = tappedDevice(keyboardTree(), calls); + const device = tappedDevice(keyboardCoveredTabBarSnapshot(), calls); const result = await device.interactions.press( { kind: 'point', x: 200, y: 810 }, - { - session: 'default', - }, + { session: 'default' }, ); assert.deepEqual(calls, [{ x: 200, y: 810 }]); @@ -158,27 +158,26 @@ test('a coordinate behind the keyboard taps anyway and discloses the reason', as test('the guard reads the point the interaction dispatches, not the rect center', async () => { // The Form button's own center sits 3 pt above the key plane, and its interactive child owns that // upper region, so the point the tap dispatches is pushed down into the keyboard's band. - const aimShifted = makeSnapshotState([ - ...keyboardCoveredTabBarSnapshot().nodes, - { - index: 5, - depth: 3, - parentIndex: 1, - type: 'Button', - label: 'Send', - rect: { x: 148, y: 500, width: 104, height: 83 }, - hittable: true, - }, - ]).nodes.map((node) => - node.index === 1 ? { ...node, rect: { x: 148, y: 500, width: 104, height: 160 } } : node, + const aimShifted = makeSnapshotState( + [ + ...keyboardCoveredTabBarSnapshot().nodes, + { + index: 7, + depth: 3, + parentIndex: 1, + type: 'Button', + label: 'Send', + rect: { x: 148, y: 500, width: 104, height: 83 }, + hittable: true, + }, + ].map((node) => + node.index === 1 ? { ...node, rect: { x: 148, y: 500, width: 104, height: 160 } } : node, + ), ); const calls: Point[] = []; await assert.rejects( - () => - tappedDevice(makeSnapshotState(aimShifted), calls).interactions.click(ref('@e2'), { - session: 'default', - }), + () => tappedDevice(aimShifted, calls).interactions.click(ref('@e2'), { session: 'default' }), /Ref @e2 is behind the visible keyboard/, ); assert.deepEqual(calls, []); @@ -186,23 +185,21 @@ test('the guard reads the point the interaction dispatches, not the rect center' // The same tree with its keyboard hauled off the bottom edge shows where that tap was aiming: below // the key plane, which is what makes this the dispatched point's refusal rather than the center's. const aim: Point[] = []; - await tappedDevice( - makeSnapshotState(liftKeyboardOffBottomEdge(aimShifted)), - aim, - ).interactions.click(ref('@e2'), { session: 'default' }); + await tappedDevice(makeSnapshotState(lifted(aimShifted.nodes)), aim).interactions.click( + ref('@e2'), + { session: 'default' }, + ); assert.equal(aim.length, 1); assert.ok((aim[0]?.y ?? 0) > 583, `expected the dispatched point below 583, got ${aim[0]?.y}`); }); test('a coordinate on a reported key is the keyboard the caller asked for', async () => { const calls: Point[] = []; - const device = tappedDevice(keyboardTree(), calls); + const device = tappedDevice(keyboardCoveredTabBarSnapshot(), calls); const result = await device.interactions.press( { kind: 'point', x: 160, y: 760 }, - { - session: 'default', - }, + { session: 'default' }, ); assert.deepEqual(calls, [{ x: 160, y: 760 }]); @@ -210,42 +207,22 @@ test('a coordinate on a reported key is the keyboard the caller asked for', asyn }); test('the out-of-viewport disclosure still wins on a stale-tree coordinate', async () => { - const device = tappedDevice(keyboardTree(), []); + const device = tappedDevice(keyboardCoveredTabBarSnapshot(), []); const result = await device.interactions.press( { kind: 'point', x: 900, y: 810 }, - { - session: 'default', - }, + { session: 'default' }, ); assert.match(result.warning ?? '', /outside the last-known viewport/); }); -// The producer-measured band (#2660). These run the acting paths, because the guarantee is that the -// fast path and the shared rule enter through the same door — a unit test on the resolver alone would -// leave the ref path free to consult the tree again on its way to the tap. - -/** The runner's own reading of the #2589 fixture's keyboard, in the app's orientation space. */ -const MEASURED_BAND: SnapshotKeyboardBandFact = { - kind: 'visible', - frame: { x: 0, y: 583, width: 402, height: 291 }, -}; - -function captured(params: { - nodes: SnapshotState['nodes']; - keyboard?: SnapshotKeyboardBandFact; -}): SnapshotState { - return makeSnapshotState( - params.nodes, - params.keyboard ? { keyboard: params.keyboard } : undefined, - ); -} +// The producer-measured band (#2660). These run the acting paths for the same reason as above. test('a measured band refuses the target the tree rule would have refused', async () => { const calls: Point[] = []; const device = tappedDevice( - captured({ nodes: keyboardTree().nodes, keyboard: MEASURED_BAND }), + captured({ nodes: keyboardCoveredTabBarSnapshot().nodes, keyboard: MEASURED_BAND }), calls, ); @@ -256,12 +233,10 @@ test('a measured band refuses the target the tree rule would have refused', asyn (error as { details?: Record }).details?.reason, 'tap_keyboard_occludes_target', ); - assert.deepEqual((error as { details?: Record }).details?.keyboardFrame, { - x: 0, - y: 583, - width: 402, - height: 291, - }); + assert.deepEqual( + (error as { details?: Record }).details?.keyboardFrame, + KEYBOARD_COVERED_BAND_RECT, + ); return true; }, ); @@ -272,16 +247,18 @@ test('the tree rule is not consulted once the capture published a band', async ( // The same screen with its keyboard hoisted off the bottom edge: the tree rule refuses to measure // that geometry and lets the tap through, which is what makes this pair the assertion that the // band came from the fact and not from the tree. - const lifted = makeSnapshotState(liftKeyboardOffBottomEdge(keyboardTree().nodes)); + const liftedNodes = keyboardCoveredTabBarSnapshot({ keyboardLifted: true }).nodes; const withoutFact: Point[] = []; - await tappedDevice(lifted, withoutFact).interactions.click(ref('@e2'), { session: 'default' }); + await tappedDevice(makeSnapshotState(liftedNodes), withoutFact).interactions.click(ref('@e2'), { + session: 'default', + }); assert.equal(withoutFact.length, 1, 'the tree rule fails open on geometry it cannot measure'); const withFact: Point[] = []; await assert.rejects( () => tappedDevice( - captured({ nodes: lifted.nodes, keyboard: MEASURED_BAND }), + captured({ nodes: liftedNodes, keyboard: MEASURED_BAND }), withFact, ).interactions.click(ref('@e2'), { session: 'default' }), /Ref @e2 is behind the visible keyboard/, @@ -291,9 +268,13 @@ test('the tree rule is not consulted once the capture published a band', async ( test('a target above the measured band still presses', async () => { const calls: Point[] = []; - const above = { x: 148, y: 470, width: 104, height: 110 }; const device = tappedDevice( - captured({ nodes: keyboardTree({ tabRect: above }).nodes, keyboard: MEASURED_BAND }), + captured({ + nodes: keyboardCoveredTabBarSnapshot({ + targetRect: { x: 148, y: 470, width: 104, height: 110 }, + }).nodes, + keyboard: MEASURED_BAND, + }), calls, ); @@ -305,7 +286,7 @@ test('a target above the measured band still presses', async () => { test('a measured band still excuses the keyboard key the caller named', async () => { const calls: Point[] = []; const device = tappedDevice( - captured({ nodes: keyboardTree().nodes, keyboard: MEASURED_BAND }), + captured({ nodes: keyboardCoveredTabBarSnapshot().nodes, keyboard: MEASURED_BAND }), calls, ); @@ -317,7 +298,7 @@ test('a measured band still excuses the keyboard key the caller named', async () test('a measured band still reads a coordinate on a reported key as the keyboard asked for', async () => { const calls: Point[] = []; const result = await tappedDevice( - captured({ nodes: keyboardTree().nodes, keyboard: MEASURED_BAND }), + captured({ nodes: keyboardCoveredTabBarSnapshot().nodes, keyboard: MEASURED_BAND }), calls, ).interactions.press({ kind: 'point', x: 160, y: 760 }, { session: 'default' }); @@ -328,7 +309,7 @@ test('a measured band still reads a coordinate on a reported key as the keyboard test('a producer that found no keyboard lets the tap the stale tree would refuse', async () => { const calls: Point[] = []; const device = tappedDevice( - captured({ nodes: keyboardTree().nodes, keyboard: { kind: 'absent' } }), + captured({ nodes: keyboardCoveredTabBarSnapshot().nodes, keyboard: { kind: 'absent' } }), calls, ); @@ -341,7 +322,7 @@ test('an unmeasurable fact leaves the acting path on the tree rule unchanged', a const calls: Point[] = []; const device = tappedDevice( captured({ - nodes: keyboardTree().nodes, + nodes: keyboardCoveredTabBarSnapshot().nodes, keyboard: { kind: 'unmeasurable', reason: 'keyboard-frame-query-timeout' }, }), calls, @@ -354,10 +335,10 @@ test('an unmeasurable fact leaves the acting path on the tree rule unchanged', a assert.deepEqual(calls, []); }); -test("a coordinate behind the measured band discloses the producer's frame", async () => { +test('a coordinate behind the measured band discloses the producer frame', async () => { const result = await tappedDevice( captured({ - nodes: liftKeyboardOffBottomEdge(keyboardTree().nodes), + nodes: keyboardCoveredTabBarSnapshot({ keyboardLifted: true }).nodes, keyboard: MEASURED_BAND, }), [], diff --git a/src/commands/interaction/runtime/target-visibility-stages.ts b/src/commands/interaction/runtime/target-visibility-stages.ts index 9da8548af5..c8f16ab15c 100644 --- a/src/commands/interaction/runtime/target-visibility-stages.ts +++ b/src/commands/interaction/runtime/target-visibility-stages.ts @@ -20,9 +20,11 @@ import { type OffscreenScrollDirection, } from '@agent-device/capture-kit/mobile-snapshot-semantics'; import { toBackendContext } from '../../runtime-common.ts'; -import { assertTapTargetClearOfVisibleKeyboard } from './keyboard-occlusion.ts'; -import { interactionVerb } from './interaction-verb.ts'; -import type { InteractionAction } from './interaction-resolution-request.ts'; +import { assertTapTargetClearOfVisibleKeyboard } from '@agent-device/selectors/keyboard-occlusion'; +import { + interactionVerb, + type InteractionAction, +} from '@agent-device/selectors/interaction-action'; /** * The one construction site for "covered by another visible element" refusals, shared by the diff --git a/test/integration/interaction-contract/find-focus-dispatch.contract.test.ts b/test/integration/interaction-contract/find-focus-dispatch.contract.test.ts new file mode 100644 index 0000000000..cab6bebe94 --- /dev/null +++ b/test/integration/interaction-contract/find-focus-dispatch.contract.test.ts @@ -0,0 +1,188 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { InteractionGuarantee } from '@agent-device/contracts/interaction-guarantees'; +import { keyboardCoveredTabBarSnapshot } from '@agent-device/test-kit'; +import { assertRpcError, assertRpcOk } from '../provider-scenarios/assertions.ts'; +import { PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS } from '../provider-scenarios/test-timeouts.ts'; +import { scenarioName, scenarioNames } from './coverage-manifest.ts'; +import { FIND_FOCUS_DISPATCH_COVERAGE } from './find-focus-dispatch.coverage.ts'; +import { runnerSnapshotEntry, runnerTapEntry, withIosContractDaemon } from './daemon-harness.ts'; + +// ADR 0011 Layer 3, find-focus-dispatch path: mutating `find focus` and the focus leg of +// `find type`. These run over the daemon harness because the seam lives in the daemon's +// find handler — it captures its own tree and dispatches the point itself, so no in-process device +// harness can reach it. The transcript is the proof: a refusal that dispatched anyway would either +// consume an unscripted tap entry or leave a scripted one behind. + +const scenario = (guarantee: InteractionGuarantee): string => + scenarioName(FIND_FOCUS_DISPATCH_COVERAGE, guarantee); + +const KEYBOARD_TREE = keyboardCoveredTabBarSnapshot().nodes; + +test( + scenario('keyboardOcclusion'), + async () => { + // One scripted capture and no tap entry: `assertComplete` is what proves the refusal stopped the + // dispatch, which is #2622's whole point. + await withIosContractDaemon([runnerSnapshotEntry(KEYBOARD_TREE)], async (daemon) => { + const error = assertRpcError( + await daemon.callCommand('find', ['label', 'Form', 'focus']), + 'COMMAND_FAILED', + /Ref @e2 is behind the visible keyboard and cannot be focused safely/, + ); + const details = readErrorDetails(error); + assert.equal(details.reason, 'tap_keyboard_occludes_target'); + assert.equal(details.ref, '@e2'); + assert.deepEqual(details.rect, { x: 148, y: 791, width: 104, height: 83 }); + assert.deepEqual(details.keyboardFrame, { x: 0, y: 583, width: 402, height: 291 }); + assert.equal(details.dispatched, 'no'); + }); + }, + PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS, +); + +test( + scenarioNames(FIND_FOCUS_DISPATCH_COVERAGE, 'keyboardOcclusion')[1]!, + async () => { + // The closest negative: the same tree with the app's button lifted above the band. + const aboveBand = KEYBOARD_TREE.map((node) => + node.index === 1 ? { ...node, rect: { x: 148, y: 470, width: 104, height: 110 } } : node, + ); + + await withIosContractDaemon( + [runnerSnapshotEntry(aboveBand), runnerTapEntry({})], + async (daemon) => { + const data = assertRpcOk(await daemon.callCommand('find', ['label', 'Form', 'focus'])); + assert.deepEqual({ x: data.x, y: data.y }, { x: 200, y: 525 }); + }, + ); + }, + PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS, +); + +test( + scenario('occlusion'), + async () => { + await withIosContractDaemon([runnerSnapshotEntry(coveredFormTree())], async (daemon) => { + const error = assertRpcError( + await daemon.callCommand('find', ['label', 'Form', 'focus']), + 'COMMAND_FAILED', + /Matched element @e2 is covered by another visible element and cannot be focused safely/, + ); + assert.equal(readErrorDetails(error).reason, 'target_covered'); + }); + }, + PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS, +); + +test( + scenario('disambiguation'), + async () => { + await withIosContractDaemon( + [ + runnerSnapshotEntry(twinFormTree()), + runnerSnapshotEntry(twinFormTree()), + runnerTapEntry({}), + ], + async (daemon) => { + const ambiguous = assertRpcError( + await daemon.callCommand('find', ['label', 'Form', 'focus']), + 'AMBIGUOUS_MATCH', + /find matched 2 elements for label "Form"/, + ); + assert.deepEqual(readErrorDetails(ambiguous).candidates, [ + '@e2 [button] "Form"', + '@e3 [button] "Form"', + ]); + + // --first is the only narrowing find offers, and the narrowed match goes through this seam. + assertRpcOk( + await daemon.callCommand('find', ['label', 'Form', 'focus'], { findFirst: true }), + ); + }, + ); + }, + PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS, +); + +test( + scenario('responseConstruction'), + async () => { + await withIosContractDaemon( + [runnerSnapshotEntry(KEYBOARD_TREE), runnerTapEntry({})], + async (daemon) => { + const data = assertRpcOk(await daemon.callCommand('find', ['label', 'space', 'focus'])); + // The focus runtime's own payload: the point it dispatched plus its success text, with nothing + // hand-rolled on top — and no `refsGeneration`, which is what would make the ref pinnable + // (ADR 0014). + assert.equal(data.message, 'Focused (202, 779)'); + assert.equal(data.refsGeneration, undefined); + }, + ); + }, + PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS, +); + +test( + scenario('errorTaxonomy'), + async () => { + await withIosContractDaemon([runnerSnapshotEntry(KEYBOARD_TREE)], async (daemon) => { + assertRpcError( + await daemon.callCommand('find', ['label', 'Nowhere', 'focus']), + 'COMMAND_FAILED', + /find did not match any element/, + ); + }); + }, + PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS, +); + +/** The wire's error envelope keeps the typed refusal details one level below its code and hint. */ +function readErrorDetails(error: Record): Record { + const details = error.details; + assert.ok(details && typeof details === 'object', 'the error carries typed details'); + return details as Record; +} + +function coveredFormTree(): unknown[] { + return [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'Button', + label: 'Form', + rect: { x: 16, y: 790, width: 140, height: 44 }, + hittable: true, + }, + { + index: 2, + parentIndex: 0, + type: 'TabBar', + rect: { x: 0, y: 760, width: 390, height: 84 }, + hittable: true, + }, + ]; +} + +function twinFormTree(): unknown[] { + return [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 400, height: 800 } }, + { + index: 1, + parentIndex: 0, + type: 'Button', + label: 'Form', + rect: { x: 40, y: 200, width: 120, height: 44 }, + hittable: true, + }, + { + index: 2, + parentIndex: 0, + type: 'Button', + label: 'Form', + rect: { x: 240, y: 200, width: 120, height: 44 }, + hittable: true, + }, + ]; +} diff --git a/test/integration/interaction-contract/find-focus-dispatch.coverage.ts b/test/integration/interaction-contract/find-focus-dispatch.coverage.ts new file mode 100644 index 0000000000..9b92cc3890 --- /dev/null +++ b/test/integration/interaction-contract/find-focus-dispatch.coverage.ts @@ -0,0 +1,15 @@ +import { definePathCoverage } from './coverage-manifest.ts'; + +export const FIND_FOCUS_DISPATCH_COVERAGE = definePathCoverage('find-focus-dispatch', { + disambiguation: + 'find-focus-dispatch disambiguation: two matches refuse with bounded candidates unless --first narrows', + occlusion: 'find-focus-dispatch occlusion: a covered match refuses to be focused', + keyboardOcclusion: [ + 'find-focus-dispatch keyboardOcclusion: a focus match behind the visible keyboard is refused with tap_keyboard_occludes_target', + 'find-focus-dispatch keyboardOcclusion: the same match above the keyboard band focuses', + ], + responseConstruction: + 'find-focus-dispatch responseConstruction: find focus returns the shared point-focus payload', + errorTaxonomy: + 'find-focus-dispatch errorTaxonomy: an unmatched find focus fails with find locator diagnostics', +}); diff --git a/test/integration/interaction-contract/fixtures.ts b/test/integration/interaction-contract/fixtures.ts index 378505b9bd..04144e7cc9 100644 --- a/test/integration/interaction-contract/fixtures.ts +++ b/test/integration/interaction-contract/fixtures.ts @@ -399,72 +399,6 @@ export const RUNNER_NON_HITTABLE_TEXT_INPUT_NODES = [ ] as const; // #2589 shape, recorded from the iOS repro: the bottom tab bar sits behind the system keyboard. -// The keyboard is its own system surface, so it is never a covering sibling of app content here -// (`occlusion` stays silent) and the tab bar is still inside the app's own window rect -// (`offscreen` passes) — the tap used to report success while the key ate the touch. -// The key rects are the bottom row measured on iPhone 17 Pro (26.2): a keyboard the guard will -// measure has to report one unbroken run of columns, so a reduced layout cannot stop at two keys. -export function keyboardCoveredTabBarSnapshot(): SnapshotState { - return makeSnapshotState([ - { - index: 0, - depth: 0, - type: 'Application', - rect: { x: 0, y: 0, width: 402, height: 874 }, - hittable: true, - }, - { - index: 1, - depth: 2, - parentIndex: 0, - type: 'Button', - label: 'Form', - rect: { x: 148, y: 791, width: 104, height: 83 }, - hittable: true, - }, - { - index: 2, - depth: 1, - parentIndex: 0, - type: 'Keyboard', - rect: { x: 0, y: 583, width: 402, height: 291 }, - hittable: false, - }, - { - index: 3, - depth: 2, - parentIndex: 2, - type: 'Key', - label: 'globe', - rect: { x: 4.67, y: 752, width: 49.33, height: 54 }, - hittable: true, - }, - { - index: 4, - depth: 2, - parentIndex: 2, - type: 'Key', - label: '.?123', - rect: { x: 54, y: 752, width: 49.33, height: 54 }, - hittable: true, - }, - { - index: 5, - depth: 2, - parentIndex: 2, - type: 'Key', - label: 'space', - rect: { x: 103.33, y: 752, width: 197.33, height: 54 }, - hittable: true, - }, - { - index: 6, - depth: 2, - parentIndex: 2, - type: 'Key', - label: 'return', - rect: { x: 300.67, y: 752, width: 99, height: 54 }, - hittable: true, - }, - ]); -} +// The tree is the shared `@agent-device/test-kit` fixture — the daemon's find focus seam and the +// selectors guard test are proven against this same shape. +export { keyboardCoveredTabBarSnapshot } from '@agent-device/test-kit'; diff --git a/test/integration/interaction-contract/index.ts b/test/integration/interaction-contract/index.ts index 436b76bc18..bf9992036c 100644 --- a/test/integration/interaction-contract/index.ts +++ b/test/integration/interaction-contract/index.ts @@ -1,5 +1,6 @@ import type { ContractCoverageEntry } from './coverage-manifest.ts'; import { COORDINATE_COVERAGE } from './coordinate.coverage.ts'; +import { FIND_FOCUS_DISPATCH_COVERAGE } from './find-focus-dispatch.coverage.ts'; import { MAESTRO_DIRECT_SELECTOR_COVERAGE } from './maestro-direct-selector.coverage.ts'; import { MAESTRO_FALLBACK_COVERAGE } from './maestro-fallback.coverage.ts'; import { NATIVE_REF_COVERAGE } from './native-ref.coverage.ts'; @@ -19,6 +20,7 @@ export const CONTRACT_COVERAGE: readonly ContractCoverageEntry[] = [ ...TARGET_DRAG_COVERAGE, ...NATIVE_REF_COVERAGE, ...COORDINATE_COVERAGE, + ...FIND_FOCUS_DISPATCH_COVERAGE, ...MAESTRO_DIRECT_SELECTOR_COVERAGE, ...MAESTRO_FALLBACK_COVERAGE, ]; diff --git a/test/repo-guards/interaction-guarantees.test.ts b/test/repo-guards/interaction-guarantees.test.ts index e50ccaad5d..e2442fce1c 100644 --- a/test/repo-guards/interaction-guarantees.test.ts +++ b/test/repo-guards/interaction-guarantees.test.ts @@ -164,6 +164,11 @@ test('acknowledged gaps are visible and bounded', () => { // (umbrella: https://github.com/callstack/agent-device/issues/1081). assert.deepEqual(gaps.sort(), [ 'coordinate/outcomeObservation', + 'find-focus-dispatch/nonHittable', + 'find-focus-dispatch/offscreen', + 'find-focus-dispatch/outcomeObservation', + 'find-focus-dispatch/parentOwnedTouchPoint', + 'find-focus-dispatch/targetStability', 'maestro-direct-selector/errorTaxonomy', 'maestro-direct-selector/outcomeObservation', 'maestro-direct-selector/responseIdentity',