From b4413ecd510bd27ed293ed93cba4e45862738e6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 16:00:33 +0200 Subject: [PATCH 1/8] refactor(move): relocate the keyboard guard to @agent-device/selectors The #2589 guard and the action vocabulary it needs (`InteractionAction`, `interactionVerb`) lived in the command surface, so the daemon's mutating-`find` focus seam could not reach either. Both move to @agent-device/selectors, the package the selector pipeline and the daemon dispatch seams already share, and the guard's call sites now import it from there. `keyboardCoveredTabBarSnapshot` moves to @agent-device/test-kit with the guard: the daemon seam test, the guard test, and the ADR 0011 contract scenarios need one copy of the #2589 capture. Its variants (a moved app target, a keyboard lifted off the bottom edge) replace the copies those suites were about to grow. --- .../__tests__/snapshot-presentation.test.ts | 2 +- packages/selectors/package.json | 8 + packages/selectors/src/interaction-action.ts | 41 ++++ .../selectors/src/keyboard-occlusion.test.ts | 145 ++++++++++++ .../selectors/src}/keyboard-occlusion.ts | 7 +- packages/test-kit/src/index.ts | 5 + .../src/keyboard-occlusion.fixtures.ts | 98 ++++++++ .../runtime/interaction-resolution-request.ts | 17 +- .../interaction/runtime/interaction-verb.ts | 20 -- .../runtime/native-ref-interaction.ts | 2 +- .../runtime/resolution-disclosure.ts | 2 +- .../interaction/runtime/resolution.ts | 8 +- .../runtime/selector-target-resolution.ts | 6 +- ...ibility-stages-keyboard-occlusion.test.ts} | 211 ++++++++---------- .../runtime/target-visibility-stages.ts | 8 +- .../interaction-contract/fixtures.ts | 72 +----- 16 files changed, 415 insertions(+), 237 deletions(-) create mode 100644 packages/selectors/src/interaction-action.ts create mode 100644 packages/selectors/src/keyboard-occlusion.test.ts rename {src/commands/interaction/runtime => packages/selectors/src}/keyboard-occlusion.ts (93%) create mode 100644 packages/test-kit/src/keyboard-occlusion.fixtures.ts delete mode 100644 src/commands/interaction/runtime/interaction-verb.ts rename src/commands/interaction/runtime/{keyboard-occlusion.test.ts => target-visibility-stages-keyboard-occlusion.test.ts} (65%) diff --git a/packages/platform-apple/src/runner/__tests__/snapshot-presentation.test.ts b/packages/platform-apple/src/runner/__tests__/snapshot-presentation.test.ts index 974a4e265e..a74d9620a9 100644 --- a/packages/platform-apple/src/runner/__tests__/snapshot-presentation.test.ts +++ b/packages/platform-apple/src/runner/__tests__/snapshot-presentation.test.ts @@ -251,7 +251,7 @@ test('a runner payload with the hittable bit absent presents without declaring i // what cannot be placed belongs to `readSnapshotKeyboardBandFact` in @agent-device/kernel, which owns // that table; this seam owns forwarding AND the proof that it forwards through that reader. The // same-process capture path feeds `result.keyboard` straight into the tap/click occlusion guard in -// src/commands/interaction/runtime/keyboard-occlusion.ts before any serialization, so a seam that +// packages/selectors/src/keyboard-occlusion.ts before any serialization, so a seam that // passed the raw wire value through would hand the guard an unvalidated shape with nothing red. // The malformed cases below are what prove the routing; kernel/src/record.test.ts owns the table. diff --git a/packages/selectors/package.json b/packages/selectors/package.json index 972a6c0f4a..d9bdcad640 100644 --- a/packages/selectors/package.json +++ b/packages/selectors/package.json @@ -40,6 +40,10 @@ "types": "./src/absence-observation-resolution.ts", "default": "./src/absence-observation-resolution.ts" }, + "./interaction-action": { + "types": "./src/interaction-action.ts", + "default": "./src/interaction-action.ts" + }, "./interaction-error": { "types": "./src/interaction-error.ts", "default": "./src/interaction-error.ts" @@ -56,6 +60,10 @@ "types": "./src/interaction-touch-point.ts", "default": "./src/interaction-touch-point.ts" }, + "./keyboard-occlusion": { + "types": "./src/keyboard-occlusion.ts", + "default": "./src/keyboard-occlusion.ts" + }, "./press-retarget": { "types": "./src/press-retarget.ts", "default": "./src/press-retarget.ts" diff --git a/packages/selectors/src/interaction-action.ts b/packages/selectors/src/interaction-action.ts new file mode 100644 index 0000000000..4248f76589 --- /dev/null +++ b/packages/selectors/src/interaction-action.ts @@ -0,0 +1,41 @@ +/** + * The action an interaction performs, and the passive verb a pre-dispatch refusal uses for it. + * + * Living below both the command surface and the daemon's dispatch seams keeps one vocabulary for + * every guard that refuses before dispatch, so a rule shared by the acting paths and a route that + * dispatches on its own (`find focus`) cannot name the same failure two ways. + */ +export type InteractionAction = + | 'click' + | 'press' + | 'fill' + | 'focus' + | 'longPress' + | 'hover' + | 'scroll' + | 'swipe' + | 'pinch' + | 'pan' + | 'drag' + | 'fling' + | 'rotate' + | 'transform'; + +/** + * The passive verb a refusal uses for the action it declined, so every acting guard that refuses + * before dispatch describes the failure in the caller's own words. + */ +export function interactionVerb(action: InteractionAction): string { + switch (action) { + case 'fill': + return 'be filled'; + case 'focus': + return 'be focused'; + case 'longPress': + return 'be long-pressed'; + case 'hover': + return 'be hovered'; + default: + return 'be tapped'; + } +} diff --git a/packages/selectors/src/keyboard-occlusion.test.ts b/packages/selectors/src/keyboard-occlusion.test.ts new file mode 100644 index 0000000000..20107b4595 --- /dev/null +++ b/packages/selectors/src/keyboard-occlusion.test.ts @@ -0,0 +1,145 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import type { + Point, + Rect, + SnapshotAddressableNode, + SnapshotKeyboardBandFact, + SnapshotState, +} from '@agent-device/kernel/snapshot'; +import { TAP_KEYBOARD_OCCLUDES_TARGET_DETAILS } from '@agent-device/contracts/tap-keyboard-occlusion'; +import { + KEYBOARD_COVERED_BAND_RECT, + KEYBOARD_COVERED_TAB_BAR_RECT, + keyboardCoveredTabBarSnapshot, +} from '@agent-device/test-kit'; +import { + assertTapTargetClearOfVisibleKeyboard, + describeKeyboardOccludedPointWarning, +} from './keyboard-occlusion.ts'; + +// The guard's own two contributions: the refusal it raises for a decision the classifier already +// made, and the disclosure string it composes. Which geometry decides is the classifier's suite +// (packages/contracts/src/tap-keyboard-occlusion.test.ts, golden-table-driven), and which doors run +// the guard is proven where each door lives — the interaction leaf's in +// src/commands/interaction/runtime/target-visibility-stages-keyboard-occlusion.test.ts, the +// mutating-`find` seam's in packages/daemon/src/execution/interaction/internal/__tests__/. Every one +// of those suites measures this same #2589 capture. + +function centerOf(rect: Rect): Point { + return { x: rect.x + rect.width / 2, y: rect.y + rect.height / 2 }; +} + +function addressableNode(snapshot: SnapshotState, index: number): SnapshotAddressableNode { + const node = snapshot.nodes.find((candidate) => candidate.index === index); + assert.ok(node && 'ref' in node && node.ref, `expected node ${index} to carry a ref`); + return node as SnapshotAddressableNode; +} + +/** + * The guard as a caller runs it: against the node it resolved, at the point it would tap. Omitting + * `tapPoint` taps that node's own center. + */ +function guard(params: { + snapshot: SnapshotState; + index?: number; + action?: 'click' | 'fill' | 'focus'; + label?: string; + keyboard?: SnapshotKeyboardBandFact; + tapPoint?: Point | null; +}): void { + const node = addressableNode(params.snapshot, params.index ?? 1); + assertTapTargetClearOfVisibleKeyboard({ + nodes: params.snapshot.nodes, + node, + action: params.action ?? 'click', + label: params.label ?? 'Ref @e2', + ...(params.keyboard ? { keyboard: params.keyboard } : {}), + tapPoint: + params.tapPoint === undefined && node.rect ? centerOf(node.rect) : (params.tapPoint ?? null), + }); +} + +/** Runs the guard expecting a refusal, and hands back that refusal's typed details. */ +function keyboardRefusalDetails(params: Parameters[0]) { + let details: Record | undefined; + assert.throws( + () => guard(params), + (error: unknown) => { + assert.ok(error instanceof Error); + assert.equal((error as { code?: string }).code, 'COMMAND_FAILED'); + assert.match(error.message, /Ref @e2 is behind the visible keyboard/); + details = (error as { details?: Record }).details; + return true; + }, + ); + assert.ok(details, 'the refusal carries details'); + return details; +} + +test('the refusal carries the shared reason, the target, and the band it measured', () => { + assert.deepEqual(keyboardRefusalDetails({ snapshot: keyboardCoveredTabBarSnapshot() }), { + ...TAP_KEYBOARD_OCCLUDES_TARGET_DETAILS, + ref: '@e2', + rect: KEYBOARD_COVERED_TAB_BAR_RECT, + keyboardFrame: KEYBOARD_COVERED_BAND_RECT, + dispatched: 'no', + }); +}); + +test('the refusal names the action it declined', () => { + for (const [action, verb] of [ + ['click', 'be tapped'], + ['fill', 'be filled'], + ['focus', 'be focused'], + ] as const) { + assert.throws( + () => guard({ snapshot: keyboardCoveredTabBarSnapshot(), action }), + new RegExp(`cannot ${verb} safely`), + ); + } +}); + +test('the refusal quotes how the caller named the target', () => { + assert.throws( + () => guard({ snapshot: keyboardCoveredTabBarSnapshot(), label: 'Selector label=Form' }), + /Selector label=Form is behind the visible keyboard/, + ); +}); + +test('a target with no measurable point cannot be refused', () => { + guard({ snapshot: keyboardCoveredTabBarSnapshot(), tapPoint: null }); +}); + +// The coordinate path discloses the same decision instead of refusing it, in one string the +// response publishes. + +function pointWarning(params: { + snapshot: SnapshotState; + point: Point; + keyboard?: SnapshotKeyboardBandFact; +}): string | undefined { + return describeKeyboardOccludedPointWarning({ + nodes: params.snapshot.nodes, + point: params.point, + viewport: params.snapshot.nodes[0]!.rect ?? null, + ...(params.keyboard ? { keyboard: params.keyboard } : {}), + }); +} + +test('a coordinate behind the keyboard discloses the point, reason, and band', () => { + const warning = pointWarning({ + snapshot: keyboardCoveredTabBarSnapshot(), + point: { x: 200, y: 810 }, + }); + assert.match(warning ?? '', /\(200, 810\) is behind the visible keyboard/); + assert.match(warning ?? '', /tap_keyboard_occludes_target/); + assert.match(warning ?? '', /keyboard frame 583\.\.874/); +}); + +test('a coordinate on a reported key discloses nothing', () => { + assert.equal( + pointWarning({ snapshot: keyboardCoveredTabBarSnapshot(), point: { x: 160, y: 760 } }), + undefined, + ); +}); diff --git a/src/commands/interaction/runtime/keyboard-occlusion.ts b/packages/selectors/src/keyboard-occlusion.ts similarity index 93% rename from src/commands/interaction/runtime/keyboard-occlusion.ts rename to packages/selectors/src/keyboard-occlusion.ts index bd45fa0bda..e28efc89aa 100644 --- a/src/commands/interaction/runtime/keyboard-occlusion.ts +++ b/packages/selectors/src/keyboard-occlusion.ts @@ -13,8 +13,7 @@ import { TAP_KEYBOARD_OCCLUDES_TARGET_DETAILS, type KeyboardSurface, } from '@agent-device/contracts/tap-keyboard-occlusion'; -import { interactionVerb } from './interaction-verb.ts'; -import type { InteractionAction } from './interaction-resolution-request.ts'; +import { interactionVerb, type InteractionAction } from './interaction-action.ts'; /** * The keyboard guard every acting path runs against the tree it already holds. @@ -24,6 +23,10 @@ import type { InteractionAction } from './interaction-resolution-request.ts'; * beside the off-screen guard rather than inside any one command keeps the fast path honest: the * native-ref preflight enters through the same door as selector and `@ref` resolution, so a backend * that "succeeds" cannot do so on a target the shared rule would refuse (ADR 0011). + * + * The rule sits below the command surface and the daemon's dispatch seams because both zones call + * it: the interaction leaf through `runInteractionPipelineStages`, and mutating `find focus`/`type`, + * which dispatch a point themselves instead of re-entering that leaf (#2622). */ /** diff --git a/packages/test-kit/src/index.ts b/packages/test-kit/src/index.ts index b24974cd52..8794236629 100644 --- a/packages/test-kit/src/index.ts +++ b/packages/test-kit/src/index.ts @@ -62,6 +62,11 @@ export { TWO_ACTIONABLE_WRAPPER_CHAIN_NODES, UNVERIFIED_HITTABILITY_WRAPPER_CHAIN_NODES, } from './interaction-targeting.fixtures.ts'; +export { + KEYBOARD_COVERED_BAND_RECT, + KEYBOARD_COVERED_TAB_BAR_RECT, + keyboardCoveredTabBarSnapshot, +} from './keyboard-occlusion.fixtures.ts'; export { closeLoopbackServer, listenOnLoopback, diff --git a/packages/test-kit/src/keyboard-occlusion.fixtures.ts b/packages/test-kit/src/keyboard-occlusion.fixtures.ts new file mode 100644 index 0000000000..9b2cf6e741 --- /dev/null +++ b/packages/test-kit/src/keyboard-occlusion.fixtures.ts @@ -0,0 +1,98 @@ +import { makeSnapshotState } from '@agent-device/capture-kit/snapshot-state-fixtures'; +import type { Rect, SnapshotState } from '@agent-device/kernel/snapshot'; + +/** The #2589 capture's app-owned tab bar, and the band the captured keyboard docks to. */ +export const KEYBOARD_COVERED_TAB_BAR_RECT: Rect = { x: 148, y: 791, width: 104, height: 83 }; +export const KEYBOARD_COVERED_BAND_RECT: Rect = { x: 0, y: 583, width: 402, height: 291 }; + +/** + * The captured #2589 screen: a docked iPhone keyboard whose band owns the app's tab bar. + * + * The keyboard is its own system surface, so it is never a covering sibling of app content here + * (`occlusion` stays silent) and the tab bar is still inside the app's own window rect + * (`offscreen` passes) — the tap used to report success while the key ate the touch. The key rects + * are the bottom row measured on iPhone 17 Pro (26.2): a keyboard the rule will measure has to + * report one unbroken run of columns, so a reduced layout cannot stop at two keys. + * + * The variants keep every keyboard-occlusion suite on this one geometry: + * - `targetRect` moves the app-owned button, so "clear of the keyboard" is the same tree. + * - `keyboardLifted` hauls the keys above the docking budget, which stops the tree-derived rule + * measuring at all — the way to prove a refusal came from the producer's band, not the tree. + */ +export function keyboardCoveredTabBarSnapshot( + params: { targetRect?: Rect; keyboardLifted?: boolean } = {}, +): SnapshotState { + const nodes = keyboardCoveredTabBarTree().nodes.map((node) => { + if (node.index === 1 && params.targetRect) return { ...node, rect: params.targetRect }; + if (!params.keyboardLifted) return node; + return node.rect && (node.type === 'Keyboard' || node.type === 'Key') + ? { ...node, rect: { ...node.rect, y: node.rect.y - 560 } } + : node; + }); + return { ...keyboardCoveredTabBarTree(), nodes }; +} + +function keyboardCoveredTabBarTree(): SnapshotState { + return makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 402, height: 874 }, + hittable: true, + }, + { + index: 1, + depth: 2, + parentIndex: 0, + type: 'Button', + label: 'Form', + rect: KEYBOARD_COVERED_TAB_BAR_RECT, + hittable: true, + }, + { + index: 2, + depth: 1, + parentIndex: 0, + type: 'Keyboard', + rect: KEYBOARD_COVERED_BAND_RECT, + hittable: false, + }, + { + index: 3, + depth: 2, + parentIndex: 2, + type: 'Key', + label: 'globe', + rect: { x: 4.67, y: 752, width: 49.33, height: 54 }, + hittable: true, + }, + { + index: 4, + depth: 2, + parentIndex: 2, + type: 'Key', + label: '.?123', + rect: { x: 54, y: 752, width: 49.33, height: 54 }, + hittable: true, + }, + { + index: 5, + depth: 2, + parentIndex: 2, + type: 'Key', + label: 'space', + rect: { x: 103.33, y: 752, width: 197.33, height: 54 }, + hittable: true, + }, + { + index: 6, + depth: 2, + parentIndex: 2, + type: 'Key', + label: 'return', + rect: { x: 300.67, y: 752, width: 99, height: 54 }, + hittable: true, + }, + ]); +} diff --git a/src/commands/interaction/runtime/interaction-resolution-request.ts b/src/commands/interaction/runtime/interaction-resolution-request.ts index 0eca2e1931..4a0ccfdc89 100644 --- a/src/commands/interaction/runtime/interaction-resolution-request.ts +++ b/src/commands/interaction/runtime/interaction-resolution-request.ts @@ -1,4 +1,5 @@ import type { ActingPipelinePolicy } from '@agent-device/selectors/selector-pipeline-policy'; +import type { InteractionAction } from '@agent-device/selectors/interaction-action'; import type { PreresolvedInteractionTarget } from '@agent-device/contracts/interaction'; import type { ReplayTargetGuardDenotation } from '@agent-device/contracts/replay'; @@ -19,22 +20,6 @@ import type { ReplayTargetGuardDenotation } from '@agent-device/contracts/replay */ export type ExpectedResolvedTarget = ReplayTargetGuardDenotation; -export type InteractionAction = - | 'click' - | 'press' - | 'fill' - | 'focus' - | 'longPress' - | 'hover' - | 'scroll' - | 'swipe' - | 'pinch' - | 'pan' - | 'drag' - | 'fling' - | 'rotate' - | 'transform'; - export type ResolveInteractionTargetParams = { action: InteractionAction; requireInteractive: boolean; diff --git a/src/commands/interaction/runtime/interaction-verb.ts b/src/commands/interaction/runtime/interaction-verb.ts deleted file mode 100644 index 31dbe7fe4a..0000000000 --- a/src/commands/interaction/runtime/interaction-verb.ts +++ /dev/null @@ -1,20 +0,0 @@ -import type { InteractionAction } from './interaction-resolution-request.ts'; - -/** - * The passive verb a refusal uses for the action it declined, so every acting guard that refuses - * before dispatch describes the failure in the caller's own words. - */ -export function interactionVerb(action: InteractionAction): string { - switch (action) { - case 'fill': - return 'be filled'; - case 'focus': - return 'be focused'; - case 'longPress': - return 'be long-pressed'; - case 'hover': - return 'be hovered'; - default: - return 'be tapped'; - } -} diff --git a/src/commands/interaction/runtime/native-ref-interaction.ts b/src/commands/interaction/runtime/native-ref-interaction.ts index 8e70ceedeb..209b62b3a1 100644 --- a/src/commands/interaction/runtime/native-ref-interaction.ts +++ b/src/commands/interaction/runtime/native-ref-interaction.ts @@ -16,7 +16,7 @@ import type { } from '@agent-device/contracts/backend'; import { toBackendContext } from '../../runtime-common.ts'; import { toBackendResult } from '../../runtime-types.ts'; -import type { InteractionAction } from './interaction-resolution-request.ts'; +import type { InteractionAction } from '@agent-device/selectors/interaction-action'; import { tryResolveRefNode } from './ref-target-resolution.ts'; import { EXACT_REF_RESOLUTION, describeNonHittableTarget } from './resolution-disclosure.ts'; import { diff --git a/src/commands/interaction/runtime/resolution-disclosure.ts b/src/commands/interaction/runtime/resolution-disclosure.ts index 6789851e5c..1d8c534903 100644 --- a/src/commands/interaction/runtime/resolution-disclosure.ts +++ b/src/commands/interaction/runtime/resolution-disclosure.ts @@ -16,7 +16,7 @@ import type { ResolutionDisclosure, SurfaceScopedNodes, } from '@agent-device/contracts/interaction'; -import type { InteractionAction } from './interaction-resolution-request.ts'; +import type { InteractionAction } from '@agent-device/selectors/interaction-action'; export type ResolvedRefNode = { ref: string; diff --git a/src/commands/interaction/runtime/resolution.ts b/src/commands/interaction/runtime/resolution.ts index 387b827caf..90931a2529 100644 --- a/src/commands/interaction/runtime/resolution.ts +++ b/src/commands/interaction/runtime/resolution.ts @@ -14,12 +14,10 @@ import type { ResolvedInteractionTarget, SurfaceScopedNodes, } from '@agent-device/contracts/interaction'; -import { describeKeyboardOccludedPointWarning } from './keyboard-occlusion.ts'; +import { describeKeyboardOccludedPointWarning } from '@agent-device/selectors/keyboard-occlusion'; +import type { InteractionAction } from '@agent-device/selectors/interaction-action'; import { assertReplayTargetResolution } from './replay-target-guard.ts'; -import type { - InteractionAction, - ResolveInteractionTargetParams, -} from './interaction-resolution-request.ts'; +import type { ResolveInteractionTargetParams } from './interaction-resolution-request.ts'; import { resolveRefInteractionTarget } from './ref-target-resolution.ts'; import { buildSelectorResolutionDisclosure, diff --git a/src/commands/interaction/runtime/selector-target-resolution.ts b/src/commands/interaction/runtime/selector-target-resolution.ts index cc7ceae36c..c9b15d541d 100644 --- a/src/commands/interaction/runtime/selector-target-resolution.ts +++ b/src/commands/interaction/runtime/selector-target-resolution.ts @@ -14,10 +14,8 @@ import { type ActionSelectorResolution, } from './selector-action-resolution.ts'; import { selectorNotFoundFailure } from './selector-read-shared.ts'; -import type { - InteractionAction, - ResolveInteractionTargetParams, -} from './interaction-resolution-request.ts'; +import type { InteractionAction } from '@agent-device/selectors/interaction-action'; +import type { ResolveInteractionTargetParams } from './interaction-resolution-request.ts'; export type SelectorResolutionAttempt = { capture: InteractionSnapshot; diff --git a/src/commands/interaction/runtime/keyboard-occlusion.test.ts b/src/commands/interaction/runtime/target-visibility-stages-keyboard-occlusion.test.ts similarity index 65% rename from src/commands/interaction/runtime/keyboard-occlusion.test.ts rename to src/commands/interaction/runtime/target-visibility-stages-keyboard-occlusion.test.ts index 324288694b..43946b0e76 100644 --- a/src/commands/interaction/runtime/keyboard-occlusion.test.ts +++ b/src/commands/interaction/runtime/target-visibility-stages-keyboard-occlusion.test.ts @@ -1,40 +1,38 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; -import type { - Point, - Rect, - SnapshotKeyboardBandFact, - SnapshotState, -} from '@agent-device/kernel/snapshot'; +import type { Point, SnapshotKeyboardBandFact, SnapshotState } from '@agent-device/kernel/snapshot'; import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; -import { keyboardCoveredTabBarSnapshot } from '../../../../test/integration/interaction-contract/fixtures.ts'; +import { + KEYBOARD_COVERED_BAND_RECT, + KEYBOARD_COVERED_TAB_BAR_RECT, + keyboardCoveredTabBarSnapshot, +} from '@agent-device/test-kit'; import { ref, selector } from './selector-read-utils.ts'; import { createInteractionDevice } from './__tests__/test-utils/index.ts'; -// #2589: the keyboard is its own system surface, so neither `occlusion` nor `offscreen` refuses a -// tap whose point belongs to it. These cover the consequences the shared classifier cannot express: -// which paths refuse, which disclose, and which stay silent. The tree itself is the contract -// fixtures' #2589 shape, so the refusal and the disclosure are measured on the same geometry the -// ADR 0011 cells claim. +// #2589 through the shared door: which acting paths refuse, which disclose, and which stay silent. +// The decision itself and the shape of its refusal are unit-proven beside the guard, in +// packages/selectors/src/keyboard-occlusion.test.ts. These run the acting paths, because the +// guarantee is that the fast path and the shared rule enter through one door — a unit test on the +// resolver alone would leave the ref path free to consult the tree again on its way to the tap. -const TAB_BAR_RECT: Rect = { x: 148, y: 791, width: 104, height: 83 }; +/** The band the #2589 capture's producer measured, in the app's orientation space. */ +const MEASURED_BAND: SnapshotKeyboardBandFact = { + kind: 'visible', + frame: KEYBOARD_COVERED_BAND_RECT, +}; -/** The contract fixture with its app-owned button moved, so the variants stay one tree, not copies. */ -function keyboardTree(params: { tabRect?: Rect } = {}): SnapshotState { - const tabRect = params.tabRect; - if (!tabRect) return keyboardCoveredTabBarSnapshot(); - return makeSnapshotState( - keyboardCoveredTabBarSnapshot().nodes.map((node) => - node.index === 1 ? { ...node, rect: tabRect } : node, - ), - ); +function tappedDevice(snapshot: SnapshotState, calls: Point[]) { + return createInteractionDevice(snapshot, { + tap: async (_context, point) => { + calls.push(point); + return { ok: true }; + }, + }); } -/** - * Keyboard-owned rects hauled above the docking budget: the tree stops describing a keyboard the - * bottom of the screen belongs to, which is what the app-drawn-keypad and aim cases measure against. - */ -function liftKeyboardOffBottomEdge(nodes: SnapshotState['nodes']): SnapshotState['nodes'] { +/** The same tree with its keyboard hoisted above the docking budget, so the tree rule stops measuring. */ +function lifted(nodes: SnapshotState['nodes']): SnapshotState['nodes'] { return nodes.map((node) => node.rect && (node.type === 'Keyboard' || node.type === 'Key') ? { ...node, rect: { ...node.rect, y: node.rect.y - 560 } } @@ -42,18 +40,19 @@ function liftKeyboardOffBottomEdge(nodes: SnapshotState['nodes']): SnapshotState ); } -function tappedDevice(snapshot: SnapshotState, calls: Point[]) { - return createInteractionDevice(snapshot, { - tap: async (_context, point) => { - calls.push(point); - return { ok: true }; - }, - }); +function captured(params: { + nodes: SnapshotState['nodes']; + keyboard?: SnapshotKeyboardBandFact; +}): SnapshotState { + return makeSnapshotState( + params.nodes, + params.keyboard ? { keyboard: params.keyboard } : undefined, + ); } test('press @ref refuses a target whose center sits behind the visible keyboard', async () => { const calls: Point[] = []; - const device = tappedDevice(keyboardTree(), calls); + const device = tappedDevice(keyboardCoveredTabBarSnapshot(), calls); await assert.rejects( () => device.interactions.click(ref('@e2'), { session: 'default' }), @@ -64,8 +63,8 @@ test('press @ref refuses a target whose center sits behind the visible keyboard' const details = (error as { details?: Record }).details; assert.equal(details?.reason, 'tap_keyboard_occludes_target'); assert.equal(details?.ref, '@e2'); - assert.deepEqual(details?.rect, TAB_BAR_RECT); - assert.deepEqual(details?.keyboardFrame, { x: 0, y: 583, width: 402, height: 291 }); + assert.deepEqual(details?.rect, KEYBOARD_COVERED_TAB_BAR_RECT); + assert.deepEqual(details?.keyboardFrame, KEYBOARD_COVERED_BAND_RECT); assert.match(String(details?.hint), /End editing first/); return true; }, @@ -74,7 +73,7 @@ test('press @ref refuses a target whose center sits behind the visible keyboard' }); test('fill refuses with the verb matching the action', async () => { - const device = tappedDevice(keyboardTree(), []); + const device = tappedDevice(keyboardCoveredTabBarSnapshot(), []); await assert.rejects( () => device.interactions.fill(ref('@e2'), 'hello', { session: 'default' }), /cannot be filled safely/, @@ -83,7 +82,7 @@ test('fill refuses with the verb matching the action', async () => { test('press selector refuses the same node the ref path refuses', async () => { const calls: Point[] = []; - const device = tappedDevice(keyboardTree(), calls); + const device = tappedDevice(keyboardCoveredTabBarSnapshot(), calls); await assert.rejects( () => device.interactions.press(selector('label=Form'), { session: 'default' }), @@ -100,7 +99,7 @@ test('press selector refuses the same node the ref path refuses', async () => { test('a keyboard key is a legitimate target: pressing one still taps', async () => { const calls: Point[] = []; - const device = tappedDevice(keyboardTree(), calls); + const device = tappedDevice(keyboardCoveredTabBarSnapshot(), calls); await device.interactions.click(ref('@e6'), { session: 'default' }); @@ -109,8 +108,10 @@ test('a keyboard key is a legitimate target: pressing one still taps', async () test('an element whose center stays above the key plane still taps', async () => { const calls: Point[] = []; - const peeking = { x: 148, y: 470, width: 104, height: 110 }; - const device = tappedDevice(keyboardTree({ tabRect: peeking }), calls); + const device = tappedDevice( + keyboardCoveredTabBarSnapshot({ targetRect: { x: 148, y: 470, width: 104, height: 110 } }), + calls, + ); await device.interactions.click(ref('@e2'), { session: 'default' }); @@ -120,7 +121,9 @@ test('an element whose center stays above the key plane still taps', async () => test('no keyboard in the tree means nothing to refuse', async () => { const calls: Point[] = []; const withoutKeyboard = makeSnapshotState( - keyboardTree().nodes.filter((node) => node.type !== 'Keyboard' && node.type !== 'Key'), + keyboardCoveredTabBarSnapshot().nodes.filter( + (node) => node.type !== 'Keyboard' && node.type !== 'Key', + ), ); const device = tappedDevice(withoutKeyboard, calls); @@ -131,8 +134,7 @@ test('no keyboard in the tree means nothing to refuse', async () => { test('an app-drawn keypad that stops short of the bottom edge is not the system keyboard', async () => { const calls: Point[] = []; - const appOwnedKeypad = makeSnapshotState(liftKeyboardOffBottomEdge(keyboardTree().nodes)); - const device = tappedDevice(appOwnedKeypad, calls); + const device = tappedDevice(keyboardCoveredTabBarSnapshot({ keyboardLifted: true }), calls); await device.interactions.click(ref('@e2'), { session: 'default' }); @@ -141,13 +143,11 @@ test('an app-drawn keypad that stops short of the bottom edge is not the system test('a coordinate behind the keyboard taps anyway and discloses the reason', async () => { const calls: Point[] = []; - const device = tappedDevice(keyboardTree(), calls); + const device = tappedDevice(keyboardCoveredTabBarSnapshot(), calls); const result = await device.interactions.press( { kind: 'point', x: 200, y: 810 }, - { - session: 'default', - }, + { session: 'default' }, ); assert.deepEqual(calls, [{ x: 200, y: 810 }]); @@ -158,27 +158,26 @@ test('a coordinate behind the keyboard taps anyway and discloses the reason', as test('the guard reads the point the interaction dispatches, not the rect center', async () => { // The Form button's own center sits 3 pt above the key plane, and its interactive child owns that // upper region, so the point the tap dispatches is pushed down into the keyboard's band. - const aimShifted = makeSnapshotState([ - ...keyboardCoveredTabBarSnapshot().nodes, - { - index: 5, - depth: 3, - parentIndex: 1, - type: 'Button', - label: 'Send', - rect: { x: 148, y: 500, width: 104, height: 83 }, - hittable: true, - }, - ]).nodes.map((node) => - node.index === 1 ? { ...node, rect: { x: 148, y: 500, width: 104, height: 160 } } : node, + const aimShifted = makeSnapshotState( + [ + ...keyboardCoveredTabBarSnapshot().nodes, + { + index: 7, + depth: 3, + parentIndex: 1, + type: 'Button', + label: 'Send', + rect: { x: 148, y: 500, width: 104, height: 83 }, + hittable: true, + }, + ].map((node) => + node.index === 1 ? { ...node, rect: { x: 148, y: 500, width: 104, height: 160 } } : node, + ), ); const calls: Point[] = []; await assert.rejects( - () => - tappedDevice(makeSnapshotState(aimShifted), calls).interactions.click(ref('@e2'), { - session: 'default', - }), + () => tappedDevice(aimShifted, calls).interactions.click(ref('@e2'), { session: 'default' }), /Ref @e2 is behind the visible keyboard/, ); assert.deepEqual(calls, []); @@ -186,23 +185,21 @@ test('the guard reads the point the interaction dispatches, not the rect center' // The same tree with its keyboard hauled off the bottom edge shows where that tap was aiming: below // the key plane, which is what makes this the dispatched point's refusal rather than the center's. const aim: Point[] = []; - await tappedDevice( - makeSnapshotState(liftKeyboardOffBottomEdge(aimShifted)), - aim, - ).interactions.click(ref('@e2'), { session: 'default' }); + await tappedDevice(makeSnapshotState(lifted(aimShifted.nodes)), aim).interactions.click( + ref('@e2'), + { session: 'default' }, + ); assert.equal(aim.length, 1); assert.ok((aim[0]?.y ?? 0) > 583, `expected the dispatched point below 583, got ${aim[0]?.y}`); }); test('a coordinate on a reported key is the keyboard the caller asked for', async () => { const calls: Point[] = []; - const device = tappedDevice(keyboardTree(), calls); + const device = tappedDevice(keyboardCoveredTabBarSnapshot(), calls); const result = await device.interactions.press( { kind: 'point', x: 160, y: 760 }, - { - session: 'default', - }, + { session: 'default' }, ); assert.deepEqual(calls, [{ x: 160, y: 760 }]); @@ -210,42 +207,22 @@ test('a coordinate on a reported key is the keyboard the caller asked for', asyn }); test('the out-of-viewport disclosure still wins on a stale-tree coordinate', async () => { - const device = tappedDevice(keyboardTree(), []); + const device = tappedDevice(keyboardCoveredTabBarSnapshot(), []); const result = await device.interactions.press( { kind: 'point', x: 900, y: 810 }, - { - session: 'default', - }, + { session: 'default' }, ); assert.match(result.warning ?? '', /outside the last-known viewport/); }); -// The producer-measured band (#2660). These run the acting paths, because the guarantee is that the -// fast path and the shared rule enter through the same door — a unit test on the resolver alone would -// leave the ref path free to consult the tree again on its way to the tap. - -/** The runner's own reading of the #2589 fixture's keyboard, in the app's orientation space. */ -const MEASURED_BAND: SnapshotKeyboardBandFact = { - kind: 'visible', - frame: { x: 0, y: 583, width: 402, height: 291 }, -}; - -function captured(params: { - nodes: SnapshotState['nodes']; - keyboard?: SnapshotKeyboardBandFact; -}): SnapshotState { - return makeSnapshotState( - params.nodes, - params.keyboard ? { keyboard: params.keyboard } : undefined, - ); -} +// The producer-measured band (#2660). These run the acting paths for the same reason as above. test('a measured band refuses the target the tree rule would have refused', async () => { const calls: Point[] = []; const device = tappedDevice( - captured({ nodes: keyboardTree().nodes, keyboard: MEASURED_BAND }), + captured({ nodes: keyboardCoveredTabBarSnapshot().nodes, keyboard: MEASURED_BAND }), calls, ); @@ -256,12 +233,10 @@ test('a measured band refuses the target the tree rule would have refused', asyn (error as { details?: Record }).details?.reason, 'tap_keyboard_occludes_target', ); - assert.deepEqual((error as { details?: Record }).details?.keyboardFrame, { - x: 0, - y: 583, - width: 402, - height: 291, - }); + assert.deepEqual( + (error as { details?: Record }).details?.keyboardFrame, + KEYBOARD_COVERED_BAND_RECT, + ); return true; }, ); @@ -272,16 +247,18 @@ test('the tree rule is not consulted once the capture published a band', async ( // The same screen with its keyboard hoisted off the bottom edge: the tree rule refuses to measure // that geometry and lets the tap through, which is what makes this pair the assertion that the // band came from the fact and not from the tree. - const lifted = makeSnapshotState(liftKeyboardOffBottomEdge(keyboardTree().nodes)); + const liftedNodes = keyboardCoveredTabBarSnapshot({ keyboardLifted: true }).nodes; const withoutFact: Point[] = []; - await tappedDevice(lifted, withoutFact).interactions.click(ref('@e2'), { session: 'default' }); + await tappedDevice(makeSnapshotState(liftedNodes), withoutFact).interactions.click(ref('@e2'), { + session: 'default', + }); assert.equal(withoutFact.length, 1, 'the tree rule fails open on geometry it cannot measure'); const withFact: Point[] = []; await assert.rejects( () => tappedDevice( - captured({ nodes: lifted.nodes, keyboard: MEASURED_BAND }), + captured({ nodes: liftedNodes, keyboard: MEASURED_BAND }), withFact, ).interactions.click(ref('@e2'), { session: 'default' }), /Ref @e2 is behind the visible keyboard/, @@ -291,9 +268,13 @@ test('the tree rule is not consulted once the capture published a band', async ( test('a target above the measured band still presses', async () => { const calls: Point[] = []; - const above = { x: 148, y: 470, width: 104, height: 110 }; const device = tappedDevice( - captured({ nodes: keyboardTree({ tabRect: above }).nodes, keyboard: MEASURED_BAND }), + captured({ + nodes: keyboardCoveredTabBarSnapshot({ + targetRect: { x: 148, y: 470, width: 104, height: 110 }, + }).nodes, + keyboard: MEASURED_BAND, + }), calls, ); @@ -305,7 +286,7 @@ test('a target above the measured band still presses', async () => { test('a measured band still excuses the keyboard key the caller named', async () => { const calls: Point[] = []; const device = tappedDevice( - captured({ nodes: keyboardTree().nodes, keyboard: MEASURED_BAND }), + captured({ nodes: keyboardCoveredTabBarSnapshot().nodes, keyboard: MEASURED_BAND }), calls, ); @@ -317,7 +298,7 @@ test('a measured band still excuses the keyboard key the caller named', async () test('a measured band still reads a coordinate on a reported key as the keyboard asked for', async () => { const calls: Point[] = []; const result = await tappedDevice( - captured({ nodes: keyboardTree().nodes, keyboard: MEASURED_BAND }), + captured({ nodes: keyboardCoveredTabBarSnapshot().nodes, keyboard: MEASURED_BAND }), calls, ).interactions.press({ kind: 'point', x: 160, y: 760 }, { session: 'default' }); @@ -328,7 +309,7 @@ test('a measured band still reads a coordinate on a reported key as the keyboard test('a producer that found no keyboard lets the tap the stale tree would refuse', async () => { const calls: Point[] = []; const device = tappedDevice( - captured({ nodes: keyboardTree().nodes, keyboard: { kind: 'absent' } }), + captured({ nodes: keyboardCoveredTabBarSnapshot().nodes, keyboard: { kind: 'absent' } }), calls, ); @@ -341,7 +322,7 @@ test('an unmeasurable fact leaves the acting path on the tree rule unchanged', a const calls: Point[] = []; const device = tappedDevice( captured({ - nodes: keyboardTree().nodes, + nodes: keyboardCoveredTabBarSnapshot().nodes, keyboard: { kind: 'unmeasurable', reason: 'keyboard-frame-query-timeout' }, }), calls, @@ -354,10 +335,10 @@ test('an unmeasurable fact leaves the acting path on the tree rule unchanged', a assert.deepEqual(calls, []); }); -test("a coordinate behind the measured band discloses the producer's frame", async () => { +test('a coordinate behind the measured band discloses the producer frame', async () => { const result = await tappedDevice( captured({ - nodes: liftKeyboardOffBottomEdge(keyboardTree().nodes), + nodes: keyboardCoveredTabBarSnapshot({ keyboardLifted: true }).nodes, keyboard: MEASURED_BAND, }), [], diff --git a/src/commands/interaction/runtime/target-visibility-stages.ts b/src/commands/interaction/runtime/target-visibility-stages.ts index 9da8548af5..c8f16ab15c 100644 --- a/src/commands/interaction/runtime/target-visibility-stages.ts +++ b/src/commands/interaction/runtime/target-visibility-stages.ts @@ -20,9 +20,11 @@ import { type OffscreenScrollDirection, } from '@agent-device/capture-kit/mobile-snapshot-semantics'; import { toBackendContext } from '../../runtime-common.ts'; -import { assertTapTargetClearOfVisibleKeyboard } from './keyboard-occlusion.ts'; -import { interactionVerb } from './interaction-verb.ts'; -import type { InteractionAction } from './interaction-resolution-request.ts'; +import { assertTapTargetClearOfVisibleKeyboard } from '@agent-device/selectors/keyboard-occlusion'; +import { + interactionVerb, + type InteractionAction, +} from '@agent-device/selectors/interaction-action'; /** * The one construction site for "covered by another visible element" refusals, shared by the diff --git a/test/integration/interaction-contract/fixtures.ts b/test/integration/interaction-contract/fixtures.ts index 378505b9bd..04144e7cc9 100644 --- a/test/integration/interaction-contract/fixtures.ts +++ b/test/integration/interaction-contract/fixtures.ts @@ -399,72 +399,6 @@ export const RUNNER_NON_HITTABLE_TEXT_INPUT_NODES = [ ] as const; // #2589 shape, recorded from the iOS repro: the bottom tab bar sits behind the system keyboard. -// The keyboard is its own system surface, so it is never a covering sibling of app content here -// (`occlusion` stays silent) and the tab bar is still inside the app's own window rect -// (`offscreen` passes) — the tap used to report success while the key ate the touch. -// The key rects are the bottom row measured on iPhone 17 Pro (26.2): a keyboard the guard will -// measure has to report one unbroken run of columns, so a reduced layout cannot stop at two keys. -export function keyboardCoveredTabBarSnapshot(): SnapshotState { - return makeSnapshotState([ - { - index: 0, - depth: 0, - type: 'Application', - rect: { x: 0, y: 0, width: 402, height: 874 }, - hittable: true, - }, - { - index: 1, - depth: 2, - parentIndex: 0, - type: 'Button', - label: 'Form', - rect: { x: 148, y: 791, width: 104, height: 83 }, - hittable: true, - }, - { - index: 2, - depth: 1, - parentIndex: 0, - type: 'Keyboard', - rect: { x: 0, y: 583, width: 402, height: 291 }, - hittable: false, - }, - { - index: 3, - depth: 2, - parentIndex: 2, - type: 'Key', - label: 'globe', - rect: { x: 4.67, y: 752, width: 49.33, height: 54 }, - hittable: true, - }, - { - index: 4, - depth: 2, - parentIndex: 2, - type: 'Key', - label: '.?123', - rect: { x: 54, y: 752, width: 49.33, height: 54 }, - hittable: true, - }, - { - index: 5, - depth: 2, - parentIndex: 2, - type: 'Key', - label: 'space', - rect: { x: 103.33, y: 752, width: 197.33, height: 54 }, - hittable: true, - }, - { - index: 6, - depth: 2, - parentIndex: 2, - type: 'Key', - label: 'return', - rect: { x: 300.67, y: 752, width: 99, height: 54 }, - hittable: true, - }, - ]); -} +// The tree is the shared `@agent-device/test-kit` fixture — the daemon's find focus seam and the +// selectors guard test are proven against this same shape. +export { keyboardCoveredTabBarSnapshot } from '@agent-device/test-kit'; From a5c9be87585aa5a9815d6fe995c27c9faa5b17ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 16:00:52 +0200 Subject: [PATCH 2/8] fix(find): run the keyboard guard before find focus dispatches `find focus` and the focus leg of `find type` aim a point themselves instead of re-entering the interaction leaf, so the #2589 guard never ran: a match behind the visible keyboard reported `Focused (x, y)` while the key ate the touch (#2622). The seam now runs the same guard the acting paths run, against the tree find matched, at the center it dispatches, ahead of the ref-frame expiry so a refusal leaves the frame pinnable. ADR 0011 gains the classified `find-focus-dispatch` row for this seam, with its enforcement proven by contract scenarios over the daemon harness and its four divergences from the acting rows recorded as owned gaps: no off-screen refusal, no parent-owned touch point, no moving-target gate, and a dispatch-only success claim. --- .../contracts/src/interaction-guarantees.ts | 98 +++++++- .../__tests__/find-keyboard-occlusion.test.ts | 93 +++++++ .../execution/interaction/internal/find.ts | 16 ++ .../selectors/src/selector-pipeline-policy.ts | 8 +- .../find-focus-dispatch.contract.test.ts | 227 ++++++++++++++++++ .../find-focus-dispatch.coverage.ts | 17 ++ .../integration/interaction-contract/index.ts | 2 + 7 files changed, 457 insertions(+), 4 deletions(-) create mode 100644 packages/daemon/src/execution/interaction/internal/__tests__/find-keyboard-occlusion.test.ts create mode 100644 test/integration/interaction-contract/find-focus-dispatch.contract.test.ts create mode 100644 test/integration/interaction-contract/find-focus-dispatch.coverage.ts diff --git a/packages/contracts/src/interaction-guarantees.ts b/packages/contracts/src/interaction-guarantees.ts index f4f7e4c9ae..dcea6216aa 100644 --- a/packages/contracts/src/interaction-guarantees.ts +++ b/packages/contracts/src/interaction-guarantees.ts @@ -96,6 +96,7 @@ export const INTERACTION_PATH_IDS = [ 'target-drag', 'native-ref', 'coordinate', + 'find-focus-dispatch', 'maestro-direct-selector', 'maestro-non-hittable-fallback', ] as const; @@ -233,7 +234,7 @@ const RUNTIME_TREE_SHARED_GUARANTEES = { // `occlusion` nor `offscreen` can reach it. keyboardOcclusion: { kind: 'runtime', - via: 'src/commands/interaction/runtime/keyboard-occlusion.ts#assertTapTargetClearOfVisibleKeyboard', + via: 'packages/selectors/src/keyboard-occlusion.ts#assertTapTargetClearOfVisibleKeyboard', }, // #1542: the base decision is the contracts-owned snapshot visibility resolver (bulk accessibility // tree), but throwIfOffscreenInteractionTarget is the actual end-to-end @@ -348,7 +349,7 @@ export const INTERACTION_DISPATCH_PATHS: Record focus`, and the focus leg of `find type`: find captures its own tree, ranks its own candidate set, runs the node stages its `findAct` row declares, then dispatches a point focus itself (`type` then dispatches the text) instead of re-entering the interaction leaf. `find click`/`find fill` are NOT this path — they hand the resolved node to `runtime-ref` as a pre-resolved target (#1654). `focus ` is the coordinate route. Every cell below classifies what the `findAct` row plus this seam actually run, not what the acting rows run.', + commands: ['find'], + guarantees: { + // find's own ranking, but under the same ambiguity contract the runtime paths use: one + // `reject-candidates` policy row (#1625), `--first`/`--last` the only narrowing, and + // `preferOnscreenMatches` as the tie-break rather than a geometry-wins rule. + disambiguation: { + kind: 'runtime', + via: 'packages/daemon/src/execution/interaction/internal/find-match-resolution.ts#resolveFindMatch', + }, + occlusion: { + kind: 'runtime', + via: 'packages/selectors/src/selector-pipeline.ts#runNodePipelineStages', + }, + // #2622: the seam dispatches the point itself, so the shared #2589 guard runs here rather + // than at the interaction leaf's door. Same refusal, same typed details, measured at the + // center the seam dispatches. + keyboardOcclusion: { + kind: 'runtime', + via: 'packages/selectors/src/keyboard-occlusion.ts#assertTapTargetClearOfVisibleKeyboard', + }, + offscreen: { + kind: 'waived', + reason: + 'gap: the `findAct` row declares `offscreen: "ignore"` because find ranks across the whole tree and prefers on-screen matches rather than refusing off-screen ones (#1625), and focus/type never re-enter the acting leaf where the runtime rows refuse. So the seam dispatches at a captured rect center the acting paths would reject as outside the root viewport — a closed drawer or an off-viewport carousel is a silent no-op focus here.', + trackingIssue: GAPS_UMBRELLA_ISSUE, + }, + parentOwnedTouchPoint: { + kind: 'waived', + reason: + 'gap: the seam aims `centerOfRect` of the resolved node rather than the parent-owned point `resolveInteractionTouchPoint` chooses, so a match whose center is tiled by an independently interactive descendant activates that descendant instead of failing closed — and the keyboard guard above measures the same center the tap uses, which is the point actually dispatched but not the point the acting paths would aim.', + trackingIssue: GAPS_UMBRELLA_ISSUE, + }, + // Promotion is the enforcement the guarantee names; the annotation half is not this seam's, + // because its payload is the focus runtime's point response and never the interaction result + // that carries `targetHittable`/`hint` (see `responseIdentity` below). + nonHittable: { + kind: 'runtime', + via: 'packages/selectors/src/selector-pipeline.ts#runNodePipelineStages', + }, + // One construction site per dispatch kind, both shared with the plain commands: the focus + // payload is `executeFocusPoint`'s (the same owner `focus ` reaches), and `type`'s + // second dispatch is `executeBoundTypeText`'s. The seam hand-rolls neither. + responseConstruction: { + kind: 'runtime', + via: 'packages/daemon/src/execution/focus-runtime.ts#executeFocusPoint', + }, + responseIdentity: { + kind: 'waived', + reason: + "Intentional: the payload is the point-dispatch result, so no refLabel/selectorChain is echoed. Mutating find's `data.ref` is pre-action diagnostic identity that ADR 0014 keeps unpinnable (no `refsGeneration`), and the seam never enters the ref/selector response builder that mints the identity fields.", + }, + verifyEvidence: { + kind: 'inapplicable', + reason: + 'find exposes no --verify flag: its flag surface is snapshot depth/raw plus --first/--last, and --verify would have no post-action capture owner on a route whose response is the dispatch payload.', + }, + settleObservation: { + kind: 'inapplicable', + reason: + 'find exposes no --settle flag for the same reason; settling is built on the tree-based runtime baseline and the captures the acting leaf owns.', + }, + // Locator misses and candidate sets use find's shared diagnostics, and a pre-action guard + // refusal travels as the AppError its owner built (`coveredInteractionDetails`, + // TAP_KEYBOARD_OCCLUDES_TARGET_DETAILS) through `normalizeError`, so code, hint, and reason + // reach the wire unchanged. + errorTaxonomy: { + kind: 'runtime', + via: 'packages/daemon/src/execution/selector-match-errors.ts#buildAmbiguousMatchError', + }, + resolutionDisclosure: { + kind: 'inapplicable', + reason: + 'The additive `resolution` field discloses the provenance of an identity the response issues; this seam issues none (ADR 0014), and its ambiguity is disclosed as the bounded candidate error the `errorTaxonomy` cell names instead.', + }, + outcomeObservation: { + kind: 'waived', + reason: + 'gap: the response reports the focus (and for `type`, the text) dispatch only. Nothing observes that the field took focus or that the text landed, and unlike the tap-shaped paths this route has no opt-in --verify/--settle that could carry such evidence.', + trackingIssue: TAP_OUTCOME_NOT_OBSERVED_GAP_ISSUE, + }, + targetStability: { + kind: 'waived', + reason: + "gap: the seam dispatches a rect center captured one runner round trip earlier without the #3471 re-observation, so a focus issued right after an action that animated the surface can aim a position the target has already vacated and report success on the vacancy. The gate itself is the acting leaf's (`guardResolvedTargetAgainstMotion`), and this route does not pass through it.", + trackingIssue: GAPS_UMBRELLA_ISSUE, + }, + }, + }, 'maestro-direct-selector': { description: 'An explicit Maestro-compatible simple-selector click completed as an XCTest element tap. This path is selected only when the runner was allowed to use the non-hittable coordinate fallback but reported that it did not use it.', diff --git a/packages/daemon/src/execution/interaction/internal/__tests__/find-keyboard-occlusion.test.ts b/packages/daemon/src/execution/interaction/internal/__tests__/find-keyboard-occlusion.test.ts new file mode 100644 index 0000000000..e86b145b48 --- /dev/null +++ b/packages/daemon/src/execution/interaction/internal/__tests__/find-keyboard-occlusion.test.ts @@ -0,0 +1,93 @@ +import { test, expect, vi, beforeEach } from 'vitest'; +import { keyboardCoveredTabBarSnapshot } from '@agent-device/test-kit'; +import type { SnapshotState } from '@agent-device/kernel/snapshot'; +import { + mockFocusPoint, + resetFindTouchRuntimeFixture, + runFindClickScenario, +} from './find-touch-runtime-fixture.ts'; +import { mockTypeText } from '../../../__tests__/interaction-get-runtime-fixture.ts'; +import { refFrameState } from '../../../../core/ref-frame.ts'; + +vi.mock('../../../../core/snapshot-interactor-capture.ts', async () => { + const fixture = await import('../../../../__tests__/support/legacy-snapshot-capture-fixture.ts'); + return { captureSnapshotWithInteractor: fixture.captureSnapshotThroughLegacyDispatchFixture }; +}); + +beforeEach(() => { + resetFindTouchRuntimeFixture(); +}); + +function keyboardTree(targetRect?: SnapshotState['nodes'][number]['rect']): SnapshotState { + return keyboardCoveredTabBarSnapshot(targetRect ? { targetRect } : {}); +} + +type Refusal = { + code?: string; + message?: string; + hint?: string; + details?: Record; +}; + +function assertKeyboardRefusal( + response: { ok: boolean; error?: Refusal }, + verb: string, +): asserts response { + expect(response.ok).toBe(false); + const error = response.error as Refusal; + expect(error.code).toBe('COMMAND_FAILED'); + expect(error.message).toBe(`Ref @e2 is behind the visible keyboard and cannot ${verb} safely`); + expect(error.details?.reason).toBe('tap_keyboard_occludes_target'); + expect(error.details?.ref).toBe('@e2'); + expect(error.details?.rect).toEqual({ x: 148, y: 791, width: 104, height: 83 }); + expect(error.details?.keyboardFrame).toEqual({ x: 0, y: 583, width: 402, height: 291 }); + expect(error.details?.dispatched).toBe('no'); + expect(error.hint).toMatch(/End editing first/); +} + +test('find focus refuses a match whose tap point sits behind the visible keyboard', async () => { + const { response, session } = await runFindClickScenario({ + positionals: ['Form', 'focus'], + nodes: keyboardTree().nodes, + }); + + assertKeyboardRefusal(response, 'be focused'); + expect(mockFocusPoint).not.toHaveBeenCalled(); + // ADR 0014: the refusal is pre-action, so the captured frame stays pinnable. + expect(refFrameState(session)).toBe('active'); +}); + +test('find type refuses before the focus or the text dispatch', async () => { + const { response, session } = await runFindClickScenario({ + positionals: ['Form', 'type', 'hello'], + nodes: keyboardTree().nodes, + }); + + assertKeyboardRefusal(response, 'be focused'); + expect(mockFocusPoint).not.toHaveBeenCalled(); + expect(mockTypeText).not.toHaveBeenCalled(); + expect(refFrameState(session)).toBe('active'); +}); + +test('find focus on a match above the keyboard band still focuses', async () => { + const { response, session } = await runFindClickScenario({ + positionals: ['Form', 'focus'], + nodes: keyboardTree({ x: 148, y: 470, width: 104, height: 110 }).nodes, + }); + + expect(response.ok).toBe(true); + expect(mockFocusPoint).toHaveBeenCalledTimes(1); + expect(refFrameState(session)).toBe('expired'); +}); + +test('find focus on a reported keyboard key still focuses it', async () => { + // The closest negative: the same guard excuses the keyboard's own control, which sits inside the + // band it would otherwise refuse. + const { response } = await runFindClickScenario({ + positionals: ['label', 'space', 'focus'], + nodes: keyboardTree().nodes, + }); + + expect(response.ok).toBe(true); + expect(mockFocusPoint).toHaveBeenCalledTimes(1); +}); diff --git a/packages/daemon/src/execution/interaction/internal/find.ts b/packages/daemon/src/execution/interaction/internal/find.ts index 6f06edcc98..2fea9bdd04 100644 --- a/packages/daemon/src/execution/interaction/internal/find.ts +++ b/packages/daemon/src/execution/interaction/internal/find.ts @@ -13,6 +13,8 @@ import { import { runNodePipelineStages } from '@agent-device/selectors/selector-pipeline'; import { SELECTOR_PIPELINE_POLICIES } from '@agent-device/selectors/selector-pipeline-policy'; import { coveredInteractionDetails } from '@agent-device/selectors/interaction-error'; +import { assertTapTargetClearOfVisibleKeyboard } from '@agent-device/selectors/keyboard-occlusion'; +import { normalizeError } from '@agent-device/kernel/errors'; import { centerOfRect, type SnapshotAddressableNode, @@ -395,6 +397,20 @@ async function dispatchFocusForFindMatch( if (!coords) { return errorResponse('COMMAND_FAILED', 'matched element has no bounds'); } + // The same keyboard decision the acting paths run (#2589/#2622): this seam dispatches the point + // itself instead of re-entering the interaction leaf, so the guard has to run here. + try { + assertTapTargetClearOfVisibleKeyboard({ + nodes: match.tree.nodes, + node: match.resolvedNode, + action: 'focus', + label: `Ref ${match.ref}`, + ...(match.keyboard ? { keyboard: match.keyboard } : {}), + tapPoint: coords, + }); + } catch (error) { + return { ok: false, error: normalizeError(error) }; + } // ADR 0014 side-effect seam: mutating find focus/type dispatch the device // command directly (they do not re-enter the interaction leaf), so expire the // frame here before the device op. Pre-seam guards above preserve the frame. diff --git a/packages/selectors/src/selector-pipeline-policy.ts b/packages/selectors/src/selector-pipeline-policy.ts index 75554a5f02..8233fbd627 100644 --- a/packages/selectors/src/selector-pipeline-policy.ts +++ b/packages/selectors/src/selector-pipeline-policy.ts @@ -181,7 +181,13 @@ export const SELECTOR_PIPELINE_POLICIES = { resolution: SELECTOR_RESOLUTION_POLICIES.readList, occlusion: 'ignore', }, - /** Mutating `find` (#1625). */ + /** + * Mutating `find` (#1625). `offscreen: 'ignore'` is deliberate for the ranking (`preferOnscreen + * Matches` prefers an on-screen twin rather than refusing an off-screen one), and each action then + * decides what to do with the winner: click/fill hand it to the acting leaf, while focus/type + * dispatch a point themselves, which is why ADR 0011 classifies this row's stages per action on + * the `find-focus-dispatch` path rather than reusing an acting row's cells. + */ findAct: { resolution: SELECTOR_RESOLUTION_POLICIES.findAct, occlusion: 'refuse', diff --git a/test/integration/interaction-contract/find-focus-dispatch.contract.test.ts b/test/integration/interaction-contract/find-focus-dispatch.contract.test.ts new file mode 100644 index 0000000000..bb44561905 --- /dev/null +++ b/test/integration/interaction-contract/find-focus-dispatch.contract.test.ts @@ -0,0 +1,227 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { InteractionGuarantee } from '@agent-device/contracts/interaction-guarantees'; +import { keyboardCoveredTabBarSnapshot } from '@agent-device/test-kit'; +import { assertRpcError, assertRpcOk } from '../provider-scenarios/assertions.ts'; +import { PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS } from '../provider-scenarios/test-timeouts.ts'; +import { scenarioName, scenarioNames } from './coverage-manifest.ts'; +import { FIND_FOCUS_DISPATCH_COVERAGE } from './find-focus-dispatch.coverage.ts'; +import { runnerSnapshotEntry, runnerTapEntry, withIosContractDaemon } from './daemon-harness.ts'; + +// ADR 0011 Layer 3, find-focus-dispatch path: mutating `find focus` and the focus leg of +// `find type`. These run over the daemon harness because the seam lives in the daemon's +// find handler — it captures its own tree and dispatches the point itself, so no in-process device +// harness can reach it. The transcript is the proof: a refusal that dispatched anyway would either +// consume an unscripted tap entry or leave a scripted one behind. + +const scenario = (guarantee: InteractionGuarantee): string => + scenarioName(FIND_FOCUS_DISPATCH_COVERAGE, guarantee); + +const KEYBOARD_TREE = keyboardCoveredTabBarSnapshot().nodes; + +test( + scenario('keyboardOcclusion'), + async () => { + // One scripted capture and no tap entry: `assertComplete` is what proves the refusal stopped the + // dispatch, which is #2622's whole point. + await withIosContractDaemon([runnerSnapshotEntry(KEYBOARD_TREE)], async (daemon) => { + const error = assertRpcError( + await daemon.callCommand('find', ['label', 'Form', 'focus']), + 'COMMAND_FAILED', + /Ref @e2 is behind the visible keyboard and cannot be focused safely/, + ); + const details = readErrorDetails(error); + assert.equal(details.reason, 'tap_keyboard_occludes_target'); + assert.equal(details.ref, '@e2'); + assert.deepEqual(details.rect, { x: 148, y: 791, width: 104, height: 83 }); + assert.deepEqual(details.keyboardFrame, { x: 0, y: 583, width: 402, height: 291 }); + assert.equal(details.dispatched, 'no'); + }); + }, + PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS, +); + +test( + scenarioNames(FIND_FOCUS_DISPATCH_COVERAGE, 'keyboardOcclusion')[1]!, + async () => { + // The closest negative: the same tree with the app's button lifted above the band. + const aboveBand = KEYBOARD_TREE.map((node) => + node.index === 1 ? { ...node, rect: { x: 148, y: 470, width: 104, height: 110 } } : node, + ); + + await withIosContractDaemon( + [runnerSnapshotEntry(aboveBand), runnerTapEntry({})], + async (daemon) => { + const data = assertRpcOk(await daemon.callCommand('find', ['label', 'Form', 'focus'])); + assert.deepEqual({ x: data.x, y: data.y }, { x: 200, y: 525 }); + }, + ); + }, + PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS, +); + +test( + scenario('occlusion'), + async () => { + await withIosContractDaemon([runnerSnapshotEntry(coveredFormTree())], async (daemon) => { + const error = assertRpcError( + await daemon.callCommand('find', ['label', 'Form', 'focus']), + 'COMMAND_FAILED', + /Matched element @e2 is covered by another visible element and cannot be focused safely/, + ); + assert.equal(readErrorDetails(error).reason, 'target_covered'); + }); + }, + PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS, +); + +test( + scenario('disambiguation'), + async () => { + await withIosContractDaemon( + [ + runnerSnapshotEntry(twinFormTree()), + runnerSnapshotEntry(twinFormTree()), + runnerTapEntry({}), + ], + async (daemon) => { + const ambiguous = assertRpcError( + await daemon.callCommand('find', ['label', 'Form', 'focus']), + 'AMBIGUOUS_MATCH', + /find matched 2 elements for label "Form"/, + ); + assert.deepEqual(readErrorDetails(ambiguous).candidates, [ + '@e2 [button] "Form"', + '@e3 [button] "Form"', + ]); + + // --first is the only narrowing find offers, and the narrowed match goes through this seam. + assertRpcOk( + await daemon.callCommand('find', ['label', 'Form', 'focus'], { findFirst: true }), + ); + }, + ); + }, + PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS, +); + +test( + scenario('nonHittable'), + async () => { + // The match is a non-hittable label inside a hittable row: the `findAct` row's promotion retargets + // the dispatch to the row, so the focus lands at the row's center rather than the label's. + await withIosContractDaemon( + [runnerSnapshotEntry(nonHittableRowTree()), runnerTapEntry({})], + async (daemon) => { + const data = assertRpcOk( + await daemon.callCommand('find', ['label', 'Notifications', 'focus']), + ); + assert.deepEqual({ x: data.x, y: data.y }, { x: 200, y: 200 }); + }, + ); + }, + PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS, +); + +test( + scenario('responseConstruction'), + async () => { + await withIosContractDaemon( + [runnerSnapshotEntry(KEYBOARD_TREE), runnerTapEntry({})], + async (daemon) => { + const data = assertRpcOk(await daemon.callCommand('find', ['label', 'space', 'focus'])); + // The focus runtime's own payload: the point it dispatched plus its success text, with nothing + // hand-rolled on top — and no `refsGeneration`, which is what would make the ref pinnable + // (ADR 0014). + assert.equal(data.message, 'Focused (202, 779)'); + assert.equal(data.refsGeneration, undefined); + }, + ); + }, + PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS, +); + +test( + scenario('errorTaxonomy'), + async () => { + await withIosContractDaemon([runnerSnapshotEntry(KEYBOARD_TREE)], async (daemon) => { + assertRpcError( + await daemon.callCommand('find', ['label', 'Nowhere', 'focus']), + 'COMMAND_FAILED', + /find did not match any element/, + ); + }); + }, + PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS, +); + +/** The wire's error envelope keeps the typed refusal details one level below its code and hint. */ +function readErrorDetails(error: Record): Record { + const details = error.details; + assert.ok(details && typeof details === 'object', 'the error carries typed details'); + return details as Record; +} + +function coveredFormTree(): unknown[] { + return [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'Button', + label: 'Form', + rect: { x: 16, y: 790, width: 140, height: 44 }, + hittable: true, + }, + { + index: 2, + parentIndex: 0, + type: 'TabBar', + rect: { x: 0, y: 760, width: 390, height: 84 }, + hittable: true, + }, + ]; +} + +function twinFormTree(): unknown[] { + return [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 400, height: 800 } }, + { + index: 1, + parentIndex: 0, + type: 'Button', + label: 'Form', + rect: { x: 40, y: 200, width: 120, height: 44 }, + hittable: true, + }, + { + index: 2, + parentIndex: 0, + type: 'Button', + label: 'Form', + rect: { x: 240, y: 200, width: 120, height: 44 }, + hittable: true, + }, + ]; +} + +function nonHittableRowTree(): unknown[] { + return [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 400, height: 800 } }, + { + index: 1, + parentIndex: 0, + type: 'Cell', + rect: { x: 40, y: 160, width: 320, height: 80 }, + hittable: true, + }, + { + index: 2, + parentIndex: 1, + type: 'StaticText', + label: 'Notifications', + rect: { x: 60, y: 180, width: 160, height: 30 }, + hittable: false, + }, + ]; +} diff --git a/test/integration/interaction-contract/find-focus-dispatch.coverage.ts b/test/integration/interaction-contract/find-focus-dispatch.coverage.ts new file mode 100644 index 0000000000..183e81adab --- /dev/null +++ b/test/integration/interaction-contract/find-focus-dispatch.coverage.ts @@ -0,0 +1,17 @@ +import { definePathCoverage } from './coverage-manifest.ts'; + +export const FIND_FOCUS_DISPATCH_COVERAGE = definePathCoverage('find-focus-dispatch', { + disambiguation: + 'find-focus-dispatch disambiguation: two matches refuse with bounded candidates unless --first narrows', + occlusion: 'find-focus-dispatch occlusion: a covered match refuses to be focused', + keyboardOcclusion: [ + 'find-focus-dispatch keyboardOcclusion: a focus match behind the visible keyboard is refused with tap_keyboard_occludes_target', + 'find-focus-dispatch keyboardOcclusion: the same match above the keyboard band focuses', + ], + nonHittable: + 'find-focus-dispatch nonHittable: a non-hittable match focuses its promoted hittable ancestor', + responseConstruction: + 'find-focus-dispatch responseConstruction: find focus returns the shared point-focus payload', + errorTaxonomy: + 'find-focus-dispatch errorTaxonomy: an unmatched find focus fails with find locator diagnostics', +}); diff --git a/test/integration/interaction-contract/index.ts b/test/integration/interaction-contract/index.ts index 436b76bc18..bf9992036c 100644 --- a/test/integration/interaction-contract/index.ts +++ b/test/integration/interaction-contract/index.ts @@ -1,5 +1,6 @@ import type { ContractCoverageEntry } from './coverage-manifest.ts'; import { COORDINATE_COVERAGE } from './coordinate.coverage.ts'; +import { FIND_FOCUS_DISPATCH_COVERAGE } from './find-focus-dispatch.coverage.ts'; import { MAESTRO_DIRECT_SELECTOR_COVERAGE } from './maestro-direct-selector.coverage.ts'; import { MAESTRO_FALLBACK_COVERAGE } from './maestro-fallback.coverage.ts'; import { NATIVE_REF_COVERAGE } from './native-ref.coverage.ts'; @@ -19,6 +20,7 @@ export const CONTRACT_COVERAGE: readonly ContractCoverageEntry[] = [ ...TARGET_DRAG_COVERAGE, ...NATIVE_REF_COVERAGE, ...COORDINATE_COVERAGE, + ...FIND_FOCUS_DISPATCH_COVERAGE, ...MAESTRO_DIRECT_SELECTOR_COVERAGE, ...MAESTRO_FALLBACK_COVERAGE, ]; From 8fe670b71b49231c1b49807ea1f7f115dab84f2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 16:00:58 +0200 Subject: [PATCH 3/8] chore(gates): register the new selectors subpaths and the find-focus-dispatch row The selectors export enumeration gains `./interaction-action` and `./keyboard-occlusion`, the guarantee matrix's three keyboard cells now name the guard's new module, and the acknowledged-gap list grows by the four cells the new row declares, each linked to its tracking issue. --- scripts/layering/package-boundaries.test.ts | 2 ++ test/repo-guards/interaction-guarantees.test.ts | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index 34b03b60a2..bc3c73aac1 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -775,10 +775,12 @@ test('the real tree parses, declares, and passes R11', () => { '@agent-device/selectors/absence-observation-resolution', '@agent-device/selectors/ast', '@agent-device/selectors/engine', + '@agent-device/selectors/interaction-action', '@agent-device/selectors/interaction-error', '@agent-device/selectors/interaction-positionals', '@agent-device/selectors/interaction-targeting', '@agent-device/selectors/interaction-touch-point', + '@agent-device/selectors/keyboard-occlusion', '@agent-device/selectors/parameterized-recorded-fill', '@agent-device/selectors/press-retarget', '@agent-device/selectors/selector-pipeline', diff --git a/test/repo-guards/interaction-guarantees.test.ts b/test/repo-guards/interaction-guarantees.test.ts index e50ccaad5d..8418710ac0 100644 --- a/test/repo-guards/interaction-guarantees.test.ts +++ b/test/repo-guards/interaction-guarantees.test.ts @@ -164,6 +164,10 @@ test('acknowledged gaps are visible and bounded', () => { // (umbrella: https://github.com/callstack/agent-device/issues/1081). assert.deepEqual(gaps.sort(), [ 'coordinate/outcomeObservation', + 'find-focus-dispatch/offscreen', + 'find-focus-dispatch/outcomeObservation', + 'find-focus-dispatch/parentOwnedTouchPoint', + 'find-focus-dispatch/targetStability', 'maestro-direct-selector/errorTaxonomy', 'maestro-direct-selector/outcomeObservation', 'maestro-direct-selector/responseIdentity', From 72c771fc1cc3da5cb492caf30d7982ba3f16971a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 17:52:27 +0200 Subject: [PATCH 4/8] fix(selectors): name every interaction action in the refusal verb interactionVerb fell through to 'be tapped' for every action it did not list, so a keyboard- or coverage-blocked drag, swipe, or scroll endpoint was refused in the words of an action nobody asked for. Each action now names itself, and with no default arm a future InteractionAction fails to compile until it chooses the verb its refusal will use. --- .../selectors/src/interaction-action.test.ts | 28 +++++++++++++++++++ packages/selectors/src/interaction-action.ts | 25 +++++++++++++++-- 2 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 packages/selectors/src/interaction-action.test.ts diff --git a/packages/selectors/src/interaction-action.test.ts b/packages/selectors/src/interaction-action.test.ts new file mode 100644 index 0000000000..bbb959240b --- /dev/null +++ b/packages/selectors/src/interaction-action.test.ts @@ -0,0 +1,28 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { interactionVerb, type InteractionAction } from './interaction-action.ts'; + +// Exhaustive mapping over `InteractionAction` is the compiler's check. This pins the words +// themselves, so a refusal that starts describing a non-tap gesture as a tap — which is what the +// catch-all used to do for `drag` — is a deliberate change here, not an accident inside the map. +test('every action names itself in the refusal', () => { + const verbs: Array<[InteractionAction, string]> = [ + ['click', 'be tapped'], + ['press', 'be tapped'], + ['fill', 'be filled'], + ['focus', 'be focused'], + ['longPress', 'be long-pressed'], + ['hover', 'be hovered'], + ['scroll', 'be scrolled'], + ['swipe', 'be swiped'], + ['pinch', 'be pinched'], + ['pan', 'be panned'], + ['drag', 'be dragged'], + ['fling', 'be flung'], + ['rotate', 'be rotated'], + ['transform', 'be transformed'], + ]; + for (const [action, verb] of verbs) { + assert.equal(interactionVerb(action), verb, action); + } +}); diff --git a/packages/selectors/src/interaction-action.ts b/packages/selectors/src/interaction-action.ts index 4248f76589..4f2b4bea43 100644 --- a/packages/selectors/src/interaction-action.ts +++ b/packages/selectors/src/interaction-action.ts @@ -23,10 +23,15 @@ export type InteractionAction = /** * The passive verb a refusal uses for the action it declined, so every acting guard that refuses - * before dispatch describes the failure in the caller's own words. + * before dispatch describes the failure in the caller's own words. Every action is named + * explicitly and there is no `default`: a new `InteractionAction` fails to compile until it picks + * the verb its refusal will use, instead of silently being refused as a tap. */ export function interactionVerb(action: InteractionAction): string { switch (action) { + case 'click': + case 'press': + return 'be tapped'; case 'fill': return 'be filled'; case 'focus': @@ -35,7 +40,21 @@ export function interactionVerb(action: InteractionAction): string { return 'be long-pressed'; case 'hover': return 'be hovered'; - default: - return 'be tapped'; + case 'scroll': + return 'be scrolled'; + case 'swipe': + return 'be swiped'; + case 'pinch': + return 'be pinched'; + case 'pan': + return 'be panned'; + case 'drag': + return 'be dragged'; + case 'fling': + return 'be flung'; + case 'rotate': + return 'be rotated'; + case 'transform': + return 'be transformed'; } } From c2117ad62fbd5516985b6b69e2c33b3671ade517 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 17:52:27 +0200 Subject: [PATCH 5/8] test(selectors): pin the typed refusal, not just its message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The action-loop assertion checked only that the message contained the verb, so it would stay green on an unrelated rejection that happened to mention the action. Each iteration now runs the full typed refusal — COMMAND_FAILED, reason tap_keyboard_occludes_target, dispatched: 'no' — and matches the verb on the message of that same refusal. --- .../selectors/src/keyboard-occlusion.test.ts | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/packages/selectors/src/keyboard-occlusion.test.ts b/packages/selectors/src/keyboard-occlusion.test.ts index 20107b4595..1d9a272f33 100644 --- a/packages/selectors/src/keyboard-occlusion.test.ts +++ b/packages/selectors/src/keyboard-occlusion.test.ts @@ -60,8 +60,12 @@ function guard(params: { }); } -/** Runs the guard expecting a refusal, and hands back that refusal's typed details. */ -function keyboardRefusalDetails(params: Parameters[0]) { +/** Runs the guard expecting a refusal, and hands back that refusal's message and typed details. */ +function keyboardRefusalDetails(params: Parameters[0]): { + message: string; + details: Record; +} { + let message = ''; let details: Record | undefined; assert.throws( () => guard(params), @@ -69,16 +73,17 @@ function keyboardRefusalDetails(params: Parameters[0]) { assert.ok(error instanceof Error); assert.equal((error as { code?: string }).code, 'COMMAND_FAILED'); assert.match(error.message, /Ref @e2 is behind the visible keyboard/); + message = error.message; details = (error as { details?: Record }).details; return true; }, ); assert.ok(details, 'the refusal carries details'); - return details; + return { message, details }; } test('the refusal carries the shared reason, the target, and the band it measured', () => { - assert.deepEqual(keyboardRefusalDetails({ snapshot: keyboardCoveredTabBarSnapshot() }), { + assert.deepEqual(keyboardRefusalDetails({ snapshot: keyboardCoveredTabBarSnapshot() }).details, { ...TAP_KEYBOARD_OCCLUDES_TARGET_DETAILS, ref: '@e2', rect: KEYBOARD_COVERED_TAB_BAR_RECT, @@ -93,9 +98,18 @@ test('the refusal names the action it declined', () => { ['fill', 'be filled'], ['focus', 'be focused'], ] as const) { - assert.throws( - () => guard({ snapshot: keyboardCoveredTabBarSnapshot(), action }), + // Each iteration runs the full typed refusal — code, reason, dispatched: 'no' — so the verb + // check cannot pass on an unrelated rejection that merely mentions the action. + const { message, details } = keyboardRefusalDetails({ + snapshot: keyboardCoveredTabBarSnapshot(), + action, + }); + assert.equal(details.reason, 'tap_keyboard_occludes_target'); + assert.equal(details.dispatched, 'no'); + assert.match( + message, new RegExp(`cannot ${verb} safely`), + `${action} refuses with the "${verb}" verb`, ); } }); From 0f4285661f7f733dea61c0c4411acc90be771e7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 17:52:37 +0200 Subject: [PATCH 6/8] fix(contracts): say what find-focus-dispatch actually enforces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit nonHittable claimed runtime for a guarantee whose second half never runs: the seam promotes a non-hittable match to its hittable ancestor but never annotates the one promotion declines, so it cannot claim the whole cell. It is an owned gap on #1081 now, and the Layer-3 scenario that would have overclaimed the waived cell is gone — the promotion half stays proven by the seam's unit test named in the reason. The responseIdentity rationale named a data.ref the focus and type payloads never emit; it describes the omission instead. --- .../contracts/src/interaction-guarantees.ts | 14 ++++--- .../find-focus-dispatch.contract.test.ts | 39 ------------------- .../find-focus-dispatch.coverage.ts | 2 - .../interaction-guarantees.test.ts | 1 + 4 files changed, 9 insertions(+), 47 deletions(-) diff --git a/packages/contracts/src/interaction-guarantees.ts b/packages/contracts/src/interaction-guarantees.ts index dcea6216aa..9a8b40e200 100644 --- a/packages/contracts/src/interaction-guarantees.ts +++ b/packages/contracts/src/interaction-guarantees.ts @@ -581,12 +581,14 @@ export const INTERACTION_DISPATCH_PATHS: Record ` reaches), and `type`'s @@ -598,7 +600,7 @@ export const INTERACTION_DISPATCH_PATHS: Record { - // The match is a non-hittable label inside a hittable row: the `findAct` row's promotion retargets - // the dispatch to the row, so the focus lands at the row's center rather than the label's. - await withIosContractDaemon( - [runnerSnapshotEntry(nonHittableRowTree()), runnerTapEntry({})], - async (daemon) => { - const data = assertRpcOk( - await daemon.callCommand('find', ['label', 'Notifications', 'focus']), - ); - assert.deepEqual({ x: data.x, y: data.y }, { x: 200, y: 200 }); - }, - ); - }, - PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS, -); - test( scenario('responseConstruction'), async () => { @@ -204,24 +186,3 @@ function twinFormTree(): unknown[] { }, ]; } - -function nonHittableRowTree(): unknown[] { - return [ - { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 400, height: 800 } }, - { - index: 1, - parentIndex: 0, - type: 'Cell', - rect: { x: 40, y: 160, width: 320, height: 80 }, - hittable: true, - }, - { - index: 2, - parentIndex: 1, - type: 'StaticText', - label: 'Notifications', - rect: { x: 60, y: 180, width: 160, height: 30 }, - hittable: false, - }, - ]; -} diff --git a/test/integration/interaction-contract/find-focus-dispatch.coverage.ts b/test/integration/interaction-contract/find-focus-dispatch.coverage.ts index 183e81adab..9b92cc3890 100644 --- a/test/integration/interaction-contract/find-focus-dispatch.coverage.ts +++ b/test/integration/interaction-contract/find-focus-dispatch.coverage.ts @@ -8,8 +8,6 @@ export const FIND_FOCUS_DISPATCH_COVERAGE = definePathCoverage('find-focus-dispa 'find-focus-dispatch keyboardOcclusion: a focus match behind the visible keyboard is refused with tap_keyboard_occludes_target', 'find-focus-dispatch keyboardOcclusion: the same match above the keyboard band focuses', ], - nonHittable: - 'find-focus-dispatch nonHittable: a non-hittable match focuses its promoted hittable ancestor', responseConstruction: 'find-focus-dispatch responseConstruction: find focus returns the shared point-focus payload', errorTaxonomy: diff --git a/test/repo-guards/interaction-guarantees.test.ts b/test/repo-guards/interaction-guarantees.test.ts index 8418710ac0..e2442fce1c 100644 --- a/test/repo-guards/interaction-guarantees.test.ts +++ b/test/repo-guards/interaction-guarantees.test.ts @@ -164,6 +164,7 @@ test('acknowledged gaps are visible and bounded', () => { // (umbrella: https://github.com/callstack/agent-device/issues/1081). assert.deepEqual(gaps.sort(), [ 'coordinate/outcomeObservation', + 'find-focus-dispatch/nonHittable', 'find-focus-dispatch/offscreen', 'find-focus-dispatch/outcomeObservation', 'find-focus-dispatch/parentOwnedTouchPoint', From ebb95613aa45ed6306f6f99d9a69411300e464f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 17:58:38 +0200 Subject: [PATCH 7/8] refactor(find): drop the dead payload fallback and share the refusal verb Both mutating focus legs ended in `response ?? { ref: match.ref }`, but executeFocusPoint and executeBoundTypeText always return a payload, so the branch never ran and it was the only place this route could have issued a data.ref. Removing it makes the responseIdentity waiver a structural fact rather than a claim about a branch. rejectCoveredFindMatch now takes the InteractionAction and names itself through interactionVerb, so the covered and keyboard refusals on this route draw their verbs from one source. --- .../src/execution/interaction/internal/find.ts | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/packages/daemon/src/execution/interaction/internal/find.ts b/packages/daemon/src/execution/interaction/internal/find.ts index 2fea9bdd04..b65ccfe46d 100644 --- a/packages/daemon/src/execution/interaction/internal/find.ts +++ b/packages/daemon/src/execution/interaction/internal/find.ts @@ -14,6 +14,10 @@ import { runNodePipelineStages } from '@agent-device/selectors/selector-pipeline import { SELECTOR_PIPELINE_POLICIES } from '@agent-device/selectors/selector-pipeline-policy'; import { coveredInteractionDetails } from '@agent-device/selectors/interaction-error'; import { assertTapTargetClearOfVisibleKeyboard } from '@agent-device/selectors/keyboard-occlusion'; +import { + interactionVerb, + type InteractionAction, +} from '@agent-device/selectors/interaction-action'; import { normalizeError } from '@agent-device/kernel/errors'; import { centerOfRect, @@ -383,7 +387,7 @@ async function handleFindType( contextFromFlags(logPath, req.flags, session.appBundleId, session.trace?.outPath), ); recordFindAction(ctx, match, 'type'); - return { ok: true, data: response ?? { ref: match.ref } }; + return { ok: true, data: response }; } async function dispatchFocusForFindMatch( @@ -391,7 +395,7 @@ async function dispatchFocusForFindMatch( match: ResolvedMatch, ): Promise { const { req, logPath } = ctx; - const coveredResponse = rejectCoveredFindMatch(match, 'be focused'); + const coveredResponse = rejectCoveredFindMatch(match, 'focus'); if (coveredResponse) return coveredResponse; const coords = match.resolvedNode.rect ? centerOfRect(match.resolvedNode.rect) : null; if (!coords) { @@ -427,15 +431,18 @@ async function dispatchFocusForFindMatch( coords, contextFromFlags(logPath, req.flags, session.appBundleId, session.trace?.outPath), ); - return { ok: true, data: response ?? { ref: match.ref } }; + return { ok: true, data: response }; } -function rejectCoveredFindMatch(match: ResolvedMatch, interaction: string): DaemonResponse | null { +function rejectCoveredFindMatch( + match: ResolvedMatch, + action: InteractionAction, +): DaemonResponse | null { const blockedNode = match.occludedNode; if (!blockedNode) return null; return errorResponse( 'COMMAND_FAILED', - `Matched element ${match.ref} is covered by another visible element and cannot ${interaction} safely`, + `Matched element ${match.ref} is covered by another visible element and cannot ${interactionVerb(action)} safely`, coveredInteractionDetails(blockedNode), ); } From a0135d0d681160155da76d5821d6a712050e9972 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 18:23:30 +0200 Subject: [PATCH 8/8] docs(contracts): point the find-focus-dispatch gaps at their own issue The four waivers this seam introduces (offscreen, parentOwnedTouchPoint, nonHittable, targetStability) now cite #3734, which owns closing them. outcomeObservation stays on #3335 and the pre-existing rows are untouched. --- packages/contracts/src/interaction-guarantees.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/contracts/src/interaction-guarantees.ts b/packages/contracts/src/interaction-guarantees.ts index 9a8b40e200..2d0a6280ae 100644 --- a/packages/contracts/src/interaction-guarantees.ts +++ b/packages/contracts/src/interaction-guarantees.ts @@ -154,6 +154,8 @@ export type InteractionPathContract = { const GAPS_UMBRELLA_ISSUE = 'https://github.com/callstack/agent-device/issues/1081'; const TAP_OUTCOME_NOT_OBSERVED_GAP_ISSUE = 'https://github.com/callstack/agent-device/issues/3335'; +// The four guarantees this route's own seam leaves open are owned by #3734. +const FIND_FOCUS_DISPATCH_GAPS_ISSUE = 'https://github.com/callstack/agent-device/issues/3734'; /** * The moving-target gate's shared cell (#3471). Scoped to the two commands that dispatch a tap the @@ -573,13 +575,13 @@ export const INTERACTION_DISPATCH_PATHS: Record ` reaches), and `type`'s @@ -635,7 +637,7 @@ export const INTERACTION_DISPATCH_PATHS: Record