fix(find): refuse a focus match behind the visible keyboard (#2622)

packages/contracts/src/interaction-guarantees.ts

@@ -96,6 +96,7 @@ export const INTERACTION_PATH_IDS = [
   'target-drag',
   'native-ref',
   'coordinate',
+  'find-focus-dispatch',
   'maestro-direct-selector',
   'maestro-non-hittable-fallback',
 ] as const;
@@ -153,6 +154,8 @@ export type InteractionPathContract = {
 
 const GAPS_UMBRELLA_ISSUE = 'https://github.com/callstack/agent-device/issues/1081';
 const TAP_OUTCOME_NOT_OBSERVED_GAP_ISSUE = 'https://github.com/callstack/agent-device/issues/3335';
+// The four guarantees this route's own seam leaves open are owned by #3734.
+const FIND_FOCUS_DISPATCH_GAPS_ISSUE = 'https://github.com/callstack/agent-device/issues/3734';
 
 /**
  * The moving-target gate's shared cell (#3471). Scoped to the two commands that dispatch a tap the
@@ -233,7 +236,7 @@ const RUNTIME_TREE_SHARED_GUARANTEES = {
   // `occlusion` nor `offscreen` can reach it.
   keyboardOcclusion: {
     kind: 'runtime',
-    via: 'src/commands/interaction/runtime/keyboard-occlusion.ts#assertTapTargetClearOfVisibleKeyboard',
+    via: 'packages/selectors/src/keyboard-occlusion.ts#assertTapTargetClearOfVisibleKeyboard',
   },
   // #1542: the base decision is the contracts-owned snapshot visibility resolver (bulk accessibility
   // tree), but throwIfOffscreenInteractionTarget is the actual end-to-end
@@ -348,7 +351,7 @@ export const INTERACTION_DISPATCH_PATHS: Record<InteractionPathId, InteractionPa
       },
       keyboardOcclusion: {
         kind: 'runtime',
-        via: 'src/commands/interaction/runtime/keyboard-occlusion.ts#assertTapTargetClearOfVisibleKeyboard',
+        via: 'packages/selectors/src/keyboard-occlusion.ts#assertTapTargetClearOfVisibleKeyboard',
       },
       offscreen: {
         kind: 'runtime',
@@ -420,7 +423,7 @@ export const INTERACTION_DISPATCH_PATHS: Record<InteractionPathId, InteractionPa
       // the snapshot annotation the pipeline reads.
       keyboardOcclusion: {
         kind: 'runtime',
-        via: 'src/commands/interaction/runtime/keyboard-occlusion.ts#assertTapTargetClearOfVisibleKeyboard',
+        via: 'packages/selectors/src/keyboard-occlusion.ts#assertTapTargetClearOfVisibleKeyboard',
       },
       parentOwnedTouchPoint: {
         kind: 'inapplicable',
@@ -545,6 +548,99 @@ export const INTERACTION_DISPATCH_PATHS: Record<InteractionPathId, InteractionPa
       },
     },
   },
+  'find-focus-dispatch': {
+    description:
+      'Mutating `find <locator> focus`, and the focus leg of `find <locator> type`: find captures its own tree, ranks its own candidate set, runs the node stages its `findAct` row declares, then dispatches a point focus itself (`type` then dispatches the text) instead of re-entering the interaction leaf. `find click`/`find fill` are NOT this path — they hand the resolved node to `runtime-ref` as a pre-resolved target (#1654). `focus <x> <y>` is the coordinate route. Every cell below classifies what the `findAct` row plus this seam actually run, not what the acting rows run.',
+    commands: ['find'],
+    guarantees: {
+      // find's own ranking, but under the same ambiguity contract the runtime paths use: one
+      // `reject-candidates` policy row (#1625), `--first`/`--last` the only narrowing, and
+      // `preferOnscreenMatches` as the tie-break rather than a geometry-wins rule.
+      disambiguation: {
+        kind: 'runtime',
+        via: 'packages/daemon/src/execution/interaction/internal/find-match-resolution.ts#resolveFindMatch',
+      },
+      occlusion: {
+        kind: 'runtime',
+        via: 'packages/selectors/src/selector-pipeline.ts#runNodePipelineStages',
+      },
+      // #2622: the seam dispatches the point itself, so the shared #2589 guard runs here rather
+      // than at the interaction leaf's door. Same refusal, same typed details, measured at the
+      // center the seam dispatches.
+      keyboardOcclusion: {
+        kind: 'runtime',
+        via: 'packages/selectors/src/keyboard-occlusion.ts#assertTapTargetClearOfVisibleKeyboard',
+      },
+      offscreen: {
+        kind: 'waived',
+        reason:
+          'gap: the `findAct` row declares `offscreen: "ignore"` because find ranks across the whole tree and prefers on-screen matches rather than refusing off-screen ones (#1625), and focus/type never re-enter the acting leaf where the runtime rows refuse. So the seam dispatches at a captured rect center the acting paths would reject as outside the root viewport — a closed drawer or an off-viewport carousel is a silent no-op focus here.',
+        trackingIssue: FIND_FOCUS_DISPATCH_GAPS_ISSUE,
+      },
+      parentOwnedTouchPoint: {
+        kind: 'waived',
+        reason:
+          'gap: the seam aims `centerOfRect` of the resolved node rather than the parent-owned point `resolveInteractionTouchPoint` chooses, so a match whose center is tiled by an independently interactive descendant activates that descendant instead of failing closed — and the keyboard guard above measures the same center the tap uses, which is the point actually dispatched but not the point the acting paths would aim.',
+        trackingIssue: FIND_FOCUS_DISPATCH_GAPS_ISSUE,
+      },
+      // The guarantee is promotion AND annotation. This seam runs only the first half, so the
+      // cell cannot claim `runtime` for the whole guarantee (the same shape `native-ref` classifies
+      // the other way round, annotation without promotion).
+      nonHittable: {
+        kind: 'waived',
+        reason:
+          "gap: the promotion half does run — the `findAct` row declares `hittable-ancestor-below-root`, so the seam dispatches the promoted ancestor's center (regression-proven in packages/daemon/src/execution/interaction/internal/__tests__/find.test.ts) — but the annotation half never does: the response is the focus runtime's point payload, so a match still reporting `hittable: false` after promotion is accepted silently, where press/click/fill answer with `targetHittable: false` plus a hint. Closing it needs the annotation composed at this seam, which is response-shape work the `responseIdentity` waiver below also records.",
+        trackingIssue: FIND_FOCUS_DISPATCH_GAPS_ISSUE,
+      },
+      // One construction site per dispatch kind, both shared with the plain commands: the focus
+      // payload is `executeFocusPoint`'s (the same owner `focus <x> <y>` reaches), and `type`'s
+      // second dispatch is `executeBoundTypeText`'s. The seam hand-rolls neither.
+      responseConstruction: {
+        kind: 'runtime',
+        via: 'packages/daemon/src/execution/focus-runtime.ts#executeFocusPoint',
+      },
+      responseIdentity: {
+        kind: 'waived',
+        reason:
+          "Intentional: the payload is the focus (or text) dispatch result, which omits `refLabel`, `selectorChain`, and `data.ref` entirely — this route issues no pinnable or diagnostic response identity, since ADR 0014 keeps mutating find's pre-action ref diagnostic-only (no `refsGeneration`) and the seam never enters the ref/selector response builder that mints the identity fields.",
+      },
+      verifyEvidence: {
+        kind: 'inapplicable',
+        reason:
+          'find exposes no --verify flag: its flag surface is snapshot depth/raw plus --first/--last, and --verify would have no post-action capture owner on a route whose response is the dispatch payload.',
+      },
+      settleObservation: {
+        kind: 'inapplicable',
+        reason:
+          'find exposes no --settle flag for the same reason; settling is built on the tree-based runtime baseline and the captures the acting leaf owns.',
+      },
+      // Locator misses and candidate sets use find's shared diagnostics, and a pre-action guard
+      // refusal travels as the AppError its owner built (`coveredInteractionDetails`,
+      // TAP_KEYBOARD_OCCLUDES_TARGET_DETAILS) through `normalizeError`, so code, hint, and reason
+      // reach the wire unchanged.
+      errorTaxonomy: {
+        kind: 'runtime',
+        via: 'packages/daemon/src/execution/selector-match-errors.ts#buildAmbiguousMatchError',
+      },
+      resolutionDisclosure: {
+        kind: 'inapplicable',
+        reason:
+          'The additive `resolution` field discloses the provenance of an identity the response issues; this seam issues none (ADR 0014), and its ambiguity is disclosed as the bounded candidate error the `errorTaxonomy` cell names instead.',
+      },
+      outcomeObservation: {
+        kind: 'waived',
+        reason:
+          'gap: the response reports the focus (and for `type`, the text) dispatch only. Nothing observes that the field took focus or that the text landed, and unlike the tap-shaped paths this route has no opt-in --verify/--settle that could carry such evidence.',
+        trackingIssue: TAP_OUTCOME_NOT_OBSERVED_GAP_ISSUE,
+      },
+      targetStability: {
+        kind: 'waived',
+        reason:
+          "gap: the seam dispatches a rect center captured one runner round trip earlier without the #3471 re-observation, so a focus issued right after an action that animated the surface can aim a position the target has already vacated and report success on the vacancy. The gate itself is the acting leaf's (`guardResolvedTargetAgainstMotion`), and this route does not pass through it.",
+        trackingIssue: FIND_FOCUS_DISPATCH_GAPS_ISSUE,
+      },
+    },
+  },
   'maestro-direct-selector': {
     description:
       'An explicit Maestro-compatible simple-selector click completed as an XCTest element tap. This path is selected only when the runner was allowed to use the non-hittable coordinate fallback but reported that it did not use it.',

packages/daemon/src/execution/interaction/internal/__tests__/find-keyboard-occlusion.test.ts

@@ -0,0 +1,93 @@
+import { test, expect, vi, beforeEach } from 'vitest';
+import { keyboardCoveredTabBarSnapshot } from '@agent-device/test-kit';
+import type { SnapshotState } from '@agent-device/kernel/snapshot';
+import {
+  mockFocusPoint,
+  resetFindTouchRuntimeFixture,
+  runFindClickScenario,
+} from './find-touch-runtime-fixture.ts';
+import { mockTypeText } from '../../../__tests__/interaction-get-runtime-fixture.ts';
+import { refFrameState } from '../../../../core/ref-frame.ts';
+
+vi.mock('../../../../core/snapshot-interactor-capture.ts', async () => {
+  const fixture = await import('../../../../__tests__/support/legacy-snapshot-capture-fixture.ts');
+  return { captureSnapshotWithInteractor: fixture.captureSnapshotThroughLegacyDispatchFixture };
+});
+
+beforeEach(() => {
+  resetFindTouchRuntimeFixture();
+});
+
+function keyboardTree(targetRect?: SnapshotState['nodes'][number]['rect']): SnapshotState {
+  return keyboardCoveredTabBarSnapshot(targetRect ? { targetRect } : {});
+}
+
+type Refusal = {
+  code?: string;
+  message?: string;
+  hint?: string;
+  details?: Record<string, unknown>;
+};
+
+function assertKeyboardRefusal(
+  response: { ok: boolean; error?: Refusal },
+  verb: string,
+): asserts response {
+  expect(response.ok).toBe(false);
+  const error = response.error as Refusal;
+  expect(error.code).toBe('COMMAND_FAILED');
+  expect(error.message).toBe(`Ref @e2 is behind the visible keyboard and cannot ${verb} safely`);
+  expect(error.details?.reason).toBe('tap_keyboard_occludes_target');
+  expect(error.details?.ref).toBe('@e2');
+  expect(error.details?.rect).toEqual({ x: 148, y: 791, width: 104, height: 83 });
+  expect(error.details?.keyboardFrame).toEqual({ x: 0, y: 583, width: 402, height: 291 });
+  expect(error.details?.dispatched).toBe('no');
+  expect(error.hint).toMatch(/End editing first/);
+}
+
+test('find focus refuses a match whose tap point sits behind the visible keyboard', async () => {
+  const { response, session } = await runFindClickScenario({
+    positionals: ['Form', 'focus'],
+    nodes: keyboardTree().nodes,
+  });
+
+  assertKeyboardRefusal(response, 'be focused');
+  expect(mockFocusPoint).not.toHaveBeenCalled();
+  // ADR 0014: the refusal is pre-action, so the captured frame stays pinnable.
+  expect(refFrameState(session)).toBe('active');
+});
+
+test('find type refuses before the focus or the text dispatch', async () => {
+  const { response, session } = await runFindClickScenario({
+    positionals: ['Form', 'type', 'hello'],
+    nodes: keyboardTree().nodes,
+  });
+
+  assertKeyboardRefusal(response, 'be focused');
+  expect(mockFocusPoint).not.toHaveBeenCalled();
+  expect(mockTypeText).not.toHaveBeenCalled();
+  expect(refFrameState(session)).toBe('active');
+});
+
+test('find focus on a match above the keyboard band still focuses', async () => {
+  const { response, session } = await runFindClickScenario({
+    positionals: ['Form', 'focus'],
+    nodes: keyboardTree({ x: 148, y: 470, width: 104, height: 110 }).nodes,
+  });
+
+  expect(response.ok).toBe(true);
+  expect(mockFocusPoint).toHaveBeenCalledTimes(1);
+  expect(refFrameState(session)).toBe('expired');
+});
+
+test('find focus on a reported keyboard key still focuses it', async () => {
+  // The closest negative: the same guard excuses the keyboard's own control, which sits inside the
+  // band it would otherwise refuse.
+  const { response } = await runFindClickScenario({
+    positionals: ['label', 'space', 'focus'],
+    nodes: keyboardTree().nodes,
+  });
+
+  expect(response.ok).toBe(true);
+  expect(mockFocusPoint).toHaveBeenCalledTimes(1);
+});

packages/daemon/src/execution/interaction/internal/find.ts

@@ -13,6 +13,12 @@ import {
 import { runNodePipelineStages } from '@agent-device/selectors/selector-pipeline';
 import { SELECTOR_PIPELINE_POLICIES } from '@agent-device/selectors/selector-pipeline-policy';
 import { coveredInteractionDetails } from '@agent-device/selectors/interaction-error';
+import { assertTapTargetClearOfVisibleKeyboard } from '@agent-device/selectors/keyboard-occlusion';
+import {
+  interactionVerb,
+  type InteractionAction,
+} from '@agent-device/selectors/interaction-action';
+import { normalizeError } from '@agent-device/kernel/errors';
 import {
   centerOfRect,
   type SnapshotAddressableNode,
@@ -381,20 +387,34 @@ async function handleFindType(
     contextFromFlags(logPath, req.flags, session.appBundleId, session.trace?.outPath),
   );
   recordFindAction(ctx, match, 'type');
-  return { ok: true, data: response ?? { ref: match.ref } };
+  return { ok: true, data: response };
 }
 
 async function dispatchFocusForFindMatch(
   ctx: FindContext,
   match: ResolvedMatch,
 ): Promise<DaemonResponse> {
   const { req, logPath } = ctx;
-  const coveredResponse = rejectCoveredFindMatch(match, 'be focused');
+  const coveredResponse = rejectCoveredFindMatch(match, 'focus');
   if (coveredResponse) return coveredResponse;
   const coords = match.resolvedNode.rect ? centerOfRect(match.resolvedNode.rect) : null;
   if (!coords) {
     return errorResponse('COMMAND_FAILED', 'matched element has no bounds');
   }
+  // The same keyboard decision the acting paths run (#2589/#2622): this seam dispatches the point
+  // itself instead of re-entering the interaction leaf, so the guard has to run here.
+  try {
+    assertTapTargetClearOfVisibleKeyboard({
+      nodes: match.tree.nodes,
+      node: match.resolvedNode,
+      action: 'focus',
+      label: `Ref ${match.ref}`,
+      ...(match.keyboard ? { keyboard: match.keyboard } : {}),
+      tapPoint: coords,
+    });
+  } catch (error) {
+    return { ok: false, error: normalizeError(error) };
+  }
   // ADR 0014 side-effect seam: mutating find focus/type dispatch the device
   // command directly (they do not re-enter the interaction leaf), so expire the
   // frame here before the device op. Pre-seam guards above preserve the frame.
@@ -411,15 +431,18 @@ async function dispatchFocusForFindMatch(
     coords,
     contextFromFlags(logPath, req.flags, session.appBundleId, session.trace?.outPath),
   );
-  return { ok: true, data: response ?? { ref: match.ref } };
+  return { ok: true, data: response };
 }
 
-function rejectCoveredFindMatch(match: ResolvedMatch, interaction: string): DaemonResponse | null {
+function rejectCoveredFindMatch(
+  match: ResolvedMatch,
+  action: InteractionAction,
+): DaemonResponse | null {
   const blockedNode = match.occludedNode;
   if (!blockedNode) return null;
   return errorResponse(
     'COMMAND_FAILED',
-    `Matched element ${match.ref} is covered by another visible element and cannot ${interaction} safely`,
+    `Matched element ${match.ref} is covered by another visible element and cannot ${interactionVerb(action)} safely`,
     coveredInteractionDetails(blockedNode),
   );
 }

packages/platform-apple/src/runner/__tests__/snapshot-presentation.test.ts

@@ -251,7 +251,7 @@ test('a runner payload with the hittable bit absent presents without declaring i
 // what cannot be placed belongs to `readSnapshotKeyboardBandFact` in @agent-device/kernel, which owns
 // that table; this seam owns forwarding AND the proof that it forwards through that reader. The
 // same-process capture path feeds `result.keyboard` straight into the tap/click occlusion guard in
-// src/commands/interaction/runtime/keyboard-occlusion.ts before any serialization, so a seam that
+// packages/selectors/src/keyboard-occlusion.ts before any serialization, so a seam that
 // passed the raw wire value through would hand the guard an unvalidated shape with nothing red.
 // The malformed cases below are what prove the routing; kernel/src/record.test.ts owns the table.
 

packages/selectors/package.json

@@ -40,6 +40,10 @@
       "types": "./src/absence-observation-resolution.ts",
       "default": "./src/absence-observation-resolution.ts"
     },
+    "./interaction-action": {
+      "types": "./src/interaction-action.ts",
+      "default": "./src/interaction-action.ts"
+    },
     "./interaction-error": {
       "types": "./src/interaction-error.ts",
       "default": "./src/interaction-error.ts"
@@ -56,6 +60,10 @@
       "types": "./src/interaction-touch-point.ts",
       "default": "./src/interaction-touch-point.ts"
     },
+    "./keyboard-occlusion": {
+      "types": "./src/keyboard-occlusion.ts",
+      "default": "./src/keyboard-occlusion.ts"
+    },
     "./press-retarget": {
       "types": "./src/press-retarget.ts",
       "default": "./src/press-retarget.ts"

packages/selectors/src/interaction-action.test.ts

@@ -0,0 +1,28 @@
+import { test } from 'vitest';
+import assert from 'node:assert/strict';
+import { interactionVerb, type InteractionAction } from './interaction-action.ts';
+
+// Exhaustive mapping over `InteractionAction` is the compiler's check. This pins the words
+// themselves, so a refusal that starts describing a non-tap gesture as a tap — which is what the
+// catch-all used to do for `drag` — is a deliberate change here, not an accident inside the map.
+test('every action names itself in the refusal', () => {
+  const verbs: Array<[InteractionAction, string]> = [
+    ['click', 'be tapped'],
+    ['press', 'be tapped'],
+    ['fill', 'be filled'],
+    ['focus', 'be focused'],
+    ['longPress', 'be long-pressed'],
+    ['hover', 'be hovered'],
+    ['scroll', 'be scrolled'],
+    ['swipe', 'be swiped'],
+    ['pinch', 'be pinched'],
+    ['pan', 'be panned'],
+    ['drag', 'be dragged'],
+    ['fling', 'be flung'],
+    ['rotate', 'be rotated'],
+    ['transform', 'be transformed'],
+  ];
+  for (const [action, verb] of verbs) {
+    assert.equal(interactionVerb(action), verb, action);
+  }
+});