diff --git a/packages/daemon/src/__tests__/support/command-surface-override-fake.ts b/packages/daemon/src/__tests__/support/command-surface-override-fake.ts new file mode 100644 index 0000000000..256754a6b2 --- /dev/null +++ b/packages/daemon/src/__tests__/support/command-surface-override-fake.ts @@ -0,0 +1,62 @@ +import type { + CreateDaemonCommandSurface, + DaemonCommandSurface, +} from '@agent-device/contracts/daemon-command-surface'; +import type { AgentDeviceRuntimeConfig } from '@agent-device/contracts/runtime-contract'; +import type { SnapshotState } from '@agent-device/kernel/snapshot'; +import { fakeCreateCommandSurface } from './command-surface-fake.ts'; + +export type CommandSurfaceMembers = { + capture?: Partial; + selectors?: Partial; + interactions?: Partial; +}; + +/** + * The inert command surface with the members a suite names answered by that suite. The daemon + * suites that reach a selector read or a touch action use it to state what the command family + * answered, so every assertion stays about what the daemon does with the answer. `members` runs + * once per surface the daemon builds and receives that request's runtime config, which is how a + * member reaches the same `backend` and `sessions` seams the real command family reaches. + */ +export function commandSurfaceWithMembers( + members: (config: AgentDeviceRuntimeConfig) => CommandSurfaceMembers, +): CreateDaemonCommandSurface { + return async (config) => { + const inert = await fakeCreateCommandSurface(config); + const own = members(config); + return { + ...inert, + signal: config.signal, + capture: { ...inert.capture, ...own.capture }, + selectors: { ...inert.selectors, ...own.selectors }, + interactions: { ...inert.interactions, ...own.interactions }, + }; + }; +} + +/** + * One capture the way the real command family takes it: through the config's backend, then + * written back as the session's stored snapshot through the config's session store. + */ +export async function captureAndStoreSnapshot( + config: AgentDeviceRuntimeConfig, + params: { + session?: string; + requestId?: string; + signal?: AbortSignal; + interactiveOnly?: boolean; + includeRects?: boolean; + }, +): Promise { + const name = params.session ?? 'default'; + const captured = await config.backend.captureSnapshot?.( + { session: name, requestId: params.requestId, signal: params.signal ?? config.signal }, + { interactiveOnly: params.interactiveOnly, includeRects: params.includeRects }, + ); + const snapshot = captured?.snapshot; + if (!snapshot) throw new Error('captureAndStoreSnapshot: the backend returned no snapshot'); + const current = await config.sessions?.get(name); + await config.sessions?.set({ ...(current ?? { name }), name, snapshot }); + return snapshot; +} diff --git a/packages/daemon/src/__tests__/support/legacy-capture-interactor-resolution.ts b/packages/daemon/src/__tests__/support/legacy-capture-interactor-resolution.ts new file mode 100644 index 0000000000..b910b87536 --- /dev/null +++ b/packages/daemon/src/__tests__/support/legacy-capture-interactor-resolution.ts @@ -0,0 +1,26 @@ +import type { Interactor } from '@agent-device/contracts/interactor-types'; +import { installInteractorResolution } from '../../core/interactor-resolution.ts'; +import { captureSnapshotThroughLegacyDispatchFixture } from './legacy-snapshot-capture-fixture.ts'; + +/** + * Routes the daemon's legacy snapshot capture through the shared capture double + * (`legacyDispatchCapture`) by installing the interactor resolution the process root would + * compose. The resolved interactor answers `snapshot` only, so a suite that reaches any other + * interactor member fails loudly instead of touching a device. + * + * Installation is process-global and each vitest file runs in its own module graph, so a suite + * calls this once at module scope. + */ +export function installLegacyCaptureInteractorResolution(): void { + installInteractorResolution({ + resolve: async (device, runnerContext) => + ({ + snapshot: async (options) => + await captureSnapshotThroughLegacyDispatchFixture({ + device, + runnerContext, + options: options ?? {}, + }), + }) as Interactor, + }); +} diff --git a/src/__tests__/daemon-composition/interaction/find-ambiguous-match.test.ts b/packages/daemon/src/execution/interaction/internal/find-ambiguous-match.test.ts similarity index 82% rename from src/__tests__/daemon-composition/interaction/find-ambiguous-match.test.ts rename to packages/daemon/src/execution/interaction/internal/find-ambiguous-match.test.ts index 0dd8a3d508..179a746085 100644 --- a/src/__tests__/daemon-composition/interaction/find-ambiguous-match.test.ts +++ b/packages/daemon/src/execution/interaction/internal/find-ambiguous-match.test.ts @@ -1,24 +1,38 @@ -import { test, expect, vi, beforeEach } from 'vitest'; - -vi.mock('@agent-device/device-selection/dispatch-resolve', async (importOriginal) => { - const actual = (await importOriginal()) as Record; - return { - ...actual, - resolveTargetDevice: actual.resolveTargetDevice, - }; -}); -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', async () => { - const fixture = - await import('@agent-device/daemon/testing/__tests__/legacy-snapshot-capture-fixture'); - return { captureSnapshotWithInteractor: fixture.captureSnapshotThroughLegacyDispatchFixture }; -}); - +import { test, expect, beforeEach } from 'vitest'; +import type { FindReadResult } from '@agent-device/contracts/interaction'; +import { refFrameScope, refFrameState } from '../../../core/ref-frame.ts'; +import { + captureAndStoreSnapshot, + commandSurfaceWithMembers, +} from '../../../__tests__/support/command-surface-override-fake.ts'; +import { fakeCreateCommandSurface } from '../../../__tests__/support/command-surface-fake.ts'; +import { installLegacyCaptureInteractorResolution } from '../../../__tests__/support/legacy-capture-interactor-resolution.ts'; import { resetFindTouchRuntimeFixture, runFindClickScenario, -} from '@agent-device/daemon/testing/interaction/internal/__tests__/find-touch-runtime-fixture'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; -import { refFrameScope, refFrameState } from '@agent-device/daemon/testing/ref-frame'; +} from './__tests__/find-touch-runtime-fixture.ts'; + +installLegacyCaptureInteractorResolution(); + +/** + * The command family's `find list`: every node the locator matches, read from a capture it + * took through the config's backend and stored as the session snapshot, exactly as the daemon's + * ref issuance expects to find it. + */ +const listingSurface = (label: string) => + commandSurfaceWithMembers((config) => ({ + selectors: { + find: async (options): Promise => { + const snapshot = await captureAndStoreSnapshot(config, options); + return { + kind: 'list', + matches: snapshot.nodes + .filter((node) => node.label === label) + .map((node) => ({ ref: `@${node.ref}`, node })), + }; + }, + }, + })); beforeEach(() => { resetFindTouchRuntimeFixture(); @@ -41,7 +55,7 @@ test('handleFindCommands ambiguous match lists snapshot-line candidates capped a }); const { response } = await runFindClickScenario({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: fakeCreateCommandSurface, positionals: ['Follow', 'click'], nodes: [ { index: 0, ref: 'e1', type: 'Application', rect: { x: 0, y: 0, width: 800, height: 1200 } }, @@ -75,7 +89,7 @@ test('handleFindCommands ambiguous match lists snapshot-line candidates capped a test('handleFindCommands ambiguous match with few candidates lists them all uncapped', async () => { const { response } = await runFindClickScenario({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: fakeCreateCommandSurface, positionals: ['Follow', 'click'], nodes: [ { index: 0, ref: 'e1', type: 'Application', rect: { x: 0, y: 0, width: 800, height: 1200 } }, @@ -130,7 +144,7 @@ test('handleFindCommands list returns every match without acting', async () => { }); const { response, invokeCalls, session } = await runFindClickScenario({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: listingSurface('Follow'), positionals: ['text', 'Follow', 'list'], nodes: [ { index: 0, ref: 'e1', type: 'Application', rect: { x: 0, y: 0, width: 800, height: 1200 } }, @@ -157,7 +171,7 @@ test('handleFindCommands list returns every match without acting', async () => { test('handleFindCommands list on a unique match still lists instead of tapping', async () => { const { response, invokeCalls } = await runFindClickScenario({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: listingSurface('Dictionary'), positionals: ['Dictionary', 'list'], nodes: [ { index: 0, ref: 'e1', type: 'Application', rect: { x: 0, y: 0, width: 800, height: 1200 } }, @@ -196,7 +210,7 @@ test('handleFindCommands selector-shaped find rejects multiple matches with cand }); const { response, invokeCalls } = await runFindClickScenario({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: fakeCreateCommandSurface, positionals: ['role=button', 'click'], nodes: [ { index: 0, ref: 'e1', type: 'Application', rect: { x: 0, y: 0, width: 800, height: 1200 } }, @@ -226,7 +240,7 @@ test('handleFindCommands selector-shaped find honors the explicit --first opt-ou }); const { response, invokeCalls } = await runFindClickScenario({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: fakeCreateCommandSurface, positionals: ['role=button', 'click'], flags: { findFirst: true }, nodes: [ diff --git a/packages/daemon/src/execution/interaction/internal/find-production-route.test.ts b/packages/daemon/src/execution/interaction/internal/find-production-route.test.ts new file mode 100644 index 0000000000..fe6e171c96 --- /dev/null +++ b/packages/daemon/src/execution/interaction/internal/find-production-route.test.ts @@ -0,0 +1,153 @@ +import type { DaemonResponse } from '@agent-device/kernel/contracts'; +import type { CommandFlags } from '@agent-device/contracts/command'; +import type { FillCommandResult, PressCommandResult } from '@agent-device/contracts/interaction'; +import { test, expect, vi, beforeEach } from 'vitest'; +import { handleInteractionCommands } from '../index.ts'; +import type { DaemonRequest } from '../../../core/daemon-request.ts'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { makeIosSession as makeSession } from '../../../__tests__/support/session-factories.ts'; +import { commandSurfaceWithMembers } from '../../../__tests__/support/command-surface-override-fake.ts'; +import { installLegacyCaptureInteractorResolution } from '../../../__tests__/support/legacy-capture-interactor-resolution.ts'; +import { + findTouchRuntimeBindings, + mockDispatch, + resetFindTouchRuntimeFixture, +} from './__tests__/find-touch-runtime-fixture.ts'; +import { invokeFindHandler } from './__tests__/find-handler-fixture.ts'; + +installLegacyCaptureInteractorResolution(); + +beforeEach(() => { + resetFindTouchRuntimeFixture(); +}); + +// --- #1654: the production route, find handler through the real interaction handler --- +// +// find.test.ts's tests stub `invoke`, so they prove what find SENDS. These drive the real +// `handleInteractionCommands` on the other end, so they prove the resolved node survives the hop: +// the command surface receives the node find selected as its `preresolvedTarget`. They fail if +// either producer (click or fill) stops attaching the channel, or the handler stops forwarding it, +// which a hand-built request cannot catch. That the command family then acts on the carried node +// rather than re-resolving the ref is pinned beside it in +// src/commands/interaction/runtime/interactions-find-carried-target.test.ts. + +const findRouteContextFromFlags = (flags: CommandFlags | undefined) => ({ + count: flags?.count, + intervalMs: flags?.intervalMs, + delayMs: flags?.delayMs, + holdMs: flags?.holdMs, + jitterPx: flags?.jitterPx, + doubleTap: flags?.doubleTap, + clickButton: flags?.clickButton, +}); + +/** The tree find's capture returns: "Save" sits at a distinctive point. */ +function freshFindTree() { + return [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'XCUIElementTypeButton', + label: 'Save', + rect: { x: 300, y: 500, width: 20, height: 20 }, + enabled: true, + hittable: true, + }, + ]; +} + +type ForwardedTarget = { ref?: string; label?: string }; + +/** + * A command surface whose `press`/`fill` record the `preresolvedTarget` the handler forwarded and + * answer with the carried node, the way the command family does for a carried target. + */ +function recordingTouchSurface(forwarded: ForwardedTarget[]) { + const answer = ( + options: { preresolvedTarget?: { ref: string; node: { label?: string } } } | undefined, + ) => { + forwarded.push({ + ref: options?.preresolvedTarget?.ref, + label: options?.preresolvedTarget?.node.label, + }); + return { + kind: 'ref' as const, + target: { kind: 'ref' as const, ref: options?.preresolvedTarget?.ref ?? '' }, + point: { x: 310, y: 510 }, + }; + }; + return commandSurfaceWithMembers(() => ({ + interactions: { + click: vi.fn(async (_target, options) => answer(options) as PressCommandResult), + press: vi.fn(async (_target, options) => answer(options) as PressCommandResult), + fill: vi.fn( + async (_target, text, options) => ({ ...answer(options), text }) as FillCommandResult, + ), + }, + })); +} + +async function runFindThroughHandler(positionals: string[], forwarded: ForwardedTarget[]) { + const sessionStore = makeSessionStore(); + const sessionName = 'default'; + sessionStore.publish(sessionName, makeSession(sessionName)); + mockDispatch.mockImplementation(async (_device, command) => + command === 'snapshot' ? { nodes: freshFindTree(), backend: 'xctest' } : {}, + ); + + const createCommandSurface = recordingTouchSurface(forwarded); + const invokeCalls: DaemonRequest[] = []; + const response = await invokeFindHandler({ + createCommandSurface, + sessionName, + sessionStore, + positionals, + invoke: async (req) => { + invokeCalls.push(req); + return ( + (await handleInteractionCommands({ + createCommandSurface, + req, + sessionName, + sessionStore, + contextFromFlags: findRouteContextFromFlags, + ...findTouchRuntimeBindings(), + })) ?? + ({ + ok: false, + error: { code: 'COMMAND_FAILED', message: 'no interaction handler' }, + } satisfies DaemonResponse) + ); + }, + }); + return { response, invokeCalls }; +} + +test('#1654 production route: find click carries its resolved node into the interaction handler', async () => { + const forwarded: ForwardedTarget[] = []; + const { response, invokeCalls } = await runFindThroughHandler( + ['text', 'Save', 'click'], + forwarded, + ); + + expect(response?.ok).toBe(true); + // The producer attaches the channel — fails if handleFindClick stops doing so. + expect(invokeCalls[0]?.internal?.findResolvedTarget?.node?.label).toBe('Save'); + expect(invokeCalls[0]?.internal?.findResolvedTarget?.ref).toBe('@e2'); + expect(forwarded).toEqual([{ ref: '@e2', label: 'Save' }]); +}); + +test('#1654 production route: find fill carries its resolved node into the interaction handler', async () => { + const forwarded: ForwardedTarget[] = []; + const { response, invokeCalls } = await runFindThroughHandler( + ['text', 'Save', 'fill', 'hello'], + forwarded, + ); + + expect(response?.ok).toBe(true); + // Fill has its own producer and its own forwarding hop; assert it separately. + expect(invokeCalls[0]?.internal?.findResolvedTarget?.node?.label).toBe('Save'); + expect(invokeCalls[0]?.internal?.findResolvedTarget?.ref).toBe('@e2'); + expect(forwarded).toEqual([{ ref: '@e2', label: 'Save' }]); +}); diff --git a/src/__tests__/daemon-composition/interaction/find-read-only.test.ts b/packages/daemon/src/execution/selector-runtime-find-read-only.test.ts similarity index 59% rename from src/__tests__/daemon-composition/interaction/find-read-only.test.ts rename to packages/daemon/src/execution/selector-runtime-find-read-only.test.ts index e55d556c7f..8638c7c0e4 100644 --- a/src/__tests__/daemon-composition/interaction/find-read-only.test.ts +++ b/packages/daemon/src/execution/selector-runtime-find-read-only.test.ts @@ -1,37 +1,64 @@ -import { test, expect, vi, beforeEach } from 'vitest'; -import type { SessionState } from '@agent-device/daemon/testing/session-state'; +import { test, expect, beforeEach } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import { isSparseSnapshotQualityVerdict } from '@agent-device/capture-kit/snapshot-quality-verdict'; import { buildSnapshotSignatures } from '@agent-device/capture-kit/snapshot-freshness'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; +import type { DaemonFindOptions } from '@agent-device/contracts/daemon-command-surface'; +import type { AgentDeviceRuntimeConfig } from '@agent-device/contracts/runtime-contract'; +import type { SessionState } from '../core/session-state.ts'; +import { makeSessionStore } from '../__tests__/support/store-factory.ts'; import { makeAuthoringSession, makeIosSession as makeSession, -} from '@agent-device/daemon/testing/__tests__/support/session-factories'; - -vi.mock('@agent-device/device-selection/dispatch-resolve', async (importOriginal) => { - const actual = (await importOriginal()) as Record; - return { - ...actual, - resolveTargetDevice: actual.resolveTargetDevice, - }; -}); -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', async () => { - const fixture = - await import('@agent-device/daemon/testing/__tests__/legacy-snapshot-capture-fixture'); - return { captureSnapshotWithInteractor: fixture.captureSnapshotThroughLegacyDispatchFixture }; -}); - +} from '../__tests__/support/session-factories.ts'; +import { commandSurfaceWithMembers } from '../__tests__/support/command-surface-override-fake.ts'; +import { installLegacyCaptureInteractorResolution } from '../__tests__/support/legacy-capture-interactor-resolution.ts'; import { mockDispatch, resetFindTouchRuntimeFixture, runFindClickScenario, -} from '@agent-device/daemon/testing/interaction/internal/__tests__/find-touch-runtime-fixture'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; -import { invokeFindHandler } from '@agent-device/daemon/testing/interaction/internal/__tests__/find-handler-fixture'; +} from './interaction/internal/__tests__/find-touch-runtime-fixture.ts'; +import { invokeFindHandler } from './interaction/internal/__tests__/find-handler-fixture.ts'; + +installLegacyCaptureInteractorResolution(); beforeEach(() => { resetFindTouchRuntimeFixture(); }); +/** + * `find wait|exists` as the command family serves it: capture through the config's backend + * up to `maxPolls` times, answer found once a node carries the query as its label, and fail with + * the timeout the command family reports otherwise. Everything the daemon decides about those + * captures (cache tier, freshness recovery, what a sparse tree may store) happens behind the + * backend, which is what these cases pin. + */ +function pollingFindSurface(maxPolls: number) { + return commandSurfaceWithMembers((config: AgentDeviceRuntimeConfig) => ({ + selectors: { + find: async (options: DaemonFindOptions) => { + for (let poll = 0; poll < maxPolls; poll += 1) { + const captured = await config.backend.captureSnapshot?.( + { session: options.session, requestId: options.requestId, signal: options.signal }, + { interactiveOnly: true }, + ); + if (!captured?.snapshot) throw new Error('the backend returned no snapshot'); + if (isSparseSnapshotQualityVerdict(captured.snapshot.snapshotQuality)) { + throw new AppError( + 'COMMAND_FAILED', + 'find could not read the current accessibility tree', + { reason: captured.snapshot.snapshotQuality?.reason }, + ); + } + if (captured.snapshot.nodes.some((node) => node.label === options.query)) { + return { kind: 'found' as const, found: true as const }; + } + } + throw new AppError('COMMAND_FAILED', 'find wait timed out'); + }, + }, + })); +} + test('handleFindCommands wait bypasses snapshot cache while Android freshness recovery is active', async () => { const sessionName = 'android-find-wait'; const session: SessionState = { @@ -91,7 +118,7 @@ test('handleFindCommands wait bypasses snapshot cache while Android freshness re }); const { response } = await runFindClickScenario({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: pollingFindSurface(1), positionals: ['text', 'Create document', 'wait', '700'], session, }); @@ -103,7 +130,7 @@ test('handleFindCommands wait bypasses snapshot cache while Android freshness re expect(mockDispatch).toHaveBeenCalledTimes(2); }); -test('handleFindCommands wait reports sparse verdict through selector runtime route', async () => { +test('handleFindCommands wait over a sparse capture hands the surface its sparse verdict and leaves the stored tree alone', async () => { const session = makeSession('default'); session.snapshot = { nodes: [ @@ -139,26 +166,25 @@ test('handleFindCommands wait reports sparse verdict through selector runtime ro }); const { response } = await runFindClickScenario({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: pollingFindSurface(1), positionals: ['text', 'Never appears', 'wait', '350'], session, }); expect(response.ok).toBe(false); expect(session.snapshot).toBe(previousSnapshot); + // The sparse verdict is the daemon's capture result reaching the surface: a timeout would carry no + // reason, so the reason proves the refusal came from the verdict and not from the poll budget. expect(!response.ok && response.error).toMatchObject({ code: 'COMMAND_FAILED', - message: 'find could not read the current accessibility tree', - details: { - reason: 'sparse tree', - hint: expect.stringContaining('snapshot quality verdict is sparse'), - }, + details: { reason: 'sparse tree' }, }); }); test('handleFindCommands wait captures fresh snapshots while polling', async () => { + const polls = 3; const { response } = await runFindClickScenario({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: pollingFindSurface(polls), positionals: ['text', 'Never appears', 'wait', '350'], nodes: [{ index: 0, depth: 0, type: 'StaticText', label: 'Other text' }], }); @@ -167,12 +193,10 @@ test('handleFindCommands wait captures fresh snapshots while polling', async () if (!response.ok) { expect(response.error.message).toContain('find wait timed out'); } - // What this test guards is that every poll re-captures instead of reusing the first tree. - // The exact poll count is a timing artifact and is not assertable: the loop's last sleep - // consumes whatever remains of the budget, so it lands on `remainingMs() === 0`, and a - // sleep that returns a millisecond early admits one more poll. Pinning this to 2 made the - // test fail under CI load on unrelated PRs. Assert the property, not the artifact. - expect(mockDispatch.mock.calls.length).toBeGreaterThanOrEqual(2); + // Every poll the surface takes reaches the device: the daemon forces a fresh capture for `find` + // instead of answering the later polls from the first tree inside the cache window. The poll + // count is the surface's here, so the property holds without waiting on a real clock. + expect(mockDispatch.mock.calls.length).toBe(polls); expect(mockDispatch.mock.calls.every(([, command]) => command === 'snapshot')).toBe(true); }); @@ -201,7 +225,7 @@ test('read-only find while recording is intentionally deferred from target-v1 ev }); const response = await invokeFindHandler({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: pollingFindSurface(1), sessionName, sessionStore, positionals: ['text', 'Save', 'exists'], diff --git a/src/__tests__/daemon-composition/interaction/find-production-route.test.ts b/src/__tests__/daemon-composition/interaction/find-production-route.test.ts deleted file mode 100644 index 4e514e59d8..0000000000 --- a/src/__tests__/daemon-composition/interaction/find-production-route.test.ts +++ /dev/null @@ -1,187 +0,0 @@ -import type { DaemonResponse } from '@agent-device/kernel/contracts'; -import { test, expect, vi, beforeEach } from 'vitest'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; -import type { CommandFlags } from '@agent-device/contracts/command'; -import type { DaemonRequest } from '@agent-device/daemon'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { makeIosSession as makeSession } from '@agent-device/daemon/testing/__tests__/support/session-factories'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; - -vi.mock('@agent-device/device-selection/dispatch-resolve', async (importOriginal) => { - const actual = (await importOriginal()) as Record; - return { - ...actual, - resolveTargetDevice: actual.resolveTargetDevice, - }; -}); -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', async () => { - const fixture = - await import('@agent-device/daemon/testing/__tests__/legacy-snapshot-capture-fixture'); - return { captureSnapshotWithInteractor: fixture.captureSnapshotThroughLegacyDispatchFixture }; -}); - -import { - findTouchRuntimeBindings, - mockDispatch, - resetFindTouchRuntimeFixture, -} from '@agent-device/daemon/testing/interaction/internal/__tests__/find-touch-runtime-fixture'; -import { invokeFindHandler } from '@agent-device/daemon/testing/interaction/internal/__tests__/find-handler-fixture'; - -beforeEach(() => { - resetFindTouchRuntimeFixture(); -}); - -// --- #1654: the production route, find handler through the real interaction leaf --- -// -// find.test.ts's tests stub `invoke`, so they prove what find SENDS. These drive the real -// `handleInteractionCommands` on the other end, so they prove what the action actually acts on — -// and they fail if either producer (click or fill) stops attaching the channel, which a -// hand-built request cannot catch. - -const findRouteContextFromFlags = (flags: CommandFlags | undefined) => ({ - count: flags?.count, - intervalMs: flags?.intervalMs, - delayMs: flags?.delayMs, - holdMs: flags?.holdMs, - jitterPx: flags?.jitterPx, - doubleTap: flags?.doubleTap, - clickButton: flags?.clickButton, -}); - -/** The tree find's capture returns: "Save" sits at a distinctive point. */ -function freshFindTree() { - return [ - { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, - { - index: 1, - parentIndex: 0, - type: 'XCUIElementTypeButton', - label: 'Save', - rect: { x: 300, y: 500, width: 20, height: 20 }, - enabled: true, - hittable: true, - }, - ]; -} - -/** - * A defensive tree in which the same ref body names a different element. No - * reachable production path is currently known to advance `session.snapshot` - * in this interval; forcing it here pins the one-resolution contract itself. - */ -function divergedSessionTree() { - return { - nodes: [ - { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 }, ref: 'e1' }, - { - index: 1, - parentIndex: 0, - type: 'XCUIElementTypeButton', - label: 'Delete', - rect: { x: 10, y: 700, width: 100, height: 40 }, - enabled: true, - hittable: true, - ref: 'e2', - }, - ], - createdAt: Date.now(), - backend: 'xctest' as const, - }; -} - -async function runFindThroughLeaf(options: { - positionals: string[]; - divergeBeforeDispatch: boolean; -}): Promise<{ response: DaemonResponse | null; invokeCalls: DaemonRequest[] }> { - const sessionStore = makeSessionStore(); - const sessionName = 'default'; - const session = makeSession(sessionName); - sessionStore.publish(sessionName, session); - - mockDispatch.mockImplementation(async (_device, command) => - command === 'snapshot' ? { nodes: freshFindTree(), backend: 'xctest' } : {}, - ); - - const invokeCalls: DaemonRequest[] = []; - const response = await invokeFindHandler({ - createCommandSurface: testCreateCommandSurface, - sessionName, - sessionStore, - positionals: options.positionals, - invoke: async (req) => { - invokeCalls.push(req); - if (options.divergeBeforeDispatch) { - const current = sessionStore.get(sessionName); - if (current) { - current.snapshot = divergedSessionTree(); - } - } - // The real leaf, not a stub. - return ( - (await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req, - sessionName, - sessionStore, - contextFromFlags: findRouteContextFromFlags, - ...findTouchRuntimeBindings(), - })) ?? { ok: false, error: { code: 'COMMAND_FAILED', message: 'no interaction handler' } } - ); - }, - }); - return { response, invokeCalls }; -} - -function dispatchedPoint(command: 'press' | 'fill'): string[] | undefined { - const call = mockDispatch.mock.calls.find((entry) => entry[1] === command); - return call?.[2] as string[] | undefined; -} - -test('#1654 production route: find click carries its resolved node into the real interaction leaf', async () => { - const { response, invokeCalls } = await runFindThroughLeaf({ - positionals: ['text', 'Save', 'click'], - divergeBeforeDispatch: false, - }); - - expect(response?.ok).toBe(true); - // The producer attaches the channel — fails if handleFindClick stops doing so. - expect(invokeCalls[0]?.internal?.findResolvedTarget?.node?.label).toBe('Save'); - expect(invokeCalls[0]?.internal?.findResolvedTarget?.ref).toBe('@e2'); - expect(dispatchedPoint('press')).toEqual(['310', '510']); -}); - -test('#1654 production route: find fill carries its resolved node into the real interaction leaf', async () => { - const { response, invokeCalls } = await runFindThroughLeaf({ - positionals: ['text', 'Save', 'fill', 'hello'], - divergeBeforeDispatch: false, - }); - - expect(response?.ok).toBe(true); - // Fill has its own producer and its own forwarding hop; assert it separately. - expect(invokeCalls[0]?.internal?.findResolvedTarget?.node?.label).toBe('Save'); - expect(invokeCalls[0]?.internal?.findResolvedTarget?.ref).toBe('@e2'); - expect(dispatchedPoint('fill')).toEqual(['310', '510', 'hello']); -}); - -test('#1654 defensive invariant: a tree that advances mid-dispatch cannot retarget find click', async () => { - // This deliberately forces an otherwise-unreached state: a second lookup of - // `@e2` would find "Delete" at (60, 720), while the resolved handoff must keep - // find's selected "Save" node at (310, 510). - const { response } = await runFindThroughLeaf({ - positionals: ['text', 'Save', 'click'], - divergeBeforeDispatch: true, - }); - - expect(response?.ok).toBe(true); - expect(dispatchedPoint('press')).toEqual(['310', '510']); -}); - -test('#1654 defensive invariant: a tree that advances mid-dispatch cannot retarget find fill', async () => { - const { response } = await runFindThroughLeaf({ - positionals: ['text', 'Save', 'fill', 'hello'], - divergeBeforeDispatch: true, - }); - - expect(response?.ok).toBe(true); - expect(dispatchedPoint('fill')).toEqual(['310', '510', 'hello']); -}); diff --git a/src/commands/interaction/runtime/interactions-find-carried-target.test.ts b/src/commands/interaction/runtime/interactions-find-carried-target.test.ts new file mode 100644 index 0000000000..ec1c104c5f --- /dev/null +++ b/src/commands/interaction/runtime/interactions-find-carried-target.test.ts @@ -0,0 +1,116 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { AgentDeviceBackend } from '@agent-device/contracts/backend'; +import { localCommandPolicy } from '@agent-device/contracts/command-policy'; +import type { PreresolvedInteractionTarget } from '@agent-device/contracts/interaction'; +import type { Point, SnapshotState } from '@agent-device/kernel/snapshot'; +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; +import { createMemorySessionStore } from '../../../command-runtime/runtime-factory.ts'; +import { createLocalArtifactAdapter } from '../../../io.ts'; +import { createAgentDevice } from '../../../runtime.ts'; +import { ref } from './selector-read-utils.ts'; + +// #1654: a mutating `find` hands the leaf the node it already resolved. The leaf acts on that node +// and does not look the `@ref` up a second time, so the point it dispatches at is the carried +// node's, whatever the stored session tree now holds under the same ref. + +const APPLICATION = { + index: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 390, height: 844 }, +}; + +function treeWithButton( + label: string, + rect: { x: number; y: number; width: number; height: number }, +) { + return makeSnapshotState([ + APPLICATION, + { + index: 1, + parentIndex: 0, + type: 'XCUIElementTypeButton', + label, + rect, + enabled: true, + hittable: true, + }, + ]); +} + +const SAVE_TREE = treeWithButton('Save', { x: 300, y: 500, width: 20, height: 20 }); +const SAVE_CENTER: Point = { x: 310, y: 510 }; + +/** The same ref body names a different element in the tree the session stores. */ +const DIVERGED_TREE = treeWithButton('Delete', { x: 10, y: 700, width: 100, height: 40 }); + +function carriedSave(): PreresolvedInteractionTarget { + return { ref: '@e2', node: SAVE_TREE.nodes[1]!, tree: { nodes: SAVE_TREE.nodes } }; +} + +function createDevice(stored: SnapshotState) { + const taps: Point[] = []; + const fills: Array<{ point: Point; text: string }> = []; + const device = createAgentDevice({ + backend: { + platform: 'ios', + captureSnapshot: async () => ({ snapshot: stored }), + tap: async (_context, point) => { + taps.push(point); + return {}; + }, + fill: async (_context, point, text) => { + fills.push({ point, text }); + return {}; + }, + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([{ name: 'default', snapshot: stored }]), + policy: localCommandPolicy(), + }); + return { device, taps, fills }; +} + +test('press acts on the carried node of a mutating find', async () => { + const { device, taps } = createDevice(SAVE_TREE); + + await device.interactions.press(ref('@e2'), { + session: 'default', + preresolvedTarget: carriedSave(), + }); + + assert.deepEqual(taps, [SAVE_CENTER]); +}); + +test('fill acts on the carried node of a mutating find', async () => { + const { device, fills } = createDevice(SAVE_TREE); + + await device.interactions.fill(ref('@e2'), 'hello', { + session: 'default', + preresolvedTarget: carriedSave(), + }); + + assert.deepEqual(fills, [{ point: SAVE_CENTER, text: 'hello' }]); +}); + +test('a stored tree that advanced past the find capture cannot retarget press', async () => { + const { device, taps } = createDevice(DIVERGED_TREE); + + await device.interactions.press(ref('@e2'), { + session: 'default', + preresolvedTarget: carriedSave(), + }); + + assert.deepEqual(taps, [SAVE_CENTER]); +}); + +test('a stored tree that advanced past the find capture cannot retarget fill', async () => { + const { device, fills } = createDevice(DIVERGED_TREE); + + await device.interactions.fill(ref('@e2'), 'hello', { + session: 'default', + preresolvedTarget: carriedSave(), + }); + + assert.deepEqual(fills, [{ point: SAVE_CENTER, text: 'hello' }]); +}); diff --git a/src/commands/interaction/runtime/selector-read-find-list.test.ts b/src/commands/interaction/runtime/selector-read-find-list.test.ts new file mode 100644 index 0000000000..6ca1d51a5b --- /dev/null +++ b/src/commands/interaction/runtime/selector-read-find-list.test.ts @@ -0,0 +1,68 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; +import { createSelectorDevice } from './__tests__/test-utils/index.ts'; + +const APPLICATION = { + index: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 800, height: 1200 }, +}; + +const followButton = (index: number, x: number) => ({ + index, + parentIndex: 0, + type: 'Button', + label: 'Follow', + hittable: true, + rect: { x, y: 100, width: 80, height: 40 }, +}); + +// #1625 decision 1: `find list` is the read-only inspection surface the ambiguity recovery +// hint points at. The test device's backend has no touch member, so a list that acted would throw. +test('runtime find list returns every matching node with its ref', async () => { + const device = createSelectorDevice( + makeSnapshotState([APPLICATION, followButton(1, 0), followButton(2, 90), followButton(3, 180)]), + ); + + const listed = await device.selectors.find({ + session: 'default', + locator: 'text', + query: 'Follow', + action: 'list', + }); + + assert.equal(listed.kind, 'list'); + assert.deepEqual( + listed.matches.map((entry) => entry.ref), + ['@e2', '@e3', '@e4'], + ); +}); + +test('runtime find list on a unique match still lists instead of acting', async () => { + const device = createSelectorDevice( + makeSnapshotState([ + APPLICATION, + { + index: 1, + parentIndex: 0, + type: 'Cell', + label: 'Dictionary', + hittable: true, + rect: { x: 0, y: 100, width: 800, height: 44 }, + }, + ]), + ); + + const listed = await device.selectors.find({ + session: 'default', + query: 'Dictionary', + action: 'list', + }); + + assert.equal(listed.kind, 'list'); + assert.deepEqual( + listed.matches.map((entry) => entry.ref), + ['@e2'], + ); +}); diff --git a/src/commands/interaction/runtime/selector-read.test.ts b/src/commands/interaction/runtime/selector-read.test.ts index ec6a6fdd1d..9916f1390c 100644 --- a/src/commands/interaction/runtime/selector-read.test.ts +++ b/src/commands/interaction/runtime/selector-read.test.ts @@ -280,10 +280,14 @@ test('runtime find wait reports sparse snapshot verdicts on the selector-read ro action: 'wait', timeoutMs: 100, }), - (error: unknown) => - error instanceof Error && - error.message === 'find could not read the current accessibility tree' && - (error as { details?: { reason?: string } }).details?.reason === 'sparse tree', + (error: unknown) => { + assert.ok(error instanceof Error); + assert.equal(error.message, 'find could not read the current accessibility tree'); + const details = (error as { details?: { reason?: string; hint?: string } }).details; + assert.equal(details?.reason, 'sparse tree'); + assert.match(details?.hint ?? '', /snapshot quality verdict is sparse/); + return true; + }, ); assert.equal(session.snapshot, initialSnapshot); });