From 2d2079c2f6564611741ab4ac97e190fa2eedcc74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 15 Oct 2026 14:24:41 +0200 Subject: [PATCH 01/10] test(daemon): add a touch-forwarding command surface fake Co-Authored-By: Claude Opus 5.5 --- .../support/touch-forwarding-surface-fake.ts | 142 ++++++++++++++++++ .../interaction-capture-disclosure.test.ts | 0 2 files changed, 142 insertions(+) create mode 100644 packages/daemon/src/__tests__/support/touch-forwarding-surface-fake.ts rename {src/__tests__/daemon-composition/interaction => packages/daemon/src/execution/interaction/internal}/interaction-capture-disclosure.test.ts (100%) diff --git a/packages/daemon/src/__tests__/support/touch-forwarding-surface-fake.ts b/packages/daemon/src/__tests__/support/touch-forwarding-surface-fake.ts new file mode 100644 index 0000000000..9f6a05c165 --- /dev/null +++ b/packages/daemon/src/__tests__/support/touch-forwarding-surface-fake.ts @@ -0,0 +1,142 @@ +import type { CreateDaemonCommandSurface } from '@agent-device/contracts/daemon-command-surface'; +import type { + FillCommandResult, + InteractionTarget, + PressCommandResult, +} from '@agent-device/contracts/interaction'; +import { AppError } from '@agent-device/kernel/errors'; +import { centerOfRect, type Point, type SnapshotNode } from '@agent-device/kernel/snapshot'; +import { + captureAndStoreSnapshot, + commandSurfaceWithMembers, +} from './command-surface-override-fake.ts'; + +type TouchResolution = { + point: Point; + target: InteractionTarget; + node?: SnapshotNode; + nodes?: SnapshotNode[]; + selectorChain?: string[]; +}; + +export type TouchForwardingSurfaceOptions = { + /** + * Which node a selector expression names. The default reads `label="X"` / `label=X` and + * `id="X"` / `id=X`; a suite that needs another selector shape states its own matcher. + */ + matchSelector?: (nodes: SnapshotNode[], selector: string) => SnapshotNode | undefined; +}; + +function defaultMatchSelector(nodes: SnapshotNode[], selector: string): SnapshotNode | undefined { + const parsed = /^(label|id)="?([^"]*)"?$/.exec(selector.trim()); + if (!parsed) return undefined; + const [, key, value] = parsed; + return nodes.find((node) => (key === 'id' ? node.identifier : node.label) === value); +} + +function refBody(ref: string): string { + return ref.replace(/^@/, '').replace(/~s\d+$/, ''); +} + +/** + * A command surface whose touch actions do what the daemon depends on the real command family + * to do, and no more: locate the target (a stored-tree lookup for a `@ref`, one interactive + * capture through the config's backend for a selector), dispatch the touch through the + * config's backend, and answer with the resolved target and the tree it was read from. Selector + * disambiguation, hittability, the moving-target gate, and every other command-family decision + * are out of scope: the suites that use this surface pin what the daemon does around them. + */ +export function touchForwardingCreateCommandSurface( + options: TouchForwardingSurfaceOptions = {}, +): CreateDaemonCommandSurface { + const matchSelector = options.matchSelector ?? defaultMatchSelector; + return commandSurfaceWithMembers((config) => { + const locate = async ( + target: InteractionTarget, + context: { session?: string; requestId?: string; signal?: AbortSignal }, + ): Promise => { + if (target.kind === 'point') return { point: { x: target.x, y: target.y }, target }; + if (target.kind === 'ref') { + const record = await config.sessions?.get(context.session ?? 'default'); + const nodes = record?.snapshot?.nodes ?? []; + const node = nodes.find((candidate) => candidate.ref === refBody(target.ref)); + if (!node?.rect) { + throw new AppError('COMMAND_FAILED', `Ref ${target.ref} not found`, { + reason: 'ref_not_found', + }); + } + return { point: centerOfRect(node.rect), target, node, nodes }; + } + const { nodes } = await captureAndStoreSnapshot(config, { + ...context, + interactiveOnly: true, + includeRects: true, + }); + const node = matchSelector(nodes, target.selector); + if (node?.rect) { + return { + point: centerOfRect(node.rect), + target, + node, + nodes, + selectorChain: [target.selector], + }; + } + throw new AppError('COMMAND_FAILED', `Selector did not match: ${target.selector}`, { + reason: 'selector_not_found', + }); + }; + + const resolvedTarget = (resolution: TouchResolution) => { + const { target, point, node, nodes, selectorChain } = resolution; + const preAction = nodes ? { nodes } : undefined; + if (target.kind === 'point') return { kind: 'point' as const, point, preAction }; + if (target.kind === 'ref') { + return { kind: 'ref' as const, point, target, node, preAction }; + } + return { + kind: 'selector' as const, + point, + target, + node: node as SnapshotNode, + selectorChain: selectorChain ?? [], + preAction, + }; + }; + + const press = async ( + target: InteractionTarget, + pressOptions?: { session?: string; requestId?: string; signal?: AbortSignal }, + ): Promise => { + const context = { + session: pressOptions?.session, + requestId: pressOptions?.requestId, + signal: pressOptions?.signal, + }; + const resolution = await locate(target, context); + const backendResult = await config.backend.tap?.(context, resolution.point); + return { ...resolvedTarget(resolution), ...(backendResult ? { backendResult } : {}) }; + }; + + const fill = async ( + target: InteractionTarget, + text: string, + fillOptions?: { session?: string; requestId?: string; signal?: AbortSignal }, + ): Promise => { + const context = { + session: fillOptions?.session, + requestId: fillOptions?.requestId, + signal: fillOptions?.signal, + }; + const resolution = await locate(target, context); + const backendResult = await config.backend.fill?.(context, resolution.point, text); + return { + ...resolvedTarget(resolution), + text, + ...(backendResult ? { backendResult } : {}), + }; + }; + + return { interactions: { press, click: press, fill } }; + }); +} diff --git a/src/__tests__/daemon-composition/interaction/interaction-capture-disclosure.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-capture-disclosure.test.ts similarity index 100% rename from src/__tests__/daemon-composition/interaction/interaction-capture-disclosure.test.ts rename to packages/daemon/src/execution/interaction/internal/interaction-capture-disclosure.test.ts From 9afd1c6fb9b37424d8c5f641329ebacfcb11fa1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 15 Oct 2026 14:24:42 +0200 Subject: [PATCH 02/10] test(interaction): capture disclosure moves beside the interaction handler (a); the full-tree retry outcome inheritance is a command test (c) interaction-capture-disclosure: all four cases are daemon disclosure decisions (a) over a touch-forwarding surface and the legacy capture interactor resolution; the unsettled-outcome inheritance by the full-tree retry, which the miss case relied on the real command for, is now pinned directly by selector-target-resolution-post-gesture (c). Co-Authored-By: Claude Opus 5.5 --- .../interaction-capture-disclosure.test.ts | 27 ++++++------ ...tor-target-resolution-post-gesture.test.ts | 42 +++++++++++++++++++ 2 files changed, 54 insertions(+), 15 deletions(-) create mode 100644 src/commands/interaction/runtime/selector-target-resolution-post-gesture.test.ts diff --git a/packages/daemon/src/execution/interaction/internal/interaction-capture-disclosure.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-capture-disclosure.test.ts index 4067928d7f..40a77481f3 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-capture-disclosure.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-capture-disclosure.test.ts @@ -6,23 +6,20 @@ import { type SnapshotState, } from '@agent-device/kernel/snapshot'; import { iosTargetActivationDisclosure } from '@agent-device/contracts/ios-target-activation'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; -import { getRuntimeBindings } from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { handleInteractionCommands } from '../index.ts'; +import { getRuntimeBindings } from '../../__tests__/interaction-get-runtime-fixture.ts'; import { contextFromFlags, makeSession, -} from '@agent-device/daemon/testing/interaction/internal/__tests__/interaction-touch-fixtures'; -import { legacyDispatchCapture } from '@agent-device/daemon/testing/__tests__/legacy-snapshot-capture-fixture'; -import { markDeferredInteractionOutcome } from '@agent-device/daemon/testing/deferred-interaction-outcome'; +} from './__tests__/interaction-touch-fixtures.ts'; +import { legacyDispatchCapture } from '../../../__tests__/support/legacy-snapshot-capture-fixture.ts'; +import { markDeferredInteractionOutcome } from '../../../core/deferred-interaction-outcome.ts'; import { formatPostGestureOutcomeWarning } from '@agent-device/capture-kit/post-gesture-stability'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; +import { installLegacyCaptureInteractorResolution } from '../../../__tests__/support/legacy-capture-interactor-resolution.ts'; +import { touchForwardingCreateCommandSurface } from '../../../__tests__/support/touch-forwarding-surface-fake.ts'; -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', async () => { - const fixture = - await import('@agent-device/daemon/testing/__tests__/legacy-snapshot-capture-fixture'); - return { captureSnapshotWithInteractor: fixture.captureSnapshotThroughLegacyDispatchFixture }; -}); +installLegacyCaptureInteractorResolution(); const FACT: IosTargetActivation = { reason: 'stale_target', @@ -62,7 +59,7 @@ async function pressSelector(params: { const sessionStore = makeSessionStore(); sessionStore.publish(params.sessionName, makeSession(params.sessionName)); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchForwardingCreateCommandSurface(), req: { token: 't', session: params.sessionName, @@ -115,7 +112,7 @@ test('a press that consumes no capture is not disclosed against an older tree', sessionStore.publish(sessionName, session); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchForwardingCreateCommandSurface(), req: { token: 't', session: sessionName, @@ -175,7 +172,7 @@ async function pressAfterUnsettledScroll(selector: string) { vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] }); let done = false; const pending = handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchForwardingCreateCommandSurface(), req: { token: 't', session: 'default', command: 'press', positionals: [selector], flags: {} }, sessionName: 'default', sessionStore, diff --git a/src/commands/interaction/runtime/selector-target-resolution-post-gesture.test.ts b/src/commands/interaction/runtime/selector-target-resolution-post-gesture.test.ts new file mode 100644 index 0000000000..cf1e04b374 --- /dev/null +++ b/src/commands/interaction/runtime/selector-target-resolution-post-gesture.test.ts @@ -0,0 +1,42 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { PostGestureOutcome } from '@agent-device/kernel/snapshot'; +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; +import { selector } from './selector-read-utils.ts'; +import { createInteractionDevice } from './__tests__/test-utils/index.ts'; + +const UNSETTLED: PostGestureOutcome = { + kind: 'unsettled', + gesture: { action: 'scroll', positionals: ['down'] }, +}; + +const movingList = (label: string) => + makeSnapshotState([ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'Cell', + label, + rect: { x: 16, y: 300, width: 370, height: 52 }, + hittable: true, + }, + ]); + +// The interactive capture that misses and the full-tree retry that follows it read the same +// moving surface, so the retry's tree carries the interactive capture's unsettled outcome: neither +// the refusal nor the session's stored tree may present the miss as settled. +test('a selector press retries a miss on the full tree and keeps the unsettled gesture outcome', async () => { + const interactive = { ...movingList('Wi-Fi'), postGestureOutcome: UNSETTLED }; + const full = movingList('Wi-Fi'); + const captures = [interactive, full]; + const device = createInteractionDevice(movingList('Wi-Fi'), { + captureSnapshot: async () => ({ snapshot: captures.shift() ?? full }), + }); + + await assert.rejects(() => + device.interactions.press(selector('label="General"'), { session: 'default' }), + ); + + assert.deepEqual(full.postGestureOutcome, UNSETTLED); +}); From 42022b31387dad3a48c042d5bc7242852a35fe07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 15 Oct 2026 14:25:24 +0200 Subject: [PATCH 03/10] test(interaction): parameterized fill scrubbing moves beside the touch finalizer (a) interaction-common: all five cases are daemon response-boundary and recorder decisions (a); the two handler-route cases run on the touch-forwarding surface, which now answers with the selector chain the command family builds for the resolved node. Co-Authored-By: Claude Opus 5.5 --- .../support/touch-forwarding-surface-fake.ts | 20 ++++++++++++++----- .../internal}/interaction-common.test.ts | 16 +++++++-------- 2 files changed, 23 insertions(+), 13 deletions(-) rename {src/__tests__/daemon-composition/interaction => packages/daemon/src/execution/interaction/internal}/interaction-common.test.ts (93%) diff --git a/packages/daemon/src/__tests__/support/touch-forwarding-surface-fake.ts b/packages/daemon/src/__tests__/support/touch-forwarding-surface-fake.ts index 9f6a05c165..e6ef2fd97a 100644 --- a/packages/daemon/src/__tests__/support/touch-forwarding-surface-fake.ts +++ b/packages/daemon/src/__tests__/support/touch-forwarding-surface-fake.ts @@ -4,6 +4,7 @@ import type { InteractionTarget, PressCommandResult, } from '@agent-device/contracts/interaction'; +import { buildSelectorChainForNode } from '@agent-device/selectors'; import { AppError } from '@agent-device/kernel/errors'; import { centerOfRect, type Point, type SnapshotNode } from '@agent-device/kernel/snapshot'; import { @@ -54,7 +55,10 @@ export function touchForwardingCreateCommandSurface( const locate = async ( target: InteractionTarget, context: { session?: string; requestId?: string; signal?: AbortSignal }, + action: 'click' | 'fill', ): Promise => { + const chainFor = (node: SnapshotNode, nodes: SnapshotNode[]) => + buildSelectorChainForNode(node, config.backend.platform, { action, nodes }); if (target.kind === 'point') return { point: { x: target.x, y: target.y }, target }; if (target.kind === 'ref') { const record = await config.sessions?.get(context.session ?? 'default'); @@ -65,7 +69,13 @@ export function touchForwardingCreateCommandSurface( reason: 'ref_not_found', }); } - return { point: centerOfRect(node.rect), target, node, nodes }; + return { + point: centerOfRect(node.rect), + target, + node, + nodes, + selectorChain: chainFor(node, nodes), + }; } const { nodes } = await captureAndStoreSnapshot(config, { ...context, @@ -79,7 +89,7 @@ export function touchForwardingCreateCommandSurface( target, node, nodes, - selectorChain: [target.selector], + selectorChain: chainFor(node, nodes), }; } throw new AppError('COMMAND_FAILED', `Selector did not match: ${target.selector}`, { @@ -92,7 +102,7 @@ export function touchForwardingCreateCommandSurface( const preAction = nodes ? { nodes } : undefined; if (target.kind === 'point') return { kind: 'point' as const, point, preAction }; if (target.kind === 'ref') { - return { kind: 'ref' as const, point, target, node, preAction }; + return { kind: 'ref' as const, point, target, node, selectorChain, preAction }; } return { kind: 'selector' as const, @@ -113,7 +123,7 @@ export function touchForwardingCreateCommandSurface( requestId: pressOptions?.requestId, signal: pressOptions?.signal, }; - const resolution = await locate(target, context); + const resolution = await locate(target, context, 'click'); const backendResult = await config.backend.tap?.(context, resolution.point); return { ...resolvedTarget(resolution), ...(backendResult ? { backendResult } : {}) }; }; @@ -128,7 +138,7 @@ export function touchForwardingCreateCommandSurface( requestId: fillOptions?.requestId, signal: fillOptions?.signal, }; - const resolution = await locate(target, context); + const resolution = await locate(target, context, 'fill'); const backendResult = await config.backend.fill?.(context, resolution.point, text); return { ...resolvedTarget(resolution), diff --git a/src/__tests__/daemon-composition/interaction/interaction-common.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-common.test.ts similarity index 93% rename from src/__tests__/daemon-composition/interaction/interaction-common.test.ts rename to packages/daemon/src/execution/interaction/internal/interaction-common.test.ts index a29df237a2..c7bbf42859 100644 --- a/src/__tests__/daemon-composition/interaction/interaction-common.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-common.test.ts @@ -1,24 +1,24 @@ import type { CommandFlags } from '@agent-device/contracts/command'; -import { legacyDispatchCapture } from '@agent-device/daemon/testing/__tests__/legacy-snapshot-capture-fixture'; +import { legacyDispatchCapture } from '../../../__tests__/support/legacy-snapshot-capture-fixture.ts'; import { beforeEach, expect, test } from 'vitest'; import { makeIosSession, makeAuthoringSession, authoringPublication, -} from '@agent-device/daemon/testing/__tests__/support/session-factories'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; +} from '../../../__tests__/support/session-factories.ts'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; import { attachRefs, type RawSnapshotNode } from '@agent-device/kernel/snapshot'; import { finalizeTouchInteraction, handleInteractionCommands, -} from '@agent-device/daemon/testing/interaction/index'; +} from '../index.ts'; import { IOS_SIMULATOR } from '@agent-device/test-kit'; import { getRuntimeBindings, mockFillPoint, resetGetRuntimeFixture, -} from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; +} from '../../__tests__/interaction-get-runtime-fixture.ts'; +import { touchForwardingCreateCommandSurface } from '../../../__tests__/support/touch-forwarding-surface-fake.ts'; const contextFromFlags = (_flags: CommandFlags | undefined) => ({}); @@ -165,7 +165,7 @@ test('parameterized fill scrubs concatenated backend values and object keys thro ); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchForwardingCreateCommandSurface(), req: { token: 't', session: sessionName, @@ -223,7 +223,7 @@ test('parameterized fill collapses whitespace-only backend echoes through the ha }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchForwardingCreateCommandSurface(), req: { token: 't', session: sessionName, From 0692d20fd885334f0d0ae245b02f2c5d2e3bb292 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 15 Oct 2026 14:27:18 +0200 Subject: [PATCH 04/10] test(interaction): dispatch-disclosure daemon rows move beside the interaction handler (a); the guard row moves beside its guard (a) interaction-dispatch-disclosure: the ten daemon rows and the four daemon verdict cases (a) drive the interaction handler over a touch-forwarding surface with get and gesture members, installing the interactor resolution instead of mocking the capture module. The post-action-guard row is driven by its own file beside interaction-android-escape; the contracts driver-owner map names both. Co-Authored-By: Claude Opus 5.5 --- .../support/touch-forwarding-surface-fake.ts | 13 +- ...android-escape-dispatch-disclosure.test.ts | 60 ++++++++ .../interaction-dispatch-disclosure.test.ts | 135 +++++++++++------- .../src/dispatch-disclosure.fixtures.ts | 5 +- 4 files changed, 155 insertions(+), 58 deletions(-) create mode 100644 packages/daemon/src/execution/interaction/internal/interaction-android-escape-dispatch-disclosure.test.ts rename {src/__tests__/daemon-composition/interaction => packages/daemon/src/execution/interaction/internal}/interaction-dispatch-disclosure.test.ts (78%) diff --git a/packages/daemon/src/__tests__/support/touch-forwarding-surface-fake.ts b/packages/daemon/src/__tests__/support/touch-forwarding-surface-fake.ts index e6ef2fd97a..afe657fd83 100644 --- a/packages/daemon/src/__tests__/support/touch-forwarding-surface-fake.ts +++ b/packages/daemon/src/__tests__/support/touch-forwarding-surface-fake.ts @@ -10,7 +10,9 @@ import { centerOfRect, type Point, type SnapshotNode } from '@agent-device/kerne import { captureAndStoreSnapshot, commandSurfaceWithMembers, + type CommandSurfaceMembers, } from './command-surface-override-fake.ts'; +import type { AgentDeviceRuntimeConfig } from '@agent-device/contracts/runtime-contract'; type TouchResolution = { point: Point; @@ -26,6 +28,8 @@ export type TouchForwardingSurfaceOptions = { * `id="X"` / `id=X`; a suite that needs another selector shape states its own matcher. */ matchSelector?: (nodes: SnapshotNode[], selector: string) => SnapshotNode | undefined; + /** Members the suite answers itself, layered over the touch actions (selector reads, gestures). */ + members?: (config: AgentDeviceRuntimeConfig) => CommandSurfaceMembers; }; function defaultMatchSelector(nodes: SnapshotNode[], selector: string): SnapshotNode | undefined { @@ -67,6 +71,7 @@ export function touchForwardingCreateCommandSurface( if (!node?.rect) { throw new AppError('COMMAND_FAILED', `Ref ${target.ref} not found`, { reason: 'ref_not_found', + dispatched: 'no', }); } return { @@ -94,6 +99,7 @@ export function touchForwardingCreateCommandSurface( } throw new AppError('COMMAND_FAILED', `Selector did not match: ${target.selector}`, { reason: 'selector_not_found', + dispatched: 'no', }); }; @@ -147,6 +153,11 @@ export function touchForwardingCreateCommandSurface( }; }; - return { interactions: { press, click: press, fill } }; + const own = options.members?.(config) ?? {}; + return { + capture: own.capture, + selectors: own.selectors, + interactions: { press, click: press, fill, ...own.interactions }, + }; }); } diff --git a/packages/daemon/src/execution/interaction/internal/interaction-android-escape-dispatch-disclosure.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-android-escape-dispatch-disclosure.test.ts new file mode 100644 index 0000000000..a93b69745a --- /dev/null +++ b/packages/daemon/src/execution/interaction/internal/interaction-android-escape-dispatch-disclosure.test.ts @@ -0,0 +1,60 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { test } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import type { AndroidObservationAdapter } from '@agent-device/contracts/android-observation'; +import { + assertDispatchDisclosureDriversMatchRows, + DISPATCH_DISCLOSURE_TABLE_PATH, + dispatchDisclosureRowsOwnedBy, +} from '@agent-device/test-kit'; +import { makeAndroidSession } from '../../../__tests__/support/session-factories.ts'; +import { discloseRequestDispatch } from '../../request-dispatch-disclosure.ts'; +import { createRequestDispatchLedger } from '../../../core/request-dispatch-ledger.ts'; +import { assertAndroidPressStayedInApp } from './interaction-android-escape.ts'; + +// contracts/fixtures/dispatch-disclosure.json, post-action guard rows: the guard is driven itself, +// inside the router's dispatch seam. + +async function pressThatLeftTheApp(): Promise { + const session = makeAndroidSession('dispatch-disclosure-android', { + appBundleId: 'com.example.app', + }); + const observation = { + readAppState: async () => ({ package: 'com.android.settings' }), + isPermissionPackage: async () => false, + } as unknown as AndroidObservationAdapter; + return await discloseRequestDispatch( + { token: 't', session: session.name, command: 'press', positionals: ['@e1'] }, + createRequestDispatchLedger(), + async () => { + await assertAndroidPressStayedInApp(session, '@e1', observation); + return null; + }, + ); +} + +const DRIVERS: Record Promise> = { + 'post-action-guard.android-press-left-app': pressThatLeftTheApp, +}; + +const ROWS = dispatchDisclosureRowsOwnedBy( + import.meta.url, + fs.readFileSync(DISPATCH_DISCLOSURE_TABLE_PATH, 'utf8'), +); + +test('every post-action guard dispatch-disclosure row has exactly one driver', () => { + assertDispatchDisclosureDriversMatchRows(ROWS, Object.keys(DRIVERS)); +}); + +for (const row of ROWS) { + test(`${row.id}: ${row.trigger} → dispatched ${row.dispatched}`, async () => { + const drive = DRIVERS[row.id]; + assert.ok(drive, `no driver for ${row.id}`); + await assert.rejects(drive(), (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.dispatched, row.dispatched); + return true; + }); + }); +} diff --git a/src/__tests__/daemon-composition/interaction/interaction-dispatch-disclosure.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-dispatch-disclosure.test.ts similarity index 78% rename from src/__tests__/daemon-composition/interaction/interaction-dispatch-disclosure.test.ts rename to packages/daemon/src/execution/interaction/internal/interaction-dispatch-disclosure.test.ts index b3deb1ef0a..a163f8de98 100644 --- a/src/__tests__/daemon-composition/interaction/interaction-dispatch-disclosure.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-dispatch-disclosure.test.ts @@ -1,8 +1,8 @@ -import { composedPlatformServicesFixture } from '../platform-services-fixture.ts'; import assert from 'node:assert/strict'; import fs from 'node:fs'; import { beforeEach, test, vi } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; +import type { GesturePlan } from '@agent-device/contracts/gesture-plan-types'; import { AppError, type DispatchDisclosure, normalizeError } from '@agent-device/kernel/errors'; import type { AndroidObservationAdapter } from '@agent-device/contracts/android-observation'; import { @@ -10,44 +10,90 @@ import { DISPATCH_DISCLOSURE_TABLE_PATH, dispatchDisclosureRowsOwnedBy, } from '@agent-device/test-kit'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { makeAndroidSession } from '@agent-device/daemon/testing/__tests__/support/session-factories'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { makeAndroidSession } from '../../../__tests__/support/session-factories.ts'; import { getRuntimeBindings, mockFillPoint, mockTapPoint, resetGetRuntimeFixture, -} from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; -import { captureSnapshotWithInteractor } from '@agent-device/daemon/testing/snapshot-interactor-capture'; -import { discloseRequestDispatch } from '@agent-device/daemon/testing/request-dispatch-disclosure'; +} from '../../__tests__/interaction-get-runtime-fixture.ts'; +import { legacyDispatchCapture } from '../../../__tests__/support/legacy-snapshot-capture-fixture.ts'; +import { installLegacyCaptureInteractorResolution } from '../../../__tests__/support/legacy-capture-interactor-resolution.ts'; +import { daemonPlatformServicesFixture } from '../../../__tests__/support/platform-services-fixture.ts'; +import { discloseRequestDispatch } from '../../request-dispatch-disclosure.ts'; import { createRequestDispatchLedger, recordBoundMutations, -} from '@agent-device/daemon/testing/request-dispatch-ledger'; -import type { BindDeviceRuntime } from '@agent-device/daemon/testing/request-runtime-binding'; -import { clearAndroidObservationFixture } from '@agent-device/daemon/testing/__tests__/android-observation-fixture'; -import { activateCompleteRefFrame, expireRefFrame } from '@agent-device/daemon/testing/ref-frame'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; -import type { InteractionRouteInput } from '@agent-device/daemon/testing/interaction/internal/types'; -import { assertAndroidPressStayedInApp } from '@agent-device/daemon/testing/interaction/internal/interaction-android-escape'; -import { gestureRuntimeBindingsFixture } from '@agent-device/daemon/testing/interaction/internal/__tests__/gesture-runtime-bindings.fixtures'; +} from '../../../core/request-dispatch-ledger.ts'; +import type { BindDeviceRuntime } from '../../../core/request-runtime-binding.ts'; +import { clearAndroidObservationFixture } from '../../../__tests__/support/android-observation-fixture.ts'; +import { activateCompleteRefFrame, expireRefFrame } from '../../../core/ref-frame.ts'; +import { handleInteractionCommands } from '../index.ts'; +import type { InteractionRouteInput } from '../../../core/interaction/internal/types.ts'; +import { assertAndroidPressStayedInApp } from './interaction-android-escape.ts'; +import { gestureRuntimeBindingsFixture } from './__tests__/gesture-runtime-bindings.fixtures.ts'; import { contextFromFlags, makeSession, makeStaleRefSession, -} from '@agent-device/daemon/testing/interaction/internal/__tests__/interaction-touch-fixtures'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; +} from './__tests__/interaction-touch-fixtures.ts'; +import { touchForwardingCreateCommandSurface } from '../../../__tests__/support/touch-forwarding-surface-fake.ts'; -// contracts/fixtures/dispatch-disclosure.json, daemon and post-action guard rows: the daemon rows -// drive a real `press` through the daemon interaction handler, inside the router's dispatch seam, -// with only the device touch mocked; the guard row drives the guard itself. +// contracts/fixtures/dispatch-disclosure.json, daemon rows: each drives a `press`, `fill`, `get`, +// or `swipe` through the daemon interaction handler, inside the router's dispatch seam, over a +// command surface that forwards the touch to the bound operations (only the device touch is +// faked). The post-action guard row is driven beside the guard, in +// interaction-android-escape-dispatch-disclosure.test.ts. -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', () => ({ - captureSnapshotWithInteractor: vi.fn(), -})); +const GESTURE_PLAN: GesturePlan = { + topology: 'single', + intent: 'fling', + executionProfile: 'timed-pan', + durationMs: 100, + viewport: { x: 0, y: 0, width: 400, height: 800 }, + pointers: [ + { + pointerId: 0, + samples: [ + { offsetMs: 0, point: { x: 100, y: 600 } }, + { offsetMs: 100, point: { x: 100, y: 200 } }, + ], + }, + ], +}; + +const createCommandSurface = touchForwardingCreateCommandSurface({ + members: (config) => ({ + selectors: { + get: async () => { + await config.backend.captureSnapshot?.({ session: 'dispatch-disclosure' }, {}); + throw new AppError('COMMAND_FAILED', 'No element matched label="Missing"', { + reason: 'selector_not_found', + }); + }, + }, + interactions: { + gesture: async (options) => { + await config.backend.performGesture?.({ session: options.session }, GESTURE_PLAN); + return { + kind: 'fling', + durationMs: GESTURE_PLAN.durationMs, + pointerCount: 1, + from: { x: 100, y: 600 }, + to: { x: 100, y: 200 }, + }; + }, + }, + }), +}); + +installLegacyCaptureInteractorResolution(); beforeEach(() => { resetGetRuntimeFixture(); + legacyDispatchCapture.mockReset(); + legacyDispatchCapture.mockResolvedValue({}); }); /** @@ -96,7 +142,7 @@ async function press({ }; sessionStore.publish(session.name, session); const response = await routeInteraction({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface, req: { token: 't', session: session.name, command, positionals, flags }, sessionName: session.name, sessionStore, @@ -133,24 +179,6 @@ async function pressAfterUnclassifiedTouchFailure( return await press({ positionals: ['@e1'] }); } -async function pressThatLeftTheApp(): Promise { - const session = makeAndroidSession('dispatch-disclosure-android', { - appBundleId: 'com.example.app', - }); - const observation = { - readAppState: async () => ({ package: 'com.android.settings' }), - isPermissionPackage: async () => false, - } as unknown as AndroidObservationAdapter; - return await discloseRequestDispatch( - { token: 't', session: session.name, command: 'press', positionals: ['@e1'] }, - createRequestDispatchLedger(), - async () => { - await assertAndroidPressStayedInApp(session, '@e1', observation); - return null; - }, - ); -} - /** `swipe --count 2` whose first repetition runs and whose second is refused before dispatch. */ async function swipeRefusedOnSecondRepetition(): Promise { const gestures = gestureRuntimeBindingsFixture(); @@ -165,8 +193,8 @@ async function swipeRefusedOnSecondRepetition(): Promise { const session = makeSession('dispatch-disclosure-swipe'); sessionStore.publish(session.name, session); const response = await routeInteraction({ - createCommandSurface: testCreateCommandSurface, - platformServices: composedPlatformServicesFixture(), + createCommandSurface, + platformServices: daemonPlatformServicesFixture(), req: { token: 't', session: session.name, @@ -199,8 +227,8 @@ async function gestureDragRefusedBeforeDispatch(): Promise { expireRefFrame(session); sessionStore.publish(session.name, session); const response = await routeInteraction({ - createCommandSurface: testCreateCommandSurface, - platformServices: composedPlatformServicesFixture(), + createCommandSurface, + platformServices: daemonPlatformServicesFixture(), req: { token: 't', session: session.name, @@ -258,8 +286,8 @@ async function refRefusedAfterAndroidRecovery(command: 'press' | 'gesture'): Pro openApp, }; const response = await routeInteraction({ - createCommandSurface: testCreateCommandSurface, - platformServices: composedPlatformServicesFixture(), + createCommandSurface, + platformServices: daemonPlatformServicesFixture(), req: { token: 't', session: session.name, @@ -315,7 +343,7 @@ async function pressThenForegroundReadRefused(): Promise { }, }; const response = await routeInteraction({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface, req: { token: 't', session: session.name, command: 'press', positionals: ['50', '40'] }, sessionName: session.name, sessionStore, @@ -343,7 +371,6 @@ const DRIVERS: Record Promise> = { 'daemon.post-dispatch.press-then-foreground-read-refused': pressThenForegroundReadRefused, 'daemon.read-only-command': () => press({ command: 'get', positionals: ['text', 'label="Missing"'] }), - 'post-action-guard.android-press-left-app': pressThatLeftTheApp, }; const ROWS = dispatchDisclosureRowsOwnedBy( @@ -351,7 +378,7 @@ const ROWS = dispatchDisclosureRowsOwnedBy( fs.readFileSync(DISPATCH_DISCLOSURE_TABLE_PATH, 'utf8'), ); -test('every daemon and post-action guard dispatch-disclosure row has exactly one driver', () => { +test('every daemon dispatch-disclosure row has exactly one driver', () => { assertDispatchDisclosureDriversMatchRows(ROWS, Object.keys(DRIVERS)); }); @@ -378,9 +405,7 @@ test('the daemon keeps a producer verdict instead of inferring its own', async ( }); test('a read-only command discloses no over a producer verdict', async () => { - const capture = vi.mocked(captureSnapshotWithInteractor); - capture.mockClear(); - capture.mockRejectedValueOnce( + legacyDispatchCapture.mockRejectedValueOnce( new AppError('COMMAND_FAILED', 'runner capture lost', { dispatched: 'unknown' }), ); await assert.rejects( @@ -392,7 +417,7 @@ test('a read-only command discloses no over a producer verdict', async () => { return true; }, ); - assert.equal(capture.mock.calls.length, 1); + assert.equal(legacyDispatchCapture.mock.calls.length, 1); }); test('a read-only command discloses no over a producer verdict it throws', async () => { @@ -425,7 +450,7 @@ test('a plain Error a backend throws reaches the wire with dispatched unknown', }) as unknown as typeof bindings.bindDevice; await assert.rejects( routeInteraction({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface, req: { token: 't', session: session.name, command: 'press', positionals: ['@e1'], flags: {} }, sessionName: session.name, sessionStore, diff --git a/packages/test-kit/src/dispatch-disclosure.fixtures.ts b/packages/test-kit/src/dispatch-disclosure.fixtures.ts index 9bee01c77d..9916c98366 100644 --- a/packages/test-kit/src/dispatch-disclosure.fixtures.ts +++ b/packages/test-kit/src/dispatch-disclosure.fixtures.ts @@ -37,11 +37,12 @@ export const DISPATCH_DISCLOSURE_TABLE_PATH = path.join( * followed by `test(\`${row.id}`, which {@link DISPATCH_DISCLOSURE_ROW_LOOP} matches. */ export const DISPATCH_DISCLOSURE_DRIVER_OWNERS: Readonly> = { - 'daemon.': 'src/__tests__/daemon-composition/interaction/interaction-dispatch-disclosure.test.ts', + 'daemon.': + 'packages/daemon/src/execution/interaction/internal/interaction-dispatch-disclosure.test.ts', 'daemon.scroll-no-progress': 'packages/daemon/src/execution/__tests__/scroll-movement.test.ts', 'daemon.route.': 'src/__tests__/daemon-composition/request-dispatch-disclosure.test.ts', 'post-action-guard.': - 'src/__tests__/daemon-composition/interaction/interaction-dispatch-disclosure.test.ts', + 'packages/daemon/src/execution/interaction/internal/interaction-android-escape-dispatch-disclosure.test.ts', 'ios-runner.': 'packages/platform-apple/src/runner/__tests__/index-dispatch-disclosure.test.ts', 'ios-runner.pre-send.': 'packages/platform-apple/src/runner/__tests__/runner-lifecycle-dispatch-disclosure.test.ts', From 9f7c181d5a2854ca9f1632eb7bf4921aa94ed2f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 15 Oct 2026 14:33:56 +0200 Subject: [PATCH 05/10] test(daemon): let the touch-forwarding surface observe a settle and carry the capture preference Co-Authored-By: Claude Opus 5.5 --- .../support/command-surface-override-fake.ts | 9 ++- .../support/touch-forwarding-surface-fake.ts | 74 +++++++++++++++++-- ...nteraction-settle-private-ax-route.test.ts | 0 .../internal}/interaction-settle.test.ts | 0 4 files changed, 76 insertions(+), 7 deletions(-) rename {src/__tests__/daemon-composition/interaction => packages/daemon/src/execution/interaction/internal}/interaction-settle-private-ax-route.test.ts (100%) rename {src/__tests__/daemon-composition/interaction => packages/daemon/src/execution/interaction/internal}/interaction-settle.test.ts (100%) diff --git a/packages/daemon/src/__tests__/support/command-surface-override-fake.ts b/packages/daemon/src/__tests__/support/command-surface-override-fake.ts index 256754a6b2..790a3ade40 100644 --- a/packages/daemon/src/__tests__/support/command-surface-override-fake.ts +++ b/packages/daemon/src/__tests__/support/command-surface-override-fake.ts @@ -3,7 +3,7 @@ import type { DaemonCommandSurface, } from '@agent-device/contracts/daemon-command-surface'; import type { AgentDeviceRuntimeConfig } from '@agent-device/contracts/runtime-contract'; -import type { SnapshotState } from '@agent-device/kernel/snapshot'; +import type { SnapshotPreferredBackend, SnapshotState } from '@agent-device/kernel/snapshot'; import { fakeCreateCommandSurface } from './command-surface-fake.ts'; export type CommandSurfaceMembers = { @@ -47,12 +47,17 @@ export async function captureAndStoreSnapshot( signal?: AbortSignal; interactiveOnly?: boolean; includeRects?: boolean; + preferredBackend?: SnapshotPreferredBackend; }, ): Promise { const name = params.session ?? 'default'; const captured = await config.backend.captureSnapshot?.( { session: name, requestId: params.requestId, signal: params.signal ?? config.signal }, - { interactiveOnly: params.interactiveOnly, includeRects: params.includeRects }, + { + interactiveOnly: params.interactiveOnly, + includeRects: params.includeRects, + preferredBackend: params.preferredBackend, + }, ); const snapshot = captured?.snapshot; if (!snapshot) throw new Error('captureAndStoreSnapshot: the backend returned no snapshot'); diff --git a/packages/daemon/src/__tests__/support/touch-forwarding-surface-fake.ts b/packages/daemon/src/__tests__/support/touch-forwarding-surface-fake.ts index afe657fd83..c530661198 100644 --- a/packages/daemon/src/__tests__/support/touch-forwarding-surface-fake.ts +++ b/packages/daemon/src/__tests__/support/touch-forwarding-surface-fake.ts @@ -3,10 +3,19 @@ import type { FillCommandResult, InteractionTarget, PressCommandResult, + SettleObservation, + SettleParams, } from '@agent-device/contracts/interaction'; import { buildSelectorChainForNode } from '@agent-device/selectors'; import { AppError } from '@agent-device/kernel/errors'; -import { centerOfRect, type Point, type SnapshotNode } from '@agent-device/kernel/snapshot'; +import { + centerOfRect, + type Point, + type SnapshotNode, + type SnapshotPreferredBackend, + type SnapshotQualityVerdict, + type SnapshotState, +} from '@agent-device/kernel/snapshot'; import { captureAndStoreSnapshot, commandSurfaceWithMembers, @@ -20,6 +29,7 @@ type TouchResolution = { node?: SnapshotNode; nodes?: SnapshotNode[]; selectorChain?: string[]; + snapshotQuality?: SnapshotQualityVerdict; }; export type TouchForwardingSurfaceOptions = { @@ -28,6 +38,21 @@ export type TouchForwardingSurfaceOptions = { * `id="X"` / `id=X`; a suite that needs another selector shape states its own matcher. */ matchSelector?: (nodes: SnapshotNode[], selector: string) => SnapshotNode | undefined; + /** + * The post-action observation a `--settle` touch owes. It runs after the touch with the + * settle parameters the daemon forwarded, the pre-action tree, and a capture taken through the + * config's backend and stored as the session snapshot; its answer rides on the result. + */ + observeSettle?: (params: { + settle: SettleParams; + baseline: SnapshotNode[] | undefined; + /** The quality verdict of the tree the touch resolved against, when it published one. */ + baselineQuality: SnapshotQualityVerdict | undefined; + capture: (options?: { + signal?: AbortSignal; + preferredBackend?: SnapshotPreferredBackend; + }) => Promise; + }) => Promise; /** Members the suite answers itself, layered over the touch actions (selector reads, gestures). */ members?: (config: AgentDeviceRuntimeConfig) => CommandSurfaceMembers; }; @@ -80,9 +105,10 @@ export function touchForwardingCreateCommandSurface( node, nodes, selectorChain: chainFor(node, nodes), + snapshotQuality: record?.snapshot?.snapshotQuality, }; } - const { nodes } = await captureAndStoreSnapshot(config, { + const { nodes, snapshotQuality } = await captureAndStoreSnapshot(config, { ...context, interactiveOnly: true, includeRects: true, @@ -95,6 +121,7 @@ export function touchForwardingCreateCommandSurface( node, nodes, selectorChain: chainFor(node, nodes), + snapshotQuality, }; } throw new AppError('COMMAND_FAILED', `Selector did not match: ${target.selector}`, { @@ -120,9 +147,34 @@ export function touchForwardingCreateCommandSurface( }; }; + const observe = async ( + context: { session?: string; requestId?: string; signal?: AbortSignal }, + resolution: TouchResolution, + settle: SettleParams | undefined, + ): Promise => { + if (!settle || !options.observeSettle) return undefined; + return await options.observeSettle({ + settle, + baseline: resolution.nodes, + baselineQuality: resolution.snapshotQuality, + capture: (capture) => + captureAndStoreSnapshot(config, { + ...context, + signal: capture?.signal ?? context.signal, + preferredBackend: capture?.preferredBackend, + includeRects: true, + }), + }); + }; + const press = async ( target: InteractionTarget, - pressOptions?: { session?: string; requestId?: string; signal?: AbortSignal }, + pressOptions?: { + session?: string; + requestId?: string; + signal?: AbortSignal; + settle?: SettleParams; + }, ): Promise => { const context = { session: pressOptions?.session, @@ -131,13 +183,23 @@ export function touchForwardingCreateCommandSurface( }; const resolution = await locate(target, context, 'click'); const backendResult = await config.backend.tap?.(context, resolution.point); - return { ...resolvedTarget(resolution), ...(backendResult ? { backendResult } : {}) }; + const settle = await observe(context, resolution, pressOptions?.settle); + return { + ...resolvedTarget(resolution), + ...(backendResult ? { backendResult } : {}), + ...(settle ? { settle } : {}), + }; }; const fill = async ( target: InteractionTarget, text: string, - fillOptions?: { session?: string; requestId?: string; signal?: AbortSignal }, + fillOptions?: { + session?: string; + requestId?: string; + signal?: AbortSignal; + settle?: SettleParams; + }, ): Promise => { const context = { session: fillOptions?.session, @@ -146,10 +208,12 @@ export function touchForwardingCreateCommandSurface( }; const resolution = await locate(target, context, 'fill'); const backendResult = await config.backend.fill?.(context, resolution.point, text); + const settle = await observe(context, resolution, fillOptions?.settle); return { ...resolvedTarget(resolution), text, ...(backendResult ? { backendResult } : {}), + ...(settle ? { settle } : {}), }; }; diff --git a/src/__tests__/daemon-composition/interaction/interaction-settle-private-ax-route.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-settle-private-ax-route.test.ts similarity index 100% rename from src/__tests__/daemon-composition/interaction/interaction-settle-private-ax-route.test.ts rename to packages/daemon/src/execution/interaction/internal/interaction-settle-private-ax-route.test.ts diff --git a/src/__tests__/daemon-composition/interaction/interaction-settle.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-settle.test.ts similarity index 100% rename from src/__tests__/daemon-composition/interaction/interaction-settle.test.ts rename to packages/daemon/src/execution/interaction/internal/interaction-settle.test.ts From f709ae9b39af3052dbe87f9e40bb4f3d0dff4d90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 15 Oct 2026 14:33:56 +0200 Subject: [PATCH 06/10] test(interaction): settle wiring moves beside the interaction handler (a); stalled-capture deadline and fill settle land as command tests (c) interaction-settle: all ten cases are daemon decisions (request signal inheritance, settle flag to parameter mapping, refsGeneration and the partial ref frame, the settled tree stored under the scoped lifetime, expired-frame refusal, flag guards) run over a settling touch-forwarding surface. The diff, tail, unavailable-observation hint, and deadline behaviors those cases leaned on are command behavior: settle.test.ts already pins the diff, tail, and broken-capture cases, and settle-capture-deadline adds the stalled-capture signal and fill --settle (c). interaction-settle-private-ax-route: (a) daemon forwarding of the settle loop's private-ax preference to the interactor snapshot options, over the installed interactor resolution; the pin decision and the runner request construction are covered by settle.test.ts and the platform-apple suites. Co-Authored-By: Claude Opus 5.5 --- ...nteraction-settle-private-ax-route.test.ts | 112 ++++++++------- .../internal/interaction-settle.test.ts | 135 ++++++++++++++---- .../runtime/settle-capture-deadline.test.ts | 80 +++++++++++ 3 files changed, 248 insertions(+), 79 deletions(-) create mode 100644 src/commands/interaction/runtime/settle-capture-deadline.test.ts diff --git a/packages/daemon/src/execution/interaction/internal/interaction-settle-private-ax-route.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-settle-private-ax-route.test.ts index 8a7b0ca9af..f0ed4659f8 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-settle-private-ax-route.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-settle-private-ax-route.test.ts @@ -1,31 +1,24 @@ import { beforeEach, expect, test, vi } from 'vitest'; +import type { Interactor, SnapshotResult } from '@agent-device/contracts/interactor-types'; +import type { SettleObservation } from '@agent-device/contracts/interaction'; +import type { SnapshotOptions } from '@agent-device/contracts/interactor-types'; import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; -import { makeIosSession } from '@agent-device/daemon/testing/__tests__/support/session-factories'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { withAppleRunnerProvider } from '@agent-device/platform-apple/runner'; -import { contextFromFlags as buildDaemonContext } from '@agent-device/daemon/testing/context'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; -import { getRuntimeBindings } from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; -import { getInteractor } from '../../../core/interactors.ts'; -import { installInteractorResolution } from '@agent-device/daemon/testing/interactor-resolution'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; +import { makeIosSession } from '../../../__tests__/support/session-factories.ts'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { contextFromFlags as buildDaemonContext } from '../../../core/context.ts'; +import { installInteractorResolution } from '../../../core/interactor-resolution.ts'; +import { handleInteractionCommands } from '../index.ts'; +import { getRuntimeBindings } from '../../__tests__/interaction-get-runtime-fixture.ts'; +import { touchForwardingCreateCommandSurface } from '../../../__tests__/support/touch-forwarding-surface-fake.ts'; -// Root composition hands the daemon its interactor resolution; this test drives the real -// settle capture, so it composes the real lookup the way the process root does. -installInteractorResolution({ resolve: getInteractor }); +// The settle loop pins `private-ax` on its captures once the observation reports that producer; the +// daemon must carry that preference through its capture path to the interactor, which is what the +// runner request is built from. Where the loop decides to pin is pinned beside it, in +// src/commands/interaction/runtime/settle.test.ts, and how the Apple interactor turns the preference +// into a runner request in packages/platform-apple. -vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) => { - const actual = - await importOriginal(); - return { - ...actual, - runAppleRunnerCommand: vi.fn(async () => ({})), - }; -}); - -import { runAppleRunnerCommand } from '@agent-device/platform-apple/runner/operations'; +const snapshotCalls: SnapshotOptions[] = []; -const mockRunnerCommand = vi.mocked(runAppleRunnerCommand); const nodes = [ { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, { @@ -51,12 +44,36 @@ const privateAxQuality = { reasonCode: 'requested-backend' as const, }; +installInteractorResolution({ + resolve: async () => + ({ + snapshot: vi.fn(async (options: SnapshotOptions): Promise => { + snapshotCalls.push(options); + return { nodes, backend: 'xctest', quality: privateAxQuality } as SnapshotResult; + }), + }) as unknown as Interactor, +}); + beforeEach(() => { - mockRunnerCommand.mockReset(); - mockRunnerCommand.mockImplementation(async (_device, command) => { - if (command.command === 'snapshot') return { nodes, snapshotQuality: privateAxQuality }; - return {}; - }); + snapshotCalls.length = 0; +}); + +/** The pin the settle loop applies: every settle capture asks for `private-ax` once it is established. */ +const pinningSurface = touchForwardingCreateCommandSurface({ + observeSettle: async ({ settle, baselineQuality, capture }): Promise => { + let pinned = baselineQuality?.backend === 'private-ax'; + for (let captures = 0; captures < 3; captures += 1) { + const settled = await capture(pinned ? { preferredBackend: 'private-ax' } : undefined); + pinned ||= settled.snapshotQuality?.backend === 'private-ax'; + } + return { + settled: true, + waitedMs: 0, + captures: 3, + quietMs: settle.quietMs ?? 500, + timeoutMs: settle.timeoutMs ?? 10_000, + }; + }, }); test('daemon press --settle pins private-ax on emitted snapshot runner requests', async () => { @@ -73,30 +90,25 @@ test('daemon press --settle pins private-ax on emitted snapshot runner requests' }), ); - const response = await withAppleRunnerProvider(mockRunnerCommand, { deviceId: 'sim-1' }, () => - handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'press', - positionals: ['label=Continue'], - flags: { settle: true, settleQuietMs: 25, timeoutMs: 1_000 }, - }, - sessionName, - sessionStore, - contextFromFlags: (flags, appBundleId, traceLogPath) => - buildDaemonContext('', flags, appBundleId, traceLogPath), - ...getRuntimeBindings(), - }), - ); + const response = await handleInteractionCommands({ + createCommandSurface: pinningSurface, + req: { + token: 't', + session: sessionName, + command: 'press', + positionals: ['label=Continue'], + flags: { settle: true, settleQuietMs: 25, timeoutMs: 1_000 }, + }, + sessionName, + sessionStore, + contextFromFlags: (flags, appBundleId, traceLogPath) => + buildDaemonContext('', flags, appBundleId, traceLogPath), + ...getRuntimeBindings(), + }); expect(response?.ok).toBe(true); - const snapshotCommands = mockRunnerCommand.mock.calls - .map(([, command]) => command) - .filter((command) => command.command === 'snapshot'); - expect(snapshotCommands.length).toBeGreaterThanOrEqual(3); - const preferredBackends = snapshotCommands.map((command) => command.preferredBackend); + expect(snapshotCalls.length).toBeGreaterThanOrEqual(3); + const preferredBackends = snapshotCalls.map((options) => options.preferredBackend); expect(preferredBackends).toEqual([ undefined, ...Array.from({ length: preferredBackends.length - 1 }, () => 'private-ax' as const), diff --git a/packages/daemon/src/execution/interaction/internal/interaction-settle.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-settle.test.ts index ad26ed494e..ec12509fb1 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-settle.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-settle.test.ts @@ -1,24 +1,22 @@ -import { composedPlatformServicesFixture } from '../platform-services-fixture.ts'; import type { CommandFlags } from '@agent-device/contracts/command'; -import { legacyDispatchCapture } from '@agent-device/daemon/testing/__tests__/legacy-snapshot-capture-fixture'; +import { legacyDispatchCapture } from '../../../__tests__/support/legacy-snapshot-capture-fixture.ts'; import { test, expect, vi, beforeEach } from 'vitest'; import { createInteractionRuntime, handleInteractionCommands, -} from '@agent-device/daemon/testing/interaction/index'; -import type { SessionStore } from '@agent-device/daemon/testing/session-store'; -import type { SessionRef, SessionState } from '@agent-device/daemon/testing/session-state'; +} from '../index.ts'; +import type { SessionStore } from '../../../core/session-store.ts'; +import type { SessionRef, SessionState } from '../../../core/session-state.ts'; import { buildSnapshotState } from '@agent-device/capture-kit/snapshot-state'; -import { setSessionSnapshot } from '@agent-device/daemon/testing/session-snapshot'; +import { setSessionSnapshot } from '../../session-snapshot.ts'; import { activateCompleteRefFrame, expireRefFrame, refFrameState, refFrameTree, -} from '@agent-device/daemon/testing/ref-frame'; -import { captureSnapshotWithInteractor } from '@agent-device/daemon/testing/snapshot-interactor-capture'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { makeIosSession } from '@agent-device/daemon/testing/__tests__/support/session-factories'; +} from '../../../core/ref-frame.ts'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { makeIosSession } from '../../../__tests__/support/session-factories.ts'; import { clearRequestAbortRegistration, registerRequestAbort, @@ -29,8 +27,12 @@ import { mockFillPoint, mockTapPoint, resetGetRuntimeFixture, -} from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; +} from '../../__tests__/interaction-get-runtime-fixture.ts'; +import { daemonPlatformServicesFixture } from '../../../__tests__/support/platform-services-fixture.ts'; +import { installLegacyCaptureInteractorResolution } from '../../../__tests__/support/legacy-capture-interactor-resolution.ts'; +import { touchForwardingCreateCommandSurface } from '../../../__tests__/support/touch-forwarding-surface-fake.ts'; +import type { SettleObservation } from '@agent-device/contracts/interaction'; +import type { SnapshotNode } from '@agent-device/kernel/snapshot'; // #1101 --settle daemon response shape: the settle payload (diff + settled + // refsGeneration) rides the wire response through the shared builder, and a @@ -38,11 +40,9 @@ import { testCreateCommandSurface } from '../command-surface-fixture.ts'; // Quiet windows are tuned down (--settle-quiet 25) so no test waits real time // beyond a few poll ticks. +installLegacyCaptureInteractorResolution(); + const mockCaptureSnapshotForSession = vi.hoisted(() => vi.fn()); -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', () => ({ - captureSnapshotWithInteractor: vi.fn(), -})); -const nativeCapture = vi.mocked(captureSnapshotWithInteractor); const BEFORE_NODES = [ { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, @@ -106,6 +106,83 @@ function seedSession(sessionName: string, sessionStore: ReturnType `${node.type}|${node.label ?? ''}`; + +/** + * A stand-in for the command family's settled diff, keyed by role and label: enough for the daemon + * to have added lines that carry refs minted from the settled tree, removed lines that carry none, + * and, when nothing was added, the unchanged interactive tail. How the real diff is computed is + * pinned beside it, in src/commands/interaction/runtime/settle.test.ts. + */ +function settledDiff(baseline: SnapshotNode[] | undefined, settled: SnapshotNode[]) { + const before = baseline ?? []; + const beforeKeys = new Set(before.map(diffKey)); + const settledKeys = new Set(settled.map(diffKey)); + const added = settled.filter((node) => !beforeKeys.has(diffKey(node))); + const removed = before.filter((node) => !settledKeys.has(diffKey(node))); + const unchanged = settled.filter((node) => beforeKeys.has(diffKey(node))); + return { + diff: { + summary: { additions: added.length, removals: removed.length, unchanged: unchanged.length }, + lines: [ + ...removed.map((node) => ({ kind: 'removed' as const, text: `${node.type} "${node.label}"` })), + ...added.map((node) => ({ + kind: 'added' as const, + text: `${node.type} "${node.label}"`, + ref: node.ref, + })), + ], + }, + tail: + added.length === 0 + ? unchanged + .filter((node) => node.hittable && node.label) + .map((node) => ({ + ref: node.ref, + role: node.type.toLowerCase(), + label: node.label, + })) + : undefined, + }; +} + +/** + * The touch-forwarding surface with the post-action observation `--settle` owes: it captures the + * settled tree through the config's backend under the settle budget's deadline and answers with the + * diff, or with the unavailable/stalled verdict the command family reports when that capture fails. + */ +const settlingSurface = touchForwardingCreateCommandSurface({ + observeSettle: async ({ settle, baseline, capture }): Promise => { + const quietMs = settle.quietMs ?? 500; + const timeoutMs = settle.timeoutMs ?? 10_000; + const deadline = AbortSignal.timeout(timeoutMs); + try { + const settled = await capture({ signal: deadline }); + const { diff, tail } = settledDiff(baseline, settled.nodes); + return { + settled: true, + waitedMs: 0, + captures: 1, + quietMs, + timeoutMs, + diff, + ...(tail ? { tail } : {}), + }; + } catch (error) { + return { + settled: false, + waitedMs: 0, + captures: 1, + quietMs, + timeoutMs, + hint: deadline.aborted + ? 'A snapshot capture stalled past the settle budget, so no settled diff is shown.' + : `Settle observation unavailable (${error instanceof Error ? error.message : String(error)}).`, + }; + } + }, +}); + test('interaction runtime inherits the registered daemon request signal', async () => { const sessionStore = makeSessionStore(); const sessionName = 'request-signal'; @@ -116,8 +193,8 @@ test('interaction runtime inherits the registered daemon request signal', async try { const runtime = await createInteractionRuntime({ - createCommandSurface: testCreateCommandSurface, - platformServices: composedPlatformServicesFixture(), + createCommandSurface: settlingSurface, + platformServices: daemonPlatformServicesFixture(), req: { token: 't', session: sessionName, @@ -175,7 +252,6 @@ beforeEach(() => { }); mockCaptureSnapshotForSession.mockReset(); mockCaptureSnapshotForSession.mockImplementation(emulateCaptureSnapshotForSession); - nativeCapture.mockReset(); }); const SETTLE_FLAGS = { settle: true, settleQuietMs: 25, timeoutMs: 2_000 }; @@ -221,7 +297,7 @@ test('press --settle responds with the settled diff, refsGeneration, and activat mockCommandDispatch({ snapshots: [BEFORE_NODES, AFTER_NODES, AFTER_NODES, AFTER_NODES] }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: settlingSurface, req: { token: 't', session: sessionName, @@ -271,7 +347,8 @@ test('held touch settle publishes refs into the current record of its scoped lif release = resolve; }); let captures = 0; - nativeCapture.mockImplementation(async () => { + legacyDispatchCapture.mockImplementation(async (_device, command) => { + if (command !== 'snapshot') return {}; captures += 1; if (captures === 2) { enter(); @@ -284,7 +361,7 @@ test('held touch settle publishes refs into the current record of its scoped lif }; }); const running = handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: settlingSurface, req: { token: 't', session: address, @@ -359,7 +436,7 @@ test('press --settle on a removals-only diff attaches the unchanged interactive }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: settlingSurface, req: { token: 't', session: sessionName, @@ -396,7 +473,7 @@ test('press --settle rejects an expired-frame ref before dispatch or observation ); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: settlingSurface, req: { token: 't', session: sessionName, @@ -438,7 +515,7 @@ test('a settle observation without a diff leaves ref staleness untouched', async }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: settlingSurface, req: { token: 't', session: sessionName, @@ -504,7 +581,7 @@ test('a stalled settle capture receives its deadline signal and leaves the inter const startedAt = Date.now(); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: settlingSurface, req: { token: 't', session: sessionName, @@ -535,7 +612,7 @@ test('bare timeout without --settle stays compatible', async () => { mockCommandDispatch({ snapshots: [BEFORE_NODES] }); const compatible = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: settlingSurface, req: { token: 't', session: sessionName, @@ -561,7 +638,7 @@ test('settle-specific tuning flags without --settle are rejected', async () => { mockCommandDispatch({ snapshots: [BEFORE_NODES] }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: settlingSurface, req: { token: 't', session: sessionName, @@ -587,7 +664,7 @@ test('fill @ref --settle carries the settle payload on the ref wire shape', asyn mockCommandDispatch({ snapshots: [AFTER_NODES, AFTER_NODES] }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: settlingSurface, req: { token: 't', session: sessionName, diff --git a/src/commands/interaction/runtime/settle-capture-deadline.test.ts b/src/commands/interaction/runtime/settle-capture-deadline.test.ts new file mode 100644 index 0000000000..4f3b7eeff6 --- /dev/null +++ b/src/commands/interaction/runtime/settle-capture-deadline.test.ts @@ -0,0 +1,80 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { AgentDeviceBackend, BackendSnapshotResult } from '@agent-device/contracts/backend'; +import { localCommandPolicy } from '@agent-device/contracts/command-policy'; +import { createMemorySessionStore } from '../../../command-runtime/runtime-factory.ts'; +import { createLocalArtifactAdapter } from '../../../io.ts'; +import { createAgentDevice } from '../../../runtime.ts'; +import { ref, selector } from './selector-read-utils.ts'; +import { + buttonSnapshot, + createFakeClock, + createSettleDevice, + welcomeSnapshot, +} from './__tests__/settle-device-fixtures.ts'; + +test('a stalled settle capture receives its deadline signal and the press still succeeds', async () => { + const before = buttonSnapshot(); + let captureCalls = 0; + let observedAbort = false; + const device = createAgentDevice({ + backend: { + platform: 'ios', + captureSnapshot: (context) => { + captureCalls += 1; + if (captureCalls === 1) return Promise.resolve({ snapshot: before }); + return new Promise((_resolve, reject) => { + assert.ok(context.signal); + context.signal.addEventListener( + 'abort', + () => { + observedAbort = true; + reject(context.signal?.reason); + }, + { once: true }, + ); + }); + }, + tap: async () => ({ ok: true }), + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([{ name: 'default', snapshot: before }]), + policy: localCommandPolicy(), + clock: createFakeClock(), + }); + + const result = await device.interactions.press(selector('label=Continue'), { + session: 'default', + settle: { quietMs: 25, timeoutMs: 75 }, + }); + + assert.equal(result.kind, 'selector'); + assert.ok(result.settle); + assert.equal(result.settle.settled, false); + assert.equal(result.settle.diff, undefined); + assert.match(result.settle.hint ?? '', /capture stalled past the settle budget/i); + assert.equal(observedAbort, true); +}); + +test('fill on a @ref with --settle carries the settle observation on its result', async () => { + const before = buttonSnapshot(); + const after = welcomeSnapshot(); + let captures = 0; + const device = createSettleDevice({ + stored: before, + captureSnapshot: () => { + captures += 1; + return { snapshot: captures === 1 ? before : after }; + }, + }); + + const result = await device.interactions.fill(ref('@e1'), 'hello', { + session: 'default', + settle: {}, + }); + + assert.equal(result.kind, 'ref'); + assert.ok(result.settle); + assert.equal(result.settle.settled, true); + assert.ok(result.settle.diff); +}); From a15f090ba95abdea945b0b22957d84de94d31466 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 15 Oct 2026 14:35:20 +0200 Subject: [PATCH 07/10] chore(test): repoint the activation checklist at the relocated capture-disclosure test and format Co-Authored-By: Claude Opus 5.5 --- .device-evidence/CHECKLIST.md | 2 +- .../interaction-capture-disclosure.test.ts | 5 +---- .../internal/interaction-common.test.ts | 5 +---- .../interaction-dispatch-disclosure.test.ts | 1 - ...nteraction-settle-private-ax-route.test.ts | 7 ++++-- .../internal/interaction-settle.test.ts | 22 +++++++++---------- 6 files changed, 18 insertions(+), 24 deletions(-) diff --git a/.device-evidence/CHECKLIST.md b/.device-evidence/CHECKLIST.md index b708fc77b8..63f61f0c85 100644 --- a/.device-evidence/CHECKLIST.md +++ b/.device-evidence/CHECKLIST.md @@ -103,7 +103,7 @@ cannot show a cache hit on command: `packages/daemon/src/execution/__tests__/sel drives a session-snapshot cache hit and pins that the consumed tree earns no repair proof and discloses nothing, `packages/daemon/src/core/__tests__/capture-disclosure-target-activation.test.ts` pins that the response builder never borrows a repair the stored snapshot happens to carry, and -`src/__tests__/daemon-composition/interaction/interaction-capture-disclosure.test.ts` pins that +`packages/daemon/src/execution/interaction/internal/interaction-capture-disclosure.test.ts` pins that a press which consumed no capture is not disclosed against an older tree. ## Why the live activation lane is manual diff --git a/packages/daemon/src/execution/interaction/internal/interaction-capture-disclosure.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-capture-disclosure.test.ts index 40a77481f3..d6cacf552a 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-capture-disclosure.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-capture-disclosure.test.ts @@ -9,10 +9,7 @@ import { iosTargetActivationDisclosure } from '@agent-device/contracts/ios-targe import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; import { handleInteractionCommands } from '../index.ts'; import { getRuntimeBindings } from '../../__tests__/interaction-get-runtime-fixture.ts'; -import { - contextFromFlags, - makeSession, -} from './__tests__/interaction-touch-fixtures.ts'; +import { contextFromFlags, makeSession } from './__tests__/interaction-touch-fixtures.ts'; import { legacyDispatchCapture } from '../../../__tests__/support/legacy-snapshot-capture-fixture.ts'; import { markDeferredInteractionOutcome } from '../../../core/deferred-interaction-outcome.ts'; import { formatPostGestureOutcomeWarning } from '@agent-device/capture-kit/post-gesture-stability'; diff --git a/packages/daemon/src/execution/interaction/internal/interaction-common.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-common.test.ts index c7bbf42859..151115d8e8 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-common.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-common.test.ts @@ -8,10 +8,7 @@ import { } from '../../../__tests__/support/session-factories.ts'; import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; import { attachRefs, type RawSnapshotNode } from '@agent-device/kernel/snapshot'; -import { - finalizeTouchInteraction, - handleInteractionCommands, -} from '../index.ts'; +import { finalizeTouchInteraction, handleInteractionCommands } from '../index.ts'; import { IOS_SIMULATOR } from '@agent-device/test-kit'; import { getRuntimeBindings, diff --git a/packages/daemon/src/execution/interaction/internal/interaction-dispatch-disclosure.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-dispatch-disclosure.test.ts index a163f8de98..d885fcb40c 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-dispatch-disclosure.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-dispatch-disclosure.test.ts @@ -31,7 +31,6 @@ import { clearAndroidObservationFixture } from '../../../__tests__/support/andro import { activateCompleteRefFrame, expireRefFrame } from '../../../core/ref-frame.ts'; import { handleInteractionCommands } from '../index.ts'; import type { InteractionRouteInput } from '../../../core/interaction/internal/types.ts'; -import { assertAndroidPressStayedInApp } from './interaction-android-escape.ts'; import { gestureRuntimeBindingsFixture } from './__tests__/gesture-runtime-bindings.fixtures.ts'; import { contextFromFlags, diff --git a/packages/daemon/src/execution/interaction/internal/interaction-settle-private-ax-route.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-settle-private-ax-route.test.ts index f0ed4659f8..9adbbd636b 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-settle-private-ax-route.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-settle-private-ax-route.test.ts @@ -1,7 +1,10 @@ import { beforeEach, expect, test, vi } from 'vitest'; -import type { Interactor, SnapshotResult } from '@agent-device/contracts/interactor-types'; +import type { + Interactor, + SnapshotOptions, + SnapshotResult, +} from '@agent-device/contracts/interactor-types'; import type { SettleObservation } from '@agent-device/contracts/interaction'; -import type { SnapshotOptions } from '@agent-device/contracts/interactor-types'; import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; import { makeIosSession } from '../../../__tests__/support/session-factories.ts'; import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; diff --git a/packages/daemon/src/execution/interaction/internal/interaction-settle.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-settle.test.ts index ec12509fb1..2cbfe26249 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-settle.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-settle.test.ts @@ -1,10 +1,7 @@ import type { CommandFlags } from '@agent-device/contracts/command'; import { legacyDispatchCapture } from '../../../__tests__/support/legacy-snapshot-capture-fixture.ts'; import { test, expect, vi, beforeEach } from 'vitest'; -import { - createInteractionRuntime, - handleInteractionCommands, -} from '../index.ts'; +import { createInteractionRuntime, handleInteractionCommands } from '../index.ts'; import type { SessionStore } from '../../../core/session-store.ts'; import type { SessionRef, SessionState } from '../../../core/session-state.ts'; import { buildSnapshotState } from '@agent-device/capture-kit/snapshot-state'; @@ -125,7 +122,10 @@ function settledDiff(baseline: SnapshotNode[] | undefined, settled: SnapshotNode diff: { summary: { additions: added.length, removals: removed.length, unchanged: unchanged.length }, lines: [ - ...removed.map((node) => ({ kind: 'removed' as const, text: `${node.type} "${node.label}"` })), + ...removed.map((node) => ({ + kind: 'removed' as const, + text: `${node.type} "${node.label}"`, + })), ...added.map((node) => ({ kind: 'added' as const, text: `${node.type} "${node.label}"`, @@ -135,13 +135,11 @@ function settledDiff(baseline: SnapshotNode[] | undefined, settled: SnapshotNode }, tail: added.length === 0 - ? unchanged - .filter((node) => node.hittable && node.label) - .map((node) => ({ - ref: node.ref, - role: node.type.toLowerCase(), - label: node.label, - })) + ? unchanged.flatMap((node) => + node.hittable && node.label && node.ref + ? [{ ref: node.ref, role: (node.type ?? '').toLowerCase(), label: node.label }] + : [], + ) : undefined, }; } From 1662807174ffe53e4de41930e8fb38e377172177 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 15 Oct 2026 21:51:07 +0200 Subject: [PATCH 08/10] chore(daemon): drop the two testing subpaths the settle and disclosure batch left without importers Co-Authored-By: Claude Opus 5.5 --- packages/daemon/package.json | 8 -------- 1 file changed, 8 deletions(-) diff --git a/packages/daemon/package.json b/packages/daemon/package.json index 32374c3b95..d3059b0bc7 100644 --- a/packages/daemon/package.json +++ b/packages/daemon/package.json @@ -184,18 +184,10 @@ "types": "./src/execution/interaction/internal/__tests__/*.ts", "default": "./src/execution/interaction/internal/__tests__/*.ts" }, - "./testing/interaction/internal/interaction-android-escape": { - "types": "./src/execution/interaction/internal/interaction-android-escape.ts", - "default": "./src/execution/interaction/internal/interaction-android-escape.ts" - }, "./testing/interaction/internal/interaction-touch-response": { "types": "./src/execution/interaction/internal/interaction-touch-response.ts", "default": "./src/execution/interaction/internal/interaction-touch-response.ts" }, - "./testing/interaction/internal/types": { - "types": "./src/core/interaction/internal/types.ts", - "default": "./src/core/interaction/internal/types.ts" - }, "./testing/interactor-resolution": { "types": "./src/core/interactor-resolution.ts", "default": "./src/core/interactor-resolution.ts" From a39b933df5319098b5df634c5b654fabbb3507b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 00:04:07 +0200 Subject: [PATCH 09/10] test(interaction): assert typed refusals in the escape and post-gesture cases, and resolve refs against the ref frame in the touch-forwarding fake Co-Authored-By: Claude Opus 5.5 --- .../touch-forwarding-surface-fake.test.ts | 32 +++++++++++++++++++ .../support/touch-forwarding-surface-fake.ts | 5 +-- ...android-escape-dispatch-disclosure.test.ts | 3 ++ ...tor-target-resolution-post-gesture.test.ts | 17 ++++++++-- 4 files changed, 52 insertions(+), 5 deletions(-) create mode 100644 packages/daemon/src/__tests__/support/touch-forwarding-surface-fake.test.ts diff --git a/packages/daemon/src/__tests__/support/touch-forwarding-surface-fake.test.ts b/packages/daemon/src/__tests__/support/touch-forwarding-surface-fake.test.ts new file mode 100644 index 0000000000..0afe844c17 --- /dev/null +++ b/packages/daemon/src/__tests__/support/touch-forwarding-surface-fake.test.ts @@ -0,0 +1,32 @@ +import { expect, test, vi } from 'vitest'; +import type { AgentDeviceRuntimeConfig } from '@agent-device/contracts/runtime-contract'; +import { attachRefs, type SnapshotState } from '@agent-device/kernel/snapshot'; +import { touchForwardingCreateCommandSurface } from './touch-forwarding-surface-fake.ts'; + +function tree(rect: { x: number; y: number; width: number; height: number }): SnapshotState { + return { + nodes: attachRefs([{ index: 0, type: 'Button', label: 'Go', rect, hittable: true }]), + createdAt: 1, + backend: 'xctest', + }; +} + +test('a ref resolves against the authorized ref frame before the latest observation', async () => { + const tap = vi.fn(async () => undefined); + const config = { + backend: { platform: 'ios', tap }, + sessions: { + get: async () => ({ + name: 'default', + snapshot: tree({ x: 200, y: 200, width: 20, height: 20 }), + refFrameSnapshot: tree({ x: 0, y: 0, width: 20, height: 20 }), + }), + set: async () => {}, + }, + } as unknown as AgentDeviceRuntimeConfig; + const surface = await touchForwardingCreateCommandSurface()(config); + + await surface.interactions.press({ kind: 'ref', ref: '@e1' }, { session: 'default' }); + + expect(tap).toHaveBeenCalledWith(expect.anything(), { x: 10, y: 10 }); +}); diff --git a/packages/daemon/src/__tests__/support/touch-forwarding-surface-fake.ts b/packages/daemon/src/__tests__/support/touch-forwarding-surface-fake.ts index c530661198..f1694f3e37 100644 --- a/packages/daemon/src/__tests__/support/touch-forwarding-surface-fake.ts +++ b/packages/daemon/src/__tests__/support/touch-forwarding-surface-fake.ts @@ -91,7 +91,8 @@ export function touchForwardingCreateCommandSurface( if (target.kind === 'point') return { point: { x: target.x, y: target.y }, target }; if (target.kind === 'ref') { const record = await config.sessions?.get(context.session ?? 'default'); - const nodes = record?.snapshot?.nodes ?? []; + const tree = record?.refFrameSnapshot ?? record?.snapshot; + const nodes = tree?.nodes ?? []; const node = nodes.find((candidate) => candidate.ref === refBody(target.ref)); if (!node?.rect) { throw new AppError('COMMAND_FAILED', `Ref ${target.ref} not found`, { @@ -105,7 +106,7 @@ export function touchForwardingCreateCommandSurface( node, nodes, selectorChain: chainFor(node, nodes), - snapshotQuality: record?.snapshot?.snapshotQuality, + snapshotQuality: tree?.snapshotQuality, }; } const { nodes, snapshotQuality } = await captureAndStoreSnapshot(config, { diff --git a/packages/daemon/src/execution/interaction/internal/interaction-android-escape-dispatch-disclosure.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-android-escape-dispatch-disclosure.test.ts index a93b69745a..da6bd3bafc 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-android-escape-dispatch-disclosure.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-android-escape-dispatch-disclosure.test.ts @@ -53,6 +53,9 @@ for (const row of ROWS) { assert.ok(drive, `no driver for ${row.id}`); await assert.rejects(drive(), (error: unknown) => { assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.details?.expectedPackage, 'com.example.app'); + assert.equal(error.details?.foregroundPackage, 'com.android.settings'); assert.equal(error.details?.dispatched, row.dispatched); return true; }); diff --git a/src/commands/interaction/runtime/selector-target-resolution-post-gesture.test.ts b/src/commands/interaction/runtime/selector-target-resolution-post-gesture.test.ts index cf1e04b374..e144928e4a 100644 --- a/src/commands/interaction/runtime/selector-target-resolution-post-gesture.test.ts +++ b/src/commands/interaction/runtime/selector-target-resolution-post-gesture.test.ts @@ -30,13 +30,24 @@ test('a selector press retries a miss on the full tree and keeps the unsettled g const interactive = { ...movingList('Wi-Fi'), postGestureOutcome: UNSETTLED }; const full = movingList('Wi-Fi'); const captures = [interactive, full]; + const taps: unknown[] = []; const device = createInteractionDevice(movingList('Wi-Fi'), { captureSnapshot: async () => ({ snapshot: captures.shift() ?? full }), + tap: async (_context, point) => { + taps.push(point); + }, }); - await assert.rejects(() => - device.interactions.press(selector('label="General"'), { session: 'default' }), + await assert.rejects( + () => device.interactions.press(selector('label="General"'), { session: 'default' }), + (error: unknown) => { + assert.ok(error instanceof Error); + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'selector_not_found'); + assert.equal(details?.dispatched, 'no'); + return true; + }, ); - + assert.deepEqual(taps, []); assert.deepEqual(full.postGestureOutcome, UNSETTLED); }); From 9c7f7103d2c7037f1a912d8d7c7711583fd06cd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 00:24:50 +0200 Subject: [PATCH 10/10] test(interaction): assert the ref-frame preference through the fill handler test and drop the standalone fake test (R84) Co-Authored-By: Claude Opus 5.5 --- .../touch-forwarding-surface-fake.test.ts | 32 ------------ .../internal/interaction-common.test.ts | 49 +++++++++++++++++++ 2 files changed, 49 insertions(+), 32 deletions(-) delete mode 100644 packages/daemon/src/__tests__/support/touch-forwarding-surface-fake.test.ts diff --git a/packages/daemon/src/__tests__/support/touch-forwarding-surface-fake.test.ts b/packages/daemon/src/__tests__/support/touch-forwarding-surface-fake.test.ts deleted file mode 100644 index 0afe844c17..0000000000 --- a/packages/daemon/src/__tests__/support/touch-forwarding-surface-fake.test.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { expect, test, vi } from 'vitest'; -import type { AgentDeviceRuntimeConfig } from '@agent-device/contracts/runtime-contract'; -import { attachRefs, type SnapshotState } from '@agent-device/kernel/snapshot'; -import { touchForwardingCreateCommandSurface } from './touch-forwarding-surface-fake.ts'; - -function tree(rect: { x: number; y: number; width: number; height: number }): SnapshotState { - return { - nodes: attachRefs([{ index: 0, type: 'Button', label: 'Go', rect, hittable: true }]), - createdAt: 1, - backend: 'xctest', - }; -} - -test('a ref resolves against the authorized ref frame before the latest observation', async () => { - const tap = vi.fn(async () => undefined); - const config = { - backend: { platform: 'ios', tap }, - sessions: { - get: async () => ({ - name: 'default', - snapshot: tree({ x: 200, y: 200, width: 20, height: 20 }), - refFrameSnapshot: tree({ x: 0, y: 0, width: 20, height: 20 }), - }), - set: async () => {}, - }, - } as unknown as AgentDeviceRuntimeConfig; - const surface = await touchForwardingCreateCommandSurface()(config); - - await surface.interactions.press({ kind: 'ref', ref: '@e1' }, { session: 'default' }); - - expect(tap).toHaveBeenCalledWith(expect.anything(), { x: 10, y: 10 }); -}); diff --git a/packages/daemon/src/execution/interaction/internal/interaction-common.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-common.test.ts index 151115d8e8..7e3a513953 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-common.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-common.test.ts @@ -8,6 +8,7 @@ import { } from '../../../__tests__/support/session-factories.ts'; import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; import { attachRefs, type RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import { activateCompleteRefFrame } from '../../../core/ref-frame.ts'; import { finalizeTouchInteraction, handleInteractionCommands } from '../index.ts'; import { IOS_SIMULATOR } from '@agent-device/test-kit'; import { @@ -295,3 +296,51 @@ test.each([ expect(session.actions[0]?.result).toEqual(response.data); }, ); + +test('a fill on a @ref acts on the node of the authorized ref frame, not on a newer observation', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'ref-frame-authority-fill'; + const session = makeIosSession(sessionName, { appBundleId: 'com.example.app' }); + const field = (rect: { x: number; y: number; width: number; height: number }) => + attachRefs([ + { + index: 0, + type: 'XCUIElementTypeTextField', + label: 'Email', + rect, + enabled: true, + hittable: true, + }, + ] as RawSnapshotNode[]); + session.snapshot = { + nodes: field({ x: 0, y: 0, width: 20, height: 20 }), + createdAt: Date.now(), + backend: 'xctest', + }; + activateCompleteRefFrame(session); + session.snapshot = { + nodes: field({ x: 200, y: 200, width: 20, height: 20 }), + createdAt: Date.now(), + backend: 'xctest', + }; + sessionStore.publish(sessionName, session); + mockFillPoint.mockResolvedValue({ filled: true }); + + const response = await handleInteractionCommands({ + createCommandSurface: touchForwardingCreateCommandSurface(), + req: { + token: 't', + session: sessionName, + command: 'fill', + positionals: ['@e1', 'hello'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(response?.ok, JSON.stringify(response)).toBe(true); + expect(mockFillPoint).toHaveBeenCalledWith(expect.objectContaining({ point: { x: 10, y: 10 } })); +});