From bfb00ef4766d169ebd6c8349b3c91cb60059120a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 28 Aug 2026 22:50:23 +0200 Subject: [PATCH] test: characterize replay application behavior --- .../session-replay-application.test.ts | 296 ++++++++++++++++++ 1 file changed, 296 insertions(+) create mode 100644 src/daemon/handlers/__tests__/session-replay-application.test.ts diff --git a/src/daemon/handlers/__tests__/session-replay-application.test.ts b/src/daemon/handlers/__tests__/session-replay-application.test.ts new file mode 100644 index 000000000..f324a51d0 --- /dev/null +++ b/src/daemon/handlers/__tests__/session-replay-application.test.ts @@ -0,0 +1,296 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { beforeEach, expect, test, vi } from 'vitest'; +import type { RequestProgressEvent } from '@agent-device/contracts/progress'; +import type { PlatformResourceCleanup } from '@agent-device/contracts/platform-resource-cleanup'; +import { + clearRequestCanceled, + markRequestCanceled, + registerRequestAbort, + withRequestProgressSink, +} from '@agent-device/host-kit/request'; +import { mkdtempForTestSync } from '../../../__tests__/test-utils/tmp-dir.ts'; +import { makeIosSession } from '../../../__tests__/test-utils/session-factories.ts'; +import { withClientReplayScriptSources } from '../../../__tests__/test-utils/replay-script-source.ts'; +import type { DaemonRequest, DaemonResponse } from '../../types.ts'; +import { SessionStore } from '../../session-store.ts'; +import { LeaseRegistry } from '../../lease-registry.ts'; +import { handleSessionReplayCommands } from '../session-replay.ts'; +import { captureSnapshotWithInteractor } from '../snapshot-interactor-capture.ts'; +import { + legacyDispatchCapture, + resetLegacySnapshotCapture, +} from '../../__tests__/legacy-snapshot-capture-fixture.ts'; +import { baseReplayRequest, writeReplayFile } from './session-replay-runtime.fixtures.ts'; + +vi.mock('../snapshot-interactor-capture.ts', () => ({ + captureSnapshotWithInteractor: vi.fn(), +})); + +const mockCaptureSnapshotWithInteractor = vi.mocked(captureSnapshotWithInteractor); + +const noopPlatformResourceCleanup = Object.freeze({ + stopSnapshotHelper: async () => {}, + closeManagedBrowser: async () => {}, + cleanupSessionlessExecutionHost: async () => {}, + retainExecutionHostAfterClose: () => false, +}) satisfies PlatformResourceCleanup; + +beforeEach(() => { + resetLegacySnapshotCapture(mockCaptureSnapshotWithInteractor); +}); + +function replayParams( + req: DaemonRequest, + root: string, + sessionStore: SessionStore, + invoke: (request: DaemonRequest) => Promise, +) { + return { + req, + sessionName: req.session, + logPath: path.join(root, 'daemon.log'), + sessionStore, + leaseRegistry: new LeaseRegistry(), + invoke, + }; +} + +function suiteParams( + req: DaemonRequest, + root: string, + sessionStore: SessionStore, + invoke: (request: DaemonRequest) => Promise, +) { + return { + ...replayParams(req, root, sessionStore, invoke), + platformResourceCleanup: noopPlatformResourceCleanup, + }; +} + +test('replay routes native and Maestro sources through one application seam and keeps the session', async () => { + const cases = [ + { + kind: 'native', + buildRequest: (root: string) => { + const filePath = writeReplayFile(root, ['open "Native"']); + return baseReplayRequest({ positionals: [filePath], flags: { platform: 'ios' } }); + }, + expectedCommands: ['open'], + }, + { + kind: 'Maestro', + buildRequest: async (root: string) => { + const filePath = path.join(root, 'flow.yaml'); + fs.writeFileSync( + filePath, + ['appId: com.example.app', '---', '- launchApp', '- inputText: typed'].join('\n'), + ); + return await withClientReplayScriptSources({ + token: 'token', + session: 'default', + command: 'replay', + positionals: [filePath], + flags: { replayBackend: 'maestro', platform: 'ios' }, + }); + }, + expectedCommands: ['open', 'type'], + }, + ] as const; + + for (const replayCase of cases) { + const root = mkdtempForTestSync(`agent-device-replay-application-${replayCase.kind}-`); + const sessionStore = new SessionStore(path.join(root, 'sessions')); + sessionStore.set('default', makeIosSession('default')); + const req = await replayCase.buildRequest(root); + const invoked: DaemonRequest[] = []; + + const response = await handleSessionReplayCommands( + replayParams(req, root, sessionStore, async (request) => { + invoked.push(request); + return request.command === 'snapshot' + ? { ok: true, data: { createdAt: 0, nodes: [] } } + : { ok: true, data: {} }; + }), + ); + + expect(response).toMatchObject({ + ok: true, + data: { replayed: replayCase.expectedCommands.length }, + }); + if (!response?.ok) continue; + expect(response.data?.sessionActive).toBe(true); + expect( + invoked.filter(({ command }) => command !== 'snapshot').map(({ command }) => command), + ).toEqual(replayCase.expectedCommands); + } +}); + +test('replay admits an annotated target before dispatching its mutation', async () => { + const root = mkdtempForTestSync('agent-device-replay-application-admission-'); + const sessionStore = new SessionStore(path.join(root, 'sessions')); + sessionStore.set('default', makeIosSession('default', { appBundleId: 'com.example.app' })); + const filePath = writeReplayFile(root, [ + '# agent-device:target-v1 {"id":"save","role":"button","label":"Save","ancestry":[],"sibling":0,"viewportOrder":0,"verification":"verified"}', + 'click label="Save"', + ]); + legacyDispatchCapture.mockResolvedValue({ + nodes: [ + { + index: 0, + depth: 0, + type: 'Button', + identifier: 'renamed-save', + label: 'Save', + rect: { x: 10, y: 10, width: 40, height: 20 }, + }, + ], + truncated: false, + backend: 'xctest', + }); + const invoked = vi.fn(async () => ({ ok: true as const, data: {} })); + + const response = await handleSessionReplayCommands( + replayParams(baseReplayRequest({ positionals: [filePath] }), root, sessionStore, invoked), + ); + + expect(invoked).not.toHaveBeenCalled(); + expect(response).toMatchObject({ + ok: false, + error: { + code: 'REPLAY_DIVERGENCE', + details: { divergence: { kind: 'identity-mismatch' } }, + }, + }); +}); + +test('replay projects the exact failure cause and retains artifacts from the failed step', async () => { + const root = mkdtempForTestSync('agent-device-replay-application-failure-'); + const sessionStore = new SessionStore(path.join(root, 'sessions')); + sessionStore.set('default', makeIosSession('default')); + const artifactPath = path.join(root, 'failure.png'); + fs.writeFileSync(artifactPath, 'artifact'); + const filePath = writeReplayFile(root, ['open "Demo"']); + mockCaptureSnapshotWithInteractor.mockRejectedValue(new Error('capture unavailable')); + + const response = await handleSessionReplayCommands( + replayParams(baseReplayRequest({ positionals: [filePath] }), root, sessionStore, async () => ({ + ok: false, + error: { + code: 'COMMAND_FAILED', + message: 'runner failed', + hint: 'retry the open', + diagnosticId: 'diag-1', + logPath: '/tmp/runner.log', + details: { reason: 'runner-timeout', artifactPaths: [artifactPath] }, + }, + })), + ); + + expect(response).toMatchObject({ + ok: false, + error: { + code: 'REPLAY_DIVERGENCE', + message: 'Replay failed at step 1 (open "Demo"): runner failed', + hint: 'retry the open', + diagnosticId: 'diag-1', + logPath: '/tmp/runner.log', + details: { + reason: 'runner-timeout', + replayPath: filePath, + step: 1, + action: 'open', + positionals: [''], + artifactPaths: [artifactPath], + divergence: { + kind: 'action-failure', + cause: { code: 'COMMAND_FAILED', message: 'runner failed', hint: 'retry the open' }, + screen: { state: 'unavailable' }, + }, + }, + }, + }); +}); + +test('test hosts native and Maestro sources format-neutrally and reports replay progress', async () => { + const root = mkdtempForTestSync('agent-device-replay-application-suite-'); + const sessionStore = new SessionStore(path.join(root, 'sessions')); + const nativePath = path.join(root, '01-native.ad'); + const maestroPath = path.join(root, '02-maestro.yaml'); + fs.writeFileSync(nativePath, 'context platform=ios\nopen "Native"\n'); + fs.writeFileSync(maestroPath, 'appId: com.example.app\n---\n- launchApp\n- inputText: typed\n'); + const req = await withClientReplayScriptSources({ + token: 'token', + session: 'default', + command: 'test', + positionals: [nativePath, maestroPath], + flags: { platform: 'ios', replayBackend: 'maestro' }, + meta: { cwd: root, requestId: 'format-neutral-suite' }, + }); + const events: RequestProgressEvent[] = []; + const invoked: DaemonRequest[] = []; + + const response = await withRequestProgressSink( + (event) => events.push(event), + async () => + await handleSessionReplayCommands( + suiteParams(req, root, sessionStore, async (request) => { + invoked.push(request); + return request.command === 'snapshot' + ? { ok: true, data: { createdAt: 0, nodes: [] } } + : { ok: true, data: {} }; + }), + ), + ); + + expect(response).toMatchObject({ ok: true, data: { total: 2, executed: 2, passed: 2 } }); + expect( + invoked.filter(({ command }) => command !== 'snapshot').map(({ command }) => command), + ).toEqual(['open', 'open', 'type']); + expect(events).toEqual( + expect.arrayContaining([ + expect.objectContaining({ type: 'replay-test-suite', status: 'start', total: 2 }), + expect.objectContaining({ type: 'replay-test', status: 'progress', stepIndex: 1 }), + expect.objectContaining({ type: 'replay-test', status: 'pass' }), + ]), + ); +}); + +test('test stops after an active cancellation and does not start the next replay', async () => { + const root = mkdtempForTestSync('agent-device-replay-application-cancel-'); + const sessionStore = new SessionStore(path.join(root, 'sessions')); + const firstPath = writeReplayFile(root, ['open "First"']); + const secondPath = path.join(root, 'second.ad'); + fs.writeFileSync(secondPath, 'open "Second"\n'); + const requestId = 'application-cancel'; + const req = await withClientReplayScriptSources({ + token: 'token', + session: 'default', + command: 'test', + positionals: [firstPath, secondPath], + meta: { cwd: root, requestId }, + }); + const invoked: DaemonRequest[] = []; + + registerRequestAbort(requestId); + try { + const response = await handleSessionReplayCommands( + suiteParams(req, root, sessionStore, async (request) => { + invoked.push(request); + markRequestCanceled(requestId); + return { + ok: false, + error: { code: 'COMMAND_FAILED', message: 'request canceled' }, + }; + }), + ); + + expect(response).toMatchObject({ + ok: true, + data: { total: 2, executed: 1, failed: 1, notRun: 1 }, + }); + expect(invoked).toHaveLength(1); + } finally { + clearRequestCanceled(requestId); + } +});