diff --git a/packages/daemon/package.json b/packages/daemon/package.json index 02692edfbb..3d7ffb08b1 100644 --- a/packages/daemon/package.json +++ b/packages/daemon/package.json @@ -48,10 +48,6 @@ "types": "./src/core/__tests__/capture-disclosure-target-activation-fact.ts", "default": "./src/core/__tests__/capture-disclosure-target-activation-fact.ts" }, - "./testing/__tests__/human-control-fixtures": { - "types": "./src/__tests__/support/human-control-fixtures.ts", - "default": "./src/__tests__/support/human-control-fixtures.ts" - }, "./testing/__tests__/interaction-get-runtime-fixture": { "types": "./src/execution/__tests__/interaction-get-runtime-fixture.ts", "default": "./src/execution/__tests__/interaction-get-runtime-fixture.ts" @@ -60,10 +56,6 @@ "types": "./src/__tests__/support/legacy-snapshot-capture-fixture.ts", "default": "./src/__tests__/support/legacy-snapshot-capture-fixture.ts" }, - "./testing/__tests__/platform-services-fixture": { - "types": "./src/__tests__/support/platform-services-fixture.ts", - "default": "./src/__tests__/support/platform-services-fixture.ts" - }, "./testing/__tests__/request-router-dispatch-mocks": { "types": "./src/__tests__/support/request-router-dispatch-mocks.ts", "default": "./src/__tests__/support/request-router-dispatch-mocks.ts" @@ -156,22 +148,10 @@ "types": "./src/core/gesture-runtime.ts", "default": "./src/core/gesture-runtime.ts" }, - "./testing/handlers/__tests__/*": { - "types": "./src/__tests__/support/handlers/*.ts", - "default": "./src/__tests__/support/handlers/*.ts" - }, - "./testing/handlers/lease": { - "types": "./src/sessions/handlers/lease.ts", - "default": "./src/sessions/handlers/lease.ts" - }, "./testing/handlers/session-app-deployment": { "types": "./src/sessions/handlers/session-app-deployment.ts", "default": "./src/sessions/handlers/session-app-deployment.ts" }, - "./testing/handlers/session-appstate": { - "types": "./src/sessions/handlers/session-appstate.ts", - "default": "./src/sessions/handlers/session-appstate.ts" - }, "./testing/handlers/snapshot": { "types": "./src/sessions/handlers/snapshot.ts", "default": "./src/sessions/handlers/snapshot.ts" @@ -284,10 +264,6 @@ "types": "./src/server/http/http-server.ts", "default": "./src/server/http/http-server.ts" }, - "./testing/server/transport": { - "types": "./src/server/http/transport.ts", - "default": "./src/server/http/transport.ts" - }, "./testing/session-action-recorder": { "types": "./src/core/session-action-recorder.ts", "default": "./src/core/session-action-recorder.ts" @@ -304,10 +280,6 @@ "types": "./src/core/session-recovery-hints.ts", "default": "./src/core/session-recovery-hints.ts" }, - "./testing/session-runtime": { - "types": "./src/sessions/session-runtime.ts", - "default": "./src/sessions/session-runtime.ts" - }, "./testing/session-snapshot": { "types": "./src/execution/session-snapshot.ts", "default": "./src/execution/session-snapshot.ts" diff --git a/packages/daemon/src/core/session-action-recorder-admission.test.ts b/packages/daemon/src/core/session-action-recorder-admission.test.ts new file mode 100644 index 0000000000..54c1549f76 --- /dev/null +++ b/packages/daemon/src/core/session-action-recorder-admission.test.ts @@ -0,0 +1,46 @@ +import assert from 'node:assert/strict'; +import { test, vi } from 'vitest'; +import { getFlagDefinitionsForKey } from '@agent-device/command-registry/flag-registry'; +import type { FlagDefinition } from '@agent-device/command-registry/flag-types'; +import { makeIosSession } from '../__tests__/support/session-factories.ts'; +import { recordActionEntry } from './session-action-recorder.ts'; + +test('recorder sanitization follows the declaration: an undeclared key is dropped, a planted declaration is recorded', async () => { + assert.equal(declarationFor('overlayRefs').recorded, false); + const shipped = recordActionEntry(makeIosSession('default'), { + command: 'screenshot', + positionals: [], + flags: { overlayRefs: true, fps: 5 }, + result: {}, + }); + assert.equal(shipped?.flags.overlayRefs, undefined, 'overlayRefs is undeclared for recording'); + assert.equal(shipped?.flags.fps, 5, 'fps is declared for recording'); + + vi.resetModules(); + const registry = await import('@agent-device/command-registry/flag-registry'); + const planted = declarationForIn(registry, 'overlayRefs'); + Object.assign(planted, { recorded: true }); + const { recordActionEntry: recordPlanted } = await import('./session-action-recorder.ts'); + const recorded = recordPlanted(makeIosSession('default'), { + command: 'screenshot', + positionals: [], + flags: { overlayRefs: true }, + result: {}, + }); + assert.equal(recorded?.flags.overlayRefs, true, 'a planted declaration is copied'); +}); + +function declarationFor(key: 'overlayRefs'): FlagDefinition { + const declaration = getFlagDefinitionsForKey(key)[0]; + assert.ok(declaration, `expected a declaration for ${key}`); + return declaration; +} + +function declarationForIn( + registry: typeof import('@agent-device/command-registry/flag-registry'), + key: 'overlayRefs', +): FlagDefinition { + const declaration = registry.getFlagDefinitionsForKey(key)[0]; + assert.ok(declaration, `expected a declaration for ${key}`); + return declaration; +} diff --git a/packages/daemon/src/sessions/handlers/__tests__/session-appstate-facts.test.ts b/packages/daemon/src/sessions/handlers/__tests__/session-appstate-facts.test.ts new file mode 100644 index 0000000000..c65c263872 --- /dev/null +++ b/packages/daemon/src/sessions/handlers/__tests__/session-appstate-facts.test.ts @@ -0,0 +1,67 @@ +import { test, expect } from 'vitest'; +import * as path from 'node:path'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { providerRuntimeOwner } from '@agent-device/contracts/platform-runtime'; +import { createUnavailableRuntimeFactsForTest, mkdtempForTestSync } from '@agent-device/test-kit'; +import { daemonPlatformServicesFixture } from '../../../__tests__/support/platform-services-fixture.ts'; +import { + handleSessionCommands, + mockBindDeviceRuntime, + mockInspectDeviceRuntimeFacts, +} from '../../../__tests__/support/handlers/session-command-harness.ts'; +import { + makeSessionStore, + noopInvoke, +} from '../../../__tests__/support/handlers/session-test-harness.ts'; +import { makeSession } from '../../../__tests__/support/session-factories.ts'; + +test('appstate refuses unavailable provider facts before binding the device', async () => { + const staleDevice: DeviceInfo = { + platform: 'android', + id: 'limrun:android:released-lease', + name: 'Released Limrun Android', + kind: 'emulator', + target: 'mobile', + booted: true, + }; + const unavailable = createUnavailableRuntimeFactsForTest( + staleDevice, + providerRuntimeOwner('limrun', 'stale'), + { + available: false, + reason: 'unsupported-provider-mode', + hint: 'Limrun requires a matching live provider session for this device.', + }, + ); + mockInspectDeviceRuntimeFacts.mockResolvedValue(unavailable); + const sessionStore = makeSessionStore(); + sessionStore.publish('default', makeSession('default', { device: staleDevice })); + + const response = await handleSessionCommands({ + createCommandSurface: async () => { + throw new Error('appstate must refuse before loading a command surface'); + }, + platformServices: daemonPlatformServicesFixture(), + req: { + token: 't', + session: 'default', + command: 'appstate', + positionals: [], + flags: { platform: 'android' }, + }, + sessionName: 'default', + logPath: path.join(mkdtempForTestSync('daemon-appstate-facts-'), 'daemon.log'), + sessionStore, + invoke: noopInvoke, + }); + + expect(response?.ok).toBe(false); + if (response && !response.ok) { + expect(response.error.code).toBe('UNSUPPORTED_OPERATION'); + expect(response.error.hint).toBe( + 'Limrun requires a matching live provider session for this device.', + ); + } + expect(mockInspectDeviceRuntimeFacts).toHaveBeenCalledOnce(); + expect(mockBindDeviceRuntime).not.toHaveBeenCalled(); +}); diff --git a/packages/daemon/src/sessions/handlers/session-progress.test.ts b/packages/daemon/src/sessions/handlers/session-progress.test.ts new file mode 100644 index 0000000000..ce17804738 --- /dev/null +++ b/packages/daemon/src/sessions/handlers/session-progress.test.ts @@ -0,0 +1,305 @@ +import type { RequestProgressEvent } from '@agent-device/contracts/progress'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { withTestDeviceInventoryProvider } from '@agent-device/device-selection/inventory-gateways-fixtures'; +import { withRequestProgressSink } from '@agent-device/host-kit/request'; +import { mkdtempForTestSync } from '@agent-device/test-kit'; +import fs from 'node:fs'; +import path from 'node:path'; +import { expect, test } from 'vitest'; +import { daemonPlatformServicesFixture } from '../../__tests__/support/platform-services-fixture.ts'; +import { fakeCreateCommandSurface } from '../../__tests__/support/command-surface-fake.ts'; +import { handleSessionCommands } from '../../__tests__/support/handlers/session-command-harness.ts'; +import type { DaemonRequest } from '../../core/daemon-request.ts'; +import { + expectOkData, + makeSessionStore, +} from '../__tests__/replay-suite/session-test-suite.fixtures.ts'; + +const ANDROID_ONE: DeviceInfo = { + platform: 'android', + id: 'emulator-5554', + name: 'Pixel 8', + kind: 'emulator', + booted: true, +}; + +const ANDROID_TWO: DeviceInfo = { + platform: 'android', + id: 'emulator-5556', + name: 'Pixel 8 Pro', + kind: 'emulator', + booted: true, +}; + +async function runSuite(params: { + root: string; + requestId: string; + flags?: DaemonRequest['flags']; + devices?: DeviceInfo[]; + invoke: (request: DaemonRequest) => Promise< + | { ok: true; data?: Record } + | { + ok: false; + error: { code: 'COMMAND_FAILED' | 'ASSERTION_FAILED'; message: string; hint?: string }; + } + >; +}): Promise<{ events: RequestProgressEvent[]; data: ReturnType }> { + const events: RequestProgressEvent[] = []; + const run = async () => + await withRequestProgressSink( + (event) => events.push(event), + async () => + await handleSessionCommands({ + createCommandSurface: fakeCreateCommandSurface, + platformServices: daemonPlatformServicesFixture(), + req: { + token: 't', + session: 'default', + command: 'test', + positionals: [params.root], + flags: params.flags, + meta: { cwd: params.root, requestId: params.requestId }, + }, + sessionName: 'default', + logPath: path.join(params.root, 'daemon.log'), + sessionStore: makeSessionStore(), + invoke: params.invoke, + }), + ); + const response = params.devices + ? await withTestDeviceInventoryProvider(async () => params.devices ?? [], run) + : await run(); + const data = expectOkData(response); + return { events, data }; +} + +function replayEvents(events: RequestProgressEvent[]) { + return events.filter( + (event): event is Extract => + event.type === 'replay-test', + ); +} + +test('native suite progress carries skip, retry hint, and attempt session values', async () => { + const root = mkdtempForTestSync('agent-device-session-test-progress-'); + const skippedPath = path.join(root, '01-untyped.ad'); + const retryPath = path.join(root, '02-retry.ad'); + fs.writeFileSync(skippedPath, 'open "Demo"\n'); + fs.writeFileSync(retryPath, 'context platform=android\nopen "Demo"\n'); + let attempts = 0; + + const { events, data } = await runSuite({ + root, + requestId: 'progress', + flags: { platform: 'android', retries: 1 }, + invoke: async () => { + attempts += 1; + if (attempts === 1) { + return { + ok: false, + error: { + code: 'COMMAND_FAILED', + message: 'first attempt failed', + hint: 'retry hint from the failed attempt', + }, + }; + } + return { ok: true, data: { replayed: 1, healed: 0 } }; + }, + }); + + expect(data.passed).toBe(1); + expect(data.skipped).toBe(1); + expect(events[0]).toMatchObject({ + type: 'replay-test-suite', + status: 'start', + total: 2, + runnable: 1, + skipped: 1, + }); + const testEvents = replayEvents(events); + expect(testEvents.map((event) => event.status)).toEqual([ + 'skip', + 'start', + 'progress', + 'fail', + 'progress', + 'pass', + ]); + expect(testEvents[0]).toMatchObject({ + file: skippedPath, + status: 'skip', + index: 1, + total: 2, + message: 'missing platform metadata for --platform android', + }); + expect(testEvents[0]?.durationMs).toBeUndefined(); + expect(testEvents[1]).toMatchObject({ + file: retryPath, + status: 'start', + index: 2, + total: 2, + maxAttempts: 2, + session: 'default:test:progress:1-02-retry:attempt-1', + }); + expect(testEvents[2]).toMatchObject({ + status: 'progress', + attempt: 1, + stepIndex: 1, + stepTotal: 1, + stepCommand: 'open', + stepValue: 'Demo', + }); + expect(testEvents[3]).toMatchObject({ + status: 'fail', + attempt: 1, + retrying: true, + message: 'Replay failed at step 1 (open "Demo"): first attempt failed', + hint: 'retry hint from the failed attempt', + session: 'default:test:progress:1-02-retry:attempt-1', + }); + expect(testEvents[4]).toMatchObject({ + status: 'progress', + attempt: 2, + stepCommand: 'open', + session: 'default:test:progress:1-02-retry:attempt-2', + }); + expect(testEvents[5]).toMatchObject({ + status: 'pass', + attempt: 2, + session: 'default:test:progress:1-02-retry:attempt-2', + }); +}); + +test('Maestro progress carries its title, step payload, and retry attempt identity', async () => { + const root = mkdtempForTestSync('agent-device-session-test-maestro-progress-'); + const flowPath = path.join(root, '01-login.yaml'); + fs.writeFileSync( + flowPath, + ['appId: demo.app', 'name: Login flow', '---', '- assertNotVisible: "Nope"', '- back', ''].join( + '\n', + ), + ); + let backCalls = 0; + + const { events, data } = await runSuite({ + root, + requestId: 'maestro-progress', + flags: { platform: 'android', replayBackend: 'maestro', retries: 1 }, + devices: [ANDROID_ONE], + invoke: async (request) => { + if (request.command === 'snapshot') return { ok: true, data: { createdAt: 0, nodes: [] } }; + if (request.command === 'runtime') { + return { ok: true, data: { viewport: { x: 0, y: 0, width: 400, height: 800 } } }; + } + if (request.command === 'back' && backCalls++ === 0) { + return { + ok: false, + error: { code: 'COMMAND_FAILED', message: 'first attempt failed' }, + }; + } + return { ok: true, data: {} }; + }, + }); + + expect(data.passed).toBe(1); + const testEvents = replayEvents(events); + const steps = testEvents.filter((event) => event.status === 'progress'); + expect(steps).toHaveLength(4); + expect(steps.map((event) => [event.attempt, event.stepIndex, event.stepCommand])).toEqual([ + [1, 1, 'assertNotVisible'], + [1, 2, 'back'], + [2, 1, 'assertNotVisible'], + [2, 2, 'back'], + ]); + expect(steps[0]).toMatchObject({ + title: 'Login flow', + stepTotal: 2, + stepValue: 'Nope', + session: 'default:test:maestro-progress:1-01-login:attempt-1', + }); + expect(steps[1]).toMatchObject({ + title: 'Login flow', + stepTotal: 2, + session: 'default:test:maestro-progress:1-01-login:attempt-1', + }); + expect(steps[1]?.stepValue).toBeUndefined(); + expect(steps[2]?.session).toBe('default:test:maestro-progress:1-01-login:attempt-2'); + expect(steps[3]?.session).toBe('default:test:maestro-progress:1-01-login:attempt-2'); + expect(testEvents.filter((event) => event.status === 'fail')[0]).toMatchObject({ + attempt: 1, + retrying: true, + hint: 'Retry with --debug and inspect diagnostics log for details.', + }); + expect(testEvents.filter((event) => event.status === 'pass')[0]).toMatchObject({ attempt: 2 }); +}); + +test('sharded suite progress carries each device and shard-scoped session', async () => { + const root = mkdtempForTestSync('agent-device-session-test-shard-progress-'); + fs.writeFileSync(path.join(root, '01-login.ad'), 'context platform=android\nopen "Demo"\n'); + + const { events } = await runSuite({ + root, + requestId: 'shard-progress', + flags: { + platform: 'android', + device: 'emulator-5554,emulator-5556', + shardAll: 2, + }, + devices: [ANDROID_ONE, ANDROID_TWO], + invoke: async () => ({ ok: true, data: { replayed: 1, healed: 0 } }), + }); + + expect(events[0]).toMatchObject({ + type: 'replay-test-suite', + status: 'start', + total: 2, + runnable: 1, + skipped: 0, + shardMode: 'all', + shardCount: 2, + }); + const starts = replayEvents(events).filter((event) => event.status === 'start'); + expect(starts.map((event) => event.session)).toEqual([ + 'default:shard-1:test:shard-progress:1-01-login:attempt-1', + 'default:shard-2:test:shard-progress:1-01-login:attempt-1', + ]); + expect(starts.map((event) => event.shardIndex)).toEqual([0, 1]); + expect(starts.map((event) => event.shardCount)).toEqual([2, 2]); + expect(starts.map((event) => event.deviceId)).toEqual(['emulator-5554', 'emulator-5556']); + expect(starts.map((event) => event.deviceName)).toEqual(['Pixel 8', 'Pixel 8 Pro']); +}); + +test('a terminal replay failure reports its message and hint without retrying', async () => { + const root = mkdtempForTestSync('agent-device-session-test-terminal-failure-'); + const testPath = path.join(root, '01-terminal-failure.ad'); + fs.writeFileSync(testPath, 'context platform=android\nopen "Demo"\n'); + let attempts = 0; + + const { events, data } = await runSuite({ + root, + requestId: 'terminal-failure', + invoke: async () => { + attempts += 1; + return { + ok: false, + error: { + code: 'ASSERTION_FAILED', + message: 'the final assertion failed', + hint: 'try replay --from 01-terminal-failure.ad', + }, + }; + }, + }); + + expect(attempts).toBe(1); + expect(data.failed).toBe(1); + const [failure] = replayEvents(events).filter((event) => event.status === 'fail'); + expect(failure).toMatchObject({ + attempt: 1, + maxAttempts: 1, + message: 'Replay failed at step 1 (open "Demo"): the final assertion failed', + hint: 'try replay --from 01-terminal-failure.ad', + }); + expect(failure?.retrying).toBeUndefined(); +}); diff --git a/packages/daemon/src/sessions/session-runtime-transport-hints.test.ts b/packages/daemon/src/sessions/session-runtime-transport-hints.test.ts new file mode 100644 index 0000000000..c57a1fe779 --- /dev/null +++ b/packages/daemon/src/sessions/session-runtime-transport-hints.test.ts @@ -0,0 +1,56 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { applyDeviceDefaultMetroHost, runtimeHintValues } from './session-runtime.ts'; + +const ANDROID_EMULATOR: DeviceInfo = { + platform: 'android', + id: 'emulator-5554', + name: 'Pixel 8', + kind: 'emulator', + booted: true, +}; + +const IOS_SIMULATOR: DeviceInfo = { + platform: 'apple', + appleOs: 'ios', + id: 'sim-1', + name: 'iPhone 16', + kind: 'simulator', + booted: true, +}; + +test.each([ + ['Android emulator', { platform: 'android', metroPort: 8084 }, ANDROID_EMULATOR, '10.0.2.2'], + ['iOS simulator', { platform: 'ios', metroPort: 8084 }, IOS_SIMULATOR, '127.0.0.1'], +] as const)( + '%s port-only runtime hints get the device loopback host', + (_label, runtime, device, host) => { + assert.deepEqual(applyDeviceDefaultMetroHost(runtime, device), { ...runtime, metroHost: host }); + }, +); + +test('physical Android port-only runtime hints stay ambiguous', () => { + const runtime = { platform: 'android', metroPort: 8084 } as const; + const device: DeviceInfo = { ...ANDROID_EMULATOR, id: 'R5CN30', kind: 'device' }; + + assert.equal(applyDeviceDefaultMetroHost(runtime, device), runtime); +}); + +test('runtime hint values serialize the configured fields and port', () => { + assert.deepEqual( + runtimeHintValues({ + platform: 'android', + metroHost: '10.0.0.10', + metroPort: 8084, + bundleUrl: 'https://10.0.0.10:8084/index.bundle', + launchUrl: 'demo://open', + }), + { + metroHost: '10.0.0.10', + metroPort: '8084', + bundleUrl: 'https://10.0.0.10:8084/index.bundle', + launchUrl: 'demo://open', + }, + ); +}); diff --git a/src/__tests__/cli-remote-diagnostics.test.ts b/src/__tests__/cli-remote-diagnostics.test.ts index 5d4d8d7eb7..933db89e9b 100644 --- a/src/__tests__/cli-remote-diagnostics.test.ts +++ b/src/__tests__/cli-remote-diagnostics.test.ts @@ -1,30 +1,20 @@ /** * #1801: a caller driving a REMOTE daemon must never be handed a diagnostics - * path on the daemon's filesystem. These run the real CLI against a real daemon - * HTTP server on loopback — the only place the whole chain (daemon error → - * locator → fetch → rendered line) is observable end to end. + * path on the daemon's filesystem. These run the real CLI against a daemon + * started through the public runtime entry, so the request record route is live. */ -import { resolveSessionRequestLogPath } from '@agent-device/daemon-contracts/session-artifact-paths'; -import type { DaemonWireRequest } from '@agent-device/contracts/command'; -import type { DaemonResponse } from '@agent-device/kernel/contracts'; import { test } from 'vitest'; import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; -import { normalizeError, AppError } from '@agent-device/kernel/errors'; -import { createDaemonHttpServer } from '@agent-device/daemon/testing/server/http-server'; -import { safeSessionName } from '@agent-device/host-kit/session-paths'; import { runCliCapture, type CapturedCliRun } from './cli-capture.ts'; import { - closeLoopbackServer, - listenOnLoopback, - mkdtempForTestSync, - skipWhenLoopbackUnavailable, -} from '@agent-device/test-kit'; + createPublicWaitCommandSurface, + startPublicDaemonRuntime, +} from './test-utils/public-daemon-runtime.ts'; +import { mkdtempForTestSync, skipWhenLoopbackUnavailable } from '@agent-device/test-kit'; -const DAEMON_TOKEN = 'daemon-secret'; -const DAEMON_SESSION = 'cwd:abcdef0123456789:default'; const RECORD_SENTINEL = 'REMOTE_RECORD_SENTINEL'; type RemoteRun = { @@ -34,78 +24,45 @@ type RemoteRun = { requestId: string; }; -/** - * Runs `argv` against a loopback daemon that always fails the command and - * writes the request's diagnostics record, exactly as the real router does. - * `serveRecord: false` models a daemon with no diagnostics route (an older - * release), so the fetch fails. - */ -async function runAgainstRemoteDaemon( - argv: string[], - options: { serveRecord: boolean }, -): Promise { +async function runAgainstRemoteDaemon(argv: string[]): Promise { const clientStateDir = mkdtempForTestSync('agent-device-remote-client-'); const daemonStateDir = mkdtempForTestSync('agent-device-remote-daemon-'); - const daemonSessionsDir = path.join(daemonStateDir, 'sessions'); - const resolveRecordPath = (session: string, requestId: string | undefined): string => - resolveSessionRequestLogPath(path.join(daemonSessionsDir, safeSessionName(session)), requestId); - let requestId = ''; - let daemonLogPath = ''; - - const handleRequest = async (req: DaemonWireRequest): Promise => { - requestId = req.meta?.requestId ?? ''; - daemonLogPath = resolveRecordPath(DAEMON_SESSION, requestId); - fs.mkdirSync(path.dirname(daemonLogPath), { recursive: true }); - fs.writeFileSync(daemonLogPath, `{"phase":"request_failed","data":"${RECORD_SENTINEL}"}\n`); - return { - ok: false, - error: normalizeError(new AppError('COMMAND_FAILED', 'wait timed out'), { - logPath: daemonLogPath, - diagnosticsRecord: { session: DAEMON_SESSION, requestId }, - }), - }; - }; - - const server = await createDaemonHttpServer({ - token: DAEMON_TOKEN, - handleRequest, - ...(options.serveRecord - ? { resolveRequestDiagnosticsPath: (ref) => resolveRecordPath(ref.session, ref.requestId) } - : {}), + const daemon = await startPublicDaemonRuntime({ + stateDir: daemonStateDir, + serverMode: 'http', + createCommandSurface: createPublicWaitCommandSurface({ fail: true }), }); try { - const port = await listenOnLoopback(server); const run = await runCliCapture(argv, { useRealDaemonClient: true, env: { AGENT_DEVICE_STATE_DIR: clientStateDir, - AGENT_DEVICE_DAEMON_BASE_URL: `http://127.0.0.1:${port}`, - AGENT_DEVICE_DAEMON_AUTH_TOKEN: DAEMON_TOKEN, + AGENT_DEVICE_DAEMON_BASE_URL: daemon.baseUrl, + AGENT_DEVICE_DAEMON_AUTH_TOKEN: daemon.runtime.token, }, }); + const daemonLogPath = findRequestLog(daemonStateDir); + const requestId = path.basename(daemonLogPath, '.ndjson'); return { run, clientStateDir, daemonLogPath, requestId }; } finally { - await closeLoopbackServer(server); - fs.rmSync(daemonStateDir, { recursive: true, force: true }); + await daemon.close(); } } test('a remote failure names a record the caller can actually read', async (t) => { if (await skipWhenLoopbackUnavailable(t)) return; - const { run, clientStateDir, daemonLogPath, requestId } = await runAgainstRemoteDaemon( - ['clipboard', 'write', 'hello'], - { serveRecord: true }, - ); + const { run, clientStateDir, daemonLogPath, requestId } = await runAgainstRemoteDaemon([ + 'wait', + '5000', + '--platform', + 'macos', + '--target', + 'desktop', + ]); try { - const localPath = path.join( - clientStateDir, - 'remote-diagnostics', - safeSessionName(DAEMON_SESSION), - 'requests', - `${requestId}.ndjson`, - ); + const localPath = localRecordPath(clientStateDir, daemonLogPath, requestId); assert.equal(run.code, 1); assert.match(run.stderr, new RegExp(`Diagnostics Log: ${escapeRegExp(localPath)}`)); assert.equal(run.stderr.includes(daemonLogPath), false, 'daemon-host path must not be printed'); @@ -118,10 +75,15 @@ test('a remote failure names a record the caller can actually read', async (t) = test('--json carries the caller-local record path, never the daemon-host one', async (t) => { if (await skipWhenLoopbackUnavailable(t)) return; - const { run, clientStateDir, daemonLogPath } = await runAgainstRemoteDaemon( - ['clipboard', 'write', 'hello', '--json'], - { serveRecord: true }, - ); + const { run, clientStateDir, daemonLogPath } = await runAgainstRemoteDaemon([ + 'wait', + '5000', + '--platform', + 'macos', + '--target', + 'desktop', + '--json', + ]); try { const payload = JSON.parse(run.stdout) as { error: { logPath?: string } }; assert.equal(payload.error.logPath?.startsWith(clientStateDir), true); @@ -135,10 +97,15 @@ test('--json carries the caller-local record path, never the daemon-host one', a test('--debug prints the fetched record inline and not the local daemon log', async (t) => { if (await skipWhenLoopbackUnavailable(t)) return; - const { run, clientStateDir } = await runAgainstRemoteDaemon( - ['clipboard', 'write', 'hello', '--debug'], - { serveRecord: true }, - ); + const { run, clientStateDir } = await runAgainstRemoteDaemon([ + 'wait', + '5000', + '--platform', + 'macos', + '--target', + 'desktop', + '--debug', + ]); try { // A local daemon log in the caller's state dir must stay unread: it belongs // to a different daemon than the one that failed. @@ -150,31 +117,30 @@ test('--debug prints the fetched record inline and not the local daemon log', as } }); -test('an unfetchable remote record says so instead of naming a path', async (t) => { - if (await skipWhenLoopbackUnavailable(t)) return; +function findRequestLog(stateDir: string): string { + const sessionsDir = path.join(stateDir, 'sessions'); + const sessionDirs = fs + .readdirSync(sessionsDir, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()); + const requestLogs = sessionDirs.flatMap((session) => { + const requestsDir = path.join(sessionsDir, session.name, 'requests'); + if (!fs.existsSync(requestsDir)) return []; + return fs.readdirSync(requestsDir).map((name) => path.join(requestsDir, name)); + }); + assert.equal(requestLogs.length, 1); + return requestLogs[0]!; +} - const { run, clientStateDir, daemonLogPath, requestId } = await runAgainstRemoteDaemon( - ['clipboard', 'write', 'hello', '--json'], - { serveRecord: false }, +function localRecordPath(clientStateDir: string, daemonLogPath: string, requestId: string): string { + const sessionDir = path.basename(path.dirname(path.dirname(daemonLogPath))); + return path.join( + clientStateDir, + 'remote-diagnostics', + sessionDir, + 'requests', + `${requestId}.ndjson`, ); - try { - const payload = JSON.parse(run.stdout) as { - error: { logPath?: string; logPathUnavailable?: string }; - }; - // A path may still be named — the CLI's own client-side record, written on - // this machine — but never the daemon host's, and never a record the fetch - // failed to produce. - assert.notEqual(payload.error.logPath, daemonLogPath); - assert.equal(payload.error.logPath?.includes('remote-diagnostics') ?? false, false); - assert.match( - payload.error.logPathUnavailable ?? '', - new RegExp(`^remote daemon http://127\\.0\\.0\\.1:\\d+, request ${requestId}: HTTP 404$`), - ); - assert.equal(run.stdout.includes(daemonLogPath), false); - } finally { - fs.rmSync(clientStateDir, { recursive: true, force: true }); - } -}); +} function escapeRegExp(value: string): string { return value.replaceAll(/[.*+?^${}()|[\]\\]/g, String.raw`\$&`); diff --git a/src/__tests__/lease-provider-profile-fields.test.ts b/src/__tests__/lease-provider-profile-fields.test.ts index 5a3c5090e1..a756af629b 100644 --- a/src/__tests__/lease-provider-profile-fields.test.ts +++ b/src/__tests__/lease-provider-profile-fields.test.ts @@ -2,25 +2,15 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; import { afterEach, test, vi } from 'vitest'; -import { AppError, normalizeError } from '@agent-device/kernel/errors'; -import { - errorResponse, - type DaemonRequest, - type DaemonResponse, -} from '@agent-device/kernel/contracts'; -import { LimrunRuntime } from '../sdk/limrun.ts'; +import { AppError } from '@agent-device/kernel/errors'; import { createAgentDeviceClient } from '../agent-device-client.ts'; -import { handleLeaseCommands } from '@agent-device/daemon/testing/handlers/lease'; -import { LeaseRegistry } from '@agent-device/daemon/testing/lease-registry'; -import { createProviderDeviceRuntimeRequestProviders } from '../provider-device-runtime.ts'; import { hashRemoteConfigFile, writeRemoteConnectionState, } from '@agent-device/remote/remote-connection-state'; -import { createTransport } from './client-transport-fixture.ts'; import { runCliCapture } from './cli-capture.ts'; import { makeTempWorkspace } from './cli-config-fixtures.ts'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; +import { startPublicDaemonRuntime } from './test-utils/public-daemon-runtime.ts'; const limrunInstances = vi.hoisted(() => ({ iosCreate: vi.fn(async () => { @@ -38,50 +28,35 @@ afterEach(() => { vi.clearAllMocks(); }); -// The daemon's lease handler over the same provider composition a daemon builds, so a request -// reaches the Limrun runtime exactly as it does in production. -function limrunDaemon(): (req: Omit) => Promise { - const runtime = new LimrunRuntime({ apiKey: 'lim_test_key' }); - const providers = createProviderDeviceRuntimeRequestProviders([runtime]); - const leaseRegistry = new LeaseRegistry(); - const sessionStore = makeSessionStore('agent-device-limrun-profile-fields-'); - return async (req) => { - try { - return (await handleLeaseCommands({ - req: { ...req, token: 'test-token' } as Parameters[0]['req'], - sessionName: req.session ?? 'default', - sessionStore, - leaseRegistry, - providerRuntimeIds: providers.providerRuntimeIds, - providerRuntimeRequiredIds: providers.providerRuntimeRequiredIds, - leaseLifecycleProvider: providers.leaseLifecycleProvider, - })) as DaemonResponse; - } catch (error) { - const normalized = normalizeError(error); - return errorResponse(normalized.code, normalized.message, normalized.details); - } - }; -} - test('leases.allocate refuses an OS version Limrun cannot honour instead of ignoring it', async () => { - const setup = createTransport(limrunDaemon()); - const client = createAgentDeviceClient(setup.config, { transport: setup.transport }); - - await assert.rejects( - client.leases.allocate({ - tenant: 'limrun', - runId: 'run-os-version', - leaseBackend: 'ios-instance', - leaseProvider: 'limrun', - platform: 'ios', - providerOsVersion: '18.0', - }), - (error: unknown) => - error instanceof AppError && - error.code === 'INVALID_ARGS' && - JSON.stringify(error.details?.flags) === '["--provider-os-version"]', - ); - assert.equal(limrunInstances.iosCreate.mock.calls.length, 0); + const daemon = await startPublicDaemonRuntime({ + serverMode: 'http', + env: { LIMRUN_API_KEY: 'lim_test_key' }, + }); + try { + const client = createAgentDeviceClient({ + daemonBaseUrl: daemon.baseUrl, + daemonAuthToken: daemon.runtime.token, + daemonTransport: 'http', + }); + await assert.rejects( + client.leases.allocate({ + tenant: 'limrun', + runId: 'run-os-version', + leaseBackend: 'ios-instance', + leaseProvider: 'limrun', + platform: 'ios', + providerOsVersion: '18.0', + }), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + JSON.stringify(error.details?.flags) === '["--provider-os-version"]', + ); + assert.equal(limrunInstances.iosCreate.mock.calls.length, 0); + } finally { + await daemon.close(); + } }); test('a remote-config profile cannot carry a field Limrun refuses past lease allocation', async () => { @@ -106,23 +81,29 @@ test('a remote-config profile cannot carry a field Limrun refuses past lease all updatedAt: now, }, }); - const daemon = limrunDaemon(); + const daemon = await startPublicDaemonRuntime({ + serverMode: 'http', + env: { LIMRUN_API_KEY: 'lim_test_key' }, + }); try { const result = await runCliCapture(['open', '--state-dir', stateDir, '--json'], { cwd: project, - env: { HOME: home }, - sendToDaemon: async (req) => await daemon(req), + env: { + HOME: home, + AGENT_DEVICE_DAEMON_BASE_URL: daemon.baseUrl, + AGENT_DEVICE_DAEMON_AUTH_TOKEN: daemon.runtime.token, + }, + useRealDaemonClient: true, }); assert.equal(result.code, 1); - assert.equal(result.calls[0]?.command, 'lease_allocate'); - assert.equal(result.calls[0]?.flags?.providerGeoLocation, 'US'); const payload = JSON.parse(result.stdout); assert.equal(payload.error.code, 'INVALID_ARGS'); assert.deepEqual(payload.error.details.flags, ['--provider-geo-location']); assert.equal(limrunInstances.iosCreate.mock.calls.length, 0); } finally { + await daemon.close(); fs.rmSync(root, { recursive: true, force: true }); } }); diff --git a/src/__tests__/limrun-runtime.test.ts b/src/__tests__/limrun-runtime.test.ts index 710ceee496..dec506e77a 100644 --- a/src/__tests__/limrun-runtime.test.ts +++ b/src/__tests__/limrun-runtime.test.ts @@ -6,7 +6,6 @@ import type { ClientOptions } from '@limrun/api'; import { createInstanceClient as createAndroidInstanceClient } from '@limrun/api/instance-client'; import { createInstanceClient as createIosInstanceClient } from '@limrun/api/ios-client'; import { LimrunRuntime } from '../sdk/limrun.ts'; -import { createExpiredProviderLeaseReleaser } from '@agent-device/daemon/testing/provider-lease-expiry'; import type { DeviceLease } from '@agent-device/contracts/device'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; @@ -261,61 +260,6 @@ test('Limrun reclaims a labeled iOS instance without an in-memory session', asyn } }); -test('Limrun recovers a failed expired lease release after a daemon restart', async () => { - const stateDir = mkdtempForTestSync('agent-device-limrun-expiry-'); - const lease: DeviceLease = { - leaseId: 'lease-recovered-after-restart', - tenantId: 'team-a', - runId: 'run-a', - backend: 'ios-instance', - leaseProvider: 'limrun', - createdAt: 1, - heartbeatAt: 1, - expiresAt: 60_001, - }; - const firstRuntime = new LimrunRuntime({ apiKey: 'lim_test_key' }); - const firstReleaser = createExpiredProviderLeaseReleaser({ - recoverExpiredLease: firstRuntime.recoverExpiredLease, - recoverableProviderIds: ['limrun'], - stateDir, - }); - let recoveredRuntime: LimrunRuntime | undefined; - let recoveredReleaser: ReturnType | undefined; - - try { - const allocateLease = firstRuntime.leaseLifecycle.allocate; - if (!allocateLease) throw new Error('Limrun runtime must provide lease allocation'); - await allocateLease(lease); - limrunMockState.iosDelete.mockRejectedValueOnce(new Error('temporary provider outage')); - - await firstReleaser.release(lease); - assert.deepEqual(limrunMockState.iosDelete.mock.calls[0], ['ios-instance-1']); - firstReleaser.shutdown(); - - recoveredRuntime = new LimrunRuntime({ apiKey: 'lim_test_key' }); - recoveredReleaser = createExpiredProviderLeaseReleaser({ - recoverExpiredLease: recoveredRuntime.recoverExpiredLease, - recoverableProviderIds: ['limrun'], - stateDir, - }); - limrunMockState.iosList.mockResolvedValueOnce({ - getPaginatedItems: () => [{ metadata: { id: 'ios-instance-recovered' } }], - }); - await recoveredReleaser.retryPending(); - - assert.deepEqual(limrunMockState.iosList.mock.calls[0], [ - { labelSelector: 'provider=limrun,leaseId=lease-recovered-after-restart' }, - ]); - assert.deepEqual(limrunMockState.iosDelete.mock.calls.at(-1), ['ios-instance-recovered']); - } finally { - // A crashed daemon cannot run the original runtime's graceful shutdown. - firstReleaser.shutdown(); - recoveredReleaser?.shutdown(); - await recoveredRuntime?.shutdown(); - fs.rmSync(stateDir, { recursive: true, force: true }); - } -}); - test('Limrun Android reverses localhost URL ports through the persistent ADB tunnel', async () => { const runtime = new LimrunRuntime({ apiKey: 'lim_test_key', diff --git a/src/__tests__/platform-runtime-runtime-hints.test.ts b/src/__tests__/platform-runtime-runtime-hints.test.ts index eef4bc3083..8ba78c701f 100644 --- a/src/__tests__/platform-runtime-runtime-hints.test.ts +++ b/src/__tests__/platform-runtime-runtime-hints.test.ts @@ -6,10 +6,6 @@ import { applyRuntimeHintValues, clearRuntimeHintValues, } from '../platform-runtime-runtime-hints.ts'; -import { - applyDeviceDefaultMetroHost, - runtimeHintValues, -} from '@agent-device/daemon/testing/session-runtime'; import { resolveRuntimeTransportHints } from '@agent-device/host-kit/runtime-transport-hints'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { @@ -131,16 +127,13 @@ test('applyRuntimeHintValues also writes the legacy ReactNativeDevPrefs.xml path }); }); -test('port-only hint on an Android emulator defaults host to 10.0.2.2 and writes the dev-server pref', async () => { +test('Android runtime hint application writes the supplied emulator loopback host', async () => { await withMockedAdb(async ({ device, readWrittenPrefsFile }) => { const packageName = 'com.example.demo'; - const runtime = applyDeviceDefaultMetroHost({ platform: 'android', metroPort: 8084 }, device); - assert.equal(runtime?.metroHost, '10.0.2.2'); - await applyRuntimeHintValues({ device, appId: packageName, - values: runtimeHintValues(runtime), + values: { metroHost: '10.0.2.2', metroPort: '8084' }, }); const defaultPayload = await readWrittenPrefsFile(defaultPrefsPath(packageName)); @@ -170,19 +163,13 @@ test('Android runtime hints escape XML text and quoted attribute delimiters', as }); }); -test('port-only hint on a physical Android device stays ambiguous and writes nothing', async () => { +test('a port-only Android runtime hint writes nothing without a host', async () => { await withMockedAdb(async ({ device, argsLogPath, readWrittenPrefsFile }) => { const physicalDevice: DeviceInfo = { ...device, id: 'R5CN30', kind: 'device' }; - const runtime = applyDeviceDefaultMetroHost( - { platform: 'android', metroPort: 8084 }, - physicalDevice, - ); - assert.equal(runtime?.metroHost, undefined); - await applyRuntimeHintValues({ device: physicalDevice, appId: 'com.example.demo', - values: runtimeHintValues(runtime), + values: { metroPort: '8084' }, }); const loggedArgs = await fs.readFile(argsLogPath, 'utf8').catch(() => ''); @@ -191,15 +178,12 @@ test('port-only hint on a physical Android device stays ambiguous and writes not }); }); -test('port-only hint on an iOS simulator defaults host to 127.0.0.1', async () => { +test('iOS runtime hint application writes the supplied simulator loopback host', async () => { await withMockedXcrun(async ({ device, argsLogPath }) => { - const runtime = applyDeviceDefaultMetroHost({ platform: 'ios', metroPort: 8084 }, device); - assert.equal(runtime?.metroHost, '127.0.0.1'); - await applyRuntimeHintValues({ device, appId: 'com.example.demo', - values: runtimeHintValues(runtime), + values: { metroHost: '127.0.0.1', metroPort: '8084' }, }); const loggedArgs = await fs.readFile(argsLogPath, 'utf8'); diff --git a/src/__tests__/remote-connection-harmonyos.test.ts b/src/__tests__/remote-connection-harmonyos.test.ts index 989820d1f5..776593a379 100644 --- a/src/__tests__/remote-connection-harmonyos.test.ts +++ b/src/__tests__/remote-connection-harmonyos.test.ts @@ -12,7 +12,6 @@ import { } from '../cli/commands/connection-runtime.ts'; import { disconnectCommand } from '../cli/commands/connection.ts'; import { readRemoteConnectionState } from '@agent-device/remote/remote-connection-state'; -import { LeaseRegistry } from '@agent-device/daemon/testing/lease-registry'; test('HarmonyOS platform resolves to its proxy lease backend', () => { expect( @@ -130,8 +129,10 @@ test('proxy HarmonyOS inventory materializes a scoped lease and closes that same clientId: 'harmony-client', }, }); - const registry = new LeaseRegistry(); - const allocated: string[] = []; + const leaseId = 'harmony-proxy-lease'; + let allocationRequest: + | Parameters['leases']['allocate']>[0] + | undefined; const released: string[] = []; const client = createTestClient({ listDevices: async () => [ @@ -147,18 +148,20 @@ test('proxy HarmonyOS inventory materializes a scoped lease and closes that same }, ], allocate: async (request) => { - expect(request.leaseBackend).toBe('harmonyos-instance'); - expect(request.deviceKey).toBe('harmonyos:mobile:127.0.0.1:16001'); - const lease = registry.allocateLease({ ...request, tenantId: request.tenant }); - allocated.push(lease.leaseId); - return lease; + allocationRequest = request; + return { + leaseId, + tenantId: request.tenant, + runId: request.runId, + backend: request.leaseBackend ?? 'harmonyos-instance', + leaseProvider: request.leaseProvider, + clientId: request.clientId, + deviceKey: request.deviceKey, + }; }, - heartbeat: async (request) => registry.heartbeatLease({ ...request, tenantId: request.tenant }), release: async (request) => { - const result = registry.releaseLease({ ...request, tenantId: request.tenant }); - expect(result.released).toBe(true); released.push(request.leaseId); - return result; + return { released: request.leaseId === leaseId }; }, }); const flags = { @@ -177,19 +180,13 @@ test('proxy HarmonyOS inventory materializes a scoped lease and closes that same expect(materialized.flags.platform).toBe('harmonyos'); expect(materialized.flags.serial).toBe('127.0.0.1:16001'); expect(materialized.flags.leaseBackend).toBe('harmonyos-instance'); - expect(materialized.flags.leaseId).toBe(allocated[0]); - expect(readRemoteConnectionState({ stateDir, session: flags.session })?.leaseId).toBe( - allocated[0], - ); - expect(() => - registry.assertLeaseAdmission({ - leaseId: allocated[0], - tenantId: 'proxy', - runId: 'harmony-run', - leaseBackend: 'android-instance', - }), - ).toThrow(); + expect(materialized.flags.leaseId).toBe(leaseId); + expect(allocationRequest).toMatchObject({ + leaseBackend: 'harmonyos-instance', + deviceKey: 'harmonyos:mobile:127.0.0.1:16001', + }); + expect(readRemoteConnectionState({ stateDir, session: flags.session })?.leaseId).toBe(leaseId); await disconnectCommand({ positionals: [], flags, client }); - expect(released).toEqual(allocated); + expect(released).toEqual([leaseId]); expect(readRemoteConnectionState({ stateDir, session: flags.session })).toBeNull(); }); diff --git a/src/__tests__/remote-connection-macos-app.test.ts b/src/__tests__/remote-connection-macos-app.test.ts index dfa5f22d69..994f615e48 100644 --- a/src/__tests__/remote-connection-macos-app.test.ts +++ b/src/__tests__/remote-connection-macos-app.test.ts @@ -7,7 +7,6 @@ import { connectCommand, disconnectCommand } from '../cli/commands/connection.ts import { materializeRemoteConnectionForCommand } from '../cli/commands/connection-runtime.ts'; import { beginStdoutCapture } from './test-utils/stream-capture.ts'; import { readRemoteConnectionState } from '@agent-device/remote/remote-connection-state'; -import { LeaseRegistry } from '@agent-device/daemon/testing/lease-registry'; const LEASE_ID = 'a1b2c3d4e5f60718293a4b5c6d7e8f90'; @@ -43,15 +42,7 @@ function flagsFor(stateDir: string, remoteConfig: string) { test('a macos-app remote config opens on the lease the host allocated, unchanged', async () => { const { stateDir, remoteConfigPath } = connectionWorkspace('macos-app-open-'); writeHostIssuedConfig(remoteConfigPath); - const registry = new LeaseRegistry(); - registry.putHostLease(LEASE_ID, { - tenantId: 'stim', - runId: 'session-1', - clientId: 'client-1', - leaseBackend: 'macos-app', - leaseProvider: 'proxy', - deviceKey: 'com.example.app@4242', - }); + const heartbeatRequests: Parameters[0][] = []; const client = createTestClient({ listDevices: async () => { throw new Error('a macos-app connection resolves no device into a new key'); @@ -59,7 +50,18 @@ test('a macos-app remote config opens on the lease the host allocated, unchanged allocate: async () => { throw new Error('a macos-app connection never allocates'); }, - heartbeat: async (request) => registry.heartbeatLease({ ...request, tenantId: request.tenant }), + heartbeat: async (request) => { + heartbeatRequests.push(request); + return { + leaseId: request.leaseId, + tenantId: request.tenant ?? 'stim', + runId: request.runId ?? 'session-1', + backend: request.leaseBackend ?? 'macos-app', + leaseProvider: request.leaseProvider, + clientId: request.clientId, + deviceKey: request.deviceKey, + }; + }, }); const flags = flagsFor(stateDir, remoteConfigPath); const materialized = await materializeRemoteConnectionForCommand({ @@ -74,6 +76,16 @@ test('a macos-app remote config opens on the lease the host allocated, unchanged platform: 'macos', }); expect(materialized.connection).toMatchObject({ deviceKey: 'com.example.app@4242' }); + expect(heartbeatRequests).toContainEqual( + expect.objectContaining({ + leaseId: LEASE_ID, + tenant: 'stim', + runId: 'session-1', + leaseBackend: 'macos-app', + leaseProvider: 'proxy', + deviceKey: 'com.example.app@4242', + }), + ); expect(readRemoteConnectionState({ stateDir, session: 'hosted' })).toMatchObject({ leaseId: LEASE_ID, deviceKey: 'com.example.app@4242', diff --git a/src/__tests__/takeover-command.test.ts b/src/__tests__/takeover-command.test.ts index 717442517f..75f53163ae 100644 --- a/src/__tests__/takeover-command.test.ts +++ b/src/__tests__/takeover-command.test.ts @@ -6,10 +6,7 @@ import { renderTakeoverStatus, takeoverCommand, } from '../cli/commands/takeover.ts'; -import { - HUMAN_CONTROL_HOLD, - createControlLatch, -} from '@agent-device/daemon/testing/__tests__/human-control-fixtures'; +import { HUMAN_CONTROL_HOLD } from '../client/human-control.fixtures.ts'; import type { DaemonRequest } from '@agent-device/kernel/contracts'; import { TAKEOVER_CLI_FLAGS } from './test-utils/client-lease-fixtures.ts'; @@ -18,6 +15,14 @@ vi.mock('../cli/commands/shared.ts', () => ({ writeCommandOutput: mocks.writeCom beforeEach(() => mocks.writeCommandOutput.mockReset()); afterEach(() => vi.useRealTimers()); +function createControlLatch() { + let resolve: () => void = () => {}; + const promise = new Promise((onResolve) => { + resolve = onResolve; + }); + return { promise, resolve }; +} + function clientForTest() { const transport = vi.fn(async (req: Omit) => ({ ok: true as const, diff --git a/src/__tests__/test-utils/public-daemon-runtime.ts b/src/__tests__/test-utils/public-daemon-runtime.ts new file mode 100644 index 0000000000..d00fca1a05 --- /dev/null +++ b/src/__tests__/test-utils/public-daemon-runtime.ts @@ -0,0 +1,125 @@ +import type { + CreateDaemonCommandSurface, + DaemonCommandSurface, + DaemonWaitResult, + DaemonWaitTarget, +} from '@agent-device/contracts/daemon-command-surface'; +import type { AgentDeviceRuntimeConfig } from '@agent-device/contracts/runtime-contract'; +import { AppError } from '@agent-device/kernel/errors'; +import { startDaemonRuntime, type DaemonRuntimeController } from '@agent-device/daemon'; +import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; +import { mkdtempForTestSync } from '@agent-device/test-kit'; +import fs from 'node:fs'; +import { createDaemonRuntimeDeps } from '../../daemon-composition.ts'; +import { currentDaemonCodeSignature } from './daemon-http-fixture.ts'; + +export type PublicDaemonRuntimeFixture = Readonly<{ + stateDir: string; + baseUrl: string; + runtime: DaemonRuntimeController; + close: () => Promise; +}>; + +export async function startPublicDaemonRuntime( + options: { + env?: NodeJS.ProcessEnv; + stateDir?: string; + serverMode?: 'http' | 'socket' | 'dual'; + createCommandSurface?: CreateDaemonCommandSurface; + } = {}, +): Promise { + const ownsStateDir = options.stateDir === undefined; + const stateDir = options.stateDir ?? mkdtempForTestSync('agent-device-public-daemon-'); + const runtime = await startDaemonRuntime({ + composeDeps: async (input) => { + const deps = await createDaemonRuntimeDeps(input); + return options.createCommandSurface + ? { + ...deps, + requestRouter: { + ...deps.requestRouter, + createCommandSurface: options.createCommandSurface, + }, + } + : deps; + }, + env: { + ...process.env, + ...options.env, + AGENT_DEVICE_STATE_DIR: stateDir, + AGENT_DEVICE_DAEMON_SERVER_MODE: options.serverMode ?? 'dual', + AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', + }, + exit: () => {}, + registerProcessHandlers: false, + stderr: { write: () => {} }, + stdout: { write: () => {} }, + }); + if (!runtime) throw new Error('The public daemon runtime did not start.'); + try { + const paths = resolveDaemonPaths(stateDir); + const info = JSON.parse(fs.readFileSync(paths.infoPath, 'utf8')) as Record; + fs.writeFileSync( + paths.infoPath, + `${JSON.stringify({ ...info, codeSignature: currentDaemonCodeSignature() })}\n`, + ); + const port = runtime.httpPort ?? runtime.socketPort; + if (port === undefined) { + throw new Error('The public daemon runtime published no transport port.'); + } + return { + stateDir, + baseUrl: `http://127.0.0.1:${port}`, + runtime, + close: async () => { + await runtime.shutdown(); + fs.rmSync(stateDir, { recursive: true, force: true }); + }, + }; + } catch (error) { + await runtime.shutdown().catch(() => {}); + if (ownsStateDir) fs.rmSync(stateDir, { recursive: true, force: true }); + throw error; + } +} + +export function createPublicWaitCommandSurface( + options: { + onStarted?: (target: DaemonWaitTarget, requestId?: string) => void; + onCanceled?: (target: DaemonWaitTarget, requestId?: string) => void; + fail?: boolean; + } = {}, +): CreateDaemonCommandSurface { + return async (config: AgentDeviceRuntimeConfig): Promise => ({ + signal: config.signal, + capture: {} as DaemonCommandSurface['capture'], + selectors: { + wait: async ({ target, requestId }): Promise => { + if (options.fail) { + throw new AppError('COMMAND_FAILED', 'REMOTE_RECORD_SENTINEL'); + } + if (target.kind === 'sleep' && target.durationMs <= 1) { + return { kind: 'sleep', waitedMs: target.durationMs }; + } + options.onStarted?.(target, requestId); + const signal = config.signal; + if (!signal) throw new Error('The daemon did not inject the request signal.'); + if (signal.aborted) { + options.onCanceled?.(target, requestId); + throw signal.reason; + } + return await new Promise((_resolve, reject) => { + signal.addEventListener( + 'abort', + () => { + options.onCanceled?.(target, requestId); + reject(signal.reason); + }, + { once: true }, + ); + }); + }, + } as DaemonCommandSurface['selectors'], + interactions: {} as DaemonCommandSurface['interactions'], + }); +} diff --git a/src/cli/replay-test/__tests__/session-test-reporter-values-maestro.test.ts b/src/cli/replay-test/__tests__/session-test-reporter-values-maestro.test.ts deleted file mode 100644 index 6d5e8e5413..0000000000 --- a/src/cli/replay-test/__tests__/session-test-reporter-values-maestro.test.ts +++ /dev/null @@ -1,286 +0,0 @@ -import type { DaemonWireRequest } from '@agent-device/contracts/command'; -import type { DaemonResponse } from '@agent-device/kernel/contracts'; -import { daemonPlatformServicesFixture } from '@agent-device/daemon/testing/__tests__/platform-services-fixture'; -// Characterization of the shipped reporter contract for the MAESTRO adapter (#1478 P3). -// -// The sibling session-test-reporter-values.test.ts covers the same contract for native `.ad`. -// Both are needed, and neither substitutes for the other: P3 replaced a request-global -// AsyncLocalStorage with an explicit per-attempt `onStep` port, and that port has two -// independent forwarding chains. The native chain is -// scheduler sink -> runReplayCommand -> executeReplayActions -> onStep?.(...) -// and the Maestro chain is a different set of links entirely: -// scheduler sink -> runReplayCommand -> runTypedMaestroReplay -// -> createMaestroReplayObserver({ onStep }) -> actionStarted -> onStep?.(...) -// -// An `.ad`-only reporter test stays green if any Maestro link is deleted, so `onTestStep` -// would silently stop firing for every `test --maestro` run with no gate objecting. That is -// the same defect class as the dropped diagnosticId/logPath (#1501) and the dropped reporter -// `hint` (#1505): a ratchet that keeps passing while a shipped value disappears. -// -// These tests document today's behavior; they do not propose behavior. - -// Maestro replay resolves a target device through core/dispatch. Gesture viewport reads re-enter -// the admitted internal runtime command; the invoke fixture below returns its typed viewport. -import { expect, test, vi } from 'vitest'; -import { testCreateCommandSurface } from '../../../__tests__/daemon-composition/command-surface-fixture.ts'; - -vi.mock('@agent-device/device-selection/dispatch-resolve', async (importOriginal) => { - const actual = - await importOriginal(); - return { - ...actual, - resolveTargetDevice: vi.fn(async () => ({ - platform: 'android', - id: 'emulator-5554', - name: 'Pixel 8', - kind: 'emulator', - booted: true, - })), - }; -}); - -import fs from 'node:fs'; - -import path from 'node:path'; -import type { ReplaySuiteResult } from '@agent-device/contracts/replay'; -import { handleSessionCommands } from '@agent-device/daemon/testing/handlers/__tests__/session-command-harness'; -import { SessionStore } from '@agent-device/daemon/testing/session-store'; -import { withRequestProgressSink } from '@agent-device/host-kit/request'; -import { - getReplayTestReporterExitCode, - runReplayTestReporterProgress, - runReplayTestReporters, -} from '../reporters/registry.ts'; -import type { ReplayTestReporter, ReplayTestReporterContext } from '../reporters/types.ts'; -import { mkdtempForTestSync } from '@agent-device/test-kit'; - -type RecordedHook = { - hook: 'onSuiteStart' | 'onTestStart' | 'onTestStep' | 'onTestResult' | 'onSuiteEnd'; - value: Record; -}; - -const reporterContext: ReplayTestReporterContext = { - stdout: { isTTY: false, write() {} }, - stderr: { isTTY: false, write() {} }, -}; - -/** - * Two steps, so `stepIndex`/`stepTotal` are distinguishable and one step carries a value while - * the other does not. The flow `name` is what populates the reporter `title` — a value only the - * Maestro path can produce, since `.ad` scripts have no title. - */ -const MAESTRO_FLOW = [ - 'appId: demo.app', - 'name: Login flow', - '---', - '- assertNotVisible: "Nope"', - '- back', - '', -].join('\n'); - -function createRecordingReporter(): { reporter: ReplayTestReporter; hooks: RecordedHook[] } { - const hooks: RecordedHook[] = []; - return { - hooks, - reporter: { - name: 'recording', - onSuiteStart: (suite) => void hooks.push({ hook: 'onSuiteStart', value: { ...suite } }), - onTestStart: (testCase) => void hooks.push({ hook: 'onTestStart', value: { ...testCase } }), - onTestStep: (testCase) => void hooks.push({ hook: 'onTestStep', value: { ...testCase } }), - onTestResult: (testCase) => void hooks.push({ hook: 'onTestResult', value: { ...testCase } }), - onSuiteEnd: (suite) => void hooks.push({ hook: 'onSuiteEnd', value: { ...suite } }), - getExitCode: (suite) => (suite.failed > 0 ? 1 : 0), - }, - }; -} - -function makeSessionStore(root: string): SessionStore { - return new SessionStore(path.join(root, 'sessions')); -} - -/** - * Runs `test --maestro` exactly as the daemon does, feeding every progress event through the - * shipped reporter registry — the same translation and dispatch a `--reporter ./mine.mjs` - * module gets. - */ -async function runMaestroSuiteThroughReporter(params: { - root: string; - requestId: string; - flags?: DaemonWireRequest['flags']; - invoke: (req: DaemonWireRequest) => Promise; -}): Promise<{ hooks: RecordedHook[]; suite: ReplaySuiteResult; exitCode: number }> { - const { reporter, hooks } = createRecordingReporter(); - const reporters = [reporter]; - const response = await withRequestProgressSink( - (event) => runReplayTestReporterProgress(reporters, event, reporterContext), - async () => - await handleSessionCommands({ - createCommandSurface: testCreateCommandSurface, - platformServices: daemonPlatformServicesFixture(), - req: { - token: 't', - session: 'default', - command: 'test', - positionals: [params.root], - meta: { cwd: params.root, requestId: params.requestId }, - flags: { replayBackend: 'maestro', platform: 'android', ...params.flags }, - }, - sessionName: 'default', - logPath: path.join(mkdtempForTestSync('daemon'), 'daemon.log'), - sessionStore: makeSessionStore(params.root), - invoke: params.invoke, - }), - ); - - expect(response?.ok, JSON.stringify(response)).toBeTruthy(); - if (!response?.ok) throw new Error('Expected a successful suite response.'); - const suite = response.data as unknown as ReplaySuiteResult; - await runReplayTestReporters(reporters, suite, reporterContext); - return { hooks, suite, exitCode: getReplayTestReporterExitCode(reporters, suite) }; -} - -/** `snapshot` returns an empty hierarchy; every other command succeeds. */ -function maestroInvoke(req: DaemonWireRequest): DaemonResponse { - if (req.command === 'snapshot') return { ok: true, data: { createdAt: 0, nodes: [] } }; - if (req.command === 'runtime') { - return { ok: true, data: { viewport: { x: 0, y: 0, width: 400, height: 800 } } }; - } - return { ok: true, data: {} }; -} - -test('a Maestro suite reaches the reporter with step payloads and attempt-scoped identity', async () => { - const root = mkdtempForTestSync('agent-device-reporter-maestro-'); - fs.writeFileSync(path.join(root, '01-login.yaml'), MAESTRO_FLOW); - - const { hooks, suite, exitCode } = await runMaestroSuiteThroughReporter({ - root, - requestId: 'suite-maestro', - invoke: async (req) => maestroInvoke(req), - }); - - expect(hooks.map((entry) => entry.hook)).toEqual([ - 'onSuiteStart', - 'onTestStart', - 'onTestStep', - 'onTestStep', - 'onTestResult', - 'onSuiteEnd', - ]); - - const suiteArtifactsDir = path.join(root, '.agent-device', 'test-artifacts', 'suite-maestro'); - const testArtifactsDir = path.join(suiteArtifactsDir, '01-login.yaml'); - const session = 'default:test:suite-maestro:1-01-login:attempt-1'; - - // The Maestro flow's `name` becomes the reporter `title`. Discovery reads it through - // `inspectMaestroFlow`, so this is a shipped reporter value the native scenario cannot cover. - expect(hooks[1]?.value).toMatchObject({ - file: path.join(root, '01-login.yaml'), - title: 'Login flow', - index: 1, - total: 1, - attempt: undefined, - maxAttempts: 1, - session, - artifactsDir: testArtifactsDir, - }); - - // The two step events: payload comes from the Maestro engine through the observer's - // `onStep`, while file/index/total/attempt/session/artifactsDir come from the scheduler's - // per-attempt context. Before P3 the latter half came from request-global AsyncLocalStorage. - const steps = hooks.filter((entry) => entry.hook === 'onTestStep').map((entry) => entry.value); - expect(steps).toHaveLength(2); - expect('status' in steps[0]!).toBe(false); - expect(steps[0]).toMatchObject({ - file: path.join(root, '01-login.yaml'), - title: 'Login flow', - index: 1, - total: 1, - attempt: 1, - maxAttempts: 1, - session, - artifactsDir: testArtifactsDir, - stepIndex: 1, - stepTotal: 2, - stepCommand: 'assertNotVisible', - stepValue: 'Nope', - }); - expect(steps[1]).toMatchObject({ - index: 1, - total: 1, - attempt: 1, - session, - artifactsDir: testArtifactsDir, - stepIndex: 2, - stepTotal: 2, - stepCommand: 'back', - }); - // `back` carries no progress value, and the observer must not invent one. - expect(steps[1]!.stepValue).toBe(undefined); - - expect(hooks[4]?.value).toMatchObject({ - status: 'pass', - title: 'Login flow', - attempt: 1, - maxAttempts: 1, - session, - artifactsDir: testArtifactsDir, - }); - expect(suite.passed).toBe(1); - expect(exitCode).toBe(0); -}); - -test('Maestro step sessions track the running attempt across a retry', async () => { - const root = mkdtempForTestSync('agent-device-reporter-maestro-retry-'); - fs.writeFileSync(path.join(root, '01-retry.yaml'), MAESTRO_FLOW); - - // The first attempt's `back` fails, so attempt 1 fails at step 2 and attempt 2 passes. - let backCalls = 0; - const { hooks, suite } = await runMaestroSuiteThroughReporter({ - root, - requestId: 'suite-maestro-retry', - flags: { retries: 1 }, - invoke: async (req) => { - if (req.command !== 'snapshot') { - backCalls += 1; - if (backCalls === 1) { - return { ok: false, error: { code: 'COMMAND_FAILED', message: 'first attempt failed' } }; - } - } - return maestroInvoke(req); - }, - }); - - const steps = hooks.filter((entry) => entry.hook === 'onTestStep').map((entry) => entry.value); - - // Every step event of attempt 1 carries attempt-1's session, and every step event of - // attempt 2 carries attempt-2's. This is the identity the removed AsyncLocalStorage used to - // supply, now threaded from the scheduler's attempt context through the Maestro observer. - const attempts = steps.map((step) => step.attempt); - const sessions = steps.map((step) => step.session); - expect(new Set(attempts)).toEqual(new Set([1, 2])); - for (const [index, step] of steps.entries()) { - expect(step.session).toBe( - `default:test:suite-maestro-retry:1-01-retry:attempt-${String(attempts[index])}`, - ); - } - expect(new Set(sessions).size).toBe(2); - expect(steps.every((step) => step.maxAttempts === 2)).toBe(true); - - // The retry result and the final result, both Maestro-produced. - const results = hooks - .filter((entry) => entry.hook === 'onTestResult') - .map((entry) => entry.value); - expect(results[0]).toMatchObject({ - status: 'fail', - attempt: 1, - retrying: true, - session: 'default:test:suite-maestro-retry:1-01-retry:attempt-1', - }); - expect(results[1]).toMatchObject({ - status: 'pass', - attempt: 2, - retrying: undefined, - session: 'default:test:suite-maestro-retry:1-01-retry:attempt-2', - }); - expect(suite.passed).toBe(1); -}); diff --git a/src/cli/replay-test/__tests__/session-test-reporter-values.test.ts b/src/cli/replay-test/__tests__/session-test-reporter-values.test.ts deleted file mode 100644 index 798784e648..0000000000 --- a/src/cli/replay-test/__tests__/session-test-reporter-values.test.ts +++ /dev/null @@ -1,386 +0,0 @@ -import type { DaemonWireRequest } from '@agent-device/contracts/command'; -import type { DaemonResponse } from '@agent-device/kernel/contracts'; -import { daemonPlatformServicesFixture } from '@agent-device/daemon/testing/__tests__/platform-services-fixture'; -// Characterization of the shipped reporter contract across the scheduler seam (#1478 P3). -// -// P3 moves the scheduler into `packages/replay-test` and makes attempt identity -// scheduler-owned, mapped back to daemon session names by the daemon adapter. The values a -// custom reporter observes today are therefore the thing most at risk of silent drift, and -// `session` most of all: it is not one value with one provenance, it is four, and they -// disagree on purpose. These tests drive the real suite handler through the real reporter -// registry and pin what a reporter sees. They document today's behavior; they do not -// propose behavior. - -// ADR 0012 attempts a post-failure screen digest through the narrow snapshot interactor seam; -// these fixtures model no runner, so reject that leaf capture deterministically. -import { expect, test, vi } from 'vitest'; -import { testCreateCommandSurface } from '../../../__tests__/daemon-composition/command-surface-fixture.ts'; - -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', () => ({ - captureSnapshotWithInteractor: vi.fn(async () => { - throw new Error('no device runner available in this test'); - }), -})); - -import fs from 'node:fs'; - -import path from 'node:path'; -import type { ReplaySuiteResult } from '@agent-device/contracts/replay'; -import { handleSessionCommands } from '@agent-device/daemon/testing/handlers/__tests__/session-command-harness'; -import { SessionStore } from '@agent-device/daemon/testing/session-store'; -import { withRequestProgressSink } from '@agent-device/host-kit/request'; -import { withTestDeviceInventoryProvider as withDeviceInventoryProvider } from '@agent-device/device-selection/inventory-gateways-fixtures'; -import type { DeviceInfo } from '@agent-device/kernel/device'; -import { - getReplayTestReporterExitCode, - runReplayTestReporterProgress, - runReplayTestReporters, -} from '../reporters/registry.ts'; -import type { ReplayTestReporter, ReplayTestReporterContext } from '../reporters/types.ts'; -import { mkdtempForTestSync } from '@agent-device/test-kit'; - -type RecordedHook = { - hook: 'onSuiteStart' | 'onTestStart' | 'onTestStep' | 'onTestResult' | 'onSuiteEnd'; - value: Record; -}; - -const ANDROID_ONE: DeviceInfo = { - platform: 'android', - id: 'emulator-5554', - name: 'Pixel 8', - kind: 'emulator', - booted: true, -}; - -const ANDROID_TWO: DeviceInfo = { - platform: 'android', - id: 'emulator-5556', - name: 'Pixel 8 Pro', - kind: 'emulator', - booted: true, -}; - -const reporterContext: ReplayTestReporterContext = { - stdout: { isTTY: false, write() {} }, - stderr: { isTTY: false, write() {} }, -}; - -function createRecordingReporter(): { reporter: ReplayTestReporter; hooks: RecordedHook[] } { - const hooks: RecordedHook[] = []; - return { - hooks, - reporter: { - name: 'recording', - onSuiteStart: (suite) => void hooks.push({ hook: 'onSuiteStart', value: { ...suite } }), - onTestStart: (testCase) => void hooks.push({ hook: 'onTestStart', value: { ...testCase } }), - onTestStep: (testCase) => void hooks.push({ hook: 'onTestStep', value: { ...testCase } }), - onTestResult: (testCase) => void hooks.push({ hook: 'onTestResult', value: { ...testCase } }), - onSuiteEnd: (suite) => void hooks.push({ hook: 'onSuiteEnd', value: { ...suite } }), - getExitCode: (suite) => (suite.failed > 0 ? 1 : 0), - }, - }; -} - -function makeSessionStore(): SessionStore { - const root = mkdtempForTestSync('agent-device-reporter-values-'); - return new SessionStore(path.join(root, 'sessions')); -} - -/** - * Runs `test` exactly as the daemon does, feeding every progress event through the shipped - * reporter registry — the same translation and dispatch a `--reporter ./mine.mjs` module gets. - */ -async function runSuiteThroughReporter(params: { - root: string; - requestId: string; - inputs?: string[]; - flags?: DaemonWireRequest['flags']; - invoke: (req: DaemonWireRequest) => Promise; - devices?: DeviceInfo[]; -}): Promise<{ hooks: RecordedHook[]; suite: ReplaySuiteResult; exitCode: number }> { - const { reporter, hooks } = createRecordingReporter(); - const reporters = [reporter]; - const call = async () => - await withRequestProgressSink( - (event) => runReplayTestReporterProgress(reporters, event, reporterContext), - async () => - await handleSessionCommands({ - createCommandSurface: testCreateCommandSurface, - platformServices: daemonPlatformServicesFixture(), - req: { - token: 't', - session: 'default', - command: 'test', - positionals: params.inputs ?? [params.root], - meta: { cwd: params.root, requestId: params.requestId }, - flags: params.flags, - }, - sessionName: 'default', - logPath: path.join(mkdtempForTestSync('daemon'), 'daemon.log'), - sessionStore: makeSessionStore(), - invoke: params.invoke, - }), - ); - - const devices = params.devices; - const response = devices - ? await withDeviceInventoryProvider(async () => devices, call) - : await call(); - - expect(response?.ok, JSON.stringify(response)).toBeTruthy(); - if (!response?.ok) throw new Error('Expected a successful suite response.'); - const suite = response.data as unknown as ReplaySuiteResult; - await runReplayTestReporters(reporters, suite, reporterContext); - return { hooks, suite, exitCode: getReplayTestReporterExitCode(reporters, suite) }; -} - -/** - * One skipped entry (no `context platform=`, filtered out by `--platform android`) followed by - * one runnable entry that fails its first attempt and passes its retry. Every reporter value a - * suite can produce shows up once. - */ -async function runRetrySuite(): Promise<{ - root: string; - hooks: RecordedHook[]; - suite: ReplaySuiteResult; - exitCode: number; -}> { - const root = mkdtempForTestSync('agent-device-reporter-session-'); - fs.writeFileSync(path.join(root, '01-untyped.ad'), 'open "Demo"\n'); - fs.writeFileSync( - path.join(root, '02-retry.ad'), - 'context platform=android retries=1\nopen "Demo"\n', - ); - - let attempts = 0; - const run = await runSuiteThroughReporter({ - root, - requestId: 'suite-reporter', - // Explicit file inputs keep argument order; a directory input enumerates - // in native readdir order, which is not lexicographic on every filesystem. - inputs: [path.join(root, '01-untyped.ad'), path.join(root, '02-retry.ad')], - flags: { platform: 'android' }, - invoke: async () => { - attempts += 1; - return attempts === 1 - ? { - ok: false, - error: { - code: 'COMMAND_FAILED', - message: 'first attempt failed', - // Carried so the retry-path `hint` below is asserted on a real hook value. The - // final-attempt path has its own hint coverage; without a hint here the retry - // emit site was reachable by no assertion at all. - hint: 'retry hint from the failed attempt', - }, - } - : { ok: true, data: { replayed: 1, healed: 0 } }; - }, - }); - return { root, ...run }; -} - -function hookValue( - hooks: RecordedHook[], - index: number, - hook: RecordedHook['hook'], -): Record { - const entry = hooks[index]; - expect(entry, `no hook recorded at ${index}`).toBeDefined(); - expect(entry?.hook).toBe(hook); - return entry?.value ?? {}; -} - -const RETRY_SUITE_HOOKS = [ - 'onSuiteStart', - 'onTestResult', - 'onTestStart', - 'onTestStep', - 'onTestResult', - 'onTestStep', - 'onTestResult', - 'onSuiteEnd', -] as const; - -test('a reporter sees the shipped suite-start, skip, and test-start values', async () => { - const { root, hooks } = await runRetrySuite(); - - expect(hooks.map((entry) => entry.hook)).toEqual([...RETRY_SUITE_HOOKS]); - - const suiteArtifactsDir = path.join(root, '.agent-device', 'test-artifacts', 'suite-reporter'); - expect(hookValue(hooks, 0, 'onSuiteStart')).toEqual({ - total: 2, - runnable: 1, - skipped: 1, - artifactsDir: suiteArtifactsDir, - shardMode: undefined, - shardCount: undefined, - }); - - // A skipped entry never opens a session, so the reporter gets no session and no duration. - expect(hookValue(hooks, 1, 'onTestResult')).toMatchObject({ - file: path.join(root, '01-untyped.ad'), - status: 'skip', - index: 1, - total: 2, - session: undefined, - artifactsDir: undefined, - durationMs: undefined, - message: 'missing platform metadata for --platform android', - }); - - // The start value is emitted once per test case, BEFORE any attempt runs, so its `session` - // is `buildReplayTestSessionName(..., attemptIndex = 0)`: always `attempt-1`, even when the - // case ends up passing on attempt 2. `attempt` is absent while `maxAttempts` is present. - // Its test number (`1`) is the runnable ordinal; `index` (`2`) is the discovery ordinal. - expect(hookValue(hooks, 2, 'onTestStart')).toEqual({ - file: path.join(root, '02-retry.ad'), - title: undefined, - index: 2, - total: 2, - attempt: undefined, - maxAttempts: 2, - session: 'default:test:suite-reporter:1-02-retry:attempt-1', - artifactsDir: path.join(suiteArtifactsDir, '02-retry.ad'), - shardIndex: undefined, - shardCount: undefined, - deviceId: undefined, - deviceName: undefined, - }); -}); - -test('reporter step and result sessions track the running attempt, not the start value', async () => { - const { root, hooks, suite, exitCode } = await runRetrySuite(); - const testArtifactsDir = path.join( - root, - '.agent-device', - 'test-artifacts', - 'suite-reporter', - '02-retry.ad', - ); - - // Step values come from the per-attempt AsyncLocalStorage context, so they track the - // running attempt: attempt-1 then attempt-2. They carry no `status` key at all. - const firstStep = hookValue(hooks, 3, 'onTestStep'); - expect('status' in firstStep).toBe(false); - expect(firstStep).toMatchObject({ - attempt: 1, - maxAttempts: 2, - session: 'default:test:suite-reporter:1-02-retry:attempt-1', - artifactsDir: testArtifactsDir, - stepIndex: 1, - stepTotal: 1, - stepCommand: 'open', - stepValue: 'Demo', - }); - expect(hookValue(hooks, 5, 'onTestStep')).toMatchObject({ - attempt: 2, - session: 'default:test:suite-reporter:1-02-retry:attempt-2', - stepIndex: 1, - stepCommand: 'open', - }); - - // The retry result carries the failed attempt's session; the final result carries the last - // attempt's session. Neither equals the start value once a retry happened. - expect(hookValue(hooks, 4, 'onTestResult')).toMatchObject({ - status: 'fail', - attempt: 1, - maxAttempts: 2, - retrying: true, - session: 'default:test:suite-reporter:1-02-retry:attempt-1', - message: 'Replay failed at step 1 (open "Demo"): first attempt failed', - // The retry emit site publishes `hint` separately from the final one. Assert it on the - // real hook value so deleting it from the scheduler fails here rather than shipping a - // reporter that silently loses the field on every retried failure. - hint: 'retry hint from the failed attempt', - }); - const finalResult = hookValue(hooks, 6, 'onTestResult'); - expect(finalResult).toMatchObject({ - status: 'pass', - attempt: 2, - maxAttempts: 2, - retrying: undefined, - session: 'default:test:suite-reporter:1-02-retry:attempt-2', - artifactsDir: testArtifactsDir, - }); - expect(finalResult.durationMs).toEqual(expect.any(Number)); - - // onSuiteEnd receives the awaited suite result; its per-test `session` is the final - // attempt's session, matching the last result value rather than the start value. - expect(hookValue(hooks, 7, 'onSuiteEnd')).toEqual(suite); - expect(suite.tests.map((entry) => ('session' in entry ? entry.session : undefined))).toEqual([ - undefined, - 'default:test:suite-reporter:1-02-retry:attempt-2', - ]); - expect(exitCode).toBe(0); -}); - -test('a failing suite reaches the reporter with the failure message, hint fields, and exit code', async () => { - const root = mkdtempForTestSync('agent-device-reporter-fail-'); - fs.writeFileSync(path.join(root, '01-fail.ad'), 'context platform=android\nopen "Demo"\n'); - - const { hooks, suite, exitCode } = await runSuiteThroughReporter({ - root, - requestId: 'suite-reporter-fail', - invoke: async () => ({ - ok: false, - error: { code: 'ASSERTION_FAILED', message: 'selector not found', hint: 'try replay --from' }, - }), - }); - - const results = hooks.filter((entry) => entry.hook === 'onTestResult'); - expect(results).toHaveLength(1); - expect(hookValue(results, 0, 'onTestResult')).toMatchObject({ - status: 'fail', - attempt: 1, - maxAttempts: 1, - retrying: undefined, - session: 'default:test:suite-reporter-fail:1-01-fail:attempt-1', - message: 'Replay failed at step 1 (open "Demo"): selector not found', - // `hint` is a shipped reporter field: assert it on the real hook value, not - // just as test input, so dropping `hint: error.hint` from the scheduler - // path cannot pass this ratchet. - hint: 'try replay --from', - }); - expect(suite.failed).toBe(1); - expect(exitCode).toBe(1); -}); - -test('sharded runs give the reporter shard-scoped sessions and device identity', async () => { - const root = mkdtempForTestSync('agent-device-reporter-shard-'); - fs.writeFileSync(path.join(root, '01-login.ad'), 'context platform=android\nopen "Demo"\n'); - - const { hooks } = await runSuiteThroughReporter({ - root, - requestId: 'suite-reporter-shard', - flags: { - platform: 'android', - device: 'emulator-5554,emulator-5556', - shardAll: 2, - }, - devices: [ANDROID_ONE, ANDROID_TWO], - invoke: async () => ({ ok: true, data: { replayed: 1, healed: 0 } }), - }); - - // Shipped asymmetry: `total` is shard-multiplied (1 file x 2 shards) while `runnable` and - // `skipped` are counted before sharding, so a sharded suite-start value reports - // total 2 / runnable 1. Pinned as-is; the extraction must not "fix" it silently. - expect(hookValue(hooks, 0, 'onSuiteStart')).toEqual({ - total: 2, - runnable: 1, - skipped: 0, - artifactsDir: path.join(root, '.agent-device', 'test-artifacts', 'suite-reporter-shard'), - shardMode: 'all', - shardCount: 2, - }); - - const starts = hooks.filter((entry) => entry.hook === 'onTestStart'); - expect(starts.map((entry) => entry.value.session)).toEqual([ - 'default:shard-1:test:suite-reporter-shard:1-01-login:attempt-1', - 'default:shard-2:test:suite-reporter-shard:1-01-login:attempt-1', - ]); - expect(starts.map((entry) => entry.value.shardIndex)).toEqual([0, 1]); - expect(starts.map((entry) => entry.value.shardCount)).toEqual([2, 2]); - expect(starts.map((entry) => entry.value.deviceId)).toEqual(['emulator-5554', 'emulator-5556']); - expect(starts.map((entry) => entry.value.deviceName)).toEqual(['Pixel 8', 'Pixel 8 Pro']); -}); diff --git a/src/client/human-control.fixtures.ts b/src/client/human-control.fixtures.ts new file mode 100644 index 0000000000..f6fded24fe --- /dev/null +++ b/src/client/human-control.fixtures.ts @@ -0,0 +1,17 @@ +import type { HumanControlHold, HumanControlHoldScope } from '@agent-device/contracts/client'; + +export const HUMAN_CONTROL_SCOPE: HumanControlHoldScope = { + backend: 'ios-instance', + leaseProvider: 'proxy', + deviceKey: 'ios:mobile:sim-1', +}; + +export const HUMAN_CONTROL_HOLD: HumanControlHold = { + id: 'operator-1', + scope: HUMAN_CONTROL_SCOPE, + state: 'active', + reason: 'Manual inspection', + createdAt: 1_000, + updatedAt: 1_000, + expiresAt: 16_000, +}; diff --git a/src/client/lease-client.test.ts b/src/client/lease-client.test.ts index a653f600be..5156365970 100644 --- a/src/client/lease-client.test.ts +++ b/src/client/lease-client.test.ts @@ -2,10 +2,7 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { AppError } from '@agent-device/kernel/errors'; import { createAgentDeviceClient } from '../agent-device-client.ts'; -import { - HUMAN_CONTROL_HOLD, - HUMAN_CONTROL_SCOPE, -} from '@agent-device/daemon/testing/__tests__/human-control-fixtures'; +import { HUMAN_CONTROL_HOLD, HUMAN_CONTROL_SCOPE } from './human-control.fixtures.ts'; import type { DaemonRequest } from '@agent-device/kernel/contracts'; test('human-control client carries normal remote authentication and full lease metadata', async () => { diff --git a/src/commands/cli-grammar/flag-declaration-admission.test.ts b/src/commands/cli-grammar/flag-declaration-admission.test.ts index 7ad93591de..2d55253268 100644 --- a/src/commands/cli-grammar/flag-declaration-admission.test.ts +++ b/src/commands/cli-grammar/flag-declaration-admission.test.ts @@ -6,8 +6,6 @@ import type { CliFlags } from '@agent-device/contracts/command'; import { getFlagDefinitionsForKey } from '@agent-device/command-registry/flag-registry'; import type { FlagDefinition } from '@agent-device/command-registry/flag-types'; import { resolveConfigBackedFlagDefaults } from '../schema/cli-config.ts'; -import { recordActionEntry } from '@agent-device/daemon/testing/session-action-recorder'; -import { makeIosSession } from '@agent-device/daemon/testing/__tests__/support/session-factories'; import { makeTempWorkspace } from '../../__tests__/cli-config-fixtures.ts'; /** @@ -79,36 +77,6 @@ test('project-config admission follows the declaration: an undeclared key is ref } }); -test('recorder sanitization follows the declaration: an undeclared key is dropped, a planted declaration is recorded', async () => { - // Shipped: `overlayRefs` declines recording so it is dropped; `fps` (recorded) is the - // control that proves the copy itself works. - assert.equal(declarationFor('overlayRefs').recorded, false); - const shipped = recordActionEntry(makeIosSession('default'), { - command: 'screenshot', - positionals: [], - flags: { overlayRefs: true, fps: 5 }, - result: {}, - }); - assert.equal(shipped?.flags.overlayRefs, undefined, 'overlayRefs is undeclared for recording'); - assert.equal(shipped?.flags.fps, 5, 'fps is declared for recording'); - - // Planted: flip the declaration before the recorder builds, and the production - // `sanitizeFlags` now copies the same value. - vi.resetModules(); - const registry = await import('@agent-device/command-registry/flag-registry'); - const planted = getInRegistry(registry, 'overlayRefs'); - Object.assign(planted, { recorded: true }); - const { recordActionEntry: recordPlanted } = - await import('@agent-device/daemon/testing/session-action-recorder'); - const recorded = recordPlanted(makeIosSession('default'), { - command: 'screenshot', - positionals: [], - flags: { overlayRefs: true }, - result: {}, - }); - assert.equal(recorded?.flags.overlayRefs, true, 'a planted declaration is copied'); -}); - test('a flag declaration without both admission fields does not compile', () => { // @ts-expect-error projectConfig and recorded are required; omission is the fail-closed gate. const declaration: FlagDefinition = { key: 'json', names: ['--json'], type: 'boolean' }; @@ -131,7 +99,7 @@ test('a CLI-only key cannot opt into recording', () => { function getInRegistry( registry: typeof import('@agent-device/command-registry/flag-registry'), - key: 'daemonBaseUrl' | 'overlayRefs', + key: 'daemonBaseUrl', ): FlagDefinition { const declaration = registry.getFlagDefinitionsForKey(key)[0]; assert.ok(declaration, `expected a declaration for ${key}`); diff --git a/src/daemon-client/__tests__/daemon-client-abort-default-transport.test.ts b/src/daemon-client/__tests__/daemon-client-abort-default-transport.test.ts index 3acd92556c..dea8cc16b2 100644 --- a/src/daemon-client/__tests__/daemon-client-abort-default-transport.test.ts +++ b/src/daemon-client/__tests__/daemon-client-abort-default-transport.test.ts @@ -1,91 +1,17 @@ -/** - * #3178: the real client route, through the default transport. - * - * `daemon-client-abort.test.ts` drives `sendRequest` directly, and `agent-device-client-abort-signal.test.ts` - * injects a fake transport, so neither proves that the caller's signal actually reaches the - * transport from `createAgentDeviceClient`. That property lives at one line — the signal handed to - * `sendRequest` in `daemon-client.ts` — and a test that skips that wiring cannot see it disappear: - * delete the hand-off and a fake-transport test still passes, because the client-level guard - * rejects the caller with `request_canceled` while the daemon keeps running the command on the - * device. That is exactly the bug #3178 reports. - * - * Each case builds a client with NO injected transport, points its state dir at a loopback daemon - * (a real `createSocketServer` / `createDaemonHttpServer` published through `daemon.json`), aborts - * a long `wait` mid-flight, and asserts the DAEMON observed the cancellation — plus that the id in - * the caller's rejection is the id the daemon canceled, which is what lets a caller correlate a - * canceled call with the daemon's own diagnostics. - */ -import type { DaemonWireRequest } from '@agent-device/contracts/command'; -import type { DaemonResponse } from '@agent-device/kernel/contracts'; import assert from 'node:assert/strict'; -import fs from 'node:fs'; import { test } from 'vitest'; import { isRequestCanceledError } from '@agent-device/kernel/errors'; -import { getRequestSignal, isRequestCanceled } from '@agent-device/host-kit/request'; -import { readProcessStartTime } from '@agent-device/host-kit/process'; -import { readVersion } from '@agent-device/host-kit/version'; -import { createDaemonHttpServer } from '@agent-device/daemon/testing/server/http-server'; -import { createSocketServer, listenNetServer } from '@agent-device/daemon/testing/server/transport'; import { createAgentDeviceClient } from '../../agent-device-client.ts'; -import { currentDaemonCodeSignature } from '../../__tests__/test-utils/daemon-http-fixture.ts'; import { - closeLoopbackServer, - listenOnLoopback, - mkdtempForTestSync, - type SkippableTestContext, - skipWhenLoopbackUnavailable, - trackLoopbackSockets, -} from '@agent-device/test-kit'; + createPublicWaitCommandSurface, + startPublicDaemonRuntime, +} from '../../__tests__/test-utils/public-daemon-runtime.ts'; +import { type SkippableTestContext, skipWhenLoopbackUnavailable } from '@agent-device/test-kit'; -type WireInvokeFn = (req: DaemonWireRequest) => Promise; - -const TOKEN = 'default-transport-abort-token'; - -type SeenRequest = { started: boolean; canceled: boolean[]; requestId?: string }; - -// `wait` is the long request the caller abandons; it never answers on its own, so only the client's -// disconnect ends it — through the daemon's own request-scoped cancellation. -function hangingWaitHandler(seen: SeenRequest): WireInvokeFn { - return async (req: DaemonWireRequest): Promise => { - const requestId = req.meta?.requestId; - seen.started = true; - seen.requestId = requestId; - const signal = getRequestSignal(requestId); - if (!signal) { - return { ok: true, data: { answeredWithoutSignal: true } }; - } - return await new Promise((_resolve, reject) => { - signal.addEventListener( - 'abort', - () => { - seen.canceled.push(isRequestCanceled(requestId)); - reject(signal.reason); - }, - { once: true }, - ); - }); - }; -} - -// The `daemon.json` a running daemon publishes, shaped so the client's takeover ladder reuses it: -// matching version and code signature, then the live probe the loopback server answers. -function publishLoopbackDaemonInfo(stateDir: string, info: Record): void { - fs.mkdirSync(stateDir, { recursive: true }); - fs.writeFileSync( - `${stateDir}/daemon.json`, - `${JSON.stringify({ - token: TOKEN, - pid: process.pid, - version: readVersion(), - codeSignature: currentDaemonCodeSignature(), - processStartTime: readProcessStartTime(process.pid) ?? undefined, - ...info, - })}\n`, - ); -} +type SeenRequest = { started: boolean; canceled: boolean; requestId?: string }; async function waitFor(condition: () => boolean, what: string): Promise { - const deadline = Date.now() + 3000; + const deadline = Date.now() + 3_000; while (!condition()) { if (Date.now() >= deadline) throw new Error(`Timed out waiting for ${what}.`); await new Promise((resolve) => setTimeout(resolve, 10)); @@ -95,29 +21,32 @@ async function waitFor(condition: () => boolean, what: string): Promise { async function runDefaultTransportAbort( t: SkippableTestContext, transport: 'socket' | 'http', - serve: (handleRequest: WireInvokeFn) => Promise<{ - server: Parameters[0]; - port: number; - destroyConnections?: () => void; - }>, ): Promise { if (await skipWhenLoopbackUnavailable(t)) return; - const seen: SeenRequest = { started: false, canceled: [] }; - const stateDir = mkdtempForTestSync(`agent-device-abort-default-${transport}-`); - const { server, port, destroyConnections } = await serve(hangingWaitHandler(seen)); + const seen: SeenRequest = { started: false, canceled: false }; + const daemon = await startPublicDaemonRuntime({ + createCommandSurface: createPublicWaitCommandSurface({ + onStarted: (_target, requestId) => { + seen.started = true; + seen.requestId = requestId; + }, + onCanceled: () => { + seen.canceled = true; + }, + }), + }); try { - publishLoopbackDaemonInfo( - stateDir, - transport === 'socket' - ? { port, transport: 'socket' } - : { httpPort: port, transport: 'http' }, - ); - // No injected transport: the client's own `sendToDaemon` and default transport have to carry the - // caller signal all the way to the connection they open. - const client = createAgentDeviceClient({ stateDir, daemonTransport: transport }); + const client = createAgentDeviceClient({ + stateDir: daemon.stateDir, + daemonTransport: transport, + }); const controller = new AbortController(); - - const call = client.command.wait({ durationMs: 5000, signal: controller.signal }); + const call = client.command.wait({ + durationMs: 5_000, + platform: 'macos', + target: 'desktop', + signal: controller.signal, + }); await waitFor(() => seen.started, 'the daemon to start the request'); controller.abort(); @@ -129,37 +58,20 @@ async function runDefaultTransportAbort( ); assert.equal(isRequestCanceledError(rejection), true); const details = (rejection as { details?: Record }).details ?? {}; - assert.equal(details.dispatched, 'unknown'); - // The proof the fake-transport tests cannot offer: the cancellation crossed the wire, because - // the daemon's own request registry saw it. Delete the signal hand-off in `daemon-client.ts` - // and this stays empty while the caller still rejects through its guard — the #3178 bug. - await waitFor(() => seen.canceled.length > 0, 'daemon-side cancellation'); - assert.deepEqual(seen.canceled, [true]); - // The caller's rejection names the same request the daemon canceled, so a caller can correlate - // the two records. assert.equal(typeof details.requestId, 'string'); + assert.equal(details.dispatched, 'unknown'); + await waitFor(() => seen.canceled, 'the daemon to observe cancellation'); + assert.equal(seen.canceled, true); assert.equal(details.requestId, seen.requestId); } finally { - destroyConnections?.(); - await closeLoopbackServer(server); - fs.rmSync(stateDir, { recursive: true, force: true }); + await daemon.close(); } } -test('the default socket transport carries the caller signal: the daemon cancels the request the client aborted', async (t) => { - await runDefaultTransportAbort(t, 'socket', async (handleRequest) => { - const server = createSocketServer(handleRequest); - // A canceled request's connection is destroyed by the client, but if an assertion above fails - // before that, the handler waits forever on an open connection and `net.Server.close()` would - // never return. Destroying the tracked sockets turns that into the failed assertion it is. - const destroyConnections = trackLoopbackSockets(server); - return { server, port: await listenNetServer(server), destroyConnections }; - }); +test('the default socket transport carries the caller signal through the public daemon runtime', async (t) => { + await runDefaultTransportAbort(t, 'socket'); }); -test('the default HTTP transport carries the caller signal: the daemon cancels the request the client aborted', async (t) => { - await runDefaultTransportAbort(t, 'http', async (handleRequest) => { - const server = await createDaemonHttpServer({ handleRequest, token: TOKEN }); - return { server, port: await listenOnLoopback(server) }; - }); +test('the default HTTP transport carries the caller signal through the public daemon runtime', async (t) => { + await runDefaultTransportAbort(t, 'http'); }); diff --git a/src/daemon-client/__tests__/daemon-client-abort-timeout.test.ts b/src/daemon-client/__tests__/daemon-client-abort-timeout.test.ts index 0115b356bb..02d6997787 100644 --- a/src/daemon-client/__tests__/daemon-client-abort-timeout.test.ts +++ b/src/daemon-client/__tests__/daemon-client-abort-timeout.test.ts @@ -5,33 +5,28 @@ * discipline, not the sweep — and each case outlives the armed budget, so the assertion proves the * timer was cleared rather than merely beaten to the settle. * - * The abort case aborts right after `sendRequest` arms its timer, so the clear is what is tested. - * The paired no-signal case lets the same budget expire and must reach the mocked seam, which - * makes the abort case's empty-calls assertion evidence instead of an absent hook: delete the - * abort path's `clearTimeout` and only the abort case fails. + * The socket abort also checks that the exact timer handle armed for the request was cleared. The + * paired no-signal case lets the same budget expire and must reach the mocked seam, so the test + * proves both the socket timer setup and the abort cleanup. */ -import type { DaemonWireRequest } from '@agent-device/contracts/command'; -import type { DaemonResponse } from '@agent-device/kernel/contracts'; import assert from 'node:assert/strict'; import http from 'node:http'; +import net from 'node:net'; import { DAEMON_HTTP_INSTANCE_MISMATCH_HEADER } from '@agent-device/contracts/daemon-http'; import { test, vi } from 'vitest'; import { AppError, isRequestCanceledError } from '@agent-device/kernel/errors'; -import { getRequestSignal } from '@agent-device/host-kit/request'; -import { createSocketServer, listenNetServer } from '@agent-device/daemon/testing/server/transport'; import { sendRequest } from '../daemon-client-transport.ts'; import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; import { closeLoopbackServer, listenOnLoopback, + type LoopbackServer, mkdtempForTestSync, type SkippableTestContext, skipWhenLoopbackUnavailable, trackLoopbackSockets, } from '@agent-device/test-kit'; -type WireInvokeFn = (req: DaemonWireRequest) => Promise; - const { handleRequestTimeoutCalls } = vi.hoisted(() => ({ handleRequestTimeoutCalls: [] as unknown[], })); @@ -50,33 +45,28 @@ const TIMEOUT_MS = 15; // Past the armed budget, so a timer the abort forgot to clear fires inside the case. const OUTLIVE_MS = 80; -// A request the daemon never answers: the only things that can end it are the client's abort and -// the transport's own budget, which is what makes the timer's fate observable. -function hangingHandler(): WireInvokeFn { - return async (req: DaemonWireRequest): Promise => { - const signal = getRequestSignal(req.meta?.requestId); - return await new Promise((_resolve, reject) => { - signal?.addEventListener('abort', () => reject(signal.reason), { once: true }); - }); - }; -} - async function runTimerDiscipline( t: SkippableTestContext, outcome: 'aborted' | 'timed-out', + transport: 'http' | 'socket' = 'http', ): Promise { if (await skipWhenLoopbackUnavailable(t)) return; const aborted = outcome === 'aborted'; handleRequestTimeoutCalls.length = 0; - const server = createSocketServer(hangingHandler()); - // The hanging handler never answers, so whichever way the request ends, a failed assertion - // before that would leave the connection open and `net.Server.close()` would hang the lane. + const server: LoopbackServer = + transport === 'http' + ? http.createServer(() => {}) + : net.createServer((socket) => socket.on('error', () => {})); const destroySockets = trackLoopbackSockets(server); + const setTimeoutSpy = vi.spyOn(globalThis, 'setTimeout'); + const clearTimeoutSpy = vi.spyOn(globalThis, 'clearTimeout'); try { - const port = await listenNetServer(server); + const port = await listenOnLoopback(server); const controller = new AbortController(); + setTimeoutSpy.mockClear(); + clearTimeoutSpy.mockClear(); const request = sendRequest( - { port, token: 't', pid: 1 }, + transport === 'http' ? { httpPort: port, token: 't', pid: 1 } : { port, token: 't', pid: 1 }, { token: 't', command: 'wait', @@ -85,15 +75,24 @@ async function runTimerDiscipline( flags: {}, meta: { requestId: `req-${outcome}-timeout-timer` }, }, - 'socket', + transport, STATE_PATHS, TIMEOUT_MS, aborted ? { signal: controller.signal } : {}, ); if (aborted) { + const armedTimeoutIndex = setTimeoutSpy.mock.calls.findIndex( + ([, timeout]) => timeout === TIMEOUT_MS, + ); + const armedTimeout = + armedTimeoutIndex === -1 ? undefined : setTimeoutSpy.mock.results[armedTimeoutIndex]?.value; + if (transport === 'socket') assert.ok(armedTimeout); // `sendRequest` armed the timeout timer synchronously by returning; this abort must disarm it. controller.abort(); await assert.rejects(request, (error: unknown) => isRequestCanceledError(error)); + if (transport === 'socket') { + assert.ok(clearTimeoutSpy.mock.calls.some(([handle]) => handle === armedTimeout)); + } } else { await assert.rejects( request, @@ -105,6 +104,8 @@ async function runTimerDiscipline( await new Promise((resolve) => setTimeout(resolve, OUTLIVE_MS)); if (aborted) assert.deepEqual(handleRequestTimeoutCalls, []); } finally { + setTimeoutSpy.mockRestore(); + clearTimeoutSpy.mockRestore(); destroySockets(); await closeLoopbackServer(server); } @@ -118,6 +119,14 @@ test('the same budget without a signal reaches the timeout seam', async (t) => { await runTimerDiscipline(t, 'timed-out'); }); +test('a socket abort with a timeout armed clears the timer instead of running the timeout path', async (t) => { + await runTimerDiscipline(t, 'aborted', 'socket'); +}); + +test('the socket budget without a signal reaches the timeout seam', async (t) => { + await runTimerDiscipline(t, 'timed-out', 'socket'); +}); + test('an abort during a restart health probe never retires the daemon or redispatches', async (t) => { if (await skipWhenLoopbackUnavailable(t)) return; handleRequestTimeoutCalls.length = 0; diff --git a/src/daemon-client/__tests__/daemon-client-abort.test.ts b/src/daemon-client/__tests__/daemon-client-abort.test.ts index f371736cb1..56620a2ec0 100644 --- a/src/daemon-client/__tests__/daemon-client-abort.test.ts +++ b/src/daemon-client/__tests__/daemon-client-abort.test.ts @@ -1,276 +1,128 @@ -/** - * #3178: the built-in daemon transports honor a per-call `AbortSignal`. - * - * These run the REAL daemon servers (`createSocketServer`, `createDaemonHttpServer`) against a - * request handler that waits on the daemon's own request-scoped signal, so each mid-flight case - * proves the whole chain: the client aborts → that one connection closes → the daemon marks the - * request canceled (`markRequestCanceled`, reached only through the transport's disconnect path) → - * the client rejects with the typed canceled-request error — and the daemon itself stays alive to - * serve a follow-up request. - */ -import type { DaemonWireRequest } from '@agent-device/contracts/command'; -import type { DaemonResponse } from '@agent-device/kernel/contracts'; import assert from 'node:assert/strict'; import { test } from 'vitest'; import { isRequestCanceledError } from '@agent-device/kernel/errors'; -import { getRequestSignal, isRequestCanceled } from '@agent-device/host-kit/request'; -import { createDaemonHttpServer } from '@agent-device/daemon/testing/server/http-server'; -import { createSocketServer, listenNetServer } from '@agent-device/daemon/testing/server/transport'; -import { sendRequest } from '../daemon-client-transport.ts'; -import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; +import { createAgentDeviceClient } from '../../agent-device-client.ts'; import { - closeLoopbackServer, - listenOnLoopback, - mkdtempForTestSync, - skipWhenLoopbackUnavailable, - trackLoopbackSockets, -} from '@agent-device/test-kit'; + createPublicWaitCommandSurface, + startPublicDaemonRuntime, +} from '../../__tests__/test-utils/public-daemon-runtime.ts'; +import { skipWhenLoopbackUnavailable, type SkippableTestContext } from '@agent-device/test-kit'; -type WireInvokeFn = (req: DaemonWireRequest) => Promise; - -const TOKEN = 'abort-signal-token'; -const STATE_PATHS = resolveDaemonPaths(mkdtempForTestSync('agent-device-client-abort-')); - -type SeenRequest = { - started: boolean; - requestId?: string; - canceled: boolean[]; - /** The command of the last non-`wait` request the RPC handler itself served. */ - servedCommand?: string; -}; - -// `wait` models the long request the caller abandons; every other command answers immediately, so -// the same handler also proves the daemon still serves requests after one was canceled. -function canceledAwareHandler(seen: SeenRequest): WireInvokeFn { - return async (req: DaemonWireRequest): Promise => { - const requestId = req.meta?.requestId; - seen.requestId = requestId; - if (req.command !== 'wait') { - seen.servedCommand = req.command; - return { ok: true, data: {} }; - } - seen.started = true; - const signal = getRequestSignal(requestId); - if (!signal) { - return { ok: true, data: { answeredWithoutSignal: true } }; - } - // The long request never answers: only the client's disconnect ends it, through the abort. - return await new Promise((_resolve, reject) => { - signal.addEventListener( - 'abort', - () => { - seen.canceled.push(isRequestCanceled(requestId)); - reject(signal.reason); - }, - { once: true }, - ); - }); - }; +async function waitFor(condition: () => boolean, what: string): Promise { + const deadline = Date.now() + 3_000; + while (!condition()) { + if (Date.now() >= deadline) throw new Error(`Timed out waiting for ${what}.`); + await new Promise((resolve) => setTimeout(resolve, 10)); + } } -function canceledRequestError(error: unknown, dispatched: 'no' | 'unknown'): boolean { - return ( - isRequestCanceledError(error) && - (error as { details?: Record }).details?.dispatched === dispatched - ); +function clientFor(stateDir: string, daemonTransport: 'socket' | 'http') { + return createAgentDeviceClient({ stateDir, daemonTransport }); } -test('socket transport: an already-aborted signal sends nothing and refuses with dispatched no', async (t) => { +async function runAlreadyAborted( + t: SkippableTestContext, + daemonTransport: 'socket' | 'http', +): Promise { if (await skipWhenLoopbackUnavailable(t)) return; - const connections: number[] = []; - const server = createSocketServer(async () => { - connections.push(1); - return { ok: true, data: {} }; + let started = false; + const daemon = await startPublicDaemonRuntime({ + createCommandSurface: createPublicWaitCommandSurface({ + onStarted: () => { + started = true; + }, + }), }); try { - const port = await listenNetServer(server); const controller = new AbortController(); controller.abort(); await assert.rejects( - sendRequest( - { port, token: TOKEN, pid: 1 }, - { - token: TOKEN, - command: 'devices', - session: 'default', - positionals: [], - flags: {}, - meta: { requestId: 'req-socket-pre-abort' }, - }, - 'socket', - STATE_PATHS, - undefined, - { signal: controller.signal }, - ), - (error: unknown) => canceledRequestError(error, 'no'), + clientFor(daemon.stateDir, daemonTransport).command.wait({ + durationMs: 5_000, + platform: 'macos', + target: 'desktop', + signal: controller.signal, + }), + (error: unknown) => + isRequestCanceledError(error) && + (error as { details?: Record }).details?.dispatched === 'no', ); - assert.deepEqual(connections, []); + assert.equal(started, false); } finally { - await closeLoopbackServer(server); + await daemon.close(); } -}); +} -test('socket transport: an abort mid-request closes the connection, the daemon marks the request canceled, and the daemon stays alive', async (t) => { +async function runAbortedInFlight( + t: SkippableTestContext, + daemonTransport: 'socket' | 'http', +): Promise { if (await skipWhenLoopbackUnavailable(t)) return; - const seen: SeenRequest = { started: false, canceled: [] }; - const server = createSocketServer(canceledAwareHandler(seen)); - // The mid-request case's connection only closes through the client's abort; if an assertion - // above that fails, the daemon-side handler waits forever and `net.Server.close()` would hang - // the whole lane instead of reporting the failure. - const destroySockets = trackLoopbackSockets(server); - try { - const port = await listenNetServer(server); - const info = { port, token: TOKEN, pid: 1 }; - const controller = new AbortController(); - const inFlight = sendRequest( - info, - { - token: TOKEN, - command: 'wait', - session: 'default', - positionals: [], - flags: {}, - meta: { requestId: 'req-socket-abort-in-flight' }, + let requestId: string | undefined; + let canceled = false; + const daemon = await startPublicDaemonRuntime({ + createCommandSurface: createPublicWaitCommandSurface({ + onStarted: (_target, startedRequestId) => { + requestId = startedRequestId; }, - 'socket', - STATE_PATHS, - undefined, - { signal: controller.signal }, - ); - // Abort only once the daemon has the request in hand, so this is genuinely a mid-request - // cancel rather than a race against connection setup. - await waitFor(() => seen.started, 'the daemon to start the request'); - controller.abort(); - await assert.rejects(inFlight, (error: unknown) => canceledRequestError(error, 'unknown')); - // The daemon's disconnect path ran: this request's id is registered-canceled, proven through - // the daemon-side registry rather than off the client's rejection. - await waitFor(() => seen.canceled.length > 0, 'daemon-side cancellation'); - assert.deepEqual(seen.canceled, [true]); - assert.equal(seen.requestId, 'req-socket-abort-in-flight'); - - // Daemon alive and serving: a follow-up request without any cancellation completes. - const followUp = await sendRequest( - info, - { - token: TOKEN, - command: 'devices', - session: 'default', - positionals: [], - flags: {}, - meta: { requestId: 'req-socket-follow-up' }, + onCanceled: (_target, canceledRequestId) => { + assert.equal(canceledRequestId, requestId); + canceled = true; }, - 'socket', - STATE_PATHS, - 5000, - {}, - ); - assert.equal(followUp.ok, true); - assert.equal(seen.servedCommand, 'devices'); - } finally { - destroySockets(); - await closeLoopbackServer(server); - } -}); - -test('http transport: an already-aborted signal sends nothing and refuses with dispatched no', async (t) => { - if (await skipWhenLoopbackUnavailable(t)) return; - const received: number[] = []; - const server = await createDaemonHttpServer({ - handleRequest: async () => { - received.push(1); - return { ok: true, data: {} }; - }, - token: TOKEN, + }), }); try { - const port = await listenOnLoopback(server); + const client = clientFor(daemon.stateDir, daemonTransport); const controller = new AbortController(); + const inFlight = client.command.wait({ + durationMs: 5_000, + platform: 'macos', + target: 'desktop', + signal: controller.signal, + }); + await waitFor(() => requestId !== undefined, 'the daemon to start the request'); controller.abort(); - await assert.rejects( - sendRequest( - { httpPort: port, token: TOKEN, pid: 1 }, - { - token: TOKEN, - command: 'devices', - session: 'default', - positionals: [], - flags: {}, - meta: { requestId: 'req-http-pre-abort' }, - }, - 'http', - STATE_PATHS, - undefined, - { signal: controller.signal }, - ), - (error: unknown) => canceledRequestError(error, 'no'), - ); - assert.deepEqual(received, []); - } finally { - await closeLoopbackServer(server); - } -}); -test('http transport: an abort mid-request closes that request, the daemon marks it canceled, and the daemon stays alive', async (t) => { - if (await skipWhenLoopbackUnavailable(t)) return; - const seen: SeenRequest = { started: false, canceled: [] }; - const server = await createDaemonHttpServer({ - handleRequest: canceledAwareHandler(seen), - token: TOKEN, - }); - try { - const port = await listenOnLoopback(server); - const info = { httpPort: port, token: TOKEN, pid: 1 }; - const controller = new AbortController(); - const inFlight = sendRequest( - info, - { - token: TOKEN, - command: 'wait', - session: 'default', - positionals: [], - flags: {}, - meta: { requestId: 'req-http-abort-in-flight' }, + const rejection = await inFlight.then( + () => { + throw new Error('expected the aborted call to reject'); }, - 'http', - STATE_PATHS, - undefined, - { signal: controller.signal }, + (error: unknown) => error, ); - await waitFor(() => seen.started, 'the daemon to start the request'); - controller.abort(); - await assert.rejects(inFlight, (error: unknown) => canceledRequestError(error, 'unknown')); - await waitFor(() => seen.canceled.length > 0, 'daemon-side cancellation'); - assert.deepEqual(seen.canceled, [true]); - assert.equal(seen.requestId, 'req-http-abort-in-flight'); + assert.equal(isRequestCanceledError(rejection), true); + const details = (rejection as { details?: Record }).details ?? {}; + assert.equal(details.dispatched, 'unknown'); + assert.equal(details.requestId, requestId); + await waitFor(() => canceled, 'the daemon to observe cancellation'); - const followUp = await sendRequest( - info, - { - token: TOKEN, - command: 'devices', - session: 'default', - positionals: [], - flags: {}, - meta: { requestId: 'req-http-follow-up' }, - }, - 'http', - STATE_PATHS, - 5000, - {}, - ); - assert.equal(followUp.ok, true); - // The follow-up ran through the RPC path `/health` does not: an aborted request's cancel must - // leave `handleRequest` serving, not just the health endpoint answering. - assert.equal(seen.servedCommand, 'devices'); + await expectQuickWait(client); } finally { - await closeLoopbackServer(server); + await daemon.close(); } -}); +} -async function waitFor(condition: () => boolean, what: string): Promise { - const deadline = Date.now() + 3000; - while (!condition()) { - if (Date.now() >= deadline) throw new Error(`Timed out waiting for ${what}.`); - await new Promise((resolve) => setTimeout(resolve, 10)); - } +async function expectQuickWait(client: ReturnType): Promise { + await expectWaitResult( + client.command.wait({ durationMs: 1, platform: 'macos', target: 'desktop' }), + ); } + +async function expectWaitResult(result: Promise): Promise { + assert.deepEqual(await result, { waitedMs: 1 }); +} + +test('socket transport refuses an already-aborted call without dispatching it', async (t) => { + await runAlreadyAborted(t, 'socket'); +}); + +test('socket transport cancels an in-flight request and serves a follow-up call', async (t) => { + await runAbortedInFlight(t, 'socket'); +}); + +test('HTTP transport refuses an already-aborted call without dispatching it', async (t) => { + await runAlreadyAborted(t, 'http'); +}); + +test('HTTP transport cancels an in-flight request and serves a follow-up call', async (t) => { + await runAbortedInFlight(t, 'http'); +}); diff --git a/src/platform-runtime-gateway.test.ts b/src/platform-runtime-gateway.test.ts index 07ef62a04a..e7706fdc37 100644 --- a/src/platform-runtime-gateway.test.ts +++ b/src/platform-runtime-gateway.test.ts @@ -1,4 +1,3 @@ -import { daemonPlatformServicesFixture } from '@agent-device/daemon/testing/__tests__/platform-services-fixture'; import { type DeviceBinding, localRuntimeOwner, @@ -16,11 +15,6 @@ import { viewportRuntimeOperationFacts } from '@agent-device/contracts/viewport- import type { DeviceInfo } from '@agent-device/kernel/device'; import { createLimrunRuntime } from '@agent-device/provider-limrun'; import { describe, expect, test, vi } from 'vitest'; -import { handleAppStateCommand } from '@agent-device/daemon/testing/handlers/session-appstate'; -import { createRequestRuntimeBindings } from '@agent-device/daemon/testing/request-runtime-binding'; -import { createRequestDispatchLedger } from '@agent-device/daemon/testing/request-dispatch-ledger'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { withTestDeviceInventory } from '@agent-device/device-selection/inventory-gateways-fixtures'; import { createComposedPlatformRuntimeGateway } from './platform-runtime-gateway.ts'; import { gatewayFixture as gateway, @@ -291,7 +285,7 @@ describe('composed platform runtime gateway', () => { expect(unrelatedLoad).not.toHaveBeenCalled(); }); - test('keeps a stale Limrun Android id unavailable through the production handler without binding or local fallback', async () => { + test('keeps a stale Limrun Android id unavailable without binding or local fallback', async () => { const registration = createLimrunRuntime( { apiKey: 'test-key', runtimeInstance: 'test-instance' }, limrunTestDependencies, @@ -334,42 +328,17 @@ describe('composed platform runtime gateway', () => { providerModules: [{ runtime: registration.runtime, module: providerModule }], }); - const bindings = createRequestRuntimeBindings({ - dispatchLedger: createRequestDispatchLedger(), - gateway: runtimeGateway, - scope, - admitDeviceClaim: async () => {}, - }); try { - const response = await withTestDeviceInventory( - { local: async () => [staleDevice] }, - async () => - await handleAppStateCommand({ - platformServices: daemonPlatformServicesFixture(), - req: { - token: 't', - session: 'default', - command: 'appstate', - positionals: [], - flags: { platform: 'android', device: staleDevice.name }, - }, - sessionName: 'default', - sessionStore: makeSessionStore('agent-device-stale-limrun-'), - inspectFacts: bindings.inspectFacts, - bindDevice: bindings.bindDevice, - }), - ); - - expect(response?.ok).toBe(false); - if (response && !response.ok) { - expect(response.error.code).toBe('UNSUPPORTED_OPERATION'); - expect(response.error.hint).toMatch(/matching live provider session/i); - } + const facts = await runtimeGateway.inspectFacts(staleDevice); + expect(facts.operations.appState).toMatchObject({ + available: false, + reason: 'owner-capability-missing', + hint: 'Limrun requires a matching live provider session for this device.', + }); expect(inspectCount).toBe(1); expect(bindCount).toBe(0); expect(localLoad).not.toHaveBeenCalled(); } finally { - await bindings[Symbol.asyncDispose](); await runtimeGateway.shutdown(); await registration.runtime.shutdown(); } diff --git a/src/provider-device-runtimes.test.ts b/src/provider-device-runtimes.test.ts new file mode 100644 index 0000000000..8ef20ecdb5 --- /dev/null +++ b/src/provider-device-runtimes.test.ts @@ -0,0 +1,107 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { test, vi } from 'vitest'; +import type { DeviceLease } from '@agent-device/contracts/device'; +import { createAgentDeviceClient } from './agent-device-client.ts'; +import { mkdtempForTestSync, skipWhenLoopbackUnavailable } from '@agent-device/test-kit'; +import { startPublicDaemonRuntime } from './__tests__/test-utils/public-daemon-runtime.ts'; + +const limrun = vi.hoisted(() => ({ + iosCreate: vi.fn(async () => ({ + metadata: { id: 'ios-instance-1' }, + status: { token: 'instance-token', apiUrl: 'https://ios.example' }, + })), + iosList: vi.fn(async () => ({ getPaginatedItems: () => [] as { metadata: { id: string } }[] })), + iosDelete: vi.fn(async (_instanceId: string) => undefined), + iosDisconnect: vi.fn(), +})); + +vi.mock('@limrun/api', () => ({ + default: class MockLimrun { + readonly iosInstances = { + create: limrun.iosCreate, + list: limrun.iosList, + delete: limrun.iosDelete, + }; + }, +})); + +vi.mock('@limrun/api/ios-client', () => ({ + createInstanceClient: vi.fn(async () => ({ disconnect: limrun.iosDisconnect })), +})); + +test('Limrun retries an expired provider lease persisted by a daemon before restart', async (t) => { + if (await skipWhenLoopbackUnavailable(t)) return; + const stateDir = mkdtempForTestSync('agent-device-limrun-expired-lease-restart-'); + const env = { LIMRUN_API_KEY: 'lim_test_key', AGENT_DEVICE_LEASE_MIN_TTL_MS: '1' }; + let now = Date.now(); + const dateNow = vi.spyOn(Date, 'now').mockImplementation(() => now); + let firstDaemon: Awaited> | undefined; + let restartedDaemon: Awaited> | undefined; + + try { + firstDaemon = await startPublicDaemonRuntime({ stateDir, env }); + const client = createAgentDeviceClient({ stateDir, daemonTransport: 'http' }); + const lease = await client.leases.allocate({ + tenant: 'team-a', + runId: 'run-a', + leaseBackend: 'ios-instance', + leaseProvider: 'limrun', + platform: 'ios', + ttlMs: 1, + }); + now += 2; + limrun.iosList.mockResolvedValue({ + getPaginatedItems: () => [{ metadata: { id: 'ios-instance-recovered' } }], + }); + limrun.iosDelete.mockRejectedValue(new Error('temporary provider outage')); + + assert.equal( + ( + await client.leases.release({ + leaseId: lease.leaseId, + tenant: 'team-a', + runId: 'run-a', + leaseBackend: 'ios-instance', + leaseProvider: 'limrun', + }) + ).released, + false, + ); + await vi.waitFor(() => assert.ok(limrun.iosDelete.mock.calls.length > 0), { + interval: 10, + timeout: 2_000, + }); + await firstDaemon.runtime.shutdown(); + assert.ok(limrun.iosDisconnect.mock.calls.length > 0); + + const pendingPath = path.join(stateDir, 'expired-provider-leases.json'); + const pending = JSON.parse(fs.readFileSync(pendingPath, 'utf8')) as { leases: DeviceLease[] }; + assert.deepEqual( + pending.leases.map(({ leaseId }) => leaseId), + [lease.leaseId], + ); + + limrun.iosDelete.mockResolvedValue(undefined); + restartedDaemon = await startPublicDaemonRuntime({ stateDir, env }); + await vi.waitFor(() => assert.ok(limrun.iosDelete.mock.calls.length > 1), { + interval: 10, + timeout: 2_000, + }); + + assert.deepEqual(limrun.iosList.mock.calls.at(-1), [ + { labelSelector: `provider=limrun,leaseId=${lease.leaseId}` }, + ]); + assert.deepEqual(limrun.iosDelete.mock.calls.at(-1), ['ios-instance-recovered']); + assert.equal(fs.existsSync(pendingPath), false); + } finally { + dateNow.mockRestore(); + if (restartedDaemon) { + await restartedDaemon.close(); + } else { + await firstDaemon?.runtime.shutdown(); + fs.rmSync(stateDir, { recursive: true, force: true }); + } + } +});