From 2e769e40353252efa66df99ac50c5e9830dd648e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 15 Oct 2026 21:56:20 +0200 Subject: [PATCH 1/2] fix(record): start the touch overlay budget when record stop starts finalizing The overlay budget started when the overlay did, after the playability checks that run first. On a cold host those compile their validator (measured 5.8 s, 21.6 s at background QoS; its own budget is 30 s), so the overlay could still spend its full 70 s on top and push record stop past the 90 s request envelope: the client exits 1 with "Daemon request timed out" while the daemon finishes with an overlayWarning that the next record stop replays. The finalizer now starts the budget before those checks and hands it to the overlay, so their time comes out of the overlay's share instead of being added to it. The overlay no longer repeats the stable-file and playability waits the finalizer has just run on the same file. --- .../src/recording/__tests__/overlay.test.ts | 95 ++++++++++++++++--- packages/capture-kit/src/recording/overlay.ts | 29 +++--- .../command-descriptor-timeout-policy.test.ts | 6 +- ...me-screen-recording-finalizer-host.test.ts | 34 ++++++- ...runtime-screen-recording-finalizer-host.ts | 7 +- website/docs/docs/commands.md | 2 +- 6 files changed, 139 insertions(+), 34 deletions(-) diff --git a/packages/capture-kit/src/recording/__tests__/overlay.test.ts b/packages/capture-kit/src/recording/__tests__/overlay.test.ts index 0426365fb2..f6114fb77d 100644 --- a/packages/capture-kit/src/recording/__tests__/overlay.test.ts +++ b/packages/capture-kit/src/recording/__tests__/overlay.test.ts @@ -24,11 +24,14 @@ vi.mock('@agent-device/host-kit/command', async (importOriginal) => { }); vi.mock('../video.ts', () => ({ - waitForStableFile: vi.fn(async () => {}), waitForPlayableVideo: vi.fn(async () => {}), })); -import { OVERLAY_BUDGET_MS, overlayRecordingTouches } from '../overlay.ts'; +import { + OVERLAY_BUDGET_MS, + overlayRecordingTouches, + startRecordingOverlayBudget, +} from '../overlay.ts'; import { AppError } from '@agent-device/kernel/errors'; import { runCmd } from '@agent-device/host-kit/command'; @@ -58,9 +61,17 @@ test('overlay burns touches through a cached helper and same-directory temp outp fs.writeFileSync(videoPath, 'original'); fs.writeFileSync(telemetryPath, '{"events":[]}'); - await overlayRecordingTouches({ videoPath, telemetryPath }); + await overlayRecordingTouches({ + videoPath, + telemetryPath, + deadline: startRecordingOverlayBudget(), + }); fs.writeFileSync(videoPath, 'second original'); - await overlayRecordingTouches({ videoPath, telemetryPath }); + await overlayRecordingTouches({ + videoPath, + telemetryPath, + deadline: startRecordingOverlayBudget(), + }); const compileCalls = mockRunCmd.mock.calls.filter(([cmd, args]) => { return cmd === 'xcrun' && args[0] === 'swiftc'; @@ -96,7 +107,9 @@ test('overlay preserves Swift helper compile hints', async () => { throw new AppError('COMMAND_FAILED', 'Timed out waiting for Swift cache lock', { hint }); }); - await expect(overlayRecordingTouches({ videoPath, telemetryPath })).rejects.toMatchObject({ + await expect( + overlayRecordingTouches({ videoPath, telemetryPath, deadline: startRecordingOverlayBudget() }), + ).rejects.toMatchObject({ code: 'COMMAND_FAILED', message: 'Failed to add touch overlays to the recording', details: { @@ -111,7 +124,11 @@ test('overlay defaults to the fast medium export preset', async () => { fs.writeFileSync(videoPath, 'original'); fs.writeFileSync(telemetryPath, '{"events":[]}'); - await overlayRecordingTouches({ videoPath, telemetryPath }); + await overlayRecordingTouches({ + videoPath, + telemetryPath, + deadline: startRecordingOverlayBudget(), + }); expect(helperScriptArgs()).toEqual( expect.arrayContaining(['--events', telemetryPath, '--quality', 'medium']), @@ -124,7 +141,12 @@ test('overlay forwards the requested high export preset', async () => { fs.writeFileSync(videoPath, 'original'); fs.writeFileSync(telemetryPath, '{"events":[]}'); - await overlayRecordingTouches({ videoPath, telemetryPath, exportQuality: 'high' }); + await overlayRecordingTouches({ + videoPath, + telemetryPath, + deadline: startRecordingOverlayBudget(), + exportQuality: 'high', + }); expect(helperScriptArgs()).toEqual( expect.arrayContaining(['--events', telemetryPath, '--quality', 'high']), @@ -137,11 +159,20 @@ test('overlay caps the helper frame rate at the caller fps only when one was ask fs.writeFileSync(videoPath, 'original'); fs.writeFileSync(telemetryPath, '{"events":[]}'); - await overlayRecordingTouches({ videoPath, telemetryPath }); + await overlayRecordingTouches({ + videoPath, + telemetryPath, + deadline: startRecordingOverlayBudget(), + }); expect(helperScriptArgs()).not.toContain('--max-fps'); mockRunCmd.mockClear(); - await overlayRecordingTouches({ videoPath, telemetryPath, fps: 15 }); + await overlayRecordingTouches({ + videoPath, + telemetryPath, + deadline: startRecordingOverlayBudget(), + fps: 15, + }); const args = helperScriptArgs(); expect(args[args.indexOf('--max-fps') + 1]).toBe('15'); }); @@ -152,7 +183,11 @@ test('overlay hands the helper what is left of its budget, inside the record req fs.writeFileSync(videoPath, 'original'); fs.writeFileSync(telemetryPath, '{"events":[]}'); - await overlayRecordingTouches({ videoPath, telemetryPath }); + await overlayRecordingTouches({ + videoPath, + telemetryPath, + deadline: startRecordingOverlayBudget(), + }); const compileCall = mockRunCmd.mock.calls.find(([cmd]) => cmd === 'xcrun'); const helperCall = mockRunCmd.mock.calls.find(([cmd]) => cmd !== 'xcrun'); @@ -166,6 +201,22 @@ test('overlay hands the helper what is left of its budget, inside the record req expect(exportMs).toBeLessThan(helperCall?.[2]?.timeoutMs ?? 0); }); +test('overlay hands the helper only what the finalization left of the budget', async () => { + vi.useFakeTimers({ toFake: ['Date'] }); + const videoPath = path.join(tmpDir, 'recording.mp4'); + const telemetryPath = path.join(tmpDir, 'recording.gesture-telemetry.json'); + fs.writeFileSync(videoPath, 'original'); + fs.writeFileSync(telemetryPath, '{"events":[]}'); + const spentMs = 25_000; + const deadline = startRecordingOverlayBudget(); + vi.setSystemTime(Date.now() + spentMs); + + await overlayRecordingTouches({ videoPath, telemetryPath, deadline }); + + const helperCall = mockRunCmd.mock.calls.find(([cmd]) => cmd !== 'xcrun'); + expect(helperCall?.[2]?.timeoutMs).toBe(OVERLAY_BUDGET_MS - spentMs); +}); + test('overlay leaves the video as recorded when compiling the helper spent its budget', async () => { vi.useFakeTimers({ toFake: ['Date'] }); const videoPath = path.join(tmpDir, 'recording.mp4'); @@ -181,7 +232,9 @@ test('overlay leaves the video as recorded when compiling the helper spent its b return { stdout: '', stderr: '', exitCode: 0 }; }); - await expect(overlayRecordingTouches({ videoPath, telemetryPath })).rejects.toMatchObject({ + await expect( + overlayRecordingTouches({ videoPath, telemetryPath, deadline: startRecordingOverlayBudget() }), + ).rejects.toMatchObject({ code: 'COMMAND_FAILED', message: 'Failed to add touch overlays to the recording', }); @@ -196,14 +249,20 @@ test('overlay names its budget when the export ran out of it', async () => { fs.writeFileSync(videoPath, 'original'); fs.writeFileSync(telemetryPath, '{"events":[]}'); - await overlayRecordingTouches({ videoPath, telemetryPath }); + await overlayRecordingTouches({ + videoPath, + telemetryPath, + deadline: startRecordingOverlayBudget(), + }); fs.writeFileSync(videoPath, 'original'); mockRunCmd.mockImplementationOnce(async () => { vi.setSystemTime(Date.now() + OVERLAY_BUDGET_MS); throw new AppError('COMMAND_FAILED', 'recording-overlay exited with code 1'); }); - await expect(overlayRecordingTouches({ videoPath, telemetryPath })).rejects.toMatchObject({ + await expect( + overlayRecordingTouches({ videoPath, telemetryPath, deadline: startRecordingOverlayBudget() }), + ).rejects.toMatchObject({ code: 'COMMAND_FAILED', message: `Failed to add touch overlays to the recording: the export did not finish within its ${OVERLAY_BUDGET_MS}ms budget; a lower --fps renders faster`, }); @@ -216,12 +275,18 @@ test('overlay keeps the plain failure when the export failed with budget to spar fs.writeFileSync(videoPath, 'original'); fs.writeFileSync(telemetryPath, '{"events":[]}'); - await overlayRecordingTouches({ videoPath, telemetryPath }); + await overlayRecordingTouches({ + videoPath, + telemetryPath, + deadline: startRecordingOverlayBudget(), + }); mockRunCmd.mockImplementationOnce(async () => { throw new AppError('COMMAND_FAILED', 'recording-overlay exited with code 1'); }); - await expect(overlayRecordingTouches({ videoPath, telemetryPath })).rejects.toMatchObject({ + await expect( + overlayRecordingTouches({ videoPath, telemetryPath, deadline: startRecordingOverlayBudget() }), + ).rejects.toMatchObject({ message: 'Failed to add touch overlays to the recording', }); }); diff --git a/packages/capture-kit/src/recording/overlay.ts b/packages/capture-kit/src/recording/overlay.ts index 79806b53b9..dc77c876bc 100644 --- a/packages/capture-kit/src/recording/overlay.ts +++ b/packages/capture-kit/src/recording/overlay.ts @@ -9,7 +9,7 @@ import { compileSwiftSourceFile, resolveRecordingScriptPath, } from './swift-cache.ts'; -import { waitForPlayableVideo, waitForStableFile } from './video.ts'; +import { waitForPlayableVideo } from './video.ts'; import { DEFAULT_RECORDING_EXPORT_QUALITY, type RecordingExportQuality, @@ -27,11 +27,18 @@ export function getRecordingOverlaySupportWarning( /** * `record stop` is one daemon request, which the client abandons after `record`'s 90 s envelope: * the caller gets "Daemon request timed out" while the daemon finishes the stop. So the overlay must - * end well inside it. Compiling the helper, its export and its exit all come out of this budget; - * past it the stop keeps the raw video and its gesture telemetry, with a warning. + * end well inside it. The budget starts with the stop's finalization, so the playability checks + * before the overlay (on a cold host they compile their validator first) come out of it, as do + * compiling the helper, its export and its exit; past it the stop keeps the raw video and its + * gesture telemetry, with a warning. */ export const OVERLAY_BUDGET_MS = 70_000; +/** Starts the overlay budget. Call it when the stop's finalization starts, not when the overlay does. */ +export function startRecordingOverlayBudget(): Deadline { + return Deadline.fromTimeoutMs(OVERLAY_BUDGET_MS); +} + /** * The helper's time outside its export: starting up, then verifying the composited output or * cancelling the export, and exiting. @@ -59,13 +66,9 @@ async function exportProcessedVideo(params: { scriptPath: string; scriptArgs: string[]; commandDescription: string; - budgetMs: number; + deadline: Deadline; }): Promise { - const { videoPath, scriptPath, scriptArgs, commandDescription } = params; - const deadline = Deadline.fromTimeoutMs(params.budgetMs); - await waitForStableFile(videoPath); - await waitForPlayableVideo(videoPath); - + const { videoPath, scriptPath, scriptArgs, commandDescription, deadline } = params; const outputPath = temporarySiblingVideoPath(videoPath); let exportOverran = false; try { @@ -80,7 +83,7 @@ async function exportProcessedVideo(params: { if (exportMs <= 0) { throw new AppError( 'COMMAND_FAILED', - `No time was left for the export within the ${params.budgetMs}ms budget`, + `No time was left for the export within the ${OVERLAY_BUDGET_MS}ms budget`, ); } try { @@ -105,7 +108,7 @@ async function exportProcessedVideo(params: { fs.renameSync(outputPath, videoPath); } catch (error) { const description = exportOverran - ? `${commandDescription}: the export did not finish within its ${params.budgetMs}ms budget; a lower --fps renders faster` + ? `${commandDescription}: the export did not finish within its ${OVERLAY_BUDGET_MS}ms budget; a lower --fps renders faster` : commandDescription; const cause = error instanceof AppError @@ -144,6 +147,8 @@ export async function overlayRecordingTouches(params: { /** The caller's `--fps`; the helper renders at most this many frames a second, capped at 30. */ fps?: number; targetLabel?: string; + /** From `startRecordingOverlayBudget`, taken when the stop's finalization started. */ + deadline: Deadline; }): Promise { const { videoPath, @@ -162,6 +167,6 @@ export async function overlayRecordingTouches(params: { ...(params.fps === undefined ? [] : ['--max-fps', String(params.fps)]), ], commandDescription: `Failed to add touch overlays to the ${targetLabel}`, - budgetMs: OVERLAY_BUDGET_MS, + deadline: params.deadline, }); } diff --git a/src/__tests__/command-descriptor-timeout-policy.test.ts b/src/__tests__/command-descriptor-timeout-policy.test.ts index 0de631a9ae..c095b867a1 100644 --- a/src/__tests__/command-descriptor-timeout-policy.test.ts +++ b/src/__tests__/command-descriptor-timeout-policy.test.ts @@ -332,9 +332,9 @@ test('snapshot uses the standard daemon request timeout with an explicit overrid }); test('the touch overlay budget leaves record stop room inside its request envelope', () => { - // The recorder stop, the copies and the playability checks around the overlay are outside its - // budget (about 1 s for a 10-minute simulator recording), so the envelope keeps a reserve for - // them beyond it. + // The recorder stop and the copies before finalization, and the playability check after the + // overlay, are outside its budget (about 1 s for a 10-minute simulator recording), so the envelope + // keeps a reserve for them beyond it. const reserveMs = 20_000; const envelopeMs = resolveCommandRequestTimeoutMs(resolveCommandTimeoutPolicy('record'), { positionals: ['stop'], diff --git a/src/platform-runtime-screen-recording-finalizer-host.test.ts b/src/platform-runtime-screen-recording-finalizer-host.test.ts index 2082e5f023..040749d6a6 100644 --- a/src/platform-runtime-screen-recording-finalizer-host.test.ts +++ b/src/platform-runtime-screen-recording-finalizer-host.test.ts @@ -1,4 +1,8 @@ import { beforeEach, expect, test, vi } from 'vitest'; +import { + OVERLAY_BUDGET_MS, + type overlayRecordingTouches, +} from '@agent-device/capture-kit/recording-overlay'; import { createScreenRecordingFinalizer } from './platform-runtime-screen-recording-finalizer-host.ts'; const video = vi.hoisted(() => ({ @@ -8,7 +12,7 @@ const video = vi.hoisted(() => ({ container: vi.fn(async () => true), })); const telemetry = vi.hoisted(() => vi.fn(() => '/tmp/capture.telemetry.json')); -const overlay = vi.hoisted(() => vi.fn(async () => {})); +const overlay = vi.hoisted(() => vi.fn(async () => {})); vi.mock('@agent-device/capture-kit/recording-video', () => ({ waitForStableFile: video.stable, waitForPlayableVideo: video.playable, @@ -18,7 +22,8 @@ vi.mock('@agent-device/capture-kit/recording-video', () => ({ vi.mock('@agent-device/capture-kit/recording-telemetry', () => ({ persistRecordingTelemetry: telemetry, })); -vi.mock('@agent-device/capture-kit/recording-overlay', () => ({ +vi.mock('@agent-device/capture-kit/recording-overlay', async (importOriginal) => ({ + ...(await importOriginal()), getRecordingOverlaySupportWarning: () => undefined, overlayRecordingTouches: overlay, })); @@ -92,6 +97,31 @@ test('hands the caller fps to the touch overlay', async () => { expect(overlay).toHaveBeenCalledWith(expect.objectContaining({ fps: 15 })); }); +test('the overlay budget covers the playability checks that run before the overlay', async () => { + // A cold host compiles the playability validator before the overlay starts; that time comes out + // of the overlay budget rather than being added to it. + vi.useFakeTimers({ toFake: ['Date'] }); + try { + const playabilityMs = 25_000; + video.stable.mockImplementationOnce(async () => { + vi.setSystemTime(Date.now() + playabilityMs); + }); + + await createScreenRecordingFinalizer().complete({ + outputPath: '/tmp/capture.mp4', + showTouches: true, + gestureEvents: [{ kind: 'tap', tMs: 500, x: 10, y: 20 }], + targetLabel: 'test recording', + }); + + expect(overlay.mock.calls[0]![0].deadline.remainingMs()).toBe( + OVERLAY_BUDGET_MS - playabilityMs, + ); + } finally { + vi.useRealTimers(); + } +}); + test('#2707 reports a dropped overlay and keeps the raw when compositing throws', async () => { // overlay.ts verifies the composite and only renames it over the raw on success, so a compositor // that fails its own #2707 checks throws before any rename: the raw file survives and the caller diff --git a/src/platform-runtime-screen-recording-finalizer-host.ts b/src/platform-runtime-screen-recording-finalizer-host.ts index 746887cf4a..d183e13a99 100644 --- a/src/platform-runtime-screen-recording-finalizer-host.ts +++ b/src/platform-runtime-screen-recording-finalizer-host.ts @@ -1,9 +1,11 @@ import { AppError } from '@agent-device/kernel/errors'; import { RECORDING_OUTPUT_UNPLAYABLE_REASON } from '@agent-device/contracts/screen-recording-runtime'; import type { ScreenRecordingRuntimeHost } from '@agent-device/contracts/screen-recording-runtime-host'; +import type { Deadline } from '@agent-device/host-kit/retry'; import { getRecordingOverlaySupportWarning, overlayRecordingTouches, + startRecordingOverlayBudget, } from '@agent-device/capture-kit/recording-overlay'; import { persistRecordingTelemetry } from '@agent-device/capture-kit/recording-telemetry'; import { @@ -24,6 +26,7 @@ async function sniffRecording(input: Readonly<{ outputPath: string }>): Promise< async function finalizeScreenRecording( input: Parameters[0], ) { + const overlayDeadline = startRecordingOverlayBudget(); await waitForStableFile(input.outputPath); await waitForPlayableVideo(input.outputPath); if (!(await isPlayableVideo(input.outputPath))) throw unplayableRecording(input.outputPath); @@ -31,12 +34,13 @@ async function finalizeScreenRecording( recording: { outPath: input.outputPath, gestureEvents: [...input.gestureEvents] }, }); if (!input.showTouches || input.gestureEvents.length === 0) return { telemetryPath }; - return await overlayTouches(input, telemetryPath); + return await overlayTouches(input, telemetryPath, overlayDeadline); } async function overlayTouches( input: Parameters[0], telemetryPath: string, + deadline: Deadline, ) { const warning = getRecordingOverlaySupportWarning(); if (warning) return { telemetryPath, overlayWarning: warning }; @@ -47,6 +51,7 @@ async function overlayTouches( exportQuality: input.exportQuality, ...(input.fps === undefined ? {} : { fps: input.fps }), targetLabel: input.targetLabel, + deadline, }); if (!(await isPlayableVideo(input.outputPath))) { throw new Error('recording post-processing produced an unplayable video'); diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 001054df8b..6a3e6106bb 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -956,7 +956,7 @@ agent-device record stop # Stop active recording - `record start` defaults to app scope and needs a session from `open `. Use `--scope device` or `--scope system` for whole-screen capture (multiple apps, settings, home screen) where the backend supports it. - iOS recording works on simulators and CoreDevice-backed physical devices. Physical devices capture runner screenshots, not a native video stream: no audio, dropped frames are expected, and the default is 15 FPS. `--fps ` (1-120) caps the rate; a busy app or long command yields fewer frames. - `--quality medium|high` (default `medium`) sets export quality on Android and Limrun; Apple export always keeps the captured resolution. Numeric `5`-`7` map to `medium` and `8`-`10` to `high`. -- With touch visualization on, recordings also produce a gesture telemetry sidecar. Burned-in touch overlays export only on macOS hosts; elsewhere `record stop` returns the raw video and telemetry with `overlayWarning`. The overlay renders at most 30 FPS (or `--fps` if lower) within a fixed budget; an export past the budget returns the raw video with `overlayWarning`. +- With touch visualization on, recordings also produce a gesture telemetry sidecar. Burned-in touch overlays export only on macOS hosts; elsewhere `record stop` returns the raw video and telemetry with `overlayWarning`. The overlay renders at most 30 FPS (or `--fps` if lower) within a fixed budget, which also covers the video checks that run before it; an export past the budget returns the raw video with `overlayWarning`. On a host that has not run a recording before, part of that budget goes to one-time tool setup, so the first `record stop` of a long recording is more likely to return the raw video. - Android uses `adb shell screenrecord` (180s limit per clip). Longer recordings are split into MP4 chunks while the daemon stays alive; after a daemon restart, `record stop` recovers only chunks listed in the device manifest. A clip can end before `record stop` when the screen stopped changing; `durationMs` is wall-clock time, and `capturedDurationMs` plus a warning report how much the video covers. - Limrun sessions always record the whole screen and refuse `--fps` and `--hide-touches` with `INVALID_ARGS`. A failed download is retried by the next `record stop`; nothing survives a daemon restart. - A busy iOS simulator recording slot returns non-retriable `DEVICE_IN_USE` with `details.reason: apple_simulator_recording_busy`. Run `record stop` in the session that owns the recording; if no recording is active, ask the host operator to restart the CoreSimulator stream service. From 3cc60037e6425b06e106c35b5cba2eed26c8bdd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 08:08:16 +0200 Subject: [PATCH 2/2] fix(record): bound the overlay's output check by its budget The playability wait after the export polled on its own schedule, so a slow check could still run past the overlay budget and rename the composited file afterwards. It now stops at the deadline, and an expired budget keeps the recorded video. --- .../src/recording/__tests__/overlay.test.ts | 23 +++++++++++++++++++ packages/capture-kit/src/recording/overlay.ts | 11 +++++++-- .../capture-kit/src/recording/video.test.ts | 11 ++++++++- packages/capture-kit/src/recording/video.ts | 8 +++++-- website/docs/docs/commands.md | 2 +- 5 files changed, 49 insertions(+), 6 deletions(-) diff --git a/packages/capture-kit/src/recording/__tests__/overlay.test.ts b/packages/capture-kit/src/recording/__tests__/overlay.test.ts index f6114fb77d..9a54eec10d 100644 --- a/packages/capture-kit/src/recording/__tests__/overlay.test.ts +++ b/packages/capture-kit/src/recording/__tests__/overlay.test.ts @@ -34,6 +34,7 @@ import { } from '../overlay.ts'; import { AppError } from '@agent-device/kernel/errors'; import { runCmd } from '@agent-device/host-kit/command'; +import { waitForPlayableVideo } from '../video.ts'; function helperScriptArgs(): string[] { const helperCall = mockRunCmd.mock.calls.find(([cmd]) => cmd !== 'xcrun'); @@ -290,3 +291,25 @@ test('overlay keeps the plain failure when the export failed with budget to spar message: 'Failed to add touch overlays to the recording', }); }); + +test('overlay keeps the video as recorded when verifying the export spent its budget', async () => { + vi.useFakeTimers({ toFake: ['Date'] }); + const videoPath = path.join(tmpDir, 'recording.mp4'); + const telemetryPath = path.join(tmpDir, 'recording.gesture-telemetry.json'); + fs.writeFileSync(videoPath, 'original'); + fs.writeFileSync(telemetryPath, '{"events":[]}'); + const deadline = startRecordingOverlayBudget(); + vi.mocked(waitForPlayableVideo).mockImplementationOnce(async () => { + vi.setSystemTime(Date.now() + OVERLAY_BUDGET_MS); + }); + + await expect( + overlayRecordingTouches({ videoPath, telemetryPath, deadline }), + ).rejects.toMatchObject({ + code: 'COMMAND_FAILED', + message: `Failed to add touch overlays to the recording: the export did not finish within its ${OVERLAY_BUDGET_MS}ms budget; a lower --fps renders faster`, + }); + expect(vi.mocked(waitForPlayableVideo)).toHaveBeenCalledWith(expect.any(String), { deadline }); + expect(fs.readFileSync(videoPath, 'utf8')).toBe('original'); + expect(fs.readdirSync(tmpDir).filter((name) => name.includes('agent-device-'))).toEqual([]); +}); diff --git a/packages/capture-kit/src/recording/overlay.ts b/packages/capture-kit/src/recording/overlay.ts index dc77c876bc..7dad0d5f51 100644 --- a/packages/capture-kit/src/recording/overlay.ts +++ b/packages/capture-kit/src/recording/overlay.ts @@ -29,7 +29,7 @@ export function getRecordingOverlaySupportWarning( * the caller gets "Daemon request timed out" while the daemon finishes the stop. So the overlay must * end well inside it. The budget starts with the stop's finalization, so the playability checks * before the overlay (on a cold host they compile their validator first) come out of it, as do - * compiling the helper, its export and its exit; past it the stop keeps the raw video and its + * compiling the helper, its export, its exit and verifying its output; past it the stop keeps the raw video and its * gesture telemetry, with a warning. */ export const OVERLAY_BUDGET_MS = 70_000; @@ -104,7 +104,14 @@ async function exportProcessedVideo(params: { exportOverran = deadline.remainingMs() <= HELPER_EXIT_GRACE_MS; throw error; } - await waitForPlayableVideo(outputPath); + await waitForPlayableVideo(outputPath, { deadline }); + if (deadline.isExpired()) { + exportOverran = true; + throw new AppError( + 'COMMAND_FAILED', + 'The composited video was not verified within the budget', + ); + } fs.renameSync(outputPath, videoPath); } catch (error) { const description = exportOverran diff --git a/packages/capture-kit/src/recording/video.test.ts b/packages/capture-kit/src/recording/video.test.ts index f334608259..c91904f010 100644 --- a/packages/capture-kit/src/recording/video.test.ts +++ b/packages/capture-kit/src/recording/video.test.ts @@ -5,7 +5,8 @@ import { runCmd } from '@agent-device/host-kit/command'; import { likelyPlayableWebmContainer } from '../__tests__/test-utils/video-fixtures.ts'; import { mkdtempForTestSync } from '../tmp-dir.fixtures.ts'; import { mp4Atom, mp4MovieHeader } from './mp4.fixtures.ts'; -import { isPlayableVideo } from './video.ts'; +import { Deadline } from '@agent-device/host-kit/retry'; +import { isPlayableVideo, waitForPlayableVideo } from './video.ts'; vi.mock('@agent-device/host-kit/command', () => ({ runCmd: vi.fn(async () => ({ exitCode: 0, stdout: '', stderr: '' })), @@ -53,3 +54,11 @@ test('refuses an MP4 container that never declares a movie header', async () => await expect(isPlayableVideo(writeFixture('no-moov.mp4', container))).resolves.toBe(false); expect(runCmd).not.toHaveBeenCalled(); }); + +test('waiting for a playable video stops polling once its deadline passes', async () => { + const filePath = path.join(mkdtempForTestSync('agent-device-video-wait-'), 'never-playable.mp4'); + fs.writeFileSync(filePath, ''); + const started = Date.now(); + await waitForPlayableVideo(filePath, { pollMs: 60_000, deadline: Deadline.fromTimeoutMs(0) }); + expect(Date.now() - started).toBeLessThan(1_000); +}); diff --git a/packages/capture-kit/src/recording/video.ts b/packages/capture-kit/src/recording/video.ts index adf1ca25b6..c6284eb1f6 100644 --- a/packages/capture-kit/src/recording/video.ts +++ b/packages/capture-kit/src/recording/video.ts @@ -1,7 +1,7 @@ import fs from 'node:fs'; import { AppError } from '@agent-device/kernel/errors'; import { runCmd } from '@agent-device/host-kit/command'; -import { sleep } from '@agent-device/host-kit/retry'; +import { type Deadline, sleep } from '@agent-device/host-kit/retry'; import { buildSwiftToolEnv, compileSwiftSourceText } from './swift-cache.ts'; import { hasPlayableWebmStructure } from './video-webm.ts'; @@ -119,9 +119,10 @@ function isSwiftVideoValidatorError(error: unknown): boolean { ); } +/** Polls until the file plays, for `attempts` polls or until `deadline` passes, whichever is first. */ export async function waitForPlayableVideo( filePath: string, - options: { pollMs?: number; attempts?: number } = {}, + options: { pollMs?: number; attempts?: number; deadline?: Deadline } = {}, ): Promise { const pollMs = options.pollMs ?? 150; const attempts = options.attempts ?? 12; @@ -130,6 +131,9 @@ export async function waitForPlayableVideo( if (await isPlayableVideo(filePath)) { return; } + if (options.deadline?.isExpired()) { + return; + } await sleep(pollMs); } } diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 6a3e6106bb..c41123127e 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -956,7 +956,7 @@ agent-device record stop # Stop active recording - `record start` defaults to app scope and needs a session from `open `. Use `--scope device` or `--scope system` for whole-screen capture (multiple apps, settings, home screen) where the backend supports it. - iOS recording works on simulators and CoreDevice-backed physical devices. Physical devices capture runner screenshots, not a native video stream: no audio, dropped frames are expected, and the default is 15 FPS. `--fps ` (1-120) caps the rate; a busy app or long command yields fewer frames. - `--quality medium|high` (default `medium`) sets export quality on Android and Limrun; Apple export always keeps the captured resolution. Numeric `5`-`7` map to `medium` and `8`-`10` to `high`. -- With touch visualization on, recordings also produce a gesture telemetry sidecar. Burned-in touch overlays export only on macOS hosts; elsewhere `record stop` returns the raw video and telemetry with `overlayWarning`. The overlay renders at most 30 FPS (or `--fps` if lower) within a fixed budget, which also covers the video checks that run before it; an export past the budget returns the raw video with `overlayWarning`. On a host that has not run a recording before, part of that budget goes to one-time tool setup, so the first `record stop` of a long recording is more likely to return the raw video. +- With touch visualization on, recordings also produce a gesture telemetry sidecar. Burned-in touch overlays export only on macOS hosts; elsewhere `record stop` returns the raw video and telemetry with `overlayWarning`. The overlay renders at most 30 FPS (or `--fps` if lower) within a fixed budget, which also covers the video checks that run before and after it; an export past the budget returns the raw video with `overlayWarning`. On a host that has not run a recording before, part of that budget goes to one-time tool setup, so the first `record stop` of a long recording is more likely to return the raw video. - Android uses `adb shell screenrecord` (180s limit per clip). Longer recordings are split into MP4 chunks while the daemon stays alive; after a daemon restart, `record stop` recovers only chunks listed in the device manifest. A clip can end before `record stop` when the screen stopped changing; `durationMs` is wall-clock time, and `capturedDurationMs` plus a warning report how much the video covers. - Limrun sessions always record the whole screen and refuse `--fps` and `--hide-touches` with `INVALID_ARGS`. A failed download is retried by the next `record stop`; nothing survives a daemon restart. - A busy iOS simulator recording slot returns non-retriable `DEVICE_IN_USE` with `details.reason: apple_simulator_recording_busy`. Run `record stop` in the session that owns the recording; if no recording is active, ask the host operator to restart the CoreSimulator stream service.