Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
118 changes: 103 additions & 15 deletions packages/capture-kit/src/recording/__tests__/overlay.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,17 @@ vi.mock('@agent-device/host-kit/command', async (importOriginal) => {
});

vi.mock('../video.ts', () => ({
waitForStableFile: vi.fn(async () => {}),
waitForPlayableVideo: vi.fn(async () => {}),
}));

import { OVERLAY_BUDGET_MS, overlayRecordingTouches } from '../overlay.ts';
import {
OVERLAY_BUDGET_MS,
overlayRecordingTouches,
startRecordingOverlayBudget,
} from '../overlay.ts';
import { AppError } from '@agent-device/kernel/errors';
import { runCmd } from '@agent-device/host-kit/command';
import { waitForPlayableVideo } from '../video.ts';

function helperScriptArgs(): string[] {
const helperCall = mockRunCmd.mock.calls.find(([cmd]) => cmd !== 'xcrun');
Expand Down Expand Up @@ -58,9 +62,17 @@ test('overlay burns touches through a cached helper and same-directory temp outp
fs.writeFileSync(videoPath, 'original');
fs.writeFileSync(telemetryPath, '{"events":[]}');

await overlayRecordingTouches({ videoPath, telemetryPath });
await overlayRecordingTouches({
videoPath,
telemetryPath,
deadline: startRecordingOverlayBudget(),
});
fs.writeFileSync(videoPath, 'second original');
await overlayRecordingTouches({ videoPath, telemetryPath });
await overlayRecordingTouches({
videoPath,
telemetryPath,
deadline: startRecordingOverlayBudget(),
});

const compileCalls = mockRunCmd.mock.calls.filter(([cmd, args]) => {
return cmd === 'xcrun' && args[0] === 'swiftc';
Expand Down Expand Up @@ -96,7 +108,9 @@ test('overlay preserves Swift helper compile hints', async () => {
throw new AppError('COMMAND_FAILED', 'Timed out waiting for Swift cache lock', { hint });
});

await expect(overlayRecordingTouches({ videoPath, telemetryPath })).rejects.toMatchObject({
await expect(
overlayRecordingTouches({ videoPath, telemetryPath, deadline: startRecordingOverlayBudget() }),
).rejects.toMatchObject({
code: 'COMMAND_FAILED',
message: 'Failed to add touch overlays to the recording',
details: {
Expand All @@ -111,7 +125,11 @@ test('overlay defaults to the fast medium export preset', async () => {
fs.writeFileSync(videoPath, 'original');
fs.writeFileSync(telemetryPath, '{"events":[]}');

await overlayRecordingTouches({ videoPath, telemetryPath });
await overlayRecordingTouches({
videoPath,
telemetryPath,
deadline: startRecordingOverlayBudget(),
});

expect(helperScriptArgs()).toEqual(
expect.arrayContaining(['--events', telemetryPath, '--quality', 'medium']),
Expand All @@ -124,7 +142,12 @@ test('overlay forwards the requested high export preset', async () => {
fs.writeFileSync(videoPath, 'original');
fs.writeFileSync(telemetryPath, '{"events":[]}');

await overlayRecordingTouches({ videoPath, telemetryPath, exportQuality: 'high' });
await overlayRecordingTouches({
videoPath,
telemetryPath,
deadline: startRecordingOverlayBudget(),
exportQuality: 'high',
});

expect(helperScriptArgs()).toEqual(
expect.arrayContaining(['--events', telemetryPath, '--quality', 'high']),
Expand All @@ -137,11 +160,20 @@ test('overlay caps the helper frame rate at the caller fps only when one was ask
fs.writeFileSync(videoPath, 'original');
fs.writeFileSync(telemetryPath, '{"events":[]}');

await overlayRecordingTouches({ videoPath, telemetryPath });
await overlayRecordingTouches({
videoPath,
telemetryPath,
deadline: startRecordingOverlayBudget(),
});
expect(helperScriptArgs()).not.toContain('--max-fps');

mockRunCmd.mockClear();
await overlayRecordingTouches({ videoPath, telemetryPath, fps: 15 });
await overlayRecordingTouches({
videoPath,
telemetryPath,
deadline: startRecordingOverlayBudget(),
fps: 15,
});
const args = helperScriptArgs();
expect(args[args.indexOf('--max-fps') + 1]).toBe('15');
});
Expand All @@ -152,7 +184,11 @@ test('overlay hands the helper what is left of its budget, inside the record req
fs.writeFileSync(videoPath, 'original');
fs.writeFileSync(telemetryPath, '{"events":[]}');

await overlayRecordingTouches({ videoPath, telemetryPath });
await overlayRecordingTouches({
videoPath,
telemetryPath,
deadline: startRecordingOverlayBudget(),
});

const compileCall = mockRunCmd.mock.calls.find(([cmd]) => cmd === 'xcrun');
const helperCall = mockRunCmd.mock.calls.find(([cmd]) => cmd !== 'xcrun');
Expand All @@ -166,6 +202,22 @@ test('overlay hands the helper what is left of its budget, inside the record req
expect(exportMs).toBeLessThan(helperCall?.[2]?.timeoutMs ?? 0);
});

test('overlay hands the helper only what the finalization left of the budget', async () => {
vi.useFakeTimers({ toFake: ['Date'] });
const videoPath = path.join(tmpDir, 'recording.mp4');
const telemetryPath = path.join(tmpDir, 'recording.gesture-telemetry.json');
fs.writeFileSync(videoPath, 'original');
fs.writeFileSync(telemetryPath, '{"events":[]}');
const spentMs = 25_000;
const deadline = startRecordingOverlayBudget();
vi.setSystemTime(Date.now() + spentMs);

await overlayRecordingTouches({ videoPath, telemetryPath, deadline });

const helperCall = mockRunCmd.mock.calls.find(([cmd]) => cmd !== 'xcrun');
expect(helperCall?.[2]?.timeoutMs).toBe(OVERLAY_BUDGET_MS - spentMs);
});

test('overlay leaves the video as recorded when compiling the helper spent its budget', async () => {
vi.useFakeTimers({ toFake: ['Date'] });
const videoPath = path.join(tmpDir, 'recording.mp4');
Expand All @@ -181,7 +233,9 @@ test('overlay leaves the video as recorded when compiling the helper spent its b
return { stdout: '', stderr: '', exitCode: 0 };
});

await expect(overlayRecordingTouches({ videoPath, telemetryPath })).rejects.toMatchObject({
await expect(
overlayRecordingTouches({ videoPath, telemetryPath, deadline: startRecordingOverlayBudget() }),
).rejects.toMatchObject({
code: 'COMMAND_FAILED',
message: 'Failed to add touch overlays to the recording',
});
Expand All @@ -196,14 +250,20 @@ test('overlay names its budget when the export ran out of it', async () => {
fs.writeFileSync(videoPath, 'original');
fs.writeFileSync(telemetryPath, '{"events":[]}');

await overlayRecordingTouches({ videoPath, telemetryPath });
await overlayRecordingTouches({
videoPath,
telemetryPath,
deadline: startRecordingOverlayBudget(),
});
fs.writeFileSync(videoPath, 'original');
mockRunCmd.mockImplementationOnce(async () => {
vi.setSystemTime(Date.now() + OVERLAY_BUDGET_MS);
throw new AppError('COMMAND_FAILED', 'recording-overlay exited with code 1');
});

await expect(overlayRecordingTouches({ videoPath, telemetryPath })).rejects.toMatchObject({
await expect(
overlayRecordingTouches({ videoPath, telemetryPath, deadline: startRecordingOverlayBudget() }),
).rejects.toMatchObject({
code: 'COMMAND_FAILED',
message: `Failed to add touch overlays to the recording: the export did not finish within its ${OVERLAY_BUDGET_MS}ms budget; a lower --fps renders faster`,
});
Expand All @@ -216,12 +276,40 @@ test('overlay keeps the plain failure when the export failed with budget to spar
fs.writeFileSync(videoPath, 'original');
fs.writeFileSync(telemetryPath, '{"events":[]}');

await overlayRecordingTouches({ videoPath, telemetryPath });
await overlayRecordingTouches({
videoPath,
telemetryPath,
deadline: startRecordingOverlayBudget(),
});
mockRunCmd.mockImplementationOnce(async () => {
throw new AppError('COMMAND_FAILED', 'recording-overlay exited with code 1');
});

await expect(overlayRecordingTouches({ videoPath, telemetryPath })).rejects.toMatchObject({
await expect(
overlayRecordingTouches({ videoPath, telemetryPath, deadline: startRecordingOverlayBudget() }),
).rejects.toMatchObject({
message: 'Failed to add touch overlays to the recording',
});
});

test('overlay keeps the video as recorded when verifying the export spent its budget', async () => {
vi.useFakeTimers({ toFake: ['Date'] });
const videoPath = path.join(tmpDir, 'recording.mp4');
const telemetryPath = path.join(tmpDir, 'recording.gesture-telemetry.json');
fs.writeFileSync(videoPath, 'original');
fs.writeFileSync(telemetryPath, '{"events":[]}');
const deadline = startRecordingOverlayBudget();
vi.mocked(waitForPlayableVideo).mockImplementationOnce(async () => {
vi.setSystemTime(Date.now() + OVERLAY_BUDGET_MS);
});

await expect(
overlayRecordingTouches({ videoPath, telemetryPath, deadline }),
).rejects.toMatchObject({
code: 'COMMAND_FAILED',
message: `Failed to add touch overlays to the recording: the export did not finish within its ${OVERLAY_BUDGET_MS}ms budget; a lower --fps renders faster`,
});
expect(vi.mocked(waitForPlayableVideo)).toHaveBeenCalledWith(expect.any(String), { deadline });
expect(fs.readFileSync(videoPath, 'utf8')).toBe('original');
expect(fs.readdirSync(tmpDir).filter((name) => name.includes('agent-device-'))).toEqual([]);
});
38 changes: 25 additions & 13 deletions packages/capture-kit/src/recording/overlay.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import {
compileSwiftSourceFile,
resolveRecordingScriptPath,
} from './swift-cache.ts';
import { waitForPlayableVideo, waitForStableFile } from './video.ts';
import { waitForPlayableVideo } from './video.ts';
import {
DEFAULT_RECORDING_EXPORT_QUALITY,
type RecordingExportQuality,
Expand All @@ -27,11 +27,18 @@ export function getRecordingOverlaySupportWarning(
/**
* `record stop` is one daemon request, which the client abandons after `record`'s 90 s envelope:
* the caller gets "Daemon request timed out" while the daemon finishes the stop. So the overlay must
* end well inside it. Compiling the helper, its export and its exit all come out of this budget;
* past it the stop keeps the raw video and its gesture telemetry, with a warning.
* end well inside it. The budget starts with the stop's finalization, so the playability checks
* before the overlay (on a cold host they compile their validator first) come out of it, as do
* compiling the helper, its export, its exit and verifying its output; past it the stop keeps the raw video and its
* gesture telemetry, with a warning.
*/
export const OVERLAY_BUDGET_MS = 70_000;

/** Starts the overlay budget. Call it when the stop's finalization starts, not when the overlay does. */
export function startRecordingOverlayBudget(): Deadline {
return Deadline.fromTimeoutMs(OVERLAY_BUDGET_MS);
}

/**
* The helper's time outside its export: starting up, then verifying the composited output or
* cancelling the export, and exiting.
Expand Down Expand Up @@ -59,13 +66,9 @@ async function exportProcessedVideo(params: {
scriptPath: string;
scriptArgs: string[];
commandDescription: string;
budgetMs: number;
deadline: Deadline;
}): Promise<void> {
const { videoPath, scriptPath, scriptArgs, commandDescription } = params;
const deadline = Deadline.fromTimeoutMs(params.budgetMs);
await waitForStableFile(videoPath);
await waitForPlayableVideo(videoPath);

const { videoPath, scriptPath, scriptArgs, commandDescription, deadline } = params;
const outputPath = temporarySiblingVideoPath(videoPath);
let exportOverran = false;
try {
Expand All @@ -80,7 +83,7 @@ async function exportProcessedVideo(params: {
if (exportMs <= 0) {
throw new AppError(
'COMMAND_FAILED',
`No time was left for the export within the ${params.budgetMs}ms budget`,
`No time was left for the export within the ${OVERLAY_BUDGET_MS}ms budget`,
);
}
try {
Expand All @@ -101,11 +104,18 @@ async function exportProcessedVideo(params: {
exportOverran = deadline.remainingMs() <= HELPER_EXIT_GRACE_MS;
throw error;
}
await waitForPlayableVideo(outputPath);
await waitForPlayableVideo(outputPath, { deadline });
if (deadline.isExpired()) {
exportOverran = true;
throw new AppError(
'COMMAND_FAILED',
'The composited video was not verified within the budget',
);
}
fs.renameSync(outputPath, videoPath);
} catch (error) {
const description = exportOverran
? `${commandDescription}: the export did not finish within its ${params.budgetMs}ms budget; a lower --fps renders faster`
? `${commandDescription}: the export did not finish within its ${OVERLAY_BUDGET_MS}ms budget; a lower --fps renders faster`
: commandDescription;
const cause =
error instanceof AppError
Expand Down Expand Up @@ -144,6 +154,8 @@ export async function overlayRecordingTouches(params: {
/** The caller's `--fps`; the helper renders at most this many frames a second, capped at 30. */
fps?: number;
targetLabel?: string;
/** From `startRecordingOverlayBudget`, taken when the stop's finalization started. */
deadline: Deadline;
}): Promise<void> {
const {
videoPath,
Expand All @@ -162,6 +174,6 @@ export async function overlayRecordingTouches(params: {
...(params.fps === undefined ? [] : ['--max-fps', String(params.fps)]),
],
commandDescription: `Failed to add touch overlays to the ${targetLabel}`,
budgetMs: OVERLAY_BUDGET_MS,
deadline: params.deadline,
Comment thread
thymikee marked this conversation as resolved.
});
}
11 changes: 10 additions & 1 deletion packages/capture-kit/src/recording/video.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ import { runCmd } from '@agent-device/host-kit/command';
import { likelyPlayableWebmContainer } from '../__tests__/test-utils/video-fixtures.ts';
import { mkdtempForTestSync } from '../tmp-dir.fixtures.ts';
import { mp4Atom, mp4MovieHeader } from './mp4.fixtures.ts';
import { isPlayableVideo } from './video.ts';
import { Deadline } from '@agent-device/host-kit/retry';
import { isPlayableVideo, waitForPlayableVideo } from './video.ts';

vi.mock('@agent-device/host-kit/command', () => ({
runCmd: vi.fn(async () => ({ exitCode: 0, stdout: '', stderr: '' })),
Expand Down Expand Up @@ -53,3 +54,11 @@ test('refuses an MP4 container that never declares a movie header', async () =>
await expect(isPlayableVideo(writeFixture('no-moov.mp4', container))).resolves.toBe(false);
expect(runCmd).not.toHaveBeenCalled();
});

test('waiting for a playable video stops polling once its deadline passes', async () => {
const filePath = path.join(mkdtempForTestSync('agent-device-video-wait-'), 'never-playable.mp4');
fs.writeFileSync(filePath, '');
const started = Date.now();
await waitForPlayableVideo(filePath, { pollMs: 60_000, deadline: Deadline.fromTimeoutMs(0) });
expect(Date.now() - started).toBeLessThan(1_000);
});
8 changes: 6 additions & 2 deletions packages/capture-kit/src/recording/video.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import fs from 'node:fs';
import { AppError } from '@agent-device/kernel/errors';
import { runCmd } from '@agent-device/host-kit/command';
import { sleep } from '@agent-device/host-kit/retry';
import { type Deadline, sleep } from '@agent-device/host-kit/retry';
import { buildSwiftToolEnv, compileSwiftSourceText } from './swift-cache.ts';

import { hasPlayableWebmStructure } from './video-webm.ts';
Expand Down Expand Up @@ -119,9 +119,10 @@ function isSwiftVideoValidatorError(error: unknown): boolean {
);
}

/** Polls until the file plays, for `attempts` polls or until `deadline` passes, whichever is first. */
export async function waitForPlayableVideo(
filePath: string,
options: { pollMs?: number; attempts?: number } = {},
options: { pollMs?: number; attempts?: number; deadline?: Deadline } = {},
): Promise<void> {
const pollMs = options.pollMs ?? 150;
const attempts = options.attempts ?? 12;
Expand All @@ -130,6 +131,9 @@ export async function waitForPlayableVideo(
if (await isPlayableVideo(filePath)) {
return;
}
if (options.deadline?.isExpired()) {
return;
}
await sleep(pollMs);
}
}
Expand Down
6 changes: 3 additions & 3 deletions src/__tests__/command-descriptor-timeout-policy.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -332,9 +332,9 @@ test('snapshot uses the standard daemon request timeout with an explicit overrid
});

test('the touch overlay budget leaves record stop room inside its request envelope', () => {
// The recorder stop, the copies and the playability checks around the overlay are outside its
// budget (about 1 s for a 10-minute simulator recording), so the envelope keeps a reserve for
// them beyond it.
// The recorder stop and the copies before finalization, and the playability check after the
// overlay, are outside its budget (about 1 s for a 10-minute simulator recording), so the envelope
// keeps a reserve for them beyond it.
const reserveMs = 20_000;
const envelopeMs = resolveCommandRequestTimeoutMs(resolveCommandTimeoutPolicy('record'), {
positionals: ['stop'],
Expand Down
Loading
Loading