From 5127962e643631789c203c94f9f30304e3d851f9 Mon Sep 17 00:00:00 2001 From: "homeboy-ci[bot]" <266378653+homeboy-ci[bot]@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:34:11 +0000 Subject: [PATCH] fix(browser-actions): record video at the CSS viewport size instead of padding a scaled frame (#2552) --- README.md | 2 +- .../src/browser-actions-runner.ts | 10 +++---- ...wser-actions-video-capture.browser.test.ts | 30 +++++++++++++++---- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 279d1f44..7df7d4e6 100644 --- a/README.md +++ b/README.md @@ -766,7 +766,7 @@ For synthetic web performance investigations, enable `capture=performance` and u Action journeys and adaptive exploration can declare their browser context with `browser-environment-json=` (inline or `@`) or the compatible `device`, `viewport`, `device-scale-factor`, `is-mobile`, `has-touch`, `user-agent`, `permissions`, `locale`, `timezone`, and `geolocation-*` arguments. These controls are applied when the Playwright context is created, so mobile/touch emulation and granted, denied, or prompt geolocation remain active for the full journey. Action summaries keep requested configuration, provider-resolved configuration, and browser-observed state separate, with explicit unsupported and inconclusive dimensions. `wordpress.browser-scenario` accepts the same arguments or an `environment` object inside `scenario-json`; probe collection and authored actions share one owned context/page, while multi-actor scenarios create one isolated configured context per actor. -For `wordpress.browser-actions capture=video`, `video-size=x` sets the WebM recording dimensions without changing its format or encoding. When omitted, recordings retain Playwright's default size unless `device-scale-factor` is greater than 1, in which case the viewport dimensions are multiplied by that scale. Requested dimensions are capped at 8,294,400 pixels total with aspect ratio preserved. Video step records include `videoOffsetMs: { startMs, endMs }` relative to recording start; named `screenshot` steps and steps with a non-empty free-form `marker` label appear in the `video.markers` timeline in `action-summary.json`. +For `wordpress.browser-actions capture=video`, `video-size=x` sets the WebM recording dimensions without changing its format or encoding. When omitted, recordings use the CSS viewport size. Playwright's recorder captures at CSS pixels and pads any larger frame, so `video-size` can only scale a recording down; upscale with an encoder afterwards when a higher-resolution output is needed. Requested dimensions are capped at 8,294,400 pixels total with aspect ratio preserved. Video step records include `videoOffsetMs: { startMs, endMs }` relative to recording start; named `screenshot` steps and steps with a non-empty free-form `marker` label appear in the `video.markers` timeline in `action-summary.json`. Step kinds: `navigate` (`url`, optional `waitFor=domcontentloaded|load|networkidle`), `click`/`hover` (`selector` or `text`), `fill`/`type` (`selector`, `value`), `press` (`key`, optional `selector`), `drag` (`from` selector, `to` as `{ "selector": ... }` or `{ "x": n, "y": n }`), `select` (`selector`, `value` or `values`), `scroll` (`selector`, `position=top|bottom`, or `by: {x,y}`), `waitFor` (`selector` or `waitFor=domcontentloaded|load|networkidle|duration|selector:`), `evaluate` (`expression`, optional `assert` to deep-equal the result), `expect` (`selector`, optional `state=visible|hidden|attached|detached|enabled|disabled|checked|unchecked|editable`), `annotate` (highlight, spotlight, arrow, anchored label, fixed caption, or clear), and `screenshot` (optional `name` and boolean `fullPage`; defaults to `true`, except with `capture=video` when it defaults to `false` to keep viewport-sized video frames. Set `fullPage: true` explicitly to request a full-document screenshot while recording). Every step may set its own `timeout=s`; the command also accepts a global `step-timeout=s` (per step) and `timeout=s` (total-script budget). Both are bounded and deterministic — the run stops cleanly on the first failing step, with no silent partial success. diff --git a/packages/runtime-playground/src/browser-actions-runner.ts b/packages/runtime-playground/src/browser-actions-runner.ts index 7e433b2c..ec394adf 100644 --- a/packages/runtime-playground/src/browser-actions-runner.ts +++ b/packages/runtime-playground/src/browser-actions-runner.ts @@ -221,7 +221,7 @@ export async function runBrowserActionsCommand({ ...topology.contextOptions(), ...(storageStateImport ? { storageState: storageStateImport.storageState } : {}), ...(runPlan.transportFaults ? { serviceWorkers: "block" as const } : {}), - ...(capture.has("video") ? { recordVideo: { dir: videoStagingDirectory, ...(browserVideoSize(runPlan.videoSize, requestedEnvironment.viewport, requestedEnvironment.deviceScaleFactor) ? { size: browserVideoSize(runPlan.videoSize, requestedEnvironment.viewport, requestedEnvironment.deviceScaleFactor) } : {}) } } : {}), + ...(capture.has("video") ? { recordVideo: { dir: videoStagingDirectory, ...(browserVideoSize(runPlan.videoSize, requestedEnvironment.viewport) ? { size: browserVideoSize(runPlan.videoSize, requestedEnvironment.viewport) } : {}) } } : {}), }, }) : undefined) const context = environmentRuntime?.context ?? null @@ -233,7 +233,7 @@ export async function runBrowserActionsCommand({ if (capture.has("video")) { videoRecording = page.video() videoStartedAt = videoRecordingOrigin - videoDimensions = browserVideoSize(runPlan.videoSize, requestedEnvironment.viewport, requestedEnvironment.deviceScaleFactor) ?? requestedEnvironment.viewport ?? { width: 1280, height: 720 } + videoDimensions = browserVideoSize(runPlan.videoSize, requestedEnvironment.viewport) ?? requestedEnvironment.viewport ?? { width: 1280, height: 720 } } navigationTracker = trackBrowserNavigation(page) if (onProgress) { @@ -891,11 +891,11 @@ function browserVideoSizeArg(args: string[]): { width: number; height: number } const width = Number(match[1]) const height = Number(match[2]) if (!Number.isSafeInteger(width) || !Number.isSafeInteger(height) || width < 1 || height < 1) throw new Error("wordpress.browser-actions video-size dimensions must be positive integers") - return browserVideoSize({ width, height }, undefined, 1) + return browserVideoSize({ width, height }, undefined) } -function browserVideoSize(explicit: { width: number; height: number } | undefined, viewport: { width: number; height: number } | undefined, scale = 1): { width: number; height: number } | undefined { - const requested = explicit ?? (viewport && scale > 1 ? { width: Math.round(viewport.width * scale), height: Math.round(viewport.height * scale) } : undefined) +function browserVideoSize(explicit: { width: number; height: number } | undefined, viewport: { width: number; height: number } | undefined): { width: number; height: number } | undefined { + const requested = explicit ?? viewport if (!requested) return undefined const size = requested const factor = Math.min(1, Math.sqrt(BROWSER_VIDEO_MAX_PIXELS / (size.width * size.height))) diff --git a/tests/browser-actions-video-capture.browser.test.ts b/tests/browser-actions-video-capture.browser.test.ts index 6baeddb4..84efcef2 100644 --- a/tests/browser-actions-video-capture.browser.test.ts +++ b/tests/browser-actions-video-capture.browser.test.ts @@ -4,6 +4,8 @@ import { createServer } from "node:http" import { tmpdir } from "node:os" import { join } from "node:path" import test from "node:test" +import { spawnSync } from "node:child_process" +import { existsSync } from "node:fs" import { runBrowserActionsCommand } from "../packages/runtime-playground/src/browser-actions-runner.js" import { wordpressRuntimeSpec } from "../scripts/test-kit.js" @@ -83,8 +85,8 @@ test("screenshot steps during video capture default to viewport-sized captures", } }) -test("browser actions scales video dimensions from a high-density viewport by default", async () => { - const fixture = await pageFixture() +test("browser actions records high-density viewport video without padding", async () => { + const fixture = await pageFixture(false, true) const artifactRoot = await mkdtemp(join(tmpdir(), "wp-codebox-browser-video-")) try { await runBrowserActionsCommand({ @@ -104,7 +106,14 @@ test("browser actions scales video dimensions from a high-density viewport by de }) const summary = JSON.parse(await readFile(join(artifactRoot, "files/browser/action-summary.json"), "utf8")) - assert.deepEqual({ width: summary.video.width, height: summary.video.height }, { width: 1075, height: 2330 }) + assert.deepEqual({ width: summary.video.width, height: summary.video.height }, { width: 430, height: 932 }) + + const pixel = bottomRightVideoPixel(join(artifactRoot, "files/browser/video.webm")) + if (pixel) { + assert(pixel[2]! > 200 && pixel[0]! < 50, `bottom-right pixel should be blue page content, got ${pixel}`) + } else { + test.diagnostic?.("ffmpeg unavailable; skipped bottom-right pixel padding check") + } } finally { await rm(artifactRoot, { recursive: true, force: true }) await fixture.close() @@ -138,10 +147,10 @@ test("browser actions rejects an unsupported capture value", async () => { } }) -async function pageFixture(tall = false) { +async function pageFixture(tall = false, fullBleed = false) { const httpServer = createServer((_request, response) => { response.setHeader("content-type", "text/html") - response.end(`video fixture
ready
`) + response.end(`video fixture
ready
`) }) await new Promise((resolve) => httpServer.listen(0, "127.0.0.1", resolve)) const address = httpServer.address() @@ -157,3 +166,14 @@ async function pageFixture(tall = false) { close: () => new Promise((resolve, reject) => httpServer.close((error) => error ? reject(error) : resolve())), } } + +function bottomRightVideoPixel(videoPath: string): number[] | undefined { + const browsers = process.env.PLAYWRIGHT_BROWSERS_PATH ?? join(process.env.HOME ?? "", ".cache/ms-playwright") + const candidates = [process.env.FFMPEG_PATH, "ffmpeg", join(browsers, "ffmpeg-1011", "ffmpeg-linux")].filter((value): value is string => Boolean(value)) + for (const ffmpeg of candidates) { + if (ffmpeg !== "ffmpeg" && !existsSync(ffmpeg)) continue + const result = spawnSync(ffmpeg, ["-v", "error", "-ss", "0.4", "-i", videoPath, "-frames:v", "1", "-vf", "crop=1:1:iw-2:ih-2", "-f", "rawvideo", "-pix_fmt", "rgb24", "-"], { timeout: 30_000 }) + if (result.status === 0 && result.stdout.length >= 3) return [...result.stdout.subarray(0, 3)] + } + return undefined +}