From 473ecff6dd7e4dcaa8b4cbbef1d615396fa17076 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Sat, 24 Oct 2026 12:05:54 -0400 Subject: [PATCH 1/5] feat(producer): fail a render that cannot finish within its maxRenderMs budget --- packages/producer/src/renderRequest.ts | 2 + .../producer/src/server.renderTooSlow.test.ts | 89 +++++++++++++ packages/producer/src/server.ts | 13 ++ .../services/render/renderEtaGuard.test.ts | 120 ++++++++++++++++++ .../src/services/render/renderEtaGuard.ts | 90 +++++++++++++ .../stages/captureStreamingStage.test.ts | 68 ++++++++++ .../src/services/renderOrchestrator.ts | 42 +++++- 7 files changed, 421 insertions(+), 3 deletions(-) create mode 100644 packages/producer/src/server.renderTooSlow.test.ts create mode 100644 packages/producer/src/services/render/renderEtaGuard.test.ts create mode 100644 packages/producer/src/services/render/renderEtaGuard.ts diff --git a/packages/producer/src/renderRequest.ts b/packages/producer/src/renderRequest.ts index 80f4de504bc..3bc695c2ed2 100644 --- a/packages/producer/src/renderRequest.ts +++ b/packages/producer/src/renderRequest.ts @@ -45,6 +45,7 @@ export interface RenderRequestOptions { gifLoop?: number; hlsSegmentSeconds?: number; workers?: number; + maxRenderMs?: number; useGpu?: boolean; debug?: boolean; /** Segmented capture: reuse segments recorded in a matching manifest. */ @@ -172,6 +173,7 @@ function assertRequestOptionScalars(options: Record): void { assertOptionalInteger(options, "gifLoop"); assertOptionalInteger(options, "hlsSegmentSeconds", 1); assertOptionalInteger(options, "workers", 1); + assertOptionalInteger(options, "maxRenderMs"); assertOptionalInteger(options, "crf"); for (const field of ["useGpu", "debug", "outputResolutionAspectAgnostic"] as const) { assertOptionalBoolean(options, field); diff --git a/packages/producer/src/server.renderTooSlow.test.ts b/packages/producer/src/server.renderTooSlow.test.ts new file mode 100644 index 00000000000..03837a0509e --- /dev/null +++ b/packages/producer/src/server.renderTooSlow.test.ts @@ -0,0 +1,89 @@ +import { Hono } from "hono"; +import { describe, expect, it, vi } from "vitest"; + +const createdConfigs = vi.hoisted(() => [] as Record[]); +vi.mock("./services/renderOrchestrator.js", async () => { + const { RenderTooSlowError } = await import("./services/render/renderEtaGuard.js"); + class RenderCancelledError extends Error {} + return { + RenderCancelledError, + createRenderJob: (config: Record) => { + createdConfigs.push(config); + return { + config, + progress: 0, + currentStage: "capture", + framesRendered: 0, + totalFrames: 0, + warnings: [], + }; + }, + executeRenderJob: async () => { + throw new RenderTooSlowError("capture", { + maxRenderMs: 120_000, + elapsedMs: 70_000, + framesCompleted: 127, + totalFrames: 21_540, + msPerFrame: 472, + etaMs: 10_000_000, + }); + }, + }; +}); + +import { createRenderHandlers } from "./server.js"; + +function post(path: string, extra: Record = {}): Promise { + const app = new Hono(); + const handlers = createRenderHandlers({ + getRequestId: () => "too-slow-test", + maxConcurrentRenders: 1, + }); + app.post("/v1/render", handlers.render); + app.post("/v1/render-stream", handlers.renderStream); + return app.request(path, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ html: "", maxRenderMs: 120_000, ...extra }), + }); +} + +describe("server render too slow", () => { + it("forwards maxRenderMs into the render config", async () => { + await post("/v1/render"); + expect(createdConfigs.at(-1)?.maxRenderMs).toBe(120_000); + }); + + it("emits RENDER_TOO_SLOW as non-retryable in blocking JSON", async () => { + const response = await post("/v1/render"); + const body = await response.json(); + expect(response.status).toBe(500); + expect(body).toMatchObject({ + success: false, + errorCode: "RENDER_TOO_SLOW", + retryable: false, + errorOwner: "user", + errorMetadata: { + stage: "capture", + framesCompleted: 127, + totalFrames: 21_540, + msPerFrame: 472, + maxRenderMs: 120_000, + }, + }); + expect(body.error).toMatch(/^render too slow:/); + }); + + it("emits the same event over SSE", async () => { + const body = await (await post("/v1/render-stream")).text(); + expect(body).toContain('"errorCode":"RENDER_TOO_SLOW"'); + expect(body).toContain('"retryable":false'); + expect(body).toContain('"error":"render too slow:'); + }); + + it.each([-1, 1.5, "90000", null])("rejects maxRenderMs=%p with a 400", async (value) => { + const response = await post("/v1/render", { maxRenderMs: value }); + expect(response.status).toBe(400); + expect((await response.json()).error).toContain("maxRenderMs"); + }); +}); diff --git a/packages/producer/src/server.ts b/packages/producer/src/server.ts index 79a8acbe714..22015644618 100644 --- a/packages/producer/src/server.ts +++ b/packages/producer/src/server.ts @@ -87,6 +87,7 @@ interface RenderInput { pageMotionBlur?: RenderConfig["pageMotionBlur"]; outputDynamicRange?: "auto" | "hdr" | "sdr"; workers?: number; + maxRenderMs?: number; crf?: number; videoBitrate?: string; useGpu: boolean; @@ -129,6 +130,7 @@ const SAFE_RENDER_ERROR_CODES = new Set([ "VIDEO_SOURCE_UNRENDERABLE", "VIDEO_EXTRACTION_FAILED", "ENCODER_INTERRUPTED", + "RENDER_TOO_SLOW", ]); /** @@ -223,6 +225,7 @@ export function parseRenderOptions(body: Record): Omit): Omit): string | undefined { const encoder = parseEncoderOptions(body, parseServerFormat(body.format) ?? "mp4"); if (!encoder.ok) return encoder.error; + if ( + body.maxRenderMs !== undefined && + (typeof body.maxRenderMs !== "number" || + !Number.isInteger(body.maxRenderMs) || + body.maxRenderMs < 0) + ) { + return "maxRenderMs must be a non-negative integer number of milliseconds"; + } if (body.variables !== undefined && !isPlainObject(body.variables)) { return 'variables must be a JSON object keyed by variable id (e.g. {"title":"Hello"})'; } diff --git a/packages/producer/src/services/render/renderEtaGuard.test.ts b/packages/producer/src/services/render/renderEtaGuard.test.ts new file mode 100644 index 00000000000..5ac7804a4f4 --- /dev/null +++ b/packages/producer/src/services/render/renderEtaGuard.test.ts @@ -0,0 +1,120 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { RenderTooSlowError, checkRenderEta, startRenderEtaGuard } from "./renderEtaGuard.js"; + +describe("checkRenderEta", () => { + const base = { maxRenderMs: 100_000, elapsedMs: 0, framesCompleted: 0, totalFrames: 100 }; + + it("passes at the 1.2x margin and fails just over it", () => { + expect(checkRenderEta("capture", { ...base, msPerFrame: 1_200 })).toBeUndefined(); + const error = checkRenderEta("capture", { ...base, msPerFrame: 1_201 }); + expect(error).toBeInstanceOf(RenderTooSlowError); + expect(error?.message).toMatch(/^render too slow:/); + expect(error?.publicMetadata).toMatchObject({ stage: "capture", etaMs: 120_100 }); + }); + + it("counts time already spent", () => { + expect( + checkRenderEta("capture", { ...base, elapsedMs: 30_000, msPerFrame: 1_000 }), + ).toBeInstanceOf(RenderTooSlowError); + }); + + it("never fires without a budget or without an estimate", () => { + expect( + checkRenderEta("capture", { ...base, maxRenderMs: undefined, msPerFrame: 1e9 }), + ).toBeUndefined(); + expect(checkRenderEta("capture", { ...base, maxRenderMs: 0, msPerFrame: 1e9 })).toBeUndefined(); + expect(checkRenderEta("capture", { ...base, msPerFrame: 0 })).toBeUndefined(); + }); + + it("matches the two prod renders on the 90 minute budget", () => { + const budget = 90 * 60_000; + // 21540 frames, 4 workers, p95 784 ms: the calibration estimate (70 min) is under budget. + expect( + checkRenderEta("capture_calibration", { + maxRenderMs: budget, + elapsedMs: 0, + framesCompleted: 0, + totalFrames: 21_540, + msPerFrame: 784 / 4, + }), + ).toBeUndefined(); + // 570 frames, 1 worker, p95 15.8 s: 150 min, caught before capture. + expect( + checkRenderEta("capture_calibration", { + maxRenderMs: budget, + elapsedMs: 600_000, + framesCompleted: 0, + totalFrames: 570, + msPerFrame: 15_800 / 1, + }), + ).toBeInstanceOf(RenderTooSlowError); + }); +}); + +describe("startRenderEtaGuard", () => { + beforeEach(() => vi.useFakeTimers()); + afterEach(() => vi.useRealTimers()); + + /** Advances 5 s per step, adding `framesPerStep` frames, until tripped or `steps` run out. */ + function run(opts: { + maxRenderMs?: number; + framesPerStep: number; + steps: number; + total: number; + }) { + const job = { framesRendered: 0 }; + let tripped: RenderTooSlowError | undefined; + const stop = startRenderEtaGuard({ + maxRenderMs: opts.maxRenderMs, + job, + totalFrames: opts.total, + pipelineStartMs: Date.now(), + onTooSlow: (error) => (tripped = error), + }); + let at = 0; + for (let i = 0; i < opts.steps && !tripped; i++) { + job.framesRendered += opts.framesPerStep; + vi.advanceTimersByTime(5_000); + at += 5_000; + } + stop(); + return { tripped, at, job }; + } + + it("fails a crawling render within about two minutes of capture", () => { + const { tripped, at } = run({ + maxRenderMs: 120_000, + framesPerStep: 1, + steps: 100, + total: 1_000, + }); + expect(tripped?.message).toMatch(/^render too slow:/); + expect(at).toBeLessThanOrEqual(120_000); + expect(tripped?.publicMetadata).toMatchObject({ + stage: "capture", + totalFrames: 1_000, + maxRenderMs: 120_000, + }); + }); + + it("leaves a render that will finish inside the budget alone", () => { + expect( + run({ maxRenderMs: 10_000_000, framesPerStep: 100, steps: 100, total: 100_000 }).tripped, + ).toBeUndefined(); + }); + + it("is a no-op without a budget", () => { + expect( + run({ maxRenderMs: undefined, framesPerStep: 1, steps: 200, total: 1_000 }).tripped, + ).toBeUndefined(); + expect( + run({ maxRenderMs: 0, framesPerStep: 1, steps: 200, total: 1_000 }).tripped, + ).toBeUndefined(); + }); + + it("does not decide on zero frames (the stall watchdog owns that)", () => { + expect( + run({ maxRenderMs: 1_000, framesPerStep: 0, steps: 200, total: 1_000 }).tripped, + ).toBeUndefined(); + }); +}); diff --git a/packages/producer/src/services/render/renderEtaGuard.ts b/packages/producer/src/services/render/renderEtaGuard.ts new file mode 100644 index 00000000000..8f245491a2c --- /dev/null +++ b/packages/producer/src/services/render/renderEtaGuard.ts @@ -0,0 +1,90 @@ +export const RENDER_TOO_SLOW = "RENDER_TOO_SLOW" as const; + +/** Fail only when the ETA clears the budget by this factor, so borderline renders survive. */ +const ETA_MARGIN = 1.2; +/** The measured rate needs this much capture time and frames; the first frame's warm-up is excluded. */ +const MIN_SAMPLE_MS = 60_000; +const MIN_SAMPLE_FRAMES = 3; +const POLL_MS = 5_000; + +interface RenderEtaInput { + /** Absent or 0 means no guard. */ + maxRenderMs?: number; + /** Wall time since the render started, so time already spent counts. */ + elapsedMs: number; + framesCompleted: number; + totalFrames: number; + /** Wall ms per frame across all workers. */ + msPerFrame: number; +} + +export class RenderTooSlowError extends Error { + readonly code = RENDER_TOO_SLOW; + readonly owner = "user" as const; + readonly retryable = false as const; + readonly publicMetadata: Readonly>; + + constructor(stage: string, input: Required & { etaMs: number }) { + const { framesCompleted, totalFrames, msPerFrame, etaMs, maxRenderMs } = input; + super( + `render too slow: stage ${stage}, ${framesCompleted}/${totalFrames} frames, ` + + `${Math.round(msPerFrame)} ms/frame, ETA ${etaMs} ms exceeds maxRenderMs ${maxRenderMs}`, + ); + this.name = "RenderTooSlowError"; + this.publicMetadata = { stage, framesCompleted, totalFrames, msPerFrame, etaMs, maxRenderMs }; + } +} + +/** The one ETA decision, shared by the post-calibration and the measured-rate checks. */ +export function checkRenderEta( + stage: string, + input: RenderEtaInput, +): RenderTooSlowError | undefined { + const { maxRenderMs, elapsedMs, framesCompleted, totalFrames, msPerFrame } = input; + if (!maxRenderMs || maxRenderMs <= 0 || !(msPerFrame > 0)) return undefined; + const etaMs = Math.round(elapsedMs + (totalFrames - framesCompleted) * msPerFrame); + if (etaMs <= maxRenderMs * ETA_MARGIN) return undefined; + return new RenderTooSlowError(stage, { ...input, maxRenderMs, etaMs }); +} + +interface RenderEtaGuardOptions { + maxRenderMs?: number; + job: { framesRendered?: number }; + totalFrames: number; + pipelineStartMs: number; + onTooSlow: (error: RenderTooSlowError) => void; + now?: () => number; + pollMs?: number; +} + +/** Polls `job.framesRendered`, which every capture path updates, and trips once. Returns stop(). */ +export function startRenderEtaGuard(options: RenderEtaGuardOptions): () => void { + const { maxRenderMs, job, totalFrames, pipelineStartMs, onTooSlow, now = Date.now } = options; + if (!maxRenderMs || maxRenderMs <= 0) return () => {}; + let base: { at: number; frames: number } | undefined; + let lastFrames = 0; + const timer = setInterval(() => { + const done = job.framesRendered ?? 0; + const at = now(); + // A capture retry resets the counter: sample the new attempt from scratch. + if (done === 0 || done < lastFrames) base = undefined; + lastFrames = done; + if (done === 0) return; + base ??= { at, frames: done }; + const sampleMs = at - base.at; + const sampleFrames = done - base.frames; + if (sampleMs < MIN_SAMPLE_MS || sampleFrames < MIN_SAMPLE_FRAMES) return; + const error = checkRenderEta("capture", { + maxRenderMs, + elapsedMs: at - pipelineStartMs, + framesCompleted: done, + totalFrames, + msPerFrame: sampleMs / sampleFrames, + }); + if (!error) return; + clearInterval(timer); + onTooSlow(error); + }, options.pollMs ?? POLL_MS); + timer.unref?.(); + return () => clearInterval(timer); +} diff --git a/packages/producer/src/services/render/stages/captureStreamingStage.test.ts b/packages/producer/src/services/render/stages/captureStreamingStage.test.ts index 9f16bf5e6bb..22442564b1c 100644 --- a/packages/producer/src/services/render/stages/captureStreamingStage.test.ts +++ b/packages/producer/src/services/render/stages/captureStreamingStage.test.ts @@ -36,6 +36,10 @@ let workerFailurePageErrors: string[] = []; // Parallel capture that reports this many frames, advancing the clock by parallelFrameMs each. let parallelFrames = 0; let parallelFrameMs = 0; +// Slow parallel capture on a virtual clock: one frame per 2 s, `slowTotal` frames, stops on abort. +let slowParallel = false; +let slowTotal = 0; +let virtualNow = 0; let injectedWorkerFailure: Error | null = null; const reorderAbortCalls: unknown[] = []; let hangSequentialUntilStall = false; @@ -154,6 +158,20 @@ mock.module("@hyperframes/engine", () => ({ signal?.addEventListener("abort", fail, { once: true }); }); } + if (slowParallel) { + for (let frame = 1; frame <= slowTotal; frame++) { + if (signal?.aborted) throw new Error("[Parallel] Capture failed: aborted"); + virtualNow += 2_000; + onProgress?.({ + totalFrames: slowTotal, + capturedFrames: frame, + activeWorkers: 2, + workerProgress: new Map(), + }); + await new Promise((resolve) => setTimeout(resolve, 1)); + } + return []; + } if (parallelFrames > 0) { const report = (capturedFrames: number, phase?: string) => onProgress?.({ @@ -454,6 +472,56 @@ describe("runCaptureStreamingStage", () => { expect(stages.at(-1)).toBe("Encoding frame 40/40"); }); + async function runSlowParallel(total: number, maxRenderMs: number | undefined) { + slowParallel = true; + slowTotal = total; + virtualNow = 0; + const controller = new AbortController(); + const { runCaptureStreamingStage } = await import("./captureStreamingStage.js"); + const { startRenderEtaGuard } = await import("../renderEtaGuard.js"); + const baseInput = createInput({ forceScreenshot: false, ffmpegStreamingTimeout: 3_600_000 }); + const input = { + ...baseInput, + totalFrames: total, + plan: { ...baseInput.plan, workerCount: 2, forceParallelStream: true }, + abortSignal: controller.signal, + }; + let tooSlow: Error | undefined; + const stop = startRenderEtaGuard({ + maxRenderMs, + job: input.job, + totalFrames: total, + pipelineStartMs: 0, + now: () => virtualNow, + pollMs: 1, + onTooSlow: (error) => { + tooSlow = error; + controller.abort(error); + }, + }); + try { + await runCaptureStreamingStage(input); + return { tooSlow, completed: true }; + } catch { + return { tooSlow, completed: false }; + } finally { + stop(); + slowParallel = false; + } + } + + it("stops a crawling parallel capture through the ETA guard", async () => { + const { tooSlow, completed } = await runSlowParallel(1_000, 120_000); + expect(completed).toBe(false); + expect(tooSlow?.message).toMatch(/^render too slow: stage capture/); + }); + + it("does not stop the same slow capture without maxRenderMs", async () => { + const { tooSlow, completed } = await runSlowParallel(60, undefined); + expect(tooSlow).toBeUndefined(); + expect(completed).toBe(true); + }); + it("does not flood the callback when frames arrive faster than the report interval", async () => { const stages = await streamParallelFrames(300, 1); const frames = stages.filter((stage) => stage.startsWith("Streaming frame")); diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index e43e9d05f66..cb5aec1cba4 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -154,6 +154,11 @@ import { type CaptureRouting, } from "./render/capturePlan.js"; import { runCaptureSegmentedStage } from "./render/stages/captureSegmentedStage.js"; +import { + RenderTooSlowError, + checkRenderEta, + startRenderEtaGuard, +} from "./render/renderEtaGuard.js"; import { resolveSegmentFrames } from "./render/segmentPlan.js"; import { resolveSegmentBrowserRecycle } from "./render/segmentRecycle.js"; import { @@ -400,6 +405,8 @@ export interface RenderConfig { motionBlur?: MotionBlurOptions; pageMotionBlur?: "off"; workers?: number; + /** Wall-time budget in ms for the whole render. Absent or 0 disables the ETA guard. */ + maxRenderMs?: number; useGpu?: boolean; debug?: boolean; /** Strict rejects correctness warnings; best-effort returns a qualified outcome. */ @@ -2864,7 +2871,11 @@ async function executeRenderPipeline(input: { const log = execution.logger; const eventPublisher = execution.events; const onProgress = execution.onProgress; - const executionSignal = execution.signal; + const tooSlow = new AbortController(); + let tooSlowError: RenderTooSlowError | undefined; + const executionSignal = job.config.maxRenderMs + ? AbortSignal.any([tooSlow.signal, ...(execution.signal ? [execution.signal] : [])]) + : execution.signal; let fileServer: FileServerHandle | null = null; let probeSession: CaptureSession | null = null; let lastBrowserConsole: string[] = []; @@ -4025,6 +4036,30 @@ async function executeRenderPipeline(input: { deParallelRouter: deParallelRouter ?? "none", }); + // No calibration estimate (explicit workers, short or forced-screenshot renders) skips the + // pre-capture check; the measured-rate guard below still covers capture. + const calibratedTooSlow = checkRenderEta("capture_calibration", { + maxRenderMs: job.config.maxRenderMs, + elapsedMs: Date.now() - pipelineStart, + framesCompleted: 0, + totalFrames, + msPerFrame: (captureCalibration?.estimate.p95Ms ?? 0) / workerCount, + }); + if (calibratedTooSlow) throw calibratedTooSlow; + execution.defer( + "stop render ETA guard", + startRenderEtaGuard({ + maxRenderMs: job.config.maxRenderMs, + job, + totalFrames, + pipelineStartMs: pipelineStart, + onTooSlow: (error) => { + tooSlowError = error; + tooSlow.abort(error); + }, + }), + ); + // Streaming encode pipes captured frames through ffmpeg's stdin to produce // a single video file. Keep the default enabled for sequential capture, but // let auto-parallel renders use disk frames: the current ordered streaming @@ -5223,8 +5258,9 @@ async function executeRenderPipeline(input: { job.outputPath = outputPath; updateJobStatus(job, "complete", "Render complete", 100, onProgress); await eventPublisher.flush(); - } catch (error) { - if (error instanceof RenderCancelledError || executionSignal?.aborted) { + } catch (caught) { + const error = tooSlowError ?? caught; + if (!tooSlowError && (error instanceof RenderCancelledError || executionSignal?.aborted)) { job.error = error instanceof Error ? error.message : "render_cancelled"; updateJobStatus(job, "cancelled", "Render cancelled", job.progress, onProgress); await eventPublisher.flush(); From 029aa3196e8912a6f8f148963087d1def0e7f4e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Sat, 24 Oct 2026 12:53:38 -0400 Subject: [PATCH 2/5] fix(producer): stop the ETA guard once capture completes From caa5b3dc24846430f6855f28cec036b0d9476a49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Sat, 24 Oct 2026 12:54:28 -0400 Subject: [PATCH 3/5] fix(producer): stop the ETA guard once capture completes --- .../services/render/renderEtaGuard.test.ts | 22 +++++++++++++++++++ .../src/services/render/renderEtaGuard.ts | 7 ++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/packages/producer/src/services/render/renderEtaGuard.test.ts b/packages/producer/src/services/render/renderEtaGuard.test.ts index 5ac7804a4f4..a6b13f5cf4f 100644 --- a/packages/producer/src/services/render/renderEtaGuard.test.ts +++ b/packages/producer/src/services/render/renderEtaGuard.test.ts @@ -103,6 +103,28 @@ describe("startRenderEtaGuard", () => { ).toBeUndefined(); }); + it("is never tripped by the encode and assemble time after capture completes", () => { + const job = { framesRendered: 0 }; + const onTooSlow = vi.fn(); + const stop = startRenderEtaGuard({ + maxRenderMs: 120_000, + job, + totalFrames: 100, + pipelineStartMs: Date.now(), + onTooSlow, + }); + // Capture finishes inside the budget... + for (let i = 0; i < 10; i++) { + job.framesRendered += 10; + vi.advanceTimersByTime(5_000); + } + // ...then a mux that runs for an hour with the counter parked at the total. + vi.advanceTimersByTime(60 * 60_000); + stop(); + expect(onTooSlow).not.toHaveBeenCalled(); + expect(vi.getTimerCount()).toBe(0); + }); + it("is a no-op without a budget", () => { expect( run({ maxRenderMs: undefined, framesPerStep: 1, steps: 200, total: 1_000 }).tripped, diff --git a/packages/producer/src/services/render/renderEtaGuard.ts b/packages/producer/src/services/render/renderEtaGuard.ts index 8f245491a2c..a9a83a039b6 100644 --- a/packages/producer/src/services/render/renderEtaGuard.ts +++ b/packages/producer/src/services/render/renderEtaGuard.ts @@ -63,6 +63,7 @@ export function startRenderEtaGuard(options: RenderEtaGuardOptions): () => void if (!maxRenderMs || maxRenderMs <= 0) return () => {}; let base: { at: number; frames: number } | undefined; let lastFrames = 0; + const stop = () => clearInterval(timer); const timer = setInterval(() => { const done = job.framesRendered ?? 0; const at = now(); @@ -70,6 +71,8 @@ export function startRenderEtaGuard(options: RenderEtaGuardOptions): () => void if (done === 0 || done < lastFrames) base = undefined; lastFrames = done; if (done === 0) return; + // Capture is complete: encode and assemble are not frame-driven, so the guard has nothing left to judge. + if (done >= totalFrames) return stop(); base ??= { at, frames: done }; const sampleMs = at - base.at; const sampleFrames = done - base.frames; @@ -82,9 +85,9 @@ export function startRenderEtaGuard(options: RenderEtaGuardOptions): () => void msPerFrame: sampleMs / sampleFrames, }); if (!error) return; - clearInterval(timer); + stop(); onTooSlow(error); }, options.pollMs ?? POLL_MS); timer.unref?.(); - return () => clearInterval(timer); + return stop; } From 732a4ad352b1eb9146d5219c94d117d537486dfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Sat, 24 Oct 2026 13:36:16 -0400 Subject: [PATCH 4/5] fix(producer): clear the dead-code and complexity findings in the ETA guard --- .../services/render/renderEtaGuard.test.ts | 6 +++ .../src/services/render/renderEtaGuard.ts | 37 ++++++++++++------- .../stages/captureStreamingStage.test.ts | 33 ++++++++++------- 3 files changed, 49 insertions(+), 27 deletions(-) diff --git a/packages/producer/src/services/render/renderEtaGuard.test.ts b/packages/producer/src/services/render/renderEtaGuard.test.ts index a6b13f5cf4f..3b3a49fe3be 100644 --- a/packages/producer/src/services/render/renderEtaGuard.test.ts +++ b/packages/producer/src/services/render/renderEtaGuard.test.ts @@ -1,6 +1,8 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { RenderTooSlowError, checkRenderEta, startRenderEtaGuard } from "./renderEtaGuard.js"; +const RENDER_TOO_SLOW = "RENDER_TOO_SLOW"; + describe("checkRenderEta", () => { const base = { maxRenderMs: 100_000, elapsedMs: 0, framesCompleted: 0, totalFrames: 100 }; @@ -10,6 +12,10 @@ describe("checkRenderEta", () => { expect(error).toBeInstanceOf(RenderTooSlowError); expect(error?.message).toMatch(/^render too slow:/); expect(error?.publicMetadata).toMatchObject({ stage: "capture", etaMs: 120_100 }); + // The producer server reads these off the error to build the wire envelope. + expect(error?.code).toBe(RENDER_TOO_SLOW); + expect(error?.owner).toBe("user"); + expect(error?.retryable).toBe(false); }); it("counts time already spent", () => { diff --git a/packages/producer/src/services/render/renderEtaGuard.ts b/packages/producer/src/services/render/renderEtaGuard.ts index a9a83a039b6..d9130b5e2ee 100644 --- a/packages/producer/src/services/render/renderEtaGuard.ts +++ b/packages/producer/src/services/render/renderEtaGuard.ts @@ -1,4 +1,4 @@ -export const RENDER_TOO_SLOW = "RENDER_TOO_SLOW" as const; +const RENDER_TOO_SLOW = "RENDER_TOO_SLOW" as const; /** Fail only when the ETA clears the budget by this factor, so borderline renders survive. */ const ETA_MARGIN = 1.2; @@ -57,32 +57,43 @@ interface RenderEtaGuardOptions { pollMs?: number; } +/** Measured wall ms per frame once enough capture time and frames are in; a counter reset restarts the sample. */ +function createRateSampler() { + let base: { at: number; frames: number } | undefined; + let lastFrames = 0; + return { + observe(done: number, at: number): number | undefined { + if (done === 0 || done < lastFrames) base = undefined; + lastFrames = done; + if (done === 0) return undefined; + base ??= { at, frames: done }; + const sampleMs = at - base.at; + const sampleFrames = done - base.frames; + if (sampleMs < MIN_SAMPLE_MS || sampleFrames < MIN_SAMPLE_FRAMES) return undefined; + return sampleMs / sampleFrames; + }, + }; +} + /** Polls `job.framesRendered`, which every capture path updates, and trips once. Returns stop(). */ export function startRenderEtaGuard(options: RenderEtaGuardOptions): () => void { const { maxRenderMs, job, totalFrames, pipelineStartMs, onTooSlow, now = Date.now } = options; if (!maxRenderMs || maxRenderMs <= 0) return () => {}; - let base: { at: number; frames: number } | undefined; - let lastFrames = 0; + const sampler = createRateSampler(); const stop = () => clearInterval(timer); const timer = setInterval(() => { const done = job.framesRendered ?? 0; - const at = now(); - // A capture retry resets the counter: sample the new attempt from scratch. - if (done === 0 || done < lastFrames) base = undefined; - lastFrames = done; - if (done === 0) return; // Capture is complete: encode and assemble are not frame-driven, so the guard has nothing left to judge. if (done >= totalFrames) return stop(); - base ??= { at, frames: done }; - const sampleMs = at - base.at; - const sampleFrames = done - base.frames; - if (sampleMs < MIN_SAMPLE_MS || sampleFrames < MIN_SAMPLE_FRAMES) return; + const at = now(); + const msPerFrame = sampler.observe(done, at); + if (msPerFrame === undefined) return; const error = checkRenderEta("capture", { maxRenderMs, elapsedMs: at - pipelineStartMs, framesCompleted: done, totalFrames, - msPerFrame: sampleMs / sampleFrames, + msPerFrame, }); if (!error) return; stop(); diff --git a/packages/producer/src/services/render/stages/captureStreamingStage.test.ts b/packages/producer/src/services/render/stages/captureStreamingStage.test.ts index 22442564b1c..378f73781b2 100644 --- a/packages/producer/src/services/render/stages/captureStreamingStage.test.ts +++ b/packages/producer/src/services/render/stages/captureStreamingStage.test.ts @@ -51,6 +51,24 @@ const browserConsoleBuffer = ["[FrameCapture:ERROR] page.goto failed"]; const closeCaptureSession = mock(async () => {}); class DrawElementVerificationError extends Error {} +async function replaySlowCapture( + signal: AbortSignal | undefined, + onProgress: ((progress: unknown) => void) | undefined, +): Promise { + for (let frame = 1; frame <= slowTotal; frame++) { + if (signal?.aborted) throw new Error("[Parallel] Capture failed: aborted"); + virtualNow += 2_000; + onProgress?.({ + totalFrames: slowTotal, + capturedFrames: frame, + activeWorkers: 2, + workerProgress: new Map(), + }); + await new Promise((resolve) => setTimeout(resolve, 1)); + } + return []; +} + mock.module("@hyperframes/engine", () => ({ calculateOptimalWorkers: () => 1, convertTransfer: () => {}, @@ -158,20 +176,7 @@ mock.module("@hyperframes/engine", () => ({ signal?.addEventListener("abort", fail, { once: true }); }); } - if (slowParallel) { - for (let frame = 1; frame <= slowTotal; frame++) { - if (signal?.aborted) throw new Error("[Parallel] Capture failed: aborted"); - virtualNow += 2_000; - onProgress?.({ - totalFrames: slowTotal, - capturedFrames: frame, - activeWorkers: 2, - workerProgress: new Map(), - }); - await new Promise((resolve) => setTimeout(resolve, 1)); - } - return []; - } + if (slowParallel) return replaySlowCapture(signal, onProgress); if (parallelFrames > 0) { const report = (capturedFrames: number, phase?: string) => onProgress?.({ From 937aa75eff7b47e4ba80ab9ea935bf9db6db2e9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Sat, 24 Oct 2026 13:48:28 -0400 Subject: [PATCH 5/5] fix(producer): mark the wire-envelope fields the server reads by name --- packages/producer/src/services/render/renderEtaGuard.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/producer/src/services/render/renderEtaGuard.ts b/packages/producer/src/services/render/renderEtaGuard.ts index d9130b5e2ee..efbc4690517 100644 --- a/packages/producer/src/services/render/renderEtaGuard.ts +++ b/packages/producer/src/services/render/renderEtaGuard.ts @@ -20,7 +20,10 @@ interface RenderEtaInput { export class RenderTooSlowError extends Error { readonly code = RENDER_TOO_SLOW; + // The producer server reads these off the error for the wire envelope. + // fallow-ignore-next-line unused-class-member readonly owner = "user" as const; + // fallow-ignore-next-line unused-class-member readonly retryable = false as const; readonly publicMetadata: Readonly>;