feat(producer): fail a render that cannot finish within its maxRenderMs budget

packages/producer/src/renderRequest.ts

@@ -45,6 +45,7 @@ export interface RenderRequestOptions {
   gifLoop?: number;
   hlsSegmentSeconds?: number;
   workers?: number;
+  maxRenderMs?: number;
   useGpu?: boolean;
   debug?: boolean;
   /** Segmented capture: reuse segments recorded in a matching manifest. */
@@ -172,6 +173,7 @@ function assertRequestOptionScalars(options: Record<string, unknown>): void {
   assertOptionalInteger(options, "gifLoop");
   assertOptionalInteger(options, "hlsSegmentSeconds", 1);
   assertOptionalInteger(options, "workers", 1);
+  assertOptionalInteger(options, "maxRenderMs");
   assertOptionalInteger(options, "crf");
   for (const field of ["useGpu", "debug", "outputResolutionAspectAgnostic"] as const) {
     assertOptionalBoolean(options, field);

packages/producer/src/server.renderTooSlow.test.ts

@@ -0,0 +1,89 @@
+import { Hono } from "hono";
+import { describe, expect, it, vi } from "vitest";
+
+const createdConfigs = vi.hoisted(() => [] as Record<string, unknown>[]);
+vi.mock("./services/renderOrchestrator.js", async () => {
+  const { RenderTooSlowError } = await import("./services/render/renderEtaGuard.js");
+  class RenderCancelledError extends Error {}
+  return {
+    RenderCancelledError,
+    createRenderJob: (config: Record<string, unknown>) => {
+      createdConfigs.push(config);
+      return {
+        config,
+        progress: 0,
+        currentStage: "capture",
+        framesRendered: 0,
+        totalFrames: 0,
+        warnings: [],
+      };
+    },
+    executeRenderJob: async () => {
+      throw new RenderTooSlowError("capture", {
+        maxRenderMs: 120_000,
+        elapsedMs: 70_000,
+        framesCompleted: 127,
+        totalFrames: 21_540,
+        msPerFrame: 472,
+        etaMs: 10_000_000,
+      });
+    },
+  };
+});
+
+import { createRenderHandlers } from "./server.js";
+
+function post(path: string, extra: Record<string, unknown> = {}): Promise<Response> {
+  const app = new Hono();
+  const handlers = createRenderHandlers({
+    getRequestId: () => "too-slow-test",
+    maxConcurrentRenders: 1,
+  });
+  app.post("/v1/render", handlers.render);
+  app.post("/v1/render-stream", handlers.renderStream);
+  return app.request(path, {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify({ html: "<html><body></body></html>", maxRenderMs: 120_000, ...extra }),
+  });
+}
+
+describe("server render too slow", () => {
+  it("forwards maxRenderMs into the render config", async () => {
+    await post("/v1/render");
+    expect(createdConfigs.at(-1)?.maxRenderMs).toBe(120_000);
+  });
+
+  it("emits RENDER_TOO_SLOW as non-retryable in blocking JSON", async () => {
+    const response = await post("/v1/render");
+    const body = await response.json();
+    expect(response.status).toBe(500);
+    expect(body).toMatchObject({
+      success: false,
+      errorCode: "RENDER_TOO_SLOW",
+      retryable: false,
+      errorOwner: "user",
+      errorMetadata: {
+        stage: "capture",
+        framesCompleted: 127,
+        totalFrames: 21_540,
+        msPerFrame: 472,
+        maxRenderMs: 120_000,
+      },
+    });
+    expect(body.error).toMatch(/^render too slow:/);
+  });
+
+  it("emits the same event over SSE", async () => {
+    const body = await (await post("/v1/render-stream")).text();
+    expect(body).toContain('"errorCode":"RENDER_TOO_SLOW"');
+    expect(body).toContain('"retryable":false');
+    expect(body).toContain('"error":"render too slow:');
+  });
+
+  it.each([-1, 1.5, "90000", null])("rejects maxRenderMs=%p with a 400", async (value) => {
+    const response = await post("/v1/render", { maxRenderMs: value });
+    expect(response.status).toBe(400);
+    expect((await response.json()).error).toContain("maxRenderMs");
+  });
+});

packages/producer/src/server.ts

@@ -87,6 +87,7 @@ interface RenderInput {
   pageMotionBlur?: RenderConfig["pageMotionBlur"];
   outputDynamicRange?: "auto" | "hdr" | "sdr";
   workers?: number;
+  maxRenderMs?: number;
   crf?: number;
   videoBitrate?: string;
   useGpu: boolean;
@@ -129,6 +130,7 @@ const SAFE_RENDER_ERROR_CODES = new Set<string>([
   "VIDEO_SOURCE_UNRENDERABLE",
   "VIDEO_EXTRACTION_FAILED",
   "ENCODER_INTERRUPTED",
+  "RENDER_TOO_SLOW",
 ]);
 
 /**
@@ -223,6 +225,7 @@ export function parseRenderOptions(body: Record<string, unknown>): Omit<RenderIn
   const fps = parseServerFps(body.fps);
   const quality = parseServerQuality(body.quality);
   const workers = typeof body.workers === "number" ? body.workers : undefined;
+  const maxRenderMs = typeof body.maxRenderMs === "number" ? body.maxRenderMs : undefined;
   const useGpu = body.gpu === true;
   const debug = body.debug === true;
   // Preserve the pre-structured-warning HTTP contract for callers that do
@@ -247,6 +250,7 @@ export function parseRenderOptions(body: Record<string, unknown>): Omit<RenderIn
     fps,
     quality,
     workers,
+    maxRenderMs,
     crf,
     videoBitrate,
     useGpu,
@@ -308,6 +312,7 @@ function buildRenderJobConfig(input: RenderInput, outputPath: string, log: Produ
       quality: input.quality,
       format: input.format ?? "mp4",
       workers: input.workers,
+      maxRenderMs: input.maxRenderMs,
       crf: input.crf,
       videoBitrate: input.videoBitrate,
       useGpu: input.useGpu,
@@ -351,6 +356,14 @@ function resolvePreparedRenderOutput(
 function validateRenderOverrides(body: Record<string, unknown>): string | undefined {
   const encoder = parseEncoderOptions(body, parseServerFormat(body.format) ?? "mp4");
   if (!encoder.ok) return encoder.error;
+  if (
+    body.maxRenderMs !== undefined &&
+    (typeof body.maxRenderMs !== "number" ||
+      !Number.isInteger(body.maxRenderMs) ||
+      body.maxRenderMs < 0)
+  ) {
+    return "maxRenderMs must be a non-negative integer number of milliseconds";
+  }
   if (body.variables !== undefined && !isPlainObject(body.variables)) {
     return 'variables must be a JSON object keyed by variable id (e.g. {"title":"Hello"})';
   }

packages/producer/src/services/render/renderEtaGuard.test.ts

@@ -0,0 +1,148 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { RenderTooSlowError, checkRenderEta, startRenderEtaGuard } from "./renderEtaGuard.js";
+
+const RENDER_TOO_SLOW = "RENDER_TOO_SLOW";
+
+describe("checkRenderEta", () => {
+  const base = { maxRenderMs: 100_000, elapsedMs: 0, framesCompleted: 0, totalFrames: 100 };
+
+  it("passes at the 1.2x margin and fails just over it", () => {
+    expect(checkRenderEta("capture", { ...base, msPerFrame: 1_200 })).toBeUndefined();
+    const error = checkRenderEta("capture", { ...base, msPerFrame: 1_201 });
+    expect(error).toBeInstanceOf(RenderTooSlowError);
+    expect(error?.message).toMatch(/^render too slow:/);
+    expect(error?.publicMetadata).toMatchObject({ stage: "capture", etaMs: 120_100 });
+    // The producer server reads these off the error to build the wire envelope.
+    expect(error?.code).toBe(RENDER_TOO_SLOW);
+    expect(error?.owner).toBe("user");
+    expect(error?.retryable).toBe(false);
+  });
+
+  it("counts time already spent", () => {
+    expect(
+      checkRenderEta("capture", { ...base, elapsedMs: 30_000, msPerFrame: 1_000 }),
+    ).toBeInstanceOf(RenderTooSlowError);
+  });
+
+  it("never fires without a budget or without an estimate", () => {
+    expect(
+      checkRenderEta("capture", { ...base, maxRenderMs: undefined, msPerFrame: 1e9 }),
+    ).toBeUndefined();
+    expect(checkRenderEta("capture", { ...base, maxRenderMs: 0, msPerFrame: 1e9 })).toBeUndefined();
+    expect(checkRenderEta("capture", { ...base, msPerFrame: 0 })).toBeUndefined();
+  });
+
+  it("matches the two prod renders on the 90 minute budget", () => {
+    const budget = 90 * 60_000;
+    // 21540 frames, 4 workers, p95 784 ms: the calibration estimate (70 min) is under budget.
+    expect(
+      checkRenderEta("capture_calibration", {
+        maxRenderMs: budget,
+        elapsedMs: 0,
+        framesCompleted: 0,
+        totalFrames: 21_540,
+        msPerFrame: 784 / 4,
+      }),
+    ).toBeUndefined();
+    // 570 frames, 1 worker, p95 15.8 s: 150 min, caught before capture.
+    expect(
+      checkRenderEta("capture_calibration", {
+        maxRenderMs: budget,
+        elapsedMs: 600_000,
+        framesCompleted: 0,
+        totalFrames: 570,
+        msPerFrame: 15_800 / 1,
+      }),
+    ).toBeInstanceOf(RenderTooSlowError);
+  });
+});
+
+describe("startRenderEtaGuard", () => {
+  beforeEach(() => vi.useFakeTimers());
+  afterEach(() => vi.useRealTimers());
+
+  /** Advances 5 s per step, adding `framesPerStep` frames, until tripped or `steps` run out. */
+  function run(opts: {
+    maxRenderMs?: number;
+    framesPerStep: number;
+    steps: number;
+    total: number;
+  }) {
+    const job = { framesRendered: 0 };
+    let tripped: RenderTooSlowError | undefined;
+    const stop = startRenderEtaGuard({
+      maxRenderMs: opts.maxRenderMs,
+      job,
+      totalFrames: opts.total,
+      pipelineStartMs: Date.now(),
+      onTooSlow: (error) => (tripped = error),
+    });
+    let at = 0;
+    for (let i = 0; i < opts.steps && !tripped; i++) {
+      job.framesRendered += opts.framesPerStep;
+      vi.advanceTimersByTime(5_000);
+      at += 5_000;
+    }
+    stop();
+    return { tripped, at, job };
+  }
+
+  it("fails a crawling render within about two minutes of capture", () => {
+    const { tripped, at } = run({
+      maxRenderMs: 120_000,
+      framesPerStep: 1,
+      steps: 100,
+      total: 1_000,
+    });
+    expect(tripped?.message).toMatch(/^render too slow:/);
+    expect(at).toBeLessThanOrEqual(120_000);
+    expect(tripped?.publicMetadata).toMatchObject({
+      stage: "capture",
+      totalFrames: 1_000,
+      maxRenderMs: 120_000,
+    });
+  });
+
+  it("leaves a render that will finish inside the budget alone", () => {
+    expect(
+      run({ maxRenderMs: 10_000_000, framesPerStep: 100, steps: 100, total: 100_000 }).tripped,
+    ).toBeUndefined();
+  });
+
+  it("is never tripped by the encode and assemble time after capture completes", () => {
+    const job = { framesRendered: 0 };
+    const onTooSlow = vi.fn();
+    const stop = startRenderEtaGuard({
+      maxRenderMs: 120_000,
+      job,
+      totalFrames: 100,
+      pipelineStartMs: Date.now(),
+      onTooSlow,
+    });
+    // Capture finishes inside the budget...
+    for (let i = 0; i < 10; i++) {
+      job.framesRendered += 10;
+      vi.advanceTimersByTime(5_000);
+    }
+    // ...then a mux that runs for an hour with the counter parked at the total.
+    vi.advanceTimersByTime(60 * 60_000);
+    stop();
+    expect(onTooSlow).not.toHaveBeenCalled();
+    expect(vi.getTimerCount()).toBe(0);
+  });
+
+  it("is a no-op without a budget", () => {
+    expect(
+      run({ maxRenderMs: undefined, framesPerStep: 1, steps: 200, total: 1_000 }).tripped,
+    ).toBeUndefined();
+    expect(
+      run({ maxRenderMs: 0, framesPerStep: 1, steps: 200, total: 1_000 }).tripped,
+    ).toBeUndefined();
+  });
+
+  it("does not decide on zero frames (the stall watchdog owns that)", () => {
+    expect(
+      run({ maxRenderMs: 1_000, framesPerStep: 0, steps: 200, total: 1_000 }).tripped,
+    ).toBeUndefined();
+  });
+});