Fork patch: Discover video window from GSAP opacity

packages/core/src/compiler/index.ts

@@ -16,12 +16,14 @@ export {
   injectDurations,
   extractResolvedMedia,
   clampDurations,
+  extendVideoSlotEnds,
   shouldClampMediaDuration,
   shouldClampResolvedMediaDuration,
   type UnresolvedElement,
   type ResolvedDuration,
   type ResolvedMediaElement,
   type CompilationResult,
+  type VideoSlotEnd,
 } from "./timingCompiler";
 
 // HTML compiler (Node.js — requires fs)

packages/core/src/compiler/timingCompiler.test.ts

@@ -7,6 +7,7 @@ import {
   injectDurations,
   extractResolvedMedia,
   clampDurations,
+  extendVideoSlotEnds,
   shouldClampResolvedMediaDuration,
 } from "./timingCompiler.js";
 
@@ -474,6 +475,28 @@ describe("extractResolvedMedia", () => {
   });
 });
 
+describe("extendVideoSlotEnds", () => {
+  it("keeps data-start and stretches a file-bounded data-duration past source EOF", () => {
+    const html = extendVideoSlotEnds(
+      `<video id="wrap" src="wrap.mp4" data-start="13" data-end="18.2" data-duration="5.2" data-hf-auto-start=""></video>`,
+      [{ id: "wrap", end: 19, duration: 6 }],
+    );
+    expect(html).toContain('data-start="13"');
+    expect(html).toContain('data-end="19"');
+    expect(html).toContain('data-duration="6"');
+    expect(html).not.toContain("data-media-start");
+  });
+
+  it("does not rewrite a video mentioned only in a comment", () => {
+    const html = extendVideoSlotEnds(
+      `<!-- <video id="wrap" data-duration="5.2" data-end="18.2"> --><video id="wrap" data-start="13" data-end="18.2" data-duration="5.2"></video>`,
+      [{ id: "wrap", end: 19, duration: 6 }],
+    );
+    expect(html).toContain(`<!-- <video id="wrap" data-duration="5.2" data-end="18.2"> -->`);
+    expect(html).toContain('data-duration="6"');
+  });
+});
+
 describe("clampDurations", () => {
   it("replaces data-duration and recomputes data-end", () => {
     const html = '<video id="v1" src="a.mp4" data-start="2" data-duration="10" data-end="12">';

packages/core/src/compiler/timingCompiler.ts

@@ -26,6 +26,7 @@ import {
   readMediaStart,
 } from "../runtime/playbackRate.js";
 import type { RateSpec } from "../speedRamp.js";
+import { MEDIA_RENDER_ID_ATTR } from "./mediaRenderIds.js";
 // ── Types ────────────────────────────────────────────────────────────────
 
 export interface UnresolvedElement {
@@ -337,6 +338,32 @@ export function extractResolvedMedia(html: string): ResolvedMediaElement[] {
   return resolved;
 }
 
+export type VideoSlotEnd = {
+  id: string;
+  end: number;
+  duration: number;
+};
+
+/**
+ * Stretch a compiled video's `data-end` / `data-duration` to an out-point
+ * past the source file (opacity hold). Leaves `data-start` / `data-media-start`.
+ */
+export function extendVideoSlotEnds(html: string, slots: readonly VideoSlotEnd[]): string {
+  if (slots.length === 0) return html;
+  const byId = new Map(slots.map((slot) => [slot.id, slot]));
+  return replaceOpeningTags(html, ["video"], (tag) => {
+    const id = getAttr(tag, MEDIA_RENDER_ID_ATTR) ?? getAttr(tag, "id");
+    if (!id) return tag;
+    const slot = byId.get(id);
+    if (!slot || !Number.isFinite(slot.end) || !(slot.duration > 0)) return tag;
+    return setAttr(
+      setAttr(tag, "data-end", String(slot.end)),
+      "data-duration",
+      String(slot.duration),
+    );
+  });
+}
+
 /**
  * Clamp existing data-duration and data-end on media elements.
  * For each resolution, replaces the declared duration with the clamped value

packages/core/src/index.ts

@@ -149,9 +149,11 @@ export {
   injectDurations,
   extractResolvedMedia,
   clampDurations,
+  extendVideoSlotEnds,
   shouldClampMediaDuration,
   shouldClampResolvedMediaDuration,
   MEDIA_DURATION_CLAMP_EPSILON_SECONDS,
+  type VideoSlotEnd,
 } from "./compiler/timingCompiler";
 
 export {

packages/producer/src/services/htmlCompiler.test.ts

@@ -2686,40 +2686,54 @@ describe("resolveCompositionDurations strict literal timing", () => {
 });
 
 describe("discoverVideoVisibilityFromTimeline", () => {
-  it("returns scene visibility windows for auto-start videos", async () => {
-    const duration = 2;
-    const sampleStep = 0.1;
-    const windows = [
-      { id: "v0", start: 0.2, end: 0.7 },
-      { id: "v1", start: 0.5, end: 1.2 },
-      { id: "v2", start: 1.0, end: 1.8 },
-      { id: "v3", start: 0.0, end: 0.4 },
-    ];
-
-    type SceneEl = { opacityAt: (t: number) => number };
+  type OpacityEl = {
+    id?: string;
+    opacityAt: (t: number) => number;
+    parentElement: OpacityEl | null;
+  };
+
+  async function runVisibilityProbe(
+    windows: { id: string; start: number; end: number }[],
+    duration: number,
+    sceneOpacity = 1,
+    hfSeek?: (t: number) => void,
+  ) {
     const videos = windows.map((win) => {
-      const sceneEl: SceneEl = {
-        opacityAt: (t) => (t >= win.start && t <= win.end ? 1 : 0),
+      const sceneEl: OpacityEl = {
+        opacityAt: () => sceneOpacity,
+        parentElement: null,
       };
-      return {
+      const videoEl: OpacityEl = {
         id: win.id,
-        closest: (selector: string) => (selector === ".scene" ? sceneEl : null),
+        opacityAt: (t) => (t >= win.start && t <= win.end ? 1 : 0),
+        parentElement: sceneEl,
       };
+      return videoEl;
     });
 
     let currentTime = 0;
     const previousWindow = globalThis.window;
     const previousDocument = globalThis.document;
 
     globalThis.window = {
+      ...(hfSeek
+        ? {
+            __hf: {
+              seek: (time: number) => {
+                currentTime = time;
+                hfSeek(time);
+              },
+            },
+          }
+        : {}),
       __timelines: {
         root: {
           totalTime: (time: number) => {
             currentTime = time;
           },
         },
       },
-      getComputedStyle: (el: SceneEl) => ({
+      getComputedStyle: (el: OpacityEl) => ({
         opacity: String(el.opacityAt(currentTime)),
       }),
     } as typeof globalThis.window;
@@ -2736,22 +2750,52 @@ describe("discoverVideoVisibilityFromTimeline", () => {
       const page = {
         evaluate: async (fn: (arg: number) => unknown, arg: number) => fn(arg),
       };
-
-      const result = await discoverVideoVisibilityFromTimeline(page as never, duration);
-
-      expect(result).toHaveLength(windows.length);
-      for (const win of windows) {
-        const found = result.find((entry) => entry.videoId === win.id);
-        expect(found).toBeDefined();
-        expect(found!.visibleStart).toBeGreaterThanOrEqual(win.start - sampleStep);
-        expect(found!.visibleStart).toBeLessThanOrEqual(win.start + sampleStep);
-        expect(found!.visibleEnd).toBeGreaterThanOrEqual(win.end - sampleStep);
-        expect(found!.visibleEnd).toBeLessThanOrEqual(win.end + sampleStep);
-      }
+      return await discoverVideoVisibilityFromTimeline(page as never, duration);
     } finally {
       globalThis.window = previousWindow;
       globalThis.document = previousDocument;
     }
+  }
+
+  it("returns video effective-opacity windows for auto-start videos", async () => {
+    const duration = 2;
+    const sampleStep = 0.1;
+    const windows = [
+      { id: "v0", start: 0.2, end: 0.7 },
+      { id: "v1", start: 0.5, end: 1.2 },
+      { id: "v2", start: 1.0, end: 1.8 },
+      { id: "v3", start: 0.0, end: 0.4 },
+    ];
+
+    const result = await runVisibilityProbe(windows, duration);
+
+    expect(result).toHaveLength(windows.length);
+    for (const win of windows) {
+      const found = result.find((entry) => entry.videoId === win.id);
+      expect(found).toBeDefined();
+      expect(found!.visibleStart).toBeGreaterThanOrEqual(win.start - sampleStep);
+      expect(found!.visibleStart).toBeLessThanOrEqual(win.start + sampleStep);
+      expect(found!.visibleEnd).toBeGreaterThanOrEqual(win.end - sampleStep);
+      expect(found!.visibleEnd).toBeLessThanOrEqual(win.end + sampleStep);
+    }
+  });
+
+  it("does not treat a fully-opaque .scene as the video window", async () => {
+    const result = await runVisibilityProbe([{ id: "clip", start: 0.8, end: 1.4 }], 2, 1);
+    expect(result).toHaveLength(1);
+    expect(result[0]!.videoId).toBe("clip");
+    expect(result[0]!.visibleStart).toBeGreaterThan(0.6);
+    expect(result[0]!.visibleStart).toBeLessThanOrEqual(0.9);
+    expect(result[0]!.visibleEnd).toBeGreaterThanOrEqual(1.3);
+    expect(result[0]!.visibleEnd).toBeLessThanOrEqual(1.5);
+  });
+
+  it("prefers window.__hf.seek so nested GSAP updates", async () => {
+    const sought: number[] = [];
+    await runVisibilityProbe([{ id: "clip", start: 0.5, end: 1.2 }], 2, 1, (t) => {
+      sought.push(t);
+    });
+    expect(sought.length).toBeGreaterThan(0);
   });
 });
 describe("sub-composition variable injection (render path, #2064)", () => {

packages/producer/src/services/htmlCompiler.ts

@@ -2606,8 +2606,10 @@ export interface VideoVisibilityWindow {
 }
 
 /**
- * Seek the GSAP timeline to discover when each video's parent scene is visible.
- * Only processes videos with the data-hf-auto-start sentinel (auto-injected timing).
+ * Seek the composition and discover when each auto-start video is visible.
+ * Visibility is the video's effective opacity (element × ancestors), not `.scene`.
+ * Prefers `window.__hf.seek` so nested scene GSAP (paused sibling timelines)
+ * actually updates — root `totalTime` does not propagate to them.
  */
 export async function discoverVideoVisibilityFromTimeline(
   page: Page,
@@ -2620,6 +2622,11 @@ export async function discoverVideoVisibilityFromTimeline(
     const videos = document.querySelectorAll("video[data-hf-auto-start]");
     if (videos.length === 0) return results;
 
+    const hfSeek = (
+      window as unknown as {
+        __hf?: { seek?: (t: number, o?: { suppressEvents?: boolean }) => void };
+      }
+    ).__hf?.seek;
     const timelines = (window as unknown as { __timelines?: Record<string, unknown> }).__timelines;
     if (!timelines) return results;
 
@@ -2643,14 +2650,35 @@ export async function discoverVideoVisibilityFromTimeline(
       }
     };
 
+    const seekComposition = (t: number) => {
+      if (typeof hfSeek === "function") {
+        try {
+          hfSeek(t, { suppressEvents: true });
+        } catch {}
+        return;
+      }
+      seekTl(t);
+    };
+
+    // Same walk as videoFrameInjector: GSAP often animates a wrapper, not the
+    // <video>. `val || 1` would turn opacity:0 into 1.
+    const getEffectiveOpacity = (node: Element): number => {
+      let opacity = 1;
+      let current: Element | null = node;
+      while (current) {
+        const val = parseFloat(window.getComputedStyle(current).opacity);
+        opacity *= Number.isNaN(val) ? 1 : val;
+        current = current.parentElement;
+      }
+      return opacity;
+    };
+
     const SAMPLE_STEP = 0.1;
     const BINARY_PRECISION = 1 / 60;
 
-    // Seek once per timestep and sample every video — seeking dominates and is
-    // independent of which element we read.
     const entries: {
       id: string;
-      sceneEl: Element;
+      videoEl: Element;
       firstVisible: number | null;
       lastVisible: number | null;
     }[] = [];
@@ -2659,49 +2687,43 @@ export async function discoverVideoVisibilityFromTimeline(
       if (!id) continue;
       entries.push({
         id,
-        sceneEl: videoEl.closest(".scene") || videoEl,
+        videoEl,
         firstVisible: null,
         lastVisible: null,
       });
     }
     if (entries.length === 0) return results;
 
     for (let t = 0; t <= duration; t += SAMPLE_STEP) {
-      seekTl(t);
+      seekComposition(t);
       for (const entry of entries) {
-        const opacity = parseFloat(window.getComputedStyle(entry.sceneEl).opacity);
-        if (opacity > 0) {
+        if (getEffectiveOpacity(entry.videoEl) > 0) {
           if (entry.firstVisible === null) entry.firstVisible = t;
           entry.lastVisible = t;
         }
       }
     }
 
-    // Per-video boundary refinement (cheap: O(log(step)) seeks each).
     for (const entry of entries) {
-      const { id, sceneEl, firstVisible, lastVisible } = entry;
+      const { id, videoEl, firstVisible, lastVisible } = entry;
       if (firstVisible === null || lastVisible === null) continue;
 
-      // Binary search left boundary
       let lo = Math.max(0, firstVisible - SAMPLE_STEP);
       let hi = firstVisible;
       while (hi - lo > BINARY_PRECISION) {
         const mid = (lo + hi) / 2;
-        seekTl(mid);
-        const opacity = parseFloat(window.getComputedStyle(sceneEl).opacity);
-        if (opacity > 0) hi = mid;
+        seekComposition(mid);
+        if (getEffectiveOpacity(videoEl) > 0) hi = mid;
         else lo = mid;
       }
       const exactStart = hi;
 
-      // Binary search right boundary
       lo = lastVisible;
       hi = Math.min(duration, lastVisible + SAMPLE_STEP);
       while (hi - lo > BINARY_PRECISION) {
         const mid = (lo + hi) / 2;
-        seekTl(mid);
-        const opacity = parseFloat(window.getComputedStyle(sceneEl).opacity);
-        if (opacity > 0) lo = mid;
+        seekComposition(mid);
+        if (getEffectiveOpacity(videoEl) > 0) lo = mid;
         else hi = mid;
       }
       const exactEnd = lo;
@@ -2713,7 +2735,7 @@ export async function discoverVideoVisibilityFromTimeline(
       });
     }
 
-    seekTl(0);
+    seekComposition(0);
     return results;
   }, compositionDuration);
 }