diff --git a/packages/studio/package.json b/packages/studio/package.json index a0f8f1fcc7..c59a03f4ff 100644 --- a/packages/studio/package.json +++ b/packages/studio/package.json @@ -52,7 +52,9 @@ "test:timeline-virtualization": "TIMELINE_ROW_VIRTUALIZATION=on TIMELINE_ELEMENT_COUNT=50000 node tests/e2e/timeline-virtualization.mjs", "test:watch": "vitest", "report:sdk-cutover": "bun src/utils/sdkCutoverPolicy.report.ts", - "test:timeline-default": "bun run test:timeline-virtualization" + "test:timeline-default": "bun run test:timeline-virtualization", + "test:studio-load": "STUDIO_LOAD_ARM=dev bun tests/e2e/studio-load.mjs", + "test:studio-load-embedded": "STUDIO_LOAD_ARM=embedded bun tests/e2e/studio-load.mjs" }, "dependencies": { "@codemirror/autocomplete": "^6.20.1", diff --git a/packages/studio/src/App.tsx b/packages/studio/src/App.tsx index 24b69e88df..fca861055a 100644 --- a/packages/studio/src/App.tsx +++ b/packages/studio/src/App.tsx @@ -64,11 +64,14 @@ import { } from "./utils/studioUrlState"; import { trackStudioSessionStart } from "./telemetry/events"; import { hasFiredSessionStart, markSessionStartFired } from "./telemetry/config"; +import { useStudioLoadMarks } from "./lib/useStudioLoadMarks"; // fallow-ignore-next-line complexity export function StudioApp() { const { projectId, resolving, waitingForServer } = useServerConnection(); + const isPastSplashGate = !(resolving || waitingForServer || !projectId); const initialUrlStateRef = useRef(readStudioUrlStateFromWindow()); const viewModeValue = useViewModeState(); + useStudioLoadMarks(isPastSplashGate, projectId); useEffect(() => { if (resolving || waitingForServer) return; if (hasFiredSessionStart()) return; diff --git a/packages/studio/src/lib/studioLoadBudgets.test.ts b/packages/studio/src/lib/studioLoadBudgets.test.ts new file mode 100644 index 0000000000..ef7dd542bd --- /dev/null +++ b/packages/studio/src/lib/studioLoadBudgets.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from "vitest"; +import { STUDIO_LOAD_BUDGETS, resolveStudioLoadBudgets } from "./studioLoadBudgets"; + +const EXPECTED_STUDIO_LOAD_BUDGETS = { + shellReadyP95Ms: 250, + constrainedShellReadyP95Ms: 600, + projectOpen1kClipsP95Ms: 1_500, + constrainedProjectOpen1kClipsP95Ms: 3_000, + projectOpen3kClipsP95Ms: 2_500, + constrainedProjectOpen3kClipsP95Ms: 5_000, + devShellReadyP95Ms: 400, + constrainedDevShellReadyP95Ms: 1_000, + devProjectOpen3kClipsP95Ms: 5_500, + constrainedDevProjectOpen3kClipsP95Ms: 9_000, + eagerEntryChunkGzipBytes: 600 * 1024, +}; + +describe("studio load budgets", () => { + it("owns the provisional studio load ceilings", () => { + expect(STUDIO_LOAD_BUDGETS).toMatchObject(EXPECTED_STUDIO_LOAD_BUDGETS); + expect(Object.isFrozen(STUDIO_LOAD_BUDGETS)).toBe(true); + }); + + it("creates an immutable result without changing production defaults", () => { + const resolved = resolveStudioLoadBudgets({}); + + expect(resolved).toMatchObject(EXPECTED_STUDIO_LOAD_BUDGETS); + expect(Object.isFrozen(resolved)).toBe(true); + expect(STUDIO_LOAD_BUDGETS).toMatchObject(EXPECTED_STUDIO_LOAD_BUDGETS); + }); + + it("overrides one budget while preserving every other default", () => { + const resolved = resolveStudioLoadBudgets({ shellReadyP95Ms: 500 }); + + expect(resolved).toMatchObject({ ...EXPECTED_STUDIO_LOAD_BUDGETS, shellReadyP95Ms: 500 }); + }); + + it.each([ + [{ eagerEntryChunkGzipBytes: -1 }, "eagerEntryChunkGzipBytes"], + [{ projectOpen3kClipsP95Ms: Number.NaN }, "projectOpen3kClipsP95Ms"], + ])("rejects an invalid override %#", (overrides, field) => { + expect(() => resolveStudioLoadBudgets(overrides)).toThrow(RangeError); + expect(() => resolveStudioLoadBudgets(overrides)).toThrow( + `Studio load budget ${field} must be a finite non-negative number`, + ); + }); +}); diff --git a/packages/studio/src/lib/studioLoadBudgets.ts b/packages/studio/src/lib/studioLoadBudgets.ts new file mode 100644 index 0000000000..b9d5a5058a --- /dev/null +++ b/packages/studio/src/lib/studioLoadBudgets.ts @@ -0,0 +1,48 @@ +export interface StudioLoadBudgets { + shellReadyP95Ms: number; + constrainedShellReadyP95Ms: number; + projectOpen1kClipsP95Ms: number; + constrainedProjectOpen1kClipsP95Ms: number; + projectOpen3kClipsP95Ms: number; + constrainedProjectOpen3kClipsP95Ms: number; + devShellReadyP95Ms: number; + constrainedDevShellReadyP95Ms: number; + devProjectOpen3kClipsP95Ms: number; + constrainedDevProjectOpen3kClipsP95Ms: number; + eagerEntryChunkGzipBytes: number; +} + +// Shell and dev-open budgets were revised once against the 2026-07-31 baseline +// per KTD1; see tests/e2e/studio-load.baseline.json. The originals (1000ms shell, +// 8000ms dev open) sat so far above measured reality that they gated nothing. +// The embedded project-open budgets are deliberately NOT revised: they are red +// today and stay red until U4 removes the quadratic. +export const STUDIO_LOAD_BUDGETS: Readonly = Object.freeze({ + shellReadyP95Ms: 250, + constrainedShellReadyP95Ms: 600, + projectOpen1kClipsP95Ms: 1_500, + constrainedProjectOpen1kClipsP95Ms: 3_000, + projectOpen3kClipsP95Ms: 2_500, + constrainedProjectOpen3kClipsP95Ms: 5_000, + devShellReadyP95Ms: 400, + constrainedDevShellReadyP95Ms: 1_000, + devProjectOpen3kClipsP95Ms: 5_500, + constrainedDevProjectOpen3kClipsP95Ms: 9_000, + eagerEntryChunkGzipBytes: 600 * 1024, +}); + +function assertValidBudget(name: string, value: unknown): void { + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) { + throw new RangeError(`Studio load budget ${name} must be a finite non-negative number`); + } +} + +export function resolveStudioLoadBudgets( + overrides: Partial = {}, +): Readonly { + const entries: Array<[string, unknown]> = Object.entries(overrides); + for (const [name, value] of entries) { + assertValidBudget(name, value); + } + return Object.freeze({ ...STUDIO_LOAD_BUDGETS, ...overrides }); +} diff --git a/packages/studio/src/lib/studioLoadMarks.test.ts b/packages/studio/src/lib/studioLoadMarks.test.ts new file mode 100644 index 0000000000..1ce88ae7f1 --- /dev/null +++ b/packages/studio/src/lib/studioLoadMarks.test.ts @@ -0,0 +1,120 @@ +import { beforeEach, describe, expect, it } from "vitest"; +import { + STUDIO_LOAD_MEASURES, + markBoot, + markClipsPainted, + markManifestDeriveEnd, + markManifestDeriveStart, + markPreviewLoaded, + markProjectOpenStart, + markShellReady, +} from "./studioLoadMarks"; + +function markCompleteStudioLoad(): void { + markBoot(); + markShellReady(); + markProjectOpenStart(); + markPreviewLoaded(); + markManifestDeriveStart(); + markManifestDeriveEnd(); + markClipsPainted(); +} + +describe("studio load marks", () => { + beforeEach(() => { + performance.clearMarks(); + performance.clearMeasures(); + }); + + it("constructs all named studio load measures", () => { + markCompleteStudioLoad(); + + for (const name of Object.values(STUDIO_LOAD_MEASURES)) { + const entries = performance.getEntriesByName(name, "measure"); + expect(entries).toHaveLength(1); + expect(entries[0]?.name).toBe(name); + expect(entries[0]?.duration).toBeGreaterThanOrEqual(0); + } + }); + + it("contains preview and manifest derivation within the project-open interval", () => { + markCompleteStudioLoad(); + + const projectOpen = performance.getEntriesByName( + STUDIO_LOAD_MEASURES.projectOpen, + "measure", + )[0]; + const subMeasures = [ + performance.getEntriesByName(STUDIO_LOAD_MEASURES.preview, "measure")[0], + performance.getEntriesByName(STUDIO_LOAD_MEASURES.manifestDerive, "measure")[0], + ]; + + expect(projectOpen).toBeDefined(); + for (const subMeasure of subMeasures) { + expect(subMeasure).toBeDefined(); + if (!projectOpen || !subMeasure) continue; + expect(subMeasure.startTime).toBeGreaterThanOrEqual(projectOpen.startTime); + expect(subMeasure.startTime + subMeasure.duration).toBeLessThanOrEqual( + projectOpen.startTime + projectOpen.duration, + ); + } + }); + + it("does not close project open at its start mark", () => { + markProjectOpenStart(); + + expect(performance.getEntriesByName(STUDIO_LOAD_MEASURES.projectOpen, "measure")).toHaveLength( + 0, + ); + }); + + it("overwrites the project-open measure on a later project open", () => { + markProjectOpenStart(); + markClipsPainted(); + markProjectOpenStart(); + markClipsPainted(); + + expect( + performance + .getEntriesByType("measure") + .filter((entry) => entry.name === STUDIO_LOAD_MEASURES.projectOpen), + ).toHaveLength(1); + }); + + it("does not fabricate project open without its start mark", () => { + expect(markClipsPainted).not.toThrow(); + expect(performance.getEntriesByName(STUDIO_LOAD_MEASURES.projectOpen, "measure")).toHaveLength( + 0, + ); + }); + + it("does not fabricate manifest derivation without its start mark", () => { + expect(markManifestDeriveEnd).not.toThrow(); + expect( + performance.getEntriesByName(STUDIO_LOAD_MEASURES.manifestDerive, "measure"), + ).toHaveLength(0); + }); + + it("degrades to no-ops when the Performance API is unavailable", () => { + const realPerformance = globalThis.performance; + try { + Object.defineProperty(globalThis, "performance", { + configurable: true, + value: { now: () => 0 }, + }); + + expect(markBoot).not.toThrow(); + expect(markShellReady).not.toThrow(); + expect(markProjectOpenStart).not.toThrow(); + expect(markPreviewLoaded).not.toThrow(); + expect(markManifestDeriveStart).not.toThrow(); + expect(markManifestDeriveEnd).not.toThrow(); + expect(markClipsPainted).not.toThrow(); + } finally { + Object.defineProperty(globalThis, "performance", { + configurable: true, + value: realPerformance, + }); + } + }); +}); diff --git a/packages/studio/src/lib/studioLoadMarks.ts b/packages/studio/src/lib/studioLoadMarks.ts new file mode 100644 index 0000000000..5a03cfe415 --- /dev/null +++ b/packages/studio/src/lib/studioLoadMarks.ts @@ -0,0 +1,99 @@ +const STUDIO_LOAD_MARKS = Object.freeze({ + boot: "hf.studio.boot", + shellReady: "hf.studio.shell-ready", + projectOpenStart: "hf.studio.project-open-start", + previewLoaded: "hf.studio.preview-loaded", + clipsPainted: "hf.studio.clips-painted", + manifestDeriveStart: "hf.studio.manifest-derive-start", + manifestDeriveEnd: "hf.studio.manifest-derive-end", +}); + +export const STUDIO_LOAD_MEASURES = Object.freeze({ + shell: "hf.studio.shell", + projectOpen: "hf.studio.project-open", + preview: "hf.studio.preview", + manifestDerive: "hf.studio.manifest-derive", +}); + +function getPerformance(): Performance | null { + const currentPerformance = globalThis.performance; + if ( + typeof currentPerformance?.mark !== "function" || + typeof currentPerformance?.measure !== "function" + ) { + return null; + } + return currentPerformance; +} + +function mark(name: string): void { + try { + const currentPerformance = getPerformance(); + if (!currentPerformance) return; + if (typeof currentPerformance.clearMarks === "function") { + currentPerformance.clearMarks(name); + } + currentPerformance.mark(name); + } catch {} +} + +function measure(name: string, startMark: string, endMark: string): void { + try { + const currentPerformance = getPerformance(); + if (!currentPerformance || typeof currentPerformance.getEntriesByName !== "function") return; + if ( + currentPerformance.getEntriesByName(startMark, "mark").length === 0 || + currentPerformance.getEntriesByName(endMark, "mark").length === 0 + ) { + return; + } + if (typeof currentPerformance.clearMeasures === "function") { + currentPerformance.clearMeasures(name); + } + currentPerformance.measure(name, startMark, endMark); + } catch {} +} + +export function markBoot(): void { + mark(STUDIO_LOAD_MARKS.boot); +} + +export function markShellReady(): void { + mark(STUDIO_LOAD_MARKS.shellReady); + measure(STUDIO_LOAD_MEASURES.shell, STUDIO_LOAD_MARKS.boot, STUDIO_LOAD_MARKS.shellReady); +} + +export function markProjectOpenStart(): void { + mark(STUDIO_LOAD_MARKS.projectOpenStart); +} + +export function markPreviewLoaded(): void { + mark(STUDIO_LOAD_MARKS.previewLoaded); + measure( + STUDIO_LOAD_MEASURES.preview, + STUDIO_LOAD_MARKS.projectOpenStart, + STUDIO_LOAD_MARKS.previewLoaded, + ); +} + +export function markManifestDeriveStart(): void { + mark(STUDIO_LOAD_MARKS.manifestDeriveStart); +} + +export function markManifestDeriveEnd(): void { + mark(STUDIO_LOAD_MARKS.manifestDeriveEnd); + measure( + STUDIO_LOAD_MEASURES.manifestDerive, + STUDIO_LOAD_MARKS.manifestDeriveStart, + STUDIO_LOAD_MARKS.manifestDeriveEnd, + ); +} + +export function markClipsPainted(): void { + mark(STUDIO_LOAD_MARKS.clipsPainted); + measure( + STUDIO_LOAD_MEASURES.projectOpen, + STUDIO_LOAD_MARKS.projectOpenStart, + STUDIO_LOAD_MARKS.clipsPainted, + ); +} diff --git a/packages/studio/src/lib/useStudioLoadMarks.ts b/packages/studio/src/lib/useStudioLoadMarks.ts new file mode 100644 index 0000000000..3ceb543000 --- /dev/null +++ b/packages/studio/src/lib/useStudioLoadMarks.ts @@ -0,0 +1,44 @@ +import { useEffect, useRef } from "react"; +import { usePlayerStore } from "../player/store/playerStore"; +import { markClipsPainted, markProjectOpenStart, markShellReady } from "./studioLoadMarks"; + +/** + * Fires the three load seams the shell owns: shell-ready once the splash gate + * clears, project-open-start whenever a distinct project resolves, and + * clips-painted on the first frame after the timeline holds any clip. + * + * Clips-painted is observed from the store rather than from Timeline because + * both App and Timeline sit at the studio 600-line cap, and because the seam + * is "the element set became non-empty" — a store fact, not a component one. + * Deliberately not anchored on timelineReady: that flips in initializeAdapter + * before clip derivation runs, which would put the derivation cost outside the + * measured window. + */ +export function useStudioLoadMarks(isPastSplashGate: boolean, projectId: string | null): void { + const shellReadyFired = useRef(false); + useEffect(() => { + if (!isPastSplashGate || shellReadyFired.current) return; + shellReadyFired.current = true; + markShellReady(); + }, [isPastSplashGate]); + + const openStartFiredFor = useRef(null); + useEffect(() => { + if (!projectId || openStartFiredFor.current === projectId) return; + openStartFiredFor.current = projectId; + markProjectOpenStart(); + }, [projectId]); + + const hasClips = usePlayerStore((state) => state.elements.length > 0); + const clipsPaintedFired = useRef(false); + useEffect(() => { + if (!hasClips) { + clipsPaintedFired.current = false; + return; + } + if (clipsPaintedFired.current) return; + clipsPaintedFired.current = true; + const frame = requestAnimationFrame(() => markClipsPainted()); + return () => cancelAnimationFrame(frame); + }, [hasClips]); +} diff --git a/packages/studio/src/main.tsx b/packages/studio/src/main.tsx index 4bf0587430..8db2e2ff84 100644 --- a/packages/studio/src/main.tsx +++ b/packages/studio/src/main.tsx @@ -2,9 +2,11 @@ import { StrictMode } from "react"; import { createRoot } from "react-dom/client"; import { StudioApp } from "./App"; import { StudioErrorBoundary } from "./components/StudioErrorBoundary"; +import { markBoot } from "./lib/studioLoadMarks"; import { trackStudioEvent } from "./utils/studioTelemetry"; import "./styles/studio.css"; +markBoot(); trackStudioEvent("session_start"); function errorProps(value: unknown): { diff --git a/packages/studio/src/player/hooks/useTimelineSyncCallbacks.ts b/packages/studio/src/player/hooks/useTimelineSyncCallbacks.ts index c4526e2609..584bf53475 100644 --- a/packages/studio/src/player/hooks/useTimelineSyncCallbacks.ts +++ b/packages/studio/src/player/hooks/useTimelineSyncCallbacks.ts @@ -28,6 +28,11 @@ import { buildMissingCompositionElements, } from "../lib/timelineIframeHelpers"; import { acceptedRuntimeMessageFps, inspectStudioRuntimeMessage } from "../lib/runtimeProtocol"; +import { + markManifestDeriveEnd, + markManifestDeriveStart, + markPreviewLoaded, +} from "../../lib/studioLoadMarks"; interface UseTimelineSyncCallbacksParams { iframeRef: React.RefObject; @@ -216,6 +221,7 @@ export function useTimelineSyncCallbacks({ } const usedHostEls = new Set(); + markManifestDeriveStart(); const els: TimelineElement[] = filtered.map((clip, index) => { const hostEl = iframeDoc ? findTimelineDomNodeForClip(iframeDoc, clip, index, usedHostEls) @@ -228,6 +234,7 @@ export function useTimelineSyncCallbacks({ hostEl, }); }); + markManifestDeriveEnd(); const rawDuration = data.durationInFrames / acceptedRuntimeMessageFps(data); // Clamp non-finite or absurdly large durations — the runtime can emit // Infinity when it detects a loop-inflated GSAP timeline without an @@ -400,6 +407,7 @@ export function useTimelineSyncCallbacks({ ]); const onIframeLoad = useCallback(() => { + markPreviewLoaded(); applyPreviewAudioState(); if (probeIntervalRef.current) clearInterval(probeIntervalRef.current); diff --git a/packages/studio/tests/e2e/generateStudioLoadFixture.mjs b/packages/studio/tests/e2e/generateStudioLoadFixture.mjs new file mode 100644 index 0000000000..de371195fd --- /dev/null +++ b/packages/studio/tests/e2e/generateStudioLoadFixture.mjs @@ -0,0 +1,207 @@ +import { mkdirSync, writeFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const ROOT_DURATION = 180; +const CLIP_DURATION = 2; +const SUB_COMPOSITION_DURATION = 120; +const SUB_COMPOSITION_PATH = "compositions/load-details.html"; +const CLIP_KINDS = ["load-card", "load-caption", "load-accent"]; +const PROJECT_CONFIG = `${JSON.stringify( + { + $schema: "https://hyperframes.heygen.com/schema/hyperframes.json", + paths: { + blocks: "compositions", + components: "compositions/components", + assets: "assets", + }, + }, + null, + 2, +)}\n`; + +function validateOptions({ clipCount, trackCount }) { + if (!Number.isInteger(clipCount) || clipCount < 0) { + throw new RangeError("clipCount must be a non-negative integer"); + } + if (!Number.isInteger(trackCount) || trackCount < 1) { + throw new RangeError("trackCount must be a positive integer"); + } +} + +function renderClip(index, start, track, indent) { + const authoredId = index % 5 === 0 ? `${indent} id="load-clip-${index}"\n` : ""; + const kind = CLIP_KINDS[index % CLIP_KINDS.length]; + const left = (index * 37) % 1820; + const top = (index * 17) % 980; + + return `${indent}
+${indent} ${index + 1} +${indent}
`; +} + +function renderRootHtml({ clipCount, trackCount }, rootClipCount) { + const rootClips = Array.from({ length: rootClipCount }, (_, index) => + renderClip( + index, + (index * 37) % (ROOT_DURATION - CLIP_DURATION), + index % Math.max(1, trackCount - 1), + " ", + ), + ).join("\n"); + const hostIndex = rootClipCount; + const hostMarkup = + clipCount === 0 + ? `
` + : `
`; + const clipMarkup = rootClips ? `${rootClips}\n${hostMarkup}` : hostMarkup; + + return ` + + + + + Studio load fixture (${clipCount} clips, ${trackCount} tracks) + + + +
+${clipMarkup} +
+ + + +`; +} + +function renderSubCompositionHtml({ trackCount }, rootClipCount, nestedClipCount) { + const firstNestedIndex = rootClipCount + 1; + const clips = Array.from({ length: nestedClipCount }, (_, offset) => { + const index = firstNestedIndex + offset; + return renderClip( + index, + (index * 17) % (SUB_COMPOSITION_DURATION - CLIP_DURATION), + index % trackCount, + " ", + ); + }).join("\n"); + const clipMarkup = clips ? `\n${clips}` : ""; + + return ` +`; +} + +/** Build every source file for a deterministic, on-disk Studio load project. */ +export function generateStudioLoadFixture(options) { + validateOptions(options); + const nestedClipCount = options.clipCount > 1 ? Math.min(100, options.clipCount - 1) : 0; + const rootClipCount = options.clipCount - nestedClipCount - (options.clipCount > 0 ? 1 : 0); + + return Object.freeze({ + "hyperframes.json": PROJECT_CONFIG, + "index.html": renderRootHtml(options, rootClipCount), + [SUB_COMPOSITION_PATH]: renderSubCompositionHtml(options, rootClipCount, nestedClipCount), + }); +} + +/** Write one generated project to disk. */ +export function writeStudioLoadFixture(outputDir, options) { + for (const [relativePath, contents] of Object.entries(generateStudioLoadFixture(options))) { + const outputPath = join(outputDir, relativePath); + mkdirSync(dirname(outputPath), { recursive: true }); + writeFileSync(outputPath, contents); + } +} + +const entryPath = process.argv[1]; +if (entryPath && import.meta.url === pathToFileURL(resolve(entryPath)).href) { + const outputDir = fileURLToPath(new URL("fixtures/studio-load/", import.meta.url)); + writeStudioLoadFixture(outputDir, { + clipCount: Number(process.argv[2] ?? 1_000), + trackCount: Number(process.argv[3] ?? 50), + }); +} diff --git a/packages/studio/tests/e2e/generateStudioLoadFixture.test.ts b/packages/studio/tests/e2e/generateStudioLoadFixture.test.ts new file mode 100644 index 0000000000..e4dc0549ae --- /dev/null +++ b/packages/studio/tests/e2e/generateStudioLoadFixture.test.ts @@ -0,0 +1,105 @@ +import { spawnSync } from "node:child_process"; +import { existsSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { afterEach, describe, expect, it } from "vitest"; +import { generateStudioLoadFixture, writeStudioLoadFixture } from "./generateStudioLoadFixture.mjs"; + +const temporaryProjects: string[] = []; +const CLI_LINT_PATH = fileURLToPath( + new URL("../../../cli/src/utils/lintProject.ts", import.meta.url), +); + +function createTemporaryProject(): string { + const projectDir = mkdtempSync(join(tmpdir(), "studio-load-fixture-")); + temporaryProjects.push(projectDir); + return projectDir; +} + +function clipTags(files: Record): string[] { + return Object.values(files).flatMap( + (contents) => contents.match(/<[^>]+\bclass="[^"]*\bclip\b[^"]*"[^>]*>/g) ?? [], + ); +} + +function expectProjectToPassLint(projectDir: string): void { + const program = ` + import { lintProject } from ${JSON.stringify(pathToFileURL(CLI_LINT_PATH).href)}; + const result = await lintProject(${JSON.stringify(projectDir)}); + const errors = result.results.flatMap(({ file, result }) => + result.findings.filter(({ severity }) => severity === "error").map((finding) => ({ file, ...finding })), + ); + console.log(JSON.stringify({ errors, totalErrors: result.totalErrors })); + if (result.totalErrors > 0) process.exitCode = 1; + `; + const result = spawnSync("bun", ["-e", program], { encoding: "utf8" }); + + expect( + { error: result.error?.message, status: result.status }, + result.stderr || result.stdout, + ).toEqual({ error: undefined, status: 0 }); +} + +afterEach(() => { + for (const projectDir of temporaryProjects.splice(0)) { + rmSync(projectDir, { force: true, recursive: true }); + } +}); + +describe("generateStudioLoadFixture", () => { + it("generates exactly 1,000 clips across every requested track", () => { + const files = generateStudioLoadFixture({ clipCount: 1_000, trackCount: 37 }); + const tags = clipTags(files); + const tracks = new Set( + tags.flatMap((tag) => { + const match = tag.match(/\bdata-track-index="(\d+)"/); + return match ? [Number(match[1])] : []; + }), + ); + + expect(tags).toHaveLength(1_000); + expect([...tracks].sort((a, b) => a - b)).toEqual( + Array.from({ length: 37 }, (_, index) => index), + ); + }); + + it("produces byte-identical files for the same inputs", () => { + expect(generateStudioLoadFixture({ clipCount: 1_000, trackCount: 50 })).toStrictEqual( + generateStudioLoadFixture({ clipCount: 1_000, trackCount: 50 }), + ); + }); + + it("keeps the authored DOM id ratio below 30 percent", () => { + const tags = clipTags(generateStudioLoadFixture({ clipCount: 1_000, trackCount: 50 })); + const authoredIdCount = tags.filter((tag) => /\sid="[^"]+"/.test(tag)).length; + + expect(authoredIdCount / tags.length).toBeLessThan(0.3); + }); + + it("passes the HyperFrames CLI structural lint", () => { + const projectDir = createTemporaryProject(); + writeStudioLoadFixture(projectDir, { clipCount: 1_000, trackCount: 50 }); + + expectProjectToPassLint(projectDir); + }, 15_000); + + it("references an existing sub-composition from the root", () => { + const projectDir = createTemporaryProject(); + const files = generateStudioLoadFixture({ clipCount: 1_000, trackCount: 50 }); + writeStudioLoadFixture(projectDir, { clipCount: 1_000, trackCount: 50 }); + const reference = files["index.html"].match(/data-composition-src="([^"]+)"/)?.[1]; + + expect(reference).toBeDefined(); + expect(reference && existsSync(join(projectDir, reference))).toBe(true); + }); + + it("generates a structurally valid zero-clip project", () => { + const files = generateStudioLoadFixture({ clipCount: 0, trackCount: 1 }); + const projectDir = createTemporaryProject(); + writeStudioLoadFixture(projectDir, { clipCount: 0, trackCount: 1 }); + + expect(clipTags(files)).toHaveLength(0); + expectProjectToPassLint(projectDir); + }); +}); diff --git a/packages/studio/tests/e2e/puppeteerEnv.mjs b/packages/studio/tests/e2e/puppeteerEnv.mjs new file mode 100644 index 0000000000..7d6cd3a518 --- /dev/null +++ b/packages/studio/tests/e2e/puppeteerEnv.mjs @@ -0,0 +1,33 @@ +/** + * Shared browser-gate helpers. Both e2e gates need to find a Chrome and take a + * percentile; keeping one copy means a fix to either lands in both. + */ +import { existsSync, readdirSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; + +export function resolveChromeExecutable() { + const chromeRoot = join(homedir(), ".cache", "puppeteer", "chrome"); + const builds = existsSync(chromeRoot) ? readdirSync(chromeRoot).sort().reverse() : []; + const installedCandidates = builds.flatMap((build) => + [ + "chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing", + "chrome-mac-x64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing", + "chrome-linux64/chrome", + ].map((relative) => join(chromeRoot, build, relative)), + ); + return [ + process.env.PUPPETEER_EXECUTABLE_PATH, + process.env.CHROME_PATH, + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + "/usr/bin/google-chrome", + "/usr/bin/chromium", + ...installedCandidates, + ].find((candidate) => candidate && existsSync(candidate)); +} + +export function percentile(values, ratio) { + if (values.length === 0) return 0; + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * ratio) - 1)]; +} diff --git a/packages/studio/tests/e2e/studio-load.baseline.json b/packages/studio/tests/e2e/studio-load.baseline.json new file mode 100644 index 0000000000..3016fbe5b5 --- /dev/null +++ b/packages/studio/tests/e2e/studio-load.baseline.json @@ -0,0 +1,130 @@ +{ + "recordedAt": "2026-07-31", + "commit": "08c47e8cd", + "machine": "Apple silicon laptop, macOS 25.5.0", + "note": "Pre-U4 baseline. Captured before any optimisation lands so later runs have something to beat. Every figure is a hf.studio.* performance.measure read from the page; nothing here comes from DOM polling.", + "arms": { + "dev-1000-virtualization-on": { + "arm": "dev", + "clips": 1000, + "rowVirtualization": "on", + "runs": 3, + "mountedClips": 152, + "hf.studio.shell": { + "median": 60.3, + "p95": 64.6 + }, + "hf.studio.project-open": { + "median": 959.3, + "p95": 964.6 + }, + "hf.studio.preview": { + "median": 578.0, + "p95": 603.3 + }, + "hf.studio.manifest-derive": { + "median": 299.2, + "p95": 300.0 + } + }, + "dev-3000-virtualization-on": { + "arm": "dev", + "clips": 3000, + "rowVirtualization": "on", + "runs": 7, + "mountedClips": 240, + "hf.studio.shell": { + "median": 59.8, + "p95": 64.5 + }, + "hf.studio.project-open": { + "median": 4219.3, + "p95": 4312.8 + }, + "hf.studio.preview": { + "median": 1235.7, + "p95": 1312.5 + }, + "hf.studio.manifest-derive": { + "median": 2879.9, + "p95": 2910.4 + } + }, + "dev-3000-virtualization-off": { + "arm": "dev", + "clips": 3000, + "rowVirtualization": "off", + "runs": 7, + "mountedClips": 2900, + "hf.studio.shell": { + "median": 63.0, + "p95": 71.0 + }, + "hf.studio.project-open": { + "median": 4914.6, + "p95": 5423.7 + }, + "hf.studio.preview": { + "median": 1411.1, + "p95": 1968.9 + }, + "hf.studio.manifest-derive": { + "median": 2926.0, + "p95": 3466.6 + } + }, + "embedded-3000-virtualization-on": { + "arm": "embedded", + "clips": 3000, + "rowVirtualization": "on", + "runs": 7, + "mountedClips": 240, + "hf.studio.shell": { + "median": 45.9, + "p95": 55.5 + }, + "hf.studio.project-open": { + "median": 4394.4, + "p95": 4518.4 + }, + "hf.studio.preview": { + "median": 1436.3, + "p95": 1508.9 + }, + "hf.studio.manifest-derive": { + "median": 2888.2, + "p95": 3034.0 + }, + "verdict": "FAIL \u2014 project-open p95 4518.4ms exceeds the 2500ms budget. Intended: the gate is red until U4 lands." + } + }, + "findings": { + "quadraticConfirmed": "manifest-derive grows 299.2ms -> 2879.9ms for a 3x clip increase, a factor of 9.6 against a predicted 9.0. The open path is quadratic in clip count, as U4 asserts.", + "previewIsLinear": "preview grows 578.0ms -> 1235.7ms over the same 3x, a factor of 2.1. Server-side parse and inlining scale with document size, not quadratically.", + "crossover": "At 1000 clips preview is the larger cost (60% of open). At 3000 clips derive overtakes it (68% of open). Both were correct about different regimes; only derive gets worse from here.", + "virtualizationDirection": "Row virtualization makes opening FASTER, not slower: 4219.3ms on versus 4914.6ms off at 3000 clips, a 14.2% improvement. The two arms' full run ranges do not overlap (on 4169.9-4312.8, off 4458.6-5423.7 across 7 runs each). derive is unchanged between arms (2879.9 vs 2926.0), consistent with it scanning the iframe document rather than the timeline DOM; the saving is in preview and in mounting 240 rather than 2900 clips into the React tree.", + "sevenSecondFigureRetired": "The 7,050ms 'time to interactive' on record came from a probe that polled a mounted-clip count every 400ms until stable for four polls. Measured properly, shell is ~60ms and open at 1000 clips is ~959ms.", + "productionBuildBarelyHelpsOpen": "The embedded arm opens in 4394.4ms against the dev arm's 4219.3ms at the same clip count and flag. A production React build does not meaningfully change open time because the cost is algorithmic JS in the derivation, not dev-server or unminified-runtime overhead. Shell time does differ (45.9ms vs 59.8ms) but both are trivial. Consequence: the dev arm is an acceptable proxy for open time, and the retired 7s figure was the old probe's polling artifact rather than dev-mode inflation as previously supposed." + }, + "notMeasured": [ + "The embedded arm at 1000 clips, and the embedded arm with row virtualization off. Neither changes any conclusion above." + ], + "eagerEntryChunkGzipBytes": { + "measured": 1111170, + "budget": 614400, + "verdict": "FAIL \u2014 intended, U5 has not landed." + }, + "instrumentationOverhead": { + "question": "Does PR 1's instrumentation cost anything? It adds five marks, four measures, and one store subscription.", + "method": "Build-agnostic A/B: baseline c6925e471 and the PR branch served from separate worktrees on the same machine, identical 3,000-clip fixture. The baseline cannot emit hf.studio.* marks, so both arms were judged on a MutationObserver firing when the first [data-clip=\"true\"] node lands. 9 runs per arm, fresh browser each run, arms INTERLEAVED with alternating order per round.", + "baselineMedianMs": 4679, + "prMedianMs": 4733, + "deltaMs": 54, + "deltaPct": 1.15, + "baselineRangeMs": [4612, 5284], + "prRangeMs": [4600, 4918], + "rangesOverlap": true, + "verdict": "No measurable cost. The delta is 1.15% with fully overlapping ranges; the PR's range is narrower than the baseline's and its fastest run is faster. Instrumentation is free at this resolution.", + "methodNote": "A first attempt ran the arms in blocks rather than interleaved, with both dev servers alive, and reported +8.77%. That was an artifact of arm ordering and drift, not a real cost. Recorded here because the blocked result was wrong in a way that looked publishable." + } +} diff --git a/packages/studio/tests/e2e/studio-load.mjs b/packages/studio/tests/e2e/studio-load.mjs new file mode 100644 index 0000000000..ea3fe9b571 --- /dev/null +++ b/packages/studio/tests/e2e/studio-load.mjs @@ -0,0 +1,252 @@ +#!/usr/bin/env node +/** + * Wall-clock gate for Studio's cold load and project-open time. + * + * Distinct from tests/e2e/timeline-virtualization.mjs, which injects a synthetic + * element set straight into the store and measures scroll. This one navigates + * cold to a real on-disk project and reads the marks Studio emits, so it covers + * the pipeline a user actually waits on. + * + * STUDIO_URL=http://127.0.0.1:5191/#project/studio-load \ + * STUDIO_LOAD_ARM=dev bun packages/studio/tests/e2e/studio-load.mjs + * + * Run under bun, not node: the budgets are imported from their TypeScript home + * so this gate and the app read one owner rather than two copies of the numbers. + * + * STUDIO_LOAD_ARM selects which serve mode is under test. "embedded" is the + * pre-built SPA the CLI ships and the only arm carrying a product promise. + * "dev" is the Vite dev server contributors run; its budgets are a regression + * signal, not a claim about what users experience. The gate asserts the arm it + * observes rather than trusting the caller, because a dev number reported as an + * embedded number is worse than no number. + * + * Every measurement comes from performance.measure entries. There is no DOM + * polling fallback: an absent measure fails the run. The probe this replaces + * polled a mounted-clip count every 400ms until it held for four polls, which + * baked ~1.6s of its own debounce into every figure it produced. + */ +import puppeteer from "puppeteer-core"; +import { percentile, resolveChromeExecutable } from "./puppeteerEnv.mjs"; +import { STUDIO_LOAD_BUDGETS } from "../../src/lib/studioLoadBudgets.ts"; + +const STUDIO_URL = process.env.STUDIO_URL; +const ARM = process.env.STUDIO_LOAD_ARM || "dev"; +const CLIP_COUNT = Number(process.env.STUDIO_LOAD_CLIPS || 1_000); +const TIER = process.env.STUDIO_LOAD_TIER || "primary"; +const ROW_VIRTUALIZATION = process.env.STUDIO_LOAD_ROW_VIRTUALIZATION || "off"; +const WARMUP_RUNS = Number(process.env.STUDIO_LOAD_WARMUP ?? 1); +const MEASURED_RUNS = Number(process.env.STUDIO_LOAD_RUNS ?? 5); +const OPEN_TIMEOUT_MS = Number(process.env.STUDIO_LOAD_TIMEOUT_MS ?? 180_000); + +const MEASURES = [ + "hf.studio.shell", + "hf.studio.project-open", + "hf.studio.preview", + "hf.studio.manifest-derive", +]; + +if (!STUDIO_URL) { + console.error( + "STUDIO_URL is required and must point at a Studio serving the studio-load fixture", + ); + process.exit(2); +} +if ( + !["embedded", "dev"].includes(ARM) || + ![1_000, 3_000].includes(CLIP_COUNT) || + !["primary", "ci"].includes(TIER) || + !["off", "on"].includes(ROW_VIRTUALIZATION) +) { + console.error( + "STUDIO_LOAD_ARM must be embedded or dev; " + + "STUDIO_LOAD_CLIPS must be 1000 or 3000; " + + "STUDIO_LOAD_TIER must be primary or ci; " + + "STUDIO_LOAD_ROW_VIRTUALIZATION must be off or on", + ); + process.exit(2); +} + +function median(values) { + return percentile(values, 0.5); +} + +/** + * The dev server serves the unbundled entry module; the pre-built SPA serves a + * hashed asset. That difference is what distinguishes the arms from inside the + * page, and it holds in both directions — unlike window.__studioTest, which the + * production build strips entirely. + */ +async function observeArm(page) { + return page.evaluate(() => { + const sources = Array.from(document.querySelectorAll("script[src]")).map( + (node) => node.getAttribute("src") || "", + ); + if (sources.some((src) => src.includes("/src/main.tsx") || src.includes("/@vite/client"))) + return "dev"; + if (sources.some((src) => /\/assets\/index-[^/]+\.js/.test(src))) return "embedded"; + return "unknown"; + }); +} + +async function runOnce(executablePath, label) { + const browser = await puppeteer.launch({ + executablePath, + headless: "new", + args: ["--no-sandbox", "--disable-dev-shm-usage", "--window-size=1600,1000"], + defaultViewport: { width: 1600, height: 1000 }, + }); + try { + const page = await browser.newPage(); + const pageErrors = []; + page.on("pageerror", (error) => pageErrors.push(String(error))); + + const startedAt = Date.now(); + await page.goto(STUDIO_URL, { waitUntil: "domcontentloaded", timeout: OPEN_TIMEOUT_MS }); + + const observedArm = await observeArm(page); + if (observedArm !== ARM) { + console.error( + `FAIL: requested arm "${ARM}" but the server under test is "${observedArm}" (${STUDIO_URL})`, + ); + await browser.close(); + process.exit(2); + } + + try { + await page.waitForFunction( + () => performance.getEntriesByName("hf.studio.project-open", "measure").length > 0, + { timeout: OPEN_TIMEOUT_MS, polling: 100 }, + ); + } catch { + throw new Error( + `${label}: hf.studio.project-open never appeared within ${OPEN_TIMEOUT_MS}ms. ` + + `Page errors: ${pageErrors.length ? pageErrors.join(" | ") : "none"}`, + ); + } + + const measures = await page.evaluate((names) => { + const read = (name) => { + const entries = performance.getEntriesByName(name, "measure"); + return entries.length === 0 + ? null + : Number(entries[entries.length - 1].duration.toFixed(1)); + }; + return Object.fromEntries(names.map((name) => [name, read(name)])); + }, MEASURES); + + const clipCount = await page.evaluate( + () => document.querySelectorAll('[data-clip="true"]').length, + ); + return { + ...measures, + wallClockMs: Date.now() - startedAt, + mountedClips: clipCount, + pageErrors: pageErrors.length, + }; + } finally { + await browser.close().catch(() => {}); + } +} + +const executablePath = resolveChromeExecutable(); +if (!executablePath) { + console.error("No Chrome executable found. Set PUPPETEER_EXECUTABLE_PATH or CHROME_PATH."); + process.exit(2); +} + +console.log( + `studio-load gate | arm=${ARM} clips=${CLIP_COUNT} tier=${TIER} rowVirtualization=${ROW_VIRTUALIZATION} ` + + `warmup=${WARMUP_RUNS} measured=${MEASURED_RUNS}`, +); +if (ARM === "dev") { + console.log( + "NOTE: dev arm — a Vite dev server, not the build users run. Regression signal only, not a product number.", + ); +} + +for (let i = 0; i < WARMUP_RUNS; i += 1) { + const result = await runOnce(executablePath, `warmup ${i + 1}`); + console.log(` warmup ${i + 1}: project-open ${result["hf.studio.project-open"]}ms (discarded)`); +} + +const runs = []; +for (let i = 0; i < MEASURED_RUNS; i += 1) { + const result = await runOnce(executablePath, `run ${i + 1}`); + runs.push(result); + console.log( + ` run ${i + 1}: shell ${result["hf.studio.shell"]}ms | open ${result["hf.studio.project-open"]}ms ` + + `(preview ${result["hf.studio.preview"]}ms, derive ${result["hf.studio.manifest-derive"]}ms) | ` + + `mounted ${result.mountedClips} clips`, + ); +} + +const summary = Object.fromEntries( + MEASURES.map((name) => { + const values = runs.map((run) => run[name]).filter((value) => typeof value === "number"); + return [ + name, + { median: median(values), p95: percentile(values, 0.95), samples: values.length }, + ]; + }), +); + +console.log("\n=== summary ==="); +for (const name of MEASURES) { + const stat = summary[name]; + console.log( + ` ${name.padEnd(28)} median ${String(stat.median).padStart(8)}ms p95 ${String(stat.p95).padStart(8)}ms n=${stat.samples}`, + ); +} + +const budgets = STUDIO_LOAD_BUDGETS; +const constrained = TIER === "ci"; +const pick = (primary, ci) => (constrained ? ci : primary); +const shellBudget = + ARM === "embedded" + ? pick(budgets.shellReadyP95Ms, budgets.constrainedShellReadyP95Ms) + : pick(budgets.devShellReadyP95Ms, budgets.constrainedDevShellReadyP95Ms); +// The plan's table defines a dev open budget at 3k clips only. Rather than +// invent one for 1k, the gate reports the number and says it is ungated. +const openBudget = + ARM === "embedded" + ? CLIP_COUNT === 3_000 + ? pick(budgets.projectOpen3kClipsP95Ms, budgets.constrainedProjectOpen3kClipsP95Ms) + : pick(budgets.projectOpen1kClipsP95Ms, budgets.constrainedProjectOpen1kClipsP95Ms) + : CLIP_COUNT === 3_000 + ? pick(budgets.devProjectOpen3kClipsP95Ms, budgets.constrainedDevProjectOpen3kClipsP95Ms) + : null; + +const failures = []; +if (summary["hf.studio.shell"].p95 > shellBudget) { + failures.push(`shell p95 ${summary["hf.studio.shell"].p95}ms exceeds ${shellBudget}ms`); +} +if (openBudget !== null && summary["hf.studio.project-open"].p95 > openBudget) { + failures.push( + `project-open p95 ${summary["hf.studio.project-open"].p95}ms exceeds ${openBudget}ms`, + ); +} + +console.log( + `\nbudgets (arm=${ARM}, tier=${TIER}): shell p95 <= ${shellBudget}ms, ` + + `project-open p95 ${openBudget === null ? "UNGATED (no budget defined for this arm at this clip count)" : `<= ${openBudget}ms`}`, +); +console.log( + JSON.stringify( + { + arm: ARM, + clips: CLIP_COUNT, + tier: TIER, + rowVirtualization: ROW_VIRTUALIZATION, + summary, + runs, + }, + null, + 2, + ), +); + +if (failures.length > 0) { + console.error(`\nFAIL: ${failures.join("; ")}`); + process.exit(1); +} +console.log("\nPASS"); diff --git a/packages/studio/tests/e2e/timeline-virtualization.mjs b/packages/studio/tests/e2e/timeline-virtualization.mjs index 7795bdbd95..a03a020ef0 100644 --- a/packages/studio/tests/e2e/timeline-virtualization.mjs +++ b/packages/studio/tests/e2e/timeline-virtualization.mjs @@ -21,10 +21,9 @@ * rather than trusting the caller: the server is configured by whoever started * it, and a mismatch would otherwise pass silently against the wrong build. */ -import { existsSync, readdirSync } from "node:fs"; -import { homedir, platform, arch } from "node:os"; -import { join } from "node:path"; +import { platform, arch } from "node:os"; import puppeteer from "puppeteer-core"; +import { percentile, resolveChromeExecutable } from "./puppeteerEnv.mjs"; const STUDIO_URL = process.env.STUDIO_URL; const PROFILE = process.env.TIMELINE_PROFILE || "dense-short"; @@ -62,32 +61,6 @@ if (ROW_VIRTUALIZATION === "off" && ELEMENT_COUNT === 50_000) { process.exit(2); } -function resolveChromeExecutable() { - const chromeRoot = join(homedir(), ".cache", "puppeteer", "chrome"); - const builds = existsSync(chromeRoot) ? readdirSync(chromeRoot).sort().reverse() : []; - const installedCandidates = builds.flatMap((build) => - [ - "chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing", - "chrome-mac-x64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing", - "chrome-linux64/chrome", - ].map((relative) => join(chromeRoot, build, relative)), - ); - return [ - process.env.PUPPETEER_EXECUTABLE_PATH, - process.env.CHROME_PATH, - "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", - "/usr/bin/google-chrome", - "/usr/bin/chromium", - ...installedCandidates, - ].find((candidate) => candidate && existsSync(candidate)); -} - -function percentile(values, ratio) { - if (values.length === 0) return 0; - const sorted = [...values].sort((a, b) => a - b); - return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * ratio) - 1)]; -} - async function collectHeapBytes(client) { const usage = await client.send("Runtime.getHeapUsage"); return usage.usedSize;