From f763e6158f691a1a6f8317084211ba4c3285000f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Tue, 18 Aug 2026 02:41:54 +0000 Subject: [PATCH 01/16] feat: make creator media edits render-safe --- packages/core/src/index.ts | 1 + .../engine/src/services/audioMixer.test.ts | 138 +++++++++++ packages/engine/src/services/audioMixer.ts | 79 +++++- .../engine/src/services/audioMixer.types.ts | 2 + .../src/services/videoFrameExtractor.test.ts | 51 ++++ .../src/services/videoFrameExtractor.ts | 144 +++++++---- .../playback-rate-av-parity.test.ts | 234 ++++++++++++++++++ .../keyframes-creator-capabilities.test.mjs | 133 ++++++++++ skills-manifest.json | 10 +- skills/hyperframes-audio/SKILL.md | 17 ++ skills/hyperframes-cli/SKILL.md | 13 + skills/hyperframes-keyframes/SKILL.md | 31 ++- skills/hyperframes/SKILL.md | 14 ++ .../references/api-map.md | 20 +- .../references/media.md | 23 +- 15 files changed, 829 insertions(+), 81 deletions(-) create mode 100644 packages/producer/tests/playback-rate-av-parity/playback-rate-av-parity.test.ts create mode 100644 scripts/keyframes-creator-capabilities.test.mjs diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 108b87bb82..28ff3d7647 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -298,6 +298,7 @@ export { // publishConfig entry points at a file the pack doesn't contain // (verify:packed-manifests catches exactly that). export { createRuntimeStartTimeResolver } from "./runtime/startResolver.js"; +export { normalizePlaybackRate } from "./runtime/playbackRate.js"; // Variable validation (CLI / tooling-side) export { diff --git a/packages/engine/src/services/audioMixer.test.ts b/packages/engine/src/services/audioMixer.test.ts index cac9b10ae8..0e2db62716 100644 --- a/packages/engine/src/services/audioMixer.test.ts +++ b/packages/engine/src/services/audioMixer.test.ts @@ -257,6 +257,129 @@ describe("processCompositionAudio", () => { expect(filter).not.toContain("weights="); }); + it("trims the consumed source span and applies pitch-preserving tempo at 2x", async () => { + const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-")); + const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-")); + tempDirs.push(baseDir, workDir); + writeFileSync(join(baseDir, "timecode.wav"), "stub"); + + const result = await processCompositionAudio( + [ + { + id: "timecode", + src: "timecode.wav", + start: 0, + end: 2, + mediaStart: 1, + playbackRate: 2, + layer: 0, + volume: 1, + type: "audio", + }, + ], + baseDir, + workDir, + join(baseDir, "out.m4a"), + 2, + ); + + expect(result.success).toBe(true); + const prepareArgs = runFfmpegMock.mock.calls[0]?.[0] ?? []; + expect(prepareArgs).toEqual(expect.arrayContaining(["-ss", "1", "-t", "4", "-af", "atempo=2"])); + expect(prepareArgs.filter((arg) => arg === "-t")).toHaveLength(2); + expect(prepareArgs.at(-3)).toBe("2"); + }); + + it.each([ + { rate: 0.1, filter: "atempo=0.5,atempo=0.5,atempo=0.5,atempo=0.8" }, + { rate: 5, filter: "atempo=2,atempo=2,atempo=1.25" }, + ])("builds a bounded atempo chain for normalized rate $rate", async ({ rate, filter }) => { + const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-")); + const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-")); + tempDirs.push(baseDir, workDir); + writeFileSync(join(baseDir, "timecode.wav"), "stub"); + + await processCompositionAudio( + [ + { + id: "timecode", + src: "timecode.wav", + start: 0, + end: 2, + mediaStart: 0, + playbackRate: rate, + layer: 0, + volume: 1, + type: "audio", + }, + ], + baseDir, + workDir, + join(baseDir, "out.m4a"), + 2, + ); + + expect(runFfmpegMock.mock.calls[0]?.[0]).toEqual(expect.arrayContaining(["-af", filter])); + }); + + it("keeps automation on authored timeline time after constant retiming", async () => { + const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-")); + const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-")); + tempDirs.push(baseDir, workDir); + writeFileSync(join(baseDir, "timecode.wav"), "stub"); + const automation = JSON.stringify({ + version: 1, + lanes: [ + { + target: "volume", + points: [ + { t: 0, v: 0 }, + { t: 2, v: 1 }, + ], + }, + ], + }); + const fxChain = JSON.stringify({ + version: 1, + nodes: [{ type: "gain", id: "gain", params: { gain: 1 } }], + }); + + await processCompositionAudio( + [ + { + id: "timecode", + src: "timecode.wav", + start: 0, + end: 2, + mediaStart: 0, + playbackRate: 2, + layer: 0, + volume: 1, + automation, + fxChain, + type: "audio", + }, + ], + baseDir, + workDir, + join(baseDir, "out.m4a"), + 2, + ); + + expect(runFfmpegMock.mock.calls[0]?.[0]).toEqual( + expect.arrayContaining(["-t", "4", "-af", "atempo=2"]), + ); + expect(applyAudioFxChainMock.mock.calls[0]?.[3]).toMatchObject({ + envelope: { + keyframes: [ + { time: 0, volume: 0 }, + { time: 2, volume: 1 }, + ], + trackStart: 0, + }, + }); + }); + it("lets an FX tail run past the clip, still bounded by the composition", async () => { // A reverb is still decaying when the clip's own audio stops. Trimming at // the clip boundary is what cut every tail short in the render. @@ -1033,6 +1156,21 @@ describe("processCompositionAudio", () => { }); describe("parseAudioElements — relative data-start resolution", () => { + it("parses and normalizes constant playback rate", () => { + const html = wrap( + '' + + '' + + '' + + '', + ); + const tracks = parseAudioElements(html); + + expect(tracks.find((track) => track.id === "fast")?.playbackRate).toBe(2); + expect(tracks.find((track) => track.id === "low")?.playbackRate).toBe(0.1); + expect(tracks.find((track) => track.id === "high")?.playbackRate).toBe(5); + expect(tracks.find((track) => track.id === "invalid")?.playbackRate).toBe(1); + }); + const wrap = (body: string) => `
${body}
`; diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.ts index e71d26587b..aa4ca18350 100644 --- a/packages/engine/src/services/audioMixer.ts +++ b/packages/engine/src/services/audioMixer.ts @@ -39,6 +39,7 @@ import { type HfAutomationLane, } from "@hyperframes/core/audio-automation"; import { chainTailSeconds } from "@hyperframes/core/audio-fx-tail"; +import { normalizePlaybackRate } from "@hyperframes/core"; import { applyAudioFxChain, AudioFxRenderError } from "./audioFxRender.js"; import type { AudioVolumeKeyframe } from "./audioMixer.types.js"; @@ -68,6 +69,39 @@ function formatFilterNumber(value: number): string { return Number(value.toFixed(6)).toString(); } +/** Build an FFmpeg-compatible, pitch-preserving tempo chain. */ +function buildAtempoFilter(playbackRate: number): string | null { + let remaining = normalizePlaybackRate(playbackRate); + if (Math.abs(remaining - 1) < 1e-9) return null; + const stages: number[] = []; + while (remaining < 0.5 - 1e-9) { + stages.push(0.5); + remaining /= 0.5; + } + while (remaining > 2 + 1e-9) { + stages.push(2); + remaining /= 2; + } + if (Math.abs(remaining - 1) >= 1e-9) stages.push(remaining); + return stages.map((stage) => `atempo=${formatFilterNumber(stage)}`).join(","); +} + +function preparedAudioOutputArgs(srcPath: string, playbackRate: number): Promise { + return stereoOutputArgs(srcPath).then((channelArgs) => { + const filters: string[] = []; + const outputArgs: string[] = []; + if (channelArgs[0] === "-af" && channelArgs[1]) { + filters.push(channelArgs[1]); + } else { + outputArgs.push(...channelArgs); + } + const atempo = buildAtempoFilter(playbackRate); + if (atempo) filters.push(atempo); + if (filters.length > 0) outputArgs.push("-af", filters.join(",")); + return outputArgs; + }); +} + function escapeExpressionCommas(expression: string): string { return expression.replace(/\\/g, "\\\\").replace(/,/g, "\\,"); } @@ -427,6 +461,8 @@ export function parseAudioElements(html: string): AudioElement[] { // and `type`; everything else (timing, layer, volume) is read identically. const build = (el: RefResolverEl, id: string, type: AudioElement["type"]): AudioElement => { const mediaStartAttr = el.getAttribute("data-media-start"); + const playbackStartAttr = el.getAttribute("data-playback-start"); + const playbackRateAttr = el.getAttribute("data-playback-rate"); const layerAttr = el.getAttribute("data-layer"); const volumeAttr = el.getAttribute("data-volume"); const fxChain = el.getAttribute(HF_AUDIO_FX_ATTR); @@ -436,7 +472,14 @@ export function parseAudioElements(html: string): AudioElement[] { src: el.getAttribute("src") as string, start: resolveStart(el), end: parseEnd(el.getAttribute("data-end")), - mediaStart: mediaStartAttr ? parseFloat(mediaStartAttr) : 0, + mediaStart: playbackStartAttr + ? parseFloat(playbackStartAttr) + : mediaStartAttr + ? parseFloat(mediaStartAttr) + : 0, + playbackRate: normalizePlaybackRate( + playbackRateAttr ? parseFloat(playbackRateAttr) : Number.NaN, + ), layer: layerAttr ? parseInt(layerAttr) : 0, volume: volumeAttr ? parseFloat(volumeAttr) : 1.0, ...(fxChain ? { fxChain } : {}), @@ -463,7 +506,7 @@ export function parseAudioElements(html: string): AudioElement[] { async function extractAudioFromVideo( videoPath: string, outputPath: string, - options?: { startTime?: number; duration?: number }, + options?: { startTime?: number; duration?: number; playbackRate?: number }, signal?: AbortSignal, config?: Partial>, ): Promise { @@ -471,11 +514,17 @@ async function extractAudioFromVideo( const outputDir = dirname(outputPath); if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true }); - const args: string[] = ["-i", videoPath]; + const playbackRate = normalizePlaybackRate(options?.playbackRate ?? 1); + const args: string[] = []; if (options?.startTime !== undefined) args.push("-ss", String(options.startTime)); - if (options?.duration !== undefined) args.push("-t", String(options.duration)); - const channelArgs = await stereoOutputArgs(videoPath); - args.push("-vn", "-acodec", "pcm_s16le", "-ar", "48000", ...channelArgs, "-y", outputPath); + if (options?.duration !== undefined) args.push("-t", String(options.duration * playbackRate)); + args.push("-i", videoPath); + const outputArgs = await preparedAudioOutputArgs(videoPath, playbackRate); + args.push("-vn", "-acodec", "pcm_s16le", "-ar", "48000", ...outputArgs); + if (playbackRate !== 1 && options?.duration !== undefined) { + args.push("-t", String(options.duration)); + } + args.push("-y", outputPath); const result = await runFfmpeg(args, { signal, timeout: ffmpegProcessTimeout }); @@ -513,29 +562,31 @@ async function prepareAudioTrack( outputPath: string, mediaStart: number, duration: number, + playbackRate = 1, signal?: AbortSignal, config?: Partial>, ): Promise { const ffmpegProcessTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout; const outputDir = dirname(outputPath); if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true }); - const channelArgs = await stereoOutputArgs(srcPath); + const normalizedPlaybackRate = normalizePlaybackRate(playbackRate); + const outputArgs = await preparedAudioOutputArgs(srcPath, normalizedPlaybackRate); const args = [ "-ss", String(mediaStart), "-t", - String(duration), + String(duration * normalizedPlaybackRate), "-i", srcPath, "-acodec", "pcm_s16le", "-ar", "48000", - ...channelArgs, - "-y", - outputPath, + ...outputArgs, ]; + if (normalizedPlaybackRate !== 1) args.push("-t", String(duration)); + args.push("-y", outputPath); const result = await runFfmpeg(args, { signal, timeout: ffmpegProcessTimeout }); @@ -888,7 +939,9 @@ export async function processCompositionAudio( ); return; } - const effectiveDuration = metadata.durationSeconds - element.mediaStart; + const effectiveDuration = + (metadata.durationSeconds - element.mediaStart) / + normalizePlaybackRate(element.playbackRate ?? 1); element.end = element.start + (effectiveDuration > 0 ? effectiveDuration : metadata.durationSeconds); } @@ -902,6 +955,7 @@ export async function processCompositionAudio( { startTime: element.mediaStart, duration: element.end - element.start, + playbackRate: element.playbackRate, }, effectiveSignal, config, @@ -929,6 +983,7 @@ export async function processCompositionAudio( trimmedPath, element.mediaStart, element.end - element.start, + element.playbackRate, effectiveSignal, config, ); diff --git a/packages/engine/src/services/audioMixer.types.ts b/packages/engine/src/services/audioMixer.types.ts index 3a0dbe626d..5879972fd5 100644 --- a/packages/engine/src/services/audioMixer.types.ts +++ b/packages/engine/src/services/audioMixer.types.ts @@ -9,6 +9,8 @@ export interface AudioElement { start: number; end: number; mediaStart: number; + /** Constant normalized source-time multiplier (0.1..5). */ + playbackRate?: number; layer: number; volume?: number; volumeKeyframes?: AudioVolumeKeyframe[]; diff --git a/packages/engine/src/services/videoFrameExtractor.test.ts b/packages/engine/src/services/videoFrameExtractor.test.ts index a5470f14a3..b940c46045 100644 --- a/packages/engine/src/services/videoFrameExtractor.test.ts +++ b/packages/engine/src/services/videoFrameExtractor.test.ts @@ -75,6 +75,7 @@ describe("resolveVideoExtractionDuration", () => { start: 0, end: Number.POSITIVE_INFINITY, mediaStart: 0, + playbackRate: 1, loop: false, hasAudio: false, ...overrides, @@ -98,6 +99,32 @@ describe("resolveVideoExtractionDuration", () => { expect(explicitLoop.loop).toBe(true); }); + it("extracts the source span consumed by an explicit 2x timeline slot", () => { + expect( + resolveVideoExtractionWindow( + video({ end: 2, mediaStart: 1, playbackRate: 2 }), + metadata(8), + 2, + ), + ).toMatchObject({ + compositionStart: 0, + mediaStart: 1, + durationSeconds: 4, + timelineDurationSeconds: 2, + }); + }); + + it("reports natural timeline duration after constant playback-rate retiming", () => { + expect( + resolveVideoExtractionWindow(video({ mediaStart: 1, playbackRate: 2 }), metadata(5), 10), + ).toMatchObject({ + compositionStart: 0, + mediaStart: 1, + durationSeconds: 4, + timelineDurationSeconds: 2, + }); + }); + it("trims materially negative preroll and advances the source offset", () => { const preroll = video({ start: -60, end: 120, mediaStart: 0 }); expect(resolveVideoExtractionWindow(preroll, metadata(120), 2)).toEqual({ @@ -648,6 +675,20 @@ describe("resolveProjectRelativeSrc — sub-composition path clamping", () => { }); describe("parseVideoElements", () => { + it("parses and normalizes constant playback rate for final rendering", () => { + const [fast, low, high, invalid] = parseVideoElements( + '' + + '' + + '' + + '', + ); + + expect(fast?.playbackRate).toBe(2); + expect(low?.playbackRate).toBe(0.1); + expect(high?.playbackRate).toBe(5); + expect(invalid?.playbackRate).toBe(1); + }); + it("parses videos without an id or data-start attribute", () => { const videos = parseVideoElements(''); @@ -675,6 +716,7 @@ describe("parseVideoElements", () => { start: 2, end: 7, mediaStart: 1.5, + playbackRate: 1, loop: false, hasAudio: true, }); @@ -815,6 +857,15 @@ describe("FrameLookupTable", () => { expect(table.getActiveFramePayloads(4.5).get("hero")?.frameIndex).toBe(15); }); + it("selects source frames at the authored constant playback rate", () => { + const videos = parseVideoElements( + '', + ); + const table = createFrameLookupTable(videos, [fakeExtracted(120, 30)]); + + expect(table.getActiveFramePayloads(1).get("hero")?.frameIndex).toBe(60); + }); + it("wraps at video-stream EOF when a mux container has longer audio", () => { const extracted = fakeExtracted(6, 2); extracted.metadata.durationSeconds = 60; diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts index 1ecc62d4eb..d4737a6d94 100644 --- a/packages/engine/src/services/videoFrameExtractor.ts +++ b/packages/engine/src/services/videoFrameExtractor.ts @@ -14,6 +14,7 @@ import { fpsToFfmpegArg, fpsToNumber, MEDIA_DURATION_CLAMP_EPSILON_SECONDS, + normalizePlaybackRate, toFps, type FpsInput, } from "@hyperframes/core"; @@ -57,6 +58,7 @@ export interface VideoElement { start: number; end: number; mediaStart: number; + playbackRate?: number; loop: boolean; hasAudio: boolean; } @@ -542,7 +544,9 @@ export function parseVideoElements(html: string): VideoElement[] { const startAttr = el.getAttribute("data-start"); const endAttr = el.getAttribute("data-end"); const durationAttr = el.getAttribute("data-duration"); - const mediaStartAttr = el.getAttribute("data-media-start"); + const mediaStartAttr = + el.getAttribute("data-playback-start") ?? el.getAttribute("data-media-start"); + const playbackRateAttr = el.getAttribute("data-playback-rate"); const hasAudioAttr = el.getAttribute("data-has-audio"); // Resolve data-start, including relative references ("intro", "intro + 2") @@ -571,6 +575,9 @@ export function parseVideoElements(html: string): VideoElement[] { start, end, mediaStart: mediaStartAttr ? parseFloat(mediaStartAttr) : 0, + playbackRate: normalizePlaybackRate( + playbackRateAttr ? parseFloat(playbackRateAttr) : Number.NaN, + ), loop: el.hasAttribute("loop"), hasAudio: hasAudioAttr === "true", }); @@ -870,6 +877,8 @@ export interface TimelineExtractionWindow { compositionStart: number; mediaStart: number; durationSeconds: number; + /** Visible duration on the authored composition timeline after retiming. */ + timelineDurationSeconds?: number; /** * Preserve the authored timeline origin and mediaStart for lookup. This is * required when a looped visible interval crosses a source boundary and @@ -893,6 +902,7 @@ export interface TimelineExtractionWindow { } type TimelineWindowVideo = Pick & + Partial> & Partial>; // Logical duration assigned to a one-frame held-tail representation. This is @@ -916,50 +926,67 @@ export function resolveTimelineExtractionWindow( timelineEnd?: number, sourceDuration?: number, ): TimelineExtractionWindow { + const playbackRate = normalizePlaybackRate(video.playbackRate ?? 1); + const withTimelineDuration = ( + window: TimelineExtractionWindow, + timelineDurationSeconds: number, + ): TimelineExtractionWindow => + playbackRate === 1 ? window : { ...window, timelineDurationSeconds }; if (timelineEnd === undefined) { - return { - compositionStart: video.start, - mediaStart: video.mediaStart, - durationSeconds: resolvedDuration, - }; + return withTimelineDuration( + { + compositionStart: video.start, + mediaStart: video.mediaStart, + durationSeconds: resolvedDuration * playbackRate, + }, + resolvedDuration, + ); } if (!Number.isFinite(timelineEnd)) { throw new Error(`Video extraction timelineEnd must be finite; got ${String(timelineEnd)}`); } const compositionStart = Math.max(0, video.start); const trimmedPreroll = compositionStart - video.start; + const trimmedSourcePreroll = trimmedPreroll * playbackRate; const timelineDuration = Math.max(0, timelineEnd - compositionStart); // Infinity means "natural source duration", not an authored infinite slot. // Explicit finite slots may outlive the source (loop or held tail), while an // omitted duration remains source-bounded exactly like the browser runtime. const resolvedVisibleDuration = resolvedDuration - trimmedPreroll; const visibleDuration = Math.max(0, Math.min(resolvedVisibleDuration, timelineDuration)); - let mediaStart = video.mediaStart + trimmedPreroll; + const visibleSourceDuration = visibleDuration * playbackRate; + let mediaStart = video.mediaStart + trimmedSourcePreroll; if (visibleDuration > 0 && sourceDuration !== undefined) { const sourceRemaining = Math.max(0, sourceDuration - video.mediaStart); if (sourceRemaining > 0 && video.loop && Number.isFinite(video.end)) { - const phaseOffset = trimmedPreroll % sourceRemaining; + const phaseOffset = trimmedSourcePreroll % sourceRemaining; const phaseRemaining = sourceRemaining - phaseOffset; // The element visibility contract includes its end boundary. Preserve a // complete cycle on equality as well, otherwise a rebased suffix would // wrap to its own first frame instead of the source cycle's first frame. - if (visibleDuration >= phaseRemaining) { - return { - compositionStart: video.start, - mediaStart: video.mediaStart, - durationSeconds: sourceRemaining, - preserveTimelinePhase: true, - }; + if (visibleSourceDuration >= phaseRemaining) { + return withTimelineDuration( + { + compositionStart: video.start, + mediaStart: video.mediaStart, + durationSeconds: sourceRemaining, + preserveTimelinePhase: true, + }, + visibleDuration, + ); } mediaStart = video.mediaStart + phaseOffset; } else if (sourceRemaining > 0) { - const sourceVisibleAfterPreroll = Math.max(0, sourceRemaining - trimmedPreroll); - if (visibleDuration <= sourceVisibleAfterPreroll) { - return { - compositionStart, - mediaStart, - durationSeconds: visibleDuration, - }; + const sourceVisibleAfterPreroll = Math.max(0, sourceRemaining - trimmedSourcePreroll); + if (visibleSourceDuration <= sourceVisibleAfterPreroll) { + return withTimelineDuration( + { + compositionStart, + mediaStart, + durationSeconds: visibleSourceDuration, + }, + visibleDuration, + ); } // The visible interval enters (or is entirely inside) the held tail. @@ -971,20 +998,26 @@ export function resolveTimelineExtractionWindow( Math.max(sourceVisibleAfterPreroll, FINAL_FRAME_LOGICAL_DURATION_SECONDS), ); const extractionOffset = sourceRemaining - extractionDuration; - return { - compositionStart: video.start + extractionOffset, - mediaStart: video.mediaStart + extractionOffset, - durationSeconds: extractionDuration, - preserveTimelineEnd: true, - ensureFinalFrame: true, - }; + return withTimelineDuration( + { + compositionStart: video.start + extractionOffset / playbackRate, + mediaStart: video.mediaStart + extractionOffset, + durationSeconds: extractionDuration, + preserveTimelineEnd: true, + ensureFinalFrame: true, + }, + visibleDuration, + ); } } - return { - compositionStart, - mediaStart, - durationSeconds: visibleDuration, - }; + return withTimelineDuration( + { + compositionStart, + mediaStart, + durationSeconds: visibleSourceDuration, + }, + visibleDuration, + ); } /** @@ -1018,6 +1051,9 @@ export async function resolveFinalFrameExtractionWindow( mediaStart: playableDuration - logicalDuration, extractionMediaStart: finalFrameTimestamp, durationSeconds: logicalDuration, + ...(window.timelineDurationSeconds !== undefined + ? { timelineDurationSeconds: window.timelineDurationSeconds } + : {}), preserveTimelineEnd: true, finalFrameOnly: true, }; @@ -1046,11 +1082,13 @@ export function resolveVideoExtractionWindow( `Video media start ${video.mediaStart}s is outside playable video duration ${playableDuration}s`, ); } - const resolvedDuration = resolveSegmentDuration( - video.end - video.start, - video.mediaStart, - playableDuration, - ); + const playbackRate = normalizePlaybackRate(video.playbackRate ?? 1); + const requestedTimelineDuration = video.end - video.start; + const resolvedDuration = + Number.isFinite(requestedTimelineDuration) && requestedTimelineDuration > 0 + ? requestedTimelineDuration + : resolveSegmentDuration(requestedTimelineDuration, video.mediaStart, playableDuration) / + playbackRate; return resolveTimelineExtractionWindow(video, resolvedDuration, timelineEnd, playableDuration); } @@ -1908,7 +1946,7 @@ export async function extractAllVideoFrames( if (!window.preserveTimelinePhase) { video.start = window.compositionStart; if (!window.preserveTimelineEnd) { - video.end = window.compositionStart + videoDuration; + video.end = window.compositionStart + (window.timelineDurationSeconds ?? videoDuration); } video.mediaStart = window.mediaStart; } @@ -2054,16 +2092,20 @@ function getFrameIndexAtTime( loop = false, mediaStart = 0, holdLastFrame = false, + playbackRate = 1, ): number | null { let localTime = globalTime - videoStart; if (localTime < 0) return null; - const loopDuration = Math.max(0, resolvePlayableVideoDuration(extracted.metadata) - mediaStart); + const normalizedPlaybackRate = normalizePlaybackRate(playbackRate); + const loopDuration = + Math.max(0, resolvePlayableVideoDuration(extracted.metadata) - mediaStart) / + normalizedPlaybackRate; if (loop && loopDuration > 0 && localTime >= loopDuration) { localTime %= loopDuration; } // Add epsilon before flooring to avoid IEEE 754 boundary errors where // e.g. 0.28 * 25 === 6.999999999999999 instead of 7. - const frameIndex = Math.floor(localTime * extracted.fps + 1e-9); + const frameIndex = Math.floor(localTime * normalizedPlaybackRate * extracted.fps + 1e-9); if (frameIndex < 0 || extracted.totalFrames <= 0) return null; if (frameIndex >= extracted.totalFrames) { return loop || holdLastFrame ? extracted.totalFrames - 1 : null; @@ -2114,6 +2156,7 @@ export class FrameLookupTable { end: number; mediaStart: number; loop: boolean; + playbackRate: number; } > = new Map(); private orderedVideos: Array<{ @@ -2123,6 +2166,7 @@ export class FrameLookupTable { end: number; mediaStart: number; loop: boolean; + playbackRate: number; }> = []; private activeVideoIds: Set = new Set(); private startCursor = 0; @@ -2134,8 +2178,16 @@ export class FrameLookupTable { end: number, mediaStart: number, loop = false, + playbackRate = 1, ): void { - this.videos.set(extracted.videoId, { extracted, start, end, mediaStart, loop }); + this.videos.set(extracted.videoId, { + extracted, + start, + end, + mediaStart, + loop, + playbackRate: normalizePlaybackRate(playbackRate), + }); this.orderedVideos = Array.from(this.videos.entries()) .map(([videoId, video]) => ({ videoId, ...video })) .sort((a, b) => a.start - b.start); @@ -2153,6 +2205,7 @@ export class FrameLookupTable { video.loop, video.mediaStart, true, + video.playbackRate, ); return frameIndex == null ? null : video.extracted.framePaths.get(frameIndex) || null; } @@ -2222,6 +2275,7 @@ export class FrameLookupTable { video.loop, video.mediaStart, true, + video.playbackRate, ); if (frameIndex == null) continue; const framePath = video.extracted.framePaths.get(frameIndex); @@ -2266,7 +2320,9 @@ export function createFrameLookupTable( for (const video of videos) { const ext = extractedMap.get(video.id); - if (ext) table.addVideo(ext, video.start, video.end, video.mediaStart, video.loop); + if (ext) { + table.addVideo(ext, video.start, video.end, video.mediaStart, video.loop, video.playbackRate); + } } return table; diff --git a/packages/producer/tests/playback-rate-av-parity/playback-rate-av-parity.test.ts b/packages/producer/tests/playback-rate-av-parity/playback-rate-av-parity.test.ts new file mode 100644 index 0000000000..357be69a46 --- /dev/null +++ b/packages/producer/tests/playback-rate-av-parity/playback-rate-av-parity.test.ts @@ -0,0 +1,234 @@ +import { afterAll, expect, test } from "bun:test"; +import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + createFrameLookupTable, + extractAllVideoFrames, + parseVideoElements, +} from "../../../engine/src/services/videoFrameExtractor.ts"; +import { + parseAudioElements, + processCompositionAudio, +} from "../../../engine/src/services/audioMixer.ts"; + +const workDirs: string[] = []; + +afterAll(() => { + if (process.env.HF_KEEP_PLAYBACK_RATE_FIXTURE === "1") return; + for (const dir of workDirs) rmSync(dir, { recursive: true, force: true }); +}); + +function ffmpeg(args: string[]) { + const result = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", ...args], { + encoding: null, + }); + if (result.status !== 0) { + throw new Error(`ffmpeg failed: ${result.stderr.toString("utf8")}`); + } + return result.stdout; +} + +function ffprobeDuration(path: string): number { + const result = spawnSync( + "ffprobe", + ["-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", "--", path], + { encoding: "utf8" }, + ); + if (result.status !== 0) throw new Error(`ffprobe failed: ${result.stderr}`); + return Number.parseFloat(result.stdout.trim()); +} + +function sampleRgb(path: string, time: number): [number, number, number] { + const rgb = ffmpeg([ + "-ss", + String(time), + "-i", + path, + "-frames:v", + "1", + "-vf", + "scale=1:1", + "-pix_fmt", + "rgb24", + "-f", + "rawvideo", + "-", + ]); + return [rgb[0] ?? 0, rgb[1] ?? 0, rgb[2] ?? 0]; +} + +function dominant(rgb: [number, number, number]): "red" | "green" | "blue" | "yellow" { + const [r, g, b] = rgb; + if (r > 130 && g > 130 && b < 100) return "yellow"; + if (r >= g && r >= b) return "red"; + if (g >= r && g >= b) return "green"; + return "blue"; +} + +function sampleFrequency(path: string, center: number): number { + const duration = 0.2; + const pcm = ffmpeg([ + "-ss", + String(center - duration / 2), + "-t", + String(duration), + "-i", + path, + "-vn", + "-ac", + "1", + "-ar", + "48000", + "-f", + "s16le", + "-", + ]); + let crossings = 0; + let previous = pcm.readInt16LE(0); + for (let offset = 2; offset + 1 < pcm.length; offset += 2) { + const current = pcm.readInt16LE(offset); + if ((previous < 0 && current >= 0) || (previous >= 0 && current < 0)) crossings += 1; + previous = current; + } + return crossings / (2 * duration); +} + +test( + "final render keeps timecoded picture and pitch-preserved sound aligned at 2x", + async () => { + const projectDir = mkdtempSync(join(tmpdir(), "hf-playback-rate-parity-")); + workDirs.push(projectDir); + const source = join(projectDir, "timecoded.mp4"); + const output = join(projectDir, "out.mp4"); + + ffmpeg([ + "-f", + "lavfi", + "-i", + "color=c=red:s=160x90:r=10:d=1", + "-f", + "lavfi", + "-i", + "color=c=green:s=160x90:r=10:d=1", + "-f", + "lavfi", + "-i", + "color=c=blue:s=160x90:r=10:d=1", + "-f", + "lavfi", + "-i", + "color=c=yellow:s=160x90:r=10:d=1", + "-f", + "lavfi", + "-i", + "sine=frequency=440:sample_rate=48000:duration=1", + "-f", + "lavfi", + "-i", + "sine=frequency=660:sample_rate=48000:duration=1", + "-f", + "lavfi", + "-i", + "sine=frequency=880:sample_rate=48000:duration=1", + "-f", + "lavfi", + "-i", + "sine=frequency=1100:sample_rate=48000:duration=1", + "-filter_complex", + "[0:v][1:v][2:v][3:v]concat=n=4:v=1:a=0[v];[4:a][5:a][6:a][7:a]concat=n=4:v=0:a=1[a]", + "-map", + "[v]", + "-map", + "[a]", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + "-shortest", + "-y", + source, + ]); + + const html = ` +
+ + +
`; + + const videoElements = parseVideoElements(html); + const extracted = await extractAllVideoFrames(videoElements, projectDir, { + fps: 10, + outputDir: join(projectDir, "extracted"), + format: "png", + timelineEnd: 2, + }); + expect(extracted.errors).toEqual([]); + + const audioElements = parseAudioElements(html); + const audioPath = join(projectDir, "audio.m4a"); + const audio = await processCompositionAudio( + audioElements, + projectDir, + join(projectDir, "audio-work"), + audioPath, + 2, + ); + expect(audio.success).toBe(true); + + const framesDir = join(projectDir, "timeline-frames"); + mkdirSync(framesDir); + const lookup = createFrameLookupTable(videoElements, extracted.extracted); + for (let frame = 0; frame < 20; frame += 1) { + const framePath = lookup.getFrame("picture", frame / 10); + if (!framePath) throw new Error(`missing picture frame ${frame}`); + copyFileSync(framePath, join(framesDir, `frame_${String(frame + 1).padStart(5, "0")}.png`)); + } + ffmpeg([ + "-framerate", + "10", + "-i", + join(framesDir, "frame_%05d.png"), + "-i", + audioPath, + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-c:a", + "copy", + "-t", + "2", + "-y", + output, + ]); + + const duration = ffprobeDuration(output); + expect(duration).toBeWithin(1.95, 2.05); + const proofTimes = [0.25, 0.75, 1.25, 1.75]; + const colors = proofTimes.map((time) => dominant(sampleRgb(output, time))); + expect(colors).toEqual(["red", "green", "blue", "yellow"]); + const frequencies = proofTimes.map((time) => sampleFrequency(output, time)); + for (const [actual, expected] of frequencies.map((value, index) => [ + value, + [440, 660, 880, 1100][index]!, + ])) { + expect(actual).toBeWithin(expected - 30, expected + 30); + } + console.log( + `[playback-rate-proof] ${JSON.stringify({ + output, + sha256: createHash("sha256").update(readFileSync(output)).digest("hex"), + duration, + proofTimes, + colors, + frequencies, + })}`, + ); + }, + 120_000, +); diff --git a/scripts/keyframes-creator-capabilities.test.mjs b/scripts/keyframes-creator-capabilities.test.mjs new file mode 100644 index 0000000000..8078ba84bb --- /dev/null +++ b/scripts/keyframes-creator-capabilities.test.mjs @@ -0,0 +1,133 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +const read = (path) => readFile(new URL(`../${path}`, import.meta.url), "utf8"); + +const files = { + router: "skills/hyperframes/SKILL.md", + keyframes: "skills/hyperframes-keyframes/SKILL.md", + cli: "skills/hyperframes-cli/SKILL.md", + audio: "skills/hyperframes-audio/SKILL.md", + remotionMedia: "skills/remotion-to-hyperframes/references/media.md", + remotionMap: "skills/remotion-to-hyperframes/references/api-map.md", +}; + +function requiresAll(text, patterns, surface) { + for (const pattern of patterns) { + assert.match(text, pattern, `${surface} is missing ${pattern}`); + } +} + +test("router loads the owning skills for creator picture and sound edits", async () => { + const router = await read(files.router); + requiresAll( + router, + [ + /cut this footage/i, + /trim[\s\S]{0,80}splice[\s\S]{0,80}reorder|trim[\s\S]{0,80}reorder[\s\S]{0,80}splice/i, + /source range/i, + /punch[- ]in.*punch[- ]out/i, + /multi-state zoom|smooth.*zoom.*reframe/i, + /Ken Burns/i, + /camera move/i, + /match cut/i, + /whip pan/i, + /fade.*crossfade/i, + /duck.*automation.*effects|automation.*duck.*effects/i, + /picture and sound|video and audio/i, + ], + files.router, + ); + assert.match(router, /cut.*trim.*splice.*reorder[\s\S]{0,500}hyperframes-core/i); + assert.match( + router, + /zoom.*punch.*reframe.*Ken Burns.*camera move[\s\S]{0,500}hyperframes-keyframes/i, + ); + assert.match( + router, + /match cut.*whip pan[\s\S]{0,500}hyperframes-animation[\s\S]{0,300}hyperframes-keyframes[\s\S]{0,300}hyperframes-registry/i, + ); + assert.match(router, /fade.*crossfade.*gain.*duck[\s\S]{0,700}hyperframes-audio/i); + assert.match( + router, + /picture and sound[\s\S]{0,700}hyperframes-core[\s\S]{0,300}hyperframes-audio/i, + ); + assert.match(router, /media-use[\s\S]{0,180}sourc|sourc[\s\S]{0,180}media-use/i); +}); + +test("keyframes states truthful creator capabilities and ownership boundaries", async () => { + const keyframes = await read(files.keyframes); + requiresAll( + keyframes, + [ + /hard cut/i, + /trim[\s\S]{0,80}splice[\s\S]{0,80}reorder|trim[\s\S]{0,80}reorder[\s\S]{0,80}splice/i, + /punch[- ]in.*punch[- ]out/i, + /multi-state zoom|multiple zoom.*reframe states/i, + /Ken Burns/i, + /camera move/i, + /match cut/i, + /whip pan/i, + /data-start/, + /data-duration/, + /data-media-start/, + /hyperframes-core/, + /hyperframes-audio/, + ], + files.keyframes, + ); + assert.match( + keyframes, + /source[\s\S]{0,80}cut[\s\S]{0,80}trim[\s\S]{0,80}reorder[\s\S]{0,500}hyperframes-core/i, + ); + assert.match(keyframes, /non-timed|non-clip/); + assert.match(keyframes, /wrapper inside the clip|inner.*wrapper/i); + assert.match(keyframes, /speed ramps?[\s\S]{0,300}(not supported|preprocess)/i); + assert.match(keyframes, /arbitrary mid-source freeze[\s\S]{0,300}(not supported|preprocess)/i); + assert.doesNotMatch(keyframes, /keyframe(?:d|ing)?\s+(?:the\s+)?data-playback-rate/i); +}); + +test("CLI requires domain skills before authoring or diagnosing creator edits", async () => { + const cli = await read(files.cli); + assert.match( + cli, + /before[\s\S]{0,120}(zoom|punch-in)[\s\S]{0,180}(reframe|camera)[\s\S]{0,180}keyframe[\s\S]{0,220}read `?\/hyperframes-keyframes/i, + ); + assert.match(cli, /before `?hyperframes keyframes`?[\s\S]{0,180}read `?\/hyperframes-keyframes/i); + assert.match(cli, /cut.*trim.*splice.*source timing[\s\S]{0,250}hyperframes-core/i); + assert.match( + cli, + /fade[\s\S]{0,100}crossfade[\s\S]{0,100}volume automation[\s\S]{0,100}carve[\s\S]{0,100}FX[\s\S]{0,300}hyperframes-audio/i, + ); +}); + +test("audio skill owns placed-track fades, automation, ducking, and effects", async () => { + const audio = await read(files.audio); + requiresAll( + audio, + [ + /fade[- ]in.*fade[- ]out/i, + /crossfade/i, + /track gain|track volume/i, + /duck/i, + /data-automation/, + /gain.*EQ.*compressor.*limiter.*gate.*saturat.*delay.*reverb.*chorus.*phaser.*bitcrush/is, + /clip timing.*hyperframes-core|hyperframes-core.*clip timing/is, + /sourcing.*media-use|media-use.*sourcing/is, + ], + files.audio, + ); + assert.match(audio, /constant.*playback rate|data-playback-rate/i); + assert.match(audio, /speed ramps?[\s\S]{0,220}(not supported|preprocess)/i); +}); + +test("Remotion media mapping uses the canonical trim and render-safe constant-rate contract", async () => { + const [media, apiMap] = await Promise.all([read(files.remotionMedia), read(files.remotionMap)]); + const combined = `${media}\n${apiMap}`; + assert.match(combined, /data-media-start/); + assert.match(combined, /data-playback-rate/); + assert.match(combined, /constant.*playback rate|playback rate.*constant/i); + assert.match(combined, /volume automation|data-automation/i); + assert.doesNotMatch(combined, /data-trim-start|data-trim-end/); +}); diff --git a/skills-manifest.json b/skills-manifest.json index c2a8e5f6a9..0b04ab2a8b 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -18,7 +18,7 @@ "files": 4 }, "hyperframes": { - "hash": "756a72f58fa3739b", + "hash": "e05aca66e618a32e", "files": 17 }, "hyperframes-animation": { @@ -26,11 +26,11 @@ "files": 121 }, "hyperframes-audio": { - "hash": "534cea75fe0f2bc6", + "hash": "d8c1204ed074a2a6", "files": 6 }, "hyperframes-cli": { - "hash": "dc900e2ef0aa16b2", + "hash": "09865d0df77d5c1d", "files": 11 }, "hyperframes-core": { @@ -42,7 +42,7 @@ "files": 78 }, "hyperframes-keyframes": { - "hash": "d00744ff0e669624", + "hash": "f6e1d87c4ee19710", "files": 3 }, "hyperframes-registry": { @@ -70,7 +70,7 @@ "files": 28 }, "remotion-to-hyperframes": { - "hash": "bf184a65059b95e8", + "hash": "87b4f7b015f3dd99", "files": 70 }, "slideshow": { diff --git a/skills/hyperframes-audio/SKILL.md b/skills/hyperframes-audio/SKILL.md index 95ccc63a46..06179a4c89 100644 --- a/skills/hyperframes-audio/SKILL.md +++ b/skills/hyperframes-audio/SKILL.md @@ -2,6 +2,7 @@ name: hyperframes-audio description: > Use when audio already placed in a HyperFrames composition needs to be mixed: + fade-in/fade-out, crossfade, track gain or volume, volume automation, ducking, a music bed that fights a voiceover (voiceover carve), effects on a track (EQ, compressor, limiter, gate, saturation, delay, reverb, chorus, phaser, bitcrush), or automation envelopes drawn on a track's volume or any effect @@ -23,6 +24,19 @@ same Web Audio graph — the studio in a live context, the engine in an offline inside the browser it already drives. There is one implementation of each effect, so what you hear while scrubbing is what gets written. You never tune twice. +Clip timing remains `/hyperframes-core`: audio/video trims and source ranges use +`data-start`, `data-duration`, and `data-media-start`, and crossfades overlap +clips on different tracks. This skill owns placed-track fade-in/fade-out, +crossfade envelopes, track gain/track volume, volume and effect automation, +ducking/voiceover carve, and the effect chain. `/media-use` owns sourcing, +generation, and preprocessing. + +Constant `data-playback-rate` (`0.1..5`) is render-safe for picture and +pitch-preserved sound when matching audio/video elements use the same timing, +source offset, and rate. Source speed ramps are not supported because there is +no rate envelope; preprocess a derived synchronized asset. HyperFrames does not +provide automatic waveform sync or drift correction. + Three attributes carry everything, all on the audio/video element itself: | Attribute | Holds | @@ -31,6 +45,9 @@ Three attributes carry everything, all on the audio/video element itself: | `data-automation` | envelopes on this track's volume or its effect parameters | | `data-fx-carve` | the carve's own settings, so it can be re-derived | +The shipped effect families are gain, EQ (highpass, lowpass, peaking, shelves), +compressor, limiter, gate, saturate, delay, reverb, chorus, phaser, and bitcrush. + Exact JSON for each, and the rules a lane must satisfy: `references/attributes.md`. Every effect with its parameters, ranges and units: `references/fx-registry.md`. How to work out what is wrong with a file you cannot hear: diff --git a/skills/hyperframes-cli/SKILL.md b/skills/hyperframes-cli/SKILL.md index 0c05d814df..ed610843a9 100644 --- a/skills/hyperframes-cli/SKILL.md +++ b/skills/hyperframes-cli/SKILL.md @@ -25,6 +25,19 @@ Run commands as `npx hyperframes ...` unless project instructions provide a wrap 8. **Render only after approval:** use draft quality for iteration and high quality for delivery. 9. **Verify the output:** confirm the file exists, is non-empty, and has a plausible duration. +## Mandatory creator-edit cross-references + +- Before authoring or diagnosing a zoom, punch-in/punch-out, reframe, camera + move, or any keyframe motion, read `/hyperframes-keyframes` first. +- Before `hyperframes keyframes`, read `/hyperframes-keyframes`; the command + surfaces animation trajectories and does not diagnose clip cuts. +- For a cut, trim, splice, reorder, or source timing edit, read + `/hyperframes-core` and use its clip/timeline contract. +- For fade-in/fade-out, crossfade, track gain, volume automation, ducking, + voiceover carve, or FX on placed audio, read `/hyperframes-audio`. Load core + alongside it when clip placement or picture timing also changes. +- Use `/media-use` only to source/generate media or preprocess a derived asset. + ```bash # Fast iteration check; repeat while authoring as needed. npx hyperframes lint diff --git a/skills/hyperframes-keyframes/SKILL.md b/skills/hyperframes-keyframes/SKILL.md index cc981fae8e..74415f53eb 100644 --- a/skills/hyperframes-keyframes/SKILL.md +++ b/skills/hyperframes-keyframes/SKILL.md @@ -1,9 +1,10 @@ --- name: hyperframes-keyframes description: > - Use when a HyperFrames composition needs seek-safe 2D/3D keyframes, GSAP - timelines, CSS keyframes, Anime.js, WAAPI, FLIP, paths, masks, SVG morph/draw, - text trails, 3D depth, or `hyperframes keyframes` diagnostics. + Use when a HyperFrames composition needs a punch-in, punch-out, zoom, reframe, + Ken Burns treatment, camera move, visual match/whip handoff, or other seek-safe + 2D/3D keyframes; also for GSAP, CSS keyframes, Anime.js, WAAPI, FLIP, paths, + masks, SVG morph/draw, text trails, 3D depth, or `hyperframes keyframes` diagnostics. Don't use for broad scene strategy, brand design, media sourcing, captions, or general video planning. --- @@ -14,6 +15,30 @@ Keyframes are a pose contract: visible states, continuous subject identity, seek Use `hyperframes-animation` for broad scene recipes. Use `hyperframes-cli` for full command docs. Use `references/keyframe-patterns.md` only when choosing implementation mechanisms, not visual style. +## Creator editing boundary + +Keyframes own visual motion, not clip assembly. Source-range hard cuts, trim, +splice, and reorder belong to `/hyperframes-core`: author one media element per +kept range, place it with `data-start` and `data-duration`, and select its source +offset with `data-media-start`. Adjacent ranges make a hard cut. A crossfade +uses overlapping clips on different tracks plus visual opacity keyframes; sound +fades use `/hyperframes-audio`. + +| Creator request | Truthful mechanism | +| -------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Punch-in / punch-out | Keyframe `scale` with `x`/`y` or percentage translation on a non-timed visual/crop wrapper inside the clip. Use a set/short tween for a hard punch and a tween for a smooth move. | +| Smooth multi-state zoom or reframe | Keep one subject wrapper alive and author multiple zoom/reframe states as a pose ladder with per-segment easing. | +| Pan, reframe, or Ken Burns camera move | Animate wrapper translation plus scale. Geometry is authored; this is not face tracking or automatic semantic reframing. | +| Chained camera moves | Chain labeled transform beats on one registered seek-safe timeline. | +| Match cut or whip pan | `/hyperframes-animation` owns the visual handoff; `/hyperframes-registry` supplies primitives; keyframes preserve authored geometry, direction, and velocity. There is no automatic matching-frame discovery. | +| Constant source retime | `/hyperframes-core` owns normalized `data-playback-rate` (`0.1..5`) for render-safe picture and pitch-preserved sound. It is constant for the whole media element. | +| Source speed ramps | Not supported: there is no time-varying playback-rate envelope. Preprocess a derived media asset, then place it through core. | +| Freeze / hold | A visual pose, final source frame, or finished sub-composition can hold. Arbitrary mid-source freeze is not supported; preprocess a still/derived segment, place it as its own clip, then resume with another source range. | + +When editing picture and sound together, load `/hyperframes-core`, this skill for +visual motion, and `/hyperframes-audio` for fades, crossfades, volume automation, +ducking/carve, or effects on the placed tracks. + ## Procedure 1. Identify the animated subject, visible states, final state, and runtime. diff --git a/skills/hyperframes/SKILL.md b/skills/hyperframes/SKILL.md index fd2168c22c..845cad1652 100644 --- a/skills/hyperframes/SKILL.md +++ b/skills/hyperframes/SKILL.md @@ -97,6 +97,20 @@ Use the bare name without `/`. If the command fails, surface the error; do not r | Registry blocks and components | `/hyperframes-registry` | | Figma assets, tokens, components, or storyboard frames as reconstructed motion | `/figma` | +Creator edit phrases are cross-domain requests. Load every skill named in the matching row: + +| Creator request | Required domains | +| ------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| “cut this footage”, hard cut, trim, splice, reorder, or use a source range | `/general-video` + `/hyperframes-core`; core owns `data-start`, `data-duration`, `data-media-start`, and track layout. | +| zoom in here, punch-in / punch-out, smooth multi-state zoom or reframe, Ken Burns, or camera move | `/general-video` + `/hyperframes-core` + `/hyperframes-keyframes`; animate the inner visual/crop wrapper, not the timed clip. | +| match cut or whip pan camera transition | `/general-video` + `/hyperframes-animation` + `/hyperframes-keyframes` + `/hyperframes-registry`; search/install a transition primitive before hand-authoring. | +| fade, crossfade, track gain/volume, automation, duck/carve, or audio effects | `/general-video` + `/hyperframes-core` + `/hyperframes-audio`; core places clips, audio mixes placed tracks. | +| picture and sound edits that combine cuts with camera motion or mixing | `/general-video` + `/hyperframes-core` + `/hyperframes-keyframes` when there is visual motion + `/hyperframes-audio` when sound is faded, mixed, ducked, automated, or processed. | +| source or generate media, or preprocess an unsupported speed ramp/mid-source freeze | `/media-use`; sourcing/generation/preprocessing only, never placed-track mixing. | + +Constant `data-playback-rate` is render-safe for picture and pitch-preserved +sound. It does not make source speed ramps keyframeable; preprocess ramps. + Broad feedback about how photographic media looks or behaves also routes to `/media-use`, even when the user never says “color grading” or “effect”: fix dark/flat/boring footage, stylize a clip, hide a face, or improve a media diff --git a/skills/remotion-to-hyperframes/references/api-map.md b/skills/remotion-to-hyperframes/references/api-map.md index 08ce72f6c8..325f09a10d 100644 --- a/skills/remotion-to-hyperframes/references/api-map.md +++ b/skills/remotion-to-hyperframes/references/api-map.md @@ -53,16 +53,16 @@ See [timing.md](timing.md) — this is the highest-leverage section. See [media.md](media.md) for trim, volume ramps, and decoder notes. -| Remotion | HyperFrames | -| -------------------------------------- | --------------------------------------------------------------------------- | -| `