diff --git a/docs/guides/voice-and-audio.mdx b/docs/guides/voice-and-audio.mdx index 4687b19db4b..31ba8df0205 100644 --- a/docs/guides/voice-and-audio.mdx +++ b/docs/guides/voice-and-audio.mdx @@ -113,6 +113,11 @@ When narration drives the project, time visual changes to its real transcript rather than estimated scene lengths. When music drives the project, cut to its analyzed structure (see [music to video](/guides/music-to-video)). +Music can start after narration begins. When the media-use skill creates a +ducking lane for that clip, its opening gain matches the speech at that point +in the composition. Starting during an attack or release continues that ramp +from the correct level; completed speech does not create a new dip. + ## Review the complete sound pass - The correct voice and source recordings are used. diff --git a/packages/cli/src/media-use/lib/duck.mjs b/packages/cli/src/media-use/lib/duck.mjs index dcf81774f39..8541f589687 100644 --- a/packages/cli/src/media-use/lib/duck.mjs +++ b/packages/cli/src/media-use/lib/duck.mjs @@ -63,11 +63,27 @@ export function duckLane(keyframes, { clipStart = 0, baseVolume = 1 } = {}) { if (t > points.at(-1).t) points.push({ t: round3(t), v }); }; for (const kf of keyframes) { - const t = Math.max(0, kf.time - start); + const t = Math.max(0, kf.time); push(t, points.at(-1).v); push(t + kf.duration, kf.volume); } - return { version: 1, lanes: [{ target: "volume", points }] }; + + const next = points.findIndex((point) => point.t > start); + const before = points[next - 1] ?? points.at(-1); + const after = points[next]; + const initial = + next === 0 + ? points[0].v + : after + ? before.v + ((after.v - before.v) * (start - before.t)) / (after.t - before.t) + : before.v; + const localPoints = [ + { t: 0, v: initial }, + ...points + .filter((point) => point.t > start) + .map((point) => ({ t: round3(point.t - start), v: point.v })), + ]; + return { version: 1, lanes: [{ target: "volume", points: localPoints }] }; } function mergeIntervals(intervals, mergeGap) { diff --git a/packages/cli/src/media-use/lib/duck.test.mjs b/packages/cli/src/media-use/lib/duck.test.mjs index 164c12bbdc5..dcb39a36235 100644 --- a/packages/cli/src/media-use/lib/duck.test.mjs +++ b/packages/cli/src/media-use/lib/duck.test.mjs @@ -169,6 +169,85 @@ test("duckLane turns composition-time keyframes into clip-local ramps", () => { }); }); +for (const [name, clipStart, expected] of [ + [ + "before the attack", + 0.5, + [ + { t: 0, v: 0.6 }, + { t: 0.5, v: 0.6 }, + { t: 0.65, v: 0.15 }, + { t: 2.5, v: 0.15 }, + { t: 2.9, v: 0.6 }, + ], + ], + [ + "during the attack", + 1.075, + [ + { t: 0, v: 0.375 }, + { t: 0.075, v: 0.15 }, + { t: 1.925, v: 0.15 }, + { t: 2.325, v: 0.6 }, + ], + ], + [ + "during speech", + 2, + [ + { t: 0, v: 0.15 }, + { t: 1, v: 0.15 }, + { t: 1.4, v: 0.6 }, + ], + ], + [ + "at the release", + 3, + [ + { t: 0, v: 0.15 }, + { t: 0.4, v: 0.6 }, + ], + ], + [ + "during the release", + 3.2, + [ + { t: 0, v: 0.375 }, + { t: 0.2, v: 0.6 }, + ], + ], + ["after narration", 4, [{ t: 0, v: 0.6 }]], +]) { + test(`duckLane preserves composition-time gain when music starts ${name}`, () => { + const keyframes = duckKeyframes([{ start: 1, end: 3 }], { baseVolume: 0.6 }); + const points = duckLane(keyframes, { clipStart, baseVolume: 0.6 }).lanes[0].points; + + assert.equal(points.length, expected.length); + for (let i = 0; i < points.length; i++) { + assert.equal(points[i].t, expected[i].t); + assert.ok( + Math.abs(points[i].v - expected[i].v) < 1e-9, + `gain at ${points[i].t}s: expected ${expected[i].v}, received ${points[i].v}`, + ); + } + }); +} + +test("duckLane discards earlier narration and retains the next speech span", () => { + const keyframes = duckKeyframes([ + { start: 1, end: 2 }, + { start: 5, end: 6 }, + ]); + + assert.deepEqual(duckLane(keyframes, { clipStart: 3 }).lanes[0].points, [ + { t: 0, v: 1 }, + { t: 2, v: 1 }, + { t: 2.15, v: 0.25 }, + { t: 3, v: 0.25 }, + { t: 3.4, v: 1 }, + ]); +}); + test("--json spans match --merge-gap semantics exactly", () => { const dir = mkdtempSync(join(tmpdir(), "media-use-duck-")); try { diff --git a/skills-manifest.json b/skills-manifest.json index e6442718b8a..6c88a545c50 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -18,7 +18,7 @@ "files": 13 }, "media-use": { - "hash": "1082bd1c4e3f033b", + "hash": "6550641984b19843", "files": 116 }, "remotion-to-hyperframes": { diff --git a/skills/media-use/scripts/lib/duck.mjs b/skills/media-use/scripts/lib/duck.mjs index dcf81774f39..8541f589687 100644 --- a/skills/media-use/scripts/lib/duck.mjs +++ b/skills/media-use/scripts/lib/duck.mjs @@ -63,11 +63,27 @@ export function duckLane(keyframes, { clipStart = 0, baseVolume = 1 } = {}) { if (t > points.at(-1).t) points.push({ t: round3(t), v }); }; for (const kf of keyframes) { - const t = Math.max(0, kf.time - start); + const t = Math.max(0, kf.time); push(t, points.at(-1).v); push(t + kf.duration, kf.volume); } - return { version: 1, lanes: [{ target: "volume", points }] }; + + const next = points.findIndex((point) => point.t > start); + const before = points[next - 1] ?? points.at(-1); + const after = points[next]; + const initial = + next === 0 + ? points[0].v + : after + ? before.v + ((after.v - before.v) * (start - before.t)) / (after.t - before.t) + : before.v; + const localPoints = [ + { t: 0, v: initial }, + ...points + .filter((point) => point.t > start) + .map((point) => ({ t: round3(point.t - start), v: point.v })), + ]; + return { version: 1, lanes: [{ target: "volume", points: localPoints }] }; } function mergeIntervals(intervals, mergeGap) {