diff --git a/docs/guides/voice-and-audio.mdx b/docs/guides/voice-and-audio.mdx index b4d03ee97b2..477657e8f65 100644 --- a/docs/guides/voice-and-audio.mdx +++ b/docs/guides/voice-and-audio.mdx @@ -76,6 +76,17 @@ JSON file: npx hyperframes transcribe subtitles.srt ``` +## Keep the ending when cutting speech + +When asking the agent to remove fillers or cut spoken passages, supply the +original audio or video along with its transcript. The source duration keeps +footage and sound after the last spoken word, and lets you keep an explicit +range in that ending. For example, removing a filler from a six-second clip +whose last word ends at four seconds preserves the remaining two seconds. + +A plan made from a transcript alone ends at the last word. Include the source +when reviewing the plan to account for the complete clip before encoding. + ## Turn the transcript into captions 1. Read the complete transcript. diff --git a/packages/cli/src/media-use/lib/cutlist.test.mjs b/packages/cli/src/media-use/lib/cutlist.test.mjs index 3ae34a91292..14a43e721a3 100644 --- a/packages/cli/src/media-use/lib/cutlist.test.mjs +++ b/packages/cli/src/media-use/lib/cutlist.test.mjs @@ -144,8 +144,6 @@ test("--plan on a fixture transcript prints the exact segment JSON", () => { process.execPath, [ SCRIPT, - "--input", - "ignored.mp4", "--transcript", transcriptPath, "--remove", diff --git a/skills-manifest.json b/skills-manifest.json index 75aabb96626..07b6e364616 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -18,7 +18,7 @@ "files": 13 }, "media-use": { - "hash": "6550641984b19843", + "hash": "66117d8c31118930", "files": 116 }, "remotion-to-hyperframes": { diff --git a/skills/media-use/references/operations.md b/skills/media-use/references/operations.md index 8c461e898d3..99723305924 100644 --- a/skills/media-use/references/operations.md +++ b/skills/media-use/references/operations.md @@ -145,7 +145,7 @@ node /scripts/transcript-cut.mjs \ resolve --from talk.cut.mp4 --type video ``` -Use `--plan` first when you want to inspect the kept segment JSON before encoding. +Use `--plan` with `--input` first when you want to inspect the kept segment JSON before encoding. The source's measured duration bounds the edits, including footage after the last word. A transcript-only `--plan` needs no media tools and uses the last word's end as its duration. ## Ducking (declare in-composition / bake for export) diff --git a/skills/media-use/scripts/transcript-cut.mjs b/skills/media-use/scripts/transcript-cut.mjs index eb0bcbfaf09..1f2ed4d52f9 100644 --- a/skills/media-use/scripts/transcript-cut.mjs +++ b/skills/media-use/scripts/transcript-cut.mjs @@ -42,7 +42,7 @@ Options: --cut-silence Remove inter-word gaps longer than this many seconds --keep Inverse mode: direct kept ranges, mutually exclusive with removal --copy Use stream copy for faster, keyframe-snapped cuts - --plan Print kept segment JSON and exit without ffmpeg + --plan Print kept segment JSON; uses source duration when --input is supplied --out Output file --json Output JSON status --help, -h Show this help`); @@ -66,8 +66,12 @@ try { function run() { if (!args.transcript) throw new Error("--transcript is required"); + const inputPath = args.input ? resolve(args.input) : null; + // A last word at 4s must not drop a 6s source's outro when removing a filler. + const totalSeconds = inputPath ? probeDuration(inputPath) : null; const transcript = JSON.parse(readFileSync(resolve(args.transcript), "utf8")); const segments = compileCutList(transcript, { + duration: totalSeconds, remove: args.remove, removeWords: args["remove-words"], removeFillers: args["remove-fillers"], @@ -80,16 +84,14 @@ function run() { return; } - if (!args.input || !args.out) + if (!inputPath || !args.out) throw new Error("--input and --out are required unless --plan is set"); if (segments.length === 0) throw new Error("cut list has no kept segments"); - const inputPath = resolve(args.input); const outPath = resolve(args.out); mkdirSync(dirname(outPath), { recursive: true }); const tmpDir = mkdtempSync(join(tmpdir(), "media-use-cut-")); const keptSeconds = sumDurations(segments); - const totalSeconds = probeDuration(inputPath); try { const parts = segments.map((segment, index) => { diff --git a/skills/media-use/scripts/transcript-cut.test.mjs b/skills/media-use/scripts/transcript-cut.test.mjs index 95852adb825..96593a2ed11 100644 --- a/skills/media-use/scripts/transcript-cut.test.mjs +++ b/skills/media-use/scripts/transcript-cut.test.mjs @@ -28,6 +28,20 @@ function run(args) { return spawnSync(process.execPath, [SCRIPT, ...args], { encoding: "utf8" }); } +function writeTone(source) { + execFileSync("ffmpeg", [ + "-y", + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + `sine=frequency=440:duration=6:sample_rate=${SAMPLE_RATE}`, + source, + ]); +} + function readPcm(filePath) { const raw = execFileSync("ffmpeg", [ "-hide_banner", @@ -47,6 +61,64 @@ function readPcm(filePath) { return samples; } +for (const { name, edits, segments, duration } of [ + { + name: "preserves the source tail when removing a filler", + edits: ["--remove-fillers", "um"], + segments: [ + { start: 0, end: 1 }, + { start: 1.5, end: 6 }, + ], + duration: 5.5, + }, + { + name: "keeps an explicit range after the last spoken word", + edits: ["--keep", "4-6"], + segments: [{ start: 4, end: 6 }], + duration: 2, + }, +]) { + test(name, { skip: !HAS_FFMPEG }, (t) => { + const { dir, cleanup } = fixture(); + t.after(cleanup); + const source = join(dir, "tone.wav"); + writeTone(source); + const transcriptPath = join(dir, "transcript.json"); + writeFileSync( + transcriptPath, + JSON.stringify({ + words: [ + { text: "hello", start: 0, end: 1 }, + { text: "um", start: 1, end: 1.5 }, + { text: "goodbye", start: 1.5, end: 4 }, + ], + }), + ); + const output = join(dir, "out.wav"); + const result = run([ + "--input", + source, + "--transcript", + transcriptPath, + ...edits, + "--out", + output, + "--json", + ]); + assert.equal(result.status, 0, result.stderr || result.stdout); + const status = JSON.parse(result.stdout); + assert.deepEqual(status.segments, segments); + assert.equal(status.kept_s, duration); + assert.equal(status.total_s, 6); + const plan = run(["--input", source, "--transcript", transcriptPath, ...edits, "--plan"]); + assert.equal(plan.status, 0, plan.stderr || plan.stdout); + assert.deepEqual(JSON.parse(plan.stdout), segments); + const samples = readPcm(output); + assert.equal(samples.length, duration * SAMPLE_RATE); + assert.ok(samples.slice(-SAMPLE_RATE).some((sample) => Math.abs(sample) > 1000)); + }); +} + // Three kept ranges, none starting or ending on a frame boundary of any rate below. const UNALIGNED_KEEP = "0.137-1.913,2.845-4.111,4.9-5.47"; const UNALIGNED_KEPT_SECONDS = 1.776 + 1.266 + 0.57; @@ -204,17 +276,7 @@ test( t.after(cleanup); const source = join(dir, "tone.wav"); - execFileSync("ffmpeg", [ - "-y", - "-hide_banner", - "-loglevel", - "error", - "-f", - "lavfi", - "-i", - `sine=frequency=440:duration=6:sample_rate=${SAMPLE_RATE}`, - source, - ]); + writeTone(source); const transcriptPath = join(dir, "transcript.json"); writeFileSync(transcriptPath, JSON.stringify({ words: [{ text: "tone", start: 0, end: 6 }] }));