Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions docs/guides/voice-and-audio.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,17 @@ JSON file:
npx hyperframes transcribe subtitles.srt
```

## Keep the ending when cutting speech

When asking the agent to remove fillers or cut spoken passages, supply the
original audio or video along with its transcript. The source duration keeps
footage and sound after the last spoken word, and lets you keep an explicit
range in that ending. For example, removing a filler from a six-second clip
whose last word ends at four seconds preserves the remaining two seconds.

A plan made from a transcript alone ends at the last word. Include the source
when reviewing the plan to account for the complete clip before encoding.

## Turn the transcript into captions

1. Read the complete transcript.
Expand Down
2 changes: 0 additions & 2 deletions packages/cli/src/media-use/lib/cutlist.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -144,8 +144,6 @@ test("--plan on a fixture transcript prints the exact segment JSON", () => {
process.execPath,
[
SCRIPT,
"--input",
"ignored.mp4",
"--transcript",
transcriptPath,
"--remove",
Expand Down
2 changes: 1 addition & 1 deletion skills-manifest.json
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
"files": 13
},
"media-use": {
"hash": "6550641984b19843",
"hash": "66117d8c31118930",
"files": 116
},
"remotion-to-hyperframes": {
Expand Down
2 changes: 1 addition & 1 deletion skills/media-use/references/operations.md
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ node <SKILL_DIR>/scripts/transcript-cut.mjs \
resolve --from talk.cut.mp4 --type video
```

Use `--plan` first when you want to inspect the kept segment JSON before encoding.
Use `--plan` with `--input` first when you want to inspect the kept segment JSON before encoding. The source's measured duration bounds the edits, including footage after the last word. A transcript-only `--plan` needs no media tools and uses the last word's end as its duration.

## Ducking (declare in-composition / bake for export)

Expand Down
10 changes: 6 additions & 4 deletions skills/media-use/scripts/transcript-cut.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ Options:
--cut-silence Remove inter-word gaps longer than this many seconds
--keep Inverse mode: direct kept ranges, mutually exclusive with removal
--copy Use stream copy for faster, keyframe-snapped cuts
--plan Print kept segment JSON and exit without ffmpeg
--plan Print kept segment JSON; uses source duration when --input is supplied
--out Output file
--json Output JSON status
--help, -h Show this help`);
Expand All @@ -66,8 +66,12 @@ try {

function run() {
if (!args.transcript) throw new Error("--transcript is required");
const inputPath = args.input ? resolve(args.input) : null;
// A last word at 4s must not drop a 6s source's outro when removing a filler.
const totalSeconds = inputPath ? probeDuration(inputPath) : null;
const transcript = JSON.parse(readFileSync(resolve(args.transcript), "utf8"));
const segments = compileCutList(transcript, {
duration: totalSeconds,
remove: args.remove,
removeWords: args["remove-words"],
removeFillers: args["remove-fillers"],
Expand All @@ -80,16 +84,14 @@ function run() {
return;
}

if (!args.input || !args.out)
if (!inputPath || !args.out)
throw new Error("--input and --out are required unless --plan is set");
if (segments.length === 0) throw new Error("cut list has no kept segments");

const inputPath = resolve(args.input);
const outPath = resolve(args.out);
mkdirSync(dirname(outPath), { recursive: true });
const tmpDir = mkdtempSync(join(tmpdir(), "media-use-cut-"));
const keptSeconds = sumDurations(segments);
const totalSeconds = probeDuration(inputPath);

try {
const parts = segments.map((segment, index) => {
Expand Down
84 changes: 73 additions & 11 deletions skills/media-use/scripts/transcript-cut.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,20 @@ function run(args) {
return spawnSync(process.execPath, [SCRIPT, ...args], { encoding: "utf8" });
}

function writeTone(source) {
execFileSync("ffmpeg", [
"-y",
"-hide_banner",
"-loglevel",
"error",
"-f",
"lavfi",
"-i",
`sine=frequency=440:duration=6:sample_rate=${SAMPLE_RATE}`,
source,
]);
}

function readPcm(filePath) {
const raw = execFileSync("ffmpeg", [
"-hide_banner",
Expand All @@ -47,6 +61,64 @@ function readPcm(filePath) {
return samples;
}

for (const { name, edits, segments, duration } of [
{
name: "preserves the source tail when removing a filler",
edits: ["--remove-fillers", "um"],
segments: [
{ start: 0, end: 1 },
{ start: 1.5, end: 6 },
],
duration: 5.5,
},
{
name: "keeps an explicit range after the last spoken word",
edits: ["--keep", "4-6"],
segments: [{ start: 4, end: 6 }],
duration: 2,
},
]) {
test(name, { skip: !HAS_FFMPEG }, (t) => {
const { dir, cleanup } = fixture();
t.after(cleanup);
const source = join(dir, "tone.wav");
writeTone(source);
const transcriptPath = join(dir, "transcript.json");
writeFileSync(
transcriptPath,
JSON.stringify({
words: [
{ text: "hello", start: 0, end: 1 },
{ text: "um", start: 1, end: 1.5 },
{ text: "goodbye", start: 1.5, end: 4 },
],
}),
);
const output = join(dir, "out.wav");
const result = run([
"--input",
source,
"--transcript",
transcriptPath,
...edits,
"--out",
output,
"--json",
]);
assert.equal(result.status, 0, result.stderr || result.stdout);
const status = JSON.parse(result.stdout);
assert.deepEqual(status.segments, segments);
assert.equal(status.kept_s, duration);
assert.equal(status.total_s, 6);
const plan = run(["--input", source, "--transcript", transcriptPath, ...edits, "--plan"]);
assert.equal(plan.status, 0, plan.stderr || plan.stdout);
assert.deepEqual(JSON.parse(plan.stdout), segments);
const samples = readPcm(output);
assert.equal(samples.length, duration * SAMPLE_RATE);
assert.ok(samples.slice(-SAMPLE_RATE).some((sample) => Math.abs(sample) > 1000));
});
}

// Three kept ranges, none starting or ending on a frame boundary of any rate below.
const UNALIGNED_KEEP = "0.137-1.913,2.845-4.111,4.9-5.47";
const UNALIGNED_KEPT_SECONDS = 1.776 + 1.266 + 0.57;
Expand Down Expand Up @@ -204,17 +276,7 @@ test(
t.after(cleanup);

const source = join(dir, "tone.wav");
execFileSync("ffmpeg", [
"-y",
"-hide_banner",
"-loglevel",
"error",
"-f",
"lavfi",
"-i",
`sine=frequency=440:duration=6:sample_rate=${SAMPLE_RATE}`,
source,
]);
writeTone(source);

const transcriptPath = join(dir, "transcript.json");
writeFileSync(transcriptPath, JSON.stringify({ words: [{ text: "tone", start: 0, end: 6 }] }));
Expand Down
Loading