Autocut: jump-cut silences, fillers, and stutters (inspired by CapCut)

README.md

@@ -174,6 +174,7 @@ dapi media grab clip.mp4 -t 0 12 45                      # decode frames to PNGs
 dapi media filmstrip clip.mp4                            # grid of video frames
 dapi media waveform track.mp3                            # audio waveform, silence flagged
 dapi media transcribe interview.wav                      # timed, word-level transcript
+dapi media autocut interview.mp4 --jsx                 # keep-ranges + optional JSX sequence
 dapi media listen interview.mp4 -p "what is said in the intro?"   # ask a multimodal model
 dapi capture intro -t 0 2 4                              # the frames a render would produce, by scene id
 ```
@@ -184,7 +185,7 @@ dapi capture intro -t 0 2 4                              # the frames a render w
 | `dapi context` | Summary of app state |
 | `dapi capture` | Render frames of a scene, as an export would, to a labelled contact sheet or one PNG per position |
 | `dapi check` | Check a node's subtree for structural mistakes (black-frame gaps, never-visible nodes, failed sources) and report subtree stats |
-| `dapi media …` | Inspect a file by id or path: `probe`, `grab`, `filmstrip`, `waveform`, `transcribe`, `listen` |
+| `dapi media …` | Inspect a file by id or path: `probe`, `grab`, `filmstrip`, `waveform`, `transcribe`, `autocut`, `listen` |
 | `dapi models` / `dapi voices` / `dapi fonts` | Discover generation models, speech voices, local fonts |
 | `dapi screenshot` / `dapi logs` | The app itself: capture the window, read recent console output |
 | `dapi fetch` | Download a video from yt/tt/ig |

apps/cli/package.json

@@ -18,6 +18,7 @@
   },
   "dependencies": {
     "@babel/core": "^7.29.7",
+    "@diffusionstudio/runtime": "*",
     "@trpc/client": "^11.18.0",
     "@trpc/server": "^11.18.0",
     "@babel/preset-typescript": "^7.27.1",

apps/cli/src/index.ts

@@ -15,6 +15,7 @@ import { editor, errnoCode, GENERATE_TIMEOUT_MS, waitForCliSocket } from "./cli-
 import { listLocalFonts } from "./fonts";
 import { buildIssueBody, createIssue } from "./report";
 import { fetchVideo } from "./ytdlp";
+import { computeAutocut, formatAutocutJsx, type Transcript } from "@diffusionstudio/runtime/media";
 import { MAX_FRAMES_PER_SHEET } from "./protocol";
 import type { AssetRef, FrameQuality, LogEntry, LogLevel, TimecodedImage } from "./protocol";
 
@@ -282,6 +283,117 @@ async function mediaWaveform(ref: string, opts: MediaPreviewOptions): Promise<vo
   }
 }
 
+type MediaAutocutOptions = {
+  silenceMin?: string;
+  pad?: string;
+  lang?: string;
+  jsx?: boolean;
+  start?: string;
+  end?: string;
+};
+
+async function mediaAutocut(ref: string, opts: MediaAutocutOptions): Promise<void> {
+  const { start, end } = parsePreviewWindow(opts);
+  const target = resolveAssetRef(ref);
+
+  let silenceMin = 0.4;
+  if (opts.silenceMin !== undefined) {
+    silenceMin = Number(opts.silenceMin);
+    if (!Number.isFinite(silenceMin) || silenceMin < 0) {
+      console.error(`--silence-min must be a non-negative number (got "${opts.silenceMin}")`);
+      process.exit(1);
+    }
+  }
+
+  let pad = 0.05;
+  if (opts.pad !== undefined) {
+    pad = Number(opts.pad);
+    if (!Number.isFinite(pad) || pad < 0) {
+      console.error(`--pad must be a non-negative number (got "${opts.pad}")`);
+      process.exit(1);
+    }
+  }
+
+  const lang = opts.lang ?? "all";
+  if (lang !== "en" && lang !== "es" && lang !== "all") {
+    console.error(`--lang must be one of en, es, all (got "${opts.lang}")`);
+    process.exit(1);
+  }
+
+  type ProbeResult = { duration?: number; width?: number; height?: number; type?: string };
+
+  const stopProbe = startSpinner("Probing asset");
+  let probe: ProbeResult;
+  try {
+    probe = (await editor.media.probe.query(target)) as ProbeResult;
+    stopProbe();
+    if (typeof probe.duration !== "number" || !Number.isFinite(probe.duration) || probe.duration <= 0) {
+      console.error("Could not determine asset duration from probe metadata.");
+      process.exit(1);
+    }
+  } catch (e) {
+    stopProbe();
+    handleSocketError(e);
+  }
+
+  const duration = probe.duration!;
+
+  const stopWave = startSpinner("Analyzing silences");
+  let silences: Array<{ start: number; end: number }>;
+  try {
+    const { silences: spans } = await editor.media.waveform.query({
+      ...target,
+      start,
+      end,
+      scale: 0.25,
+    });
+    stopWave();
+    silences = spans;
+  } catch (e) {
+    stopWave();
+    handleSocketError(e);
+  }
+
+  const stopTranscribe = startSpinner("Transcribing asset");
+  let transcript: Transcript;
+  try {
+    transcript = await editor.media.transcribe.query(target, GENERATE);
+    stopTranscribe();
+  } catch (e) {
+    stopTranscribe();
+    const code = errnoCode(e);
+    if (code === "ENOENT" || code === "ECONNREFUSED") handleSocketError(e);
+    const msg = e instanceof Error ? e.message : String(e);
+    if (/no speech detected/i.test(msg)) {
+      console.error("No speech detected; continuing with silence-only cuts.");
+    } else {
+      console.error(`${msg}; continuing with silence-only cuts.`);
+    }
+    transcript = { segments: [] };
+  }
+
+  const result = computeAutocut(
+    {
+      duration,
+      silences,
+      transcript,
+      ...(start !== undefined || end !== undefined ? { window: { start: start ?? 0, end: end ?? duration } } : {}),
+    },
+    { silenceMin, pad, lang: lang as "en" | "es" | "all" },
+  );
+
+  const output: Record<string, unknown> = { ...result };
+  if (opts.jsx) {
+    output.jsx = formatAutocutJsx(ref, result.keep, {
+      width: probe.width,
+      height: probe.height,
+      kind: probe.type === "AUDIO" ? "audio" : "video",
+    });
+  }
+
+  console.log(JSON.stringify(output));
+}
+
 type CaptureOptions = { time?: string[]; output?: string; separate?: boolean; perSheet?: string };
 
 async function captureNode(id: string, opts: CaptureOptions): Promise<void> {
@@ -700,6 +812,20 @@ media
   .option("-o, --output <path>", "write the PNG here instead of a temp file")
   .action((ref: string, opts: MediaPreviewOptions) => mediaWaveform(ref, opts));
 
+media
+  .command("autocut")
+  .description(
+    `Propose keep-ranges for a jump-cut edit by composing waveform silence detection with a timed transcript: drops silent stretches, immediate word repeats (stutters), and vocal fillers (um/uh/eh/em) plus safe phrases (you know, i mean, o sea). Does not re-encode — returns second ranges to trim with \`sourceIn\`/\`sourceOut\`, and optionally a JSX \`<sequence>\` of back-to-back clips. Uses cloud STT when signed in; continues with silence-only cuts when transcription is unavailable or no speech is detected.`,
+  )
+  .argument("<path>", "local video or audio file path, or library path")
+  .option("-s, --start <time>", `start of the window to analyze — seconds, "45f" frames, or "MM:SS" (default: 0)`)
+  .option("-e, --end <time>", `end of the window to analyze — seconds, "45f" frames, or "MM:SS" (default: asset duration)`)
+  .option("--silence-min <seconds>", "drop silences at least this long (default: 0.4; waveform amplitude threshold is fixed in the app)")
+  .option("--pad <seconds>", "keep this much audio on each side of a cut so speech does not clip (default: 0.05)")
+  .option("--lang <code>", "filler vocabulary: en, es, or all (default: all)")
+  .option("--jsx", "include a JSX <sequence> string using sourceIn/sourceOut for each kept span")
+  .action((ref: string, opts: MediaAutocutOptions) => mediaAutocut(ref, opts));
+
 media
   .command("listen")
   .description(

apps/web/src/components/sidebar-left/project-menu/object-menu.tsx

@@ -22,6 +22,16 @@ import {
   wrapSelectionInScene,
   wrapSelectionInSequence,
 } from "@/engine";
+import {
+  analyzeClipForAutocut,
+  applyAutocutToClip,
+  autocutWouldChange,
+  isAutocutClip,
+  planAutocutTimeline,
+  timelineStartSeconds,
+} from "@/engine/autocut";
+import { getEditHistory } from "@/engine/history";
+import { toast } from "somoto";
 
 export function ObjectMenu() {
   const world = useWorld();
@@ -186,6 +196,51 @@ export function ObjectMenu() {
 }
 
 export function ObjectAiMenu() {
+  const world = useWorld();
+  const { nodes } = useSelection();
+
+  const canRemoveSilences = () => {
+    const eligible = nodes().filter((entity) => isAutocutClip(world, entity));
+    return eligible.length === 1;
+  };
+
+  const removeSilences = async () => {
+    const eligible = nodes().filter((entity) => isAutocutClip(world, entity));
+    const entity = eligible.length === 1 ? eligible[0] : null;
+    if (!entity) return;
+
+    const history = getEditHistory(world);
+
+    try {
+      const result = await analyzeClipForAutocut(world, entity);
+      if (!autocutWouldChange(result.removed)) {
+        toast("Nothing to cut", { description: "No silences, fillers, or stutters were found in this clip." });
+        return;
+      }
+
+      const specs = planAutocutTimeline(result.keep, timelineStartSeconds(world, entity));
+      if (specs.length === 0) {
+        toast("Nothing to cut", { description: "Remove silences would remove the entire clip." });
+        return;
+      }
+
+      history.beginGesture();
+      try {
+        applyAutocutToClip(world, entity, specs);
+      } finally {
+        history.endGesture();
+      }
+
+      toast("Silences removed", {
+        description: `${specs.length} clip${specs.length === 1 ? "" : "s"} on the timeline.`,
+      });
+    } catch (err) {
+      toast.error("Remove silences failed", {
+        description: err instanceof Error ? err.message : String(err),
+      });
+    }
+  };
+
   return (
     <>
       <DropdownMenuGroup>
@@ -196,7 +251,9 @@ export function ObjectAiMenu() {
       <DropdownMenuSeparator />
 
       <DropdownMenuGroup>
-        <DropdownMenuItem>Remove silences</DropdownMenuItem>
+        <DropdownMenuItem disabled={!canRemoveSilences()} onSelect={() => void removeSilences()}>
+          Remove silences
+        </DropdownMenuItem>
         <DropdownMenuItem>Lip sync</DropdownMenuItem>
       </DropdownMenuGroup>
     </>