diff --git a/README.md b/README.md index 5158a6c8..bcd63a47 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,7 @@ dapi media grab clip.mp4 -t 0 12 45 # decode frames to PNGs dapi media filmstrip clip.mp4 # grid of video frames dapi media waveform track.mp3 # audio waveform, silence flagged dapi media transcribe interview.wav # timed, word-level transcript +dapi media autocut interview.mp4 --jsx # keep-ranges + optional JSX sequence dapi media listen interview.mp4 -p "what is said in the intro?" # ask a multimodal model dapi capture intro -t 0 2 4 # the frames a render would produce, by scene id ``` @@ -184,7 +185,7 @@ dapi capture intro -t 0 2 4 # the frames a render w | `dapi context` | Summary of app state | | `dapi capture` | Render frames of a scene, as an export would, to a labelled contact sheet or one PNG per position | | `dapi check` | Check a node's subtree for structural mistakes (black-frame gaps, never-visible nodes, failed sources) and report subtree stats | -| `dapi media …` | Inspect a file by id or path: `probe`, `grab`, `filmstrip`, `waveform`, `transcribe`, `listen` | +| `dapi media …` | Inspect a file by id or path: `probe`, `grab`, `filmstrip`, `waveform`, `transcribe`, `autocut`, `listen` | | `dapi models` / `dapi voices` / `dapi fonts` | Discover generation models, speech voices, local fonts | | `dapi screenshot` / `dapi logs` | The app itself: capture the window, read recent console output | | `dapi fetch` | Download a video from yt/tt/ig | diff --git a/apps/cli/package.json b/apps/cli/package.json index 06ea360d..3055b9c3 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -18,6 +18,7 @@ }, "dependencies": { "@babel/core": "^7.29.7", + "@diffusionstudio/runtime": "*", "@trpc/client": "^11.18.0", "@trpc/server": "^11.18.0", "@babel/preset-typescript": "^7.27.1", diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts index c869ab38..d4b9f138 100644 --- a/apps/cli/src/index.ts +++ b/apps/cli/src/index.ts @@ -15,6 +15,7 @@ import { editor, errnoCode, GENERATE_TIMEOUT_MS, waitForCliSocket } from "./cli- import { listLocalFonts } from "./fonts"; import { buildIssueBody, createIssue } from "./report"; import { fetchVideo } from "./ytdlp"; +import { computeAutocut, formatAutocutJsx, type Transcript } from "@diffusionstudio/runtime/media"; import { MAX_FRAMES_PER_SHEET } from "./protocol"; import type { AssetRef, FrameQuality, LogEntry, LogLevel, TimecodedImage } from "./protocol"; @@ -282,6 +283,117 @@ async function mediaWaveform(ref: string, opts: MediaPreviewOptions): Promise { + const { start, end } = parsePreviewWindow(opts); + const target = resolveAssetRef(ref); + + let silenceMin = 0.4; + if (opts.silenceMin !== undefined) { + silenceMin = Number(opts.silenceMin); + if (!Number.isFinite(silenceMin) || silenceMin < 0) { + console.error(`--silence-min must be a non-negative number (got "${opts.silenceMin}")`); + process.exit(1); + } + } + + let pad = 0.05; + if (opts.pad !== undefined) { + pad = Number(opts.pad); + if (!Number.isFinite(pad) || pad < 0) { + console.error(`--pad must be a non-negative number (got "${opts.pad}")`); + process.exit(1); + } + } + + const lang = opts.lang ?? "all"; + if (lang !== "en" && lang !== "es" && lang !== "all") { + console.error(`--lang must be one of en, es, all (got "${opts.lang}")`); + process.exit(1); + } + + type ProbeResult = { duration?: number; width?: number; height?: number; type?: string }; + + const stopProbe = startSpinner("Probing asset"); + let probe: ProbeResult; + try { + probe = (await editor.media.probe.query(target)) as ProbeResult; + stopProbe(); + if (typeof probe.duration !== "number" || !Number.isFinite(probe.duration) || probe.duration <= 0) { + console.error("Could not determine asset duration from probe metadata."); + process.exit(1); + } + } catch (e) { + stopProbe(); + handleSocketError(e); + } + + const duration = probe.duration!; + + const stopWave = startSpinner("Analyzing silences"); + let silences: Array<{ start: number; end: number }>; + try { + const { silences: spans } = await editor.media.waveform.query({ + ...target, + start, + end, + scale: 0.25, + }); + stopWave(); + silences = spans; + } catch (e) { + stopWave(); + handleSocketError(e); + } + + const stopTranscribe = startSpinner("Transcribing asset"); + let transcript: Transcript; + try { + transcript = await editor.media.transcribe.query(target, GENERATE); + stopTranscribe(); + } catch (e) { + stopTranscribe(); + const code = errnoCode(e); + if (code === "ENOENT" || code === "ECONNREFUSED") handleSocketError(e); + const msg = e instanceof Error ? e.message : String(e); + if (/no speech detected/i.test(msg)) { + console.error("No speech detected; continuing with silence-only cuts."); + } else { + console.error(`${msg}; continuing with silence-only cuts.`); + } + transcript = { segments: [] }; + } + + const result = computeAutocut( + { + duration, + silences, + transcript, + ...(start !== undefined || end !== undefined ? { window: { start: start ?? 0, end: end ?? duration } } : {}), + }, + { silenceMin, pad, lang: lang as "en" | "es" | "all" }, + ); + + const output: Record = { ...result }; + if (opts.jsx) { + output.jsx = formatAutocutJsx(ref, result.keep, { + width: probe.width, + height: probe.height, + kind: probe.type === "AUDIO" ? "audio" : "video", + }); + } + + console.log(JSON.stringify(output)); +} + type CaptureOptions = { time?: string[]; output?: string; separate?: boolean; perSheet?: string }; async function captureNode(id: string, opts: CaptureOptions): Promise { @@ -700,6 +812,20 @@ media .option("-o, --output ", "write the PNG here instead of a temp file") .action((ref: string, opts: MediaPreviewOptions) => mediaWaveform(ref, opts)); +media + .command("autocut") + .description( + `Propose keep-ranges for a jump-cut edit by composing waveform silence detection with a timed transcript: drops silent stretches, immediate word repeats (stutters), and vocal fillers (um/uh/eh/em) plus safe phrases (you know, i mean, o sea). Does not re-encode — returns second ranges to trim with \`sourceIn\`/\`sourceOut\`, and optionally a JSX \`\` of back-to-back clips. Uses cloud STT when signed in; continues with silence-only cuts when transcription is unavailable or no speech is detected.`, + ) + .argument("", "local video or audio file path, or library path") + .option("-s, --start