From b1ac87a430756ca4585b250ebc5ef8b57ac30616 Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Fri, 31 Jul 2026 02:09:19 -0700 Subject: [PATCH] feat(engine,producer): apply VST chains in the offline audio mix A track carrying data-vst-chain is bounced through the VST host after it is trimmed and before volume automation is baked, so plugins see the dry signal and the envelope applies to their output. A VST failure is deliberately fatal for the whole mix rather than a per-track soft failure. Every other audio failure mode degrades gracefully (source missing, download failure, extract/prepare failure): the track drops, a failure is recorded, siblings continue. Silently substituting the untreated signal for a carved one would ship a render that sounds plausible and is wrong, which is worse than a loud failure, so VstChainProcessingError escapes the per-element catch and rejects the whole call. Because the per-element work races under Promise.all, that rejection does not wait for in-flight siblings. An internal AbortController (chained off the caller's signal) is threaded through extract, prepare and the bounce so a sibling's sidecar subprocess is aborted before workDir is removed from under it. Lambda has no plugin host, so `lambda render` and `lambda render-batch` both reject a composition with any VST chain up front, before the AWS-calling code path is reached. --- packages/cli/src/commands/lambda.test.ts | 128 +++++ packages/cli/src/commands/lambda.ts | 28 ++ .../engine/src/services/audioMixer.test.ts | 452 ++++++++++++------ packages/engine/src/services/audioMixer.ts | 358 ++++++++------ .../engine/src/services/audioMixer.types.ts | 2 + .../engine/src/services/vstBounce.test.ts | 62 +++ packages/engine/src/services/vstBounce.ts | 144 ++++++ .../src/services/vstSidecarTestFixture.ts | 20 + .../src/services/vstRenderParity.test.ts | 149 ++++++ 9 files changed, 1046 insertions(+), 297 deletions(-) create mode 100644 packages/cli/src/commands/lambda.test.ts create mode 100644 packages/engine/src/services/vstBounce.test.ts create mode 100644 packages/engine/src/services/vstBounce.ts create mode 100644 packages/engine/src/services/vstSidecarTestFixture.ts create mode 100644 packages/producer/src/services/vstRenderParity.test.ts diff --git a/packages/cli/src/commands/lambda.test.ts b/packages/cli/src/commands/lambda.test.ts new file mode 100644 index 0000000000..6fc6fe0c4f --- /dev/null +++ b/packages/cli/src/commands/lambda.test.ts @@ -0,0 +1,128 @@ +/** + * Regression coverage for the VST-chain guard in `hyperframes lambda render` + * (Task 8): plugins can't run in Lambda, so a composition with any + * `data-vst-chain` audio track must be rejected up front — before the + * dynamic import of `./lambda/render.js` (and everything that pulls in: + * S3 upload, Step Functions, real AWS calls) ever runs. This guard shipped + * with no automated test; this file closes that gap. + */ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { CliRuntimeError } from "../utils/commandResult.js"; + +const runRenderMock = vi.fn(async () => undefined); +vi.mock("./lambda/render.js", () => ({ runRender: runRenderMock })); +// The command dynamically imports `@hyperframes/aws-lambda/sdk` up front for +// every AWS-calling verb (a "is it installed" probe, not otherwise used by +// this guard) — stub it out so this test never pays for that package's real +// (heavier) module graph or its resolution cost under load, which is +// unrelated to what this test verifies. +vi.mock("@hyperframes/aws-lambda/sdk", () => ({})); + +/** Invoke `hyperframes lambda --width 1920 --height 1080`. */ +async function runLambdaRenderCommand( + projectDir: string, + subcommand: "render" | "render-batch" = "render", + extraArgs: Record = {}, +): Promise { + const command = (await import("./lambda.js")).default; + return command.run?.({ + args: { + subcommand, + target: projectDir, + width: "1920", + height: "1080", + ...extraArgs, + }, + } as never); +} + +describe("lambda render VST guard", () => { + let projectDir: string | undefined; + + afterEach(() => { + vi.restoreAllMocks(); + if (projectDir) rmSync(projectDir, { recursive: true, force: true }); + projectDir = undefined; + }); + + it("exits 1 naming the VST track before any AWS-calling code runs", async () => { + projectDir = mkdtempSync(join(tmpdir(), "hf-lambda-vst-guard-")); + writeFileSync( + join(projectDir, "index.html"), + ` + +
+ +`, + ); + + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + + // The guard fails the command at exit code 1. If it hadn't fired and + // execution had instead reached `./lambda/render.js`'s real AWS calls, this + // would either hang on network/credentials or throw an unrelated error + // instead of this specific, synchronous rejection. + const error = await runLambdaRenderCommand(projectDir).then( + () => undefined, + (err: unknown) => err, + ); + expect(error).toBeInstanceOf(CliRuntimeError); + expect((error as CliRuntimeError).result.exitCode).toBe(1); + + const loggedError = errorSpy.mock.calls.map((call) => String(call[0])).join("\n"); + expect(loggedError).toContain("Lambda rendering does not support VST audio chains"); + expect(loggedError).toContain("a1"); + expect(runRenderMock).not.toHaveBeenCalled(); + }, 20_000); + + it("does not fire for a composition with no VST chains", async () => { + projectDir = mkdtempSync(join(tmpdir(), "hf-lambda-vst-guard-clean-")); + writeFileSync( + join(projectDir, "index.html"), + ` + +
+ +`, + ); + + vi.spyOn(console, "error").mockImplementation(() => undefined); + runRenderMock.mockClear(); + + // The guard didn't block this clean composition — execution reached the + // (mocked) real render path instead of failing for a VST reason. + await runLambdaRenderCommand(projectDir); + + expect(runRenderMock).toHaveBeenCalledTimes(1); + }, 20_000); + + it("also guards render-batch, not just render", async () => { + projectDir = mkdtempSync(join(tmpdir(), "hf-lambda-vst-guard-batch-")); + writeFileSync( + join(projectDir, "index.html"), + ` + +
+ +`, + ); + writeFileSync(join(projectDir, "batch.jsonl"), `{"outputKey":"out.mp4"}\n`); + + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + + const error = await runLambdaRenderCommand(projectDir, "render-batch", { + batch: join(projectDir, "batch.jsonl"), + }).then( + () => undefined, + (err: unknown) => err, + ); + expect(error).toBeInstanceOf(CliRuntimeError); + expect((error as CliRuntimeError).result.exitCode).toBe(1); + expect(errorSpy.mock.calls.map((call) => String(call[0])).join("\n")).toContain( + "Lambda rendering does not support VST audio chains", + ); + }, 20_000); +}); diff --git a/packages/cli/src/commands/lambda.ts b/packages/cli/src/commands/lambda.ts index 5265602a08..cb0ddce85c 100644 --- a/packages/cli/src/commands/lambda.ts +++ b/packages/cli/src/commands/lambda.ts @@ -9,14 +9,40 @@ import { failCommand } from "../utils/commandResult.js"; * lives in `@hyperframes/aws-lambda/sdk`. */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; import { defineCommand } from "citty"; import type { DistributedFormat } from "@hyperframes/aws-lambda/sdk"; import { type CanvasResolution } from "@hyperframes/core"; +import { parseAudioElements } from "@hyperframes/engine"; import { parseOutputResolutionFlag } from "../utils/parseOutputResolution.js"; import type { Example } from "./_examples.js"; import { c } from "../ui/colors.js"; import { readAllowedCompositionFpsFromDir } from "../utils/compositionFps.js"; +/** + * Lambda has no VST plugin host, so a composition whose audio depends on a + * plugin chain would render silently wrong (dry signal, no carve). Fail before + * the upload instead of shipping an untreated mix. + */ +function rejectVstChains(projectDir: string, usageContext: string): void { + let compositionHtml: string; + try { + compositionHtml = readFileSync(join(projectDir, "index.html"), "utf-8"); + } catch { + // Composition file missing or unreadable — let the render path report it. + return; + } + const vstTracks = parseAudioElements(compositionHtml).filter((el) => el.vstChain); + if (vstTracks.length === 0) return; + console.error( + `${usageContext} Lambda rendering does not support VST audio chains (plugins cannot run in Lambda).\n` + + `Tracks with VST chains: ${vstTracks.map((t) => t.id).join(", ")}.\n` + + `Render locally with: hyperframes render`, + ); + failCommand(); +} + export const examples: Example[] = [ ["Deploy the Lambda render stack to AWS", "hyperframes lambda deploy"], [ @@ -304,6 +330,7 @@ export default defineCommand({ console.error(`[lambda render] --fps must be 24, 30, or 60; got ${fpsRaw}.`); failCommand(); } + rejectVstChains(projectDir, "[lambda render]"); const { runRender } = await import("./lambda/render.js"); const renderResolution = parseOutputResolution(args["output-resolution"]); await runRender({ @@ -361,6 +388,7 @@ export default defineCommand({ console.error(`[lambda render-batch] --fps must be 24, 30, or 60; got ${fpsRaw}.`); failCommand(); } + rejectVstChains(projectDir, "[lambda render-batch]"); const { runRenderBatch } = await import("./lambda/render-batch.js"); const batchResolution = parseOutputResolution(args["output-resolution"]); await runRenderBatch({ diff --git a/packages/engine/src/services/audioMixer.test.ts b/packages/engine/src/services/audioMixer.test.ts index bc49a149ed..95744e913f 100644 --- a/packages/engine/src/services/audioMixer.test.ts +++ b/packages/engine/src/services/audioMixer.test.ts @@ -1,8 +1,17 @@ // fallow-ignore-file code-duplication import { afterEach, describe, expect, it, vi } from "vitest"; -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; +import { makeFakeSidecar } from "./vstSidecarTestFixture"; + +async function waitFor(predicate: () => boolean, timeoutMs: number, intervalMs = 20) { + const start = Date.now(); + while (Date.now() - start < timeoutMs) { + if (predicate()) return; + await new Promise((r) => setTimeout(r, intervalMs)); + } +} // The mix filter graph is written to a temp file and passed via // a file-valued filter option (not inlined via -filter_complex) so the command @@ -46,7 +55,48 @@ vi.mock("../utils/ffprobe.js", async (importOriginal) => { return { ...actual, extractAudioMetadata: extractAudioMetadataMock }; }); -import { parseAudioElements, processCompositionAudio } from "./audioMixer.js"; +import { parseAudioElements, processCompositionAudio, type AudioElement } from "./audioMixer.js"; + +/** Create a base/work temp-dir pair and register both for the test's `afterEach` cleanup. */ +function setupTempDirs(tempDirs: string[]): { baseDir: string; workDir: string } { + const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-")); + const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-")); + tempDirs.push(baseDir, workDir); + return { baseDir, workDir }; +} + +/** Build an `AudioElement` fixture with sensible test defaults, overridden per test. */ +function makeAudioElement( + overrides: Partial & Pick, +): AudioElement { + return { + start: 0, + end: 2, + mediaStart: 0, + layer: 0, + volume: 1, + type: "audio", + ...overrides, + }; +} + +/** + * Queue the mock ffmpeg runner to succeed on the first call (the prepare + * step) then fail the second (the mix) with the given stderr/exitCode. + * Shared by the automation-fallback and legacy-filter-option-retry tests, + * both of which need a successful prepare followed by a failing mix. + */ +function queueSuccessfulPrepareThenFailingMix(stderr: string, exitCode: number): void { + runFfmpegMock + .mockImplementationOnce(async () => { + capturedFilterScripts.push(""); + return { success: true, durationMs: 1, stderr: "", exitCode: 0 }; + }) + .mockImplementationOnce(async () => { + capturedFilterScripts.push(""); + return { success: false, durationMs: 1, stderr, exitCode }; + }); +} describe("processCompositionAudio", () => { const tempDirs: string[] = []; @@ -116,25 +166,12 @@ describe("processCompositionAudio", () => { }); it("preserves muted tracks and uses unity master gain by default", async () => { - const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-")); - const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-")); - tempDirs.push(baseDir, workDir); + const { baseDir, workDir } = setupTempDirs(tempDirs); writeFileSync(join(baseDir, "voice.wav"), "stub"); const result = await processCompositionAudio( - [ - { - id: "voice", - src: "voice.wav", - start: 0, - end: 2, - mediaStart: 0, - layer: 0, - volume: 0, - type: "audio", - }, - ], + [makeAudioElement({ id: "voice", src: "voice.wav", volume: 0 })], baseDir, workDir, join(baseDir, "out.m4a"), @@ -155,9 +192,7 @@ describe("processCompositionAudio", () => { }); it("compensates amix normalization so multi-track master gain equals track count", async () => { - const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-")); - const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-")); - tempDirs.push(baseDir, workDir); + const { baseDir, workDir } = setupTempDirs(tempDirs); writeFileSync(join(baseDir, "a.wav"), "stub"); writeFileSync(join(baseDir, "b.wav"), "stub"); @@ -165,36 +200,9 @@ describe("processCompositionAudio", () => { const result = await processCompositionAudio( [ - { - id: "a", - src: "a.wav", - start: 0, - end: 2, - mediaStart: 0, - layer: 0, - volume: 0.8, - type: "audio", - }, - { - id: "b", - src: "b.wav", - start: 0, - end: 2, - mediaStart: 0, - layer: 1, - volume: 1, - type: "audio", - }, - { - id: "c", - src: "c.wav", - start: 0, - end: 2, - mediaStart: 0, - layer: 2, - volume: 0.5, - type: "audio", - }, + makeAudioElement({ id: "a", src: "a.wav", layer: 0, volume: 0.8 }), + makeAudioElement({ id: "b", src: "b.wav", layer: 1, volume: 1 }), + makeAudioElement({ id: "c", src: "c.wav", layer: 2, volume: 0.5 }), ], baseDir, workDir, @@ -467,29 +475,24 @@ describe("processCompositionAudio", () => { }); it("uses frame-evaluated volume automation when keyframes are present", async () => { - const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-")); - const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-")); - tempDirs.push(baseDir, workDir); + const { baseDir, workDir } = setupTempDirs(tempDirs); writeFileSync(join(baseDir, "voice.wav"), "stub"); const result = await processCompositionAudio( [ - { + makeAudioElement({ id: "voice", src: "voice.wav", start: 2, end: 5, - mediaStart: 0, - layer: 0, volume: 0, volumeKeyframes: [ { time: 2, volume: 0 }, { time: 3, volume: 1 }, { time: 5, volume: 0.5 }, ], - type: "audio", - }, + }), ], baseDir, workDir, @@ -508,9 +511,7 @@ describe("processCompositionAudio", () => { }); it("bounds expression nesting for dense keyframe automation without dropping the envelope", async () => { - const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-")); - const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-")); - tempDirs.push(baseDir, workDir); + const { baseDir, workDir } = setupTempDirs(tempDirs); writeFileSync(join(baseDir, "bgm.wav"), "stub"); @@ -527,17 +528,13 @@ describe("processCompositionAudio", () => { const result = await processCompositionAudio( [ - { + makeAudioElement({ id: "bgm", src: "bgm.wav", - start: 0, end: 10, - mediaStart: 0, - layer: 0, volume: 0, volumeKeyframes: keyframes, - type: "audio", - }, + }), ], baseDir, workDir, @@ -561,9 +558,7 @@ describe("processCompositionAudio", () => { }); it("falls back to a static-volume mix instead of dropping audio when the automated mix fails", async () => { - const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-")); - const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-")); - tempDirs.push(baseDir, workDir); + const { baseDir, workDir } = setupTempDirs(tempDirs); writeFileSync(join(baseDir, "bgm.wav"), "stub"); @@ -573,37 +568,20 @@ describe("processCompositionAudio", () => { // one-time overrides bypass the default mock's capturedFilterScripts // push, so they push an empty placeholder themselves to keep the array // index-aligned with call order for the fallback mix's assertion below. - runFfmpegMock - .mockImplementationOnce(async () => { - capturedFilterScripts.push(""); - return { success: true, durationMs: 1, stderr: "", exitCode: 0 }; - }) - .mockImplementationOnce(async () => { - capturedFilterScripts.push(""); - return { - success: false, - durationMs: 1, - stderr: "Error initializing filters", - exitCode: 234, - }; - }); + queueSuccessfulPrepareThenFailingMix("Error initializing filters", 234); const result = await processCompositionAudio( [ - { + makeAudioElement({ id: "bgm", src: "bgm.wav", - start: 0, end: 5, - mediaStart: 0, - layer: 0, volume: 0.8, volumeKeyframes: [ { time: 0, volume: 0.8 }, { time: 5, volume: 0 }, ], - type: "audio", - }, + }), ], baseDir, workDir, @@ -624,9 +602,7 @@ describe("processCompositionAudio", () => { }); it("keeps the ffmpeg command line short with a large track count (regression for spawn ENAMETOOLONG)", async () => { - const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-")); - const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-")); - tempDirs.push(baseDir, workDir); + const { baseDir, workDir } = setupTempDirs(tempDirs); // Reported in the wild at 146 timed audio clips: the old inline // -filter_complex string scaled with track count and blew past the OS @@ -635,16 +611,13 @@ describe("processCompositionAudio", () => { const elements = Array.from({ length: trackCount }, (_, i) => { const filename = `clip-${i}.wav`; writeFileSync(join(baseDir, filename), "stub"); - return { + return makeAudioElement({ id: `clip-${i}`, src: filename, start: i * 0.1, end: i * 0.1 + 0.5, - mediaStart: 0, layer: i, - volume: 1, - type: "audio" as const, - }; + }); }); const result = await processCompositionAudio( @@ -677,40 +650,17 @@ describe("processCompositionAudio", () => { }); it("retries with the current file-valued filter option when a nightly removes the legacy alias", async () => { - const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-")); - const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-")); - tempDirs.push(baseDir, workDir); + const { baseDir, workDir } = setupTempDirs(tempDirs); writeFileSync(join(baseDir, "voice.wav"), "stub"); - runFfmpegMock - .mockImplementationOnce(async () => { - capturedFilterScripts.push(""); - return { success: true, durationMs: 1, stderr: "", exitCode: 0 }; - }) - .mockImplementationOnce(async () => { - capturedFilterScripts.push(""); - return { - success: false, - durationMs: 1, - stderr: "Unrecognized option 'filter_complex_script'.\nError splitting the argument list", - exitCode: 8, - }; - }); + queueSuccessfulPrepareThenFailingMix( + "Unrecognized option 'filter_complex_script'.\nError splitting the argument list", + 8, + ); const result = await processCompositionAudio( - [ - { - id: "voice", - src: "voice.wav", - start: 0, - end: 2, - mediaStart: 0, - layer: 0, - volume: 1, - type: "audio", - }, - ], + [makeAudioElement({ id: "voice", src: "voice.wav" })], baseDir, workDir, join(baseDir, "out.m4a"), @@ -728,9 +678,7 @@ describe("processCompositionAudio", () => { }); it("prepares percent-encoded non-Latin audio srcs from decoded filesystem paths", async () => { - const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-")); - const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-")); - tempDirs.push(baseDir, workDir); + const { baseDir, workDir } = setupTempDirs(tempDirs); const encodedFilename = "%D9%87%D9%86%D8%A7%20%D9%85%D8%B1%D9%88%D8%A7%20-%20%D9%85%D8%A8%D8%A7%D8%B1%D9%83.mp4"; @@ -739,18 +687,7 @@ describe("processCompositionAudio", () => { writeFileSync(join(baseDir, "assets", filename), "stub"); const result = await processCompositionAudio( - [ - { - id: "voice", - src: `assets/${encodedFilename}`, - start: 0, - end: 2, - mediaStart: 0, - layer: 0, - volume: 1, - type: "audio", - }, - ], + [makeAudioElement({ id: "voice", src: `assets/${encodedFilename}` })], baseDir, workDir, join(baseDir, "out.m4a"), @@ -766,26 +703,13 @@ describe("processCompositionAudio", () => { }); it("prepares browser root-absolute audio srcs from the project root", async () => { - const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-")); - const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-")); - tempDirs.push(baseDir, workDir); + const { baseDir, workDir } = setupTempDirs(tempDirs); mkdirSync(join(baseDir, ".media"), { recursive: true }); writeFileSync(join(baseDir, ".media", "tone.wav"), "stub"); const result = await processCompositionAudio( - [ - { - id: "tone", - src: "/.media/tone.wav", - start: 0, - end: 1, - mediaStart: 0, - layer: 0, - volume: 1, - type: "audio", - }, - ], + [makeAudioElement({ id: "tone", src: "/.media/tone.wav", end: 1 })], baseDir, workDir, join(baseDir, "out.m4a"), @@ -798,6 +722,206 @@ describe("processCompositionAudio", () => { }); }); +describe("processCompositionAudio VST chain application", () => { + const tempDirs: string[] = []; + + afterEach(() => { + delete process.env.HF_VST_HOST_CMD; + delete process.env.HF_TEST_PIDFILE; + delete process.env.HF_TEST_SENTINEL; + runFfmpegMock.mockClear(); + capturedFilterScripts.length = 0; + for (const dir of tempDirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }); + } + }); + + /** Writes a stub dry `music.wav` + empty `chain.json` into `baseDir` — the + * fixture every test below needs before pointing `HF_VST_HOST_CMD` at its + * own fake sidecar behavior. */ + function writeMusicChainFixture(baseDir: string): void { + writeFileSync(join(baseDir, "music.wav"), "stub"); + writeFileSync(join(baseDir, "chain.json"), "{}"); + } + + it("applies the chain via the sidecar before the volume-envelope bake, with no errors", async () => { + const { baseDir, workDir } = setupTempDirs(tempDirs); + + writeMusicChainFixture(baseDir); + process.env.HF_VST_HOST_CMD = makeFakeSidecar( + workDir, + ` +out="" +prev="" +for a in "$@"; do + if [ "$prev" = "--output" ]; then out="$a"; fi + prev="$a" +done +echo processed > "$out" +`, + ); + + const result = await processCompositionAudio( + [makeAudioElement({ id: "music", src: "music.wav", vstChain: "chain.json" })], + baseDir, + workDir, + join(baseDir, "out.m4a"), + 2, + ); + + expect(result.success).toBe(true); + expect(result.error).toBeUndefined(); + }); + + it("hard-fails the track (never falls back to unprocessed audio) and names the missing plugin", async () => { + const { baseDir, workDir } = setupTempDirs(tempDirs); + + writeMusicChainFixture(baseDir); + process.env.HF_VST_HOST_CMD = makeFakeSidecar( + workDir, + `echo "PLUGIN_MISSING FabFilter Pro-Q 3" >&2; exit 3`, + ); + + // A hard failure must actually block a successful-looking render, not + // just leave an error string somewhere on an otherwise-`success: true` + // result — assert the call rejects (never resolves with `success: true` + // and the track silently dropped). + await expect( + processCompositionAudio( + [makeAudioElement({ id: "music", src: "music.wav", vstChain: "chain.json" })], + baseDir, + workDir, + join(baseDir, "out.m4a"), + 2, + ), + ).rejects.toThrow('for track "music": plugin "FabFilter Pro-Q 3" is not installed'); + }); + + it("hard-fails when the referenced VST chain file doesn't exist on disk", async () => { + const { baseDir, workDir } = setupTempDirs(tempDirs); + + writeFileSync(join(baseDir, "music.wav"), "stub"); + + // Same requirement as above: the missing chain file must reject the call + // (naming the track and the missing file), not degrade to a "successful" + // mix with the track quietly dropped. + await expect( + processCompositionAudio( + [makeAudioElement({ id: "music", src: "music.wav", vstChain: "does-not-exist.json" })], + baseDir, + workDir, + join(baseDir, "out.m4a"), + 2, + ), + ).rejects.toThrow('VST chain file not found for track "music"'); + }); + + it("kills a sibling's still-running VST sidecar when another track's chain hard-fails", async () => { + const { baseDir, workDir } = setupTempDirs(tempDirs); + // Separate from workDir on purpose: workDir is deleted by + // processCompositionAudio's `finally` block the moment the call rejects, + // so a sentinel/pid file written there would disappear regardless of + // whether the sidecar was actually killed — that would make this test + // pass even without the fix. Writing to an independent control dir keeps + // the assertions about the sidecar's own lifecycle. + const controlDir = mkdtempSync(join(tmpdir(), "hf-audio-control-")); + tempDirs.push(controlDir); + + writeFileSync(join(baseDir, "music-slow.wav"), "stub"); + writeFileSync(join(baseDir, "music-fail.wav"), "stub"); + writeFileSync(join(baseDir, "chain-slow.json"), "{}"); + writeFileSync(join(baseDir, "chain-fail.json"), "{}"); + + const pidFile = join(controlDir, "slow.pid"); + const sentinelFile = join(controlDir, "slow.done"); + process.env.HF_TEST_PIDFILE = pidFile; + process.env.HF_TEST_SENTINEL = sentinelFile; + // Branches on the `--chain` filename: the "fail" track exits 3 (missing + // plugin) immediately; the "slow" track records its own pid, sleeps + // (simulating a slow bounce/convolution reverb), then — only if it ran to + // completion uninterrupted — writes a sentinel and its output. + process.env.HF_VST_HOST_CMD = makeFakeSidecar( + workDir, + ` +chain="" +out="" +prev="" +for a in "$@"; do + if [ "$prev" = "--chain" ]; then chain="$a"; fi + if [ "$prev" = "--output" ]; then out="$a"; fi + prev="$a" +done +case "$chain" in + *fail*) + echo "PLUGIN_MISSING FabFilter Pro-Q 3" >&2 + exit 3 + ;; + *) + echo $$ > "$HF_TEST_PIDFILE" + sleep 1.2 + echo done > "$HF_TEST_SENTINEL" + echo processed > "$out" + ;; +esac +`, + ); + + const startedAt = Date.now(); + await expect( + processCompositionAudio( + [ + makeAudioElement({ + id: "musicSlow", + src: "music-slow.wav", + end: 1, + vstChain: "chain-slow.json", + }), + makeAudioElement({ + id: "musicFail", + src: "music-fail.wav", + end: 1, + vstChain: "chain-fail.json", + }), + ], + baseDir, + workDir, + join(baseDir, "out.m4a"), + 1, + ), + ).rejects.toThrow(/plugin "FabFilter Pro-Q 3" is not installed/); + + // Rejects promptly on the failing sibling — it must not wait out the + // slow sibling's full sleep. + expect(Date.now() - startedAt).toBeLessThan(1000); + + // The slow sidecar had actually started (recorded its own pid) before + // the rejection tore things down. + await waitFor(() => existsSync(pidFile), 500); + const pid = Number(readFileSync(pidFile, "utf8").trim()); + expect(Number.isFinite(pid)).toBe(true); + + // The fix under test: once the sibling's VstChainProcessingError rejects + // processCompositionAudio, the still-running slow sidecar must actually + // be terminated, not left running unmanaged. + await waitFor(() => { + try { + process.kill(pid, 0); + return false; // still alive + } catch { + return true; // ESRCH — process is gone + } + }, 1500); + expect(() => process.kill(pid, 0)).toThrow(); + + // Let the sidecar's full sleep duration elapse; the sentinel — written + // only on an uninterrupted run — must never appear, proving the process + // was killed rather than merely racing the assertions above. + const remaining = 1500 - (Date.now() - startedAt); + if (remaining > 0) await new Promise((r) => setTimeout(r, remaining)); + expect(existsSync(sentinelFile)).toBe(false); + }); +}); + describe("parseAudioElements — relative data-start resolution", () => { const wrap = (body: string) => `
${body}
`; @@ -845,6 +969,20 @@ describe("parseAudioElements — relative data-start resolution", () => { }); }); +describe("parseAudioElements data-vst-chain", () => { + it("captures the chain path when present", () => { + const html = ``; + const [el] = parseAudioElements(html); + expect(el.vstChain).toBe("fx/music.vstchain.json"); + }); + + it("leaves vstChain undefined when absent", () => { + const html = ``; + const [el] = parseAudioElements(html); + expect(el.vstChain).toBeUndefined(); + }); +}); + describe("parseAudioElements — hidden tracks", () => { it("excludes directly hidden audio and audible video from the render mix", () => { const html = diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.ts index 1b063f3fd5..82114ccd08 100644 --- a/packages/engine/src/services/audioMixer.ts +++ b/packages/engine/src/services/audioMixer.ts @@ -23,9 +23,24 @@ import type { MixResult, } from "./audioMixer.types.js"; import { applyVolumeEnvelopeToWav } from "./audioVolumeEnvelope.js"; +import { applyVstChainToWav } from "./vstBounce.js"; export type { AudioElement, MixResult } from "./audioMixer.types.js"; +/** + * Thrown when applying a track's VST plugin chain fails — missing plugin, + * missing chain file, sidecar crash. Distinct from the soft per-element + * failures collected into `failures`: a soft failure degrades one track, a + * VST failure never does — the per-element catch below rethrows it so the + * whole mix fails loudly rather than silently shipping the dry signal. + */ +class VstChainProcessingError extends Error { + constructor(message: string) { + super(message); + this.name = "VstChainProcessingError"; + } +} + function clampVolume(volume: number): number { if (!Number.isFinite(volume)) return 1; return Math.max(0, Math.min(1, volume)); @@ -338,6 +353,7 @@ export function parseAudioElements(html: string): AudioElement[] { const mediaStartAttr = el.getAttribute("data-media-start"); const layerAttr = el.getAttribute("data-layer"); const volumeAttr = el.getAttribute("data-volume"); + const vstChain = el.getAttribute("data-vst-chain"); return { id, src: el.getAttribute("src") as string, @@ -346,6 +362,7 @@ export function parseAudioElements(html: string): AudioElement[] { mediaStart: mediaStartAttr ? parseFloat(mediaStartAttr) : 0, layer: layerAttr ? parseInt(layerAttr) : 0, volume: volumeAttr ? parseFloat(volumeAttr) : 1.0, + ...(vstChain ? { vstChain } : {}), type, }; }; @@ -689,163 +706,224 @@ export async function processCompositionAudio( if (!existsSync(workDir)) mkdirSync(workDir, { recursive: true }); - await Promise.all( - elements.map(async (element) => { - if (signal?.aborted) { - failures.push({ - stage: "cancelled", - reason: "cancelled", - owner: "user", - retryable: false, - elementId: element.id, - detail: boundedDetail(`Cancelled audio element ${element.id}`), - }); - return; - } - try { - let srcPath = element.src; - if (!isHttpUrl(srcPath)) { - // Same browser-vs-filesystem path semantics as videos — see - // resolveProjectRelativeSrc in videoFrameExtractor for the full why. - srcPath = resolveProjectRelativeSrc(element.src, baseDir, compiledDir); - } - - if (isHttpUrl(srcPath)) { - try { - srcPath = await downloadToTemp(srcPath, workDir); - } catch (err: unknown) { - failures.push( - downloadFailure(err instanceof Error ? err.message : String(err), element.id), - ); - return; - } - } + // Every element's async work (extract / prepare / VST-bounce) races + // concurrently below via Promise.all, which rejects as soon as the FIRST + // element's chain rejects (a VstChainProcessingError) — it does NOT wait for + // siblings still in flight. Without this controller a sibling's VST sidecar + // subprocess keeps running unmanaged after workDir is deleted out from under + // it. This internal signal is threaded into every element's chain and + // aborted the moment Promise.all rejects, so in-flight siblings get a chance + // to kill their subprocess first. It also aborts when the caller's own + // `signal` fires, preserving external-cancellation behavior. + const internalController = new AbortController(); + const effectiveSignal = internalController.signal; + if (signal) { + if (signal.aborted) internalController.abort(); + else signal.addEventListener("abort", () => internalController.abort(), { once: true }); + } - if (!existsSync(srcPath)) { + try { + await Promise.all( + elements.map(async (element) => { + if (effectiveSignal.aborted) { failures.push({ - stage: "source", - reason: "source_not_found", + stage: "cancelled", + reason: "cancelled", owner: "user", retryable: false, elementId: element.id, - detail: boundedDetail(`Source not found for audio element ${element.id}`), + detail: boundedDetail(`Cancelled audio element ${element.id}`), }); return; } + try { + let srcPath = element.src; + if (!isHttpUrl(srcPath)) { + // Same browser-vs-filesystem path semantics as videos — see + // resolveProjectRelativeSrc in videoFrameExtractor for the full why. + srcPath = resolveProjectRelativeSrc(element.src, baseDir, compiledDir); + } - // Fallback: if no duration was specified, probe the actual file - if (element.end - element.start <= 0) { - let metadata; - try { - metadata = await extractAudioMetadata(srcPath); - } catch (err: unknown) { - failures.push( - probeFailure(err instanceof Error ? err.message : String(err), element.id), - ); - return; + if (isHttpUrl(srcPath)) { + try { + srcPath = await downloadToTemp(srcPath, workDir); + } catch (err: unknown) { + failures.push( + downloadFailure(err instanceof Error ? err.message : String(err), element.id), + ); + return; + } } - const effectiveDuration = metadata.durationSeconds - element.mediaStart; - element.end = - element.start + (effectiveDuration > 0 ? effectiveDuration : metadata.durationSeconds); - } - let audioSrcPath = srcPath; - if (element.type === "video") { - const extractedPath = join(workDir, `${element.id}-extracted.wav`); - const extractResult = await extractAudioFromVideo( - srcPath, - extractedPath, - { - startTime: element.mediaStart, - duration: element.end - element.start, - }, - signal, - config, - ); - if (!extractResult.success) { - failures.push( - extractResult.failure - ? { ...extractResult.failure, elementId: element.id } - : { - stage: "extract", - reason: "ffmpeg_failed", - owner: "system", - retryable: false, - elementId: element.id, - detail: boundedDetail(`Audio extract failed for element ${element.id}`), - }, - ); + if (!existsSync(srcPath)) { + failures.push({ + stage: "source", + reason: "source_not_found", + owner: "user", + retryable: false, + elementId: element.id, + detail: boundedDetail(`Source not found for audio element ${element.id}`), + }); return; } - audioSrcPath = extractedPath; - } else { - const trimmedPath = join(workDir, `${element.id}-trimmed.wav`); - const prepResult = await prepareAudioTrack( - srcPath, - trimmedPath, - element.mediaStart, - element.end - element.start, - signal, - config, - ); - if (!prepResult.success) { - failures.push( - prepResult.failure - ? { ...prepResult.failure, elementId: element.id } - : { - stage: "prepare", - reason: "ffmpeg_failed", - owner: "system", - retryable: false, - elementId: element.id, - detail: boundedDetail(`Audio prepare failed for element ${element.id}`), - }, + + // Fallback: if no duration was specified, probe the actual file + if (element.end - element.start <= 0) { + let metadata; + try { + metadata = await extractAudioMetadata(srcPath); + } catch (err: unknown) { + failures.push( + probeFailure(err instanceof Error ? err.message : String(err), element.id), + ); + return; + } + const effectiveDuration = metadata.durationSeconds - element.mediaStart; + element.end = + element.start + + (effectiveDuration > 0 ? effectiveDuration : metadata.durationSeconds); + } + + let audioSrcPath = srcPath; + if (element.type === "video") { + const extractedPath = join(workDir, `${element.id}-extracted.wav`); + const extractResult = await extractAudioFromVideo( + srcPath, + extractedPath, + { + startTime: element.mediaStart, + duration: element.end - element.start, + }, + effectiveSignal, + config, ); - return; + if (!extractResult.success) { + failures.push( + extractResult.failure + ? { ...extractResult.failure, elementId: element.id } + : { + stage: "extract", + reason: "ffmpeg_failed", + owner: "system", + retryable: false, + elementId: element.id, + detail: boundedDetail(`Audio extract failed for element ${element.id}`), + }, + ); + return; + } + audioSrcPath = extractedPath; + } else { + const trimmedPath = join(workDir, `${element.id}-trimmed.wav`); + const prepResult = await prepareAudioTrack( + srcPath, + trimmedPath, + element.mediaStart, + element.end - element.start, + effectiveSignal, + config, + ); + if (!prepResult.success) { + failures.push( + prepResult.failure + ? { ...prepResult.failure, elementId: element.id } + : { + stage: "prepare", + reason: "ffmpeg_failed", + owner: "system", + retryable: false, + elementId: element.id, + detail: boundedDetail(`Audio prepare failed for element ${element.id}`), + }, + ); + return; + } + audioSrcPath = trimmedPath; + } + + // Apply the track's VST plugin chain (if any) to the dry, trimmed WAV + // before volume automation is baked in — plugins should see the raw + // signal, and the envelope should be applied to their output. A missing + // plugin or sidecar failure is a hard failure for this track: never + // silently fall back to unprocessed audio. + if (element.vstChain) { + const chainAbsPath = resolveProjectRelativeSrc(element.vstChain, baseDir, compiledDir); + if (!existsSync(chainAbsPath)) { + throw new VstChainProcessingError( + `VST chain file not found for track "${element.id}": ${element.vstChain}`, + ); + } + try { + audioSrcPath = await applyVstChainToWav( + audioSrcPath, + chainAbsPath, + workDir, + element.id, + { signal: effectiveSignal }, + ); + } catch (err: unknown) { + throw new VstChainProcessingError(err instanceof Error ? err.message : String(err)); + } } - audioSrcPath = trimmedPath; - } - // Primary volume-automation path: bake the envelope into the PCM samples - // (sample-accurate, no keyframe ceiling). If the WAV isn't the expected - // 16-bit PCM, fall back to the ffmpeg expression path by leaving the - // keyframes on the track for buildVolumeExpression to handle. - let bakedEnvelope = false; - if (element.volumeKeyframes && element.volumeKeyframes.length > 0) { - bakedEnvelope = applyVolumeEnvelopeToWav( - audioSrcPath, - element.volumeKeyframes, - element.start, - element.volume ?? 1.0, - ); + // Primary volume-automation path: bake the envelope into the PCM samples + // (sample-accurate, no keyframe ceiling). If the WAV isn't the expected + // 16-bit PCM, fall back to the ffmpeg expression path by leaving the + // keyframes on the track for buildVolumeExpression to handle. + let bakedEnvelope = false; + if (element.volumeKeyframes && element.volumeKeyframes.length > 0) { + bakedEnvelope = applyVolumeEnvelopeToWav( + audioSrcPath, + element.volumeKeyframes, + element.start, + element.volume ?? 1.0, + ); + } + tracks.push({ + id: element.id, + srcPath: audioSrcPath, + start: element.start, + end: element.end, + mediaStart: element.mediaStart, + duration: element.end - element.start, + // Gain is already in the samples when baked, so mix at unity. + volume: bakedEnvelope ? 1.0 : (element.volume ?? 1.0), + volumeKeyframes: bakedEnvelope ? undefined : element.volumeKeyframes, + }); + } catch (err: unknown) { + // A VST failure is fatal for the whole call — rethrow so it escapes + // this element's promise, rejects the Promise.all, and propagates out + // of processCompositionAudio. Every other failure mode (missing source, + // download failure, extract/prepare failure) keeps degrading + // gracefully: recorded as a failure, track dropped, siblings continue. + if (err instanceof VstChainProcessingError) throw err; + failures.push({ + stage: "internal", + reason: "internal", + owner: "system", + retryable: false, + elementId: element.id, + detail: boundedDetail( + `Audio processing failed for element ${element.id}: ${ + err instanceof Error ? err.message : String(err) + }`, + ), + }); } - tracks.push({ - id: element.id, - srcPath: audioSrcPath, - start: element.start, - end: element.end, - mediaStart: element.mediaStart, - duration: element.end - element.start, - // Gain is already in the samples when baked, so mix at unity. - volume: bakedEnvelope ? 1.0 : (element.volume ?? 1.0), - volumeKeyframes: bakedEnvelope ? undefined : element.volumeKeyframes, - }); - } catch (err: unknown) { - failures.push({ - stage: "internal", - reason: "internal", - owner: "system", - retryable: false, - elementId: element.id, - detail: boundedDetail( - `Audio processing failed for element ${element.id}: ${ - err instanceof Error ? err.message : String(err) - }`, - ), - }); - } - }), - ); + }), + ); + } catch (err) { + // Rejected early on a sibling's VstChainProcessingError, without waiting + // for other in-flight elements — abort them now so their subprocess (e.g. a + // VST sidecar) can see the signal and stop before workDir is removed. + internalController.abort(); + try { + rmSync(workDir, { recursive: true, force: true }); + } catch { + /* ignore */ + } + throw err; + } // Never turn a per-track preparation failure into a successful partial mix. // The producer only surfaces audio failures when `success` is false; mixing diff --git a/packages/engine/src/services/audioMixer.types.ts b/packages/engine/src/services/audioMixer.types.ts index a599e1d675..6ffca553cf 100644 --- a/packages/engine/src/services/audioMixer.types.ts +++ b/packages/engine/src/services/audioMixer.types.ts @@ -12,6 +12,8 @@ export interface AudioElement { layer: number; volume?: number; volumeKeyframes?: AudioVolumeKeyframe[]; + /** Project-relative path to a .vstchain.json effect chain (data-vst-chain). */ + vstChain?: string; type: "audio" | "video"; } diff --git a/packages/engine/src/services/vstBounce.test.ts b/packages/engine/src/services/vstBounce.test.ts new file mode 100644 index 0000000000..95cd5b485f --- /dev/null +++ b/packages/engine/src/services/vstBounce.test.ts @@ -0,0 +1,62 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { mkdtempSync, writeFileSync, existsSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { applyVstChainToWav, resolveVstHostCommand } from "./vstBounce"; +import { makeFakeSidecar } from "./vstSidecarTestFixture"; + +const cleanupEnv = () => { + delete process.env.HF_VST_HOST_CMD; +}; +afterEach(cleanupEnv); + +/** Sets up a fresh temp dir with a stub dry `.wav` + empty chain file, and + * points `HF_VST_HOST_CMD` at a fake sidecar running `sidecarBody`. */ +function setupBounceFixture(sidecarBody: string): { dir: string; wav: string; chain: string } { + const dir = mkdtempSync(join(tmpdir(), "vst-")); + process.env.HF_VST_HOST_CMD = makeFakeSidecar(dir, sidecarBody); + const wav = join(dir, "dry.wav"); + const chain = join(dir, "chain.json"); + writeFileSync(wav, "RIFF"); + writeFileSync(chain, "{}"); + return { dir, wav, chain }; +} + +describe("applyVstChainToWav", () => { + it("returns the output path written by the sidecar", async () => { + // fake sidecar: copy input to output (args: bounce --input X --chain C --output O) + const { dir, wav, chain } = setupBounceFixture(` +out="" +prev="" +for a in "$@"; do + if [ "$prev" = "--output" ]; then out="$a"; fi + prev="$a" +done +cp "$3" "$out" 2>/dev/null || echo processed > "$out" +`); + const result = await applyVstChainToWav(wav, chain, dir, "music"); + expect(existsSync(result)).toBe(true); + expect(result).not.toBe(wav); + }); + + it("names the missing plugin on exit code 3", async () => { + const { dir, wav, chain } = setupBounceFixture( + `echo "PLUGIN_MISSING FabFilter Pro-Q 3" >&2; exit 3`, + ); + await expect(applyVstChainToWav(wav, chain, dir, "music")).rejects.toThrow( + /plugin "FabFilter Pro-Q 3" is not installed/, + ); + }); +}); + +describe("resolveVstHostCommand", () => { + it("prefers HF_VST_HOST_CMD", () => { + process.env.HF_VST_HOST_CMD = "/opt/custom vst-host"; + expect(resolveVstHostCommand()).toEqual(["/opt/custom", "vst-host"]); + }); + + it("falls back to the bare hyperframes-vst command on PATH", () => { + delete process.env.HF_VST_HOST_CMD; + expect(resolveVstHostCommand()).toEqual(["hyperframes-vst"]); + }); +}); diff --git a/packages/engine/src/services/vstBounce.ts b/packages/engine/src/services/vstBounce.ts new file mode 100644 index 0000000000..5090c5491a --- /dev/null +++ b/packages/engine/src/services/vstBounce.ts @@ -0,0 +1,144 @@ +/** + * VST Bounce Service + * + * Spawns the `hyperframes-vst` Python sidecar to apply a VST plugin chain to + * a dry WAV track before it's mixed into the composition. The sidecar lives + * in the standalone `heygen-com/hyperframes-vst-host` repo, published to + * PyPI as `hyperframes-vst-host` (`uv tool install hyperframes-vst-host`). + */ + +import { spawn } from "node:child_process"; +import { basename, join } from "node:path"; +import { trackChildProcess } from "../utils/processTracker.js"; + +export interface ApplyVstChainOptions { + signal?: AbortSignal; +} + +// Plugin scanning + audio bounce through a DAW-grade chain can be slow, +// especially on first run (plugin validation) or with convolution reverbs. +const BOUNCE_TIMEOUT_MS = 10 * 60 * 1000; + +/** + * Resolves the command used to invoke the VST host sidecar. + * + * Precedence: + * 1. `HF_VST_HOST_CMD` env var (space-split) — lets CI/dev machines point at + * an arbitrary executable (or, in tests, a fake shell script). + * 2. Bare `hyperframes-vst` on PATH (an installed/published sidecar — see + * `uv tool install hyperframes-vst-host`). + * + * Duplicated in `@hyperframes/studio-server`'s `vstSidecar.ts` — this package + * doesn't export the function from its public entry point or a subpath, so + * that module carries its own copy (see its docblock for the full why). + */ +// fallow-ignore-next-line code-duplication +export function resolveVstHostCommand(): string[] { + const override = process.env.HF_VST_HOST_CMD; + if (override && override.trim().length > 0) { + return override.trim().split(/\s+/); + } + + return ["hyperframes-vst"]; +} + +/** + * Applies a VST chain to a dry WAV track by spawning the sidecar's `bounce` + * subcommand. Resolves with the path to the processed WAV on success. + * + * Rejects — never falls back to the unprocessed `wavPath` — when the sidecar + * fails. A missing plugin (sidecar exit code 3 with `PLUGIN_MISSING ` + * on stderr) is reported with the specific plugin name and track id so the + * failure is actionable rather than a silent swap to dry audio. + * + * `options.signal`, when provided, kills the sidecar (SIGTERM) if aborted — + * e.g. by `processCompositionAudio` when a sibling track's VST chain fails + * and the whole render is about to be torn down, so this sidecar isn't left + * running against a `workDir` that's about to be deleted. + */ +export function applyVstChainToWav( + wavPath: string, + chainAbsPath: string, + workDir: string, + trackId: string, + options?: ApplyVstChainOptions, +): Promise { + const outputPath = join(workDir, `${basename(wavPath, ".wav")}_vst.wav`); + const commandParts = resolveVstHostCommand(); + const cmd = commandParts[0]; + if (!cmd) { + return Promise.reject( + new Error(`VST render failed for track "${trackId}": no VST host command resolved`), + ); + } + const baseArgs = commandParts.slice(1); + const args = [ + ...baseArgs, + "bounce", + "--input", + wavPath, + "--chain", + chainAbsPath, + "--output", + outputPath, + ]; + + const signal = options?.signal; + + return new Promise((resolvePromise, reject) => { + const child = spawn(cmd, args); + trackChildProcess(child); + let stderr = ""; + child.stderr.on("data", (data: Buffer) => { + stderr += data.toString(); + }); + + const onAbort = () => { + child.kill("SIGTERM"); + }; + if (signal) { + if (signal.aborted) { + onAbort(); + } else { + signal.addEventListener("abort", onAbort, { once: true }); + } + } + + const timer = setTimeout(() => { + child.kill("SIGTERM"); + reject(new Error(`VST render timed out for track "${trackId}"`)); + }, BOUNCE_TIMEOUT_MS); + + child.on("error", (err) => { + clearTimeout(timer); + if (signal) signal.removeEventListener("abort", onAbort); + reject(new Error(`VST sidecar could not be started for track "${trackId}": ${err.message}`)); + }); + + child.on("close", (code) => { + clearTimeout(timer); + if (signal) signal.removeEventListener("abort", onAbort); + if (signal?.aborted) { + reject(new Error(`VST render cancelled for track "${trackId}"`)); + return; + } + if (code === 0) { + resolvePromise(outputPath); + return; + } + const missing = stderr.match(/PLUGIN_MISSING (.+)/); + const missingPlugin = missing ? missing[1] : undefined; + if (code === 3 && missingPlugin) { + reject( + new Error( + `VST render failed for track "${trackId}": plugin "${missingPlugin.trim()}" is not installed on this machine`, + ), + ); + return; + } + reject( + new Error(`VST render failed for track "${trackId}" (exit ${code}): ${stderr.trim()}`), + ); + }); + }); +} diff --git a/packages/engine/src/services/vstSidecarTestFixture.ts b/packages/engine/src/services/vstSidecarTestFixture.ts new file mode 100644 index 0000000000..472775d9c1 --- /dev/null +++ b/packages/engine/src/services/vstSidecarTestFixture.ts @@ -0,0 +1,20 @@ +/** + * Shared test fixture for faking the `hyperframes-vst` sidecar process — + * used by both `audioMixer.test.ts` (the render pipeline's caller) and + * `vstBounce.test.ts` (the sidecar-invocation layer itself) so the two + * suites' fake-process setup can't drift apart. + */ + +import { writeFileSync, chmodSync } from "node:fs"; +import { join } from "node:path"; + +/** Writes an executable shell script to `dir` that stands in for the real + * `hyperframes-vst` binary when pointed at via `HF_VST_HOST_CMD` — `body` + * is the fake process's behavior (e.g. copy input to output, or exit with + * a specific plugin-missing error). */ +export function makeFakeSidecar(dir: string, body: string): string { + const script = join(dir, "fake-vst.sh"); + writeFileSync(script, `#!/bin/sh\n${body}\n`); + chmodSync(script, 0o755); + return script; +} diff --git a/packages/producer/src/services/vstRenderParity.test.ts b/packages/producer/src/services/vstRenderParity.test.ts new file mode 100644 index 0000000000..42d93d9389 --- /dev/null +++ b/packages/producer/src/services/vstRenderParity.test.ts @@ -0,0 +1,149 @@ +/** + * End-to-end integration test for the VST render path: a composition with a + * `data-vst-chain` audio track is rendered through the real + * `processCompositionAudio` mixer, which shells out to the Python VST host + * sidecar (via `applyVstChainToWav`, `packages/engine/src/services/vstBounce.ts`) + * to bounce the dry track through a plugin chain before mixing. The sidecar + * itself lives in the standalone `heygen-com/hyperframes-vst-host` repo, + * published to PyPI as `hyperframes-vst-host` — install with + * `uv tool install hyperframes-vst-host` to run this test locally. + * + * Uses a BUILTIN pedalboard plugin (`Gain`) rather than a real VST3/AU + * bundle: builtins are deterministic (see chain.py / Task 7's nondeterminism + * caveat about some external plugins), so this test can assert bit-for-bit + * reproducibility across two independent runs without depending on any + * plugin being installed on the host machine. + * + * Skips when `hyperframes-vst` isn't on PATH — required to spawn the sidecar + * via `resolveVstHostCommand()`'s bare-PATH fallback. + */ +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { parseAudioElements, processCompositionAudio } from "@hyperframes/engine"; +import { computeAudioResidualRmsDb } from "../utils/audioRegression.js"; + +// Mirrors resolveVstHostCommand's own bare-PATH fallback +// (packages/engine/src/services/vstBounce.ts) so the skip condition matches +// exactly what the sidecar spawn will look for. +const HAS_VST_HOST_CLI = + spawnSync("hyperframes-vst", ["--help"], { encoding: "utf-8" }).status === 0; + +describe.skipIf(!HAS_VST_HOST_CLI)("VST render parity (integration)", () => { + let projectDir: string; + let workRoot: string; + let outRoot: string; + + beforeAll(() => { + projectDir = mkdtempSync(join(tmpdir(), "hf-vst-parity-project-")); + workRoot = mkdtempSync(join(tmpdir(), "hf-vst-parity-work-")); + outRoot = mkdtempSync(join(tmpdir(), "hf-vst-parity-out-")); + + const assetsDir = join(projectDir, "assets"); + const fxDir = join(projectDir, "fx"); + mkdirSync(assetsDir, { recursive: true }); + mkdirSync(fxDir, { recursive: true }); + + // 2-second 440 Hz sine dry source, same generation approach as + // audioRegression.test.ts (spawn ffmpeg's `sine` lavfi source directly — + // no hand-rolled PCM writer needed). + const toneResult = spawnSync( + "ffmpeg", + [ + "-nostdin", + "-v", + "error", + "-f", + "lavfi", + "-i", + "sine=frequency=440:duration=2:sample_rate=48000", + "-ac", + "2", + "-c:a", + "pcm_s16le", + join(assetsDir, "tone.wav"), + ], + { encoding: "utf-8" }, + ); + if (toneResult.status !== 0) { + throw new Error(`ffmpeg sine generation failed: ${toneResult.stderr}`); + } + + // Builtin pedalboard Gain(gain_db=-12) chain — deterministic, no + // external plugin bundle required. Shape per + // packages/studio/src/utils/vstChainFile.ts's ChainFileJson contract. + const gainStateB64 = Buffer.from(JSON.stringify({ gain_db: -12 })).toString("base64"); + const chainJson = { + version: 1, + plugins: [ + { + format: "builtin", + path: "Gain", + pluginName: null, + name: "Gain", + stateB64: gainStateB64, + }, + ], + }; + writeFileSync(join(fxDir, "t.vstchain.json"), JSON.stringify(chainJson, null, 2)); + }); + + afterAll(() => { + rmSync(projectDir, { recursive: true, force: true }); + rmSync(workRoot, { recursive: true, force: true }); + rmSync(outRoot, { recursive: true, force: true }); + }); + + const wetHtml = ` + + +
+ + +`; + // Same composition, minus the VST chain attribute — the dry baseline mix + // of the identical source track. + const dryHtml = wetHtml.replace(' data-vst-chain="fx/t.vstchain.json"', ""); + + async function renderMix(html: string, label: string): Promise { + const elements = parseAudioElements(html); + const workDir = join(workRoot, `work-${label}`); + const outputPath = join(outRoot, `${label}.m4a`); + const result = await processCompositionAudio(elements, projectDir, workDir, outputPath, 2); + if (!result.success) { + throw new Error( + `processCompositionAudio failed for "${label}": ${result.error ?? "unknown"}`, + ); + } + expect(result.tracksProcessed).toBe(1); + return outputPath; + } + + it("applies a measurable, real gain change vs. the dry mix", async () => { + const dryOut = await renderMix(dryHtml, "dry"); + const wetOut = await renderMix(wetHtml, "wet-a"); + + const residual = computeAudioResidualRmsDb(wetOut, dryOut); + // A real -12 dB gain change is well outside the -50 dBFS noise floor + // used to treat two streams as "effectively identical" — this proves + // the plugin chain actually ran and altered the signal, rather than + // silently falling back to the unprocessed dry track. + expect(residual.error).toBeUndefined(); + expect(residual.ok).toBe(false); + expect(residual.overallDb).toBeGreaterThan(-50); + }, 30_000); + + it("is deterministic for a builtin plugin across two independent runs", async () => { + const wetOut1 = await renderMix(wetHtml, "wet-b1"); + const wetOut2 = await renderMix(wetHtml, "wet-b2"); + + const residual = computeAudioResidualRmsDb(wetOut1, wetOut2); + // Two from-scratch renders of the same builtin-chain composition must + // cancel to within the noise floor — no external-plugin-style + // nondeterminism (see chain.py / Task 7 design notes) for builtins. + expect(residual.error).toBeUndefined(); + expect(residual.ok).toBe(true); + }, 30_000); +});