Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/cli/src/commands/init.ts
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ function probeVideo(filePath: string): VideoMeta | undefined {
if (!ffprobePath) return undefined;
const raw = execFileSync(
ffprobePath,
["-v", "quiet", "-print_format", "json", "-show_format", "-show_streams", filePath],
["-v", "quiet", "-print_format", "json", "-show_format", "-show_streams", "--", filePath],
{ encoding: "utf-8", timeout: 15_000 },
);

Expand Down
1 change: 1 addition & 0 deletions packages/cli/src/utils/webmAlphaCheck.ts
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ function probeWebmAlpha(filePath: string): WebmAlphaProbe {
"stream=codec_name:stream_tags=alpha_mode",
"-of",
"json",
"--",
filePath,
],
{ encoding: "utf-8", timeout: 15_000 },
Expand Down
3 changes: 2 additions & 1 deletion packages/cli/src/whisper/transcribe.ts
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,7 @@ function getMediaDurationSeconds(filePath: string): number | null {
"format=duration",
"-of",
"default=noprint_wrappers=1:nokey=1",
"--",
filePath,
],
{ encoding: "utf-8", timeout: 10_000 },
Expand Down Expand Up @@ -329,7 +330,7 @@ function isWav16kMono(filePath: string): boolean {
if (!ffprobePath) return false;
const raw = execFileSync(
ffprobePath,
["-v", "quiet", "-print_format", "json", "-show_streams", filePath],
["-v", "quiet", "-print_format", "json", "-show_streams", "--", filePath],
{ encoding: "utf-8", timeout: 10_000 },
);
const parsed: {
Expand Down
2 changes: 1 addition & 1 deletion packages/core/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,7 @@ export {
quantizeTimeToFrame,
type MediaVisualStyleProperty,
} from "./inline-scripts/parityContract";
export { redactTelemetryString } from "./telemetryRedaction";
export { redactKnownPaths, redactTelemetryString } from "./telemetryRedaction";
export { isSafePath, resolveWithinProject } from "./safePath";
export type {
HyperframePickerApi,
Expand Down
1 change: 1 addition & 0 deletions packages/core/src/mediaGradeAnalyzer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ function probeMedia(mediaPath: string, ffprobePath: string): GradeMediaProbe {
"stream=color_space,color_transfer,color_primaries,pix_fmt,duration:format=duration",
"-of",
"json",
"--",
mediaPath,
],
{ encoding: "utf8", timeout: 5_000, stdio: ["ignore", "pipe", "pipe"] },
Expand Down
99 changes: 98 additions & 1 deletion packages/core/src/telemetryRedaction.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest";
import { redactTelemetryString } from "./telemetryRedaction.js";
import { redactKnownPaths, redactTelemetryString } from "./telemetryRedaction.js";

describe("redactTelemetryString", () => {
it("redacts macOS, Linux, Windows, file URLs, and URL query strings", () => {
Expand All @@ -16,4 +16,101 @@ describe("redactTelemetryString", () => {
),
).toBe("[path] [path] [path] [path] [file-url] https://example.com/video.mp4?…");
});

// The redactor used to enumerate roots (/Users, /home, /opt, /tmp, …). Any
// root outside that list reached telemetry verbatim, which is most of them.
it.each([
"/data/media/interview.mov",
"/mnt2/nfs/share/take3.wav",
"/srv2/renders/2026/final.mp4",
"/nix/store/abc123/asset.png",
])("redacts the non-allowlisted absolute root in %s", (path) => {
const out = redactTelemetryString(`ffprobe failed reading ${path}`);
expect(out).not.toContain("/");
expect(out).toContain("[path]");
});

it("redacts relative paths, including a dash-prefixed one", () => {
expect(redactTelemetryString("could not open ./assets/-weird-name.mp3")).toBe(
"could not open [path]",
);
expect(redactTelemetryString("could not open ../-out.wav")).toBe("could not open [path]");
expect(redactTelemetryString("could not open .\\tmp\\-x.aac")).toBe("could not open [path]");
});

it("redacts a bare basename — a caller may pass one instead of a path", () => {
expect(redactTelemetryString("Invalid data found in my-client-cut.mp4")).toBe(
"Invalid data found in [file]",
);
});

// Over-redaction is cheap; these are ordinary in ffprobe stderr and turning
// them into [path] would make a diagnostic string useless.
it.each(["N/A", "24/1", "Stream #0:0", "moov atom not found", "48000/1001"])(
"leaves %s alone",
(text) => {
expect(redactTelemetryString(`ffprobe: ${text}`)).toBe(`ffprobe: ${text}`);
},
);

// A `?` is illegal in a Windows filename, so this is not a query string —
// the whole token is path, and must not survive by hiding behind a `?`.
it("consumes the rest of the token once a path is established", () => {
expect(redactTelemetryString("Navigation failed for C:\\Users\\A\\v.mov?not-a-query")).toBe(
"Navigation failed for [path]",
);
});

it("truncates after redacting, so a long path cannot survive by being cut", () => {
const out = redactTelemetryString(`/data/${"x".repeat(500)}/a.mp4`, 40);
expect(out).not.toContain("xxx");
});

// Named explicitly in review: a relative path with NO `./` prefix was
// missed by both the absolute rule (needs a leading slash) and the `./`
// rule (needs the dot), so it reached telemetry completely unredacted.
it.each([
"customer/acme-secret/video.mp4",
"assets/bgm.mp3",
"projects/client-name/cut/final.mov",
"a\\b\\c.wav",
])("redacts the bare relative path %s", (path) => {
const out = redactTelemetryString(`Invalid data found when processing ${path}`);
expect(out).toBe("Invalid data found when processing [path]");
});

it("redacts a dash-prefixed bare basename", () => {
expect(redactTelemetryString("could not open -customer-secret-intro.mp4")).toBe(
"could not open [file]",
);
});
});

describe("redactKnownPaths", () => {
// Shape matching has holes by construction. A caller that built the argv
// knows the exact path, so it can name it instead of hoping a regex does.
it("redacts an exact path a regex would not recognise as one", () => {
const weird = "acme_secret_project";
expect(redactKnownPaths(`ffprobe: ${weird}: Invalid data`, [weird])).toBe(
"ffprobe: [path]: Invalid data",
);
});

it("redacts the basename too — ffprobe often reports only that", () => {
const out = redactKnownPaths("moov atom not found in secret-cut.mp4", [
"/data/x/secret-cut.mp4",
]);
expect(out).toContain("[path]");
expect(out).not.toContain("secret-cut");
});

it("leaves the message alone when no path was supplied", () => {
expect(redactKnownPaths("moov atom not found", [])).toBe("moov atom not found");
});

// Guards against a one/two-character basename turning every occurrence of
// that letter into [path].
it("ignores paths too short to be distinctive", () => {
expect(redactKnownPaths("a stream at a rate", ["a"])).toBe("a stream at a rate");
});
});
120 changes: 114 additions & 6 deletions packages/core/src/telemetryRedaction.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,121 @@ function redactUrlQueryStrings(value: string): string {
return value.replace(/\b(https?:\/\/[^\s?]+)\?[^\s]*/g, "$1?…");
}

/**
* Path characters we treat as part of a single segment. Space is deliberately
* excluded: including it would let a match run past the path and swallow the
* prose after it, and a path with a space still gets its remaining segments
* redacted, which is the part that carries the identifying information.
*/
const SEGMENT = String.raw`[\w.\-@+()~]+`;

/** Same, minus the dot, so a trailing `.ext` can be matched separately. */
const SEGMENT_NODOT = String.raw`[\w\-@+()~]+`;

/**
* Once a match is established as a path, consume the rest of the token.
* Windows forbids `?` in a filename, so `video.mov?not-a-query` is not a real
* query string — but stopping at the `?` would emit the remainder verbatim.
* Redacting to the next delimiter cannot leak; stopping early can.
*/
const TOKEN_TAIL = String.raw`[^\s'")]*`;

/**
* Absolute path, any root — NOT an allowlist of roots.
*
* The previous version enumerated `/Users`, `/home`, `/opt`, `/tmp`… which
* meant a project on `/data`, `/Volumes/External`, an NFS mount or any root a
* user invented reached telemetry verbatim. Two or more segments are required
* so `N/A` and a `24/1` frame rate — both ordinary in ffprobe stderr — are not
* mistaken for paths.
*
* The lookbehind keeps this off URLs: after `https:` the slash is preceded by
* `:`, the second by `/`, and the path segment by a word character, so no
* position inside a URL can start a match. URLs are handled above, where the
* host is kept and only the query is dropped.
*/
const ABSOLUTE_PATH = new RegExp(
String.raw`(?<![:\w/\\])(?:[A-Za-z]:)?(?:[\\/]${SEGMENT}){2,}${TOKEN_TAIL}`,
"g",
);

/** `./assets/bgm.mp3`, `../out.wav`, `.\tmp\x` — relative paths leak the same
* project structure absolute ones do, and were previously untouched. */
const RELATIVE_PATH = new RegExp(
String.raw`(?<![\w/\\.])\.{1,2}(?:[\\/]${SEGMENT})+${TOKEN_TAIL}`,
"g",
);

/**
* A bare basename with an asset extension. ffprobe reports the input by the
* name it was given, so a caller that passes a basename (or a path this
* flattened to its last segment) still names the user's file.
*
* The lookbehind excludes a slash so this cannot re-redact the tail of a URL
* whose host we deliberately keep.
*/
const ASSET_BASENAME =
/(?<![\w/\\])[\w.\-@+()~]+\.(?:mp4|mov|mkv|webm|avi|m4v|mpe?g|ts|mp3|wav|aac|m4a|flac|ogg|opus|png|jpe?g|gif|webp|svg|html?|json|srt|vtt|ass)\b/gi;

/**
* A relative path with NO `./` prefix — `assets/bgm.mp3`,
* `customer/acme-secret/video.mp4`. These leak exactly as much as an absolute
* path and were missed by both rules above: the absolute rule requires a
* leading slash, and the `./` rule requires the dot.
*
* Qualifying needs either two separators or one plus a file extension, so the
* ordinary non-paths in ffprobe stderr — `N/A`, a `24/1` frame rate,
* `48000/1001` — do not match. The lookbehind keeps it off URL paths, whose
* host is deliberately kept.
*/
const BARE_RELATIVE_PATH = new RegExp(
[
// Two or more separators: `customer/acme/video.mp4`. No extension needed —
// that much structure is already a path.
String.raw`(?<![\w/\\.:@-])(?:${SEGMENT}[\\/]){2,}${SEGMENT}${TOKEN_TAIL}`,
// One separator, but the last segment carries a file extension:
// `assets/bgm.mp3`. That segment is dot-free on purpose — SEGMENT includes
// `.`, so a greedy one swallows the extension this rule needs.
String.raw`(?<![\w/\\.:@-])${SEGMENT}[\\/]${SEGMENT_NODOT}\.\w{1,8}\b${TOKEN_TAIL}`,
].join("|"),
"g",
);

/**
* Redact literal strings — the exact paths a caller KNOWS it passed — before
* any shape-based rule runs.
*
* Shape matching is a net with holes by construction; this is not. When the
* caller has the path in hand (it built the argv), spell it out rather than
* hoping a regex recognises it, and take the basename too since ffprobe often
* reports only that.
*/
export function redactKnownPaths(value: string, paths: readonly string[]): string {
let out = value;
for (const path of paths) {
if (typeof path !== "string" || path.length === 0) continue;
// Longest first: replacing the basename before the full path would leave
// the directory prefix stranded.
const basename = path.split(/[\\/]/).pop() ?? "";
for (const literal of [path, basename].filter((v) => v.length > 2)) {
out = out.split(literal).join("[path]");
}
}
return out;
}

function redactFilePaths(value: string): string {
return value
.replace(/file:\/\/[^\s'")]+/g, "[file-url]")
.replace(/\/Users\/[^\s'")]+/g, "[path]")
.replace(/\/(?:home|root|opt|app|workspace|srv|mnt)\/[^\s'")]+/g, "[path]")
.replace(/\/(?:private\/)?(?:var|tmp)\/[^\s'")]+/g, "[path]")
.replace(/[A-Za-z]:\\[^\s'")]+/g, "[path]");
return (
value
.replace(/file:\/\/[^\s'")]+/g, "[file-url]")
// Relative BEFORE absolute: `./assets/x.mp3` has an absolute-looking tail
// (`/assets/x.mp3`), so the absolute rule would consume it and leave the
// leading `.` stranded outside the redaction.
.replace(RELATIVE_PATH, "[path]")
.replace(ABSOLUTE_PATH, "[path]")
.replace(BARE_RELATIVE_PATH, "[path]")
.replace(ASSET_BASENAME, "[file]")
);
}

export function redactTelemetryString(
Expand Down
1 change: 1 addition & 0 deletions packages/lint/src/hevcPreviewLint.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ async function probeIsHevc(ffprobePath: string, filePath: string): Promise<boole
"stream=codec_name",
"-of",
"json",
"--",
filePath,
]);
return hasHevcStream(JSON.parse(stdout));
Expand Down
1 change: 1 addition & 0 deletions packages/producer/src/plan-parity-analysis.ts
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@ function probeMetadata(outputPath: string): PlanParityStreamMetadata {
].join(":"),
"-of",
"json",
"--",
outputPath,
]);
return normalizeFfprobeMetadata(JSON.parse(bytes.toString("utf-8")) as unknown);
Expand Down
20 changes: 18 additions & 2 deletions packages/producer/src/services/render/audioPadTrim.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import {
trackChildProcess,
type AudioMetadata,
} from "@hyperframes/engine";
import { redactKnownPaths, redactTelemetryString } from "@hyperframes/core";

/**
* Tolerance used to decide whether an audio file is already short enough to
Expand Down Expand Up @@ -361,6 +362,7 @@ async function defaultProbeVideoFrameInfo(
"stream=nb_frames,r_frame_rate",
"-of",
"json",
"--",
videoPath,
],
signal,
Expand All @@ -381,6 +383,7 @@ async function defaultProbeVideoFrameInfo(
"stream=nb_read_packets,r_frame_rate",
"-of",
"json",
"--",
videoPath,
],
signal,
Expand Down Expand Up @@ -434,7 +437,13 @@ async function defaultRunFfmpeg(
// ── ffprobe JSON runner (shared between fast/slow video probe paths) ─────

async function runFfprobeJson<T>(args: string[], signal?: AbortSignal): Promise<T> {
const proc = spawn(getFfprobeBinary(), args);
// Callers bake the input path into `args` (terminated with "--"), so this
// helper cannot add the terminator itself — assert they did rather than
// let a dash-prefixed path silently reach ffprobe as an option.
if (!args.includes("--")) {
throw new Error('[audioPadTrim] ffprobe args must terminate options with "--".');
}
const proc = spawn(getFfprobeBinary(), args, { stdio: ["ignore", "pipe", "pipe"] });
trackChildProcess(proc);
let stdout = "";
proc.stdout.on("data", (data: Buffer) => {
Expand All @@ -452,7 +461,14 @@ async function runFfprobeJson<T>(args: string[], signal?: AbortSignal): Promise<
throw outcome.error ?? new Error(outcome.stderr);
}
if (outcome.reason !== "exit" || outcome.exitCode !== 0) {
throw new Error(`ffprobe ${outcome.reason}: ${outcome.stderr}`);
// Redacted twice, deliberately. The shape-based scrub is a net with
// holes — it cannot know that `customer/acme-secret/video.mp4` is a path
// and `48000/1001` is not — but THIS caller knows the exact path it put
// in the argv, so it names it literally first. The message reaches logs,
// telemetry, and `PadTrimAudioResult.error`.
const probed = args[args.length - 1];
const scrubbed = redactKnownPaths(outcome.stderr, probed === undefined ? [] : [probed]);
throw new Error(`ffprobe ${outcome.reason}: ${redactTelemetryString(scrubbed, 2000)}`);
}
try {
return JSON.parse(stdout) as T;
Expand Down
1 change: 1 addition & 0 deletions packages/producer/src/utils/audioRegression.ts
Original file line number Diff line number Diff line change
Expand Up @@ -315,6 +315,7 @@ function probeAudioDuration(file: string): { seconds: number; error?: string } {
"stream=duration",
"-of",
"default=noprint_wrappers=1:nokey=1",
"--",
file,
],
{ encoding: "utf-8" },
Expand Down
Loading
Loading