diff --git a/docs/developers/cli.mdx b/docs/developers/cli.mdx index 9ec21a74861..e795e354203 100644 --- a/docs/developers/cli.mdx +++ b/docs/developers/cli.mdx @@ -48,6 +48,10 @@ The CLI also renders in batches and from variables, writes image sequences and several video formats, and runs in the cloud — managed, on AWS Lambda, or on Google Cloud Run. Most commands take `--json` when you need to read the result from a script. +For workers launched through `runCancellableProcess`, diagnostics preserve +Unicode filenames and non-English text across output chunks. Its output limit +counts the combined UTF-8 bytes from standard output and standard error. + Start with the simple command, then open the [full reference](/packages/cli) for the exact options needed by automation or infrastructure. diff --git a/packages/cli/src/utils/cancellableProcess.test.ts b/packages/cli/src/utils/cancellableProcess.test.ts index 5161085b14b..d418b255b34 100644 --- a/packages/cli/src/utils/cancellableProcess.test.ts +++ b/packages/cli/src/utils/cancellableProcess.test.ts @@ -8,6 +8,73 @@ import { testProcessIsAlive, waitForTestCondition } from "./processTestUtils.js" const IS_POSIX = process.platform !== "win32"; +const splitOutputSource = [ + "const [output, exitCode, keepAlive] = process.argv.slice(1);", + "const bytes = Buffer.from(output);", + "process.stdout.write(bytes.subarray(0, 1));", + "process.stderr.write(bytes.subarray(0, 1));", + "setTimeout(() => {", + " process.stdout.write(bytes.subarray(1));", + " process.stderr.write(bytes.subarray(1));", + " process.exitCode = Number(exitCode);", + " if (keepAlive === '1') setTimeout(() => {}, 10_000);", + "}, 50);", +].join("\n"); + +describe("cancellable process output", () => { + it.each(["é", "项目/音声.wav", "🎬 render", "ASCII output"])( + "preserves split output on both streams for %s", + async (output) => { + const chunks: string[] = []; + const result = await runCancellableProcess( + process.execPath, + ["--eval", splitOutputSource, output, "0"], + { onStdout: (chunk) => chunks.push(chunk) }, + ); + expect(result).toEqual({ stdout: output, stderr: output }); + expect(chunks.join("")).toBe(output); + }, + ); + + it("preserves split diagnostics when the process fails", async () => { + const output = "診断: 🎬"; + await expect( + runCancellableProcess(process.execPath, ["--eval", splitOutputSource, output, "1"]), + ).rejects.toMatchObject({ status: 1, stdout: output, stderr: output }); + }); + + it("counts a split UTF-8 character once at the combined byte limit", async () => { + const result = await runCancellableProcess( + process.execPath, + ["--eval", splitOutputSource, "é", "0"], + { maxBufferBytes: 4 }, + ); + expect(result).toEqual({ stdout: "é", stderr: "é" }); + }); + + it("still rejects output that exceeds the combined raw byte limit", async () => { + await expect( + runCancellableProcess(process.execPath, ["--eval", splitOutputSource, "é", "0", "1"], { + maxBufferBytes: 3, + }), + ).rejects.toMatchObject({ code: "ENOBUFS", killed: true }); + }); + + it("flushes incomplete trailing UTF-8 bytes when the streams end", async () => { + const chunks: string[] = []; + const result = await runCancellableProcess( + process.execPath, + [ + "--eval", + "process.stdout.write(Buffer.from([0xe2, 0x82])); process.stderr.write(Buffer.from([0xf0, 0x9f]));", + ], + { onStdout: (chunk) => chunks.push(chunk) }, + ); + expect(result).toEqual({ stdout: "�", stderr: "�" }); + expect(chunks.join("")).toBe("�"); + }); +}); + function processGroupOf(pid: number): string { return execFileSync("ps", ["-o", "pgid=", "-p", String(pid)], { encoding: "utf8" }).trim(); } diff --git a/packages/cli/src/utils/cancellableProcess.ts b/packages/cli/src/utils/cancellableProcess.ts index 59985b8a63d..9aa3b0aaa9f 100644 --- a/packages/cli/src/utils/cancellableProcess.ts +++ b/packages/cli/src/utils/cancellableProcess.ts @@ -1,4 +1,5 @@ import { spawn, type ChildProcess, type SpawnOptions } from "node:child_process"; +import { StringDecoder } from "node:string_decoder"; import { fileURLToPath } from "node:url"; import { terminateProcessTree } from "./processTree.js"; import { renderSetupFailureFrom, renderSetupResultFrom } from "../renderSetupWorkerLifecycle.js"; @@ -78,6 +79,9 @@ export function runCancellableProcess( if (child.pid) options.onSpawn?.(child.pid); let stdout = ""; let stderr = ""; + const stdoutDecoder = new StringDecoder("utf8"); + const stderrDecoder = new StringDecoder("utf8"); + let outputBytes = 0; let settled = false; let terminationReason: unknown; let terminationSignal: NodeJS.Signals | undefined; @@ -112,15 +116,18 @@ export function runCancellableProcess( const onAbort = (): void => terminate(signal?.reason ?? new Error("Operation aborted")); child.stdout?.on("data", (chunk: Buffer | string) => { - stdout += chunk.toString(); - options.onStdout?.(chunk.toString()); - if (Buffer.byteLength(stdout) + Buffer.byteLength(stderr) > maxBufferBytes) { + const decoded = typeof chunk === "string" ? chunk : stdoutDecoder.write(chunk); + stdout += decoded; + options.onStdout?.(decoded); + outputBytes += typeof chunk === "string" ? Buffer.byteLength(chunk) : chunk.length; + if (outputBytes > maxBufferBytes) { terminate(new Error(`Process output exceeded ${maxBufferBytes} bytes`), "ENOBUFS"); } }); child.stderr?.on("data", (chunk: Buffer | string) => { - stderr += chunk.toString(); - if (Buffer.byteLength(stdout) + Buffer.byteLength(stderr) > maxBufferBytes) { + stderr += typeof chunk === "string" ? chunk : stderrDecoder.write(chunk); + outputBytes += typeof chunk === "string" ? Buffer.byteLength(chunk) : chunk.length; + if (outputBytes > maxBufferBytes) { terminate(new Error(`Process output exceeded ${maxBufferBytes} bytes`), "ENOBUFS"); } }); @@ -128,6 +135,10 @@ export function runCancellableProcess( // fallow-ignore-next-line complexity child.once("close", async (status, closeSignal) => { if (settled) return; + const trailingStdout = stdoutDecoder.end(); + stdout += trailingStdout; + if (trailingStdout) options.onStdout?.(trailingStdout); + stderr += stderrDecoder.end(); if (terminationTask) terminationSignal = await terminationTask.catch(() => terminationSignal); if (settled) return; settled = true;