fix(cli): preserve UTF-8 across subprocess output chunks

docs/developers/cli.mdx

@@ -48,6 +48,10 @@ The CLI also renders in batches and from variables, writes image sequences and
 several video formats, and runs in the cloud — managed, on AWS Lambda, or on
 Google Cloud Run. Most commands take `--json` when you need to read the result from a script.
 
+For workers launched through `runCancellableProcess`, diagnostics preserve
+Unicode filenames and non-English text across output chunks. Its output limit
+counts the combined UTF-8 bytes from standard output and standard error.
+
 Start with the simple command, then open the [full reference](/packages/cli) for the exact options needed by automation or infrastructure.
 
 <Tip>

packages/cli/src/utils/cancellableProcess.test.ts

@@ -8,6 +8,73 @@ import { testProcessIsAlive, waitForTestCondition } from "./processTestUtils.js"
 
 const IS_POSIX = process.platform !== "win32";
 
+const splitOutputSource = [
+  "const [output, exitCode, keepAlive] = process.argv.slice(1);",
+  "const bytes = Buffer.from(output);",
+  "process.stdout.write(bytes.subarray(0, 1));",
+  "process.stderr.write(bytes.subarray(0, 1));",
+  "setTimeout(() => {",
+  "  process.stdout.write(bytes.subarray(1));",
+  "  process.stderr.write(bytes.subarray(1));",
+  "  process.exitCode = Number(exitCode);",
+  "  if (keepAlive === '1') setTimeout(() => {}, 10_000);",
+  "}, 50);",
+].join("\n");
+
+describe("cancellable process output", () => {
+  it.each(["é", "项目/音声.wav", "🎬 render", "ASCII output"])(
+    "preserves split output on both streams for %s",
+    async (output) => {
+      const chunks: string[] = [];
+      const result = await runCancellableProcess(
+        process.execPath,
+        ["--eval", splitOutputSource, output, "0"],
+        { onStdout: (chunk) => chunks.push(chunk) },
+      );
+      expect(result).toEqual({ stdout: output, stderr: output });
+      expect(chunks.join("")).toBe(output);
+    },
+  );
+
+  it("preserves split diagnostics when the process fails", async () => {
+    const output = "診断: 🎬";
+    await expect(
+      runCancellableProcess(process.execPath, ["--eval", splitOutputSource, output, "1"]),
+    ).rejects.toMatchObject({ status: 1, stdout: output, stderr: output });
+  });
+
+  it("counts a split UTF-8 character once at the combined byte limit", async () => {
+    const result = await runCancellableProcess(
+      process.execPath,
+      ["--eval", splitOutputSource, "é", "0"],
+      { maxBufferBytes: 4 },
+    );
+    expect(result).toEqual({ stdout: "é", stderr: "é" });
+  });
+
+  it("still rejects output that exceeds the combined raw byte limit", async () => {
+    await expect(
+      runCancellableProcess(process.execPath, ["--eval", splitOutputSource, "é", "0", "1"], {
+        maxBufferBytes: 3,
+      }),
+    ).rejects.toMatchObject({ code: "ENOBUFS", killed: true });
+  });
+
+  it("flushes incomplete trailing UTF-8 bytes when the streams end", async () => {
+    const chunks: string[] = [];
+    const result = await runCancellableProcess(
+      process.execPath,
+      [
+        "--eval",
+        "process.stdout.write(Buffer.from([0xe2, 0x82])); process.stderr.write(Buffer.from([0xf0, 0x9f]));",
+      ],
+      { onStdout: (chunk) => chunks.push(chunk) },
+    );
+    expect(result).toEqual({ stdout: "�", stderr: "�" });
+    expect(chunks.join("")).toBe("�");
+  });
+});
+
 function processGroupOf(pid: number): string {
   return execFileSync("ps", ["-o", "pgid=", "-p", String(pid)], { encoding: "utf8" }).trim();
 }

packages/cli/src/utils/cancellableProcess.ts

@@ -1,4 +1,5 @@
 import { spawn, type ChildProcess, type SpawnOptions } from "node:child_process";
+import { StringDecoder } from "node:string_decoder";
 import { fileURLToPath } from "node:url";
 import { terminateProcessTree } from "./processTree.js";
 import { renderSetupFailureFrom, renderSetupResultFrom } from "../renderSetupWorkerLifecycle.js";
@@ -78,6 +79,9 @@ export function runCancellableProcess(
     if (child.pid) options.onSpawn?.(child.pid);
     let stdout = "";
     let stderr = "";
+    const stdoutDecoder = new StringDecoder("utf8");
+    const stderrDecoder = new StringDecoder("utf8");
+    let outputBytes = 0;
     let settled = false;
     let terminationReason: unknown;
     let terminationSignal: NodeJS.Signals | undefined;
@@ -112,22 +116,29 @@ export function runCancellableProcess(
     const onAbort = (): void => terminate(signal?.reason ?? new Error("Operation aborted"));
 
     child.stdout?.on("data", (chunk: Buffer | string) => {
-      stdout += chunk.toString();
-      options.onStdout?.(chunk.toString());
-      if (Buffer.byteLength(stdout) + Buffer.byteLength(stderr) > maxBufferBytes) {
+      const decoded = typeof chunk === "string" ? chunk : stdoutDecoder.write(chunk);
+      stdout += decoded;
+      options.onStdout?.(decoded);
+      outputBytes += typeof chunk === "string" ? Buffer.byteLength(chunk) : chunk.length;
+      if (outputBytes > maxBufferBytes) {
         terminate(new Error(`Process output exceeded ${maxBufferBytes} bytes`), "ENOBUFS");
       }
     });
     child.stderr?.on("data", (chunk: Buffer | string) => {
-      stderr += chunk.toString();
-      if (Buffer.byteLength(stdout) + Buffer.byteLength(stderr) > maxBufferBytes) {
+      stderr += typeof chunk === "string" ? chunk : stderrDecoder.write(chunk);
+      outputBytes += typeof chunk === "string" ? Buffer.byteLength(chunk) : chunk.length;
+      if (outputBytes > maxBufferBytes) {
         terminate(new Error(`Process output exceeded ${maxBufferBytes} bytes`), "ENOBUFS");
       }
     });
     child.once("error", (error) => void finishReject(error));
     // fallow-ignore-next-line complexity
     child.once("close", async (status, closeSignal) => {
       if (settled) return;
+      const trailingStdout = stdoutDecoder.end();
+      stdout += trailingStdout;
+      if (trailingStdout) options.onStdout?.(trailingStdout);
+      stderr += stderrDecoder.end();
       if (terminationTask) terminationSignal = await terminationTask.catch(() => terminationSignal);
       if (settled) return;
       settled = true;