diff --git a/packages/stack/src/runtime/Container.integration.test.ts b/packages/stack/src/runtime/Container.integration.test.ts index 3cd82ba7b4..02cf48fa10 100644 --- a/packages/stack/src/runtime/Container.integration.test.ts +++ b/packages/stack/src/runtime/Container.integration.test.ts @@ -1549,6 +1549,34 @@ describe("container process adapter", { timeout: 120_000 }, () => { }).pipe(Effect.provide(NodeServices.layer)), ); + it.effect( + "names the stalled engine subcommand and its stderr when an engine command times out", + () => + Effect.gen(function* () { + const engine = yield* makeStalledStartSpawner("Error: waiting for network sandbox"); + const runtime = yield* makeContainerRuntime({ + target: containerTarget, + root: ".", + hostGateway: sharedHostGateway, + }).pipe(Effect.provide(engine.layer)); + const launch = yield* Effect.scoped( + runtime.launch({ image, stackId: "s".repeat(64), instanceId: "stalled-start", env: {} }), + ).pipe(Effect.provide(engine.layer), Effect.exit, Effect.forkChild); + yield* Deferred.await(engine.stderrRead); + yield* TestClock.adjust("30 seconds"); + const result = yield* Fiber.join(launch); + + const failure = Exit.isFailure(result) + ? Option.getOrUndefined(Cause.findErrorOption(result.cause)) + : undefined; + expect(failure instanceof ContainerLaunchError).toBe(true); + expect(failureKind(failure)).toBe("engine-timeout"); + expect(failure instanceof ContainerLaunchError ? failure.failure.message : undefined).toBe( + "Engine command 'start' did not finish within 30s; stderr: Error: waiting for network sandbox", + ); + }).pipe(Effect.provide(NodeServices.layer)), + ); + it.effect("removes the container a hung docker create may still create when interrupted", () => Effect.gen(function* () { const engine = yield* makeHangingCreateSpawner(); @@ -2202,6 +2230,39 @@ const successfulHandle = (stdout = "") => unref: Effect.succeed(Effect.void), }); +/** Answers every engine command except `start`, which writes `stderr` and never exits. */ +const makeStalledStartSpawner = Effect.fn("ContainerTest.stalledStartSpawner")(function* ( + stderr: string, +) { + const stderrRead = yield* Deferred.make(); + const spawner = ChildProcessSpawner.make((command) => { + if (!ChildProcess.isStandardCommand(command)) return Effect.die("unexpected piped command"); + if (command.args[0] === "info") return Effect.succeed(successfulHandle("fake-daemon-id")); + if (command.args[0] !== "start") return Effect.succeed(successfulHandle()); + // The second pull only happens once the reader has consumed the stderr line. + const stalledStderr = Stream.succeed(new TextEncoder().encode(stderr)).pipe( + Stream.concat(Stream.fromEffectDrain(Deferred.succeed(stderrRead, undefined))), + Stream.concat(Stream.never), + ); + return Effect.succeed( + ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(0), + exitCode: Effect.never, + isRunning: Effect.succeed(true), + kill: () => Effect.void, + stdin: Sink.drain, + stdout: Stream.empty, + stderr: stalledStderr, + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }), + ); + }); + return { stderrRead, layer: Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner) }; +}); + const makeHangingCreateSpawner = Effect.fn("ContainerTest.hangingCreateSpawner")(function* () { const commands: string[][] = []; const createStarted = yield* Deferred.make(); diff --git a/packages/stack/src/runtime/Container.ts b/packages/stack/src/runtime/Container.ts index 7e64d2ae34..aaf1866570 100644 --- a/packages/stack/src/runtime/Container.ts +++ b/packages/stack/src/runtime/Container.ts @@ -5,7 +5,7 @@ import { Crypto, Data, Deferred, - type Duration, + Duration, Effect, Exit, FileSystem, @@ -134,6 +134,32 @@ const engineKind = ( ? "port-allocation" : undefined; +const MANAGEMENT_COMMANDS = new Set([ + "container", + "context", + "image", + "network", + "system", + "volume", +]); + +/** Names the stalled engine subcommand and its latest stderr, never the full argv. */ +const commandTimedOut = ( + args: ReadonlyArray, + timeout: Duration.Input, + stderr: string, +): Cause.TimeoutError => { + const [first = "command", second] = args; + const subcommand = + MANAGEMENT_COMMANDS.has(first) && second !== undefined && !second.startsWith("-") + ? `${first} ${second}` + : first; + const tail = stderr.trim().slice(-2000); + return new Cause.TimeoutError( + `Engine command '${subcommand}' did not finish within ${Duration.format(Duration.fromInputUnsafe(timeout))}${tail.length > 0 ? `; stderr: ${tail}` : ""}`, + ); +}; + const errorFor = (operation: string, cause: unknown): ContainerError => { const message = cause instanceof Error ? cause.message : String(cause); const kind = engineKind(operation, cause, message); @@ -527,19 +553,25 @@ export const makeContainerRuntime = (options: { "process.arg_count": args.length, "container.command": args[0] ?? "", }); + const stderrSoFar = yield* Ref.make(""); return yield* Effect.scoped( Effect.gen(function* () { const child = yield* spawner.spawn(command(args)); - const tail = (stream: typeof child.stdout) => + const tail = (stream: typeof child.stdout, progress?: Ref.Ref) => stream.pipe( Stream.decodeText, - Stream.runFold( + Stream.runFoldEffect( () => "", - (text, chunk) => (text + chunk).slice(-65536), + (text, chunk) => { + const next = (text + chunk).slice(-65536); + return progress === undefined + ? Effect.succeed(next) + : Ref.set(progress, next).pipe(Effect.as(next)); + }, ), ); const [stdout, stderr, code] = yield* Effect.all( - [tail(child.stdout), tail(child.stderr), child.exitCode], + [tail(child.stdout), tail(child.stderr, stderrSoFar), child.exitCode], { concurrency: "unbounded" }, ); yield* Effect.annotateCurrentSpan("process.exit_code", Number(code)); @@ -551,10 +583,22 @@ export const makeContainerRuntime = (options: { return stdout.trim(); }), ).pipe( - (effect) => - commandOptions.timeout === undefined + (effect) => { + const timeout = commandOptions.timeout; + return timeout === undefined ? effect - : effect.pipe(Effect.timeout(commandOptions.timeout)), + : effect.pipe( + Effect.timeoutOrElse({ + duration: timeout, + orElse: () => + Ref.get(stderrSoFar).pipe( + Effect.flatMap((stderr) => + Effect.fail(commandTimedOut(args, timeout, stderr)), + ), + ), + }), + ); + }, Effect.mapError((cause) => errorFor(args[0] ?? "command", cause)), ); });