diff --git a/apps/server/src/__tests__/fixtures/sandbox-rpc-worker.ts b/apps/server/src/__tests__/fixtures/sandbox-rpc-worker.ts index b4203eb..fe18437 100644 --- a/apps/server/src/__tests__/fixtures/sandbox-rpc-worker.ts +++ b/apps/server/src/__tests__/fixtures/sandbox-rpc-worker.ts @@ -1,9 +1,20 @@ import { DurableObject } from "cloudflare:workers" -import { getSandbox as getCloudflareSandbox, type Sandbox } from "@cloudflare/sandbox" +import { getSandbox as getCloudflareSandbox, Sandbox } from "@cloudflare/sandbox" import { freshRpc } from "../../lib/containers/fresh-rpc" import { getSandbox } from "../../lib/containers/sandbox-client" export class ResettableSandbox extends DurableObject { + envVars: Record = {} + + setEnvVars(vars: Record): Promise { + // Exercise the installed SDK's environment storage, not a durable mock of it. + return Sandbox.prototype.setEnvVars.call(this as unknown as Sandbox, vars) + } + + hasCommandAuth() { + return !!this.envVars.GH_TOKEN + } + async configure(configuration: Record) { await this.ctx.storage.put("configuration", configuration) } @@ -78,6 +89,13 @@ export default { return checkSdkConfigurationOrder(env.Sandbox, mode === "/sdk-fresh") } const connect = () => env.Sandbox.getByName(mode) + if (mode === "/volatile-env") { + const client = freshRpc(connect) + await client.setEnvVars({ GH_TOKEN: "test-only" }) + const before = await client.hasCommandAuth() + await client.mutateAndReset().catch(() => {}) + return Response.json({ before, after: await client.hasCommandAuth() }) + } const stub = mode === "/fresh" ? freshRpc(connect) : connect() // Capture before the reset, like a long-lived tool or retry callback. const read = stub.readCount.bind(stub) diff --git a/apps/server/src/lib/containers/__tests__/command-environment.test.ts b/apps/server/src/lib/containers/__tests__/command-environment.test.ts new file mode 100644 index 0000000..03fa1ff --- /dev/null +++ b/apps/server/src/lib/containers/__tests__/command-environment.test.ts @@ -0,0 +1,48 @@ +import type { Sandbox } from "@cloudflare/sandbox" +import { describe, expect, it, vi } from "vitest" +import { GITHUB_COMMAND_ENV, withGitHubCommandEnv } from "../command-environment" +import { getSandbox } from "../sandbox-client" + +const mocks = vi.hoisted(() => ({ getSandbox: vi.fn() })) +vi.mock("@cloudflare/sandbox", () => ({ getSandbox: mocks.getSandbox })) + +describe("sessionless command environment", () => { + it.each(["exec", "execStream"] as const)("adds bootstrap options lazily to %s", async (method) => { + const call = vi.fn(async () => "ok") + mocks.getSandbox.mockReset().mockReturnValue({ [method]: call }) + const client = getSandbox({} as DurableObjectNamespace, "test", { enableDefaultSession: false }) + const options = { + cwd: "/repo", + timeout: 123, + env: { TEST: "value" }, + onOutput: vi.fn(), + signal: new AbortController().signal, + } + const retained = client[method].bind(client) + expect(mocks.getSandbox).not.toHaveBeenCalled() + await retained("probe", options) + expect(call).toHaveBeenCalledExactlyOnceWith("probe", { + ...options, + env: { BASH_ENV: GITHUB_COMMAND_ENV, TEST: "value" }, + }) + expect(options.env).toEqual({ TEST: "value" }) + }) + + it.each([ + { env: { GH_TOKEN: "explicit" } }, + { env: { GH_TOKEN: undefined } }, + { env: { BASH_ENV: "/custom.sh" } }, + { env: { BASH_ENV: undefined } }, + { sessionId: "explicit-session", env: { TEST: "value" } }, + ])("preserves explicit credentials, startup hooks, and sessions: %j", (options) => { + expect(withGitHubCommandEnv(options)).toEqual(options) + }) + + it("does not modify default-session clients", async () => { + const exec = vi.fn(async () => "ok") + mocks.getSandbox.mockReset().mockReturnValue({ exec }) + const client = getSandbox({} as DurableObjectNamespace, "test") + await client.exec("probe") + expect(exec).toHaveBeenCalledExactlyOnceWith("probe") + }) +}) diff --git a/apps/server/src/lib/containers/__tests__/sandbox-rpc-runtime.test.ts b/apps/server/src/lib/containers/__tests__/sandbox-rpc-runtime.test.ts index 2a8b6a6..d42f854 100644 --- a/apps/server/src/lib/containers/__tests__/sandbox-rpc-runtime.test.ts +++ b/apps/server/src/lib/containers/__tests__/sandbox-rpc-runtime.test.ts @@ -34,6 +34,14 @@ function createRuntime() { } describe("Sandbox connections across real Workers RPC resets", () => { + it("reproduces the SDK command environment being lost on DO reset", async () => { + const runtime = createRuntime() + expect(await (await runtime.dispatchFetch("https://test/volatile-env")).json()).toEqual({ + before: true, + after: false, + }) + }, 30_000) + it("reproduces a permanently broken stub and reconnects without replaying a write", async () => { const runtime = createRuntime() expect(await (await runtime.dispatchFetch("https://test/stale")).json()).toEqual({ @@ -63,7 +71,11 @@ describe("Sandbox connections across real Workers RPC resets", () => { nonThenable: true, command: "probe", sessionToken: "__DISABLE_SESSION__", - options: { cwd: "/review", timeout: 123 }, + options: { + cwd: "/review", + timeout: 123, + ...(mode === "fresh" ? { env: { BASH_ENV: "/tmp/jared-github-env.sh" } } : {}), + }, configuration: { sandboxName: { name: `sdk-${mode}`, normalizeId: true }, sleepAfter: "10m" }, }) }, 30_000) diff --git a/apps/server/src/lib/containers/__tests__/thin-prep.test.ts b/apps/server/src/lib/containers/__tests__/thin-prep.test.ts index 904e5d2..ecb67b9 100644 --- a/apps/server/src/lib/containers/__tests__/thin-prep.test.ts +++ b/apps/server/src/lib/containers/__tests__/thin-prep.test.ts @@ -4,6 +4,7 @@ import { tmpdir } from "node:os" import { join } from "node:path" import type { getSandbox } from "@cloudflare/sandbox" import { afterEach, describe, expect, it } from "vitest" +import { GITHUB_COMMAND_ENV, withGitHubCommandEnv } from "../command-environment" import { buildThinSandboxPrepScript, ensureSandboxReady, @@ -17,6 +18,8 @@ const baseOpts: SandboxSetupOpts = { installationToken: "ghs_exampletoken1234567890", entityKey: "getsentry/cli#1365", openaiApiKey: "openai-test-key", + anthropicApiKey: "anthropic-test-key", + openrouterApiKey: "openrouter-test-key", } const fixtureDirs: string[] = [] @@ -56,11 +59,12 @@ printf '%s' "$GH_TOKEN" > "$TEST_AUTH_RECEIPT" value .replaceAll("/workspace", join(dir, "workspace")) .replaceAll("/tmp/flue-env", join(dir, "tmp/flue-env")) + .replaceAll("/tmp/jared-github-env", join(dir, "tmp/jared-github-env")) .replaceAll("/root/", `${dir}/root/`) .replaceAll("/opt/flue/", `${dir}/opt/flue/`) .replaceAll("https://github.com/getsentry/cli.git", join(dir, "seed.git")) const commandEnv: Record = {} - const exec = (command: string, fail = false) => + const exec = (command: string, fail = false, env: Record = {}) => spawnSync("/bin/bash", ["--noprofile", "--norc", "-c", mapPaths(command)], { encoding: "utf8", timeout: 5_000, @@ -75,6 +79,7 @@ printf '%s' "$GH_TOKEN" > "$TEST_AUTH_RECEIPT" TEST_AUTH_RECEIPT: join(dir, "auth-receipt"), TEST_SETUP_FAIL: fail ? "1" : "0", ...commandEnv, + ...Object.fromEntries(Object.entries(env).map(([key, value]) => [key, value && mapPaths(value)])), }, }) const run = (token: string, fail = false) => { @@ -88,8 +93,8 @@ printf '%s' "$GH_TOKEN" > "$TEST_AUTH_RECEIPT" setEnvVars: async (vars: Record) => { Object.assign(commandEnv, vars) }, - exec: async (command: string) => { - const result = exec(command) + exec: async (command: string, options?: { env?: Record }) => { + const result = exec(command, false, withGitHubCommandEnv(options)?.env) return { success: result.status === 0, stderr: result.stderr, stdout: result.stdout, exitCode: result.status } }, } as unknown as ReturnType @@ -123,19 +128,63 @@ describe("buildThinSandboxPrepScript", () => { expect(readFileSync(skill, "utf8")).toBe("local agent edit") expect(readFileSync(join(f.dir, "workspace/repo/.agents/skills/new.md"), "utf8")).toBe("new skill") }) - it("authenticates later commands and detects lost command auth after a DO reset", async () => { - const { sandbox } = prepFixture() + it("keeps command auth after DO memory is lost without exposing provider keys", async () => { + const { dir, sandbox } = prepFixture() for (const token of ["initial-installation-token", "refreshed-installation-token"]) { await ensureSandboxReady(sandbox, { ...baseOpts, installationToken: token, thinSandbox: true }) const command = await sandbox.exec('printf "%s" "$GH_TOKEN"') expect(command.success).toBe(true) expect(command.stdout).toBe(token) + await sandbox.setEnvVars({ GH_TOKEN: undefined }) + expect((await sandbox.exec(THIN_SANDBOX_READY_CHECK)).success).toBe(true) + expect((await sandbox.exec('printf "%s" "$GH_TOKEN"')).stdout).toBe(token) + const bootstrap = readFileSync(join(dir, "tmp/jared-github-env.sh"), "utf8") + expect(bootstrap).not.toContain(baseOpts.openaiApiKey!) + expect(bootstrap).not.toContain(baseOpts.anthropicApiKey!) + expect(bootstrap).not.toContain(baseOpts.openrouterApiKey!) + expect(statSync(join(dir, "tmp/jared-github-env.sh")).mode & 0o777).toBe(0o600) + expect((await sandbox.exec(`test -z "\${OPENAI_API_KEY:-}"`)).success).toBe(true) + expect( + (await sandbox.exec(`test -z "\${ANTHROPIC_API_KEY:-}" && test -z "\${OPENROUTER_API_KEY:-}"`)).success, + ).toBe(true) + expect((await sandbox.exec(`test -z "\${BASH_ENV:-}"`)).success).toBe(true) } - await sandbox.setEnvVars({ GH_TOKEN: undefined }) - expect((await sandbox.exec("test -d /workspace/repo/.git")).success).toBe(true) + expect((await sandbox.exec('printf "%s" "$GH_TOKEN"', { env: { GH_TOKEN: "explicit-token" } })).stdout).toBe( + "explicit-token", + ) + expect((await sandbox.exec("exit 44")).exitCode).toBe(44) + expect(GITHUB_COMMAND_ENV).toBe("/tmp/jared-github-env.sh") + }) + + it("does not consider a legacy in-memory token ready without the command bootstrap", async () => { + const { dir, sandbox } = prepFixture() + await ensureSandboxReady(sandbox, { ...baseOpts, thinSandbox: true }) + await sandbox.setEnvVars({ GH_TOKEN: "legacy-token" }) + rmSync(join(dir, "tmp/jared-github-env.sh")) expect((await sandbox.exec(THIN_SANDBOX_READY_CHECK)).success).toBe(false) }) + it("does not reuse stale bootstrap credentials when fresh preparation has no token", async () => { + const { sandbox } = prepFixture() + await ensureSandboxReady(sandbox, { ...baseOpts, thinSandbox: true }) + await expect( + ensureSandboxReady(sandbox, { ...baseOpts, installationToken: "", thinSandbox: true }), + ).rejects.toThrow() + expect((await sandbox.exec('test -z "$GH_TOKEN"')).success).toBe(true) + }) + + it("fails closed without deleting an existing directory at the credential path", () => { + const { dir, run } = prepFixture() + const target = join(dir, "tmp/jared-github-env.sh") + mkdirSync(target) + writeFileSync(join(target, "preserved"), "existing data") + const result = run("test-token") + expect(result.status).toBe(73) + expect(result.stderr).toContain("GitHub command auth path is a directory") + expect(readFileSync(join(target, "preserved"), "utf8")).toBe("existing data") + expect(existsSync(join(dir, "workspace/.thin-sandbox-prep.lock"))).toBe(false) + }) + it("configures headless auth with fresh tokens on cold and warm workspaces", () => { const { dir, run } = prepFixture() for (const token of ["first-installation-token", "refreshed-installation-token"]) { diff --git a/apps/server/src/lib/containers/__tests__/workspace-checkpoint.test.ts b/apps/server/src/lib/containers/__tests__/workspace-checkpoint.test.ts index cee1174..fa8fbaf 100644 --- a/apps/server/src/lib/containers/__tests__/workspace-checkpoint.test.ts +++ b/apps/server/src/lib/containers/__tests__/workspace-checkpoint.test.ts @@ -42,10 +42,14 @@ function fixture() { writeFileSync(join(repo, ".agents/skills/test.md"), "test skill") writeFileSync(join(repo, ".git/jared-workspace-generation"), `${generation}\n`) writeFileSync(join(dir, "flue-env.sh"), "export GH_TOKEN='test-only'\n") + writeFileSync(join(dir, "jared-github-env.sh"), "export GH_TOKEN='test-only'\nunset BASH_ENV\n") } seed("generation-1") const map = (script: string) => - script.replaceAll("/workspace", join(dir, "workspace")).replaceAll("/tmp/flue-env.sh", join(dir, "flue-env.sh")) + script + .replaceAll("/workspace", join(dir, "workspace")) + .replaceAll("/tmp/flue-env.sh", join(dir, "flue-env.sh")) + .replaceAll("/tmp/jared-github-env.sh", join(dir, "jared-github-env.sh")) const sandbox = { exec: async (command: string, options: { cwd?: string } = {}) => { const result = spawnSync("/bin/bash", ["--noprofile", "--norc", "-c", map(command)], { diff --git a/apps/server/src/lib/containers/command-environment.ts b/apps/server/src/lib/containers/command-environment.ts new file mode 100644 index 0000000..45e554d --- /dev/null +++ b/apps/server/src/lib/containers/command-environment.ts @@ -0,0 +1,12 @@ +/** Container-owned credential bootstrap; never source the provider-key env file. */ +export const GITHUB_COMMAND_ENV = "/tmp/jared-github-env.sh" + +type CommandOptions = { env?: Record; sessionId?: string } + +export function withGitHubCommandEnv(options?: T) { + if (options?.sessionId !== undefined) return options + const env = { ...options?.env } + // Explicit per-command credentials/startup hooks retain their SDK semantics. + if (!Object.hasOwn(env, "GH_TOKEN") && !Object.hasOwn(env, "BASH_ENV")) env.BASH_ENV = GITHUB_COMMAND_ENV + return { ...options, env } +} diff --git a/apps/server/src/lib/containers/dispatch.ts b/apps/server/src/lib/containers/dispatch.ts index e3f9cc9..313be6c 100644 --- a/apps/server/src/lib/containers/dispatch.ts +++ b/apps/server/src/lib/containers/dispatch.ts @@ -15,6 +15,7 @@ import type { DrizzleD1Database } from "drizzle-orm/d1" import type * as dbSchema from "@/db/schema" import { startAgentGeneration } from "@/lib/agents/lifecycle" import { FLUE_INTERNAL_HEADER, resolveFlueInternalToken } from "@/middlewares/flue-auth" +import { GITHUB_COMMAND_ENV } from "./command-environment" import { toAgentInstanceId } from "./ids" import type { getSandbox } from "./sandbox-client" import { isTransientSandboxError } from "./sandbox-errors" @@ -157,10 +158,6 @@ export async function ensureSandboxReady( await sandbox.writeFile(envSource, buildEnvFileContents(opts, opts.installationToken)) const result = await sandbox.exec(buildThinSandboxPrepScript(opts, envSource), { cwd: "/workspace" }) if (!result.success) throw new Error(`thin sandbox prep failed: ${result.stderr}`) - // Sourcing the file only authenticates the prep shell. Native Flue runs - // later commands in separate shells, so give them the installation token - // through the Sandbox API as well (never interpolate it into exec). - await sandbox.setEnvVars({ GH_TOKEN: opts.installationToken || undefined }) }, 5) // Guard against a "successful" prep that silently dropped work: a mid-exec @@ -339,6 +336,8 @@ export function buildThinSandboxPrepScript(opts: SandboxSetupOpts, envSource = " `REPO=${shellQuote(repo)}`, `CLONE_URL=${shellQuote(cloneUrl)}`, `ENV_SOURCE=${shellQuote(envSource)}`, + `GITHUB_ENV_FILE=${shellQuote(GITHUB_COMMAND_ENV)}`, + "GITHUB_ENV_PENDING=", `BOT_LOGIN=${shellQuote(opts.botLogin)}`, `BOT_EMAIL=${shellQuote(botEmail)}`, "LOCK=/workspace/.thin-sandbox-prep.lock", @@ -349,10 +348,12 @@ export function buildThinSandboxPrepScript(opts: SandboxSetupOpts, envSource = " " i=$((i+1)); sleep 1", "done", 'if [ "$HELD" != 1 ]; then echo "sandbox prep lock timed out" >&2; exit 75; fi', - 'cleanup() { rm -f "$ENV_SOURCE"; [ "$HELD" = 1 ] && rmdir "$LOCK" 2>/dev/null || true; }', + 'cleanup() { rm -f "$ENV_SOURCE"; [ -z "$GITHUB_ENV_PENDING" ] || rm -f "$GITHUB_ENV_PENDING"; [ "$HELD" = 1 ] && rmdir "$LOCK" 2>/dev/null || true; }', "trap cleanup EXIT", "", 'test -f "$ENV_SOURCE"', + // Never let a previous BASH_ENV bootstrap mask missing fresh credentials. + "unset GH_TOKEN GITHUB_TOKEN", 'source "$ENV_SOURCE"', `TOKEN="\${GH_TOKEN:-}"`, "", @@ -385,6 +386,12 @@ export function buildThinSandboxPrepScript(opts: SandboxSetupOpts, envSource = " 'mv "$ENV_SOURCE" /tmp/flue-env.sh.tmp', "mv /tmp/flue-env.sh.tmp /tmp/flue-env.sh", "chmod 600 /tmp/flue-env.sh", + // Bash loads this for each implicit command, even after the Sandbox DO is + // evicted. Publish only GitHub auth, never the provider keys in flue-env.sh. + "GITHUB_ENV_PENDING=$(mktemp /tmp/jared-github-env.XXXXXX)", + `printf 'export GH_TOKEN=%q\\nunset BASH_ENV\\n' "$TOKEN" > "$GITHUB_ENV_PENDING"`, + 'if test -d "$GITHUB_ENV_FILE"; then echo "GitHub command auth path is a directory" >&2; exit 73; fi', + 'mv "$GITHUB_ENV_PENDING" "$GITHUB_ENV_FILE"', "mkdir -p /workspace/repo/.agents", "[ -d /root/.agents/skills ] && copy_workspace_file -R /root/.agents/skills /workspace/repo/.agents/ || true", "[ -f /root/AGENTS.md ] && copy_workspace_file /root/AGENTS.md /workspace/repo/AGENTS.md || true", @@ -409,6 +416,7 @@ async function writeEnvFile(sandbox: ReturnType, opts: Sandbo export const THIN_SANDBOX_READY_CHECK = "test -d /workspace/repo/.git && " + '[ -n "$(ls -A /workspace/repo/.agents/skills 2>/dev/null)" ] && ' + + `test -s ${GITHUB_COMMAND_ENV} && ` + `test -n "\${GH_TOKEN:-}" && ` + "grep -q '^export GH_TOKEN=' /tmp/flue-env.sh" diff --git a/apps/server/src/lib/containers/flue-session-adapt.ts b/apps/server/src/lib/containers/flue-session-adapt.ts index f938f4d..d92f5de 100644 --- a/apps/server/src/lib/containers/flue-session-adapt.ts +++ b/apps/server/src/lib/containers/flue-session-adapt.ts @@ -228,8 +228,11 @@ function extractSettlements(history: Record | null): AnyRecord[ * source of truth. Used to guard destructive actions (Clear Idle) from deleting * a container that is actually mid-task. */ -export function isFlueHistoryBusy(history: Record | null): boolean { - return deriveFlueBusyStatus(extractRawMessages(history), extractSettlements(history)) +export function isFlueHistoryBusy( + history: Record | null, + opts?: { ignoreSettledParts?: boolean }, +): boolean { + return deriveFlueBusyStatus(extractRawMessages(history), extractSettlements(history), opts) } /** @@ -255,8 +258,9 @@ export function deriveFlueBusyStatus( return true } - // Dashboard-only: a terminal receipt can survive best-effort tool repair. - // Keep the runtime admission guard conservative by default. + // A terminal receipt can survive best-effort tool repair. Recovery callers + // may opt in only when the DO atomically fences against new admissions. + // Keep destructive-action and admission guards conservative by default. if (opts?.ignoreSettledParts && submissionId && settled.has(submissionId)) continue const parts = Array.isArray(m.parts) ? (m.parts as AnyRecord[]) : [] diff --git a/apps/server/src/lib/containers/sandbox-client.ts b/apps/server/src/lib/containers/sandbox-client.ts index 23e9be1..09f06ec 100644 --- a/apps/server/src/lib/containers/sandbox-client.ts +++ b/apps/server/src/lib/containers/sandbox-client.ts @@ -1,7 +1,18 @@ import { getSandbox as getCloudflareSandbox, type Sandbox, type SandboxOptions } from "@cloudflare/sandbox" +import { withGitHubCommandEnv } from "./command-environment" import { freshRpc } from "./fresh-rpc" /** Keep SDK options/semantics, but never retain a poisoned RPC stub between calls. */ export function getSandbox(ns: DurableObjectNamespace, id: string, options?: SandboxOptions) { - return freshRpc(() => getCloudflareSandbox(ns, id, options)) + const client = freshRpc(() => getCloudflareSandbox(ns, id, options)) + if (options?.enableDefaultSession !== false) return client + return new Proxy(client, { + get(target, key) { + const method = Reflect.get(target, key, target) + if (key !== "exec" && key !== "execStream") return method + if (typeof method !== "function") throw new TypeError("Sandbox command method is unavailable") + return (command: string, commandOptions?: Parameters[0]) => + Reflect.apply(method, target, [command, withGitHubCommandEnv(commandOptions)]) + }, + }) } diff --git a/apps/server/src/routes/__tests__/workspace-recovery.test.ts b/apps/server/src/routes/__tests__/workspace-recovery.test.ts index 2d2ab96..e9d7492 100644 --- a/apps/server/src/routes/__tests__/workspace-recovery.test.ts +++ b/apps/server/src/routes/__tests__/workspace-recovery.test.ts @@ -32,6 +32,31 @@ function fixture(authenticated = true) { } describe("operator workspace acknowledgement", () => { + it.each([ + { type: "dynamic-tool", state: "input-available" }, + { type: "text", state: "streaming" }, + { type: "tool", state: { status: "running" } }, + ])("can acknowledge a settled failure with an unfinished historical part: %j", async (part) => { + const f = fixture() + const messages = [{ role: "assistant", submissionId: "sub_one", parts: [part] }] + const settlements = [{ submissionId: "sub_one", outcome: "failed", error: { type: "workspace_lost" } }] + read.mockResolvedValue({ ok: true, history: { messages, settlements } }) + expect((await f.request({ runId: "sub_one", acknowledgeDataLoss: true })).status).toBe(200) + expect(f.acknowledge).toHaveBeenCalledExactlyOnceWith("sub_one") + + // The DO is still authoritative if a new admission races the history read. + f.acknowledge.mockResolvedValue(false) + expect((await f.request({ runId: "sub_one", acknowledgeDataLoss: true })).status).toBe(409) + + f.acknowledge.mockClear() + read.mockResolvedValue({ + ok: true, + history: { messages: [...messages, { role: "user", submissionId: "sub_new", parts: [] }], settlements }, + }) + expect((await f.request({ runId: "sub_one", acknowledgeDataLoss: true })).status).toBe(409) + expect(f.acknowledge).not.toHaveBeenCalled() + }) + it("can acknowledge an exact durable blocker after Flue terminalizes an interruption", async () => { for (const receipt of [ { submissionId: "sub_one", outcome: "aborted" }, diff --git a/apps/server/src/routes/containers/workspace-recovery.ts b/apps/server/src/routes/containers/workspace-recovery.ts index e296568..de8b84d 100644 --- a/apps/server/src/routes/containers/workspace-recovery.ts +++ b/apps/server/src/routes/containers/workspace-recovery.ts @@ -43,7 +43,10 @@ export default new Hono().post("/:entityKey/workspace/acknowledge", isA )) if ( !read.ok || - isFlueHistoryBusy(read.history) || + // An interrupted tool can remain input-available after terminal settlement. + // The exact blocker and all unsettled submissions are checked atomically by + // acknowledgeWorkspaceLoss below; stale parts must not prevent recovery. + isFlueHistoryBusy(read.history, { ignoreSettledParts: true }) || (!interruption && submissionSettlementStatus(read.history, body.runId) !== "failed:workspace_lost") ) { return c.json({ error: "Only a settled workspace failure on an inactive run can be acknowledged" }, 409)