From 2d46b9522071705f165467a66c1c0e24952b7976 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Fri, 31 Jul 2026 14:57:02 -0700 Subject: [PATCH] fix(desktop,web): contain renderer memory growth and recover from renderer crashes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Long sessions OOM the Electron renderer at V8's heap ceiling (~3.7GB), leaving a dead white window while agents keep running invisibly. Three containments, no user-visible behavior change: - Collapse superseded context-window.updated activities in the live reducer path. Providers stream these continuously and consumers only read the latest value per turn, but the client retained every row — thousands per long thread. Mirrors the rule the server already applies to snapshots (dropStaleContextWindowActivities), including its malformed-row and per-turn revert semantics. - Drop the sidebar thread-detail prewarm from 10 to 3. Each prewarmed row pins a fully hydrated, live-growing thread subscription for as long as it is visible; cold opens still render instantly from the cached snapshot. - Reload the window on render-process-gone (crashed/oom/abnormal-exit), bounded to 3 attempts per rolling minute so a boot-crash cannot reload-loop. The backend is unaffected by renderer death, so a reload fully rehydrates. Co-Authored-By: Claude Fable 5 --- apps/desktop/src/window/DesktopWindow.ts | 45 +++++++++- apps/web/src/components/Sidebar.logic.ts | 8 +- .../src/state/threadReducer.test.ts | 88 +++++++++++++++++++ .../client-runtime/src/state/threadReducer.ts | 40 ++++++++- 4 files changed, 173 insertions(+), 8 deletions(-) diff --git a/apps/desktop/src/window/DesktopWindow.ts b/apps/desktop/src/window/DesktopWindow.ts index 40788009d3b..ed77ae1a0c8 100644 --- a/apps/desktop/src/window/DesktopWindow.ts +++ b/apps/desktop/src/window/DesktopWindow.ts @@ -1,3 +1,4 @@ +import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; @@ -25,6 +26,12 @@ const TITLEBAR_LIGHT_SYMBOL_COLOR = "#1f2937"; const TITLEBAR_DARK_SYMBOL_COLOR = "#f8fafc"; const MAIN_WINDOW_BOUNDS_PERSIST_DEBOUNCE_MS = 500; const DEVELOPMENT_LOAD_RETRY_DELAYS_MS = [100, 250, 500, 1_000, 2_000] as const; +// Renderer crash (usually V8 OOM on long sessions) recovery: reload after a +// short delay, at most MAX_ATTEMPTS times per rolling WINDOW so a renderer +// that dies on boot cannot reload-loop forever. +const RENDERER_RECOVERY_RELOAD_DELAY_MS = 500; +const RENDERER_RECOVERY_MAX_ATTEMPTS = 3; +const RENDERER_RECOVERY_WINDOW_MS = 60_000; const DEVELOPMENT_RETRYABLE_LOAD_ERROR_CODES = new Set([ -2, // ERR_FAILED -7, // ERR_TIMED_OUT @@ -537,6 +544,7 @@ export const make = Effect.gen(function* () { let developmentLoadRetryIndex = 0; let developmentLoadRetryFiber: Fiber.Fiber | undefined; + let rendererRecoveryTimestamps: number[] = []; const clearDevelopmentLoadRetry = () => { if (developmentLoadRetryFiber === undefined) { return; @@ -618,10 +626,39 @@ export const make = Effect.gen(function* () { }, ); window.webContents.on("render-process-gone", (_event, details) => { - void runPromise( - logWindowWarning("main window render process gone", { - reason: details.reason, - exitCode: details.exitCode, + const recoverable = + details.reason === "crashed" || + details.reason === "oom" || + details.reason === "abnormal-exit"; + // Long sessions can OOM the renderer (V8 heap exhaustion from + // accumulated thread state). Without a reload the user is left staring + // at a dead white window while agents keep running invisibly, so + // recover by reloading — the renderer rehydrates from the backend, + // which is unaffected. Recovery attempts are bounded so a renderer + // that dies immediately on boot cannot reload-loop forever. + runFork( + Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis; + rendererRecoveryTimestamps = rendererRecoveryTimestamps.filter( + (timestamp) => now - timestamp < RENDERER_RECOVERY_WINDOW_MS, + ); + const shouldRecover = + recoverable && + !window.isDestroyed() && + rendererRecoveryTimestamps.length < RENDERER_RECOVERY_MAX_ATTEMPTS; + yield* logWindowWarning("main window render process gone", { + reason: details.reason, + exitCode: details.exitCode, + recovering: shouldRecover, + }); + if (!shouldRecover) { + return; + } + rendererRecoveryTimestamps.push(now); + yield* Effect.sleep(RENDERER_RECOVERY_RELOAD_DELAY_MS); + if (!window.isDestroyed()) { + loadApplication(); + } }), ); }); diff --git a/apps/web/src/components/Sidebar.logic.ts b/apps/web/src/components/Sidebar.logic.ts index 7b8a0eff3ff..8025dffe3c3 100644 --- a/apps/web/src/components/Sidebar.logic.ts +++ b/apps/web/src/components/Sidebar.logic.ts @@ -16,8 +16,12 @@ import { resolveServerBackedAppStageLabel } from "../branding.logic"; export const THREAD_SELECTION_SAFE_SELECTOR = "[data-thread-item], [data-thread-selection-safe]"; export const THREAD_JUMP_HINT_SHOW_DELAY_MS = 100; // Visible sidebar rows are prewarmed into the thread-detail cache so opening a -// nearby thread usually reuses an already-hot subscription. -export const SIDEBAR_THREAD_PREWARM_LIMIT = 10; +// nearby thread usually reuses an already-hot subscription. Each prewarmed +// thread holds a live, fully hydrated detail subscription (all messages and +// activities, growing as agents work) for as long as the row stays visible, +// so this limit is a direct renderer-heap and server-load multiplier — keep +// it small; cold opens still render instantly from the cached snapshot. +export const SIDEBAR_THREAD_PREWARM_LIMIT = 3; type SidebarProject = { id: string; diff --git a/packages/client-runtime/src/state/threadReducer.test.ts b/packages/client-runtime/src/state/threadReducer.test.ts index 967c30960f9..dffae6cf6dc 100644 --- a/packages/client-runtime/src/state/threadReducer.test.ts +++ b/packages/client-runtime/src/state/threadReducer.test.ts @@ -633,6 +633,94 @@ describe("applyThreadDetailEvent", () => { expect(result.thread.activities[0]?.id).toBe("activity-0"); } }); + + it("replaces earlier resolvable context-window updates for the same turn", () => { + const contextWindowActivity = (id: string, sequence: number, usedTokens: unknown) => ({ + id: EventId.make(id), + tone: "info" as const, + kind: "context-window.updated", + summary: "Context window updated", + payload: { usedTokens }, + turnId: TurnId.make("turn-1"), + sequence, + createdAt: "2026-04-01T11:00:00.000Z", + }); + const otherTurnActivity = contextWindowActivity("activity-other-turn", 2, 500); + const existingActivities = [ + contextWindowActivity("activity-cw-1", 1, 1_000), + { ...otherTurnActivity, turnId: TurnId.make("turn-0") }, + // Malformed row (no usedTokens): must survive, and must not be + // treated as the latest value by consumers. + contextWindowActivity("activity-cw-malformed", 3, undefined), + contextWindowActivity("activity-cw-2", 4, 2_000), + ]; + + const result = applyThreadDetailEvent( + { ...baseThread, activities: existingActivities }, + { + ...baseEventFields, + sequence: 20, + occurredAt: "2026-04-01T11:02:00.000Z", + aggregateKind: "thread", + aggregateId: ThreadId.make("thread-1"), + type: "thread.activity-appended", + payload: { + threadId: ThreadId.make("thread-1"), + activity: contextWindowActivity("activity-cw-3", 5, 3_000), + }, + }, + ); + + expect(result.kind).toBe("updated"); + if (result.kind === "updated") { + const ids = result.thread.activities.map((activity) => activity.id); + // Same-turn resolvable rows collapse to the newest; the other turn's + // row and the malformed row are untouched. + expect(ids).toEqual(["activity-other-turn", "activity-cw-malformed", "activity-cw-3"]); + } + }); + + it("does not collapse context-window history for a malformed update", () => { + const resolvable = { + id: EventId.make("activity-cw-resolvable"), + tone: "info" as const, + kind: "context-window.updated", + summary: "Context window updated", + payload: { usedTokens: 1_000 }, + turnId: TurnId.make("turn-1"), + sequence: 1, + createdAt: "2026-04-01T11:00:00.000Z", + }; + + const result = applyThreadDetailEvent( + { ...baseThread, activities: [resolvable] }, + { + ...baseEventFields, + sequence: 21, + occurredAt: "2026-04-01T11:03:00.000Z", + aggregateKind: "thread", + aggregateId: ThreadId.make("thread-1"), + type: "thread.activity-appended", + payload: { + threadId: ThreadId.make("thread-1"), + activity: { + ...resolvable, + id: EventId.make("activity-cw-broken"), + payload: { usedTokens: Number.NaN }, + sequence: 2, + }, + }, + }, + ); + + expect(result.kind).toBe("updated"); + if (result.kind === "updated") { + // The resolvable row must survive so consumers can still derive a + // usage value by walking backwards past the malformed row. + const ids = result.thread.activities.map((activity) => activity.id); + expect(ids).toEqual(["activity-cw-resolvable", "activity-cw-broken"]); + } + }); }); describe("thread.turn-diff-completed", () => { diff --git a/packages/client-runtime/src/state/threadReducer.ts b/packages/client-runtime/src/state/threadReducer.ts index 6b04f094d82..7b484b8358f 100644 --- a/packages/client-runtime/src/state/threadReducer.ts +++ b/packages/client-runtime/src/state/threadReducer.ts @@ -35,6 +35,24 @@ const activityOrder = O.combineAll([ O.mapInput(O.String, (a) => a.id), ]); +/** + * Matches the validity rule in `deriveLatestContextWindowSnapshot` (and the + * server's snapshot-side `dropStaleContextWindowActivities`): rows without a + * finite, non-negative `usedTokens` are skipped during the consumer's backward + * walk, so they must not replace an earlier resolvable row here. + */ +function isResolvableContextWindowActivity(activity: OrchestrationThreadActivity): boolean { + if (activity.kind !== "context-window.updated") { + return false; + } + const payload = + activity.payload && typeof activity.payload === "object" + ? (activity.payload as Record) + : null; + const usedTokens = payload?.usedTokens; + return typeof usedTokens === "number" && Number.isFinite(usedTokens) && usedTokens >= 0; +} + /** * Apply a single orchestration event to an `OrchestrationThread`, returning * the updated thread, a deletion signal, or an "unchanged" marker when the @@ -509,10 +527,28 @@ export function applyThreadDetailEvent( // ── Activities ────────────────────────────────────────────────── case "thread.activity-appended": { + const activity = event.payload.activity; + // A resolvable context-window update supersedes earlier resolvable ones + // for the same turn: consumers only read the latest value (walking the + // array backwards), and providers stream these updates continuously, so + // retaining the history grows the thread by thousands of rows over a + // long session. Mirrors the server-side snapshot rule in + // dropStaleContextWindowActivities; retention stays per turn so a + // thread.reverted that discards turns can still resolve a value from + // the turns that survive. + const supersedesContextWindow = isResolvableContextWindowActivity(activity); const activities = pipe( thread.activities, - Arr.filter((activity) => activity.id !== event.payload.activity.id), - Arr.append(event.payload.activity), + Arr.filter( + (entry) => + entry.id !== activity.id && + !( + supersedesContextWindow && + entry.turnId === activity.turnId && + isResolvableContextWindowActivity(entry) + ), + ), + Arr.append(activity), Arr.sort(activityOrder), );