From 11281400a50eb082aa74931cd408d2496a34ab21 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 15:52:43 -0400 Subject: [PATCH 01/60] =?UTF-8?q?fix(fleet):=20fan-in=20serves=20every=20l?= =?UTF-8?q?istener=20=E2=80=94=20one=20connection=20per=20listener,=20no?= =?UTF-8?q?=20eviction=20(#1772)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/amicode_service/sse_fanin_driver.ts | 26 ++-- ...code_service_observe_event_routing.test.ts | 86 +++++++++++ .../test/sse_fanin_driver_pump.test.ts | 144 +++++++++++++++++- 3 files changed, 238 insertions(+), 18 deletions(-) diff --git a/packages/extension/src/amicode_service/sse_fanin_driver.ts b/packages/extension/src/amicode_service/sse_fanin_driver.ts index 2ef85910f..b6d5463c0 100644 --- a/packages/extension/src/amicode_service/sse_fanin_driver.ts +++ b/packages/extension/src/amicode_service/sse_fanin_driver.ts @@ -380,11 +380,15 @@ class FanInConnection { } } -/** The `/event` fan-in driver (ADR 0033 §D1). One live connection at a time - * (ADR 0027 single-origin: the app opens exactly one `/event`); a second - * `handle` tears the prior connection down first. */ +/** The `/event` fan-in driver (ADR 0033 §D1). #1772 — ONE FanInConnection PER + * LISTENER: the single-origin assumption (ADR 0027: the app opens exactly one + * `/event`) no longer holds with multiple windows, Chat Deck panes and split + * panes onto the same hub. Evicting the prior listener made two views re-open + * against each other once per second, indefinitely, silently dropping whatever + * the evicted view would have received. Each listener now owns its aggregator, + * upstream arms and reconcile timer; the driver only tracks the live set. */ export class SseFanInDriver implements EventFanInDriver { - private active?: FanInConnection; + private readonly connections = new Set(); constructor(private readonly deps: SseFanInDriverDeps) {} @@ -392,20 +396,22 @@ export class SseFanInDriver implements EventFanInDriver { // #1565: always accept on the observation path. Zero non-local owners starts // in fleet-of-one mode (the aggregator's §D4 byte-identity relay); reconcile // opens peer arms as the OwnerMapFeed discovers them. - this.active?.close(); const conn = new FanInConnection(this.deps, req, res); - this.active = conn; + this.connections.add(conn); conn.start(); + // A departing listener closes ONLY its own connection (and so only its own + // upstream arms); every other listener keeps streaming. res.on("close", () => { - if (this.active === conn) this.active = undefined; + this.connections.delete(conn); conn.close(); }); return true; } - /** Reconcile the live connection's membership NOW (the per-connection timer - * target; also called directly by the arm-lifecycle tests for determinism). */ + /** Reconcile EVERY live connection's membership NOW (each connection's own + * timer is the cadence; also called directly by the arm-lifecycle tests for + * determinism). */ reconcile(): void { - this.active?.reconcile(); + for (const conn of this.connections) conn.reconcile(); } } diff --git a/packages/extension/test/amicode_service_observe_event_routing.test.ts b/packages/extension/test/amicode_service_observe_event_routing.test.ts index 4334ca60a..442b32ff1 100644 --- a/packages/extension/test/amicode_service_observe_event_routing.test.ts +++ b/packages/extension/test/amicode_service_observe_event_routing.test.ts @@ -482,3 +482,89 @@ describe("#1580 — observation fan-in serves /global/event (v1 global-instance expect(joined).toContain('"from":"studio"'); }); }); + +// ══════════════════════════════════════════════════════════════════════════════ +// #1772 — the observation fan-in serves EVERY listener through the REAL dispatch +// path. Two concurrent `/global/event` readers (two windows, a Chat Deck pane, a +// split pane onto the same hub) both stay connected and both receive every +// frame; a listener beyond the cap is refused with a retryable 503 + Retry-After +// and no existing listener is closed to make room. The upstream origins are +// modeled as BROADCAST buses: each arm open is a new subscription, and a frame +// reaches every OPEN subscription — so an evicted listener visibly misses it. +// ══════════════════════════════════════════════════════════════════════════════ +interface BroadcastOrigins { + openUpstream: (r: { namespace: string }) => SseFrameSource; + broadcast(namespace: string, frame: string): void; + opens(namespace: string): number; + live(namespace: string): number; +} +function broadcastOrigins(): BroadcastOrigins { + const subs = new Map>(); + const openCount = new Map(); + return { + openUpstream(r) { + const ctl = controllableSource(); + const set = subs.get(r.namespace) ?? new Set(); + subs.set(r.namespace, set); + set.add(ctl); + openCount.set(r.namespace, (openCount.get(r.namespace) ?? 0) + 1); + return { + next: () => ctl.source.next(), + close: () => { + set.delete(ctl); + ctl.end(); + }, + }; + }, + broadcast(namespace, frame) { + for (const ctl of subs.get(namespace) ?? []) ctl.push(frame); + }, + opens: (ns) => openCount.get(ns) ?? 0, + live: (ns) => subs.get(ns)?.size ?? 0, + }; +} + +async function until(cond: () => boolean, timeoutMs = 2000): Promise { + const start = Date.now(); + while (!cond() && Date.now() - start < timeoutMs) await new Promise((r) => setTimeout(r, 10)); +} + +function observeDriver(origins: BroadcastOrigins, extra: Partial[0]> = {}): SseFanInDriver { + const ownerMap = new SessionOwnerMap(); + ownerMap.update([{ id: "ses-studio", amicode_owner: { owner_machine_id: "studio", owner_name: "studio", is_local: false } }]); + return new SseFanInDriver({ + ownerMap, + localMachineId: "macbook", + localEventUrl: () => "http://local.invalid", + peerBaseUrl: (id) => (id === "studio" ? "http://studio.invalid" : undefined), + peerToken: (id) => (id === "studio" ? { ok: true, credential: { baseUrl: "http://studio.invalid", token: "tok-studio" } } : { ok: false, reason: "absent" }), + reconcileMs: 999999, + openUpstream: origins.openUpstream, + ...extra, + }); +} + +describe("#1772 — the real dispatch path accepts concurrent global-stream listeners", () => { + it("AC1 — two concurrent /global/event readers both receive the local AND the peer frame", async () => { + const origins = broadcastOrigins(); + const svc = new AmicodeServiceServer({ password: PW }); + svc.attachObservationEventPlane(observeDriver(origins)); + const origin = (await svc.start()).toString().replace(/\/$/, ""); + try { + const readA = readSseFrames(`${origin}/global/event`, authed, { maxFrames: 2, timeoutMs: 3000 }); + await until(() => origins.opens("local") >= 1); + const readB = readSseFrames(`${origin}/global/event`, authed, { maxFrames: 2, timeoutMs: 3000 }); + await until(() => origins.opens("local") >= 2); + origins.broadcast("local", sseFrame("event: message", 'data: {"from":"macbook"}', "id: l1")); + origins.broadcast("studio", sseFrame("event: message", 'data: {"from":"studio"}', "id: p1")); + const [a, b] = await Promise.all([readA, readB]); + for (const frames of [a, b]) { + const joined = frames.join(""); + expect(joined).toContain('"from":"macbook"'); + expect(joined).toContain('"from":"studio"'); + } + } finally { + await svc.stop(); + } + }); +}); diff --git a/packages/extension/test/sse_fanin_driver_pump.test.ts b/packages/extension/test/sse_fanin_driver_pump.test.ts index 4d56ed51a..c059bc64f 100644 --- a/packages/extension/test/sse_fanin_driver_pump.test.ts +++ b/packages/extension/test/sse_fanin_driver_pump.test.ts @@ -54,32 +54,63 @@ function controllableSource(): Ctl { }; } -/** A mock ServerResponse that collects writes. */ -function mockRes(): http.ServerResponse & { text(): string; fireClose(): void } { +/** A mock ServerResponse that collects writes. #1772 — also records whether the + * response was ENDED (an evicted listener is ended by the driver) and the + * status/headers/body of a non-stream answer (the beyond-cap refusal). */ +type MockRes = http.ServerResponse & { + text(): string; + fireClose(): void; + ended: boolean; + status?: number; + headerMap: Record; + body: string; +}; +function mockRes(): MockRes { const chunks: string[] = []; const closeListeners: Array<() => void> = []; - return { + const res = { headersSent: false, + ended: false, + status: undefined as number | undefined, + headerMap: {} as Record, + body: "", write(chunk: string) { chunks.push(chunk); return true; }, - writeHead() { - return this; + writeHead(status: number, headers?: Record) { + res.status = status; + for (const [k, v] of Object.entries(headers ?? {})) res.headerMap[k.toLowerCase()] = String(v); + res.headersSent = true; + return res; + }, + setHeader(k: string, v: string | number) { + res.headerMap[k.toLowerCase()] = String(v); + return res; + }, + set statusCode(s: number) { + res.status = s; + }, + get statusCode(): number { + return res.status ?? 200; }, flushHeaders() {}, on(event: string, cb: (...args: unknown[]) => void) { if (event === "close") closeListeners.push(cb as () => void); - return this; + return res; + }, + end(chunk?: string) { + if (typeof chunk === "string") res.body += chunk; + res.ended = true; }, - end() {}, text() { return chunks.join(""); }, fireClose() { for (const cb of closeListeners) cb(); }, - } as unknown as http.ServerResponse & { text(): string; fireClose(): void }; + }; + return res as unknown as MockRes; } /** #1617 — a mock ServerResponse that models REAL Node backpressure: `write()` @@ -793,3 +824,100 @@ describe("#1638 — re-open resumes from last-delivered id + composite-mode reco res.fireClose() }) }) + +// ══════════════════════════════════════════════════════════════════════════════ +// #1772 — the fan-in serves EVERY listener. Each global-stream listener gets its +// own FanInConnection (aggregator + upstream arms + reconcile); a second view +// (another window, a Chat Deck pane, a split pane) no longer evicts the first. +// The harness models the upstream origins as BROADCAST buses (the engine's +// global bus delivers every event to every open subscription), so a listener +// whose arms were torn down by an eviction visibly stops receiving frames. +// ══════════════════════════════════════════════════════════════════════════════ + +/** A fake set of upstream origins: every `openUpstream` call is a NEW + * subscription on its namespace's bus; `broadcast` delivers a frame to every + * OPEN subscription of that namespace. Records every open and every close. */ +interface FakeOrigins { + openUpstream: (r: { namespace: string; lastEventId?: string }) => SseFrameSource; + broadcast(namespace: string, frame: string): void; + opens(namespace: string): number; + closes(namespace: string): number; + live(namespace: string): number; +} +function fakeOrigins(): FakeOrigins { + const subs = new Map>(); + const openCount = new Map(); + const closeCount = new Map(); + const bump = (m: Map, ns: string) => m.set(ns, (m.get(ns) ?? 0) + 1); + return { + openUpstream(r) { + const ctl = controllableSource(); + const set = subs.get(r.namespace) ?? new Set(); + subs.set(r.namespace, set); + set.add(ctl); + bump(openCount, r.namespace); + return { + next: () => ctl.source.next(), + close: () => { + if (set.delete(ctl)) bump(closeCount, r.namespace); + ctl.end(); + }, + }; + }, + broadcast(namespace, frame) { + for (const ctl of subs.get(namespace) ?? []) ctl.push(frame); + }, + opens: (ns) => openCount.get(ns) ?? 0, + closes: (ns) => closeCount.get(ns) ?? 0, + live: (ns) => subs.get(ns)?.size ?? 0, + }; +} + +const STUDIO_OWNED = [ + { id: "ses-studio", amicode_owner: { owner_machine_id: "studio", owner_name: "studio", is_local: false } }, +]; + +function twoListenerDriver(origins: FakeOrigins, ownerMap: SessionOwnerMap, extra: Partial[0]> = {}): SseFanInDriver { + return new SseFanInDriver({ + ownerMap, + localMachineId: "macbook", + localEventUrl: () => "http://local.invalid", + peerBaseUrl: (id) => (id === "studio" ? "http://studio.invalid" : undefined), + peerToken: (id) => + id === "studio" + ? { ok: true as const, credential: { baseUrl: "http://studio.invalid", token: "tok-studio" } } + : { ok: false as const, reason: "absent" as const }, + reconcileMs: 999_999, + openUpstream: origins.openUpstream, + ...extra, + }); +} + +describe("#1772 AC1 — two concurrent listeners both stay connected and both receive every frame", () => { + it("listener A and listener B each receive the local arm's AND the peer arm's frames; neither is ended", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const origins = fakeOrigins(); + const driver = twoListenerDriver(origins, ownerMap); + + const resA = mockRes(); + const resB = mockRes(); + expect(driver.handle(mockReq("/global/event"), resA)).toBe(true); + await tick(); + expect(driver.handle(mockReq("/global/event"), resB)).toBe(true); + await tick(); + + origins.broadcast(LOCAL_NAMESPACE, sseFrame("event: message", 'data: {"from":"macbook","n":1}', "id: l1")); + origins.broadcast("studio", sseFrame("event: message", 'data: {"from":"studio","n":2}', "id: p1")); + await tick(); + + for (const res of [resA, resB]) { + expect(res.ended).toBe(false); // still connected — no eviction + expect(res.text()).toContain('"from":"macbook"'); + expect(res.text()).toContain('"from":"studio"'); + } + + resA.fireClose(); + resB.fireClose(); + }); +}); From b6c66fbcfccd39616fdd416acf63c6710c818a96 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 15:53:09 -0400 Subject: [PATCH 02/60] test(fleet): closing one fan-in listener closes only its own arms (#1772) --- .../test/sse_fanin_driver_pump.test.ts | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/packages/extension/test/sse_fanin_driver_pump.test.ts b/packages/extension/test/sse_fanin_driver_pump.test.ts index c059bc64f..01e33d929 100644 --- a/packages/extension/test/sse_fanin_driver_pump.test.ts +++ b/packages/extension/test/sse_fanin_driver_pump.test.ts @@ -921,3 +921,55 @@ describe("#1772 AC1 — two concurrent listeners both stay connected and both re resB.fireClose(); }); }); + +describe("#1772 AC2 — closing one listener closes ONLY that listener's upstream arms", () => { + it("A departs → exactly A's local + peer arms close; B keeps its arms and keeps streaming, uninterrupted", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const origins = fakeOrigins(); + const driver = twoListenerDriver(origins, ownerMap); + + const resA = mockRes(); + const resB = mockRes(); + driver.handle(mockReq("/global/event"), resA); + await tick(); + driver.handle(mockReq("/global/event"), resB); + await tick(); + // Both listeners hold their own local + peer arm. + expect(origins.live(LOCAL_NAMESPACE)).toBe(2); + expect(origins.live("studio")).toBe(2); + + origins.broadcast(LOCAL_NAMESPACE, sseFrame("event: message", 'data: {"n":"before"}', "id: l1")); + await tick(); + + // Listener A departs (its socket closes). + resA.fireClose(); + await tick(); + + // Exactly A's two arms closed; B's two arms untouched. + expect(origins.closes(LOCAL_NAMESPACE)).toBe(1); + expect(origins.closes("studio")).toBe(1); + expect(origins.live(LOCAL_NAMESPACE)).toBe(1); + expect(origins.live("studio")).toBe(1); + + // B keeps streaming on the SAME arms (no re-open), from both origins. + driver.reconcile(); + await tick(); + origins.broadcast(LOCAL_NAMESPACE, sseFrame("event: message", 'data: {"n":"after-local"}', "id: l2")); + origins.broadcast("studio", sseFrame("event: message", 'data: {"n":"after-peer"}', "id: p2")); + await tick(); + expect(resB.ended).toBe(false); + expect(resB.text()).toContain('"n":"before"'); + expect(resB.text()).toContain('"n":"after-local"'); + expect(resB.text()).toContain('"n":"after-peer"'); + expect(origins.opens(LOCAL_NAMESPACE)).toBe(2); // one per listener, never re-opened + expect(origins.opens("studio")).toBe(2); + // The departed listener receives nothing further. + expect(resA.text()).not.toContain('"n":"after-local"'); + + resB.fireClose(); + await tick(); + expect(origins.live(LOCAL_NAMESPACE)).toBe(0); + expect(origins.live("studio")).toBe(0); + }); +}); From dc0c31066a5abcfd271a90c407605c166a4bf992 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 15:53:44 -0400 Subject: [PATCH 03/60] test(fleet): reconcile updates every live fan-in connection on its cadence (#1772) --- .../test/sse_fanin_driver_pump.test.ts | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/packages/extension/test/sse_fanin_driver_pump.test.ts b/packages/extension/test/sse_fanin_driver_pump.test.ts index 01e33d929..ce835cca6 100644 --- a/packages/extension/test/sse_fanin_driver_pump.test.ts +++ b/packages/extension/test/sse_fanin_driver_pump.test.ts @@ -973,3 +973,74 @@ describe("#1772 AC2 — closing one listener closes ONLY that listener's upstrea expect(origins.live("studio")).toBe(0); }); }); + +/** Poll until `cond` holds (or time out) — for the cadence-driven tests, where + * each connection's own reconcile timer (not a manual call) does the work. */ +async function until(cond: () => boolean, timeoutMs = 2000): Promise { + const start = Date.now(); + while (!cond() && Date.now() - start < timeoutMs) await tick(5); + return cond(); +} + +describe("#1772 AC3 — reconcile updates EVERY live connection's membership", () => { + it("driver.reconcile() opens a newly-owned peer's arm on BOTH listeners, and closes it on both when it goes dark", async () => { + const ownerMap = new SessionOwnerMap(); // fleet-of-one at connect + const origins = fakeOrigins(); + const driver = twoListenerDriver(origins, ownerMap); + + const resA = mockRes(); + const resB = mockRes(); + driver.handle(mockReq("/global/event"), resA); + driver.handle(mockReq("/global/event"), resB); + await tick(); + expect(origins.opens("studio")).toBe(0); + + ownerMap.update(STUDIO_OWNED); + driver.reconcile(); + await tick(); + expect(origins.live("studio")).toBe(2); // one peer arm per listener + origins.broadcast("studio", sseFrame("event: message", 'data: {"from":"studio"}', "id: p1")); + await tick(); + expect(resA.text()).toContain('"from":"studio"'); + expect(resB.text()).toContain('"from":"studio"'); + + ownerMap.update([]); + driver.reconcile(); + await tick(); + expect(origins.live("studio")).toBe(0); // both listeners dropped the dark peer + expect(origins.live(LOCAL_NAMESPACE)).toBe(2); // local arms untouched (§D4) + + resA.fireClose(); + resB.fireClose(); + }); + + it("on its CADENCE (no manual reconcile): both listeners pick up a new owner, then drop it", async () => { + const ownerMap = new SessionOwnerMap(); + const origins = fakeOrigins(); + const driver = twoListenerDriver(origins, ownerMap, { reconcileMs: 20 }); + + const resA = mockRes(); + const resB = mockRes(); + driver.handle(mockReq("/global/event"), resA); + driver.handle(mockReq("/global/event"), resB); + await tick(); + + ownerMap.update(STUDIO_OWNED); + expect(await until(() => origins.live("studio") === 2)).toBe(true); + origins.broadcast("studio", sseFrame("event: message", 'data: {"from":"studio"}', "id: p1")); + await tick(); + expect(resA.text()).toContain('"from":"studio"'); + expect(resB.text()).toContain('"from":"studio"'); + + ownerMap.update([]); + expect(await until(() => origins.live("studio") === 0)).toBe(true); + + resA.fireClose(); + resB.fireClose(); + // A closed listener's timer is stopped: a later owner change opens nothing. + const opensAfterClose = origins.opens("studio"); + ownerMap.update(STUDIO_OWNED); + await tick(80); + expect(origins.opens("studio")).toBe(opensAfterClose); + }); +}); From 31db217e8ce9819d66254b4f9d1535cd03b0df06 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 15:55:06 -0400 Subject: [PATCH 04/60] =?UTF-8?q?fix(fleet):=20cap=20concurrent=20fan-in?= =?UTF-8?q?=20listeners=20=E2=80=94=20refuse=20beyond=20it=20with=20a=20re?= =?UTF-8?q?tryable=20503,=20never=20evict=20(#1772)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/amicode_service/sse_fanin_driver.ts | 48 ++++++++++++++++- ...code_service_observe_event_routing.test.ts | 27 ++++++++++ .../test/sse_fanin_driver_pump.test.ts | 54 +++++++++++++++++++ 3 files changed, 128 insertions(+), 1 deletion(-) diff --git a/packages/extension/src/amicode_service/sse_fanin_driver.ts b/packages/extension/src/amicode_service/sse_fanin_driver.ts index b6d5463c0..e050dd0da 100644 --- a/packages/extension/src/amicode_service/sse_fanin_driver.ts +++ b/packages/extension/src/amicode_service/sse_fanin_driver.ts @@ -77,8 +77,20 @@ export interface SseFanInDriverDeps { /** The live reconcile cadence per connection (ms). Default 1000. Tests set a * large value and call `reconcile()` directly for determinism. */ reconcileMs?: number; + /** #1772 — the cap on CONCURRENT listeners (one FanInConnection each). A + * listener beyond it is refused with a retryable 503 + Retry-After; no live + * listener is ever closed to make room. Default DEFAULT_FANIN_MAX_LISTENERS. */ + maxListeners?: number; } +/** #1772 — default concurrent-listener cap per fan-in service. A service rarely + * has more than a handful of views (windows, Chat Deck panes, split panes). */ +export const DEFAULT_FANIN_MAX_LISTENERS = 16; +/** #1772 — the Retry-After hint (seconds) on a beyond-cap refusal. */ +export const FANIN_LISTENER_CAP_RETRY_AFTER_S = 5; +/** #1772 — the named error on a beyond-cap refusal. */ +export const FANIN_LISTENER_CAP_ERROR = "fan-in listener cap reached"; + /** The production upstream reader: a frame-preserving http SSE source. Splits the * byte stream on the SSE blank-line delimiter into WHOLE frames (delimiter- * inclusive) so `event:`/`data:`/`retry:`/`id:` lines survive intact for the @@ -380,19 +392,53 @@ class FanInConnection { } } +/** #1772 — the explicit, retryable beyond-cap refusal: 503 + Retry-After and a + * named JSON body (the `{ ok: false, error, reason }` shape of the service's + * other 503s). The app's stream loop treats any non-200 as a disconnect and + * retries; nothing about this response is an eviction of anyone else. */ +function refuseBeyondCap(res: http.ServerResponse, cap: number): void { + try { + res.writeHead(503, { + "content-type": "application/json", + "cache-control": "no-cache", + "retry-after": String(FANIN_LISTENER_CAP_RETRY_AFTER_S), + }); + res.end( + JSON.stringify({ + ok: false, + error: FANIN_LISTENER_CAP_ERROR, + reason: "listener-cap", + max_listeners: cap, + retry_after_s: FANIN_LISTENER_CAP_RETRY_AFTER_S, + }), + ); + } catch { + /* the refused socket is already gone — nothing to answer */ + } +} + /** The `/event` fan-in driver (ADR 0033 §D1). #1772 — ONE FanInConnection PER * LISTENER: the single-origin assumption (ADR 0027: the app opens exactly one * `/event`) no longer holds with multiple windows, Chat Deck panes and split * panes onto the same hub. Evicting the prior listener made two views re-open * against each other once per second, indefinitely, silently dropping whatever * the evicted view would have received. Each listener now owns its aggregator, - * upstream arms and reconcile timer; the driver only tracks the live set. */ + * upstream arms and reconcile timer; the driver only tracks the live set, + * bounded by `maxListeners` (refusal beyond it, never an eviction). */ export class SseFanInDriver implements EventFanInDriver { private readonly connections = new Set(); constructor(private readonly deps: SseFanInDriverDeps) {} handle(req: http.IncomingMessage, res: http.ServerResponse): boolean { + // #1772: beyond the cap the listener is REFUSED — answered here (so dispatch + // never falls through to an un-fanned engine stream) with a retryable 503 + // and a Retry-After hint. No existing listener is closed to make room. + const cap = this.deps.maxListeners ?? DEFAULT_FANIN_MAX_LISTENERS; + if (this.connections.size >= cap) { + refuseBeyondCap(res, cap); + return true; + } // #1565: always accept on the observation path. Zero non-local owners starts // in fleet-of-one mode (the aggregator's §D4 byte-identity relay); reconcile // opens peer arms as the OwnerMapFeed discovers them. diff --git a/packages/extension/test/amicode_service_observe_event_routing.test.ts b/packages/extension/test/amicode_service_observe_event_routing.test.ts index 442b32ff1..e8591bb42 100644 --- a/packages/extension/test/amicode_service_observe_event_routing.test.ts +++ b/packages/extension/test/amicode_service_observe_event_routing.test.ts @@ -568,3 +568,30 @@ describe("#1772 — the real dispatch path accepts concurrent global-stream list } }); }); + +describe("#1772 AC4 — beyond the (configured) cap, the real dispatch path refuses — retryable, no eviction", () => { + it("maxListeners: 1 → a second /global/event gets 503 + Retry-After; the first listener keeps streaming", async () => { + const origins = broadcastOrigins(); + const svc = new AmicodeServiceServer({ password: PW }); + svc.attachObservationEventPlane(observeDriver(origins, { maxListeners: 1 })); + const origin = (await svc.start()).toString().replace(/\/$/, ""); + try { + const readA = readSseFrames(`${origin}/global/event`, authed, { maxFrames: 1, timeoutMs: 3000 }); + await until(() => origins.opens("local") >= 1); + + const refused = await fetch(`${origin}/global/event`, { headers: authed, signal: AbortSignal.timeout(3000) }); + expect(refused.status).toBe(503); + expect(Number(refused.headers.get("retry-after"))).toBeGreaterThan(0); + const body = (await refused.json()) as { ok: boolean; reason: string; max_listeners: number }; + expect(body).toMatchObject({ ok: false, reason: "listener-cap", max_listeners: 1 }); + + // The refusal opened no arm and closed none: the first listener is intact. + expect(origins.opens("local")).toBe(1); + expect(origins.live("local")).toBe(1); + origins.broadcast("local", sseFrame("event: message", 'data: {"n":"after-refusal"}', "id: l9")); + expect((await readA).join("")).toContain('"n":"after-refusal"'); + } finally { + await svc.stop(); + } + }); +}); diff --git a/packages/extension/test/sse_fanin_driver_pump.test.ts b/packages/extension/test/sse_fanin_driver_pump.test.ts index ce835cca6..d9de91ad2 100644 --- a/packages/extension/test/sse_fanin_driver_pump.test.ts +++ b/packages/extension/test/sse_fanin_driver_pump.test.ts @@ -1044,3 +1044,57 @@ describe("#1772 AC3 — reconcile updates EVERY live connection's membership", ( expect(origins.opens("studio")).toBe(opensAfterClose); }); }); + +describe("#1772 AC4 — a listener beyond the cap is refused (retryable), never by evicting one", () => { + it("default cap 16: the 17th listener gets 503 + Retry-After, opens no arm, and all 16 keep streaming; a freed slot is reusable", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const origins = fakeOrigins(); + const driver = twoListenerDriver(origins, ownerMap); + + const accepted: MockRes[] = []; + for (let i = 0; i < 16; i++) { + const res = mockRes(); + expect(driver.handle(mockReq("/global/event"), res)).toBe(true); + accepted.push(res); + } + await tick(); + expect(origins.live(LOCAL_NAMESPACE)).toBe(16); + + // The 17th is ANSWERED (handle owns the response) with a retryable refusal. + const refused = mockRes(); + expect(driver.handle(mockReq("/global/event"), refused)).toBe(true); + await tick(); + expect(refused.status).toBe(503); + expect(Number(refused.headerMap["retry-after"])).toBeGreaterThan(0); + expect(refused.ended).toBe(true); + const body = JSON.parse(refused.body) as { ok: boolean; reason: string; max_listeners: number }; + expect(body.ok).toBe(false); + expect(body.reason).toBe("listener-cap"); + expect(body.max_listeners).toBe(16); + + // No existing listener was closed to make room, and the refused one opened nothing. + expect(origins.opens(LOCAL_NAMESPACE)).toBe(16); + expect(origins.closes(LOCAL_NAMESPACE)).toBe(0); + expect(origins.live("studio")).toBe(16); + origins.broadcast(LOCAL_NAMESPACE, sseFrame("event: message", 'data: {"n":"after-refusal"}', "id: l1")); + await tick(); + for (const res of accepted) { + expect(res.ended).toBe(false); + expect(res.text()).toContain('"n":"after-refusal"'); + } + expect(refused.text()).toBe(""); + + // The cap bounds CONCURRENT listeners: once one departs, a retry is accepted. + accepted[0].fireClose(); + await tick(); + const retry = mockRes(); + driver.handle(mockReq("/global/event"), retry); + await tick(); + expect(retry.status).toBe(200); + expect(retry.ended).toBe(false); + expect(origins.live(LOCAL_NAMESPACE)).toBe(16); + + for (const res of [...accepted, retry]) res.fireClose(); + }); +}); From 691a8af88b8e1511d6ed687c9af9252d44ebc5e4 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 15:55:47 -0400 Subject: [PATCH 05/60] =?UTF-8?q?test(fleet):=20two-listener=20harness=20?= =?UTF-8?q?=E2=80=94=20one=20local=20arm=20each,=20no=20re-open=20storm=20?= =?UTF-8?q?(#1772)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../test/sse_fanin_driver_pump.test.ts | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/packages/extension/test/sse_fanin_driver_pump.test.ts b/packages/extension/test/sse_fanin_driver_pump.test.ts index d9de91ad2..ac1c9152e 100644 --- a/packages/extension/test/sse_fanin_driver_pump.test.ts +++ b/packages/extension/test/sse_fanin_driver_pump.test.ts @@ -1098,3 +1098,71 @@ describe("#1772 AC4 — a listener beyond the cap is refused (retryable), never for (const res of [...accepted, retry]) res.fireClose(); }); }); + +/** #1772 — a VIEW that behaves like the app's stream loop: when the driver ENDS + * its stream (an eviction is a clean end), the socket closes and the view + * reconnects after `delayMs` (the client's reconnect delay, shortened). Two + * such views against an evicting driver re-open against each other forever — + * the Studio's ~340-reconnects-in-15-minutes storm. */ +function reconnectingView(driver: SseFanInDriver, delayMs = 5): { connects(): number; current(): MockRes; stop(): void } { + let stopped = false; + let connects = 0; + let current!: MockRes; + const connect = (): void => { + if (stopped) return; + connects += 1; + const res = mockRes(); + const end = res.end.bind(res); + res.end = ((chunk?: string) => { + end(chunk); + setTimeout(() => { + res.fireClose(); + connect(); + }, delayMs); + }) as typeof res.end; + current = res; + driver.handle(mockReq("/global/event"), res); + }; + connect(); + return { + connects: () => connects, + current: () => current, + stop() { + stopped = true; + current.fireClose(); + }, + }; +} + +describe("#1772 AC5 — two listeners: one local arm each, kept open — no repeating re-open pattern", () => { + it("two reconnecting views over many live reconcile ticks: 2 local arms opened, 0 closed, 0 view reconnects", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const origins = fakeOrigins(); + const driver = twoListenerDriver(origins, ownerMap, { reconcileMs: 10 }); + + const viewA = reconnectingView(driver); + await tick(); + const viewB = reconnectingView(driver); + await tick(250); // ~25 reconcile ticks per connection + + try { + expect(viewA.connects()).toBe(1); + expect(viewB.connects()).toBe(1); + expect(origins.opens(LOCAL_NAMESPACE)).toBe(2); // one local arm per listener + expect(origins.closes(LOCAL_NAMESPACE)).toBe(0); // …and each is kept open + expect(origins.live(LOCAL_NAMESPACE)).toBe(2); + expect(origins.opens("studio")).toBe(2); + expect(origins.closes("studio")).toBe(0); + + // Both views are live on those same arms. + origins.broadcast(LOCAL_NAMESPACE, sseFrame("event: message", 'data: {"n":"steady"}', "id: l1")); + await tick(); + expect(viewA.current().text()).toContain('"n":"steady"'); + expect(viewB.current().text()).toContain('"n":"steady"'); + } finally { + viewA.stop(); + viewB.stop(); + } + }); +}); From bebeba845506f8e02051371661da92ec6682b367 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 15:57:03 -0400 Subject: [PATCH 06/60] test(fleet): #1264/#1601/#1617 hold per fan-in listener; retire single-origin notes (#1772) --- .../extension/src/amicode_service/server.ts | 4 +- .../src/amicode_service/sse_fanin_driver.ts | 12 +- .../test/sse_fanin_driver_pump.test.ts | 104 ++++++++++++++++++ 3 files changed, 115 insertions(+), 5 deletions(-) diff --git a/packages/extension/src/amicode_service/server.ts b/packages/extension/src/amicode_service/server.ts index 98368ecfc..1cdf9c2ee 100644 --- a/packages/extension/src/amicode_service/server.ts +++ b/packages/extension/src/amicode_service/server.ts @@ -581,7 +581,9 @@ export class AmicodeServiceServer { return; } // #1519 (W1c, ADR 0033 §D1): the SSE fan-in interception for the app's - // ONE global /event stream (ADR 0027 single-origin). AHEAD of every + // global /event stream. #1772: one fan-in connection PER LISTENER (windows, + // Chat Deck panes, split panes) — a second view no longer evicts the first; + // beyond the driver's cap a listener gets a retryable 503. AHEAD of every // downstream path (the fleet hub block, the engine proxy) so a machine // that owns peer sessions fans their events IN onto this one response. // A SINGLE flag-gated expression: with the flag OFF (default) the `&&` diff --git a/packages/extension/src/amicode_service/sse_fanin_driver.ts b/packages/extension/src/amicode_service/sse_fanin_driver.ts index e050dd0da..72260e628 100644 --- a/packages/extension/src/amicode_service/sse_fanin_driver.ts +++ b/packages/extension/src/amicode_service/sse_fanin_driver.ts @@ -5,8 +5,11 @@ // #1511 built the synchronous frame-routing CORE (sse_fanin_aggregator.ts) and // unit-tested all 7 ACs against a deterministic sink. What was deliberately NOT // wired was the driver that opens the REAL upstream SSE connections and pumps -// their frames onto the ONE downstream `res` the app reads (ADR 0027 single- -// origin). This module is that driver. +// their frames onto the downstream `res` the app reads. This module is that +// driver. #1772: ADR 0027's single-origin assumption (the app opens exactly one +// `/event`) no longer holds — multiple windows, Chat Deck panes and split panes +// each open their own — so every listener gets its own connection (aggregator, +// arms, reconcile) up to a cap; none is ever evicted to make room for another. // // On the observation path (#1543/#1565), `handle()` ALWAYS accepts (returns // true) — zero non-local owners starts in fleet-of-one mode where the @@ -150,8 +153,9 @@ export function openHttpSseSource(r: FanInUpstreamRequest): SseFrameSource { }; } -/** One live `/event` fan-in connection: the aggregator on this response, the - * open upstream sources keyed by namespace, and the reconcile timer. */ +/** One live `/event` fan-in connection — ONE PER LISTENER (#1772): the + * aggregator on this response, the open upstream sources keyed by namespace, + * and this connection's own reconcile timer. Closing it closes only its arms. */ class FanInConnection { private readonly agg: SseFanInAggregator; private readonly sources = new Map(); diff --git a/packages/extension/test/sse_fanin_driver_pump.test.ts b/packages/extension/test/sse_fanin_driver_pump.test.ts index ac1c9152e..a903c8b42 100644 --- a/packages/extension/test/sse_fanin_driver_pump.test.ts +++ b/packages/extension/test/sse_fanin_driver_pump.test.ts @@ -1166,3 +1166,107 @@ describe("#1772 AC5 — two listeners: one local arm each, kept open — no repe } }); }); + +describe("#1772 invariants — #1264 / #1601 / #1617 hold PER LISTENER", () => { + it("#1264 fleet-of-one: each of two listeners receives the local stream byte-identical", async () => { + const origins = fakeOrigins(); + const driver = twoListenerDriver(origins, new SessionOwnerMap()); + const resA = mockRes(); + const resB = mockRes(); + driver.handle(mockReq("/global/event"), resA); + driver.handle(mockReq("/global/event"), resB); + await tick(); + + const F1 = sseFrame("event: message", 'data: {"a":1}', "id: 1"); + const F2 = sseFrame("event: message", 'data: {"b":2}', "id: 2"); + origins.broadcast(LOCAL_NAMESPACE, F1); + origins.broadcast(LOCAL_NAMESPACE, F2); + await tick(); + expect(resA.text()).toBe(F1 + F2); + expect(resB.text()).toBe(F1 + F2); + + resA.fireClose(); + resB.fireClose(); + }); + + it("#1601 per connection: one listener's dead local arm is re-opened for THAT listener only", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const localCtls: Ctl[] = []; + const driver = twoListenerDriver(fakeOrigins(), ownerMap, { + openUpstream: (r) => { + const ctl = controllableSource(); + if (r.namespace === LOCAL_NAMESPACE) localCtls.push(ctl); + return ctl.source; + }, + }); + const resA = mockRes(); + const resB = mockRes(); + driver.handle(mockReq("/global/event"), resA); // localCtls[0] + driver.handle(mockReq("/global/event"), resB); // localCtls[1] + await tick(); + expect(localCtls.length).toBe(2); + + localCtls[0].end(); // A's local upstream drops + await tick(); + driver.reconcile(); + await tick(); + expect(localCtls.length).toBe(3); // exactly one re-open — A's + + localCtls[2].push(sseFrame("event: message", 'data: {"to":"A"}', "id: a1")); + localCtls[1].push(sseFrame("event: message", 'data: {"to":"B"}', "id: b1")); + await tick(); + expect(resA.text()).toContain('"to":"A"'); + expect(resA.text()).not.toContain('"to":"B"'); + expect(resB.text()).toContain('"to":"B"'); + expect(resB.text()).not.toContain('"to":"A"'); + + localCtls.forEach((c) => c.end()); + resA.fireClose(); + resB.fireClose(); + }); + + it("#1617 per connection: a backpressured listener pauses only ITS upstreams and heals on ITS drain", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const ctls: PausableCtl[] = []; + const driver = twoListenerDriver(fakeOrigins(), ownerMap, { + openUpstream: () => { + const ctl = pausableSource(); + ctls.push(ctl); + return ctl.source; + }, + }); + const resA = backpressureRes(); + driver.handle(mockReq("/global/event"), resA); // ctls[0] local, ctls[1] studio + await tick(); + const resB = mockRes(); + driver.handle(mockReq("/global/event"), resB); // ctls[2] local, ctls[3] studio + await tick(); + const [aLocal, aPeer, bLocal, bPeer] = ctls; + + resA.block(); + aLocal.push(sseFrame("event: message", 'data: {"n":1}', "id: l1")); // trips A's backpressure + await tick(); + aLocal.push(sseFrame("event: message", 'data: {"n":2}', "id: l2")); // stranded behind it + await tick(); + expect(aLocal.pauseCalls).toBeGreaterThan(0); + expect(aPeer.pauseCalls).toBeGreaterThan(0); + expect(bLocal.pauseCalls).toBe(0); // B is not throttled by A's slow socket + expect(bPeer.pauseCalls).toBe(0); + + bLocal.push(sseFrame("event: message", 'data: {"to":"B"}', "id: b1")); + await tick(); + expect(resB.text()).toContain('"to":"B"'); // B streams on while A is backpressured + expect(resA.text()).not.toContain('"n":2'); + + resA.fireDrain(); + await tick(); + expect(resA.text()).toContain('"n":2'); + expect(aLocal.resumeCalls).toBeGreaterThan(0); + + ctls.forEach((c) => c.end()); + resA.fireClose(); + resB.fireClose(); + }); +}); From d401ea00cfffbbe3b6698cfaf9c3129fb1e21e4a Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 16:07:32 -0400 Subject: [PATCH 07/60] feat(engine): stamp every global-stream frame with an : id (#1776) One sequencer at the global bus assigns a strictly increasing sequence at emit, shared by every subscriber; a sync copy carries its source event's sequence. A pre-dispatch tap serializes each event once into a frame the global stream writes with its SSE id. Connected and heartbeat frames carry the connection's current sequence without consuming one. --- packages/app-bundle/manifest.json | 16 +- .../opencode/src/bus/global-replay.ts | 75 +++++++++ .../packages/opencode/src/bus/global.ts | 76 +++++++++ .../instance/httpapi/handlers/global.ts | 141 +++++++++++++++++ .../test/server/httpapi-global-event.test.ts | 147 ++++++++++++++++++ 5 files changed, 451 insertions(+), 4 deletions(-) create mode 100644 packages/app-bundle/overlay/packages/opencode/src/bus/global-replay.ts create mode 100644 packages/app-bundle/overlay/packages/opencode/src/bus/global.ts create mode 100644 packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts create mode 100644 packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts diff --git a/packages/app-bundle/manifest.json b/packages/app-bundle/manifest.json index db47e49e5..cc18daf60 100644 --- a/packages/app-bundle/manifest.json +++ b/packages/app-bundle/manifest.json @@ -316,6 +316,8 @@ "packages/llm/src/protocols/bedrock-converse.ts": "834a415fa8ddf9ad1224133f8352d0a926694f19f0954201d182111c171ec03b", "packages/llm/test/provider/bedrock-converse.test.ts": "e8e14d60e8502e928f170f8ac5b406c44bf16d2dd34996ddc2fb7b9261decf85", "packages/opencode/src/agent/agent.ts": "512372a4c62997e9565266638ada19b9dc0a3685b0c0483fd28fede2abdb5c9b", + "packages/opencode/src/bus/global-replay.ts": "bc5d4e77c2aa8ef0444177e6174af0642428d1bcdf47fc668a4799d57163597e", + "packages/opencode/src/bus/global.ts": "65b3b380936fc34b2c5ef9f3ab419a6c8e6d4cfc696302a266188f26ccdc320a", "packages/opencode/src/effect/runner.ts": "b660b5b22df810b16293c23c6d832e5e8dcfb12b9a8a858c875d2c051c855b81", "packages/opencode/src/mcp/browser.ts": "ecdda42cbd55a97b8d14050aa15c6e6450a1d63b0a641c8c8e5b81e50f957c87", "packages/opencode/src/mcp/index.ts": "da45165e6791a5b30c8962c7fd0ea391bb8e1931cc2357c6b90e9fd090cdeb0d", @@ -390,6 +392,7 @@ "packages/opencode/test/server/httpapi-compression.test.ts": "5f7603d7018e0b95f1c2f36c01dd297c01770bfcdd91ce29778e03b7d6143f75", "packages/opencode/test/server/httpapi-file-write-gate.test.ts": "0c91b7417e615850edfde3f2878207dab6b6624d4cfbb6e6783b2e6d9aa4dff0", "packages/opencode/test/server/httpapi-file.test.ts": "d3386129bc81467ecbcc57a3861f53767ee56ac41b4ff6ad58d8b168fb3b3373", + "packages/opencode/test/server/httpapi-global-event.test.ts": "eaecf3b6e0a148f48e9a056c1b24184cda8f730aeec84aed2d7b89a50548d637", "packages/opencode/test/server/httpapi-instance-route-auth.test.ts": "4b73a89d2431c02bc3b665d93f9002914e51eb1add979dc1867efbcbc0c36e25", "packages/opencode/test/server/httpapi-provider.test.ts": "9e8b3e085b478e3f99b689ffff600e10c0b4b50b3cabdbd749f88dae19bf5904", "packages/opencode/test/server/httpapi-session.test.ts": "eda6fd763ac4c17891cedaa4ccd49597c03e7227fcfdcf0f94a1b4545ba623ed", @@ -924,6 +927,7 @@ "packages/opencode/src/server/routes/instance/httpapi/groups/session.ts": "1421f8c07948b121c7e2005bd9b0fcd6b2c456565ddd1da32d8e6c35d03fac9c", "packages/opencode/src/server/routes/instance/httpapi/handlers/experimental.ts": "69ca3dd48d95157fc822e1f76eb42ac873df50f22616b85194619ca658613867", "packages/opencode/src/server/routes/instance/httpapi/handlers/file.ts": "10bcb79b117beebee04184fb48c1ad789e782e2f7ae4b0c68603edc00f03aee9", + "packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts": "efbbbcebff05007137bbab19486fa6bd4602f84f11f463b4d56dd0da5cb76de7", "packages/opencode/src/server/routes/instance/httpapi/handlers/mcp.ts": "7da08d58765cdf2b6ce4648fc70de371402949f5bb7b75696f674d43f2d5365b", "packages/opencode/src/server/routes/instance/httpapi/handlers/provider.ts": "8baf16cad0992c5351b61b0c0fb27903bb25b17e1b25fc7c5d77f807c24413a6", "packages/opencode/src/server/routes/instance/httpapi/handlers/session.ts": "b37ff4136c4aea85b23083bf441f03e09820e3e4cdd95e4118ca570d8435b94f", @@ -942,7 +946,7 @@ "packages/ui/src/v2/components/text-shimmer-v2.tsx" ], "exceptions": [], - "extracted_at": "2026-10-12T14:32:31.142Z", + "extracted_at": "2026-10-12T20:07:24.375Z", "per_package": { "packages/app": { "A": 237, @@ -960,8 +964,8 @@ "D": 0 }, "packages/opencode": { - "A": 75, - "M": 89, + "A": 77, + "M": 91, "D": 0 }, "packages/schema": { @@ -991,7 +995,7 @@ } }, "counts": { - "overlay_total": 920, + "overlay_total": 924, "deletions": 7, "server_coupled": 0 }, @@ -1304,6 +1308,8 @@ "packages/llm/src/protocols/bedrock-converse.ts": "M", "packages/llm/test/provider/bedrock-converse.test.ts": "M", "packages/opencode/src/agent/agent.ts": "M", + "packages/opencode/src/bus/global-replay.ts": "A", + "packages/opencode/src/bus/global.ts": "M", "packages/opencode/src/effect/runner.ts": "M", "packages/opencode/src/mcp/browser.ts": "M", "packages/opencode/src/mcp/index.ts": "M", @@ -1378,6 +1384,7 @@ "packages/opencode/test/server/httpapi-compression.test.ts": "M", "packages/opencode/test/server/httpapi-file-write-gate.test.ts": "A", "packages/opencode/test/server/httpapi-file.test.ts": "M", + "packages/opencode/test/server/httpapi-global-event.test.ts": "A", "packages/opencode/test/server/httpapi-instance-route-auth.test.ts": "M", "packages/opencode/test/server/httpapi-provider.test.ts": "M", "packages/opencode/test/server/httpapi-session.test.ts": "M", @@ -1912,6 +1919,7 @@ "packages/opencode/src/server/routes/instance/httpapi/groups/session.ts": "M", "packages/opencode/src/server/routes/instance/httpapi/handlers/experimental.ts": "M", "packages/opencode/src/server/routes/instance/httpapi/handlers/file.ts": "M", + "packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts": "M", "packages/opencode/src/server/routes/instance/httpapi/handlers/mcp.ts": "M", "packages/opencode/src/server/routes/instance/httpapi/handlers/provider.ts": "M", "packages/opencode/src/server/routes/instance/httpapi/handlers/session.ts": "M", diff --git a/packages/app-bundle/overlay/packages/opencode/src/bus/global-replay.ts b/packages/app-bundle/overlay/packages/opencode/src/bus/global-replay.ts new file mode 100644 index 000000000..84f1f4b89 --- /dev/null +++ b/packages/app-bundle/overlay/packages/opencode/src/bus/global-replay.ts @@ -0,0 +1,75 @@ +// The global event stream's shared frame source (#1776). One tap on the global +// bus serializes each event once, at emit, into a frame stamped with its +// `:` id, and fans it out to every live subscription. +import { GlobalBus } from "./global" + +export type Frame = { + /** The SSE id: `:`. */ + readonly id: string + readonly seq: number + /** The serialized global event — the SSE `data`. */ + readonly data: string +} + +export function formatID(epoch: string, seq: number) { + return `${epoch}:${seq}` +} + +export type Subscription = { + /** The connection's starting position: the sequence its first live frame follows. */ + readonly position: number + /** Begin delivering live frames. */ + start(deliver: (frame: Frame) => void): void + /** Unregister. Idempotent. */ + close(): void +} + +export class FrameSource { + readonly #listeners = new Set<(frame: Frame) => void>() + + constructor(readonly epoch: string) {} + + /** Live subscriptions currently registered. */ + get subscribers() { + return this.#listeners.size + } + + push(seq: number, data: string): Frame { + const frame: Frame = { id: formatID(this.epoch, seq), seq, data } + for (const listener of this.#listeners) listener(frame) + return frame + } + + subscribe(): Subscription { + let deliver: ((frame: Frame) => void) | undefined + const pending: Frame[] = [] + const listener = (frame: Frame) => (deliver ? deliver(frame) : pending.push(frame)) + this.#listeners.add(listener) + return { + position: GlobalBus.seq, + start: (fn) => { + for (const frame of pending.splice(0)) fn(frame) + deliver = fn + }, + close: () => { + this.#listeners.delete(listener) + pending.length = 0 + }, + } + } +} + +export const GlobalEventFrames = new FrameSource(GlobalBus.epoch) + +GlobalBus.tap((event, seq) => { + let data: string + try { + data = JSON.stringify(event) + } catch (error) { + // An unserializable event cannot be framed; its sequence becomes a hole the + // client detects (heartbeats carry the bus sequence), never a dead stream. + console.error("global event not serializable", { seq, type: event.payload?.type, error }) + return + } + GlobalEventFrames.push(seq, data) +}) diff --git a/packages/app-bundle/overlay/packages/opencode/src/bus/global.ts b/packages/app-bundle/overlay/packages/opencode/src/bus/global.ts new file mode 100644 index 000000000..2e6ee4e20 --- /dev/null +++ b/packages/app-bundle/overlay/packages/opencode/src/bus/global.ts @@ -0,0 +1,76 @@ +import { EventEmitter } from "events" +import { randomBytes } from "crypto" +import { Identifier } from "@/id/id" + +export type GlobalEvent = { + directory?: string + project?: string + workspace?: string + payload: any +} + +/** + * A pre-dispatch observer. Taps see every emitted event with its sequence + * BEFORE any ordinary listener runs, so they observe events in strict + * sequence order even when a listener emits from inside its handler (a nested + * emit is sequenced and tapped only after the outer event was). A tap must not + * throw and must not emit. + */ +export type GlobalBusTap = (event: GlobalEvent, seq: number) => void + +// How many recent source-event ids are remembered so a sync copy (which the +// event-v2 bridge emits right after its source) can share the source's +// sequence. The copy follows its source immediately; the window only has to +// absorb events a listener emits in between. +const MIRROR_WINDOW = 256 + +class GlobalBusEmitter extends EventEmitter<{ + event: [GlobalEvent] +}> { + /** Random per-process token; `:` is a global-stream frame id (#1776). */ + readonly epoch = randomBytes(6).toString("base64url") + #seq = 0 + #mirrors = new Map() + #taps = new Set() + + /** The current sequence: the one assigned to the last emitted event (0 before any). */ + get seq() { + return this.#seq + } + + /** Register a pre-dispatch tap; returns its unregister function. */ + tap(fn: GlobalBusTap): () => void { + this.#taps.add(fn) + return () => { + this.#taps.delete(fn) + } + } + + override emit(eventName: "event", event: GlobalEvent): boolean { + if (event.payload && typeof event.payload === "object" && !("id" in event.payload)) { + event.payload.id = event.payload.syncEvent?.id ?? Identifier.create("evt", "ascending") + } + const seq = this.#sequence(event.payload) + for (const tap of this.#taps) tap(event, seq) + return super.emit(eventName, event) + } + + // One strictly increasing counter, assigned once per event at emit and shared + // by every subscriber. A sync copy is not a new event: it carries the + // sequence of the event it mirrors, so filtering sync copies leaves no holes. + #sequence(payload: any): number { + if (payload?.type === "sync") { + const source = typeof payload.syncEvent?.id === "string" ? this.#mirrors.get(payload.syncEvent.id) : undefined + if (source !== undefined) return source + return ++this.#seq + } + const seq = ++this.#seq + if (typeof payload?.id === "string") { + this.#mirrors.set(payload.id, seq) + if (this.#mirrors.size > MIRROR_WINDOW) this.#mirrors.delete(this.#mirrors.keys().next().value!) + } + return seq + } +} + +export const GlobalBus = new GlobalBusEmitter() diff --git a/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts b/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts new file mode 100644 index 000000000..a724a2314 --- /dev/null +++ b/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts @@ -0,0 +1,141 @@ +import { Config } from "@/config/config" +import { GlobalBus } from "@/bus/global" +import { GlobalEventFrames, formatID, type Frame } from "@/bus/global-replay" +import { EffectBridge } from "@/effect/bridge" +import { EventV2 } from "@opencode-ai/core/event" +import { Installation } from "@/installation" +import { disposeAllInstancesAndEmitGlobalDisposed } from "@/server/global-lifecycle" +import { InstallationVersion } from "@opencode-ai/core/installation/version" +import { Effect, Queue } from "effect" +import * as Stream from "effect/Stream" +import { HttpServerResponse } from "effect/unstable/http" +import { HttpApiBuilder } from "effect/unstable/httpapi" +import * as Sse from "effect/unstable/encoding/Sse" +import { RootHttpApi } from "../api" +import { GlobalUpgradeInput } from "../groups/global" + +function frameEvent(frame: Pick): Sse.Event { + return { _tag: "Event", event: "message", id: frame.id, data: frame.data } +} + +// A per-connection control frame (connected, heartbeat). It never consumes a +// sequence — that would punch a hole in every other subscriber's stream — and +// carries the connection's current position as its id instead. +function controlEvent(seq: number, type: string, properties: Record = {}): Sse.Event { + return frameEvent({ + id: formatID(GlobalBus.epoch, seq), + data: JSON.stringify({ payload: { id: EventV2.ID.create(), type, properties } }), + }) +} + +function eventResponse() { + return Effect.gen(function* () { + // Registration is eager and synchronous: a bus event emitted while the + // response body is starting is queued, never lost. + const queue = yield* Queue.unbounded() + const subscription = yield* Effect.acquireRelease( + Effect.sync(() => { + const subscription = GlobalEventFrames.subscribe() + Queue.offerUnsafe(queue, controlEvent(subscription.position, "server.connected")) + subscription.start((frame) => Queue.offerUnsafe(queue, frameEvent(frame))) + return subscription + }), + (subscription) => Effect.sync(() => subscription.close()), + ) + yield* Effect.logInfo("global event connected") + const heartbeat = Stream.tick("10 seconds").pipe( + Stream.drop(1), + Stream.map(() => controlEvent(GlobalBus.seq, "server.heartbeat")), + ) + + return HttpServerResponse.stream( + Stream.fromQueue(queue).pipe( + Stream.merge(heartbeat, { haltStrategy: "left" }), + Stream.pipeThroughChannel(Sse.encode()), + Stream.encodeText, + Stream.ensuring( + Effect.suspend(() => { + subscription.close() + return Effect.logInfo("global event disconnected") + }), + ), + ), + { + contentType: "text/event-stream", + headers: { + "Cache-Control": "no-cache, no-transform", + "X-Accel-Buffering": "no", + "X-Content-Type-Options": "nosniff", + }, + }, + ) + }) +} + +export const globalHandlers = HttpApiBuilder.group(RootHttpApi, "global", (handlers) => + Effect.gen(function* () { + const config = yield* Config.Service + const installation = yield* Installation.Service + const bridge = yield* EffectBridge.make() + + const health = Effect.fn("GlobalHttpApi.health")(function* () { + return { healthy: true as const, version: InstallationVersion } + }) + + const event = Effect.fn("GlobalHttpApi.event")(function* () { + return yield* eventResponse() + }) + + const configGet = Effect.fn("GlobalHttpApi.configGet")(function* () { + return yield* config.getGlobal() + }) + + const configUpdate = Effect.fn("GlobalHttpApi.configUpdate")(function* (ctx) { + const result = yield* config.updateGlobal(ctx.payload) + if (result.changed) bridge.fork(disposeAllInstancesAndEmitGlobalDisposed({ swallowErrors: true })) + return result.info + }) + + const dispose = Effect.fn("GlobalHttpApi.dispose")(function* () { + yield* disposeAllInstancesAndEmitGlobalDisposed() + return true + }) + + const upgrade = Effect.fn("GlobalHttpApi.upgrade")(function* (ctx: { payload: typeof GlobalUpgradeInput.Type }) { + const method = yield* installation.method() + if (method === "unknown") { + return HttpServerResponse.jsonUnsafe( + { success: false as const, error: "Unknown installation method" }, + { status: 400 }, + ) + } + const target = ctx.payload.target + const result = yield* installation.upgrade(method, target).pipe( + Effect.as({ success: true as const, version: target }), + Effect.catch((err) => + Effect.succeed({ + success: false as const, + error: err instanceof Error ? err.message : String(err), + }), + ), + ) + if (!result.success) return HttpServerResponse.jsonUnsafe(result, { status: 500 }) + GlobalBus.emit("event", { + directory: "global", + payload: { + type: Installation.Event.Updated.type, + properties: { version: target }, + }, + }) + return HttpServerResponse.jsonUnsafe(result) + }) + + return handlers + .handle("health", health) + .handleRaw("event", event) + .handle("configGet", configGet) + .handle("configUpdate", configUpdate) + .handle("dispose", dispose) + .handle("upgrade", upgrade) + }), +) diff --git a/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts b/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts new file mode 100644 index 000000000..00c1b2946 --- /dev/null +++ b/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts @@ -0,0 +1,147 @@ +// #1776 — the global event stream is sequenced and replayable: every frame +// carries an `:` SSE id, a resume cursor replays the gap, an +// unservable cursor gets an explicit reset, heartbeats carry time + seq, and a +// client abort releases its subscription. Exercised through a raw node:http +// client against the in-process server (so headers and aborts are exact), on +// the same Bun node:http shim the compiled binary runs on. +import { NodeHttpServer } from "@effect/platform-node" +import { describe, expect } from "bun:test" +import http from "node:http" +import { Context, Effect, Layer, Option, Queue } from "effect" +import { HttpRouter, HttpServer } from "effect/unstable/http" +import { HttpApiBuilder } from "effect/unstable/httpapi" +import { Auth } from "../../src/auth" +import { GlobalBus } from "../../src/bus/global" +import { Config } from "../../src/config/config" +import { Installation } from "../../src/installation" +import { MoveSession } from "@opencode-ai/core/control-plane/move-session" +import { ServerAuth } from "../../src/server/auth" +import { RootHttpApi } from "../../src/server/routes/instance/httpapi/api" +import { GlobalPaths } from "../../src/server/routes/instance/httpapi/groups/global" +import { controlHandlers } from "../../src/server/routes/instance/httpapi/handlers/control" +import { controlPlaneHandlers } from "../../src/server/routes/instance/httpapi/handlers/control-plane" +import { globalHandlers } from "../../src/server/routes/instance/httpapi/handlers/global" +import { authorizationLayer } from "../../src/server/routes/instance/httpapi/middleware/authorization" +import { schemaErrorLayer } from "../../src/server/routes/instance/httpapi/middleware/schema-error" +import { testEffect } from "../lib/effect" + +const apiLayer = HttpRouter.serve( + HttpApiBuilder.layer(RootHttpApi).pipe( + Layer.provide([controlHandlers, controlPlaneHandlers, globalHandlers]), + Layer.provide([authorizationLayer, schemaErrorLayer]), + // Raw HttpApi routes expose an opaque handler context at the request boundary. + // oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion + HttpRouter.provideRequest(Layer.succeedContext(Context.empty() as Context.Context)), + ), + { disableListenLog: true, disableLogger: true }, +).pipe( + Layer.provideMerge(NodeHttpServer.layerTest), + Layer.provide(Layer.mock(Auth.Service)({})), + Layer.provide(Layer.mock(Config.Service)({})), + Layer.provide(Layer.mock(MoveSession.Service)({})), + Layer.provide(Layer.mock(Installation.Service)({})), + Layer.provide(ServerAuth.Config.configLayer({ password: Option.none(), username: "opencode" })), +) +const it = testEffect(apiLayer) + +const ID = /^([A-Za-z0-9_-]+):(\d+)$/ + +type Frame = { id: string | undefined; event: string | undefined; data: any } + +/** Split an SSE id into its epoch and sequence (fails the test if malformed). */ +function idOf(frame: Frame) { + const match = ID.exec(frame.id ?? "") + if (!match) throw new Error(`frame id ${JSON.stringify(frame.id)} is not : (${JSON.stringify(frame.data)})`) + return { epoch: match[1], seq: Number(match[2]) } +} + +function parseFrame(block: string): Frame { + const frame: Frame = { id: undefined, event: undefined, data: undefined } + const data: string[] = [] + for (const line of block.split("\n")) { + if (line.startsWith("id: ")) frame.id = line.slice(4) + else if (line.startsWith("event: ")) frame.event = line.slice(7) + else if (line.startsWith("data: ")) data.push(line.slice(6)) + } + frame.data = data.length ? JSON.parse(data.join("\n")) : undefined + return frame +} + +/** Open a raw SSE connection to the in-process server's global stream. */ +const connect = (input: { headers?: Record; query?: string } = {}) => + Effect.gen(function* () { + const server = yield* HttpServer.HttpServer + const address = server.address + if (address._tag !== "TcpAddress") throw new Error("expected a TCP test server") + const frames = yield* Queue.unbounded() + const request = yield* Effect.callback((resume) => { + const req = http.get( + { + host: "127.0.0.1", + port: address.port, + path: GlobalPaths.event + (input.query ?? ""), + headers: { accept: "text/event-stream", ...input.headers }, + }, + (res) => { + let buffer = "" + res.setEncoding("utf8") + res.on("data", (chunk: string) => { + buffer += chunk + let end = buffer.indexOf("\n\n") + while (end !== -1) { + const block = buffer.slice(0, end) + buffer = buffer.slice(end + 2) + if (block.trim() !== "") Queue.offerUnsafe(frames, parseFrame(block)) + end = buffer.indexOf("\n\n") + } + }) + res.on("error", () => {}) + resume(Effect.succeed(req)) + }, + ) + req.on("error", (error) => resume(Effect.fail(error))) + }) + yield* Effect.addFinalizer(() => Effect.sync(() => request.destroy())) + const next = Queue.take(frames).pipe( + Effect.timeoutOrElse({ duration: "5 seconds", orElse: () => Effect.fail(new Error("timed out waiting for frame")) }), + ) + const take = (n: number) => Effect.forEach(Array.from({ length: n }), () => next) + return { next, take, abort: () => request.destroy() } + }) + +const emit = (type: string, properties: Record = {}) => + GlobalBus.emit("event", { directory: "/tmp/i1776", payload: { type, properties } }) + +describe("global event stream — sequenced ids (#1776)", () => { + it.live("every frame carries an : id shared by all subscribers; a sync copy carries its source's id", () => + Effect.gen(function* () { + const a = yield* connect() + const b = yield* connect() + const connectedA = yield* a.next + const connectedB = yield* b.next + expect(connectedA.data.payload.type).toBe("server.connected") + const start = idOf(connectedA) + expect(idOf(connectedB)).toEqual(start) + + emit("test.first") + // the event-v2 bridge's twin emission: the durable event, then its sync copy + GlobalBus.emit("event", { directory: "/tmp/i1776", payload: { id: "evt_i1776_durable", type: "test.durable", properties: {} } }) + GlobalBus.emit("event", { + directory: "/tmp/i1776", + payload: { + type: "sync", + syncEvent: { id: "evt_i1776_durable", type: "test.durable.1", seq: 1, aggregateID: "agg", data: {} }, + }, + }) + emit("test.last") + + const framesA = yield* a.take(4) + const framesB = yield* b.take(4) + expect(framesA.map((f) => f.data.payload.type)).toEqual(["test.first", "test.durable", "sync", "test.last"]) + expect(framesA.map((f) => f.id)).toEqual(framesB.map((f) => f.id)) + const seqs = framesA.map((f) => idOf(f).seq) + expect(framesA.every((f) => idOf(f).epoch === start.epoch)).toBe(true) + expect(seqs).toEqual([start.seq + 1, start.seq + 2, start.seq + 2, start.seq + 3]) + }), + ) +}) From a4cb7202cf6c57e03377d68b7a53d09a3e411e1a Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 16:09:26 -0400 Subject: [PATCH 08/60] feat(engine): resume the global stream from a cursor via a shared replay buffer (#1776) The stream accepts the cursor from the Last-Event-ID header or the existing lastEventID query parameter and replays the frames after it before going live. The live listener is registered before the buffer is read and frames already covered by the replay are dropped, so events published while the reconnect is in flight are neither missed nor duplicated. --- packages/app-bundle/manifest.json | 8 +-- .../opencode/src/bus/global-replay.ts | 59 ++++++++++++++----- .../instance/httpapi/handlers/global.ts | 18 +++++- .../test/server/httpapi-global-event.test.ts | 45 +++++++++++++- 4 files changed, 108 insertions(+), 22 deletions(-) diff --git a/packages/app-bundle/manifest.json b/packages/app-bundle/manifest.json index cc18daf60..efaae606f 100644 --- a/packages/app-bundle/manifest.json +++ b/packages/app-bundle/manifest.json @@ -316,7 +316,7 @@ "packages/llm/src/protocols/bedrock-converse.ts": "834a415fa8ddf9ad1224133f8352d0a926694f19f0954201d182111c171ec03b", "packages/llm/test/provider/bedrock-converse.test.ts": "e8e14d60e8502e928f170f8ac5b406c44bf16d2dd34996ddc2fb7b9261decf85", "packages/opencode/src/agent/agent.ts": "512372a4c62997e9565266638ada19b9dc0a3685b0c0483fd28fede2abdb5c9b", - "packages/opencode/src/bus/global-replay.ts": "bc5d4e77c2aa8ef0444177e6174af0642428d1bcdf47fc668a4799d57163597e", + "packages/opencode/src/bus/global-replay.ts": "d0793e1d7ed161bc2844fede247bc42a6637be40ba1ef468260a256916299eae", "packages/opencode/src/bus/global.ts": "65b3b380936fc34b2c5ef9f3ab419a6c8e6d4cfc696302a266188f26ccdc320a", "packages/opencode/src/effect/runner.ts": "b660b5b22df810b16293c23c6d832e5e8dcfb12b9a8a858c875d2c051c855b81", "packages/opencode/src/mcp/browser.ts": "ecdda42cbd55a97b8d14050aa15c6e6450a1d63b0a641c8c8e5b81e50f957c87", @@ -392,7 +392,7 @@ "packages/opencode/test/server/httpapi-compression.test.ts": "5f7603d7018e0b95f1c2f36c01dd297c01770bfcdd91ce29778e03b7d6143f75", "packages/opencode/test/server/httpapi-file-write-gate.test.ts": "0c91b7417e615850edfde3f2878207dab6b6624d4cfbb6e6783b2e6d9aa4dff0", "packages/opencode/test/server/httpapi-file.test.ts": "d3386129bc81467ecbcc57a3861f53767ee56ac41b4ff6ad58d8b168fb3b3373", - "packages/opencode/test/server/httpapi-global-event.test.ts": "eaecf3b6e0a148f48e9a056c1b24184cda8f730aeec84aed2d7b89a50548d637", + "packages/opencode/test/server/httpapi-global-event.test.ts": "cc319e7593820b6eb41b838cc6615fad8af607848dcb4955ca2ba4443cfc5401", "packages/opencode/test/server/httpapi-instance-route-auth.test.ts": "4b73a89d2431c02bc3b665d93f9002914e51eb1add979dc1867efbcbc0c36e25", "packages/opencode/test/server/httpapi-provider.test.ts": "9e8b3e085b478e3f99b689ffff600e10c0b4b50b3cabdbd749f88dae19bf5904", "packages/opencode/test/server/httpapi-session.test.ts": "eda6fd763ac4c17891cedaa4ccd49597c03e7227fcfdcf0f94a1b4545ba623ed", @@ -927,7 +927,7 @@ "packages/opencode/src/server/routes/instance/httpapi/groups/session.ts": "1421f8c07948b121c7e2005bd9b0fcd6b2c456565ddd1da32d8e6c35d03fac9c", "packages/opencode/src/server/routes/instance/httpapi/handlers/experimental.ts": "69ca3dd48d95157fc822e1f76eb42ac873df50f22616b85194619ca658613867", "packages/opencode/src/server/routes/instance/httpapi/handlers/file.ts": "10bcb79b117beebee04184fb48c1ad789e782e2f7ae4b0c68603edc00f03aee9", - "packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts": "efbbbcebff05007137bbab19486fa6bd4602f84f11f463b4d56dd0da5cb76de7", + "packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts": "b0abcc243fe79e2e41e88e7d8a40a1f18cffb068945d25f354b0cb20dac3d23e", "packages/opencode/src/server/routes/instance/httpapi/handlers/mcp.ts": "7da08d58765cdf2b6ce4648fc70de371402949f5bb7b75696f674d43f2d5365b", "packages/opencode/src/server/routes/instance/httpapi/handlers/provider.ts": "8baf16cad0992c5351b61b0c0fb27903bb25b17e1b25fc7c5d77f807c24413a6", "packages/opencode/src/server/routes/instance/httpapi/handlers/session.ts": "b37ff4136c4aea85b23083bf441f03e09820e3e4cdd95e4118ca570d8435b94f", @@ -946,7 +946,7 @@ "packages/ui/src/v2/components/text-shimmer-v2.tsx" ], "exceptions": [], - "extracted_at": "2026-10-12T20:07:24.375Z", + "extracted_at": "2026-10-12T20:09:26.217Z", "per_package": { "packages/app": { "A": 237, diff --git a/packages/app-bundle/overlay/packages/opencode/src/bus/global-replay.ts b/packages/app-bundle/overlay/packages/opencode/src/bus/global-replay.ts index 84f1f4b89..110b3b987 100644 --- a/packages/app-bundle/overlay/packages/opencode/src/bus/global-replay.ts +++ b/packages/app-bundle/overlay/packages/opencode/src/bus/global-replay.ts @@ -1,6 +1,8 @@ -// The global event stream's shared frame source (#1776). One tap on the global -// bus serializes each event once, at emit, into a frame stamped with its -// `:` id, and fans it out to every live subscription. +// The global event stream's replay buffer (#1776). One tap on the global bus +// serializes each event once, at emit, into a frame stamped with its +// `:` id; the buffer keeps recent frames (shared by every +// subscriber) and fans each one out to the live subscriptions. A connection +// that resumes from a cursor gets the frames after it, then live frames. import { GlobalBus } from "./global" export type Frame = { @@ -9,23 +11,36 @@ export type Frame = { readonly seq: number /** The serialized global event — the SSE `data`. */ readonly data: string + /** Push order; unlike `seq` it is unique per frame (a sync copy shares its source's seq). */ + readonly ordinal: number } export function formatID(epoch: string, seq: number) { return `${epoch}:${seq}` } +/** Parse an `:` cursor; `undefined` when absent or malformed. */ +export function parseCursor(raw: string | null | undefined): { epoch: string; seq: number } | undefined { + const match = /^([A-Za-z0-9_-]+):(\d+)$/.exec(raw?.trim() ?? "") + if (!match) return undefined + return { epoch: match[1], seq: Number(match[2]) } +} + export type Subscription = { - /** The connection's starting position: the sequence its first live frame follows. */ + /** The connection's starting position: the sequence its first frame follows. */ readonly position: number - /** Begin delivering live frames. */ + /** Frames after the cursor, in order — delivered before any live frame. */ + readonly replay: readonly Frame[] + /** Begin delivering live frames (any that arrived since subscribing first, minus duplicates of `replay`). */ start(deliver: (frame: Frame) => void): void /** Unregister. Idempotent. */ close(): void } -export class FrameSource { +export class ReplayBuffer { readonly #listeners = new Set<(frame: Frame) => void>() + readonly #frames: Frame[] = [] + #ordinal = 0 constructor(readonly epoch: string) {} @@ -35,31 +50,47 @@ export class FrameSource { } push(seq: number, data: string): Frame { - const frame: Frame = { id: formatID(this.epoch, seq), seq, data } + const frame: Frame = { id: formatID(this.epoch, seq), seq, data, ordinal: ++this.#ordinal } + this.#frames.push(frame) for (const listener of this.#listeners) listener(frame) return frame } - subscribe(): Subscription { + /** + * Subscribe, optionally resuming after `cursor`. The live listener is + * registered BEFORE the buffer is read, and live frames already covered by + * the replay are dropped, so nothing published around the reconnect is + * missed or delivered twice. + */ + subscribe(cursor?: string | null): Subscription { let deliver: ((frame: Frame) => void) | undefined - const pending: Frame[] = [] + let pending: Frame[] = [] const listener = (frame: Frame) => (deliver ? deliver(frame) : pending.push(frame)) this.#listeners.add(listener) + + const watermark = this.#ordinal + const parsed = parseCursor(cursor) + const resume = parsed && parsed.epoch === this.epoch ? parsed.seq : undefined + const replay = resume === undefined ? [] : this.#frames.filter((frame) => frame.seq > resume) + return { - position: GlobalBus.seq, + position: resume ?? GlobalBus.seq, + replay, start: (fn) => { - for (const frame of pending.splice(0)) fn(frame) + const queued = pending + pending = [] + for (const frame of queued) if (frame.ordinal > watermark) fn(frame) deliver = fn }, close: () => { this.#listeners.delete(listener) - pending.length = 0 + pending = [] }, } } } -export const GlobalEventFrames = new FrameSource(GlobalBus.epoch) +export const GlobalEventReplay = new ReplayBuffer(GlobalBus.epoch) GlobalBus.tap((event, seq) => { let data: string @@ -71,5 +102,5 @@ GlobalBus.tap((event, seq) => { console.error("global event not serializable", { seq, type: event.payload?.type, error }) return } - GlobalEventFrames.push(seq, data) + GlobalEventReplay.push(seq, data) }) diff --git a/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts b/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts index a724a2314..d18808420 100644 --- a/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts +++ b/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts @@ -1,6 +1,6 @@ import { Config } from "@/config/config" import { GlobalBus } from "@/bus/global" -import { GlobalEventFrames, formatID, type Frame } from "@/bus/global-replay" +import { GlobalEventReplay, formatID, type Frame } from "@/bus/global-replay" import { EffectBridge } from "@/effect/bridge" import { EventV2 } from "@opencode-ai/core/event" import { Installation } from "@/installation" @@ -8,12 +8,21 @@ import { disposeAllInstancesAndEmitGlobalDisposed } from "@/server/global-lifecy import { InstallationVersion } from "@opencode-ai/core/installation/version" import { Effect, Queue } from "effect" import * as Stream from "effect/Stream" -import { HttpServerResponse } from "effect/unstable/http" +import { HttpServerRequest, HttpServerResponse } from "effect/unstable/http" import { HttpApiBuilder } from "effect/unstable/httpapi" import * as Sse from "effect/unstable/encoding/Sse" import { RootHttpApi } from "../api" import { GlobalUpgradeInput } from "../groups/global" +// The resume cursor: the standard `Last-Event-ID` header (what an SSE client +// sends once it has seen an `id:` line), else the `lastEventID` query +// parameter clients already send. +function cursorOf(request: HttpServerRequest.HttpServerRequest) { + const header = request.headers["last-event-id"]?.trim() + if (header) return header + return new URL(request.url, "http://localhost").searchParams.get("lastEventID")?.trim() || undefined +} + function frameEvent(frame: Pick): Sse.Event { return { _tag: "Event", event: "message", id: frame.id, data: frame.data } } @@ -30,13 +39,16 @@ function controlEvent(seq: number, type: string, properties: Record() const subscription = yield* Effect.acquireRelease( Effect.sync(() => { - const subscription = GlobalEventFrames.subscribe() + const subscription = GlobalEventReplay.subscribe(cursor) Queue.offerUnsafe(queue, controlEvent(subscription.position, "server.connected")) + for (const frame of subscription.replay) Queue.offerUnsafe(queue, frameEvent(frame)) subscription.start((frame) => Queue.offerUnsafe(queue, frameEvent(frame))) return subscription }), diff --git a/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts b/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts index 00c1b2946..b007890ef 100644 --- a/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts +++ b/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts @@ -7,7 +7,7 @@ import { NodeHttpServer } from "@effect/platform-node" import { describe, expect } from "bun:test" import http from "node:http" -import { Context, Effect, Layer, Option, Queue } from "effect" +import { Context, Effect, Fiber, Layer, Option, Queue } from "effect" import { HttpRouter, HttpServer } from "effect/unstable/http" import { HttpApiBuilder } from "effect/unstable/httpapi" import { Auth } from "../../src/auth" @@ -145,3 +145,46 @@ describe("global event stream — sequenced ids (#1776)", () => { }), ) }) + +describe("global event stream — resume from a cursor (#1776)", () => { + for (const carrier of ["Last-Event-ID header", "lastEventID query"] as const) { + it.live(`replays exactly the frames after a ${carrier} cursor, then continues live`, () => + Effect.gen(function* () { + const a = yield* connect() + yield* a.next + emit("test.one") + emit("test.two") + emit("test.three") + const [one] = yield* a.take(3) + const cursor = idOf(one) + a.abort() + + emit("test.gap") + const resume = + carrier === "Last-Event-ID header" ? { headers: { "last-event-id": one.id! } } : { query: `?lastEventID=${encodeURIComponent(one.id!)}` } + // events keep arriving while the reconnect is in flight + const reconnect = yield* Effect.forkChild(connect(resume)) + for (let i = 0; i < 20; i++) { + emit("test.inflight", { i }) + yield* Effect.sleep("1 millis") + } + const b = yield* Fiber.join(reconnect) + emit("test.after") + + const connected = yield* b.next + expect(connected.data.payload.type).toBe("server.connected") + expect(idOf(connected)).toEqual(cursor) + const frames = yield* b.take(2 + 1 + 20 + 1) + const types = frames.map((f) => f.data.payload.type) + expect(types).toEqual(["test.two", "test.three", "test.gap", ...Array(20).fill("test.inflight"), "test.after"]) + expect(frames.map((f) => idOf(f).seq)).toEqual(Array.from({ length: 24 }, (_, i) => cursor.seq + 1 + i)) + expect(frames.filter((f) => f.data.payload.type === "test.inflight").map((f) => f.data.payload.properties.i)).toEqual( + Array.from({ length: 20 }, (_, i) => i), + ) + // nothing further: no duplicate trails the live frames + const extra = yield* b.next.pipe(Effect.timeoutOrElse({ duration: "200 millis", orElse: () => Effect.succeed(undefined) })) + expect(extra).toBeUndefined() + }), + ) + } +}) From a13c802fd4dc0bacf28d5ec211aa20d4d707180b Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 16:12:31 -0400 Subject: [PATCH 09/60] feat(engine): answer an unservable global-stream cursor with a reset frame (#1776) A cursor from another epoch, a malformed one, one ahead of the stream, or one older than the buffer's oldest retained frame gets a server.cursor.unservable frame first (properties: the current epoch and sequence), then the connected frame and live frames. The shared buffer evicts oldest-first past its byte bound (default 32 MiB) and tracks the floor below which cursors are stale. --- packages/app-bundle/manifest.json | 8 +- .../opencode/src/bus/global-replay.ts | 113 +++++++++++++++--- .../instance/httpapi/handlers/global.ts | 10 ++ .../test/server/httpapi-global-event.test.ts | 43 ++++++- 4 files changed, 153 insertions(+), 21 deletions(-) diff --git a/packages/app-bundle/manifest.json b/packages/app-bundle/manifest.json index efaae606f..e57b5562a 100644 --- a/packages/app-bundle/manifest.json +++ b/packages/app-bundle/manifest.json @@ -316,7 +316,7 @@ "packages/llm/src/protocols/bedrock-converse.ts": "834a415fa8ddf9ad1224133f8352d0a926694f19f0954201d182111c171ec03b", "packages/llm/test/provider/bedrock-converse.test.ts": "e8e14d60e8502e928f170f8ac5b406c44bf16d2dd34996ddc2fb7b9261decf85", "packages/opencode/src/agent/agent.ts": "512372a4c62997e9565266638ada19b9dc0a3685b0c0483fd28fede2abdb5c9b", - "packages/opencode/src/bus/global-replay.ts": "d0793e1d7ed161bc2844fede247bc42a6637be40ba1ef468260a256916299eae", + "packages/opencode/src/bus/global-replay.ts": "6bd6b68baf46e0570be268bb99989e8b3f041a584720aac0c917be2e887e2bec", "packages/opencode/src/bus/global.ts": "65b3b380936fc34b2c5ef9f3ab419a6c8e6d4cfc696302a266188f26ccdc320a", "packages/opencode/src/effect/runner.ts": "b660b5b22df810b16293c23c6d832e5e8dcfb12b9a8a858c875d2c051c855b81", "packages/opencode/src/mcp/browser.ts": "ecdda42cbd55a97b8d14050aa15c6e6450a1d63b0a641c8c8e5b81e50f957c87", @@ -392,7 +392,7 @@ "packages/opencode/test/server/httpapi-compression.test.ts": "5f7603d7018e0b95f1c2f36c01dd297c01770bfcdd91ce29778e03b7d6143f75", "packages/opencode/test/server/httpapi-file-write-gate.test.ts": "0c91b7417e615850edfde3f2878207dab6b6624d4cfbb6e6783b2e6d9aa4dff0", "packages/opencode/test/server/httpapi-file.test.ts": "d3386129bc81467ecbcc57a3861f53767ee56ac41b4ff6ad58d8b168fb3b3373", - "packages/opencode/test/server/httpapi-global-event.test.ts": "cc319e7593820b6eb41b838cc6615fad8af607848dcb4955ca2ba4443cfc5401", + "packages/opencode/test/server/httpapi-global-event.test.ts": "92e3cba9fb54d16e229ced46c1bd23ee8f1a5ef24b1134baf77a6e414230c2a5", "packages/opencode/test/server/httpapi-instance-route-auth.test.ts": "4b73a89d2431c02bc3b665d93f9002914e51eb1add979dc1867efbcbc0c36e25", "packages/opencode/test/server/httpapi-provider.test.ts": "9e8b3e085b478e3f99b689ffff600e10c0b4b50b3cabdbd749f88dae19bf5904", "packages/opencode/test/server/httpapi-session.test.ts": "eda6fd763ac4c17891cedaa4ccd49597c03e7227fcfdcf0f94a1b4545ba623ed", @@ -927,7 +927,7 @@ "packages/opencode/src/server/routes/instance/httpapi/groups/session.ts": "1421f8c07948b121c7e2005bd9b0fcd6b2c456565ddd1da32d8e6c35d03fac9c", "packages/opencode/src/server/routes/instance/httpapi/handlers/experimental.ts": "69ca3dd48d95157fc822e1f76eb42ac873df50f22616b85194619ca658613867", "packages/opencode/src/server/routes/instance/httpapi/handlers/file.ts": "10bcb79b117beebee04184fb48c1ad789e782e2f7ae4b0c68603edc00f03aee9", - "packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts": "b0abcc243fe79e2e41e88e7d8a40a1f18cffb068945d25f354b0cb20dac3d23e", + "packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts": "809ba02a180d2450a0f5928af83960a4fd3cd13e914b2606dcbcdbc601e2e201", "packages/opencode/src/server/routes/instance/httpapi/handlers/mcp.ts": "7da08d58765cdf2b6ce4648fc70de371402949f5bb7b75696f674d43f2d5365b", "packages/opencode/src/server/routes/instance/httpapi/handlers/provider.ts": "8baf16cad0992c5351b61b0c0fb27903bb25b17e1b25fc7c5d77f807c24413a6", "packages/opencode/src/server/routes/instance/httpapi/handlers/session.ts": "b37ff4136c4aea85b23083bf441f03e09820e3e4cdd95e4118ca570d8435b94f", @@ -946,7 +946,7 @@ "packages/ui/src/v2/components/text-shimmer-v2.tsx" ], "exceptions": [], - "extracted_at": "2026-10-12T20:09:26.217Z", + "extracted_at": "2026-10-12T20:12:31.400Z", "per_package": { "packages/app": { "A": 237, diff --git a/packages/app-bundle/overlay/packages/opencode/src/bus/global-replay.ts b/packages/app-bundle/overlay/packages/opencode/src/bus/global-replay.ts index 110b3b987..889773fc0 100644 --- a/packages/app-bundle/overlay/packages/opencode/src/bus/global-replay.ts +++ b/packages/app-bundle/overlay/packages/opencode/src/bus/global-replay.ts @@ -1,8 +1,9 @@ // The global event stream's replay buffer (#1776). One tap on the global bus // serializes each event once, at emit, into a frame stamped with its -// `:` id; the buffer keeps recent frames (shared by every -// subscriber) and fans each one out to the live subscriptions. A connection -// that resumes from a cursor gets the frames after it, then live frames. +// `:` id; the buffer keeps recent frames (one buffer shared by +// every subscriber, bounded by bytes) and fans each frame out to the live +// subscriptions. A connection that resumes from a servable cursor gets the +// frames after it, then live frames; any other cursor gets an explicit reset. import { GlobalBus } from "./global" export type Frame = { @@ -11,24 +12,38 @@ export type Frame = { readonly seq: number /** The serialized global event — the SSE `data`. */ readonly data: string + /** UTF-8 size of `data`, charged against the byte bound. */ + readonly bytes: number /** Push order; unlike `seq` it is unique per frame (a sync copy shares its source's seq). */ readonly ordinal: number } +export type Bounds = { + /** Retained frame bytes; the oldest frames are evicted past it. */ + readonly maxBytes: number +} + +export const DEFAULT_BOUNDS: Bounds = { maxBytes: 32 * 1024 * 1024 } + export function formatID(epoch: string, seq: number) { return `${epoch}:${seq}` } -/** Parse an `:` cursor; `undefined` when absent or malformed. */ -export function parseCursor(raw: string | null | undefined): { epoch: string; seq: number } | undefined { - const match = /^([A-Za-z0-9_-]+):(\d+)$/.exec(raw?.trim() ?? "") +/** Parse an `:` cursor; `undefined` when malformed. */ +export function parseCursor(raw: string): { epoch: string; seq: number } | undefined { + const match = /^([A-Za-z0-9_-]+):(\d+)$/.exec(raw.trim()) if (!match) return undefined return { epoch: match[1], seq: Number(match[2]) } } +/** Why a cursor cannot be served. */ +export type ResetReason = "malformed" | "epoch" | "ahead" | "evicted" + export type Subscription = { /** The connection's starting position: the sequence its first frame follows. */ readonly position: number + /** Set when the cursor could not be served: the connection must start with a reset frame. */ + readonly reset: ResetReason | undefined /** Frames after the cursor, in order — delivered before any live frame. */ readonly replay: readonly Frame[] /** Begin delivering live frames (any that arrived since subscribing first, minus duplicates of `replay`). */ @@ -39,10 +54,30 @@ export type Subscription = { export class ReplayBuffer { readonly #listeners = new Set<(frame: Frame) => void>() - readonly #frames: Frame[] = [] + // Retained frames are #frames[#head..]; evicted slots before #head are + // compacted away in batches so eviction stays O(1) amortized. + #frames: (Frame | undefined)[] = [] + #head = 0 + #bytes = 0 #ordinal = 0 + #current: number + // The highest sequence whose frames are no longer all retained (evicted, or + // emitted before this buffer existed). A cursor below it cannot be served. + #floor: number + + constructor( + readonly epoch: string, + readonly bounds: Bounds = DEFAULT_BOUNDS, + start = 0, + ) { + this.#current = start + this.#floor = start + } - constructor(readonly epoch: string) {} + /** The current sequence: the last one pushed (or skipped). */ + get current() { + return this.#current + } /** Live subscriptions currently registered. */ get subscribers() { @@ -50,31 +85,44 @@ export class ReplayBuffer { } push(seq: number, data: string): Frame { - const frame: Frame = { id: formatID(this.epoch, seq), seq, data, ordinal: ++this.#ordinal } + const frame: Frame = { + id: formatID(this.epoch, seq), + seq, + data, + bytes: Buffer.byteLength(data), + ordinal: ++this.#ordinal, + } + this.#current = Math.max(this.#current, seq) this.#frames.push(frame) + this.#bytes += frame.bytes for (const listener of this.#listeners) listener(frame) + this.#evict() return frame } + /** Advance past a sequence that produced no frame (the hole stays visible to clients). */ + skip(seq: number) { + this.#current = Math.max(this.#current, seq) + } + /** * Subscribe, optionally resuming after `cursor`. The live listener is * registered BEFORE the buffer is read, and live frames already covered by * the replay are dropped, so nothing published around the reconnect is * missed or delivered twice. */ - subscribe(cursor?: string | null): Subscription { + subscribe(cursor?: string): Subscription { let deliver: ((frame: Frame) => void) | undefined let pending: Frame[] = [] const listener = (frame: Frame) => (deliver ? deliver(frame) : pending.push(frame)) this.#listeners.add(listener) const watermark = this.#ordinal - const parsed = parseCursor(cursor) - const resume = parsed && parsed.epoch === this.epoch ? parsed.seq : undefined - const replay = resume === undefined ? [] : this.#frames.filter((frame) => frame.seq > resume) + const resolved = this.#resolve(cursor) + const replay = resolved.reset ? [] : this.#retained().filter((frame) => frame.seq > resolved.position) return { - position: resume ?? GlobalBus.seq, + ...resolved, replay, start: (fn) => { const queued = pending @@ -88,9 +136,41 @@ export class ReplayBuffer { }, } } + + #resolve(cursor: string | undefined): { position: number; reset: ResetReason | undefined } { + if (cursor === undefined) return { position: this.#current, reset: undefined } + const parsed = parseCursor(cursor) + if (!parsed) return { position: this.#current, reset: "malformed" } + const reset: ResetReason | undefined = + parsed.epoch !== this.epoch + ? "epoch" + : parsed.seq > this.#current + ? "ahead" + : parsed.seq < this.#floor + ? "evicted" + : undefined + return reset ? { position: this.#current, reset } : { position: parsed.seq, reset: undefined } + } + + #retained(): Frame[] { + return this.#frames.slice(this.#head) as Frame[] + } + + #evict() { + while (this.#head < this.#frames.length && this.#bytes > this.bounds.maxBytes) { + const frame = this.#frames[this.#head]! + this.#frames[this.#head++] = undefined + this.#bytes -= frame.bytes + this.#floor = Math.max(this.#floor, frame.seq) + } + if (this.#head > 1024 && this.#head * 2 > this.#frames.length) { + this.#frames = this.#frames.slice(this.#head) + this.#head = 0 + } + } } -export const GlobalEventReplay = new ReplayBuffer(GlobalBus.epoch) +export const GlobalEventReplay = new ReplayBuffer(GlobalBus.epoch, DEFAULT_BOUNDS, GlobalBus.seq) GlobalBus.tap((event, seq) => { let data: string @@ -98,8 +178,9 @@ GlobalBus.tap((event, seq) => { data = JSON.stringify(event) } catch (error) { // An unserializable event cannot be framed; its sequence becomes a hole the - // client detects (heartbeats carry the bus sequence), never a dead stream. + // client detects (heartbeats carry the current sequence), never a dead stream. console.error("global event not serializable", { seq, type: event.payload?.type, error }) + GlobalEventReplay.skip(seq) return } GlobalEventReplay.push(seq, data) diff --git a/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts b/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts index d18808420..d5ed6f6b8 100644 --- a/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts +++ b/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts @@ -47,6 +47,16 @@ function eventResponse() { const subscription = yield* Effect.acquireRelease( Effect.sync(() => { const subscription = GlobalEventReplay.subscribe(cursor) + // An unservable cursor is never silent: a reset frame comes first, + // carrying the current epoch and sequence, and the client resyncs. + if (subscription.reset) + Queue.offerUnsafe( + queue, + controlEvent(subscription.position, "server.cursor.unservable", { + epoch: GlobalEventReplay.epoch, + seq: subscription.position, + }), + ) Queue.offerUnsafe(queue, controlEvent(subscription.position, "server.connected")) for (const frame of subscription.replay) Queue.offerUnsafe(queue, frameEvent(frame)) subscription.start((frame) => Queue.offerUnsafe(queue, frameEvent(frame))) diff --git a/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts b/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts index b007890ef..7d0d9900f 100644 --- a/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts +++ b/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts @@ -7,7 +7,7 @@ import { NodeHttpServer } from "@effect/platform-node" import { describe, expect } from "bun:test" import http from "node:http" -import { Context, Effect, Fiber, Layer, Option, Queue } from "effect" +import { Context, Effect, Fiber, Layer, Option, Queue, Scope } from "effect" import { HttpRouter, HttpServer } from "effect/unstable/http" import { HttpApiBuilder } from "effect/unstable/httpapi" import { Auth } from "../../src/auth" @@ -188,3 +188,44 @@ describe("global event stream — resume from a cursor (#1776)", () => { ) } }) + +describe("global event stream — unservable cursor resets (#1776)", () => { + const unservable: Record Effect.Effect> = { + "a cursor from another epoch": () => Effect.succeed(`x${GlobalBus.epoch}:1`), + "a cursor that is not : (a payload id)": () => Effect.succeed("evt_131b06142001C9nKEJ5DvEnx4G"), + "a cursor ahead of the stream": () => Effect.succeed(`${GlobalBus.epoch}:${GlobalBus.seq + 1000}`), + "a cursor older than the buffer's oldest entry": () => + Effect.gen(function* () { + const a = yield* connect() + const cursor = (yield* a.next).id! + a.abort() + // more than the default 32 MiB byte bound: the frames after the cursor are evicted + const big = "x".repeat(1024 * 1024) + for (let i = 0; i < 33; i++) emit("test.big", { big }) + return cursor + }), + } + for (const [name, cursorFor] of Object.entries(unservable)) { + it.live(`${name} yields a reset frame first, then live frames`, () => + Effect.gen(function* () { + emit("test.before") + const cursor = yield* cursorFor() + const b = yield* connect({ headers: { "last-event-id": cursor } }) + const current = { epoch: GlobalBus.epoch, seq: GlobalBus.seq } + + const reset = yield* b.next + expect(reset.data.payload.type).toBe("server.cursor.unservable") + expect(reset.data.payload.properties).toEqual(current) + expect(idOf(reset)).toEqual(current) + const connected = yield* b.next + expect(connected.data.payload.type).toBe("server.connected") + expect(idOf(connected)).toEqual(current) + + emit("test.live") + const live = yield* b.next + expect(live.data.payload.type).toBe("test.live") + expect(idOf(live)).toEqual({ epoch: current.epoch, seq: current.seq + 1 }) + }), + ) + } +}) From d2f51c9238f95ce4b61a899ab157fcf7c2432d02 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 16:14:13 -0400 Subject: [PATCH 10/60] feat(engine): bound the global replay buffer by bytes and age, configurable by env (#1776) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shared buffer evicts oldest-first until retained bytes and the oldest frame's age are both within bounds — on every push and before a cursor is resolved. OPENCODE_GLOBAL_EVENT_REPLAY_MAX_BYTES (default 32 MiB) and OPENCODE_GLOBAL_EVENT_REPLAY_MAX_AGE_MS (default 5 min) set the bounds. --- packages/app-bundle/manifest.json | 10 ++- .../opencode/src/bus/global-replay.ts | 61 ++++++++++--- .../test/server/global-event-replay.test.ts | 85 +++++++++++++++++++ 3 files changed, 142 insertions(+), 14 deletions(-) create mode 100644 packages/app-bundle/overlay/packages/opencode/test/server/global-event-replay.test.ts diff --git a/packages/app-bundle/manifest.json b/packages/app-bundle/manifest.json index e57b5562a..f56bb5610 100644 --- a/packages/app-bundle/manifest.json +++ b/packages/app-bundle/manifest.json @@ -316,7 +316,7 @@ "packages/llm/src/protocols/bedrock-converse.ts": "834a415fa8ddf9ad1224133f8352d0a926694f19f0954201d182111c171ec03b", "packages/llm/test/provider/bedrock-converse.test.ts": "e8e14d60e8502e928f170f8ac5b406c44bf16d2dd34996ddc2fb7b9261decf85", "packages/opencode/src/agent/agent.ts": "512372a4c62997e9565266638ada19b9dc0a3685b0c0483fd28fede2abdb5c9b", - "packages/opencode/src/bus/global-replay.ts": "6bd6b68baf46e0570be268bb99989e8b3f041a584720aac0c917be2e887e2bec", + "packages/opencode/src/bus/global-replay.ts": "bfbdeb214b6ba36d65afa4fdac1dacfa903bebe0c5d88adccfb19a0fbaea25b9", "packages/opencode/src/bus/global.ts": "65b3b380936fc34b2c5ef9f3ab419a6c8e6d4cfc696302a266188f26ccdc320a", "packages/opencode/src/effect/runner.ts": "b660b5b22df810b16293c23c6d832e5e8dcfb12b9a8a858c875d2c051c855b81", "packages/opencode/src/mcp/browser.ts": "ecdda42cbd55a97b8d14050aa15c6e6450a1d63b0a641c8c8e5b81e50f957c87", @@ -389,6 +389,7 @@ "packages/opencode/test/server/amicode-widgets-src.test.ts": "fa95521084f95e0c421e7f7f13bf61de7e6b5cc9f42aa9fade486f5c1b53a65d", "packages/opencode/test/server/amicode-widgets.test.ts": "14fc6ccff4e83a5b005a724936e9f0d7ce411fab74109f319e8bc748665f7ab8", "packages/opencode/test/server/auth-accept-set.test.ts": "8fe698812ed9a809d21ace1a9751bd3f1976ab1e58ad5bda65dbfff58df8e3bc", + "packages/opencode/test/server/global-event-replay.test.ts": "a2ccc81b21eb99cb3eeeecd3973738c5ac3a0ecaba59de0516b6d0ab4fcb726c", "packages/opencode/test/server/httpapi-compression.test.ts": "5f7603d7018e0b95f1c2f36c01dd297c01770bfcdd91ce29778e03b7d6143f75", "packages/opencode/test/server/httpapi-file-write-gate.test.ts": "0c91b7417e615850edfde3f2878207dab6b6624d4cfbb6e6783b2e6d9aa4dff0", "packages/opencode/test/server/httpapi-file.test.ts": "d3386129bc81467ecbcc57a3861f53767ee56ac41b4ff6ad58d8b168fb3b3373", @@ -946,7 +947,7 @@ "packages/ui/src/v2/components/text-shimmer-v2.tsx" ], "exceptions": [], - "extracted_at": "2026-10-12T20:12:31.400Z", + "extracted_at": "2026-10-12T20:14:12.948Z", "per_package": { "packages/app": { "A": 237, @@ -964,7 +965,7 @@ "D": 0 }, "packages/opencode": { - "A": 77, + "A": 78, "M": 91, "D": 0 }, @@ -995,7 +996,7 @@ } }, "counts": { - "overlay_total": 924, + "overlay_total": 925, "deletions": 7, "server_coupled": 0 }, @@ -1381,6 +1382,7 @@ "packages/opencode/test/server/amicode-widgets-src.test.ts": "A", "packages/opencode/test/server/amicode-widgets.test.ts": "A", "packages/opencode/test/server/auth-accept-set.test.ts": "A", + "packages/opencode/test/server/global-event-replay.test.ts": "A", "packages/opencode/test/server/httpapi-compression.test.ts": "M", "packages/opencode/test/server/httpapi-file-write-gate.test.ts": "A", "packages/opencode/test/server/httpapi-file.test.ts": "M", diff --git a/packages/app-bundle/overlay/packages/opencode/src/bus/global-replay.ts b/packages/app-bundle/overlay/packages/opencode/src/bus/global-replay.ts index 889773fc0..2eca1e4fe 100644 --- a/packages/app-bundle/overlay/packages/opencode/src/bus/global-replay.ts +++ b/packages/app-bundle/overlay/packages/opencode/src/bus/global-replay.ts @@ -14,6 +14,8 @@ export type Frame = { readonly data: string /** UTF-8 size of `data`, charged against the byte bound. */ readonly bytes: number + /** Push time (ms), checked against the age bound. */ + readonly time: number /** Push order; unlike `seq` it is unique per frame (a sync copy shares its source's seq). */ readonly ordinal: number } @@ -21,9 +23,30 @@ export type Frame = { export type Bounds = { /** Retained frame bytes; the oldest frames are evicted past it. */ readonly maxBytes: number + /** Retained frame age in milliseconds; older frames are evicted. */ + readonly maxAgeMs: number } -export const DEFAULT_BOUNDS: Bounds = { maxBytes: 32 * 1024 * 1024 } +/** About 32 MB and 5 minutes of history: covers link blips and window reloads. */ +export const DEFAULT_BOUNDS: Bounds = { maxBytes: 32 * 1024 * 1024, maxAgeMs: 5 * 60_000 } + +export const BOUNDS_ENV = { + maxBytes: "OPENCODE_GLOBAL_EVENT_REPLAY_MAX_BYTES", + maxAgeMs: "OPENCODE_GLOBAL_EVENT_REPLAY_MAX_AGE_MS", +} as const + +/** The buffer bounds from the environment; an unset or invalid value keeps its default. */ +export function boundsFromEnv(env: Record): Bounds { + const read = (key: string, fallback: number) => { + const raw = env[key]?.trim() + if (!raw || !/^\d+$/.test(raw)) return fallback + return Number(raw) + } + return { + maxBytes: read(BOUNDS_ENV.maxBytes, DEFAULT_BOUNDS.maxBytes), + maxAgeMs: read(BOUNDS_ENV.maxAgeMs, DEFAULT_BOUNDS.maxAgeMs), + } +} export function formatID(epoch: string, seq: number) { return `${epoch}:${seq}` @@ -65,13 +88,22 @@ export class ReplayBuffer { // emitted before this buffer existed). A cursor below it cannot be served. #floor: number - constructor( - readonly epoch: string, - readonly bounds: Bounds = DEFAULT_BOUNDS, - start = 0, - ) { - this.#current = start - this.#floor = start + readonly epoch: string + readonly bounds: Bounds + readonly #now: () => number + + constructor(input: { + epoch: string + bounds?: Bounds + /** The sequence the buffer starts at (events before it were never recorded). */ + start?: number + now?: () => number + }) { + this.epoch = input.epoch + this.bounds = input.bounds ?? DEFAULT_BOUNDS + this.#now = input.now ?? Date.now + this.#current = input.start ?? 0 + this.#floor = this.#current } /** The current sequence: the last one pushed (or skipped). */ @@ -90,6 +122,7 @@ export class ReplayBuffer { seq, data, bytes: Buffer.byteLength(data), + time: this.#now(), ordinal: ++this.#ordinal, } this.#current = Math.max(this.#current, seq) @@ -117,6 +150,7 @@ export class ReplayBuffer { const listener = (frame: Frame) => (deliver ? deliver(frame) : pending.push(frame)) this.#listeners.add(listener) + this.#evict() const watermark = this.#ordinal const resolved = this.#resolve(cursor) const replay = resolved.reset ? [] : this.#retained().filter((frame) => frame.seq > resolved.position) @@ -156,9 +190,12 @@ export class ReplayBuffer { return this.#frames.slice(this.#head) as Frame[] } + // Oldest-first, until retained bytes and the oldest frame's age are both within bounds. #evict() { - while (this.#head < this.#frames.length && this.#bytes > this.bounds.maxBytes) { + const oldest = this.#now() - this.bounds.maxAgeMs + while (this.#head < this.#frames.length) { const frame = this.#frames[this.#head]! + if (this.#bytes <= this.bounds.maxBytes && frame.time >= oldest) break this.#frames[this.#head++] = undefined this.#bytes -= frame.bytes this.#floor = Math.max(this.#floor, frame.seq) @@ -170,7 +207,11 @@ export class ReplayBuffer { } } -export const GlobalEventReplay = new ReplayBuffer(GlobalBus.epoch, DEFAULT_BOUNDS, GlobalBus.seq) +export const GlobalEventReplay = new ReplayBuffer({ + epoch: GlobalBus.epoch, + bounds: boundsFromEnv(process.env), + start: GlobalBus.seq, +}) GlobalBus.tap((event, seq) => { let data: string diff --git a/packages/app-bundle/overlay/packages/opencode/test/server/global-event-replay.test.ts b/packages/app-bundle/overlay/packages/opencode/test/server/global-event-replay.test.ts new file mode 100644 index 000000000..b63645966 --- /dev/null +++ b/packages/app-bundle/overlay/packages/opencode/test/server/global-event-replay.test.ts @@ -0,0 +1,85 @@ +// #1776 — the global stream's replay buffer: one shared buffer bounded by +// bytes and age, evicting oldest-first, both bounds configurable by environment. +import { describe, expect, test } from "bun:test" +import path from "node:path" +import { DEFAULT_BOUNDS, ReplayBuffer, boundsFromEnv } from "../../src/bus/global-replay" + +const frameOf = (bytes: number) => "x".repeat(bytes) + +function buffer(bounds: { maxBytes: number; maxAgeMs: number }, clock = { now: 0 }) { + return new ReplayBuffer({ epoch: "ep0ch", bounds, now: () => clock.now }) +} + +const seqs = (b: ReplayBuffer, cursor: string) => { + const sub = b.subscribe(cursor) + sub.close() + return sub.reset ?? sub.replay.map((frame) => frame.seq) +} + +describe("global event replay buffer — bounds (#1776)", () => { + test("evicts oldest-first once retained bytes exceed the byte bound", () => { + const b = buffer({ maxBytes: 30, maxAgeMs: 60_000 }) + for (const seq of [1, 2, 3]) b.push(seq, frameOf(10)) + expect(seqs(b, "ep0ch:0")).toEqual([1, 2, 3]) + + b.push(4, frameOf(10)) // 40 bytes > 30: the oldest frame goes + expect(seqs(b, "ep0ch:0")).toBe("evicted") + expect(seqs(b, "ep0ch:1")).toEqual([2, 3, 4]) + + b.push(5, frameOf(25)) // 55 bytes: evict 2, 3, 4 (oldest first) until within bound + expect(seqs(b, "ep0ch:3")).toBe("evicted") + expect(seqs(b, "ep0ch:4")).toEqual([5]) + }) + + test("evicts oldest-first once frames are older than the age bound", () => { + const clock = { now: 0 } + const b = buffer({ maxBytes: 1 << 20, maxAgeMs: 1_500 }, clock) + b.push(1, frameOf(10)) + clock.now = 1_000 + b.push(2, frameOf(10)) + clock.now = 1_400 + expect(seqs(b, "ep0ch:0")).toEqual([1, 2]) + + clock.now = 2_000 // frame 1 is 2000 ms old (> 1500), frame 2 is 1000 ms old + expect(seqs(b, "ep0ch:0")).toBe("evicted") + expect(seqs(b, "ep0ch:1")).toEqual([2]) + + clock.now = 3_000 // both aged out; a cursor at the current sequence is still servable + b.push(3, frameOf(10)) + expect(seqs(b, "ep0ch:1")).toBe("evicted") + expect(seqs(b, "ep0ch:2")).toEqual([3]) + expect(seqs(b, "ep0ch:3")).toEqual([]) + }) + + test("both bounds are read from the environment, defaulting to 32 MiB and 5 minutes", () => { + expect(DEFAULT_BOUNDS).toEqual({ maxBytes: 32 * 1024 * 1024, maxAgeMs: 5 * 60_000 }) + expect( + boundsFromEnv({ + OPENCODE_GLOBAL_EVENT_REPLAY_MAX_BYTES: "1024", + OPENCODE_GLOBAL_EVENT_REPLAY_MAX_AGE_MS: "60000", + }), + ).toEqual({ maxBytes: 1024, maxAgeMs: 60_000 }) + expect(boundsFromEnv({})).toEqual(DEFAULT_BOUNDS) + expect( + boundsFromEnv({ OPENCODE_GLOBAL_EVENT_REPLAY_MAX_BYTES: "lots", OPENCODE_GLOBAL_EVENT_REPLAY_MAX_AGE_MS: "-5" }), + ).toEqual(DEFAULT_BOUNDS) + }) + + test("the engine's shared buffer takes its bounds from the process environment", () => { + const result = Bun.spawnSync({ + cmd: [ + process.execPath, + "-e", + 'const { GlobalEventReplay } = await import("./src/bus/global-replay.ts"); console.log(JSON.stringify(GlobalEventReplay.bounds))', + ], + cwd: path.join(import.meta.dir, "..", ".."), + env: { + ...process.env, + OPENCODE_GLOBAL_EVENT_REPLAY_MAX_BYTES: "4096", + OPENCODE_GLOBAL_EVENT_REPLAY_MAX_AGE_MS: "2500", + }, + }) + expect(result.stderr.toString()).toBe("") + expect(JSON.parse(result.stdout.toString())).toEqual({ maxBytes: 4096, maxAgeMs: 2500 }) + }) +}) From 13e59afc9486bfca070e6210a1ab32d6708387a9 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 16:15:37 -0400 Subject: [PATCH 11/60] feat(engine): global-stream heartbeats carry server time and the current sequence (#1776) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Heartbeat properties are {time, seq} — server time in milliseconds and the current sequence — and the heartbeat's SSE id is that sequence. It is queued behind every frame up to that sequence, so a backlogged connection never sees a heartbeat claim a sequence it has not been sent (lag and idle holes become measurable without false holes). --- packages/app-bundle/manifest.json | 6 ++-- .../instance/httpapi/handlers/global.ts | 22 +++++++++++---- .../test/server/httpapi-global-event.test.ts | 28 ++++++++++++++++++- 3 files changed, 46 insertions(+), 10 deletions(-) diff --git a/packages/app-bundle/manifest.json b/packages/app-bundle/manifest.json index f56bb5610..0494392e8 100644 --- a/packages/app-bundle/manifest.json +++ b/packages/app-bundle/manifest.json @@ -393,7 +393,7 @@ "packages/opencode/test/server/httpapi-compression.test.ts": "5f7603d7018e0b95f1c2f36c01dd297c01770bfcdd91ce29778e03b7d6143f75", "packages/opencode/test/server/httpapi-file-write-gate.test.ts": "0c91b7417e615850edfde3f2878207dab6b6624d4cfbb6e6783b2e6d9aa4dff0", "packages/opencode/test/server/httpapi-file.test.ts": "d3386129bc81467ecbcc57a3861f53767ee56ac41b4ff6ad58d8b168fb3b3373", - "packages/opencode/test/server/httpapi-global-event.test.ts": "92e3cba9fb54d16e229ced46c1bd23ee8f1a5ef24b1134baf77a6e414230c2a5", + "packages/opencode/test/server/httpapi-global-event.test.ts": "5433a19e30cbdc6dc3ab12c76c3dd0239dacaa08cab9ca5fe9c4057ea33d986f", "packages/opencode/test/server/httpapi-instance-route-auth.test.ts": "4b73a89d2431c02bc3b665d93f9002914e51eb1add979dc1867efbcbc0c36e25", "packages/opencode/test/server/httpapi-provider.test.ts": "9e8b3e085b478e3f99b689ffff600e10c0b4b50b3cabdbd749f88dae19bf5904", "packages/opencode/test/server/httpapi-session.test.ts": "eda6fd763ac4c17891cedaa4ccd49597c03e7227fcfdcf0f94a1b4545ba623ed", @@ -928,7 +928,7 @@ "packages/opencode/src/server/routes/instance/httpapi/groups/session.ts": "1421f8c07948b121c7e2005bd9b0fcd6b2c456565ddd1da32d8e6c35d03fac9c", "packages/opencode/src/server/routes/instance/httpapi/handlers/experimental.ts": "69ca3dd48d95157fc822e1f76eb42ac873df50f22616b85194619ca658613867", "packages/opencode/src/server/routes/instance/httpapi/handlers/file.ts": "10bcb79b117beebee04184fb48c1ad789e782e2f7ae4b0c68603edc00f03aee9", - "packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts": "809ba02a180d2450a0f5928af83960a4fd3cd13e914b2606dcbcdbc601e2e201", + "packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts": "3be6ff6c008d83f3c775c8e7b896397ca28ca31356dff3ae6a4891bc35da92a9", "packages/opencode/src/server/routes/instance/httpapi/handlers/mcp.ts": "7da08d58765cdf2b6ce4648fc70de371402949f5bb7b75696f674d43f2d5365b", "packages/opencode/src/server/routes/instance/httpapi/handlers/provider.ts": "8baf16cad0992c5351b61b0c0fb27903bb25b17e1b25fc7c5d77f807c24413a6", "packages/opencode/src/server/routes/instance/httpapi/handlers/session.ts": "b37ff4136c4aea85b23083bf441f03e09820e3e4cdd95e4118ca570d8435b94f", @@ -947,7 +947,7 @@ "packages/ui/src/v2/components/text-shimmer-v2.tsx" ], "exceptions": [], - "extracted_at": "2026-10-12T20:14:12.948Z", + "extracted_at": "2026-10-12T20:15:37.380Z", "per_package": { "packages/app": { "A": 237, diff --git a/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts b/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts index d5ed6f6b8..06892f4b6 100644 --- a/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts +++ b/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts @@ -6,7 +6,7 @@ import { EventV2 } from "@opencode-ai/core/event" import { Installation } from "@/installation" import { disposeAllInstancesAndEmitGlobalDisposed } from "@/server/global-lifecycle" import { InstallationVersion } from "@opencode-ai/core/installation/version" -import { Effect, Queue } from "effect" +import { Clock, Effect, Queue } from "effect" import * as Stream from "effect/Stream" import { HttpServerRequest, HttpServerResponse } from "effect/unstable/http" import { HttpApiBuilder } from "effect/unstable/httpapi" @@ -27,12 +27,13 @@ function frameEvent(frame: Pick): Sse.Event { return { _tag: "Event", event: "message", id: frame.id, data: frame.data } } -// A per-connection control frame (connected, heartbeat). It never consumes a -// sequence — that would punch a hole in every other subscriber's stream — and -// carries the connection's current position as its id instead. +// A per-connection control frame (connected, heartbeat, reset). It never +// consumes a sequence — that would punch a hole in every other subscriber's +// stream — and carries the connection's current position as its id instead, +// so ids on one connection never decrease. function controlEvent(seq: number, type: string, properties: Record = {}): Sse.Event { return frameEvent({ - id: formatID(GlobalBus.epoch, seq), + id: formatID(GlobalEventReplay.epoch, seq), data: JSON.stringify({ payload: { id: EventV2.ID.create(), type, properties } }), }) } @@ -65,9 +66,18 @@ function eventResponse() { (subscription) => Effect.sync(() => subscription.close()), ) yield* Effect.logInfo("global event connected") + // The heartbeat enters the connection's own queue, behind every frame up to + // the current sequence, so a backlogged connection never sees a heartbeat + // claim a sequence it has not been sent yet (that would read as a hole). const heartbeat = Stream.tick("10 seconds").pipe( Stream.drop(1), - Stream.map(() => controlEvent(GlobalBus.seq, "server.heartbeat")), + Stream.mapEffect(() => + Effect.map(Clock.currentTimeMillis, (time) => { + const seq = GlobalEventReplay.current + Queue.offerUnsafe(queue, controlEvent(seq, "server.heartbeat", { time, seq })) + }), + ), + Stream.drain, ) return HttpServerResponse.stream( diff --git a/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts b/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts index 7d0d9900f..0d22c2c48 100644 --- a/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts +++ b/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts @@ -7,7 +7,8 @@ import { NodeHttpServer } from "@effect/platform-node" import { describe, expect } from "bun:test" import http from "node:http" -import { Context, Effect, Fiber, Layer, Option, Queue, Scope } from "effect" +import { Clock, Context, Effect, Fiber, Layer, Option, Queue, Scope } from "effect" +import * as TestClock from "effect/testing/TestClock" import { HttpRouter, HttpServer } from "effect/unstable/http" import { HttpApiBuilder } from "effect/unstable/httpapi" import { Auth } from "../../src/auth" @@ -229,3 +230,28 @@ describe("global event stream — unservable cursor resets (#1776)", () => { ) } }) + +describe("global event stream — heartbeats (#1776)", () => { + it.effect("a heartbeat carries server time in milliseconds and the current sequence, and ids itself with it", () => + Effect.gen(function* () { + const a = yield* connect() + yield* a.next + emit("test.beat") + const live = yield* a.next + + yield* TestClock.adjust("10 seconds") + const beat = yield* a.next + expect(beat.data.payload.type).toBe("server.heartbeat") + expect(beat.data.payload.properties).toEqual({ time: yield* Clock.currentTimeMillis, seq: idOf(live).seq }) + expect(idOf(beat)).toEqual(idOf(live)) + + emit("test.after-beat") + yield* TestClock.adjust("10 seconds") + const [after, second] = yield* a.take(2) + expect(after.data.payload.type).toBe("test.after-beat") + expect(second.data.payload.type).toBe("server.heartbeat") + expect(second.data.payload.properties).toEqual({ time: yield* Clock.currentTimeMillis, seq: idOf(after).seq }) + expect(idOf(second)).toEqual(idOf(after)) + }), + ) +}) From 0589f1f080d9c766ab2feab741c3cd2eaa5bcf72 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 16:18:15 -0400 Subject: [PATCH 12/60] fix(engine): release a global-stream subscription when its client aborts (#1776) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bun's node:http shim — the compiled engine's runtime — never emits the response 'close' event on a client abort, which is the only signal the node:http adapter interrupts the request fiber on; writes to the dead socket keep succeeding. Measured on a scratch engine built from main: 6 connects, 0 disconnects after every client aborted. The stream now also ends when the request's socket closes; the end of the stream unregisters the listener, drops the undelivered frames with the queue, and logs the disconnect with the remaining subscriber count. --- packages/app-bundle/manifest.json | 6 +-- .../instance/httpapi/handlers/global.ts | 50 +++++++++++++++++-- .../test/server/httpapi-global-event.test.ts | 47 ++++++++++++++++- 3 files changed, 95 insertions(+), 8 deletions(-) diff --git a/packages/app-bundle/manifest.json b/packages/app-bundle/manifest.json index 0494392e8..4e1044f80 100644 --- a/packages/app-bundle/manifest.json +++ b/packages/app-bundle/manifest.json @@ -393,7 +393,7 @@ "packages/opencode/test/server/httpapi-compression.test.ts": "5f7603d7018e0b95f1c2f36c01dd297c01770bfcdd91ce29778e03b7d6143f75", "packages/opencode/test/server/httpapi-file-write-gate.test.ts": "0c91b7417e615850edfde3f2878207dab6b6624d4cfbb6e6783b2e6d9aa4dff0", "packages/opencode/test/server/httpapi-file.test.ts": "d3386129bc81467ecbcc57a3861f53767ee56ac41b4ff6ad58d8b168fb3b3373", - "packages/opencode/test/server/httpapi-global-event.test.ts": "5433a19e30cbdc6dc3ab12c76c3dd0239dacaa08cab9ca5fe9c4057ea33d986f", + "packages/opencode/test/server/httpapi-global-event.test.ts": "29b60da36fa77e0a1a251bbeb52d6b026aa5f2e528dc189edd6951c7a9545aac", "packages/opencode/test/server/httpapi-instance-route-auth.test.ts": "4b73a89d2431c02bc3b665d93f9002914e51eb1add979dc1867efbcbc0c36e25", "packages/opencode/test/server/httpapi-provider.test.ts": "9e8b3e085b478e3f99b689ffff600e10c0b4b50b3cabdbd749f88dae19bf5904", "packages/opencode/test/server/httpapi-session.test.ts": "eda6fd763ac4c17891cedaa4ccd49597c03e7227fcfdcf0f94a1b4545ba623ed", @@ -928,7 +928,7 @@ "packages/opencode/src/server/routes/instance/httpapi/groups/session.ts": "1421f8c07948b121c7e2005bd9b0fcd6b2c456565ddd1da32d8e6c35d03fac9c", "packages/opencode/src/server/routes/instance/httpapi/handlers/experimental.ts": "69ca3dd48d95157fc822e1f76eb42ac873df50f22616b85194619ca658613867", "packages/opencode/src/server/routes/instance/httpapi/handlers/file.ts": "10bcb79b117beebee04184fb48c1ad789e782e2f7ae4b0c68603edc00f03aee9", - "packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts": "3be6ff6c008d83f3c775c8e7b896397ca28ca31356dff3ae6a4891bc35da92a9", + "packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts": "ffb0bdeae81c205434e08f540b12bf8eb49fe2949c51b3cd88cb380cee640f0c", "packages/opencode/src/server/routes/instance/httpapi/handlers/mcp.ts": "7da08d58765cdf2b6ce4648fc70de371402949f5bb7b75696f674d43f2d5365b", "packages/opencode/src/server/routes/instance/httpapi/handlers/provider.ts": "8baf16cad0992c5351b61b0c0fb27903bb25b17e1b25fc7c5d77f807c24413a6", "packages/opencode/src/server/routes/instance/httpapi/handlers/session.ts": "b37ff4136c4aea85b23083bf441f03e09820e3e4cdd95e4118ca570d8435b94f", @@ -947,7 +947,7 @@ "packages/ui/src/v2/components/text-shimmer-v2.tsx" ], "exceptions": [], - "extracted_at": "2026-10-12T20:15:37.380Z", + "extracted_at": "2026-10-12T20:18:15.709Z", "per_package": { "packages/app": { "A": 237, diff --git a/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts b/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts index 06892f4b6..debaa48f9 100644 --- a/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts +++ b/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts @@ -6,11 +6,12 @@ import { EventV2 } from "@opencode-ai/core/event" import { Installation } from "@/installation" import { disposeAllInstancesAndEmitGlobalDisposed } from "@/server/global-lifecycle" import { InstallationVersion } from "@opencode-ai/core/installation/version" -import { Clock, Effect, Queue } from "effect" +import { Clock, Deferred, Effect, Exit, Queue } from "effect" import * as Stream from "effect/Stream" import { HttpServerRequest, HttpServerResponse } from "effect/unstable/http" import { HttpApiBuilder } from "effect/unstable/httpapi" import * as Sse from "effect/unstable/encoding/Sse" +import type { Socket } from "node:net" import { RootHttpApi } from "../api" import { GlobalUpgradeInput } from "../groups/global" @@ -38,10 +39,34 @@ function controlEvent(seq: number, type: string, properties: Record() + const done = () => Deferred.doneUnsafe(gone, Exit.void) + const socket = (request.source as { socket?: Partial } | undefined)?.socket + if (!socket || typeof socket.once !== "function" || typeof socket.off !== "function") return gone + if (socket.destroyed) { + done() + return gone + } + socket.once("close", done) + yield* Effect.addFinalizer(() => Effect.sync(() => socket.off?.("close", done))) + return gone + }) +} + function eventResponse() { return Effect.gen(function* () { const request = yield* HttpServerRequest.HttpServerRequest const cursor = cursorOf(request) + const gone = yield* clientGone(request) // Registration is eager and synchronous: a bus event emitted while the // response body is starting is queued, never lost. const queue = yield* Queue.unbounded() @@ -63,9 +88,15 @@ function eventResponse() { subscription.start((frame) => Queue.offerUnsafe(queue, frameEvent(frame))) return subscription }), - (subscription) => Effect.sync(() => subscription.close()), + (subscription) => Effect.andThen(Effect.sync(() => subscription.close()), Queue.shutdown(queue)), + ) + yield* Effect.logInfo("global event connected").pipe( + Effect.annotateLogs({ + subscribers: GlobalEventReplay.subscribers, + resume: subscription.reset ? `reset:${subscription.reset}` : cursor ? "replay" : "live", + replayed: subscription.replay.length, + }), ) - yield* Effect.logInfo("global event connected") // The heartbeat enters the connection's own queue, behind every frame up to // the current sequence, so a backlogged connection never sees a heartbeat // claim a sequence it has not been sent yet (that would read as a hole). @@ -83,12 +114,23 @@ function eventResponse() { return HttpServerResponse.stream( Stream.fromQueue(queue).pipe( Stream.merge(heartbeat, { haltStrategy: "left" }), + Stream.interruptWhen(Deferred.await(gone)), Stream.pipeThroughChannel(Sse.encode()), Stream.encodeText, Stream.ensuring( + // Release at the stream's end, not only at the request scope's: the + // listener is unregistered and the undelivered frames are dropped + // with the queue before the disconnect is logged. Effect.suspend(() => { subscription.close() - return Effect.logInfo("global event disconnected") + const queued = Queue.sizeUnsafe(queue) + return Queue.shutdown(queue).pipe( + Effect.andThen( + Effect.logInfo("global event disconnected").pipe( + Effect.annotateLogs({ subscribers: GlobalEventReplay.subscribers, dropped: queued }), + ), + ), + ) }), ), ), diff --git a/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts b/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts index 0d22c2c48..822f84ed9 100644 --- a/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts +++ b/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts @@ -7,12 +7,13 @@ import { NodeHttpServer } from "@effect/platform-node" import { describe, expect } from "bun:test" import http from "node:http" -import { Clock, Context, Effect, Fiber, Layer, Option, Queue, Scope } from "effect" +import { Clock, Context, Effect, Fiber, Layer, Logger, Option, Queue, References, Scope } from "effect" import * as TestClock from "effect/testing/TestClock" import { HttpRouter, HttpServer } from "effect/unstable/http" import { HttpApiBuilder } from "effect/unstable/httpapi" import { Auth } from "../../src/auth" import { GlobalBus } from "../../src/bus/global" +import { GlobalEventReplay } from "../../src/bus/global-replay" import { Config } from "../../src/config/config" import { Installation } from "../../src/installation" import { MoveSession } from "@opencode-ai/core/control-plane/move-session" @@ -42,9 +43,23 @@ const apiLayer = HttpRouter.serve( Layer.provide(Layer.mock(MoveSession.Service)({})), Layer.provide(Layer.mock(Installation.Service)({})), Layer.provide(ServerAuth.Config.configLayer({ password: Option.none(), username: "opencode" })), + Layer.provide( + Logger.layer([ + Logger.make((options) => { + logs.push({ + message: ([] as unknown[]).concat(options.message).join(" "), + annotations: { ...options.fiber.getRef(References.CurrentLogAnnotations) }, + }) + }), + ]), + ), ) const it = testEffect(apiLayer) +// The server's own log lines (the stream's connect/disconnect census). +const logs: { message: string; annotations: Record }[] = [] +const logged = (message: string) => logs.filter((line) => line.message === message) + const ID = /^([A-Za-z0-9_-]+):(\d+)$/ type Frame = { id: string | undefined; event: string | undefined; data: any } @@ -255,3 +270,33 @@ describe("global event stream — heartbeats (#1776)", () => { }), ) }) + +describe("global event stream — client abort (#1776)", () => { + it.live("a client that disconnects mid-stream releases its subscription and logs the disconnect", () => + Effect.gen(function* () { + const baseline = GlobalEventReplay.subscribers + const disconnects = logged("global event disconnected").length + const a = yield* connect() + yield* a.next + expect(GlobalEventReplay.subscribers).toBe(baseline + 1) + expect(logged("global event connected").at(-1)?.annotations).toMatchObject({ subscribers: baseline + 1 }) + + emit("test.unread") + emit("test.unread") + a.abort() + + const released = yield* Effect.gen(function* () { + while (GlobalEventReplay.subscribers !== baseline) yield* Effect.sleep("20 millis") + return true + }).pipe(Effect.timeoutOrElse({ duration: "3 seconds", orElse: () => Effect.succeed(false) })) + expect(released).toBe(true) + const lines = logged("global event disconnected").slice(disconnects) + expect(lines).toHaveLength(1) + expect(lines[0].annotations).toMatchObject({ subscribers: baseline }) + + // released for good: later events are queued for no one + for (let i = 0; i < 100; i++) emit("test.after-abort", { i }) + expect(GlobalEventReplay.subscribers).toBe(baseline) + }), + ) +}) From 647e13871b176a94645cd04590488e65884f4b6c Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 16:22:36 -0400 Subject: [PATCH 13/60] test(engine): scratch-engine check of global-stream replay and abort release on the compiled binary (#1776) Opt-in (SCRATCH_ENGINE_E2E=1): spawns this checkout's compiled engine on a spare port (4190-4199) with isolated state dirs and DB, resumes from a real engine id, and aborts five clients mid-stream. Red on a binary built from main (no ids; 0 disconnects), green after. --- .../slow/engine_global_stream_scratch.test.ts | 173 ++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 packages/extension/test/slow/engine_global_stream_scratch.test.ts diff --git a/packages/extension/test/slow/engine_global_stream_scratch.test.ts b/packages/extension/test/slow/engine_global_stream_scratch.test.ts new file mode 100644 index 000000000..5a31b337f --- /dev/null +++ b/packages/extension/test/slow/engine_global_stream_scratch.test.ts @@ -0,0 +1,173 @@ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { spawn, type ChildProcess } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import http from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +// ============================================================================ +// Scratch-engine check of the sequenced, replayable global stream (#1776), +// against the COMPILED engine binary — the runtime (Bun's node:http shim) +// whose client-abort behaviour the in-process suite cannot vouch for alone. +// +// Opt-in: SCRATCH_ENGINE_E2E=1, after `pnpm --filter amicode run build:binary` +// (not an AMICODE_* name: test/setup_hermetic_env.ts deletes those before this +// module's gate is read). It spawns the vendored binary of THIS checkout on a spare port +// (default 4195; never a hub port) with every state dir and the session DB in +// one temp dir, and kills it afterwards (SCRATCH_ENGINE_BIN points it at another +// binary, e.g. a baseline build). Live hubs are never touched. +// ============================================================================ + +const ENABLED = process.env.SCRATCH_ENGINE_E2E === "1"; +const BIN = + process.env.SCRATCH_ENGINE_BIN ?? join(__dirname, "..", "..", "vendor", "opencode", `${process.platform}-${process.arch}`, "opencode"); +const PORT = Number(process.env.SCRATCH_ENGINE_PORT ?? 4195); +const ID = /^([A-Za-z0-9_-]+):(\d+)$/; + +type Frame = { id: string; type: string; data: string }; + +function seqOf(frame: Frame) { + const match = ID.exec(frame.id); + if (!match) throw new Error(`frame id ${JSON.stringify(frame.id)} is not :`); + return { epoch: match[1], seq: Number(match[2]) }; +} + +function stream(headers: Record = {}) { + const frames: Frame[] = []; + let buffer = ""; + let req!: http.ClientRequest; + const opened = new Promise((resolve, reject) => { + req = http.get({ host: "127.0.0.1", port: PORT, path: "/global/event", headers: { accept: "text/event-stream", ...headers } }, (res) => { + res.setEncoding("utf8"); + res.on("data", (chunk: string) => { + buffer += chunk; + let end = buffer.indexOf("\n\n"); + while (end !== -1) { + const block = buffer.slice(0, end); + buffer = buffer.slice(end + 2); + const id = /^id: (.*)$/m.exec(block)?.[1] ?? ""; + const data = block.split("\n").filter((l) => l.startsWith("data: ")).map((l) => l.slice(6)).join("\n"); + if (data) frames.push({ id, data, type: JSON.parse(data).payload?.type }); + end = buffer.indexOf("\n\n"); + } + }); + res.on("error", () => {}); + resolve(); + }); + req.on("error", (e: NodeJS.ErrnoException) => (e.code === "ECONNRESET" ? undefined : reject(e))); + }); + return { frames, opened, abort: () => req.destroy() }; +} + +function post(path: string): Promise { + return new Promise((resolve, reject) => { + const req = http.request({ host: "127.0.0.1", port: PORT, path, method: "POST", headers: { "content-type": "application/json" } }, (res) => { + res.resume(); + resolve(res.statusCode ?? 0); + }); + req.on("error", reject); + req.end("{}"); + }); +} + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); +async function until(probe: () => T | undefined, what: string, ms = 10_000): Promise { + const deadline = Date.now() + ms; + for (;;) { + const value = probe(); + if (value !== undefined && value !== false) return value; + if (Date.now() > deadline) throw new Error(`timed out waiting for ${what}`); + await sleep(25); + } +} + +describe.skipIf(!ENABLED)("scratch engine: sequenced, replayable global stream (#1776)", () => { + let engine: ChildProcess; + let scratch: string; + let log = ""; + const lines = (message: string) => log.split("\n").filter((l) => l.includes(`message="${message}"`)); + + beforeAll(async () => { + if (!(PORT >= 4190 && PORT <= 4199)) throw new Error(`scratch port ${PORT} outside 4190-4199`); + if (!existsSync(BIN)) throw new Error(`no compiled engine at ${BIN} — run build:binary first`); + scratch = mkdtempSync(join(tmpdir(), "amicode-scratch-engine-")); + const proj = join(scratch, "proj"); + mkdirSync(proj); + mkdirSync(join(scratch, "home")); + const env: Record = {}; + for (const [k, v] of Object.entries(process.env)) if (v !== undefined && !/^(OPENCODE|AMICO)/.test(k)) env[k] = v; + Object.assign(env, { + HOME: join(scratch, "home"), + XDG_DATA_HOME: join(scratch, "data"), + XDG_CONFIG_HOME: join(scratch, "config"), + XDG_STATE_HOME: join(scratch, "state"), + XDG_CACHE_HOME: join(scratch, "cache"), + OPENCODE_DB: join(scratch, "db.sqlite"), + }); + engine = spawn(BIN, ["serve", "--port", String(PORT), "--hostname", "127.0.0.1", "--print-logs", "--log-level", "INFO", "--pure"], { + cwd: proj, + env, + stdio: ["ignore", "pipe", "pipe"], + }); + engine.stdout?.on("data", (d: Buffer) => (log += d)); + engine.stderr?.on("data", (d: Buffer) => (log += d)); + await until(() => log.includes(`listening on http://127.0.0.1:${PORT}`) || undefined, "engine listen", 60_000); + }, 90_000); + + afterAll(async () => { + engine?.kill("SIGTERM"); + await sleep(1_000); + if (engine && engine.exitCode === null && engine.signalCode === null) engine.kill("SIGKILL"); + if (scratch) rmSync(scratch, { recursive: true, force: true }); + }); + + it("replays exactly the frames after a Last-Event-ID cursor, then continues live", async () => { + const a = stream(); + await a.opened; + const connected = await until(() => a.frames[0], "connected frame"); + expect(connected.type).toBe("server.connected"); + const { epoch } = seqOf(connected); + + expect(await post("/session")).toBe(200); + await until(() => a.frames.length >= 3 || undefined, "session events on the first connection"); + await sleep(500); + const [first, ...afterCursor] = a.frames.slice(1); + a.abort(); + + expect(await post("/session")).toBe(200); // published while no one is connected + const b = stream({ "last-event-id": first.id }); + await b.opened; + await until(() => b.frames.length >= afterCursor.length + 3 || undefined, "replay and gap frames"); + await sleep(500); + + expect(b.frames[0].type).toBe("server.connected"); + expect(b.frames[0].id).toBe(first.id); + const replayed = b.frames.slice(1); + // the frames the first connection saw after the cursor, byte for byte, then the gap + expect(replayed.slice(0, afterCursor.length)).toEqual(afterCursor); + const gap = replayed.slice(afterCursor.length).filter((f) => f.type !== "server.heartbeat"); + expect(gap.map((f) => f.type)).toContain("session.created"); + // one epoch; ids never decrease and bus events never skip a sequence + let previous = seqOf(first).seq; + for (const frame of replayed) { + const { epoch: e, seq } = seqOf(frame); + expect(e).toBe(epoch); + expect(seq - previous === 0 || seq - previous === 1).toBe(true); + previous = seq; + } + b.abort(); + }, 30_000); + + it("a client that aborts mid-stream releases its subscription, and the release is logged", async () => { + const clients = Array.from({ length: 5 }, () => stream()); + await Promise.all(clients.map((c) => c.opened)); + await until(() => clients.every((c) => c.frames.length > 0) || undefined, "connected frames"); + for (const c of clients) c.abort(); + await until( + () => lines("global event disconnected").length === lines("global event connected").length || undefined, + "one disconnect per connect", + ); + expect(lines("global event connected").length).toBeGreaterThanOrEqual(5); + expect(lines("global event disconnected").at(-1)).toContain("subscribers=0"); + }, 30_000); +}); From c6039fe6284370edd0d73721f9f758f47ed47cd9 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 16:42:42 -0400 Subject: [PATCH 14/60] refactor(engine): one release path for the global stream; log the disconnect after it (#1776) The subscription release (unregister, drop the queue) is shared by the stream's end and the request scope's. The disconnect log is now built after the release runs, so its subscriber count is the post-release one (an eager annotation reported the pre-release count) and it carries how many undelivered frames were dropped. The in-process test client reads with fetch, waits for its test server to serve a real route, and reports frames seen on a timeout. --- packages/app-bundle/manifest.json | 6 +- .../instance/httpapi/handlers/global.ts | 24 ++-- .../test/server/httpapi-global-event.test.ts | 114 ++++++++++++------ 3 files changed, 92 insertions(+), 52 deletions(-) diff --git a/packages/app-bundle/manifest.json b/packages/app-bundle/manifest.json index 4e1044f80..31453756b 100644 --- a/packages/app-bundle/manifest.json +++ b/packages/app-bundle/manifest.json @@ -393,7 +393,7 @@ "packages/opencode/test/server/httpapi-compression.test.ts": "5f7603d7018e0b95f1c2f36c01dd297c01770bfcdd91ce29778e03b7d6143f75", "packages/opencode/test/server/httpapi-file-write-gate.test.ts": "0c91b7417e615850edfde3f2878207dab6b6624d4cfbb6e6783b2e6d9aa4dff0", "packages/opencode/test/server/httpapi-file.test.ts": "d3386129bc81467ecbcc57a3861f53767ee56ac41b4ff6ad58d8b168fb3b3373", - "packages/opencode/test/server/httpapi-global-event.test.ts": "29b60da36fa77e0a1a251bbeb52d6b026aa5f2e528dc189edd6951c7a9545aac", + "packages/opencode/test/server/httpapi-global-event.test.ts": "357e374d73bfb167869f19cbdd1090d171b0e7ef4e51071247428aaa03c6f7a2", "packages/opencode/test/server/httpapi-instance-route-auth.test.ts": "4b73a89d2431c02bc3b665d93f9002914e51eb1add979dc1867efbcbc0c36e25", "packages/opencode/test/server/httpapi-provider.test.ts": "9e8b3e085b478e3f99b689ffff600e10c0b4b50b3cabdbd749f88dae19bf5904", "packages/opencode/test/server/httpapi-session.test.ts": "eda6fd763ac4c17891cedaa4ccd49597c03e7227fcfdcf0f94a1b4545ba623ed", @@ -928,7 +928,7 @@ "packages/opencode/src/server/routes/instance/httpapi/groups/session.ts": "1421f8c07948b121c7e2005bd9b0fcd6b2c456565ddd1da32d8e6c35d03fac9c", "packages/opencode/src/server/routes/instance/httpapi/handlers/experimental.ts": "69ca3dd48d95157fc822e1f76eb42ac873df50f22616b85194619ca658613867", "packages/opencode/src/server/routes/instance/httpapi/handlers/file.ts": "10bcb79b117beebee04184fb48c1ad789e782e2f7ae4b0c68603edc00f03aee9", - "packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts": "ffb0bdeae81c205434e08f540b12bf8eb49fe2949c51b3cd88cb380cee640f0c", + "packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts": "f09578d5d85fbbd63c58ddf73b38daf802602165a7fc920a06b0e344202c70d2", "packages/opencode/src/server/routes/instance/httpapi/handlers/mcp.ts": "7da08d58765cdf2b6ce4648fc70de371402949f5bb7b75696f674d43f2d5365b", "packages/opencode/src/server/routes/instance/httpapi/handlers/provider.ts": "8baf16cad0992c5351b61b0c0fb27903bb25b17e1b25fc7c5d77f807c24413a6", "packages/opencode/src/server/routes/instance/httpapi/handlers/session.ts": "b37ff4136c4aea85b23083bf441f03e09820e3e4cdd95e4118ca570d8435b94f", @@ -947,7 +947,7 @@ "packages/ui/src/v2/components/text-shimmer-v2.tsx" ], "exceptions": [], - "extracted_at": "2026-10-12T20:18:15.709Z", + "extracted_at": "2026-10-12T20:42:42.315Z", "per_package": { "packages/app": { "A": 237, diff --git a/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts b/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts index debaa48f9..358e9b80e 100644 --- a/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts +++ b/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts @@ -1,6 +1,6 @@ import { Config } from "@/config/config" import { GlobalBus } from "@/bus/global" -import { GlobalEventReplay, formatID, type Frame } from "@/bus/global-replay" +import { GlobalEventReplay, formatID, type Frame, type Subscription } from "@/bus/global-replay" import { EffectBridge } from "@/effect/bridge" import { EventV2 } from "@opencode-ai/core/event" import { Installation } from "@/installation" @@ -70,6 +70,10 @@ function eventResponse() { // Registration is eager and synchronous: a bus event emitted while the // response body is starting is queued, never lost. const queue = yield* Queue.unbounded() + // Unregister the listener and drop the undelivered frames with the queue. + // Idempotent: it runs at the stream's end and again at the request scope's. + const release = (subscription: Subscription) => + Effect.andThen(Effect.sync(() => subscription.close()), Queue.shutdown(queue)) const subscription = yield* Effect.acquireRelease( Effect.sync(() => { const subscription = GlobalEventReplay.subscribe(cursor) @@ -88,7 +92,7 @@ function eventResponse() { subscription.start((frame) => Queue.offerUnsafe(queue, frameEvent(frame))) return subscription }), - (subscription) => Effect.andThen(Effect.sync(() => subscription.close()), Queue.shutdown(queue)), + release, ) yield* Effect.logInfo("global event connected").pipe( Effect.annotateLogs({ @@ -117,17 +121,17 @@ function eventResponse() { Stream.interruptWhen(Deferred.await(gone)), Stream.pipeThroughChannel(Sse.encode()), Stream.encodeText, + // Release when the stream ends (client gone, or server shutdown), then + // log the disconnect with what remains (read after the release ran). Stream.ensuring( - // Release at the stream's end, not only at the request scope's: the - // listener is unregistered and the undelivered frames are dropped - // with the queue before the disconnect is logged. Effect.suspend(() => { - subscription.close() - const queued = Queue.sizeUnsafe(queue) - return Queue.shutdown(queue).pipe( + const dropped = Queue.sizeUnsafe(queue) + return release(subscription).pipe( Effect.andThen( - Effect.logInfo("global event disconnected").pipe( - Effect.annotateLogs({ subscribers: GlobalEventReplay.subscribers, dropped: queued }), + Effect.suspend(() => + Effect.logInfo("global event disconnected").pipe( + Effect.annotateLogs({ subscribers: GlobalEventReplay.subscribers, dropped }), + ), ), ), ) diff --git a/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts b/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts index 822f84ed9..16116e589 100644 --- a/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts +++ b/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts @@ -1,12 +1,11 @@ // #1776 — the global event stream is sequenced and replayable: every frame // carries an `:` SSE id, a resume cursor replays the gap, an // unservable cursor gets an explicit reset, heartbeats carry time + seq, and a -// client abort releases its subscription. Exercised through a raw node:http -// client against the in-process server (so headers and aborts are exact), on -// the same Bun node:http shim the compiled binary runs on. +// client abort releases its subscription. Exercised with a raw fetch client +// (exact headers, real aborts) against the in-process server, which runs on the +// same Bun node:http shim as the compiled binary. import { NodeHttpServer } from "@effect/platform-node" import { describe, expect } from "bun:test" -import http from "node:http" import { Clock, Context, Effect, Fiber, Layer, Logger, Option, Queue, References, Scope } from "effect" import * as TestClock from "effect/testing/TestClock" import { HttpRouter, HttpServer } from "effect/unstable/http" @@ -60,6 +59,13 @@ const it = testEffect(apiLayer) const logs: { message: string; annotations: Record }[] = [] const logged = (message: string) => logs.filter((line) => line.message === message) +/** Poll until `probe` holds (true), or give up after `ms` (false). */ +const eventually = (probe: () => boolean, ms = 3_000) => + Effect.gen(function* () { + while (!probe()) yield* Effect.sleep("20 millis") + return true + }).pipe(Effect.timeoutOrElse({ duration: ms, orElse: () => Effect.succeed(false) })) + const ID = /^([A-Za-z0-9_-]+):(\d+)$/ type Frame = { id: string | undefined; event: string | undefined; data: any } @@ -83,46 +89,77 @@ function parseFrame(block: string): Frame { return frame } +// The test server starts accepting connections a moment before its request +// handler is attached (the router is served from a forked fiber); a request in +// that window gets an empty 200 instead of a route. Wait until the server +// answers a real route — polled in real time, so TestClock cannot stall it. +const ready = new Set() +const awaitReady = (port: number) => + Effect.promise(async () => { + if (ready.has(port)) return + for (let attempt = 0; attempt < 200; attempt++) { + const response = await fetch(`http://127.0.0.1:${port}${GlobalPaths.health}`).catch(() => undefined) + const json = response?.headers.get("content-type")?.includes("application/json") + await response?.body?.cancel() + if (response?.status === 200 && json) { + ready.add(port) + return + } + await new Promise((resolve) => setTimeout(resolve, 10)) + } + throw new Error(`test server on ${port} never became ready`) + }) + /** Open a raw SSE connection to the in-process server's global stream. */ const connect = (input: { headers?: Record; query?: string } = {}) => Effect.gen(function* () { const server = yield* HttpServer.HttpServer const address = server.address if (address._tag !== "TcpAddress") throw new Error("expected a TCP test server") + yield* awaitReady(address.port) const frames = yield* Queue.unbounded() - const request = yield* Effect.callback((resume) => { - const req = http.get( - { - host: "127.0.0.1", - port: address.port, - path: GlobalPaths.event + (input.query ?? ""), - headers: { accept: "text/event-stream", ...input.headers }, - }, - (res) => { - let buffer = "" - res.setEncoding("utf8") - res.on("data", (chunk: string) => { - buffer += chunk - let end = buffer.indexOf("\n\n") - while (end !== -1) { - const block = buffer.slice(0, end) - buffer = buffer.slice(end + 2) - if (block.trim() !== "") Queue.offerUnsafe(frames, parseFrame(block)) - end = buffer.indexOf("\n\n") - } - }) - res.on("error", () => {}) - resume(Effect.succeed(req)) - }, - ) - req.on("error", (error) => resume(Effect.fail(error))) - }) - yield* Effect.addFinalizer(() => Effect.sync(() => request.destroy())) + const seen: string[] = [] + let ended = "" + const controller = new AbortController() + const response = yield* Effect.promise(() => + fetch(`http://127.0.0.1:${address.port}${GlobalPaths.event}${input.query ?? ""}`, { + headers: { accept: "text/event-stream", ...input.headers }, + signal: controller.signal, + }), + ) + const status = `${response.status} ${response.headers.get("content-type") ?? "-"}` + // Read the body in the background, splitting it into SSE frames. + void (async () => { + const reader = response.body!.pipeThrough(new TextDecoderStream()).getReader() + let buffer = "" + for (;;) { + const chunk = await reader.read() + if (chunk.done) return + buffer += chunk.value + for (let end = buffer.indexOf("\n\n"); end !== -1; end = buffer.indexOf("\n\n")) { + const block = buffer.slice(0, end) + buffer = buffer.slice(end + 2) + if (block.trim() === "") continue + const frame = parseFrame(block) + seen.push(`${frame.id}=${frame.data?.payload?.type}`) + Queue.offerUnsafe(frames, frame) + } + } + })().then( + () => (ended ||= `end (${status})`), + (error) => (ended ||= `${error?.name ?? "error"} (${status})`), + ) + const abort = () => controller.abort() + yield* Effect.addFinalizer(() => Effect.sync(abort)) const next = Queue.take(frames).pipe( - Effect.timeoutOrElse({ duration: "5 seconds", orElse: () => Effect.fail(new Error("timed out waiting for frame")) }), + Effect.timeoutOrElse({ + duration: "4 seconds", + orElse: () => + Effect.fail(new Error(`timed out waiting for frame; seen [${seen.join(", ")}], response ${ended || "open"}`)), + }), ) const take = (n: number) => Effect.forEach(Array.from({ length: n }), () => next) - return { next, take, abort: () => request.destroy() } + return { next, take, abort } }) const emit = (type: string, properties: Record = {}) => @@ -274,6 +311,9 @@ describe("global event stream — heartbeats (#1776)", () => { describe("global event stream — client abort (#1776)", () => { it.live("a client that disconnects mid-stream releases its subscription and logs the disconnect", () => Effect.gen(function* () { + // earlier tests' connections (destroyed at their end) finish releasing first + const settled = () => logged("global event connected").length === logged("global event disconnected").length + expect(yield* eventually(settled)).toBe(true) const baseline = GlobalEventReplay.subscribers const disconnects = logged("global event disconnected").length const a = yield* connect() @@ -285,11 +325,7 @@ describe("global event stream — client abort (#1776)", () => { emit("test.unread") a.abort() - const released = yield* Effect.gen(function* () { - while (GlobalEventReplay.subscribers !== baseline) yield* Effect.sleep("20 millis") - return true - }).pipe(Effect.timeoutOrElse({ duration: "3 seconds", orElse: () => Effect.succeed(false) })) - expect(released).toBe(true) + expect(yield* eventually(() => GlobalEventReplay.subscribers === baseline)).toBe(true) const lines = logged("global event disconnected").slice(disconnects) expect(lines).toHaveLength(1) expect(lines[0].annotations).toMatchObject({ subscribers: baseline }) From bf594cf713657e0ab55b2c1e96133fdd35d2f165 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 17:24:24 -0400 Subject: [PATCH 15/60] =?UTF-8?q?test(engine):=20deterministic=20global-st?= =?UTF-8?q?ream=20harness=20=E2=80=94=20wildcard=20bind=20let=20a=20foreig?= =?UTF-8?q?n=20127.0.0.1=20listener=20take=20the=20port=20(#1776)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NodeHttpServer.layerTest listens on `{ port: 0 }` (the dual-stack wildcard). macOS only skips ephemeral ports already bound on the SAME address, so the wildcard bind can share its port with another process's 127.0.0.1 listener, and a client dialing 127.0.0.1: reaches that process instead. On this host ~296 leaked `fake_serve.mjs` processes (extension test/server_detach.test.ts, `res.end("ok")`) hold 127.0.0.1 ephemeral ports: they answered the probe with 200 / no content-type / content-length 2, and their request logs hold 6000 `/global/health`, 18 `/global/event` and 21 `/global/upgrade` requests. Bind the test server to 127.0.0.1 (the upstream pattern in workspace-proxy.test.ts / control-plane/workspace.test.ts) and dial its formatted address. Drop the readiness probe: the router attaches during layer build (Layer.effectDiscard(server.serve)), and with the fix every one of 360 probes succeeded on its first attempt — the "startup window" was this collision. Re-stamp the manifest for the edited overlay test file. --- packages/app-bundle/manifest.json | 4 +- .../test/server/httpapi-global-event.test.ts | 37 +++++-------------- 2 files changed, 12 insertions(+), 29 deletions(-) diff --git a/packages/app-bundle/manifest.json b/packages/app-bundle/manifest.json index 31453756b..a4fc94773 100644 --- a/packages/app-bundle/manifest.json +++ b/packages/app-bundle/manifest.json @@ -393,7 +393,7 @@ "packages/opencode/test/server/httpapi-compression.test.ts": "5f7603d7018e0b95f1c2f36c01dd297c01770bfcdd91ce29778e03b7d6143f75", "packages/opencode/test/server/httpapi-file-write-gate.test.ts": "0c91b7417e615850edfde3f2878207dab6b6624d4cfbb6e6783b2e6d9aa4dff0", "packages/opencode/test/server/httpapi-file.test.ts": "d3386129bc81467ecbcc57a3861f53767ee56ac41b4ff6ad58d8b168fb3b3373", - "packages/opencode/test/server/httpapi-global-event.test.ts": "357e374d73bfb167869f19cbdd1090d171b0e7ef4e51071247428aaa03c6f7a2", + "packages/opencode/test/server/httpapi-global-event.test.ts": "f7c80f685ea285d7c034baeb3aff3500711bf3605a261bb5ac95900c4200447c", "packages/opencode/test/server/httpapi-instance-route-auth.test.ts": "4b73a89d2431c02bc3b665d93f9002914e51eb1add979dc1867efbcbc0c36e25", "packages/opencode/test/server/httpapi-provider.test.ts": "9e8b3e085b478e3f99b689ffff600e10c0b4b50b3cabdbd749f88dae19bf5904", "packages/opencode/test/server/httpapi-session.test.ts": "eda6fd763ac4c17891cedaa4ccd49597c03e7227fcfdcf0f94a1b4545ba623ed", @@ -947,7 +947,7 @@ "packages/ui/src/v2/components/text-shimmer-v2.tsx" ], "exceptions": [], - "extracted_at": "2026-10-12T20:42:42.315Z", + "extracted_at": "2026-10-12T21:17:20.369Z", "per_package": { "packages/app": { "A": 237, diff --git a/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts b/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts index 16116e589..c4d424606 100644 --- a/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts +++ b/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts @@ -5,6 +5,7 @@ // (exact headers, real aborts) against the in-process server, which runs on the // same Bun node:http shim as the compiled binary. import { NodeHttpServer } from "@effect/platform-node" +import Http from "node:http" import { describe, expect } from "bun:test" import { Clock, Context, Effect, Fiber, Layer, Logger, Option, Queue, References, Scope } from "effect" import * as TestClock from "effect/testing/TestClock" @@ -36,7 +37,11 @@ const apiLayer = HttpRouter.serve( ), { disableListenLog: true, disableLogger: true }, ).pipe( - Layer.provideMerge(NodeHttpServer.layerTest), + // Bound to 127.0.0.1 — the address the raw fetch client dials — not + // `NodeHttpServer.layerTest`'s wildcard. The kernel only skips ephemeral ports + // already bound on the SAME address, so a wildcard bind can share its port + // with another process's 127.0.0.1 listener, which then answers our requests. + Layer.provideMerge(NodeHttpServer.layer(Http.createServer, { host: "127.0.0.1", port: 0 })), Layer.provide(Layer.mock(Auth.Service)({})), Layer.provide(Layer.mock(Config.Service)({})), Layer.provide(Layer.mock(MoveSession.Service)({})), @@ -89,40 +94,18 @@ function parseFrame(block: string): Frame { return frame } -// The test server starts accepting connections a moment before its request -// handler is attached (the router is served from a forked fiber); a request in -// that window gets an empty 200 instead of a route. Wait until the server -// answers a real route — polled in real time, so TestClock cannot stall it. -const ready = new Set() -const awaitReady = (port: number) => - Effect.promise(async () => { - if (ready.has(port)) return - for (let attempt = 0; attempt < 200; attempt++) { - const response = await fetch(`http://127.0.0.1:${port}${GlobalPaths.health}`).catch(() => undefined) - const json = response?.headers.get("content-type")?.includes("application/json") - await response?.body?.cancel() - if (response?.status === 200 && json) { - ready.add(port) - return - } - await new Promise((resolve) => setTimeout(resolve, 10)) - } - throw new Error(`test server on ${port} never became ready`) - }) - /** Open a raw SSE connection to the in-process server's global stream. */ const connect = (input: { headers?: Record; query?: string } = {}) => Effect.gen(function* () { - const server = yield* HttpServer.HttpServer - const address = server.address - if (address._tag !== "TcpAddress") throw new Error("expected a TCP test server") - yield* awaitReady(address.port) + // The router is attached while the layer builds, so the server routes from + // the first request; dial exactly the address it is bound to. + const url = HttpServer.formatAddress((yield* HttpServer.HttpServer).address) const frames = yield* Queue.unbounded() const seen: string[] = [] let ended = "" const controller = new AbortController() const response = yield* Effect.promise(() => - fetch(`http://127.0.0.1:${address.port}${GlobalPaths.event}${input.query ?? ""}`, { + fetch(`${url}${GlobalPaths.event}${input.query ?? ""}`, { headers: { accept: "text/event-stream", ...input.headers }, signal: controller.signal, }), From dea6cd43dba88354b7f92130825db98ead1988be Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 17:54:48 -0400 Subject: [PATCH 16/60] feat(engine): sync copies are opt-in per global-stream subscription; the workspace control plane asks for them (#1778) --- packages/app-bundle/manifest.json | 18 +- .../opencode/src/bus/global-replay.ts | 31 +- .../opencode/src/control-plane/workspace.ts | 970 ++++++++++++++++++ .../instance/httpapi/handlers/global.ts | 16 +- .../workspace-sync-optin.test.ts | 255 +++++ .../test/server/httpapi-global-event.test.ts | 96 +- 6 files changed, 1370 insertions(+), 16 deletions(-) create mode 100644 packages/app-bundle/overlay/packages/opencode/src/control-plane/workspace.ts create mode 100644 packages/app-bundle/overlay/packages/opencode/test/control-plane/workspace-sync-optin.test.ts diff --git a/packages/app-bundle/manifest.json b/packages/app-bundle/manifest.json index a4fc94773..41c60bbc8 100644 --- a/packages/app-bundle/manifest.json +++ b/packages/app-bundle/manifest.json @@ -316,8 +316,9 @@ "packages/llm/src/protocols/bedrock-converse.ts": "834a415fa8ddf9ad1224133f8352d0a926694f19f0954201d182111c171ec03b", "packages/llm/test/provider/bedrock-converse.test.ts": "e8e14d60e8502e928f170f8ac5b406c44bf16d2dd34996ddc2fb7b9261decf85", "packages/opencode/src/agent/agent.ts": "512372a4c62997e9565266638ada19b9dc0a3685b0c0483fd28fede2abdb5c9b", - "packages/opencode/src/bus/global-replay.ts": "bfbdeb214b6ba36d65afa4fdac1dacfa903bebe0c5d88adccfb19a0fbaea25b9", + "packages/opencode/src/bus/global-replay.ts": "3b172a01b322af8282b3451446316834aafae840b9d48e41ef5447986fbaf496", "packages/opencode/src/bus/global.ts": "65b3b380936fc34b2c5ef9f3ab419a6c8e6d4cfc696302a266188f26ccdc320a", + "packages/opencode/src/control-plane/workspace.ts": "cc56fb9a6dc6e0fe88e14237ff8aa29a46ce12971ea69a9d80982e0c58128c13", "packages/opencode/src/effect/runner.ts": "b660b5b22df810b16293c23c6d832e5e8dcfb12b9a8a858c875d2c051c855b81", "packages/opencode/src/mcp/browser.ts": "ecdda42cbd55a97b8d14050aa15c6e6450a1d63b0a641c8c8e5b81e50f957c87", "packages/opencode/src/mcp/index.ts": "da45165e6791a5b30c8962c7fd0ea391bb8e1931cc2357c6b90e9fd090cdeb0d", @@ -362,6 +363,7 @@ "packages/opencode/src/tool/write.ts": "094661a3bdd3fab2cd439c0b6cb942b6a2653d3e9451420523b33acbc762d1a7", "packages/opencode/src/worktree/index.ts": "a86f542b579ef40f2940d36e2c8011ce5592d4811cd5ed437bafe614789fb87e", "packages/opencode/test/acp/service-session.test.ts": "5810c4a0c935f06389c5f01df791e4ae82a4c32268407baf32dc4502d487f282", + "packages/opencode/test/control-plane/workspace-sync-optin.test.ts": "ba35d79f25bf2d9215c60dfba1d34746684fd22b4a6d2870cc7d81de1b57ec2d", "packages/opencode/test/effect/runner-queue.test.ts": "9ce3074dc4a86f620381e2c7ada08b9034b500a62b5da1a7d63896f2519085ca", "packages/opencode/test/effect/runner.test.ts": "6be02ad6a7d173642100a9e17b323410bad7ece842a00018d2170b91d4894c5a", "packages/opencode/test/fixture/fixture.ts": "357bdaa06c6233a91069db20cc5f1cbb2d6eafa3bc7ba3e89b019859d195d6ef", @@ -393,7 +395,7 @@ "packages/opencode/test/server/httpapi-compression.test.ts": "5f7603d7018e0b95f1c2f36c01dd297c01770bfcdd91ce29778e03b7d6143f75", "packages/opencode/test/server/httpapi-file-write-gate.test.ts": "0c91b7417e615850edfde3f2878207dab6b6624d4cfbb6e6783b2e6d9aa4dff0", "packages/opencode/test/server/httpapi-file.test.ts": "d3386129bc81467ecbcc57a3861f53767ee56ac41b4ff6ad58d8b168fb3b3373", - "packages/opencode/test/server/httpapi-global-event.test.ts": "f7c80f685ea285d7c034baeb3aff3500711bf3605a261bb5ac95900c4200447c", + "packages/opencode/test/server/httpapi-global-event.test.ts": "3c51b701e9912f359f992df89afb3a436fcedbe003a41a51ca6cdedb88b8e994", "packages/opencode/test/server/httpapi-instance-route-auth.test.ts": "4b73a89d2431c02bc3b665d93f9002914e51eb1add979dc1867efbcbc0c36e25", "packages/opencode/test/server/httpapi-provider.test.ts": "9e8b3e085b478e3f99b689ffff600e10c0b4b50b3cabdbd749f88dae19bf5904", "packages/opencode/test/server/httpapi-session.test.ts": "eda6fd763ac4c17891cedaa4ccd49597c03e7227fcfdcf0f94a1b4545ba623ed", @@ -928,7 +930,7 @@ "packages/opencode/src/server/routes/instance/httpapi/groups/session.ts": "1421f8c07948b121c7e2005bd9b0fcd6b2c456565ddd1da32d8e6c35d03fac9c", "packages/opencode/src/server/routes/instance/httpapi/handlers/experimental.ts": "69ca3dd48d95157fc822e1f76eb42ac873df50f22616b85194619ca658613867", "packages/opencode/src/server/routes/instance/httpapi/handlers/file.ts": "10bcb79b117beebee04184fb48c1ad789e782e2f7ae4b0c68603edc00f03aee9", - "packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts": "f09578d5d85fbbd63c58ddf73b38daf802602165a7fc920a06b0e344202c70d2", + "packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts": "d0b4e3431b20d52a1e94fc7742cf073a0ca74398cdb7150a49a07bd32d0a563b", "packages/opencode/src/server/routes/instance/httpapi/handlers/mcp.ts": "7da08d58765cdf2b6ce4648fc70de371402949f5bb7b75696f674d43f2d5365b", "packages/opencode/src/server/routes/instance/httpapi/handlers/provider.ts": "8baf16cad0992c5351b61b0c0fb27903bb25b17e1b25fc7c5d77f807c24413a6", "packages/opencode/src/server/routes/instance/httpapi/handlers/session.ts": "b37ff4136c4aea85b23083bf441f03e09820e3e4cdd95e4118ca570d8435b94f", @@ -947,7 +949,7 @@ "packages/ui/src/v2/components/text-shimmer-v2.tsx" ], "exceptions": [], - "extracted_at": "2026-10-12T21:17:20.369Z", + "extracted_at": "2026-10-12T21:54:35.577Z", "per_package": { "packages/app": { "A": 237, @@ -965,8 +967,8 @@ "D": 0 }, "packages/opencode": { - "A": 78, - "M": 91, + "A": 79, + "M": 92, "D": 0 }, "packages/schema": { @@ -996,7 +998,7 @@ } }, "counts": { - "overlay_total": 925, + "overlay_total": 927, "deletions": 7, "server_coupled": 0 }, @@ -1311,6 +1313,7 @@ "packages/opencode/src/agent/agent.ts": "M", "packages/opencode/src/bus/global-replay.ts": "A", "packages/opencode/src/bus/global.ts": "M", + "packages/opencode/src/control-plane/workspace.ts": "M", "packages/opencode/src/effect/runner.ts": "M", "packages/opencode/src/mcp/browser.ts": "M", "packages/opencode/src/mcp/index.ts": "M", @@ -1355,6 +1358,7 @@ "packages/opencode/src/tool/write.ts": "M", "packages/opencode/src/worktree/index.ts": "M", "packages/opencode/test/acp/service-session.test.ts": "M", + "packages/opencode/test/control-plane/workspace-sync-optin.test.ts": "A", "packages/opencode/test/effect/runner-queue.test.ts": "A", "packages/opencode/test/effect/runner.test.ts": "M", "packages/opencode/test/fixture/fixture.ts": "M", diff --git a/packages/app-bundle/overlay/packages/opencode/src/bus/global-replay.ts b/packages/app-bundle/overlay/packages/opencode/src/bus/global-replay.ts index 2eca1e4fe..12bd3e2f3 100644 --- a/packages/app-bundle/overlay/packages/opencode/src/bus/global-replay.ts +++ b/packages/app-bundle/overlay/packages/opencode/src/bus/global-replay.ts @@ -18,6 +18,12 @@ export type Frame = { readonly time: number /** Push order; unlike `seq` it is unique per frame (a sync copy shares its source's seq). */ readonly ordinal: number + /** + * A sync copy sharing the sequence of the frame it mirrors (#1778). Sent only + * to subscribers that request sync copies; dropping it leaves no hole in a + * subscriber's sequence, because its source carries the same one. + */ + readonly syncCopy: boolean } export type Bounds = { @@ -116,7 +122,7 @@ export class ReplayBuffer { return this.#listeners.size } - push(seq: number, data: string): Frame { + push(seq: number, data: string, options: { syncCopy?: boolean } = {}): Frame { const frame: Frame = { id: formatID(this.epoch, seq), seq, @@ -124,6 +130,7 @@ export class ReplayBuffer { bytes: Buffer.byteLength(data), time: this.#now(), ordinal: ++this.#ordinal, + syncCopy: options.syncCopy === true, } this.#current = Math.max(this.#current, seq) this.#frames.push(frame) @@ -142,18 +149,26 @@ export class ReplayBuffer { * Subscribe, optionally resuming after `cursor`. The live listener is * registered BEFORE the buffer is read, and live frames already covered by * the replay are dropped, so nothing published around the reconnect is - * missed or delivered twice. + * missed or delivered twice. Sync copies reach the subscription only when it + * asks for them (`sync`), in its replay and its live frames alike. */ - subscribe(cursor?: string): Subscription { + subscribe(cursor?: string, options: { sync?: boolean } = {}): Subscription { + const wanted = options.sync === true ? () => true : (frame: Frame) => !frame.syncCopy let deliver: ((frame: Frame) => void) | undefined let pending: Frame[] = [] - const listener = (frame: Frame) => (deliver ? deliver(frame) : pending.push(frame)) + const listener = (frame: Frame) => { + if (!wanted(frame)) return + if (deliver) deliver(frame) + else pending.push(frame) + } this.#listeners.add(listener) this.#evict() const watermark = this.#ordinal const resolved = this.#resolve(cursor) - const replay = resolved.reset ? [] : this.#retained().filter((frame) => frame.seq > resolved.position) + const replay = resolved.reset + ? [] + : this.#retained().filter((frame) => frame.seq > resolved.position && wanted(frame)) return { ...resolved, @@ -214,6 +229,10 @@ export const GlobalEventReplay = new ReplayBuffer({ }) GlobalBus.tap((event, seq) => { + // A sync copy whose sequence was already handed out mirrors its source. One + // that consumed a fresh sequence (no source framed nearby) is a frame of its + // own and reaches every subscriber, so filtering never opens a hole. + const syncCopy = event.payload?.type === "sync" && seq <= GlobalEventReplay.current let data: string try { data = JSON.stringify(event) @@ -224,5 +243,5 @@ GlobalBus.tap((event, seq) => { GlobalEventReplay.skip(seq) return } - GlobalEventReplay.push(seq, data) + GlobalEventReplay.push(seq, data, { syncCopy }) }) diff --git a/packages/app-bundle/overlay/packages/opencode/src/control-plane/workspace.ts b/packages/app-bundle/overlay/packages/opencode/src/control-plane/workspace.ts new file mode 100644 index 000000000..b89c5a188 --- /dev/null +++ b/packages/app-bundle/overlay/packages/opencode/src/control-plane/workspace.ts @@ -0,0 +1,970 @@ +import { LayerNode } from "@opencode-ai/core/effect/layer-node" +import { httpClient } from "@opencode-ai/core/effect/app-node-platform" +import { Context, Effect, FiberMap, Iterable, Layer, Schema, Stream } from "effect" +import { serviceUse } from "@opencode-ai/core/effect/service-use" +import { FetchHttpClient, HttpBody, HttpClient, HttpClientError, HttpClientRequest } from "effect/unstable/http" +import { Database } from "@opencode-ai/core/database/database" +import { asc } from "drizzle-orm" +import { eq } from "drizzle-orm" +import { inArray } from "drizzle-orm" +import { Project } from "@/project/project" +import { GlobalBus } from "@/bus/global" +import { Auth } from "@/auth" +import { EventV2 } from "@opencode-ai/core/event" +import { EventV2Bridge } from "@/event-v2-bridge" +import { EventSequenceTable, EventTable } from "@opencode-ai/core/event/sql" +import { FSUtil } from "@opencode-ai/core/fs-util" +import { RuntimeFlags } from "@/effect/runtime-flags" +import { ProjectV2 } from "@opencode-ai/core/project" +import { Slug } from "@opencode-ai/core/util/slug" +import { WorkspaceTable } from "@opencode-ai/core/control-plane/workspace.sql" +import { getAdapter, registeredAdapters } from "./adapters" +import { type Target, type WorkspaceInfo, WorkspaceInfo as WorkspaceInfoSchema } from "./types" +import { WorkspaceV2 } from "@opencode-ai/core/workspace" +import { Session } from "@/session/session" +import { SessionPrompt } from "@/session/prompt" +import { SessionTable } from "@opencode-ai/core/session/sql" +import { SessionID } from "@/session/schema" +import { NotFoundError } from "@/storage/storage" +import { errorData } from "@/util/error" +import { waitEvent } from "./util" +import { WorkspaceRef } from "@/effect/instance-ref" +import { Vcs } from "@/project/vcs" +import { InstanceStore } from "@/project/instance-store" +import { WorkspaceAdapterRuntime } from "./workspace-adapter-runtime" +import { AppNodeBuilderV1 } from "@/effect/app-node-builder-v1" +import { WorkspaceEvent } from "@opencode-ai/schema/workspace-event" + +export const Info = Schema.Struct({ + ...WorkspaceInfoSchema.fields, + timeUsed: Schema.Number, +}).annotate({ identifier: "Workspace" }) +export type Info = WorkspaceInfo & { timeUsed: number } + +export const ConnectionStatus = WorkspaceEvent.ConnectionStatus +export type ConnectionStatus = WorkspaceEvent.ConnectionStatus + +export const Event = WorkspaceEvent + +function fromRow(row: typeof WorkspaceTable.$inferSelect): Info { + return { + id: row.id, + type: row.type, + branch: row.branch, + name: row.name, + directory: row.directory, + extra: row.extra, + projectID: row.project_id, + timeUsed: row.time_used, + } +} + +export const CreateInput = Schema.Struct({ + id: Schema.optional(WorkspaceV2.ID), + type: Info.fields.type, + branch: Info.fields.branch, + projectID: ProjectV2.ID, + extra: Schema.optional(Info.fields.extra), +}) +export type CreateInput = Schema.Schema.Type + +export const SessionWarpInput = Schema.Struct({ + workspaceID: Schema.NullOr(WorkspaceV2.ID), + sessionID: SessionID, + copyChanges: Schema.optional(Schema.Boolean), +}) +export type SessionWarpInput = Schema.Schema.Type + +export class SyncHttpError extends Schema.TaggedErrorClass()("WorkspaceSyncHttpError", { + message: Schema.String, + status: Schema.Number, + body: Schema.optional(Schema.String), +}) {} + +export class WorkspaceNotFoundError extends Schema.TaggedErrorClass()( + "WorkspaceNotFoundError", + { + message: Schema.String, + workspaceID: WorkspaceV2.ID, + }, +) {} + +export class SessionEventsNotFoundError extends Schema.TaggedErrorClass()( + "WorkspaceSessionEventsNotFoundError", + { + message: Schema.String, + sessionID: SessionID, + }, +) {} + +export class SessionWarpHttpError extends Schema.TaggedErrorClass()( + "WorkspaceSessionWarpHttpError", + { + message: Schema.String, + workspaceID: WorkspaceV2.ID, + sessionID: SessionID, + status: Schema.Number, + body: Schema.String, + }, +) {} + +export class SyncTimeoutError extends Schema.TaggedErrorClass()("WorkspaceSyncTimeoutError", { + message: Schema.String, + state: Schema.Record(Schema.String, Schema.Number), +}) {} + +export class SyncAbortedError extends Schema.TaggedErrorClass()("WorkspaceSyncAbortedError", { + message: Schema.String, + cause: Schema.optional(Schema.Defect()), +}) {} + +type CreateError = Auth.AuthError +type SessionWarpError = + | WorkspaceNotFoundError + | SessionEventsNotFoundError + | SessionWarpHttpError + | Vcs.PatchApplyError + | HttpClientError.HttpClientError +type WaitForSyncError = SyncTimeoutError | SyncAbortedError +type SyncLoopError = SyncHttpError | HttpClientError.HttpClientError + +export interface Interface { + readonly create: (input: CreateInput) => Effect.Effect + readonly sessionWarp: (input: SessionWarpInput) => Effect.Effect + readonly list: (project: Project.Info) => Effect.Effect + readonly syncList: (project: Project.Info) => Effect.Effect + readonly get: (id: WorkspaceV2.ID) => Effect.Effect + readonly remove: (id: WorkspaceV2.ID) => Effect.Effect + readonly status: () => Effect.Effect + readonly isSyncing: (workspaceID: WorkspaceV2.ID) => Effect.Effect + readonly waitForSync: ( + workspaceID: WorkspaceV2.ID, + state: Record, + signal?: AbortSignal, + timeout?: number, + ) => Effect.Effect + readonly startWorkspaceSyncing: (projectID: ProjectV2.ID) => Effect.Effect +} + +export class Service extends Context.Service()("@opencode/Workspace") {} + +export const use = serviceUse(Service) + +const layer = Layer.effect( + Service, + Effect.gen(function* () { + const auth = yield* Auth.Service + const session = yield* Session.Service + const prompt = yield* SessionPrompt.Service + const http = yield* HttpClient.HttpClient + const events = yield* EventV2Bridge.Service + const vcs = yield* Vcs.Service + const flags = yield* RuntimeFlags.Service + const fs = yield* FSUtil.Service + const { db } = yield* Database.Service + const connections = new Map() + const syncFibers = yield* FiberMap.make() + + const setStatus = (id: WorkspaceV2.ID, status: ConnectionStatus["status"]) => { + const prev = connections.get(id) + if (prev?.status === status) return + const next = { workspaceID: id, status } + connections.set(id, next) + + GlobalBus.emit("event", { + directory: "global", + workspace: id, + payload: { + type: Event.Status.type, + properties: next, + }, + }) + } + + const connectSSE = Effect.fn("Workspace.connectSSE")(function* ( + url: URL | string, + headers: HeadersInit | undefined, + ) { + // Sync copies are opt-in per subscription (#1778, amicode overlay): this + // loop replays them into the local store, so it asks for them. + const request = new Headers(headers) + request.set("x-opencode-sync-copies", "true") + const response = yield* http.execute( + HttpClientRequest.get(route(url, "/global/event"), { + headers: request, + accept: "text/event-stream", + }), + ) + if (response.status < 200 || response.status >= 300) { + return yield* new SyncHttpError({ + message: `Workspace sync HTTP failure: ${response.status}`, + status: response.status, + }) + } + return response.stream + }) + + const parseSSE = Effect.fn("Workspace.parseSSE")(function* ( + stream: Stream.Stream, + onEvent: (event: unknown) => Effect.Effect, + ) { + yield* stream.pipe( + Stream.decodeText(), + Stream.splitLines, + Stream.mapAccum( + () => ({ data: [] as string[], id: undefined as string | undefined, retry: 1000 }), + (state, line) => { + if (line === "") { + if (!state.data.length) return [state, []] + return [{ ...state, data: [] }, [{ data: state.data.join("\n"), id: state.id, retry: state.retry }]] + } + + const index = line.indexOf(":") + const field = index === -1 ? line : line.slice(0, index) + const value = index === -1 ? "" : line.slice(index + (line[index + 1] === " " ? 2 : 1)) + + if (field === "data") return [{ ...state, data: [...state.data, value] }, []] + if (field === "id") return [{ ...state, id: value }, []] + if (field === "retry") { + const retry = Number.parseInt(value, 10) + return [Number.isNaN(retry) ? state : { ...state, retry }, []] + } + return [state, []] + }, + { + onHalt: (state) => + state.data.length ? [{ data: state.data.join("\n"), id: state.id, retry: state.retry }] : [], + }, + ), + Stream.map((event) => { + try { + return JSON.parse(event.data) as unknown + } catch { + return { + type: "sse.message", + properties: { + data: event.data, + id: event.id || undefined, + retry: event.retry, + }, + } + } + }), + Stream.runForEach(onEvent), + ) + }) + + const runInWorkspace = (input: { + workspaceID?: WorkspaceV2.ID + local: () => Effect.Effect + remote: (input: { + workspace: Info + target: Extract + }) => HttpClientRequest.HttpClientRequest + fallback: A + response?: "json" | "text" + }) => + Effect.gen(function* () { + if (!input.workspaceID) return yield* input.local() + + const workspace = yield* get(input.workspaceID) + if (!workspace) return input.fallback + + const target = yield* WorkspaceAdapterRuntime.target(workspace) + + if (target.type === "local") { + const store = yield* InstanceStore.Service + return yield* store.provide({ directory: target.directory }, input.local()) + } + + const response = yield* http.execute(input.remote({ workspace, target })).pipe( + Effect.catch((error) => + Effect.logWarning("workspace target request failed", { + workspaceID: workspace.id, + error: errorData(error), + }).pipe(Effect.as(undefined)), + ), + ) + if (!response) return input.fallback + if (response.status < 200 || response.status >= 300) { + const body = yield* response.text.pipe(Effect.catch(() => Effect.succeed(""))) + yield* Effect.logWarning("workspace target request failed", { + workspaceID: workspace.id, + status: response.status, + body, + }) + return input.fallback + } + + const body = input.response === "text" ? response.text : response.json + return yield* body.pipe( + Effect.map((result) => result as A), + Effect.catch((error) => + Effect.logWarning("workspace target response decode failed", { + workspaceID: workspace.id, + error: errorData(error), + }).pipe(Effect.as(input.fallback)), + ), + ) + }) + + const syncHistory = Effect.fn("Workspace.syncHistory")(function* ( + space: Info, + url: URL | string, + headers: HeadersInit | undefined, + ) { + const sessionIDs = (yield* db + .select({ id: SessionTable.id }) + .from(SessionTable) + .where(eq(SessionTable.workspace_id, space.id)) + .all() + .pipe(Effect.orDie)).map((row) => row.id) + const state = sessionIDs.length + ? Object.fromEntries( + (yield* db + .select() + .from(EventSequenceTable) + .where(inArray(EventSequenceTable.aggregate_id, sessionIDs)) + .all() + .pipe(Effect.orDie)).map((row) => [row.aggregate_id, row.seq]), + ) + : {} + + const response = yield* http.execute( + HttpClientRequest.post(route(url, "/sync/history"), { + headers: new Headers(headers), + body: HttpBody.jsonUnsafe(state), + }), + ) + + if (response.status < 200 || response.status >= 300) { + const body = yield* response.text + return yield* new SyncHttpError({ + message: `Workspace history HTTP failure: ${response.status} ${body}`, + status: response.status, + body, + }) + } + + const history = (yield* response.json) as HistoryEvent[] + + yield* Effect.forEach( + history, + (event) => + events + .replay( + { + id: EventV2.ID.make(event.id), + aggregateID: event.aggregate_id, + seq: event.seq, + type: event.type, + data: event.data, + }, + { publish: true, ownerID: space.id }, + ) + .pipe(Effect.provideService(WorkspaceRef, space.id)), + { discard: true }, + ) + }) + + const syncWorkspaceLoop = Effect.fn("Workspace.syncWorkspaceLoop")(function* (space: Info) { + const target = yield* WorkspaceAdapterRuntime.target(space) + + if (target.type === "local") return + + let attempt = 0 + + while (true) { + setStatus(space.id, "connecting") + + const stream = yield* connectSSE(target.url, target.headers).pipe( + Effect.tap(() => syncHistory(space, target.url, target.headers)), + Effect.catch((err) => + Effect.gen(function* () { + setStatus(space.id, "error") + yield* Effect.logWarning("failed to connect to global sync", { + workspace: space.name, + error: errorData(err), + }) + return null + }), + ), + ) + + if (stream) { + attempt = 0 + + setStatus(space.id, "connected") + + yield* parseSSE(stream, (evt) => + Effect.gen(function* () { + if (!evt || typeof evt !== "object" || !("payload" in evt)) return + const payload = evt.payload as { type?: string; syncEvent?: EventV2.SerializedEvent } + if (payload.type === "server.heartbeat") return + + if (payload.type === "sync" && payload.syncEvent) { + const failed = yield* events.replay(payload.syncEvent, { publish: true, ownerID: space.id }).pipe( + Effect.as(false), + Effect.catchCause((error) => + Effect.logWarning("failed to replay global event", error).pipe( + Effect.annotateLogs({ workspaceID: space.id }), + Effect.as(true), + ), + ), + ) + if (failed) return + } + + try { + const event = evt as { directory?: string; project?: string; payload: unknown } + GlobalBus.emit("event", { + directory: event.directory, + project: event.project, + workspace: space.id, + payload: event.payload, + }) + } catch (error) { + yield* Effect.logWarning("failed to emit global event", { + workspaceID: space.id, + error: errorData(error), + }) + } + }), + ) + + setStatus(space.id, "disconnected") + } + + // Back off reconnect attempts up to 2 minutes while the workspace + // stays unavailable. + yield* Effect.sleep(`${Math.min(120_000, 1_000 * 2 ** attempt)} millis`) + attempt += 1 + } + }) + + const startSync = Effect.fn("Workspace.startSync")(function* (space: Info) { + if (!flags.experimentalWorkspaces) return + + const target = yield* WorkspaceAdapterRuntime.target(space).pipe( + Effect.catch((error) => + Effect.gen(function* () { + setStatus(space.id, "error") + yield* Effect.logWarning("workspace target failed", { + workspaceID: space.id, + error: errorData(error), + }) + return null + }), + ), + ) + if (!target) return + + if (target.type === "local") { + setStatus(space.id, (yield* fs.existsSafe(target.directory)) ? "connected" : "error") + return + } + + const exists = yield* FiberMap.has(syncFibers, space.id) + if (exists && connections.get(space.id)?.status !== "error") return + + setStatus(space.id, "disconnected") + + yield* FiberMap.run( + syncFibers, + space.id, + // TODO: look into `tapError` to set the status but still + // allow the fiber to fail and automatically get removed + syncWorkspaceLoop(space).pipe( + Effect.catch((error) => + Effect.gen(function* () { + setStatus(space.id, "error") + yield* Effect.logWarning("workspace listener failed", { + workspaceID: space.id, + error: errorData(error), + }) + }), + ), + ), + ) + }) + + const stopSync = Effect.fn("Workspace.stopSync")(function* (id: WorkspaceV2.ID) { + yield* FiberMap.remove(syncFibers, id) + connections.delete(id) + }) + + const create = Effect.fn("Workspace.create")(function* (input: CreateInput) { + const id = WorkspaceV2.ID.ascending(input.id) + const adapter = getAdapter(input.projectID, input.type) + const config = yield* WorkspaceAdapterRuntime.configure(adapter, { + ...input, + id, + name: Slug.create(), + directory: null, + extra: input.extra ?? null, + }) + + const info: Info = { + id, + type: config.type, + branch: config.branch ?? null, + name: config.name ?? null, + directory: config.directory ?? null, + extra: config.extra ?? null, + projectID: input.projectID, + timeUsed: Date.now(), + } + + yield* db + .insert(WorkspaceTable) + .values({ + id: info.id, + type: info.type, + branch: info.branch, + name: info.name, + directory: info.directory, + extra: info.extra, + project_id: info.projectID, + time_used: info.timeUsed, + }) + .run() + .pipe(Effect.orDie) + + const env = { + OPENCODE_AUTH_CONTENT: JSON.stringify(yield* auth.all()), + OPENCODE_WORKSPACE_ID: config.id, + OPENCODE_EXPERIMENTAL_WORKSPACES: "true", + OTEL_EXPORTER_OTLP_HEADERS: process.env.OTEL_EXPORTER_OTLP_HEADERS, + OTEL_EXPORTER_OTLP_ENDPOINT: process.env.OTEL_EXPORTER_OTLP_ENDPOINT, + OTEL_RESOURCE_ATTRIBUTES: process.env.OTEL_RESOURCE_ATTRIBUTES, + } + + yield* WorkspaceAdapterRuntime.create(adapter, config, env) + yield* Effect.all( + [ + waitEvent({ + timeout: TIMEOUT, + fn(event) { + if (event.workspace === info.id && event.payload.type === Event.Status.type) { + const { status } = event.payload.properties + return status === "error" || status === "connected" + } + return false + }, + }), + startSync(info), + ], + { concurrency: 2, discard: true }, + ) + + return info + }) + + const sessionWarp = Effect.fn("Workspace.sessionWarp")(function* (input: SessionWarpInput) { + return yield* Effect.gen(function* () { + const current = yield* db + .select({ workspaceID: SessionTable.workspace_id }) + .from(SessionTable) + .where(eq(SessionTable.id, input.sessionID)) + .get() + .pipe(Effect.orDie) + + if (current?.workspaceID) { + const previous = yield* get(current.workspaceID) + if (previous) { + const target = yield* WorkspaceAdapterRuntime.target(previous) + + if (target.type === "remote") { + yield* syncHistory(previous, target.url, target.headers).pipe( + Effect.catch((error) => + Effect.logWarning("session warp final source sync failed", { + workspaceID: previous.id, + sessionID: input.sessionID, + error: errorData(error), + }), + ), + ) + } else { + yield* prompt.cancel(input.sessionID) + } + + // "claim" this session so any future events coming from + // the old workspace are ignored + yield* events.claim(input.sessionID, input.workspaceID ?? previous.projectID) + } + } + + const sourcePatch = + input.copyChanges && current?.workspaceID + ? yield* runInWorkspace({ + workspaceID: current?.workspaceID ?? undefined, + local: () => vcs.diffRaw(), + remote: ({ target }) => + HttpClientRequest.get(route(target.url, "/vcs/diff/raw"), { + headers: new Headers(target.headers), + }), + fallback: "", + response: "text", + }).pipe(Effect.provide(AppNodeBuilderV1.build(InstanceStore.node))) + : "" + + if (sourcePatch) { + // Attempt to apply the file changes to the new workspace. + // We intentionally do first so if it fails we don't warp + // the session. + yield* runInWorkspace({ + workspaceID: input.workspaceID ?? undefined, + local: () => vcs.apply({ patch: sourcePatch }), + remote: ({ target }) => + HttpClientRequest.post(route(target.url, "/vcs/apply"), { + headers: new Headers(target.headers), + body: HttpBody.jsonUnsafe({ patch: sourcePatch }), + }), + fallback: { applied: false }, + }).pipe(Effect.provide(AppNodeBuilderV1.build(InstanceStore.node))) + } + + if (input.workspaceID === null) { + yield* session.setWorkspace({ sessionID: input.sessionID, workspaceID: undefined }) + + return + } + + const workspaceID = input.workspaceID + const space = yield* get(workspaceID) + if (!space) + return yield* new WorkspaceNotFoundError({ + message: `Workspace not found: ${workspaceID}`, + workspaceID, + }) + + const target = yield* WorkspaceAdapterRuntime.target(space) + + if (target.type === "local") { + yield* session.setWorkspace({ sessionID: input.sessionID, workspaceID: input.workspaceID }) + + return + } + + const rows = yield* db + .select({ + id: EventTable.id, + aggregateID: EventTable.aggregate_id, + seq: EventTable.seq, + type: EventTable.type, + data: EventTable.data, + }) + .from(EventTable) + .where(eq(EventTable.aggregate_id, input.sessionID)) + .orderBy(asc(EventTable.seq)) + .all() + .pipe(Effect.orDie) + if (rows.length === 0) + return yield* new SessionEventsNotFoundError({ + message: `No events found for session: ${input.sessionID}`, + sessionID: input.sessionID, + }) + + const batches = Iterable.chunksOf(rows, 10) + const total = Iterable.size(batches) + + yield* Effect.forEach( + batches, + (events, i) => + Effect.gen(function* () { + const response = yield* http.execute( + HttpClientRequest.post(route(target.url, "/sync/replay"), { + headers: new Headers(target.headers), + body: HttpBody.jsonUnsafe({ + directory: space.directory ?? "", + events, + }), + }), + ) + + if (response.status < 200 || response.status >= 300) { + const body = yield* response.text + return yield* new SessionWarpHttpError({ + message: `Failed to warp session ${input.sessionID} into workspace ${workspaceID}: HTTP ${response.status} ${body}`, + workspaceID, + sessionID: input.sessionID, + status: response.status, + body, + }) + } + }), + { discard: true }, + ) + + const response = yield* http.execute( + HttpClientRequest.post(route(target.url, "/sync/steal"), { + headers: new Headers(target.headers), + body: HttpBody.jsonUnsafe({ sessionID: input.sessionID }), + }), + ) + if (response.status < 200 || response.status >= 300) { + const body = yield* response.text + return yield* new SessionWarpHttpError({ + message: `Failed to steal session ${input.sessionID} into workspace ${workspaceID}: HTTP ${response.status} ${body}`, + workspaceID, + sessionID: input.sessionID, + status: response.status, + body, + }) + } + + yield* session.setWorkspace({ sessionID: input.sessionID, workspaceID: input.workspaceID }) + }) + }) + + const list = Effect.fn("Workspace.list")(function* (project: Project.Info) { + if (!flags.experimentalWorkspaces) return [] + return (yield* db + .select() + .from(WorkspaceTable) + .where(eq(WorkspaceTable.project_id, project.id)) + .all() + .pipe(Effect.orDie)) + .map(fromRow) + .sort((a, b) => a.id.localeCompare(b.id)) + }) + + const syncList = Effect.fn("Workspace.syncList")(function* (project: Project.Info) { + const names = new Set((yield* list(project)).map((workspace) => workspace.name)) + const discovered = yield* Effect.forEach( + registeredAdapters(project.id), + ([type, adapter]) => + WorkspaceAdapterRuntime.list(adapter).pipe( + Effect.catchCause((error) => + Effect.logWarning("workspace adapter list failed", { type, error }).pipe(Effect.as([])), + ), + ), + { concurrency: "unbounded" }, + ).pipe(Effect.map((items) => items.flat())) + + yield* Effect.forEach( + discovered, + (item) => + Effect.gen(function* () { + if (names.has(item.name)) return + names.add(item.name) + + const info: Info = { + id: WorkspaceV2.ID.ascending(), + type: item.type, + branch: item.branch, + name: item.name, + directory: item.directory, + extra: item.extra, + projectID: item.projectID, + timeUsed: Date.now(), + } + + yield* db + .insert(WorkspaceTable) + .values({ + id: info.id, + type: info.type, + branch: info.branch, + name: info.name, + directory: info.directory, + extra: info.extra, + project_id: info.projectID, + time_used: info.timeUsed, + }) + .run() + .pipe(Effect.orDie) + + yield* startSync(info) + }), + { concurrency: 1 }, + ) + }) + + const get = Effect.fn("Workspace.get")(function* (id: WorkspaceV2.ID) { + const row = yield* db.select().from(WorkspaceTable).where(eq(WorkspaceTable.id, id)).get().pipe(Effect.orDie) + if (!row) return + return fromRow(row) + }) + + const remove = Effect.fn("Workspace.remove")(function* (id: WorkspaceV2.ID) { + const sessions = yield* db + .select({ id: SessionTable.id, parentID: SessionTable.parent_id }) + .from(SessionTable) + .where(eq(SessionTable.workspace_id, id)) + .all() + .pipe(Effect.orDie) + const sessionIDs = new Set(sessions.map((sessionInfo) => sessionInfo.id)) + yield* Effect.forEach( + sessions.filter((sessionInfo) => !sessionInfo.parentID || !sessionIDs.has(sessionInfo.parentID)), + (sessionInfo) => + session.remove(sessionInfo.id).pipe(Effect.catchIf(NotFoundError.isInstance, () => Effect.void)), + { discard: true }, + ) + + const row = yield* db.select().from(WorkspaceTable).where(eq(WorkspaceTable.id, id)).get().pipe(Effect.orDie) + if (!row) return + + yield* stopSync(id) + + const info = fromRow(row) + yield* Effect.catchCause( + Effect.gen(function* () { + yield* WorkspaceAdapterRuntime.remove(info) + }), + () => Effect.logError("adapter not available when removing workspace", { type: row.type }), + ) + + yield* db.delete(WorkspaceTable).where(eq(WorkspaceTable.id, id)).run().pipe(Effect.orDie) + return info + }) + + const status = Effect.fn("Workspace.status")(function* () { + return [...connections.values()] + }) + + const isSyncing = Effect.fn("Workspace.isSyncing")(function* (workspaceID: WorkspaceV2.ID) { + const exists = yield* FiberMap.has(syncFibers, workspaceID) + return exists && connections.get(workspaceID)?.status !== "error" + }) + + const waitForSync = Effect.fn("Workspace.waitForSync")(function* ( + workspaceID: WorkspaceV2.ID, + state: Record, + signal?: AbortSignal, + timeout = TIMEOUT, + ) { + if (yield* synced(db, state)) return + + yield* Effect.catch( + waitUntilSynced({ db, workspaceID, state, signal, timeout }), + (): Effect.Effect => + signal?.aborted + ? Effect.fail( + new SyncAbortedError({ + message: signal.reason instanceof Error ? signal.reason.message : "Request aborted", + cause: signal.reason, + }), + ) + : Effect.fail( + new SyncTimeoutError({ + message: `Timed out waiting for sync fence: ${JSON.stringify(state)}`, + state, + }), + ), + ) + }) + + const startWorkspaceSyncing = Effect.fn("Workspace.startWorkspaceSyncing")(function* (projectID: ProjectV2.ID) { + if (!flags.experimentalWorkspaces) return + const rows = yield* db + .selectDistinct({ workspace: WorkspaceTable }) + .from(WorkspaceTable) + .where(eq(WorkspaceTable.project_id, projectID)) + .all() + .pipe(Effect.orDie) + + for (const { workspace } of rows) { + yield* startSync(fromRow(workspace)).pipe( + Effect.catch((error) => + Effect.sync(() => { + setStatus(workspace.id, "error") + }), + ), + Effect.forkDetach, + ) + } + }) + + return Service.of({ + create, + sessionWarp, + list, + syncList, + get, + remove, + status, + isSyncing, + waitForSync, + startWorkspaceSyncing, + }) + }), +) + +const TIMEOUT = 5000 + +type HistoryEvent = { + id: string + aggregate_id: string + seq: number + type: string + data: Record +} + +function waitUntilSynced(input: { + db: Database.Interface["db"] + workspaceID: WorkspaceV2.ID + state: Record + signal?: AbortSignal + timeout: number +}): Effect.Effect { + return Effect.suspend(() => + waitEvent({ + timeout: input.timeout, + signal: input.signal, + fn(event) { + return event.workspace === input.workspaceID || event.payload.type === "sync" + }, + }).pipe( + Effect.andThen(synced(input.db, input.state)), + Effect.flatMap((done): Effect.Effect => (done ? Effect.void : waitUntilSynced(input))), + ), + ) +} + +function synced(db: Database.Interface["db"], state: Record): Effect.Effect { + const ids = Object.keys(state) + if (ids.length === 0) return Effect.succeed(true) + + return db + .select({ + id: EventSequenceTable.aggregate_id, + seq: EventSequenceTable.seq, + }) + .from(EventSequenceTable) + .where(inArray(EventSequenceTable.aggregate_id, ids)) + .all() + .pipe( + Effect.orDie, + Effect.map((rows) => { + const done = Object.fromEntries(rows.map((row) => [row.id, row.seq])) as Record + return ids.every((id) => (done[id] ?? -1) >= state[id]) + }), + ) +} + +function route(url: string | URL, path: string) { + const next = new URL(url) + next.pathname = `${next.pathname.replace(/\/$/, "")}${path}` + next.search = "" + next.hash = "" + return next +} + +export const node = LayerNode.make({ + service: Service, + layer: layer, + deps: [ + Auth.node, + Session.node, + SessionPrompt.node, + httpClient, + EventV2Bridge.node, + Vcs.node, + RuntimeFlags.node, + FSUtil.node, + Database.node, + ], +}) + +export * as Workspace from "./workspace" diff --git a/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts b/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts index 358e9b80e..d905bd7f7 100644 --- a/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts +++ b/packages/app-bundle/overlay/packages/opencode/src/server/routes/instance/httpapi/handlers/global.ts @@ -24,6 +24,18 @@ function cursorOf(request: HttpServerRequest.HttpServerRequest) { return new URL(request.url, "http://localhost").searchParams.get("lastEventID")?.trim() || undefined } +// Sync copies (the event-v2 bridge's `type: "sync"` twin of every durable +// event) are opt-in per subscription (#1778): only a subscriber that replays +// them (the workspace control plane, control-plane/workspace.ts) asks — with +// the `x-opencode-sync-copies` header, else the `sync` query parameter; every +// other subscriber discards them, so they stay off its wire. +function syncRequested(request: HttpServerRequest.HttpServerRequest) { + const raw = + request.headers["x-opencode-sync-copies"] ?? new URL(request.url, "http://localhost").searchParams.get("sync") + const value = raw?.trim().toLowerCase() + return value === "true" || value === "1" +} + function frameEvent(frame: Pick): Sse.Event { return { _tag: "Event", event: "message", id: frame.id, data: frame.data } } @@ -66,6 +78,7 @@ function eventResponse() { return Effect.gen(function* () { const request = yield* HttpServerRequest.HttpServerRequest const cursor = cursorOf(request) + const sync = syncRequested(request) const gone = yield* clientGone(request) // Registration is eager and synchronous: a bus event emitted while the // response body is starting is queued, never lost. @@ -76,7 +89,7 @@ function eventResponse() { Effect.andThen(Effect.sync(() => subscription.close()), Queue.shutdown(queue)) const subscription = yield* Effect.acquireRelease( Effect.sync(() => { - const subscription = GlobalEventReplay.subscribe(cursor) + const subscription = GlobalEventReplay.subscribe(cursor, { sync }) // An unservable cursor is never silent: a reset frame comes first, // carrying the current epoch and sequence, and the client resyncs. if (subscription.reset) @@ -99,6 +112,7 @@ function eventResponse() { subscribers: GlobalEventReplay.subscribers, resume: subscription.reset ? `reset:${subscription.reset}` : cursor ? "replay" : "live", replayed: subscription.replay.length, + sync, }), ) // The heartbeat enters the connection's own queue, behind every frame up to diff --git a/packages/app-bundle/overlay/packages/opencode/test/control-plane/workspace-sync-optin.test.ts b/packages/app-bundle/overlay/packages/opencode/test/control-plane/workspace-sync-optin.test.ts new file mode 100644 index 000000000..34074ab41 --- /dev/null +++ b/packages/app-bundle/overlay/packages/opencode/test/control-plane/workspace-sync-optin.test.ts @@ -0,0 +1,255 @@ +// #1778 — sync copies are opt-in per global-stream subscription. The workspace +// control plane is the one subscriber that replays them, so it must ask for +// them; the remote here behaves like an engine with the opt-in filter (engine- +// shaped frames, `id: :`, sync copies only for `?sync=true`). +// Harness trimmed from workspace.test.ts (upstream) — same layers and helpers. +import { afterEach, beforeEach, describe, expect, mock } from "bun:test" +import Http from "node:http" +import { NodeHttpServer } from "@effect/platform-node" +import { Effect, Layer } from "effect" +import { HttpServer, HttpServerRequest, HttpServerResponse } from "effect/unstable/http" +import { eq } from "drizzle-orm" +import { GlobalBus, type GlobalEvent } from "@/bus/global" +import { Database } from "@opencode-ai/core/database/database" +import { ProjectV2 } from "@opencode-ai/core/project" +import { Session as SessionNs } from "@/session/session" +import { SessionID } from "@/session/schema" +import { SessionTable } from "@opencode-ai/core/session/sql" +import { SessionProjector } from "@opencode-ai/core/session/projector" +import { EventSequenceTable } from "@opencode-ai/core/event/sql" +import { resetDatabase } from "../fixture/db" +import { disposeAllInstances, provideTmpdirInstance, requireInstance } from "../fixture/fixture" +import { testEffect } from "../lib/effect" +import { registerAdapter } from "../../src/control-plane/adapters" +import { WorkspaceV2 } from "@opencode-ai/core/workspace" +import { WorkspaceTable } from "@opencode-ai/core/control-plane/workspace.sql" +import type { WorkspaceAdapter } from "../../src/control-plane/types" +import * as Workspace from "../../src/control-plane/workspace" +import { InstanceStore } from "@/project/instance-store" +import { InstanceBootstrap } from "@/project/bootstrap" +import { RuntimeFlags } from "@/effect/runtime-flags" +import { Ripgrep } from "@opencode-ai/core/ripgrep" +import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder" +import { LayerNode } from "@opencode-ai/core/effect/layer-node" + +const originalWorkspacesFlag = process.env.OPENCODE_EXPERIMENTAL_WORKSPACES + +const it = testEffect( + Layer.mergeAll( + NodeHttpServer.layer(Http.createServer, { host: "127.0.0.1", port: 0 }), + AppNodeBuilder.build( + LayerNode.group([ + Workspace.node, + SessionNs.node, + SessionProjector.node, + Database.node, + InstanceStore.node, + Ripgrep.node, + ]), + [ + [RuntimeFlags.node, RuntimeFlags.layer({ experimentalWorkspaces: true })], + [ + InstanceStore.bootstrapNode, + Layer.succeed(InstanceBootstrap.Service, InstanceBootstrap.Service.of({ run: Effect.void })), + ], + ], + ), + ), +) + +beforeEach(() => { + process.env.OPENCODE_EXPERIMENTAL_WORKSPACES = "true" +}) + +afterEach(async () => { + mock.restore() + await disposeAllInstances() + if (originalWorkspacesFlag === undefined) delete process.env.OPENCODE_EXPERIMENTAL_WORKSPACES + else process.env.OPENCODE_EXPERIMENTAL_WORKSPACES = originalWorkspacesFlag + await resetDatabase() +}) + +const unique = (prefix: string) => `${prefix}-${Math.random().toString(36).slice(2)}` + +function remoteAdapter(url: string): WorkspaceAdapter { + return { + name: "remote", + description: "remote", + configure: (info) => info, + async create() {}, + async remove() {}, + target: () => ({ type: "remote", url }), + } +} + +/** An engine-shaped global stream: `id: :` + `data:` per frame, then end. */ +function engineStream(frames: { seq: number; event: unknown }[]) { + const encoder = new TextEncoder() + return new Response( + new ReadableStream({ + start(controller) { + for (const frame of frames) + controller.enqueue(encoder.encode(`id: ep0ch:${frame.seq}\ndata: ${JSON.stringify(frame.event)}\n\n`)) + controller.close() + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } }, + ) +} + +function captureGlobalEvents() { + const events: GlobalEvent[] = [] + const handler = (event: GlobalEvent) => events.push(event) + GlobalBus.on("event", handler) + return { events, dispose: () => GlobalBus.off("event", handler) } +} + +function eventuallyEffect(effect: Effect.Effect, timeout = 3000) { + return Effect.gen(function* () { + const started = Date.now() + let last: unknown + while (Date.now() - started < timeout) { + const exit = yield* Effect.exit(effect) + if (exit._tag === "Success") return + last = exit.cause + yield* Effect.sleep("10 millis") + } + throw last ?? new Error("Timed out waiting for condition") + }) +} + +const insertWorkspace = (info: Workspace.Info) => + Database.Service.use(({ db }) => + db + .insert(WorkspaceTable) + .values({ + id: info.id, + type: info.type, + branch: info.branch, + name: info.name, + directory: info.directory, + extra: info.extra, + project_id: info.projectID, + time_used: info.timeUsed, + }) + .run() + .pipe(Effect.orDie), + ) + +const attachSessionToWorkspace = (sessionID: SessionID, workspaceID: WorkspaceV2.ID) => + Database.Service.use(({ db }) => + db.update(SessionTable).set({ workspace_id: workspaceID }).where(eq(SessionTable.id, sessionID)).run().pipe(Effect.orDie), + ) + +const sessionSequence = (sessionID: SessionID) => + Database.Service.use(({ db }) => + db + .select({ seq: EventSequenceTable.seq }) + .from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, sessionID)) + .get() + .pipe( + Effect.orDie, + Effect.map((row) => row?.seq), + ), + ) + +describe("workspace control plane — sync copies are opt-in (#1778)", () => { + it.live("the control-plane subscription requests sync copies and replays them into the local store", () => { + const requests: { url: URL; syncCopies: string | undefined }[] = [] + let remote: { sessionID: SessionID; info: SessionNs.Info; seq: number } | undefined + return Effect.gen(function* () { + yield* HttpServer.serveEffect()( + Effect.gen(function* () { + const req = yield* HttpServerRequest.HttpServerRequest + const url = new URL(req.url, "http://localhost") + if (url.pathname === "/optin/global/event") { + const syncCopies = req.headers["x-opencode-sync-copies"] + requests.push({ url, syncCopies }) + if (!remote) return HttpServerResponse.fromWeb(engineStream([])) + const id = `evt_${unique("optin")}` + const event = { type: "session.updated", properties: { sessionID: remote.sessionID, info: remote.info } } + // the engine's twin emission: the durable event, then its sync copy (same sequence) + const frames = [{ seq: 7, event: { directory: "remote-dir", payload: { id, ...event } } }] + if (syncCopies === "true") + frames.push({ + seq: 7, + event: { + directory: "remote-dir", + payload: { + id, + type: "sync", + syncEvent: { + id, + aggregateID: remote.sessionID, + seq: remote.seq, + type: "session.updated.1", + data: event.properties, + }, + }, + } as never, + }) + return HttpServerResponse.fromWeb(engineStream(frames)) + } + if (url.pathname === "/optin/sync/history") return HttpServerResponse.fromWeb(Response.json([])) + return HttpServerResponse.text("unexpected", { status: 500 }) + }), + ) + const url = HttpServer.formatAddress((yield* HttpServer.HttpServer).address) + yield* provideTmpdirInstance( + () => + Effect.gen(function* () { + const workspace = yield* Workspace.Service + const sessions = yield* SessionNs.Service + const instance = yield* requireInstance + const captured = captureGlobalEvents() + try { + const type = unique("optin") + const info: Workspace.Info = { + id: WorkspaceV2.ID.ascending(), + type, + name: unique("workspace"), + branch: null, + directory: null, + extra: null, + projectID: instance.project.id as ProjectV2.ID, + timeUsed: Date.now(), + } + yield* insertWorkspace(info) + registerAdapter(instance.project.id, type, remoteAdapter(`${url}/optin`)) + const session = yield* sessions.create({ title: "before sync" }) + yield* attachSessionToWorkspace(session.id, info.id) + remote = { + sessionID: session.id, + info: { ...session, workspaceID: info.id, title: "from the remote's sync copy" }, + seq: ((yield* sessionSequence(session.id)) ?? -1) + 1, + } + + yield* workspace.startWorkspaceSyncing(instance.project.id) + + // replayed correctly: the sync copy reached the local store + yield* eventuallyEffect( + Effect.gen(function* () { + expect((yield* sessions.get(session.id).pipe(Effect.orDie)).title).toBe("from the remote's sync copy") + }), + ) + expect(requests.length).toBeGreaterThan(0) + expect(requests.every((request) => request.syncCopies === "true")).toBe(true) + // the opt-in rides a header: the event URL keeps its routed form + expect(requests.every((request) => request.url.pathname === "/optin/global/event" && !request.url.search)).toBe( + true, + ) + // and the remote's frames are still forwarded onto the local bus + expect( + captured.events.some((event) => event.workspace === info.id && event.payload.type === "session.updated"), + ).toBe(true) + yield* workspace.remove(info.id) + } finally { + captured.dispose() + } + }), + { git: true }, + ) + }) + }) +}) diff --git a/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts b/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts index c4d424606..e11a37eff 100644 --- a/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts +++ b/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts @@ -148,11 +148,27 @@ const connect = (input: { headers?: Record; query?: string } = { const emit = (type: string, properties: Record = {}) => GlobalBus.emit("event", { directory: "/tmp/i1776", payload: { type, properties } }) +/** The event-v2 bridge's twin emission: a durable event, then its sync copy. */ +const twin = (id: string, type = "test.durable") => { + GlobalBus.emit("event", { directory: "/tmp/i1778", payload: { id, type, properties: {} } }) + GlobalBus.emit("event", { + directory: "/tmp/i1778", + payload: { type: "sync", syncEvent: { id, type: `${type}.1`, seq: 1, aggregateID: "agg", data: {} } }, + }) +} + +const typesOf = (frames: Frame[]) => frames.map((frame) => frame.data.payload.type) +const seqsOf = (frames: Frame[]) => frames.map((frame) => idOf(frame).seq) +/** Resolves `undefined` when no further frame arrives within `ms`. */ +const quiet = (stream: { next: Effect.Effect }, ms = 200) => + stream.next.pipe(Effect.timeoutOrElse({ duration: ms, orElse: () => Effect.succeed(undefined) })) + describe("global event stream — sequenced ids (#1776)", () => { it.live("every frame carries an : id shared by all subscribers; a sync copy carries its source's id", () => Effect.gen(function* () { - const a = yield* connect() - const b = yield* connect() + // sync copies are opt-in (#1778): both subscribers request them + const a = yield* connect({ query: "?sync=true" }) + const b = yield* connect({ query: "?sync=true" }) const connectedA = yield* a.next const connectedB = yield* b.next expect(connectedA.data.payload.type).toBe("server.connected") @@ -182,6 +198,82 @@ describe("global event stream — sequenced ids (#1776)", () => { ) }) +describe("global event stream — sync copies are opt-in (#1778)", () => { + it.live("a default subscriber gets no sync copies and its sequence stays gap-free; an opt-in subscriber gets them all", () => + Effect.gen(function* () { + const plain = yield* connect() + // the header carrier (what the workspace control plane sends); `?sync=true` is the query twin + const synced = yield* connect({ headers: { "x-opencode-sync-copies": "true" } }) + const start = idOf(yield* plain.next) + expect(idOf(yield* synced.next)).toEqual(start) + + emit("test.first") + twin("evt_i1778_optin_a") + twin("evt_i1778_optin_b") + emit("test.last") + + const plainFrames = yield* plain.take(4) + expect(typesOf(plainFrames)).toEqual(["test.first", "test.durable", "test.durable", "test.last"]) + // filtering the copies punches no hole: a copy shares its source's sequence + expect(seqsOf(plainFrames)).toEqual([1, 2, 3, 4].map((n) => start.seq + n)) + expect(yield* quiet(plain)).toBeUndefined() + + const syncedFrames = yield* synced.take(6) + expect(typesOf(syncedFrames)).toEqual(["test.first", "test.durable", "sync", "test.durable", "sync", "test.last"]) + expect(seqsOf(syncedFrames)).toEqual([1, 2, 2, 3, 3, 4].map((n) => start.seq + n)) + // the frames both subscribers get are the same frames + expect(syncedFrames.filter((f) => f.data.payload.type !== "sync").map((f) => f.id)).toEqual( + plainFrames.map((f) => f.id), + ) + }), + ) + + it.live("a resumed connection replays sync copies only when it requests them", () => + Effect.gen(function* () { + const a = yield* connect() + const cursor = (yield* a.next).id! + a.abort() + twin("evt_i1778_replay_a") + emit("test.between") + twin("evt_i1778_replay_b") + + const plain = yield* connect({ headers: { "last-event-id": cursor } }) + const synced = yield* connect({ headers: { "last-event-id": cursor }, query: "?sync=true" }) + const base = idOf(yield* plain.next).seq + yield* synced.next + emit("test.live") + + const plainFrames = yield* plain.take(4) + expect(typesOf(plainFrames)).toEqual(["test.durable", "test.between", "test.durable", "test.live"]) + expect(seqsOf(plainFrames)).toEqual([1, 2, 3, 4].map((n) => base + n)) + expect(yield* quiet(plain)).toBeUndefined() + + const syncedFrames = yield* synced.take(6) + expect(typesOf(syncedFrames)).toEqual(["test.durable", "sync", "test.between", "test.durable", "sync", "test.live"]) + expect(seqsOf(syncedFrames)).toEqual([1, 1, 2, 3, 3, 4].map((n) => base + n)) + }), + ) + + it.live("a sync copy with no framed source keeps its own sequence and reaches every subscriber, so no hole opens", () => + Effect.gen(function* () { + const plain = yield* connect() + const start = idOf(yield* plain.next) + emit("test.before") + GlobalBus.emit("event", { + directory: "/tmp/i1778", + payload: { + type: "sync", + syncEvent: { id: "evt_i1778_orphan", type: "test.orphan.1", seq: 1, aggregateID: "agg", data: {} }, + }, + }) + emit("test.after") + const frames = yield* plain.take(3) + expect(typesOf(frames)).toEqual(["test.before", "sync", "test.after"]) + expect(seqsOf(frames)).toEqual([1, 2, 3].map((n) => start.seq + n)) + }), + ) +}) + describe("global event stream — resume from a cursor (#1776)", () => { for (const carrier of ["Last-Event-ID header", "lastEventID query"] as const) { it.live(`replays exactly the frames after a ${carrier} cursor, then continues live`, () => From f2b1b25a543e57122168e3fe80f03b9589ad5201 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 17:55:00 -0400 Subject: [PATCH 17/60] =?UTF-8?q?feat(fleet):=20fan-in=20census=20line=20e?= =?UTF-8?q?very=2030=20s=20=E2=80=94=20listeners,=20accepted/closed,=20evi?= =?UTF-8?q?ctions,=20per-arm=20traffic=20(#1771)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/amicode_service/fanin_census.ts | 240 +++++++++++++++ .../extension/src/amicode_service/index.ts | 51 ++-- .../amicode_service/sse_fanin_aggregator.ts | 102 ++++++- .../src/amicode_service/sse_fanin_driver.ts | 135 +++++++-- .../extension/src/amicode_service_runner.ts | 2 + .../extension/src/amicode_service_wiring.ts | 3 + ...code_service_observe_event_routing.test.ts | 26 +- .../fixtures/engine_stream/engine_frames.ts | 281 ++++++++++++++++++ .../test/sse_fanin_driver_pump.test.ts | 154 +++++++++- 9 files changed, 948 insertions(+), 46 deletions(-) create mode 100644 packages/extension/src/amicode_service/fanin_census.ts create mode 100644 packages/extension/test/fixtures/engine_stream/engine_frames.ts diff --git a/packages/extension/src/amicode_service/fanin_census.ts b/packages/extension/src/amicode_service/fanin_census.ts new file mode 100644 index 000000000..16046ef47 --- /dev/null +++ b/packages/extension/src/amicode_service/fanin_census.ts @@ -0,0 +1,240 @@ +// FAN-IN CENSUS (#1771, part of #1770) — every fan-in service's counters and +// its log lines. A wedged, evicted, or flapping event arm used to look exactly +// like a quiet one in every log: nothing recorded who subscribed, why an arm +// re-opened, or what was dropped (the Oct 12 eviction storm was found only by +// reverse-engineering engine connect timestamps). The census makes that visible +// without a probe run: +// +// - one CENSUS line every 30 s (FANIN_CENSUS_INTERVAL_MS): active listeners, +// listeners accepted / closed since boot, evictions, refusals, and per arm +// (local and each peer, per listener connection) frames, bytes, largest +// frame, time since the last frame, opens, closes by reason, pending depth, +// overflow drops, and gaps emitted; +// - one TRANSITION line immediately on each listener accepted / closed / +// refused, arm opened / closed, gap emitted, overflow drop — never per frame. +// +// Observation only: the census writes to its SINK (the hub runner's log, the +// window's Amicode output channel), never to a stream. It adds no frame and +// changes no delivery decision. The clock and the sink are injected so tests +// drive it deterministically. +import type { ArmTraffic, GapCause } from "./sse_fanin_aggregator"; + +/** Why a listener or an upstream arm closed — an ENUMERATED set, so later + * slices and the verification slice can assert on it. `evicted` is kept even + * though #1772 removed the eviction path: it must read 0, and a regression + * that brings eviction back must have a name to show up under. */ +export const FANIN_CLOSE_REASONS = [ + "upstream-end", + "upstream-error", + "membership-change", + "listener-closed", + "evicted", + "shutdown", +] as const; +export type FanInCloseReason = (typeof FANIN_CLOSE_REASONS)[number]; + +/** The steady-state census cadence. */ +export const FANIN_CENSUS_INTERVAL_MS = 30_000; + +/** One arm of one live listener connection, as the census reports it. */ +export interface ArmCensus { + /** The listener connection (`c1`, `c2`, … in accept order). */ + conn: string; + /** The arm's namespace: `local` or the peer's machine id. */ + arm: string; + frames: number; + bytes: number; + largestFrame: number; + /** ms since the arm's last frame; null when it has delivered none. */ + sinceLastFrameMs: number | null; + opens: number; + closes: Record; + /** Frames waiting in the aggregator's pending stream for this arm. */ + pending: number; + /** Frames dropped on overflow, since the connection opened. */ + drops: number; + /** Gap frames emitted naming this arm. */ + gaps: number; +} + +export interface FanInCensusSnapshot { + plane: string; + listeners: number; + accepted: number; + closed: number; + evictions: number; + refusals: number; + arms: ArmCensus[]; +} + +/** What a live listener connection exposes to the census: its id and its + * arms' current counters (read at census time — nothing is pushed per frame). */ +export interface CensusConnection { + readonly id: string; + arms(): Array<{ + arm: string; + traffic: ArmTraffic; + pending: number; + opens: number; + closes: Partial>; + }>; +} + +export interface FanInCensusOptions { + /** Which fan-in this is (`observe` — the observation path; `premium` — the + * flag-gated fleet plane). Rides every line. */ + plane?: string; + /** Where the lines go. Default: nowhere (a census nobody wired is silent). */ + sink?: (line: string) => void; + /** The clock (ms). Default Date.now. */ + now?: () => number; + /** The census cadence. Default FANIN_CENSUS_INTERVAL_MS. */ + intervalMs?: number; +} + +export function emptyCloses(): Record { + const out = {} as Record; + for (const r of FANIN_CLOSE_REASONS) out[r] = 0; + return out; +} + +function closesToken(closes: Record): string { + const parts = FANIN_CLOSE_REASONS.filter((r) => closes[r] > 0).map((r) => `${r}:${closes[r]}`); + return parts.length === 0 ? "-" : parts.join(","); +} + +export class FanInCensus { + readonly plane: string; + readonly now: () => number; + private readonly sink: (line: string) => void; + private readonly intervalMs: number; + private readonly live = new Map(); + private accepted = 0; + private closed = 0; + private evictions = 0; + private refusals = 0; + private timer?: ReturnType; + + constructor(opts: FanInCensusOptions = {}) { + this.plane = opts.plane ?? "fanin"; + this.sink = opts.sink ?? (() => undefined); + this.now = opts.now ?? Date.now; + this.intervalMs = opts.intervalMs ?? FANIN_CENSUS_INTERVAL_MS; + } + + /** Start the steady-state cadence (idempotent; the timer never keeps the + * process alive on its own). */ + start(): void { + if (this.timer) return; + this.timer = setInterval(() => this.emit(), this.intervalMs); + this.timer.unref?.(); + } + + /** Stop the cadence (idempotent). Transition lines still log. */ + stop(): void { + if (!this.timer) return; + clearInterval(this.timer); + this.timer = undefined; + } + + // ── transitions (each logs ONE line, immediately) ───────────────────────── + + listenerAccepted(conn: CensusConnection): void { + this.accepted += 1; + this.live.set(conn.id, conn); + this.log(`listener-accepted conn=${conn.id} listeners=${this.live.size}`); + } + + listenerClosed(connId: string, reason: FanInCloseReason): void { + if (!this.live.delete(connId)) return; + this.closed += 1; + if (reason === "evicted") this.evictions += 1; + this.log(`listener-closed conn=${connId} reason=${reason} listeners=${this.live.size}`); + } + + listenerRefused(cap: number): void { + this.refusals += 1; + this.log(`listener-refused listeners=${this.live.size} cap=${cap}`); + } + + armOpened(connId: string, arm: string, opens: number): void { + this.log(`arm-opened conn=${connId} arm=${arm} opens=${opens}`); + } + + armClosed(connId: string, arm: string, reason: FanInCloseReason): void { + this.log(`arm-closed conn=${connId} arm=${arm} reason=${reason}`); + } + + gapEmitted(connId: string, arms: string[], cause: GapCause): void { + this.log(`gap-emitted conn=${connId} arm=${arms.join(",")} cause=${cause}`); + } + + /** The FIRST drop of an overflow episode (the census counts every drop; the + * line is per episode — a drop storm never becomes a per-frame log). */ + overflowDrop(connId: string, arm: string, pending: number): void { + this.log(`overflow-drop conn=${connId} arm=${arm} pending=${pending}`); + } + + // ── the census ──────────────────────────────────────────────────────────── + + snapshot(): FanInCensusSnapshot { + const now = this.now(); + const arms: ArmCensus[] = []; + for (const conn of this.live.values()) { + for (const a of conn.arms()) { + arms.push({ + conn: conn.id, + arm: a.arm, + frames: a.traffic.frames, + bytes: a.traffic.bytes, + largestFrame: a.traffic.largestFrame, + sinceLastFrameMs: a.traffic.lastFrameAt === undefined ? null : Math.max(0, now - a.traffic.lastFrameAt), + opens: a.opens, + closes: { ...emptyCloses(), ...a.closes }, + pending: a.pending, + drops: a.traffic.drops, + gaps: a.traffic.gaps, + }); + } + } + return { + plane: this.plane, + listeners: this.live.size, + accepted: this.accepted, + closed: this.closed, + evictions: this.evictions, + refusals: this.refusals, + arms, + }; + } + + /** The census line: the service-level head, then ` | `-separated per-arm + * segments (`key=value` tokens throughout). */ + line(): string { + const s = this.snapshot(); + const head = `[fanin] census plane=${s.plane} listeners=${s.listeners} accepted=${s.accepted} closed=${s.closed} evictions=${s.evictions} refusals=${s.refusals} arms=${s.arms.length}`; + const segs = s.arms.map( + (a) => + `conn=${a.conn} arm=${a.arm} frames=${a.frames} bytes=${a.bytes} largest=${a.largestFrame} since_last_ms=${a.sinceLastFrameMs ?? "-"} opens=${a.opens} closes=${closesToken(a.closes)} pending=${a.pending} drops=${a.drops} gaps=${a.gaps}`, + ); + return [head, ...segs].join(" | "); + } + + /** Write the census line to the sink now (the cadence calls this). */ + emit(): void { + this.write(this.line()); + } + + private log(event: string): void { + const [kind, ...rest] = event.split(" "); + this.write(`[fanin] ${kind} plane=${this.plane} ${rest.join(" ")}`); + } + + private write(line: string): void { + try { + this.sink(line); + } catch { + /* a broken sink must never break the fan-in */ + } + } +} diff --git a/packages/extension/src/amicode_service/index.ts b/packages/extension/src/amicode_service/index.ts index 3e02f2134..72c7f865c 100644 --- a/packages/extension/src/amicode_service/index.ts +++ b/packages/extension/src/amicode_service/index.ts @@ -45,6 +45,7 @@ import { type SessionEntry, } from "./session_multiplexer"; import { SseFanInDriver } from "./sse_fanin_driver"; +import { FanInCensus } from "./fanin_census"; import { readFleetSessionStatus, type StatusOwnerLookup } from "./fleet_session_status"; import { createObservationReadPlane } from "./observation_read_plane"; import { createObservationWritePlane } from "./observation_write_plane"; @@ -930,6 +931,10 @@ export function createAmicodeService( * credential gate's refresh loop). Absent → the surface renders empty * roles (no cards known) with providers from the stored snapshot. */ modelRouting?: ModelRoutingDeps; + /** #1771: the service's diagnostic log (the hub runner's stdout → + * /tmp/amico-hub.log; the window's Amicode output channel). The fan-in + * census and the owner-map refresh outcomes write here. Absent → silent. */ + log?: (line: string) => void; fleet?: { /** Resolved entitlements (injectable for tests); null resolves the * machine's real set. */ @@ -1207,7 +1212,12 @@ export function createAmicodeService( let eventFanIn: SseFanInDriver | undefined; if (fleetPeers && !isClient) { const peers = fleetPeers; - eventFanIn = new SseFanInDriver({ + // #1771: the fan-in census — 30 s census lines + immediate transition + // lines to the service log; torn down (and closes attributed to + // `shutdown`) with the server. + const census = new FanInCensus({ plane: "premium", ...(opts.log ? { sink: opts.log } : {}) }); + census.start(); + const driver = new SseFanInDriver({ ownerMap, localMachineId: peers.localMachineId, localEventUrl: opts.engine?.getUrl ?? ((): string | undefined => undefined), @@ -1223,7 +1233,10 @@ export function createAmicodeService( // the aggregator so the connect frame carries real focus data (not the // named-empty default). Absent → the aggregator's named-empty fallback. ...(opts.fleet.focusSnapshot ? { focusSnapshot: opts.fleet.focusSnapshot } : {}), + census, }); + server.registerCleanup(() => driver.shutdown()); + eventFanIn = driver; } const fleetPlaneObj: import("./server").FleetPlane = { getMode, @@ -1436,22 +1449,26 @@ export function createAmicodeService( // — fleet-of-one byte-identity holds only absent a focus provider (D6). // Per-peer arms auth AS THEMSELVES with their OWN reader token (decision // A); the local arm rides the app's incoming credential. - server.attachObservationEventPlane( - new SseFanInDriver({ - ownerMap, - localMachineId: fleetPeers.localMachineId, - localEventUrl: opts.engine?.getUrl ?? ((): string | undefined => undefined), - peerBaseUrl: (machineId) => { - if (!fleetPeers.getServingPeers().some((p) => p.machineId === machineId)) return undefined; - const r = fleetPeers.readPeerToken(machineId); - return r.ok ? r.credential.baseUrl : undefined; - }, - peerToken: (machineId) => { - const r = fleetPeers.readPeerToken(machineId); - return r.ok ? { ok: true, credential: r.credential } : { ok: false, reason: "absent" }; - }, - }), - ); + // #1771: with its census (30 s census + transition lines → the log). + const observeCensus = new FanInCensus({ plane: "observe", ...(opts.log ? { sink: opts.log } : {}) }); + observeCensus.start(); + const observeDriver = new SseFanInDriver({ + ownerMap, + localMachineId: fleetPeers.localMachineId, + localEventUrl: opts.engine?.getUrl ?? ((): string | undefined => undefined), + peerBaseUrl: (machineId) => { + if (!fleetPeers.getServingPeers().some((p) => p.machineId === machineId)) return undefined; + const r = fleetPeers.readPeerToken(machineId); + return r.ok ? r.credential.baseUrl : undefined; + }, + peerToken: (machineId) => { + const r = fleetPeers.readPeerToken(machineId); + return r.ok ? { ok: true, credential: r.credential } : { ok: false, reason: "absent" }; + }, + census: observeCensus, + }); + server.registerCleanup(() => observeDriver.shutdown()); + server.attachObservationEventPlane(observeDriver); } } } diff --git a/packages/extension/src/amicode_service/sse_fanin_aggregator.ts b/packages/extension/src/amicode_service/sse_fanin_aggregator.ts index c5b1fa49f..9857e416c 100644 --- a/packages/extension/src/amicode_service/sse_fanin_aggregator.ts +++ b/packages/extension/src/amicode_service/sse_fanin_aggregator.ts @@ -49,6 +49,10 @@ export interface SseFrameSource { pause?(): void; /** #1617 — resume pulling once the downstream has drained. */ resume?(): void; + /** #1771 — once `next()` has resolved null: did the upstream END cleanly or + * fail with an ERROR? Census only (the close reason); both still end the + * source the same way. Absent / undefined ⇒ a clean end. */ + endReason?(): "end" | "error" | undefined; } /** The downstream response, narrowed to what the relay needs. In production an @@ -210,6 +214,37 @@ export async function pipeToResponse( // ── the aggregator core ────────────────────────────────────────────────────── +/** #1771 — one arm's traffic counters on this connection (the census reads + * them; nothing about delivery depends on them). Counted per INGESTED frame — + * whether it is delivered, buffered, or dropped. */ +export interface ArmTraffic { + frames: number; + /** UTF-8 bytes of the upstream frames as received. */ + bytes: number; + largestFrame: number; + /** Clock time of the last frame (the aggregator's `now`); undefined if none. */ + lastFrameAt?: number; + /** Frames dropped on overflow, cumulative (never zeroed by a gap). */ + drops: number; + /** Gap frames emitted that named this arm. */ + gaps: number; +} + +/** #1771 — why a gap frame was emitted: an arm re-opened after delivering + * (#1638), or frames were dropped on overflow (#1617). */ +export type GapCause = "reopen" | "overflow"; + +/** #1771 — the aggregator's transitions, for the census. Observation only: + * callbacks run after the aggregator decided, and a throwing observer is + * swallowed. */ +export interface SseFanInObserver { + /** The FIRST overflow drop of an episode for `namespace` (an episode ends + * when the overflow gap naming it is emitted). `pending` is its live depth. */ + onOverflowDrop?(namespace: string, pending: number): void; + /** A gap frame was emitted naming `namespaces`. */ + onGap?(namespaces: string[], cause: GapCause): void; +} + export interface SseFanInOptions { /** The one downstream `/event` response every arm fans into. */ sink: SseSink; @@ -226,6 +261,10 @@ export interface SseFanInOptions { * buffers cannot overflow-drop. Absent ⇒ buffer-and-drain only (today's * behavior for injected test sources). */ onBackpressure?: () => void; + /** #1771 — the clock stamping each arm's last-frame time. Default Date.now. */ + now?: () => number; + /** #1771 — the census's view of drops and gaps. Absent ⇒ unobserved. */ + observer?: SseFanInObserver; } export class SseFanInAggregator { @@ -233,6 +272,10 @@ export class SseFanInAggregator { private readonly bufferBound: number; private readonly focusProvider?: () => FocusSnapshot | undefined; private readonly onBackpressure?: () => void; + private readonly now: () => number; + private readonly observer?: SseFanInObserver; + /** #1771 — per-arm traffic counters (census only). */ + private readonly trafficByArm = new Map(); /** namespace → last delivered upstream id (the composite cursor state). */ private readonly composite = new Map(); @@ -275,6 +318,8 @@ export class SseFanInAggregator { this.bufferBound = opts.bufferBound ?? 256; this.focusProvider = opts.focusSnapshot; this.onBackpressure = opts.onBackpressure; + this.now = opts.now ?? Date.now; + this.observer = opts.observer; } /** Begin (or reconnect): parse the client's opaque composite cursor into the @@ -334,6 +379,7 @@ export class SseFanInAggregator { /** Ingest one raw SSE frame from an arm. `namespace === "local"` for the local * arm; a machineId for a peer arm. */ ingest(namespace: string, rawFrame: string): void { + this.countFrame(namespace, rawFrame); // #1771 census — observation only const rawId = frameRawId(rawFrame); const fleetOfOne = this.peerArms.size === 0; @@ -466,6 +512,7 @@ export class SseFanInAggregator { if (!this.everDelivered.has(namespace)) return; // initial open → no gap const data = JSON.stringify({ type: "amicode.sync.gap", namespaces: [namespace] }); this.writeControl(`event: amicode.sync.gap\ndata: ${data}\n\n`); + this.noteGap([namespace], "reopen"); } /** #1617 — the overflow defense-in-depth. When resume() observes a nonzero @@ -483,6 +530,7 @@ export class SseFanInAggregator { const data = JSON.stringify({ type: "amicode.sync.gap", namespaces: dropped }); this.writeControl(`event: amicode.sync.gap\ndata: ${data}\n\n`); for (const ns of dropped) this.droppedCount.set(ns, 0); + this.noteGap(dropped, "overflow"); } /** The number of frames DROPPED for a namespace on buffer overflow (decision @@ -506,8 +554,57 @@ export class SseFanInAggregator { return this.composite.get(namespace); } + /** #1771 — one arm's traffic counters on this connection (a copy; zeros for + * an arm that has ingested nothing). */ + traffic(namespace: string): ArmTraffic { + const t = this.trafficByArm.get(namespace); + return t ? { ...t } : { frames: 0, bytes: 0, largestFrame: 0, drops: 0, gaps: 0 }; + } + + /** #1771 — how many of this arm's frames wait undelivered right now: its data + * entries on the ordered pending stream, plus (local) the fleet-of-one + * verbatim buffer. */ + pendingDepth(namespace: string): number { + let n = 0; + for (const entry of this.pending) if (entry.kind === "data" && entry.namespace === namespace) n++; + if (namespace === LOCAL_NAMESPACE) n += this.verbatimBuffer.length; + return n; + } + // ── internals ──────────────────────────────────────────────────────────── + private trafficOf(namespace: string): ArmTraffic { + let t = this.trafficByArm.get(namespace); + if (!t) { + t = { frames: 0, bytes: 0, largestFrame: 0, drops: 0, gaps: 0 }; + this.trafficByArm.set(namespace, t); + } + return t; + } + + private countFrame(namespace: string, rawFrame: string): void { + const t = this.trafficOf(namespace); + const bytes = Buffer.byteLength(rawFrame, "utf8"); + t.frames += 1; + t.bytes += bytes; + if (bytes > t.largestFrame) t.largestFrame = bytes; + t.lastFrameAt = this.now(); + } + + private noteGap(namespaces: string[], cause: GapCause): void { + for (const ns of namespaces) this.trafficOf(ns).gaps += 1; + this.observe((o) => o.onGap?.([...namespaces], cause)); + } + + private observe(fn: (o: SseFanInObserver) => void): void { + if (!this.observer) return; + try { + fn(this.observer); + } catch { + /* the census must never break delivery */ + } + } + /** Write one frame downstream in composite mode: advance the namespace's * cursor from the frame's id, stamp the full composite as the wire `id:`, * write + flush. A `write()` returning false backpressures the shared @@ -543,7 +640,10 @@ export class SseFanInAggregator { if (entry.kind === "data" && entry.namespace === namespace) live++; } if (live >= this.bufferBound) { - this.droppedCount.set(namespace, (this.droppedCount.get(namespace) ?? 0) + 1); + const before = this.droppedCount.get(namespace) ?? 0; + this.droppedCount.set(namespace, before + 1); + this.trafficOf(namespace).drops += 1; // #1771 census + if (before === 0) this.observe((o) => o.onOverflowDrop?.(namespace, live)); return; } this.pending.push({ kind: "data", namespace, frame: rawFrame }); diff --git a/packages/extension/src/amicode_service/sse_fanin_driver.ts b/packages/extension/src/amicode_service/sse_fanin_driver.ts index 72260e628..9b7c37a67 100644 --- a/packages/extension/src/amicode_service/sse_fanin_driver.ts +++ b/packages/extension/src/amicode_service/sse_fanin_driver.ts @@ -33,6 +33,7 @@ import { } from "./sse_fanin_aggregator"; import type { SessionOwnerMap } from "./session_multiplexer"; import type { PeerTokenRead } from "./fleet_peer_store"; +import { FanInCensus, emptyCloses, type FanInCloseReason, type CensusConnection } from "./fanin_census"; /** The one thing dispatch needs from the driver (server.ts imports this type and * exposes it on FleetPlane.eventFanIn). `handle` always returns true on the @@ -84,6 +85,10 @@ export interface SseFanInDriverDeps { * listener beyond it is refused with a retryable 503 + Retry-After; no live * listener is ever closed to make room. Default DEFAULT_FANIN_MAX_LISTENERS. */ maxListeners?: number; + /** #1771 — this service's census (counters + log lines). The caller owns its + * cadence (`start()`) and its sink. Absent ⇒ a silent census (counted, + * logged nowhere). */ + census?: FanInCensus; } /** #1772 — default concurrent-listener cap per fan-in service. A service rarely @@ -98,20 +103,24 @@ export const FANIN_LISTENER_CAP_ERROR = "fan-in listener cap reached"; * byte stream on the SSE blank-line delimiter into WHOLE frames (delimiter- * inclusive) so `event:`/`data:`/`retry:`/`id:` lines survive intact for the * aggregator's §D2 relay. Never throws — a connection error just ends the - * source (the aggregator names the gap; D3 replays it on reconnect). */ + * source (the aggregator names the gap; D3 replays it on reconnect). #1771: + * the source remembers WHICH ended it — a clean upstream end or an error — for + * the census's close reason; both still end it identically. */ export function openHttpSseSource(r: FanInUpstreamRequest): SseFrameSource { const queue: string[] = []; const waiters: Array<(v: string | null) => void> = []; let ended = false; + let reason: "end" | "error" | undefined; let clientReq: http.ClientRequest | undefined; const pushFrame = (f: string): void => { const w = waiters.shift(); if (w) w(f); else queue.push(f); }; - const end = (): void => { + const end = (why: "end" | "error"): void => { if (ended) return; ended = true; + reason = why; let w: ((v: string | null) => void) | undefined; while ((w = waiters.shift())) w(null); }; @@ -132,12 +141,12 @@ export function openHttpSseSource(r: FanInUpstreamRequest): SseFrameSource { buf = buf.slice(idx + 2); } }); - res.on("end", () => end()); - res.on("error", () => end()); + res.on("end", () => end("end")); + res.on("error", () => end("error")); }); - clientReq.on("error", () => end()); + clientReq.on("error", () => end("error")); } catch { - end(); + end("error"); } return { next(): Promise { @@ -148,15 +157,18 @@ export function openHttpSseSource(r: FanInUpstreamRequest): SseFrameSource { }, close(): void { clientReq?.destroy(); - end(); + end("end"); }, + endReason: () => reason, }; } /** One live `/event` fan-in connection — ONE PER LISTENER (#1772): the * aggregator on this response, the open upstream sources keyed by namespace, - * and this connection's own reconcile timer. Closing it closes only its arms. */ -class FanInConnection { + * and this connection's own reconcile timer. Closing it closes only its arms. + * #1771: it is also the census's view of this listener — per-arm opens and + * closes-by-reason live here, traffic/pending/drops/gaps in the aggregator. */ +class FanInConnection implements CensusConnection { private readonly agg: SseFanInAggregator; private readonly sources = new Map(); private resume = new Map(); @@ -164,9 +176,16 @@ class FanInConnection { private readonly authHeader?: string; private timer?: ReturnType; private closed = false; + /** #1771 — per arm: how often it opened, and why it closed. */ + private readonly armLife = new Map }>(); + /** #1771 — sources THIS connection closed (their reason is already recorded; + * the pump's exit must not re-count them as an upstream end). */ + private readonly closedByUs = new WeakSet(); constructor( + readonly id: string, private readonly deps: SseFanInDriverDeps, + private readonly census: FanInCensus, private readonly req: http.IncomingMessage, private readonly res: http.ServerResponse, ) { @@ -188,9 +207,55 @@ class FanInConnection { // cursor). The `drain` edge (wired in start()) resumes them and calls // agg.resume() to flush the buffers in order and re-arm `flowing`. onBackpressure: () => this.pauseUpstreams(), + // #1771 — the census clock and its view of drops and gaps. + now: census.now, + observer: { + onOverflowDrop: (ns, pending) => census.overflowDrop(this.id, ns, pending), + onGap: (nss, cause) => census.gapEmitted(this.id, nss, cause), + }, }); } + /** #1771 — this listener's arms as the census reports them (read at census + * time, never pushed per frame). */ + arms(): ReturnType { + return [...this.armLife.entries()].map(([arm, life]) => ({ + arm, + traffic: this.agg.traffic(arm), + pending: this.agg.pendingDepth(arm), + opens: life.opens, + closes: { ...life.closes }, + })); + } + + private lifeOf(namespace: string): { opens: number; closes: Record } { + let life = this.armLife.get(namespace); + if (!life) { + life = { opens: 0, closes: emptyCloses() }; + this.armLife.set(namespace, life); + } + return life; + } + + /** #1771 — record (and log) an arm's close, once. */ + private noteArmClosed(namespace: string, reason: FanInCloseReason): void { + this.lifeOf(namespace).closes[reason] += 1; + this.census.armClosed(this.id, namespace, reason); + } + + /** Close an upstream THIS connection decided to close (membership change, + * listener closed, shutdown) — recorded with its reason before the source's + * own end reaches the pump. */ + private closeSource(namespace: string, source: SseFrameSource, reason: FanInCloseReason): void { + this.closedByUs.add(source); + this.noteArmClosed(namespace, reason); + try { + source.close(); + } catch { + /* already gone */ + } + } + /** #1617 — pause every live upstream source that supports it. A source * without pause()/resume() (an injected test source, or a transport that * cannot pause) simply keeps flowing into the aggregator's bounded buffer. */ @@ -289,11 +354,7 @@ class FanInConnection { for (const machineId of [...this.sources.keys()]) { if (machineId === LOCAL_NAMESPACE) continue; if (!active.has(machineId)) { - try { - this.sources.get(machineId)?.close(); - } catch { - /* already gone */ - } + this.closeSource(machineId, this.sources.get(machineId)!, "membership-change"); this.sources.delete(machineId); } } @@ -350,6 +411,9 @@ class FanInConnection { ...(lastEventId ? { lastEventId } : {}), }); this.sources.set(namespace, source); + const life = this.lifeOf(namespace); + life.opens += 1; + this.census.armOpened(this.id, namespace, life.opens); void this.pump(namespace, source); } @@ -358,35 +422,35 @@ class FanInConnection { * the client tracks — NOT the bare `pipeToResponse` relay, which would emit * the NS_SEP form the composite cursor cannot round-trip). */ private async pump(namespace: string, source: SseFrameSource): Promise { + let threw = false; for (;;) { let frame: string | null; try { frame = await source.next(); } catch { + threw = true; break; } if (frame === null || this.closed) break; this.agg.ingest(namespace, frame); } + // #1771 — the upstream's OWN end (not one this connection caused): name it. + if (!this.closed && !this.closedByUs.has(source)) { + this.noteArmClosed(namespace, threw || source.endReason?.() === "error" ? "upstream-error" : "upstream-end"); + } // The upstream ended or errored — remove the dead source so the next // reconcile() tick can re-open the arm (#1566). this.sources.delete(namespace); } - close(): void { + close(reason: FanInCloseReason = "listener-closed"): void { if (this.closed) return; this.closed = true; if (this.timer) { clearInterval(this.timer); this.timer = undefined; } - for (const s of this.sources.values()) { - try { - s.close(); - } catch { - /* already gone */ - } - } + for (const [namespace, s] of this.sources) this.closeSource(namespace, s, reason); this.sources.clear(); try { this.res.end(); @@ -431,8 +495,14 @@ function refuseBeyondCap(res: http.ServerResponse, cap: number): void { * bounded by `maxListeners` (refusal beyond it, never an eviction). */ export class SseFanInDriver implements EventFanInDriver { private readonly connections = new Set(); + /** #1771 — this service's census (a silent one when none was injected). */ + readonly census: FanInCensus; + private nextConnection = 0; + private shuttingDown = false; - constructor(private readonly deps: SseFanInDriverDeps) {} + constructor(private readonly deps: SseFanInDriverDeps) { + this.census = deps.census ?? new FanInCensus(); + } handle(req: http.IncomingMessage, res: http.ServerResponse): boolean { // #1772: beyond the cap the listener is REFUSED — answered here (so dispatch @@ -441,19 +511,24 @@ export class SseFanInDriver implements EventFanInDriver { const cap = this.deps.maxListeners ?? DEFAULT_FANIN_MAX_LISTENERS; if (this.connections.size >= cap) { refuseBeyondCap(res, cap); + this.census.listenerRefused(cap); return true; } // #1565: always accept on the observation path. Zero non-local owners starts // in fleet-of-one mode (the aggregator's §D4 byte-identity relay); reconcile // opens peer arms as the OwnerMapFeed discovers them. - const conn = new FanInConnection(this.deps, req, res); + const conn = new FanInConnection(`c${++this.nextConnection}`, this.deps, this.census, req, res); this.connections.add(conn); + this.census.listenerAccepted(conn); conn.start(); // A departing listener closes ONLY its own connection (and so only its own - // upstream arms); every other listener keeps streaming. + // upstream arms); every other listener keeps streaming. #1771: a close the + // service's own teardown caused is attributed to `shutdown`. res.on("close", () => { this.connections.delete(conn); - conn.close(); + const reason: FanInCloseReason = this.shuttingDown ? "shutdown" : "listener-closed"; + conn.close(reason); + this.census.listenerClosed(conn.id, reason); }); return true; } @@ -464,4 +539,12 @@ export class SseFanInDriver implements EventFanInDriver { reconcile(): void { for (const conn of this.connections) conn.reconcile(); } + + /** #1771 — the service is stopping: the listener and arm closes its socket + * teardown triggers next are attributed `shutdown` (not `listener-closed`), + * and the census cadence stops. Closes nothing itself — delivery unchanged. */ + shutdown(): void { + this.shuttingDown = true; + this.census.stop(); + } } diff --git a/packages/extension/src/amicode_service_runner.ts b/packages/extension/src/amicode_service_runner.ts index 5575a35fa..b3edfc416 100644 --- a/packages/extension/src/amicode_service_runner.ts +++ b/packages/extension/src/amicode_service_runner.ts @@ -555,6 +555,8 @@ export async function bootAmicodeServiceRunner(opts: AmicodeServiceRunnerOptions engine: { password: unarmed ? undefined : password, getUrl: () => engineUrl }, shelf: { distRoot: opts.appDistRoot }, ...(hubFleet ? { fleet: hubFleet } : {}), + // #1771: the fan-in census + owner-map refresh outcomes → the runner's log. + log, }); const servicePort = opts.servicePort ?? 4095; let url: URL; diff --git a/packages/extension/src/amicode_service_wiring.ts b/packages/extension/src/amicode_service_wiring.ts index 77193ee00..d60e86a90 100644 --- a/packages/extension/src/amicode_service_wiring.ts +++ b/packages/extension/src/amicode_service_wiring.ts @@ -331,6 +331,9 @@ export async function startAmicodeService( shelf: opts.appDistRoot !== undefined ? { distRoot: opts.appDistRoot } : undefined, fleet: fleet ?? opts.fleet, modelRouting: opts.modelRouting, + // #1771: the fan-in census + owner-map refresh outcomes → the Amicode + // output channel. + log: (line) => log.appendLine(line), }); let url: URL; if (opts.port && opts.port > 0) { diff --git a/packages/extension/test/amicode_service_observe_event_routing.test.ts b/packages/extension/test/amicode_service_observe_event_routing.test.ts index e8591bb42..4ca15ca07 100644 --- a/packages/extension/test/amicode_service_observe_event_routing.test.ts +++ b/packages/extension/test/amicode_service_observe_event_routing.test.ts @@ -10,7 +10,7 @@ // The route dispatch → the observation `/event` interception → the #1511 // aggregator → the real downstream `res` is exercised deterministically via an // INJECTED upstream opener; the live cross-machine delivery is the opt-in E2E. -import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { describe, it, expect, beforeAll, afterAll, vi } from "vitest"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -358,6 +358,30 @@ describe("#1543 — production wiring (createAmicodeService observation-only pat await svc.stop(); } }); + + it("#1771: the observation fan-in's census goes to the service log every 30 s, and stops with the service", async () => { + vi.useFakeTimers({ toFake: ["setInterval", "clearInterval"] }); + const lines: string[] = []; + try { + const svc = createAmicodeService({ + password: PW, + engine: { password: "engine-mint", getUrl: () => engineStub.url }, + fleet: { hub: { getUrl: () => undefined }, observationOnly: true, fleetPeers: servingPeerProvider() }, + log: (line) => lines.push(line), + }); + const census = () => lines.filter((l) => l.startsWith("[fanin] census ")); + expect(census()).toHaveLength(0); + vi.advanceTimersByTime(30_000); + expect(census()).toHaveLength(1); + expect(census()[0]).toContain("plane=observe"); + expect(census()[0]).toContain("listeners=0 accepted=0 closed=0 evictions=0"); + await svc.stop(); + vi.advanceTimersByTime(90_000); + expect(census()).toHaveLength(1); // the cadence is torn down with the service + } finally { + vi.useRealTimers(); + } + }); }); // ══════════════════════════════════════════════════════════════════════════════ diff --git a/packages/extension/test/fixtures/engine_stream/engine_frames.ts b/packages/extension/test/fixtures/engine_stream/engine_frames.ts new file mode 100644 index 000000000..1321e2ed5 --- /dev/null +++ b/packages/extension/test/fixtures/engine_stream/engine_frames.ts @@ -0,0 +1,281 @@ +// ============================================================================ +// #1771 — ENGINE-SHAPED global-stream fixtures (shared test surface; #1773 and +// #1779 build on these). +// +// The fan-in suites used to feed synthetic `event: message` / `id: 1` frames — +// a wire the engine never sends. These builders produce the frames the engine's +// `/global/event` route ACTUALLY writes, so the suites exercise production's +// wire format: +// +// - The route serializes each global event once with `JSON.stringify(event)` +// (`{directory, project, workspace?, payload}`, payload `{id, type, +// properties}`) and writes it through effect's SSE encoder: an `id:` line +// when the event has one, NO `event:` line (the event name is "message", +// which the encoder omits), one `data:` line, then the blank line. +// - LEGACY (pre-#1776) engines send NO `id:` line at all — every frame is +// id-less. This is what a peer running an old engine sends (#1773 needs it). +// - SEQUENCED (#1776) engines stamp `id: :`. Bus events take a +// strictly increasing seq; the per-connection control frames +// (`server.connected`, `server.heartbeat`) consume NO seq and carry the +// CURRENT one, so ids on one connection never decrease. +// - A stored (durable) event is emitted TWICE by the event-v2 bridge: the +// plain event, then its `sync` copy (`payload.type === "sync"`, the source +// under `syncEvent`). The copy shares its source's seq, so on a sequenced +// engine both frames carry the SAME id. +// - A per-turn summary (`message.updated` with `info.summary.diffs[].patch`) +// can be ~10 MB in ONE frame (measured 12.3 MB on the Studio), which the +// network delivers in 64 KB chunks. +// +// Pure builders + one stateful `fakeEngine` that sequences like the engine. +// Deterministic by construction: no clock or randomness unless injected. +// ============================================================================ +import * as http from "node:http"; +import type { AddressInfo } from "node:net"; + +/** A representative engine process epoch (`randomBytes(6).toString("base64url")`). */ +export const LOCAL_EPOCH = "bG9jYWwx"; +/** A second process epoch — a peer's engine. */ +export const PEER_EPOCH = "c3R1ZGlv"; + +export const ENGINE_DIRECTORY = "/Users/jj/harmoniqs/amicode"; +export const ENGINE_PROJECT = "prj_0fixture"; + +/** The payload of one global event (what the engine's bus carries). */ +export interface EnginePayload { + id?: string; + type: string; + properties?: Record; + syncEvent?: Record; + [key: string]: unknown; +} + +/** One global event as the engine serializes it (`JSON.stringify(event)`). */ +export interface EngineGlobalEvent { + directory?: string; + project?: string; + workspace?: string; + payload: EnginePayload; +} + +/** The #1776 global-stream frame id: `:`. */ +export function engineId(epoch: string, seq: number): string { + return `${epoch}:${seq}`; +} + +/** Serialize one global event as the engine's SSE encoder writes it: an `id:` + * line only when `id` is given (sequenced engines), no `event:` line, one + * `data:` line, the blank-line terminator. */ +export function engineFrame(event: EngineGlobalEvent, opts: { id?: string } = {}): string { + const idLine = opts.id !== undefined ? `id: ${opts.id}\n` : ""; + return `${idLine}data: ${JSON.stringify(event)}\n\n`; +} + +/** A bus event as the event-v2 bridge emits it (`{directory, project, + * workspace?, payload: {id, type, properties}}` — key order preserved). */ +export function busEvent(opts: { + id: string; + type: string; + properties: Record; + directory?: string; + project?: string; + workspace?: string; +}): EngineGlobalEvent { + return { + directory: opts.directory ?? ENGINE_DIRECTORY, + project: opts.project ?? ENGINE_PROJECT, + ...(opts.workspace !== undefined ? { workspace: opts.workspace } : {}), + payload: { id: opts.id, type: opts.type, properties: opts.properties }, + }; +} + +/** The `sync` copy the bridge emits right after a durable event. The bus + * assigns the copy's `payload.id` from `syncEvent.id` at emit (so the key lands + * AFTER `syncEvent`, exactly as the engine serializes it). */ +export function syncCopyOf( + source: EngineGlobalEvent, + durable: { seq: number; aggregateID: string; version?: number }, +): EngineGlobalEvent { + const id = source.payload.id; + return { + ...(source.directory !== undefined ? { directory: source.directory } : {}), + ...(source.project !== undefined ? { project: source.project } : {}), + ...(source.workspace !== undefined ? { workspace: source.workspace } : {}), + payload: { + type: "sync", + syncEvent: { + id, + type: `${source.payload.type}.${durable.version ?? 1}`, + seq: durable.seq, + aggregateID: durable.aggregateID, + data: source.payload.properties ?? {}, + }, + id, + }, + }; +} + +/** A per-connection control event (`server.connected` / `server.heartbeat` / + * `server.cursor.unservable`): payload only, no directory. */ +export function controlEvent(id: string, type: string, properties: Record = {}): EngineGlobalEvent { + return { payload: { id, type, properties } }; +} + +// ── a stateful engine that sequences like the real one ────────────────────── + +export interface FakeEngine { + /** The process epoch, or undefined for a LEGACY (id-less) engine. */ + readonly epoch: string | undefined; + /** The current sequence (the last bus event's), 0 before any. */ + readonly seq: number; + /** The SSE id a sequenced engine stamps for `seq`; undefined when legacy. */ + idAt(seq: number): string | undefined; + /** The `server.connected` frame (first frame of every connection). */ + connected(): string; + /** A heartbeat frame (legacy: empty properties; sequenced: `{time, seq}`). */ + heartbeat(): string; + /** One non-durable bus event frame (consumes one seq). */ + event(type: string, properties: Record): string; + /** A stored (durable) event: the plain frame then its `sync` twin — one seq, + * shared (on a sequenced engine both frames carry the same id). */ + stored(type: string, properties: Record, durable: { aggregateID: string; durableSeq?: number }): [plain: string, sync: string]; +} + +/** An engine stream: `{ legacy: true }` (pre-#1776, NO `id:` lines) or + * `{ epoch }` (#1776, `id: :`). `now` stamps heartbeat time. */ +export function fakeEngine( + opts: { legacy: true; now?: () => number; directory?: string } | { epoch: string; now?: () => number; directory?: string }, +): FakeEngine { + const epoch = "epoch" in opts ? opts.epoch : undefined; + const now = opts.now ?? (() => 0); + const directory = opts.directory ?? ENGINE_DIRECTORY; + let seq = 0; + let evt = 0; + const nextPayloadId = (): string => `evt_${String(++evt).padStart(6, "0")}`; + const idAt = (s: number): string | undefined => (epoch === undefined ? undefined : engineId(epoch, s)); + const frame = (event: EngineGlobalEvent, s: number): string => { + const id = idAt(s); + return engineFrame(event, id !== undefined ? { id } : {}); + }; + let durableSeq = 0; + return { + epoch, + get seq() { + return seq; + }, + idAt, + connected: () => frame(controlEvent(nextPayloadId(), "server.connected"), seq), + heartbeat: () => + frame(controlEvent(nextPayloadId(), "server.heartbeat", epoch === undefined ? {} : { time: now(), seq }), seq), + event: (type, properties) => { + const s = ++seq; + return frame(busEvent({ id: nextPayloadId(), type, properties, directory }), s); + }, + stored: (type, properties, durable) => { + const s = ++seq; + const source = busEvent({ id: nextPayloadId(), type, properties, directory }); + const twin = syncCopyOf(source, { seq: durable.durableSeq ?? ++durableSeq, aggregateID: durable.aggregateID }); + return [frame(source, s), frame(twin, s)]; + }, + }; +} + +/** A one-off sequenced bus-event frame at an explicit `seq` — for suites that + * script many namespaces' positions directly. `properties` ride verbatim. */ +export function sequencedEvent(epoch: string, seq: number, properties: Record, type = "message.part.updated"): string { + return engineFrame(busEvent({ id: `evt_${epoch}_${seq}`, type, properties }), { id: engineId(epoch, seq) }); +} + +/** A one-off LEGACY (id-less) bus-event frame. */ +export function legacyEvent(properties: Record, type = "message.part.updated", payloadId = "evt_legacy"): string { + return engineFrame(busEvent({ id: payloadId, type, properties })); +} + +// ── the large frame + its chunked delivery ────────────────────────────────── + +export const TEN_MB = 10 * 1024 * 1024; +export const CHUNK_64K = 64 * 1024; + +/** A `message.updated` frame whose `info.summary.diffs[].patch` text brings the + * frame to at least `targetBytes` UTF-8 bytes (default 10 MB) — the shape of + * the measured multi-MB summary events. The patch text carries non-ASCII + * characters, so bytes ≠ string length and a 64 KB byte-chunk boundary can + * fall inside a code point (as it does on the wire). */ +export function largeSummaryFrame(opts: { targetBytes?: number; epoch?: string; seq?: number } = {}): string { + const target = opts.targetBytes ?? TEN_MB; + const lines: string[] = []; + for (let i = 0; i < 1400; i++) lines.push(`+ const row${i} = "résumé → ${"x".repeat(60)}";`); + const patch = `@@ -1,0 +1,1400 @@\n${lines.join("\n")}\n`; + // one diff entry's serialized size (patch JSON-escapes its newlines) + const perDiff = Buffer.byteLength(JSON.stringify(patch)) + 96; + const count = Math.max(1, Math.ceil(target / perDiff)); + const diffs = Array.from({ length: count }, (_, i) => ({ + file: `src/module_${i}.ts`, + patch, + additions: 1400, + deletions: 0, + status: "modified", + })); + const event = busEvent({ + id: "evt_summary", + type: "message.updated", + properties: { info: { id: "msg_summary", sessionID: "ses_large", role: "user", summary: { diffs } } }, + }); + const id = opts.epoch !== undefined ? engineId(opts.epoch, opts.seq ?? 1) : undefined; + return engineFrame(event, id !== undefined ? { id } : {}); +} + +/** Split text's UTF-8 BYTES into `chunkSize` pieces (the last may be short) — + * how the network delivers a large frame. Concatenating the chunks yields the + * exact original bytes. */ +export function chunkBytes(text: string, chunkSize = CHUNK_64K): Buffer[] { + const bytes = Buffer.from(text, "utf8"); + const out: Buffer[] = []; + for (let off = 0; off < bytes.length; off += chunkSize) out.push(bytes.subarray(off, off + chunkSize)); + return out; +} + +// ── a scripted engine `/global/event` server (127.0.0.1, port 0) ──────────── + +export interface EngineSseServer { + /** `http://127.0.0.1:` — dial THIS address (never a wildcard bind). */ + url: string; + /** Every request's path + query, in arrival order. */ + requests: string[]; + stop(): Promise; +} + +/** Serve `/global/event` from a script: `script(res)` writes the stream (whole + * frames, or raw chunks) and decides how it ends (`res.end()` = a clean + * upstream end; `res.destroy()` = an upstream error). Bound to 127.0.0.1 on an + * ephemeral port and dialed at its own address, so no foreign listener can + * share it. */ +export function startEngineSseServer(script: (res: http.ServerResponse, req: http.IncomingMessage) => void | Promise): Promise { + const requests: string[] = []; + const server = http.createServer((req, res) => { + requests.push(req.url ?? "/"); + res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache, no-transform" }); + res.flushHeaders(); + void script(res, req); + }); + return new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => { + const { port } = server.address() as AddressInfo; + resolve({ + url: `http://127.0.0.1:${port}`, + requests, + stop: () => + new Promise((r) => { + server.close(() => r()); + server.closeAllConnections?.(); + }), + }); + }); + }); +} + +/** Write chunks one at a time, honoring backpressure (wait for `drain`). */ +export async function writeChunks(res: http.ServerResponse, chunks: Buffer[]): Promise { + for (const c of chunks) { + if (!res.write(c)) await new Promise((r) => res.once("drain", () => r())); + } +} diff --git a/packages/extension/test/sse_fanin_driver_pump.test.ts b/packages/extension/test/sse_fanin_driver_pump.test.ts index a903c8b42..e0ec54f5a 100644 --- a/packages/extension/test/sse_fanin_driver_pump.test.ts +++ b/packages/extension/test/sse_fanin_driver_pump.test.ts @@ -3,12 +3,14 @@ // pump() method removes it from the internal sources Map so the next // reconcile() tick can re-open the arm — not leave a zombie that blocks // reconnection indefinitely. -import { describe, it, expect } from "vitest"; +import { describe, it, expect, vi, afterEach } from "vitest"; import * as http from "node:http"; import { SseFanInDriver } from "../src/amicode_service/sse_fanin_driver"; import type { SseFrameSource } from "../src/amicode_service/sse_fanin_aggregator"; import { LOCAL_NAMESPACE } from "../src/amicode_service/sse_fanin_aggregator"; import { SessionOwnerMap } from "../src/amicode_service/session_multiplexer"; +import { FanInCensus, FANIN_CENSUS_INTERVAL_MS } from "../src/amicode_service/fanin_census"; +import { fakeEngine, LOCAL_EPOCH, PEER_EPOCH } from "./fixtures/engine_stream/engine_frames"; // ── helpers ────────────────────────────────────────────────────────────────── @@ -1270,3 +1272,153 @@ describe("#1772 invariants — #1264 / #1601 / #1617 hold PER LISTENER", () => { resB.fireClose(); }); }); + +// ══════════════════════════════════════════════════════════════════════════════ +// #1771 — the fan-in CENSUS. A wedged, evicted, or flapping arm used to look +// identical to a quiet one in every log: nothing recorded who subscribed, why an +// arm re-opened, or what was dropped. Every fan-in service now keeps counters per +// listener connection and per arm, logs one census line every 30 s, and one line +// immediately on each transition. The upstream frames below are ENGINE-SHAPED +// (test/fixtures/engine_stream): id-less or `:` data frames, the +// plain + sync twin of a stored event, heartbeats. +// ══════════════════════════════════════════════════════════════════════════════ + +/** A census wired to a collecting sink and a hand-driven clock. */ +function testCensus(plane = "observe") { + const lines: string[] = []; + let t = 1_000_000; + const census = new FanInCensus({ plane, sink: (l) => lines.push(l), now: () => t }); + return { + census, + lines, + advance(ms: number) { + t += ms; + }, + censusLines: () => lines.filter((l) => l.startsWith("[fanin] census ")), + }; +} + +/** One `key=value key=value` segment → a record. */ +function kv(segment: string): Record { + const out: Record = {}; + for (const tok of segment.trim().split(" ")) { + const i = tok.indexOf("="); + if (i > 0) out[tok.slice(0, i)] = tok.slice(i + 1); + } + return out; +} + +/** A census line: its service-level head, then one segment per (connection, arm). */ +function parseCensus(line: string): { head: Record; arms: Array> } { + const [head, ...arms] = line.split(" | "); + return { head: kv(head), arms: arms.map(kv) }; +} + +/** Drain the pumps: every pending microtask runs before the immediate fires — + * no wall-clock sleep. */ +const flush = () => new Promise((r) => setImmediate(r)); + +const bytesOf = (frames: string[]) => frames.reduce((n, f) => n + Buffer.byteLength(f), 0); + +describe("#1771 AC1 — a fan-in service logs a census line every 30 s", () => { + afterEach(() => vi.useRealTimers()); + + it("listeners, accepted/closed since boot, evictions, and per arm: frames, bytes, largest, time since last frame, opens, closes", async () => { + vi.useFakeTimers({ toFake: ["setInterval", "clearInterval"] }); + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const origins = fakeOrigins(); + const { census, advance, censusLines } = testCensus(); + census.start(); + const driver = twoListenerDriver(origins, ownerMap, { census }); + const local = fakeEngine({ epoch: LOCAL_EPOCH }); + const studio = fakeEngine({ epoch: PEER_EPOCH }); + + const resA = mockRes(); + const resB = mockRes(); + driver.handle(mockReq("/global/event"), resA); + driver.handle(mockReq("/global/event"), resB); + await flush(); + + const localFrames = [ + local.connected(), + local.event("session.status", { sessionID: "ses_1", status: { type: "busy" } }), + ...local.stored("message.part.updated", { part: { id: "prt_1", text: "bravo" } }, { aggregateID: "ses_1" }), + local.heartbeat(), + ]; + for (const f of localFrames) origins.broadcast(LOCAL_NAMESPACE, f); + const studioFrame = studio.event("message.part.updated", { from: "studio" }); + origins.broadcast("studio", studioFrame); + await flush(); + resA.fireClose(); // listener A departs; B stays + await flush(); + advance(1500); + + expect(censusLines()).toHaveLength(0); // nothing before the first period + vi.advanceTimersByTime(FANIN_CENSUS_INTERVAL_MS); + expect(censusLines()).toHaveLength(1); + + const { head, arms } = parseCensus(censusLines()[0]); + expect(head).toMatchObject({ plane: "observe", listeners: "1", accepted: "2", closed: "1", evictions: "0" }); + // Only the LIVE listener's arms are listed: B's local arm and B's peer arm. + expect(arms.map((a) => a.arm).sort()).toEqual(["local", "studio"]); + expect(new Set(arms.map((a) => a.conn)).size).toBe(1); + const bLocal = arms.find((a) => a.arm === "local")!; + expect(bLocal).toMatchObject({ + frames: "5", + bytes: String(bytesOf(localFrames)), + largest: String(Math.max(...localFrames.map((f) => Buffer.byteLength(f)))), + since_last_ms: "1500", + opens: "1", + closes: "-", + pending: "0", + drops: "0", + gaps: "0", + }); + expect(arms.find((a) => a.arm === "studio")).toMatchObject({ + frames: "1", + bytes: String(Buffer.byteLength(studioFrame)), + largest: String(Buffer.byteLength(studioFrame)), + since_last_ms: "1500", + opens: "1", + closes: "-", + }); + + // The cadence repeats every 30 s, and stops with the census. + vi.advanceTimersByTime(FANIN_CENSUS_INTERVAL_MS); + expect(censusLines()).toHaveLength(2); + census.stop(); + vi.advanceTimersByTime(FANIN_CENSUS_INTERVAL_MS * 3); + expect(censusLines()).toHaveLength(2); + + resB.fireClose(); + }); + + it("per arm: pending depth while backpressured, overflow drops, and gaps emitted", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const origins = fakeOrigins(); // its sources cannot pause → frames keep arriving + const { census, censusLines } = testCensus(); + const driver = twoListenerDriver(origins, ownerMap, { census }); + const studio = fakeEngine({ epoch: PEER_EPOCH }); + + const res = backpressureRes(); + driver.handle(mockReq("/global/event"), res); + await flush(); + const studioArm = () => parseCensus(censusLines().at(-1)!).arms.find((a) => a.arm === "studio")!; + + res.block(); + // 1 trip-frame written, 256 buffered (the per-namespace bound), 43 dropped. + for (let i = 0; i < 300; i++) origins.broadcast("studio", studio.event("message.part.delta", { i })); + await flush(); + census.emit(); + expect(studioArm()).toMatchObject({ frames: "300", pending: "256", drops: "43", gaps: "0" }); + + res.fireDrain(); // the buffer flushes in order, then ONE overflow gap names the arm + await flush(); + census.emit(); + expect(studioArm()).toMatchObject({ frames: "300", pending: "0", drops: "43", gaps: "1" }); + + res.fireClose(); + }); +}); From 7062f9820127495d799acd9ca34a597003da8668 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 17:56:20 -0400 Subject: [PATCH 18/60] test(fleet): census close reasons are the enumerated six; the real reader tells upstream end from error (#1771) --- .../test/sse_fanin_driver_pump.test.ts | 156 +++++++++++++++++- 1 file changed, 154 insertions(+), 2 deletions(-) diff --git a/packages/extension/test/sse_fanin_driver_pump.test.ts b/packages/extension/test/sse_fanin_driver_pump.test.ts index e0ec54f5a..67ec5b59e 100644 --- a/packages/extension/test/sse_fanin_driver_pump.test.ts +++ b/packages/extension/test/sse_fanin_driver_pump.test.ts @@ -9,8 +9,8 @@ import { SseFanInDriver } from "../src/amicode_service/sse_fanin_driver"; import type { SseFrameSource } from "../src/amicode_service/sse_fanin_aggregator"; import { LOCAL_NAMESPACE } from "../src/amicode_service/sse_fanin_aggregator"; import { SessionOwnerMap } from "../src/amicode_service/session_multiplexer"; -import { FanInCensus, FANIN_CENSUS_INTERVAL_MS } from "../src/amicode_service/fanin_census"; -import { fakeEngine, LOCAL_EPOCH, PEER_EPOCH } from "./fixtures/engine_stream/engine_frames"; +import { FanInCensus, FANIN_CENSUS_INTERVAL_MS, FANIN_CLOSE_REASONS } from "../src/amicode_service/fanin_census"; +import { fakeEngine, startEngineSseServer, LOCAL_EPOCH, PEER_EPOCH } from "./fixtures/engine_stream/engine_frames"; // ── helpers ────────────────────────────────────────────────────────────────── @@ -1422,3 +1422,155 @@ describe("#1771 AC1 — a fan-in service logs a census line every 30 s", () => { res.fireClose(); }); }); + +/** Resolve once the census sink has received a line matching `pred` — + * event-driven (the sink notifies), never a polling sleep. */ +function lineWaiter(lines: string[]): { sink: (l: string) => void; waitFor(pred: (l: string) => boolean): Promise } { + const waiters: Array<{ pred: (l: string) => boolean; resolve: (l: string) => void }> = []; + return { + sink(l: string) { + lines.push(l); + for (const w of [...waiters]) { + if (w.pred(l)) { + waiters.splice(waiters.indexOf(w), 1); + w.resolve(l); + } + } + }, + waitFor(pred) { + const hit = lines.find(pred); + if (hit !== undefined) return Promise.resolve(hit); + return new Promise((resolve) => waiters.push({ pred, resolve })); + }, + }; +} + +describe("#1771 AC2 — close reasons are an enumerated set", () => { + it("the set is exactly: upstream end, upstream error, membership change, listener closed, evicted, shutdown", () => { + expect([...FANIN_CLOSE_REASONS]).toEqual([ + "upstream-end", + "upstream-error", + "membership-change", + "listener-closed", + "evicted", + "shutdown", + ]); + }); + + it("an arm's closes are counted under their reason: upstream end, upstream error, membership change", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const { census, lines, censusLines } = testCensus(); + const localCtls: Ctl[] = []; + const failing: Array<(err: Error) => void> = []; + const driver = twoListenerDriver(fakeOrigins(), ownerMap, { + census, + openUpstream: (r) => { + if (r.namespace === LOCAL_NAMESPACE) { + const ctl = controllableSource(); + localCtls.push(ctl); + return ctl.source; + } + // the peer's upstream FAILS: next() rejects + return { + next: () => new Promise((_, reject) => failing.push(reject)), + close() {}, + }; + }, + }); + const res = mockRes(); + driver.handle(mockReq("/global/event"), res); + await flush(); + + localCtls[0].end(); // the local engine ends the stream cleanly + failing.shift()!(new Error("ECONNRESET")); // the peer's stream errors + await flush(); + expect(lines).toContainEqual(expect.stringMatching(/^\[fanin\] arm-closed plane=observe conn=c1 arm=local reason=upstream-end$/)); + expect(lines).toContainEqual(expect.stringMatching(/^\[fanin\] arm-closed plane=observe conn=c1 arm=studio reason=upstream-error$/)); + + driver.reconcile(); // both re-open + await flush(); + ownerMap.update([]); // the peer's sessions end → its arm leaves the membership + driver.reconcile(); + await flush(); + expect(lines).toContainEqual(expect.stringMatching(/^\[fanin\] arm-closed plane=observe conn=c1 arm=studio reason=membership-change$/)); + + census.emit(); + const arms = parseCensus(censusLines().at(-1)!).arms; + expect(arms.find((a) => a.arm === "local")).toMatchObject({ opens: "2", closes: "upstream-end:1" }); + expect(arms.find((a) => a.arm === "studio")).toMatchObject({ opens: "2", closes: "upstream-error:1,membership-change:1" }); + expect(parseCensus(censusLines().at(-1)!).head.evictions).toBe("0"); // no eviction path exists (#1772) + + localCtls.forEach((c) => c.end()); + res.fireClose(); + }); + + it("the REAL upstream reader tells a clean end (upstream-end) from a mid-stream failure (upstream-error)", async () => { + const local = fakeEngine({ legacy: true }); // an old engine: id-less frames + const peer = fakeEngine({ legacy: true }); + const ends = await startEngineSseServer((res) => { + res.write(local.connected()); + res.end(); // clean end of stream + }); + const dies = await startEngineSseServer((res) => { + res.write(peer.connected()); + setImmediate(() => res.destroy()); // the link dies mid-stream + }); + const lines: string[] = []; + const w = lineWaiter(lines); + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const driver = new SseFanInDriver({ + ownerMap, + localMachineId: "macbook", + localEventUrl: () => ends.url, + peerBaseUrl: (id) => (id === "studio" ? dies.url : undefined), + peerToken: (id) => + id === "studio" + ? { ok: true as const, credential: { baseUrl: dies.url, token: "tok-studio" } } + : { ok: false as const, reason: "absent" as const }, + reconcileMs: 999_999, + census: new FanInCensus({ plane: "observe", sink: w.sink }), + // no openUpstream → the production http reader + }); + const res = mockRes(); + try { + driver.handle(mockReq("/global/event"), res); + expect(await w.waitFor((l) => l.includes("arm=local reason="))).toMatch(/reason=upstream-end$/); + expect(await w.waitFor((l) => l.includes("arm=studio reason="))).toMatch(/reason=upstream-error$/); + } finally { + res.fireClose(); + await ends.stop(); + await dies.stop(); + } + }); + + it("a departing listener's arms close as listener-closed; after shutdown() the teardown's closes are shutdown", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const { census, lines } = testCensus(); + const driver = twoListenerDriver(fakeOrigins(), ownerMap, { census }); + const resA = mockRes(); + const resB = mockRes(); + driver.handle(mockReq("/global/event"), resA); + driver.handle(mockReq("/global/event"), resB); + await flush(); + + resA.fireClose(); // a view goes away + expect(lines.filter((l) => l.includes("conn=c1") && l.includes("reason="))).toEqual([ + "[fanin] arm-closed plane=observe conn=c1 arm=local reason=listener-closed", + "[fanin] arm-closed plane=observe conn=c1 arm=studio reason=listener-closed", + "[fanin] listener-closed plane=observe conn=c1 reason=listener-closed listeners=1", + ]); + + driver.shutdown(); // the service stops: its socket teardown closes the rest + resB.fireClose(); + expect(lines.filter((l) => l.includes("conn=c2") && l.includes("reason="))).toEqual([ + "[fanin] arm-closed plane=observe conn=c2 arm=local reason=shutdown", + "[fanin] arm-closed plane=observe conn=c2 arm=studio reason=shutdown", + "[fanin] listener-closed plane=observe conn=c2 reason=shutdown listeners=0", + ]); + // shutdown() closed nothing itself — B's stream ended only with its socket. + expect(lines.some((l) => l.includes("reason=evicted"))).toBe(false); + }); +}); From 5f8349f1394fd61c38bd50c51a89da21a6d923e3 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 17:57:30 -0400 Subject: [PATCH 19/60] test(fleet): census transitions log one line immediately with conn + arm; never per frame (#1771) --- .../test/sse_fanin_aggregator.test.ts | 104 ++++++++++++++++++ .../test/sse_fanin_driver_pump.test.ts | 86 +++++++++++++++ 2 files changed, 190 insertions(+) diff --git a/packages/extension/test/sse_fanin_aggregator.test.ts b/packages/extension/test/sse_fanin_aggregator.test.ts index bfa3ca665..1690d1789 100644 --- a/packages/extension/test/sse_fanin_aggregator.test.ts +++ b/packages/extension/test/sse_fanin_aggregator.test.ts @@ -41,6 +41,7 @@ import { import { SessionOwnerMap } from "../src/amicode_service/session_multiplexer"; import { peerAuthHeader } from "../src/amicode_service/merged_projection"; import type { PeerTokenRead } from "../src/amicode_service/fleet_peer_store"; +import { fakeEngine, sequencedEvent, LOCAL_EPOCH, PEER_EPOCH } from "./fixtures/engine_stream/engine_frames"; // ── test helpers ────────────────────────────────────────────────────────────── @@ -943,3 +944,106 @@ describe("#1638 — control/gap/focus writes are return-checked and drain in tru expect(n2Idx).toBeLessThan(gapIdx); // control frame AFTER the preceding data frame }); }); + +// ══════════════════════════════════════════════════════════════════════════════ +// #1771 — the aggregator's census counters (per arm, per connection) and its +// transition observer. Observation only: the frames on the sink are unchanged. +// ══════════════════════════════════════════════════════════════════════════════ +describe("#1771 — per-arm traffic counters, pending depth, and the drop/gap observer", () => { + function blockableSink() { + const chunks: string[] = []; + let blocked = false; + return { + write(chunk: string): boolean { + chunks.push(chunk); + return !blocked; + }, + flush() {}, + block() { + blocked = true; + }, + unblock() { + blocked = false; + }, + text: () => chunks.join(""), + }; + } + + it("counts every ingested frame per arm — frames, UTF-8 bytes, largest, last-frame time — whether delivered or buffered", () => { + let t = 500; + const sink = blockableSink(); + const agg = new SseFanInAggregator({ sink, now: () => t }); + agg.connect(); + agg.setMembership([{ machineId: "studio", reachable: true, token: "tok-studio" }]); + const local = fakeEngine({ epoch: LOCAL_EPOCH }); + const studio = fakeEngine({ legacy: true }); // an old peer engine: id-less frames + const localFrames = [local.connected(), ...local.stored("message.part.updated", { text: "résumé" }, { aggregateID: "ses_1" })]; + for (const f of localFrames) agg.ingest(LOCAL_NAMESPACE, f); + t = 900; + sink.block(); + const studioFrames = [studio.event("session.status", { status: "busy" }), studio.heartbeat()]; + for (const f of studioFrames) agg.ingest("studio", f); // the 1st trips backpressure, the 2nd buffers + + expect(agg.traffic(LOCAL_NAMESPACE)).toEqual({ + frames: 3, + bytes: localFrames.reduce((n, f) => n + Buffer.byteLength(f), 0), + largestFrame: Math.max(...localFrames.map((f) => Buffer.byteLength(f))), + lastFrameAt: 500, + drops: 0, + gaps: 0, + }); + expect(agg.traffic("studio")).toMatchObject({ frames: 2, lastFrameAt: 900, drops: 0 }); + expect(agg.pendingDepth("studio")).toBe(1); + expect(agg.pendingDepth(LOCAL_NAMESPACE)).toBe(0); + expect(agg.traffic("mini")).toEqual({ frames: 0, bytes: 0, largestFrame: 0, drops: 0, gaps: 0 }); // never ingested + }); + + it("the observer sees the FIRST drop of an overflow episode, then the gap that ends it; drops stay cumulative", () => { + const sink = blockableSink(); + const drops: Array<[string, number]> = []; + const gaps: Array<[string[], string]> = []; + const agg = new SseFanInAggregator({ + sink, + bufferBound: 2, + observer: { onOverflowDrop: (ns, pending) => drops.push([ns, pending]), onGap: (nss, cause) => gaps.push([nss, cause]) }, + }); + agg.connect(); + agg.setMembership([{ machineId: "studio", reachable: true, token: "tok-studio" }]); + sink.block(); + for (let n = 1; n <= 6; n++) agg.ingest("studio", sequencedEvent(PEER_EPOCH, n, { n })); // 1 written, 2 buffered, 3 dropped + expect(drops).toEqual([["studio", 2]]); // one episode → one callback + expect(agg.traffic("studio").drops).toBe(3); + + sink.unblock(); + agg.resume(); + expect(gaps).toEqual([[["studio"], "overflow"]]); + expect(agg.traffic("studio")).toMatchObject({ drops: 3, gaps: 1 }); + expect(agg.pendingDepth("studio")).toBe(0); + + // a NEW episode after the gap is a new first drop + sink.block(); + for (let n = 7; n <= 10; n++) agg.ingest("studio", sequencedEvent(PEER_EPOCH, n, { n })); + expect(drops).toHaveLength(2); + + agg.reopenArm("studio"); // composite mode, already delivered → a reopen gap + expect(gaps.at(-1)).toEqual([["studio"], "reopen"]); + }); + + it("a throwing observer never breaks delivery", () => { + const sink = collectingSink(); + const agg = new SseFanInAggregator({ + sink, + observer: { + onGap: () => { + throw new Error("broken census"); + }, + }, + }); + agg.connect(); + agg.setMembership([{ machineId: "studio", reachable: true, token: "tok-studio" }]); + const f = sequencedEvent(PEER_EPOCH, 1, { n: 1 }); + agg.ingest("studio", f); + expect(() => agg.reopenArm("studio")).not.toThrow(); + expect(sink.text()).toContain("amicode.sync.gap"); + }); +}); diff --git a/packages/extension/test/sse_fanin_driver_pump.test.ts b/packages/extension/test/sse_fanin_driver_pump.test.ts index 67ec5b59e..53165b90a 100644 --- a/packages/extension/test/sse_fanin_driver_pump.test.ts +++ b/packages/extension/test/sse_fanin_driver_pump.test.ts @@ -1574,3 +1574,89 @@ describe("#1771 AC2 — close reasons are an enumerated set", () => { expect(lines.some((l) => l.includes("reason=evicted"))).toBe(false); }); }); + +describe("#1771 AC3 — each transition logs one line immediately, with its connection and arm", () => { + it("listener accepted → arm opened (local, peer) — synchronously, before any frame or timer", () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const { census, lines } = testCensus(); + const driver = twoListenerDriver(fakeOrigins(), ownerMap, { census }); + const res = mockRes(); + driver.handle(mockReq("/global/event"), res); + // no flush, no timer: the lines are already there + expect(lines).toEqual([ + "[fanin] listener-accepted plane=observe conn=c1 listeners=1", + "[fanin] arm-opened plane=observe conn=c1 arm=local opens=1", + "[fanin] arm-opened plane=observe conn=c1 arm=studio opens=1", + ]); + res.fireClose(); + }); + + it("frames are never logged; an arm that drops and re-opens logs closed → gap emitted → opened", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const { census, lines } = testCensus(); + const peerCtls: Ctl[] = []; + const driver = twoListenerDriver(fakeOrigins(), ownerMap, { + census, + openUpstream: (r) => { + const ctl = controllableSource(); + if (r.namespace === "studio") peerCtls.push(ctl); + return ctl.source; + }, + }); + const studio = fakeEngine({ epoch: PEER_EPOCH }); + const res = mockRes(); + driver.handle(mockReq("/global/event"), res); + const before = lines.length; + for (let i = 0; i < 25; i++) peerCtls[0].push(studio.event("message.part.delta", { i })); + peerCtls[0].push(studio.heartbeat()); + await flush(); + expect(lines.length).toBe(before); // 26 frames delivered, zero lines + + peerCtls[0].end(); + await flush(); + driver.reconcile(); + expect(lines.slice(before)).toEqual([ + "[fanin] arm-closed plane=observe conn=c1 arm=studio reason=upstream-end", + "[fanin] gap-emitted plane=observe conn=c1 arm=studio cause=reopen", + "[fanin] arm-opened plane=observe conn=c1 arm=studio opens=2", + ]); + peerCtls.forEach((c) => c.end()); + res.fireClose(); + }); + + it("an overflow episode logs ONE overflow-drop line (not one per dropped frame), then its gap on drain", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const origins = fakeOrigins(); + const { census, lines } = testCensus(); + const driver = twoListenerDriver(origins, ownerMap, { census }); + const studio = fakeEngine({ epoch: PEER_EPOCH }); + const res = backpressureRes(); + driver.handle(mockReq("/global/event"), res); + await flush(); + const before = lines.length; + + res.block(); + for (let i = 0; i < 300; i++) origins.broadcast("studio", studio.event("message.part.delta", { i })); + await flush(); + expect(lines.slice(before)).toEqual(["[fanin] overflow-drop plane=observe conn=c1 arm=studio pending=256"]); + + res.fireDrain(); + await flush(); + expect(lines.slice(before + 1)).toEqual(["[fanin] gap-emitted plane=observe conn=c1 arm=studio cause=overflow"]); + res.fireClose(); + }); + + it("a beyond-cap refusal logs one line naming the cap (and counts in the census)", () => { + const { census, lines } = testCensus(); + const driver = twoListenerDriver(fakeOrigins(), new SessionOwnerMap(), { census, maxListeners: 1 }); + const ok = mockRes(); + driver.handle(mockReq("/global/event"), ok); + driver.handle(mockReq("/global/event"), mockRes()); + expect(lines.at(-1)).toBe("[fanin] listener-refused plane=observe listeners=1 cap=1"); + expect(census.snapshot()).toMatchObject({ listeners: 1, accepted: 1, refusals: 1, evictions: 0 }); + ok.fireClose(); + }); +}); From c6c0e43066c1d96e477c2933046fe0688ef71340 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 18:03:28 -0400 Subject: [PATCH 20/60] feat(fleet): log each ownership refresh's per-peer outcome, duration and entry count (#1771) --- .../extension/src/amicode_service/index.ts | 14 +- .../src/amicode_service/merged_projection.ts | 131 +++++++++++++++--- .../amicode_service/session_multiplexer.ts | 31 ++++- ...de_service_multiplex_owner_routing.test.ts | 81 +++++++++++ .../test/merged_projection_archived.test.ts | 83 +++++++++++ 5 files changed, 309 insertions(+), 31 deletions(-) diff --git a/packages/extension/src/amicode_service/index.ts b/packages/extension/src/amicode_service/index.ts index 72c7f865c..275fc989b 100644 --- a/packages/extension/src/amicode_service/index.ts +++ b/packages/extension/src/amicode_service/index.ts @@ -61,7 +61,7 @@ import { readPendingRequests, } from "./fleet_control_request"; import { HubCredentialRead, mintRegistry, readHubCredential } from "./hub_credential"; -import { buildMergedProjection, buildFleetProjection, buildOwnerRoutingProjection, type UpstreamMode, type MergedProjection, type FleetProjection } from "./merged_projection"; +import { buildMergedProjection, buildFleetProjection, buildOwnerRoutingProjection, type UpstreamMode, type MergedProjection, type FleetProjection, type OwnerRefreshPeer } from "./merged_projection"; import { buildControlResolver } from "./remote_session_state"; import { FleetPostureDetector, type FleetPostureTuning } from "./fleet_posture"; import { handleFleetWrite, type FleetWriteDeps } from "./fleet_writes"; @@ -1173,7 +1173,7 @@ export function createAmicodeService( if (fleetPeers) { const ownerMapFeed = new OwnerMapFeed({ ownerMap, - buildProjection: async (): Promise<{ sessions: SessionEntry[] }> => { + buildProjection: async (): Promise<{ sessions: SessionEntry[]; peers: OwnerRefreshPeer[] }> => { const peers = fleetPeers.getServingPeers().map((p) => { const tokenRead = fleetPeers.readPeerToken(p.machineId); return { @@ -1188,8 +1188,10 @@ export function createAmicodeService( peers, rosterLookup: fleetPeers.rosterLookup, }); - return { sessions: projection.sessions as SessionEntry[] }; + return { sessions: projection.sessions as SessionEntry[], peers: projection.peers }; }, + // #1771: each refresh's per-peer outcome → the service log. + ...(opts.log ? { log: opts.log } : {}), }); ownerMapFeed.start(); server.registerCleanup(() => ownerMapFeed.stop()); @@ -1366,7 +1368,7 @@ export function createAmicodeService( const ownerMap = new SessionOwnerMap(); const ownerMapFeed = new OwnerMapFeed({ ownerMap, - buildProjection: async (): Promise<{ sessions: SessionEntry[] }> => { + buildProjection: async (): Promise<{ sessions: SessionEntry[]; peers: OwnerRefreshPeer[] }> => { const peers = fleetPeers.getServingPeers().map((p) => { const tokenRead = fleetPeers.readPeerToken(p.machineId); return { @@ -1381,8 +1383,10 @@ export function createAmicodeService( peers, rosterLookup: fleetPeers.rosterLookup, }); - return { sessions: projection.sessions as SessionEntry[] }; + return { sessions: projection.sessions as SessionEntry[], peers: projection.peers }; }, + // #1771: each refresh's per-peer outcome → the service log. + ...(opts.log ? { log: opts.log } : {}), }); ownerMapFeed.start(); server.registerCleanup(() => ownerMapFeed.stop()); diff --git a/packages/extension/src/amicode_service/merged_projection.ts b/packages/extension/src/amicode_service/merged_projection.ts index 03e7efd41..e2446199c 100644 --- a/packages/extension/src/amicode_service/merged_projection.ts +++ b/packages/extension/src/amicode_service/merged_projection.ts @@ -98,6 +98,26 @@ export interface FleetProjectionOptions { archived?: boolean; fetchImpl?: typeof fetch; timeoutMs?: number; + /** #1771: the clock timing each source's fetch (the owner-map refresh's + * per-peer duration). Default Date.now. */ + now?: () => number; +} + +/** #1771: one peer's ownership-refresh outcome — what the OwnerMapFeed logs. + * `timeout` is the fetch guard firing (AbortSignal.timeout); `unauthorized` a + * 401; every other absence (fetch failure, no upstream, missing credential, + * untrusted) is `error`, with its named `reason`. */ +export type OwnerRefreshOutcome = "ok" | "timeout" | "error" | "unauthorized"; + +export interface OwnerRefreshPeer { + machineId: string; + outcome: OwnerRefreshOutcome; + /** The peer's fetch duration (ms; the slower of its active/archived reads). */ + durationMs: number; + /** The peer-owned sessions this refresh put in the routing union. */ + entries: number; + reason?: SourceAbsenceReason; + detail?: string; } /** The fleet-wide projection result — N sources, machine-keyed. */ @@ -213,19 +233,27 @@ async function fetchSessions( * Peers/hub keep the old behavior (false): no credential ⇒ do not fire an * unauthenticated cross-machine request. */ allowUnauthenticated = false, -): Promise<{ record: SourceFetchRecord; entries: Record[] }> { + /** #1771: the clock timing this fetch. */ + now: () => number = Date.now, +): Promise { + const started = now(); + const done = (r: { record: SourceFetchRecord; entries: Record[] }, timedOut = false): SourceFetch => ({ + ...r, + durationMs: Math.max(0, now() - started), + timedOut, + }); const base: SourceFetchRecord = { source: tag, present: false }; const origin = opts.getUrl(); - if (!origin) return { record: { ...base, reason: "no-upstream" }, entries: [] }; + if (!origin) return done({ record: { ...base, reason: "no-upstream" }, entries: [] }); if (!authHeader && !allowUnauthenticated) { - return { + return done({ record: { ...base, reason: tag === "hub" ? "credential-missing" : "credential-missing", detail: tag === "hub" ? "hub credential absent/malformed/incomplete" : "engine mint not armed", }, entries: [], - }; + }); } // Send Authorization only when we actually hold one — the open loopback // engine is queried bare (allowUnauthenticated with no password). @@ -235,13 +263,13 @@ async function fetchSessions( headers: authHeaders, signal: AbortSignal.timeout(timeoutMs), }); - if (res.status === 401) return { record: { ...base, reason: "unauthorized" }, entries: [] }; + if (res.status === 401) return done({ record: { ...base, reason: "unauthorized" }, entries: [] }); if (!res.ok) { - return { record: { ...base, reason: "fetch-failed", detail: `HTTP ${res.status}` }, entries: [] }; + return done({ record: { ...base, reason: "fetch-failed", detail: `HTTP ${res.status}` }, entries: [] }); } const parsed: unknown = await res.json(); if (!Array.isArray(parsed)) { - return { record: { ...base, reason: "fetch-failed", detail: "session list is not an array" }, entries: [] }; + return done({ record: { ...base, reason: "fetch-failed", detail: "session list is not an array" }, entries: [] }); } const entries = parsed.filter( (e): e is Record => typeof e === "object" && e !== null && !Array.isArray(e), @@ -272,15 +300,31 @@ async function fetchSessions( /* the stamp fails open to null — the record stays honest via the field */ } const record: SourceFetchRecord = { source: tag, present: true, count: entries.length, max, sum, version }; - return { record, entries }; + return done({ record, entries }); } catch (e) { - return { - record: { ...base, reason: "fetch-failed", detail: e instanceof Error ? e.message : String(e) }, - entries: [], - }; + // #1771: the fetch guard firing is a TIMEOUT (the record keeps its named + // `fetch-failed` reason; only the refresh outcome distinguishes it). + const timedOut = (e as { name?: unknown } | null)?.name === "TimeoutError"; + return done( + { + record: { ...base, reason: "fetch-failed", detail: e instanceof Error ? e.message : String(e) }, + entries: [], + }, + timedOut, + ); } } +/** One source fetch: the NAMED record + entries, plus (#1771) how long it took + * and whether the fetch guard timed it out. Internal — the served projection + * carries only the record. */ +interface SourceFetch { + record: SourceFetchRecord; + entries: Record[]; + durationMs: number; + timedOut: boolean; +} + /** Merge both stores into ONE provenance-tagged list. The hub wins conflicts * (the store of record for fleet sessions); locals stay tagged local. */ export function mergeSessions( @@ -419,8 +463,15 @@ function mergeNSources( * Never throws: every peer failure is a NAMED record inside an otherwise-valid * projection — the read path degrades by naming, not by vanishing. */ export async function buildFleetProjection(opts: FleetProjectionOptions): Promise { + return (await fleetFanOut(opts)).projection; +} + +/** The fan-out behind buildFleetProjection, also returning each PEER's raw + * fetch (#1771: its duration + timeout flag) in `opts.peers` order. */ +async function fleetFanOut(opts: FleetProjectionOptions): Promise<{ projection: FleetProjection; peerFetches: SourceFetch[] }> { const fetchImpl = opts.fetchImpl ?? fetch; const timeoutMs = opts.timeoutMs ?? 10_000; + const now = opts.now ?? Date.now; const localAuth = opts.local.password !== undefined ? serverAuthHeader(opts.local.password) : undefined; // Fan out: local + all peers in parallel. @@ -436,8 +487,8 @@ export async function buildFleetProjection(opts: FleetProjectionOptions): Promis const fleetEndpoint = opts.archived ? "/experimental/session?archived=true&limit=10000" : "/experimental/session?limit=10000"; - const localPromise = fetchSessions(opts.localMachineId, opts.local, localAuth, fetchImpl, timeoutMs, fleetEndpoint, true); - const peerPromises = opts.peers.map((peer) => { + const localPromise = fetchSessions(opts.localMachineId, opts.local, localAuth, fetchImpl, timeoutMs, fleetEndpoint, true, now); + const peerPromises = opts.peers.map((peer): Promise => { // #1481 (AC1): TRUST gates Observe. An untrusted peer (no Observe grant) is // never contacted — it resolves to a NAMED `untrusted` record with zero // entries, so no session metadata can leak. Trust is checked BEFORE the @@ -446,10 +497,12 @@ export async function buildFleetProjection(opts: FleetProjectionOptions): Promis return Promise.resolve({ record: { source: peer.machineId, present: false, reason: "untrusted" as const } satisfies SourceFetchRecord, entries: [] as Record[], + durationMs: 0, + timedOut: false, }); } const auth = peer.token !== undefined ? peerAuthHeader(peer.token) : undefined; - return fetchSessions(peer.machineId, { getUrl: peer.getUrl }, auth, fetchImpl, timeoutMs, fleetEndpoint); + return fetchSessions(peer.machineId, { getUrl: peer.getUrl }, auth, fetchImpl, timeoutMs, fleetEndpoint, false, now); }); const [localResult, ...peerResults] = await Promise.all([localPromise, ...peerPromises]); @@ -496,7 +549,18 @@ export async function buildFleetProjection(opts: FleetProjectionOptions): Promis const allRecords = [localResult.record, ...peerResults.map((r) => r.record)]; const currency = deriveCurrency(allRecords); - return { ok: true, mode: "fleet", sessions, sources, currency: { ...currency, derived_over: "fetched" } }; + return { + projection: { ok: true, mode: "fleet", sessions, sources, currency: { ...currency, derived_over: "fetched" } }, + peerFetches: peerResults, + }; +} + +/** #1771: classify one peer fetch as a refresh outcome. */ +function refreshOutcomeOf(f: SourceFetch): OwnerRefreshOutcome { + if (f.record.present) return "ok"; + if (f.timedOut) return "timeout"; + if (f.record.reason === "unauthorized") return "unauthorized"; + return "error"; } /** #1647 (S2): the OWNER-ROUTING session set — active AND archived unioned. @@ -505,18 +569,41 @@ export async function buildFleetProjection(opts: FleetProjectionOptions): Promis * ARCHIVED session (unarchive / delete of an archived row) has no owner entry * and misroutes to the LOCAL engine (the #1382 silent-local failure). This * runs BOTH variants and unions by id so every peer-owned session — active or - * archived — is routable. Owner tags are carried through unchanged. */ + * archived — is routable. Owner tags are carried through unchanged. + * #1771: also returns each PEER's refresh outcome (`peers`, in `opts.peers` + * order) — the first non-ok of its two reads, the slower read's duration, and + * the peer-owned entries this union carries — which the OwnerMapFeed logs. */ export async function buildOwnerRoutingProjection( opts: Omit, -): Promise<{ sessions: FleetProjection["sessions"] }> { +): Promise<{ sessions: FleetProjection["sessions"]; peers: OwnerRefreshPeer[] }> { const [active, archived] = await Promise.all([ - buildFleetProjection({ ...opts, archived: false }), - buildFleetProjection({ ...opts, archived: true }), + fleetFanOut({ ...opts, archived: false }), + fleetFanOut({ ...opts, archived: true }), ]); const byId = new Map(); - for (const s of [...active.sessions, ...archived.sessions]) { + for (const s of [...active.projection.sessions, ...archived.projection.sessions]) { const id = typeof s.id === "string" ? s.id : undefined; if (id && !byId.has(id)) byId.set(id, s); } - return { sessions: [...byId.values()] }; + const sessions = [...byId.values()]; + const owned = new Map(); + for (const s of sessions) { + const owner = s.amicode_owner?.owner_machine_id; + if (owner !== undefined) owned.set(owner, (owned.get(owner) ?? 0) + 1); + } + const peers = opts.peers.map((peer, i): OwnerRefreshPeer => { + const a = active.peerFetches[i]; + const b = archived.peerFetches[i]; + const worst = refreshOutcomeOf(a) !== "ok" ? a : b; + const outcome = refreshOutcomeOf(worst); + return { + machineId: peer.machineId, + outcome, + durationMs: Math.max(a.durationMs, b.durationMs), + entries: owned.get(peer.machineId) ?? 0, + ...(outcome !== "ok" && worst.record.reason !== undefined ? { reason: worst.record.reason } : {}), + ...(outcome !== "ok" && worst.record.detail !== undefined ? { detail: worst.record.detail } : {}), + }; + }); + return { sessions, peers }; } diff --git a/packages/extension/src/amicode_service/session_multiplexer.ts b/packages/extension/src/amicode_service/session_multiplexer.ts index 0abf3fd72..3cb9fd755 100644 --- a/packages/extension/src/amicode_service/session_multiplexer.ts +++ b/packages/extension/src/amicode_service/session_multiplexer.ts @@ -17,7 +17,7 @@ // it is a proxy, never a redirect. The app holds ONE base URL throughout. import * as http from "node:http"; -import type { SessionOwnerTag } from "./merged_projection"; +import type { SessionOwnerTag, OwnerRefreshPeer } from "./merged_projection"; import { evaluateRemoteWriteGate, type WriteGrantRead } from "./remote_write_gate"; // ── owner-routing header ───────────────────────────────────────────────────── @@ -196,12 +196,25 @@ export class SessionOwnerMap { export interface OwnerMapFeedOpts { ownerMap: SessionOwnerMap; /** Build the fleet-wide projection whose `.sessions` carry the amicode_owner - * overlay. Injected so the feed is unit-testable without a real fan-out. */ - buildProjection: () => Promise<{ sessions: SessionEntry[] }>; + * overlay. Injected so the feed is unit-testable without a real fan-out. + * #1771: `peers` (optional) carries each peer's refresh outcome — the feed + * logs one line per peer, so a failed peer refresh is visible even though + * neither the builder nor the feed ever throws. */ + buildProjection: () => Promise<{ sessions: SessionEntry[]; peers?: OwnerRefreshPeer[] }>; /** The refresh cadence in ms. Default 5000. */ intervalMs?: number; /** Observe a build failure (diagnostics only — the feed never throws). */ onError?: (err: unknown) => void; + /** #1771: the service log for per-peer refresh outcomes. Absent → silent. */ + log?: (line: string) => void; +} + +/** #1771: one per-peer refresh line: outcome, duration, entry count, and for a + * failure its named reason and detail. */ +export function ownerRefreshLine(p: OwnerRefreshPeer): string { + const reason = p.reason !== undefined ? ` reason=${p.reason}` : ""; + const detail = p.detail !== undefined ? ` detail=${JSON.stringify(p.detail)}` : ""; + return `[owner-map] refresh peer=${p.machineId} outcome=${p.outcome} duration_ms=${p.durationMs} entries=${p.entries}${reason}${detail}`; } /** The default owner-map refresh cadence. */ @@ -216,15 +229,25 @@ export class OwnerMapFeed { /** Rebuild the projection ONCE and update the owner-map. Never throws: a * build failure is reported via `onError` and the last-good map is kept. */ async refreshOnce(): Promise { - let projection: { sessions: SessionEntry[] }; + let projection: { sessions: SessionEntry[]; peers?: OwnerRefreshPeer[] }; try { projection = await this.opts.buildProjection(); } catch (err) { this.opts.onError?.(err); + this.log(`[owner-map] refresh failed error=${JSON.stringify(err instanceof Error ? err.message : String(err))}`); return; // last-good map intact — never clear on a transient failure } if (this.stopped) return; // a stop() raced the in-flight build this.opts.ownerMap.update(projection.sessions ?? []); + for (const peer of projection.peers ?? []) this.log(ownerRefreshLine(peer)); + } + + private log(line: string): void { + try { + this.opts.log?.(line); + } catch { + /* a broken log sink never breaks the feed */ + } } /** Start the loop: fire an immediate refresh, then re-refresh on the diff --git a/packages/extension/test/amicode_service_multiplex_owner_routing.test.ts b/packages/extension/test/amicode_service_multiplex_owner_routing.test.ts index 73c0f36e9..e55fc8889 100644 --- a/packages/extension/test/amicode_service_multiplex_owner_routing.test.ts +++ b/packages/extension/test/amicode_service_multiplex_owner_routing.test.ts @@ -40,6 +40,7 @@ import { EngineProxy } from "../src/amicode_service/engine_proxy"; import { writeAttachmentPointerFile } from "../src/amicode_service/attachment_pointer"; import { writeKeeperPointerFile } from "../src/amicode_service/keeper_pointer"; import { writeHubCredential, readHubCredential } from "../src/amicode_service/hub_credential"; +import { buildOwnerRoutingProjection } from "../src/amicode_service/merged_projection"; import { serverAuthHeader } from "../src/server_auth"; // ── a marker stub: returns a distinct marker for every request; records paths ── @@ -148,6 +149,86 @@ describe("#1449 AC1 — SessionOwnerMap fed by a live projection loop (not a tes }); }); +// ══════════════════════════════════════════════════════════════════════════════ +// #1771 — every ownership refresh logs, PER PEER, its outcome (ok / timeout / +// error / unauthorized), duration and entry count. The projection builder never +// throws and the feed never throws, so before this a peer whose refresh failed +// was silently missing from the map (its event arm closed, its sessions routed +// local) with nothing in any log. +// ══════════════════════════════════════════════════════════════════════════════ +describe("#1771 — each ownership refresh logs a per-peer outcome record", () => { + it("one line per peer: outcome, duration, entries — a failed peer refresh is visible though nothing throws", async () => { + const ownerMap = new SessionOwnerMap(); + const lines: string[] = []; + const feed = new OwnerMapFeed({ + ownerMap, + log: (l) => lines.push(l), + buildProjection: async () => ({ + sessions: [{ id: "ses-A", amicode_owner: { owner_machine_id: "studio", owner_name: "Studio", is_local: false } }], + peers: [ + { machineId: "studio", outcome: "ok", durationMs: 120, entries: 1 }, + { machineId: "mini", outcome: "timeout", durationMs: 10_004, entries: 0, reason: "fetch-failed", detail: "The operation was aborted due to timeout" }, + ], + }), + }); + await expect(feed.refreshOnce()).resolves.toBeUndefined(); + expect(ownerMap.resolveOwner("ses-A")).toBe("studio"); + expect(lines).toEqual([ + "[owner-map] refresh peer=studio outcome=ok duration_ms=120 entries=1", + '[owner-map] refresh peer=mini outcome=timeout duration_ms=10004 entries=0 reason=fetch-failed detail="The operation was aborted due to timeout"', + ]); + }); + + it("through the REAL routing projection: an unauthorized peer is a named outcome; the healthy peer still lands", async () => { + const ownerMap = new SessionOwnerMap(); + const lines: string[] = []; + const fetchImpl = (async (input: RequestInfo | URL) => { + const url = String(input); + if (url.startsWith("http://peer-b")) return new Response("no", { status: 401 }); + if (url.endsWith("/global/health")) return new Response(JSON.stringify({ version: "t" }), { status: 200 }); + if (url.startsWith("http://peer-a")) return new Response(JSON.stringify([{ id: "ses-a1" }, { id: "ses-a2" }]), { status: 200 }); + return new Response(JSON.stringify([]), { status: 200 }); // local + }) as unknown as typeof fetch; + const feed = new OwnerMapFeed({ + ownerMap, + log: (l) => lines.push(l), + buildProjection: async () => { + const p = await buildOwnerRoutingProjection({ + localMachineId: "macbook", + local: { getUrl: () => "http://local" }, + peers: [ + { machineId: "peer-a", getUrl: () => "http://peer-a", token: "tok-a" }, + { machineId: "peer-b", getUrl: () => "http://peer-b", token: "tok-b" }, + ], + rosterLookup: () => undefined, + fetchImpl, + }); + return { sessions: p.sessions as SessionEntry[], peers: p.peers }; + }, + }); + await expect(feed.refreshOnce()).resolves.toBeUndefined(); // never throws + expect(ownerMap.resolveOwner("ses-a1")).toBe("peer-a"); + const a = lines.find((l) => l.includes("peer=peer-a "))!; + const b = lines.find((l) => l.includes("peer=peer-b "))!; + expect(a).toMatch(/^\[owner-map\] refresh peer=peer-a outcome=ok duration_ms=\d+ entries=2$/); + expect(b).toMatch(/^\[owner-map\] refresh peer=peer-b outcome=unauthorized duration_ms=\d+ entries=0 reason=unauthorized$/); + expect(lines).toHaveLength(2); // per PEER — the local engine is not a peer + }); + + it("a projection BUILD failure is logged too (and still never throws)", async () => { + const lines: string[] = []; + const feed = new OwnerMapFeed({ + ownerMap: new SessionOwnerMap(), + log: (l) => lines.push(l), + buildProjection: async () => { + throw new Error("peer fan-out failed"); + }, + }); + await expect(feed.refreshOnce()).resolves.toBeUndefined(); + expect(lines).toEqual(['[owner-map] refresh failed error="peer fan-out failed"']); + }); +}); + // ══════════════════════════════════════════════════════════════════════════════ // FREE-TIER REMOTE CREATE (feature/free-tier-fleet) — pending create-time owner // bindings. A session just created on a peer is not yet in the pull-only fleet diff --git a/packages/extension/test/merged_projection_archived.test.ts b/packages/extension/test/merged_projection_archived.test.ts index 67d91b1a8..60e9d8f38 100644 --- a/packages/extension/test/merged_projection_archived.test.ts +++ b/packages/extension/test/merged_projection_archived.test.ts @@ -128,3 +128,86 @@ describe("#1647 S2 — buildOwnerRoutingProjection unions active + archived for expect(arch.amicode_control).toEqual({ controlState: "interactive", reason: null, eligibility: "none" }); }); }); + +// ══════════════════════════════════════════════════════════════════════════════ +// #1771 — the owner-routing projection reports EACH PEER's refresh outcome (the +// OwnerMapFeed logs it). Never throws: every failure is a classified record. +// The served projection shape is unchanged — this rides only the routing union. +// ══════════════════════════════════════════════════════════════════════════════ +describe("#1771 — buildOwnerRoutingProjection classifies each peer's refresh", () => { + const json = (v: unknown, status = 200) => new Response(JSON.stringify(v), { status }); + + it("ok / unauthorized / timeout / error — with each peer's entry count (active ∪ archived)", async () => { + const fetchImpl = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + if (url.endsWith("/global/health")) return json({ version: "t" }); + if (url.startsWith("http://ok")) return json(/archived=true/.test(url) ? [{ id: "ses-ok-arch" }] : [{ id: "ses-ok-1" }]); + if (url.startsWith("http://denied")) return new Response("no", { status: 401 }); + if (url.startsWith("http://broken")) return new Response("bad gateway", { status: 502 }); + if (url.startsWith("http://slow")) { + // a peer that never answers: the projection's own AbortSignal.timeout fires + return new Promise((_, reject) => init?.signal?.addEventListener("abort", () => reject(init.signal!.reason))); + } + return json([]); // local + }) as unknown as typeof fetch; + const p = await buildOwnerRoutingProjection({ + localMachineId: "local-mac", + local: { getUrl: () => "http://local" }, + peers: [ + { machineId: "ok", getUrl: () => "http://ok", token: "t" }, + { machineId: "denied", getUrl: () => "http://denied", token: "t" }, + { machineId: "slow", getUrl: () => "http://slow", token: "t" }, + { machineId: "broken", getUrl: () => "http://broken", token: "t" }, + { machineId: "gone", getUrl: () => undefined, token: "t" }, + ], + rosterLookup: () => undefined, + fetchImpl, + timeoutMs: 1, + }); + const summary = p.peers.map(({ machineId, outcome, entries, reason, detail }) => ({ machineId, outcome, entries, reason, detail })); + expect(summary).toEqual([ + { machineId: "ok", outcome: "ok", entries: 2, reason: undefined, detail: undefined }, + { machineId: "denied", outcome: "unauthorized", entries: 0, reason: "unauthorized", detail: undefined }, + { machineId: "slow", outcome: "timeout", entries: 0, reason: "fetch-failed", detail: expect.stringContaining("timeout") }, + { machineId: "broken", outcome: "error", entries: 0, reason: "fetch-failed", detail: "HTTP 502" }, + { machineId: "gone", outcome: "error", entries: 0, reason: "no-upstream", detail: undefined }, + ]); + // the routing union itself is unchanged: only the healthy peer's sessions land + expect(p.sessions.map((s) => s.id).sort()).toEqual(["ses-ok-1", "ses-ok-arch"]); + }); + + it("each peer's duration is measured on the injected clock", async () => { + let t = 1_000; + const held: Array<() => void> = []; + // plain promise-backed responses: every read is a microtask, nothing waits + // on a stream or a timer + const fake = (v: unknown) => ({ ok: true, status: 200, json: async () => v }) as unknown as Response; + const fetchImpl = (async (input: RequestInfo | URL) => { + const url = String(input); + if (url.endsWith("/global/health")) return fake({ version: "t" }); + if (url.startsWith("http://slowish")) return new Promise((resolve) => held.push(() => resolve(fake([{ id: "ses-s" }])))); + return fake([]); + }) as unknown as typeof fetch; + const pending = buildOwnerRoutingProjection({ + localMachineId: "local-mac", + local: { getUrl: () => "http://local" }, + peers: [ + { machineId: "quick", getUrl: () => "http://quick", token: "t" }, + { machineId: "slowish", getUrl: () => "http://slowish", token: "t" }, + ], + rosterLookup: () => undefined, + fetchImpl, + now: () => t, + }); + // Every fetch but the held ones resolves at once, so one macrotask turn + // drains them all (microtasks only) — the quick peer is done, the slow + // peer's two reads (active + archived) are in flight. + await new Promise((r) => setImmediate(r)); + expect(held).toHaveLength(2); + t = 2_500; // 1.5 s pass while the slow peer answers + held.forEach((release) => release()); + const p = await pending; + expect(p.peers.find((r) => r.machineId === "quick")).toMatchObject({ outcome: "ok", durationMs: 0, entries: 0 }); + expect(p.peers.find((r) => r.machineId === "slowish")).toMatchObject({ outcome: "ok", durationMs: 1_500, entries: 1 }); + }); +}); From cb427c09d0ab9c6c6b9775bacca71361cdd5b925 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 18:04:54 -0400 Subject: [PATCH 21/60] =?UTF-8?q?test(fleet):=20the=20census=20adds=20no?= =?UTF-8?q?=20frame=20and=20changes=20no=20delivered=20byte=20=E2=80=94=20?= =?UTF-8?q?fleet-of-one=20and=20composite=20(#1771)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../test/sse_fanin_aggregator.test.ts | 31 ++++++ .../test/sse_fanin_driver_pump.test.ts | 99 +++++++++++++++++++ 2 files changed, 130 insertions(+) diff --git a/packages/extension/test/sse_fanin_aggregator.test.ts b/packages/extension/test/sse_fanin_aggregator.test.ts index 1690d1789..bc81920c3 100644 --- a/packages/extension/test/sse_fanin_aggregator.test.ts +++ b/packages/extension/test/sse_fanin_aggregator.test.ts @@ -1047,3 +1047,34 @@ describe("#1771 — per-arm traffic counters, pending depth, and the drop/gap ob expect(sink.text()).toContain("amicode.sync.gap"); }); }); + +describe("#1771 AC5 — the census observer changes no delivered byte", () => { + it("an overflow + gap + reopen sequence writes the identical bytes with and without an observer", () => { + function run(observed: boolean): string { + const chunks: string[] = []; + let blocked = false; + const sink = { write: (c: string) => (chunks.push(c), !blocked), flush() {} }; + const agg = new SseFanInAggregator({ + sink, + bufferBound: 3, + ...(observed ? { now: () => 7, observer: { onOverflowDrop() {}, onGap() {} } } : {}), + }); + agg.connect(`local=${LOCAL_EPOCH}:0;studio=${PEER_EPOCH}:0`); + agg.setMembership([{ machineId: "studio", reachable: true, token: "tok-studio" }]); + const local = fakeEngine({ legacy: true }); + const studio = fakeEngine({ epoch: PEER_EPOCH }); + for (const f of [local.connected(), ...local.stored("message.updated", { n: 1 }, { aggregateID: "ses_1" })]) agg.ingest(LOCAL_NAMESPACE, f); + blocked = true; + for (let i = 0; i < 9; i++) agg.ingest("studio", studio.event("message.part.delta", { i })); + agg.ingest(LOCAL_NAMESPACE, local.heartbeat()); + blocked = false; + agg.resume(); + agg.reopenArm("studio"); + agg.ingest("studio", studio.heartbeat()); + return chunks.join(""); + } + const plain = run(false); + expect(run(true)).toBe(plain); + expect(plain).toContain("amicode.sync.gap"); // the sequence did exercise drop + gap + }); +}); diff --git a/packages/extension/test/sse_fanin_driver_pump.test.ts b/packages/extension/test/sse_fanin_driver_pump.test.ts index 53165b90a..4b0f6dc86 100644 --- a/packages/extension/test/sse_fanin_driver_pump.test.ts +++ b/packages/extension/test/sse_fanin_driver_pump.test.ts @@ -1660,3 +1660,102 @@ describe("#1771 AC3 — each transition logs one line immediately, with its conn ok.fireClose(); }); }); + +describe("#1771 AC5 — the census adds no frames to any stream and changes no delivery", () => { + afterEach(() => vi.useRealTimers()); + + it("FLEET-OF-ONE under a live census (cadence ticking, every transition logging): the stream is the engine's frames, byte-identical (#1264)", async () => { + vi.useFakeTimers({ toFake: ["setInterval", "clearInterval"] }); + const { census, lines } = testCensus(); + census.start(); + const localCtls: Ctl[] = []; + const driver = twoListenerDriver(fakeOrigins(), new SessionOwnerMap(), { + census, + openUpstream: () => { + const ctl = controllableSource(); + localCtls.push(ctl); + return ctl.source; + }, + }); + for (const engine of [fakeEngine({ legacy: true }), fakeEngine({ epoch: LOCAL_EPOCH })]) { + localCtls.length = 0; + const res = mockRes(); + driver.handle(mockReq("/global/event"), res); + const first = [engine.connected(), engine.event("session.status", { status: "busy" }), ...engine.stored("message.part.updated", { text: "ciao" }, { aggregateID: "ses_1" })]; + for (const f of first) localCtls[0].push(f); + await flush(); + vi.advanceTimersByTime(FANIN_CENSUS_INTERVAL_MS); // a census line mid-stream + localCtls[0].end(); // the arm drops … + await flush(); + driver.reconcile(); // … and re-opens (no gap in fleet-of-one) + const second = [engine.heartbeat(), engine.event("session.status", { status: "idle" })]; + for (const f of second) localCtls[1].push(f); + await flush(); + expect(res.text()).toBe([...first, ...second].join("")); + res.fireClose(); + } + expect(lines.some((l) => l.startsWith("[fanin] census "))).toBe(true); + expect(lines.some((l) => l.includes("arm-closed"))).toBe(true); + }); + + it("COMPOSITE: the same scripted sequence (overflow, gap, membership change, re-open) is byte-identical with and without a census sink", async () => { + async function run(sink: ((l: string) => void) | undefined): Promise { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const origins = fakeOrigins(); + const driver = twoListenerDriver(origins, ownerMap, sink ? { census: new FanInCensus({ sink, now: () => 0 }) } : {}); + const local = fakeEngine({ legacy: true }); // an old local engine: id-less + const studio = fakeEngine({ epoch: PEER_EPOCH }); + const res = backpressureRes(); + driver.handle(mockReq(`/global/event?lastEventID=studio=${studio.idAt(0)}`), res); + await flush(); + for (const f of [local.connected(), ...local.stored("message.updated", { n: 1 }, { aggregateID: "ses_1" }), local.heartbeat()]) { + origins.broadcast(LOCAL_NAMESPACE, f); + } + origins.broadcast("studio", studio.connected()); + await flush(); + res.block(); + for (let i = 0; i < 300; i++) origins.broadcast("studio", studio.event("message.part.delta", { i })); + await flush(); + res.fireDrain(); + await flush(); + ownerMap.update([]); + driver.reconcile(); // membership change closes the peer arm + ownerMap.update(STUDIO_OWNED); + driver.reconcile(); // … and it re-opens (a reopen gap) + await flush(); + origins.broadcast("studio", studio.heartbeat()); + await flush(); + res.fireClose(); + return res.text(); + } + const lines: string[] = []; + const silent = await run(undefined); + const logged = await run((l) => lines.push(l)); + expect(logged).toBe(silent); + expect(lines.length).toBeGreaterThan(5); // the census did see it all … + expect(logged).not.toContain("[fanin]"); // … and none of it reached the stream + expect(logged).not.toContain("[owner-map]"); + }); + + it("a census sink that throws never breaks delivery", async () => { + const census = new FanInCensus({ + sink: () => { + throw new Error("log pipe closed"); + }, + }); + census.start(); + const origins = fakeOrigins(); + const driver = twoListenerDriver(origins, new SessionOwnerMap(), { census }); + const engine = fakeEngine({ epoch: LOCAL_EPOCH }); + const res = mockRes(); + expect(() => driver.handle(mockReq("/global/event"), res)).not.toThrow(); + const f = engine.event("session.status", { status: "busy" }); + origins.broadcast(LOCAL_NAMESPACE, f); + await flush(); + expect(() => census.emit()).not.toThrow(); + expect(res.text()).toBe(f); + res.fireClose(); + census.stop(); + }); +}); From 66306c4d11f63cc1ccf541ca78b49b6788c1d785 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 18:06:30 -0400 Subject: [PATCH 22/60] test(fleet): engine-shaped stream fixtures pinned to the engine wire; 10 MB frame in 64 KB chunks through the real reader (#1771) --- .../test/engine_stream_fixtures.test.ts | 118 ++++++++++++++++++ .../test/sse_fanin_driver_pump.test.ts | 56 ++++++++- 2 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 packages/extension/test/engine_stream_fixtures.test.ts diff --git a/packages/extension/test/engine_stream_fixtures.test.ts b/packages/extension/test/engine_stream_fixtures.test.ts new file mode 100644 index 000000000..f0c7f1d1f --- /dev/null +++ b/packages/extension/test/engine_stream_fixtures.test.ts @@ -0,0 +1,118 @@ +// engine_stream_fixtures.test.ts — #1771: the engine-shaped global-stream +// fixtures (test/fixtures/engine_stream) are SHARED test surface — the fan-in +// suites use them, and the loud-gap (#1773) and linear-splitting (#1779) slices +// build on them. This pins them to the wire the engine's `/global/event` route +// actually writes (overlay handlers/global.ts + bus/global.ts + the event-v2 +// bridge), so a fixture can never drift into a frame production never sends. +import { describe, it, expect } from "vitest"; +import { + fakeEngine, + engineFrame, + busEvent, + syncCopyOf, + sequencedEvent, + legacyEvent, + largeSummaryFrame, + chunkBytes, + CHUNK_64K, + TEN_MB, + LOCAL_EPOCH, + ENGINE_DIRECTORY, +} from "./fixtures/engine_stream/engine_frames"; + +/** Split one SSE frame into its field lines (the trailing blank line dropped). */ +function fields(frame: string): Array<[string, string]> { + expect(frame.endsWith("\n\n")).toBe(true); + return frame + .slice(0, -2) + .split("\n") + .map((l) => { + const i = l.indexOf(":"); + return [l.slice(0, i), l.slice(i + 1).replace(/^ /, "")] as [string, string]; + }); +} +const idOf = (frame: string) => fields(frame).find(([k]) => k === "id")?.[1]; +const dataOf = (frame: string) => JSON.parse(fields(frame).find(([k]) => k === "data")![1]); + +describe("#1771 — engine-shaped fixtures match the engine's global-stream wire", () => { + it("a frame is the SSE encoder's: optional `id:`, NO `event:` line (the name is \"message\"), one `data:` line, blank-line terminated", () => { + const f = engineFrame(busEvent({ id: "evt_1", type: "session.status", properties: { a: 1 } }), { id: `${LOCAL_EPOCH}:3` }); + expect(fields(f).map(([k]) => k)).toEqual(["id", "data"]); + expect(f).not.toContain("event:"); + // JSON.stringify(event) — the bridge's key order, payload {id, type, properties} + expect(fields(f)[1][1]).toBe( + `{"directory":"${ENGINE_DIRECTORY}","project":"prj_0fixture","payload":{"id":"evt_1","type":"session.status","properties":{"a":1}}}`, + ); + }); + + it("LEGACY (pre-#1776) engine: every frame is id-less — events, twins, heartbeats, connected", () => { + const e = fakeEngine({ legacy: true }); + const frames = [e.connected(), e.event("session.status", { s: 1 }), ...e.stored("message.updated", { m: 1 }, { aggregateID: "ses_1" }), e.heartbeat()]; + for (const f of frames) { + expect(idOf(f)).toBeUndefined(); + expect(fields(f).map(([k]) => k)).toEqual(["data"]); + } + expect(dataOf(frames.at(-1)!).payload).toMatchObject({ type: "server.heartbeat", properties: {} }); + expect(legacyEvent({ x: 1 })).not.toContain("id:"); + }); + + it("SEQUENCED (#1776) engine: bus events take strictly increasing `:`; control frames carry the CURRENT seq and consume none", () => { + let now = 1_700_000_000_000; + const e = fakeEngine({ epoch: LOCAL_EPOCH, now: () => now }); + const connected = e.connected(); + const a = e.event("session.status", { s: 1 }); + now += 10_000; + const hb = e.heartbeat(); + const b = e.event("session.status", { s: 2 }); + expect([idOf(connected), idOf(a), idOf(hb), idOf(b)]).toEqual([`${LOCAL_EPOCH}:0`, `${LOCAL_EPOCH}:1`, `${LOCAL_EPOCH}:1`, `${LOCAL_EPOCH}:2`]); + expect(dataOf(hb).payload).toMatchObject({ type: "server.heartbeat", properties: { time: 1_700_000_010_000, seq: 1 } }); + expect(dataOf(connected).payload).toMatchObject({ type: "server.connected", properties: {} }); + expect(e.seq).toBe(2); + expect(sequencedEvent(LOCAL_EPOCH, 9, { n: 9 })).toMatch(new RegExp(`^id: ${LOCAL_EPOCH}:9\\n`)); + }); + + it("a stored event is the plain + sync TWIN: the copy mirrors its source and shares its id (one seq)", () => { + const e = fakeEngine({ epoch: LOCAL_EPOCH }); + const [plain, sync] = e.stored("message.part.updated", { part: { id: "prt_1" } }, { aggregateID: "ses_9", durableSeq: 41 }); + expect(idOf(plain)).toBe(`${LOCAL_EPOCH}:1`); + expect(idOf(sync)).toBe(idOf(plain)); // filtering sync copies leaves no hole + const p = dataOf(plain); + const s = dataOf(sync); + expect(p.payload).toEqual({ id: p.payload.id, type: "message.part.updated", properties: { part: { id: "prt_1" } } }); + expect(s.directory).toBe(p.directory); + expect(s.payload).toEqual({ + type: "sync", + syncEvent: { id: p.payload.id, type: "message.part.updated.1", seq: 41, aggregateID: "ses_9", data: { part: { id: "prt_1" } } }, + id: p.payload.id, + }); + // the bus assigns the copy's payload.id at emit → it serializes AFTER syncEvent + expect(fields(sync)[1][1].indexOf('"syncEvent"')).toBeLessThan(fields(sync)[1][1].lastIndexOf('"id"')); + expect(syncCopyOf(busEvent({ id: "evt_x", type: "t", properties: {} }), { seq: 1, aggregateID: "a" }).payload.type).toBe("sync"); + // on a legacy engine the twin is two id-less frames + const [lp, ls] = fakeEngine({ legacy: true }).stored("message.updated", {}, { aggregateID: "ses_1" }); + expect([idOf(lp), idOf(ls)]).toEqual([undefined, undefined]); + }); + + it("the large frame: ≥ 10 MB of message.updated summary patch text, delivered in exact 64 KB byte chunks", () => { + const big = largeSummaryFrame(); + const bytes = Buffer.byteLength(big); + expect(bytes).toBeGreaterThanOrEqual(TEN_MB); + expect(bytes).toBeLessThan(TEN_MB + 1024 * 1024); + expect(big.length).toBeLessThan(bytes); // non-ASCII: bytes ≠ string length + const data = dataOf(big); + expect(data.payload.type).toBe("message.updated"); + expect(data.payload.properties.info.summary.diffs[0]).toEqual( + expect.objectContaining({ file: expect.any(String), patch: expect.any(String), additions: 1400, deletions: 0, status: "modified" }), + ); + expect(idOf(big)).toBeUndefined(); // legacy by default … + expect(idOf(largeSummaryFrame({ epoch: LOCAL_EPOCH, seq: 7 }))).toBe(`${LOCAL_EPOCH}:7`); // … or sequenced + + const chunks = chunkBytes(big); + expect(chunks.slice(0, -1).every((c) => c.length === CHUNK_64K)).toBe(true); + expect(chunks.at(-1)!.length).toBeGreaterThan(0); + expect(chunks.at(-1)!.length).toBeLessThanOrEqual(CHUNK_64K); + expect(Buffer.concat(chunks).equals(Buffer.from(big, "utf8"))).toBe(true); + // at least one boundary falls inside a multi-byte code point, as on the wire + expect(chunks.some((c) => (c[0] & 0xc0) === 0x80)).toBe(true); + }); +}); diff --git a/packages/extension/test/sse_fanin_driver_pump.test.ts b/packages/extension/test/sse_fanin_driver_pump.test.ts index 4b0f6dc86..ce649bcb0 100644 --- a/packages/extension/test/sse_fanin_driver_pump.test.ts +++ b/packages/extension/test/sse_fanin_driver_pump.test.ts @@ -10,7 +10,17 @@ import type { SseFrameSource } from "../src/amicode_service/sse_fanin_aggregator import { LOCAL_NAMESPACE } from "../src/amicode_service/sse_fanin_aggregator"; import { SessionOwnerMap } from "../src/amicode_service/session_multiplexer"; import { FanInCensus, FANIN_CENSUS_INTERVAL_MS, FANIN_CLOSE_REASONS } from "../src/amicode_service/fanin_census"; -import { fakeEngine, startEngineSseServer, LOCAL_EPOCH, PEER_EPOCH } from "./fixtures/engine_stream/engine_frames"; +import { + fakeEngine, + startEngineSseServer, + writeChunks, + largeSummaryFrame, + chunkBytes, + CHUNK_64K, + TEN_MB, + LOCAL_EPOCH, + PEER_EPOCH, +} from "./fixtures/engine_stream/engine_frames"; // ── helpers ────────────────────────────────────────────────────────────────── @@ -1759,3 +1769,47 @@ describe("#1771 AC5 — the census adds no frames to any stream and changes no d census.stop(); }); }); + +describe("#1771 AC6 — a 10 MB engine frame delivered in 64 KB chunks (the #1779 surface)", () => { + it("the real reader reassembles it whole; the stream carries it byte-identical; the census counts its bytes and largest frame", async () => { + const engine = fakeEngine({ epoch: LOCAL_EPOCH }); + const connected = engine.connected(); + const big = largeSummaryFrame({ epoch: LOCAL_EPOCH, seq: 1 }); + const after = engine.heartbeat(); + const chunks = chunkBytes(connected + big + after, CHUNK_64K); + expect(Buffer.byteLength(big)).toBeGreaterThanOrEqual(TEN_MB); + expect(chunks.length).toBeGreaterThan(160); + const server = await startEngineSseServer(async (res) => { + await writeChunks(res, chunks); + res.end(); + }); + const lines: string[] = []; + const w = lineWaiter(lines); + const census = new FanInCensus({ plane: "observe", sink: w.sink, now: () => 0 }); + const driver = new SseFanInDriver({ + ownerMap: new SessionOwnerMap(), // fleet-of-one: the verbatim relay + localMachineId: "macbook", + localEventUrl: () => server.url, + peerBaseUrl: () => undefined, + peerToken: () => ({ ok: false as const, reason: "absent" as const }), + reconcileMs: 999_999, + census, + }); + const res = mockRes(); + try { + driver.handle(mockReq("/global/event"), res); + await w.waitFor((l) => l.includes("arm=local reason=")); + expect(res.text()).toBe(connected + big + after); // whole, in order, byte-identical + census.emit(); + const local = parseCensus(lines.at(-1)!).arms.find((a) => a.arm === "local")!; + expect(local).toMatchObject({ + frames: "3", + bytes: String(Buffer.byteLength(connected + big + after)), + largest: String(Buffer.byteLength(big)), + }); + } finally { + res.fireClose(); + await server.stop(); + } + }); +}); From 06eac85109a855f11dcbfaa95a0d94c0d4c1c584 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 18:06:36 -0400 Subject: [PATCH 23/60] feat(engine): summary diffs carry no patch bodies on the wire; the per-turn diff view fetches a patch on expand (#1778) --- packages/app-bundle/manifest.json | 32 +++-- .../app/src/pages/directory-layout.tsx | 135 ++++++++++++++++++ .../packages/app/src/utils/diffs.test.ts | 120 ++++++++++++++++ .../overlay/packages/app/src/utils/diffs.ts | 53 +++++++ .../opencode/src/bus/global-replay.ts | 43 +++++- .../test/server/httpapi-global-event.test.ts | 96 ++++++++++++- .../test/server/summary-diff-wire.test.ts | 110 ++++++++++++++ .../src/components/session-turn.tsx | 28 +++- .../src/components/summary-patch.test.ts | 124 ++++++++++++++++ .../src/components/summary-patch.ts | 77 ++++++++++ .../packages/session-ui/src/context/data.tsx | 82 +++++++++++ 11 files changed, 883 insertions(+), 17 deletions(-) create mode 100644 packages/app-bundle/overlay/packages/app/src/pages/directory-layout.tsx create mode 100644 packages/app-bundle/overlay/packages/app/src/utils/diffs.test.ts create mode 100644 packages/app-bundle/overlay/packages/app/src/utils/diffs.ts create mode 100644 packages/app-bundle/overlay/packages/opencode/test/server/summary-diff-wire.test.ts create mode 100644 packages/app-bundle/overlay/packages/session-ui/src/components/summary-patch.test.ts create mode 100644 packages/app-bundle/overlay/packages/session-ui/src/components/summary-patch.ts create mode 100644 packages/app-bundle/overlay/packages/session-ui/src/context/data.tsx diff --git a/packages/app-bundle/manifest.json b/packages/app-bundle/manifest.json index 41c60bbc8..60a61ff6a 100644 --- a/packages/app-bundle/manifest.json +++ b/packages/app-bundle/manifest.json @@ -233,6 +233,7 @@ "packages/app/src/i18n/uk.ts": "c1f2fb529563fa211e3e322aca0d9417a34909fd056ae80b65c3861b0d6caee4", "packages/app/src/i18n/zh.ts": "50755c05ef47e910c3de7c1471f66b7cd21cea93c005480636ec085a11d5e09b", "packages/app/src/i18n/zht.ts": "c65962a29d6d772d1333ce85a7b3a6f8f3246fbef634ec148cc6531253bbeae9", + "packages/app/src/pages/directory-layout.tsx": "e580e6197408358ddd44eaa76cddd0828e87db255a9068971998f787851369d6", "packages/app/src/pages/empty-workspace-landing.tsx": "ac9410f5b223b30e2d82a6d222992fed2226b6afc1213ff0c8305a230cda7cb2", "packages/app/src/pages/error.tsx": "db9ba0847cfb205100ad6133cce30c6ea2ce983802af44947e6d2b746c2a6ec5", "packages/app/src/pages/hold-debug-badge-gate.test.ts": "661f59e172a5b074a6807896f46c18338faa59b6d9c0912fd824df2dae4a5730", @@ -265,6 +266,8 @@ "packages/app/src/utils/amicode-workspace-projects.test.ts": "8b5c32172e2039b3d9c52f2257023263fc846132e7c185e4a63ae014bc2e2bfb", "packages/app/src/utils/amicode-workspace-projects.ts": "cbe091c463177375c93cb43676c202bd7bf06baf36fdec779f7d641558ff6bee", "packages/app/src/utils/chrome-dropdown.ts": "f8b878c5df4e1afd96b0751c73daaad0a8127411030724cb8f9a5089ebb29eac", + "packages/app/src/utils/diffs.test.ts": "38a5b5dbe23014660204fe08e30599dad16c93665eeaf555284915d0de83a569", + "packages/app/src/utils/diffs.ts": "313ec1085e24b5b8d2844cf42ea2caa93cda46081a6aa5691de5bce8a6686744", "packages/app/src/utils/global-clipboard.test.ts": "18ac0ea486d36eed00597f2718f03bffce70a6e6a8f77c79aefe0795790d8771", "packages/app/src/utils/global-clipboard.ts": "b26373289d7402c88e8038575703b11fa5e4ef5a7f50ee05bceb45488022ab26", "packages/app/src/utils/pane-bridge.ts": "a8a8a40dc60a367086f3c4e4c561a289fb85d8b2086a8d62c2b66102de652ce4", @@ -316,7 +319,7 @@ "packages/llm/src/protocols/bedrock-converse.ts": "834a415fa8ddf9ad1224133f8352d0a926694f19f0954201d182111c171ec03b", "packages/llm/test/provider/bedrock-converse.test.ts": "e8e14d60e8502e928f170f8ac5b406c44bf16d2dd34996ddc2fb7b9261decf85", "packages/opencode/src/agent/agent.ts": "512372a4c62997e9565266638ada19b9dc0a3685b0c0483fd28fede2abdb5c9b", - "packages/opencode/src/bus/global-replay.ts": "3b172a01b322af8282b3451446316834aafae840b9d48e41ef5447986fbaf496", + "packages/opencode/src/bus/global-replay.ts": "3ce244c2dd0fd75cc1a9907f65d758833fae5da7165860189ca80fcd73e7ef60", "packages/opencode/src/bus/global.ts": "65b3b380936fc34b2c5ef9f3ab419a6c8e6d4cfc696302a266188f26ccdc320a", "packages/opencode/src/control-plane/workspace.ts": "cc56fb9a6dc6e0fe88e14237ff8aa29a46ce12971ea69a9d80982e0c58128c13", "packages/opencode/src/effect/runner.ts": "b660b5b22df810b16293c23c6d832e5e8dcfb12b9a8a858c875d2c051c855b81", @@ -395,7 +398,7 @@ "packages/opencode/test/server/httpapi-compression.test.ts": "5f7603d7018e0b95f1c2f36c01dd297c01770bfcdd91ce29778e03b7d6143f75", "packages/opencode/test/server/httpapi-file-write-gate.test.ts": "0c91b7417e615850edfde3f2878207dab6b6624d4cfbb6e6783b2e6d9aa4dff0", "packages/opencode/test/server/httpapi-file.test.ts": "d3386129bc81467ecbcc57a3861f53767ee56ac41b4ff6ad58d8b168fb3b3373", - "packages/opencode/test/server/httpapi-global-event.test.ts": "3c51b701e9912f359f992df89afb3a436fcedbe003a41a51ca6cdedb88b8e994", + "packages/opencode/test/server/httpapi-global-event.test.ts": "4c02060af1d2d9585f41293f2d62560de9df6b85b542c93ed6e2aa4171c145fc", "packages/opencode/test/server/httpapi-instance-route-auth.test.ts": "4b73a89d2431c02bc3b665d93f9002914e51eb1add979dc1867efbcbc0c36e25", "packages/opencode/test/server/httpapi-provider.test.ts": "9e8b3e085b478e3f99b689ffff600e10c0b4b50b3cabdbd749f88dae19bf5904", "packages/opencode/test/server/httpapi-session.test.ts": "eda6fd763ac4c17891cedaa4ccd49597c03e7227fcfdcf0f94a1b4545ba623ed", @@ -404,6 +407,7 @@ "packages/opencode/test/server/session-compact-errors.test.ts": "2ae2044d66db1ba41a5e294afedea5271f11d5bf384685a06ea05ac515b16fb2", "packages/opencode/test/server/session-diff-missing-patch.test.ts": "70526ca7fb86741843fd167f8f2f9c6f6d7e5e415a20a5448d17d24229bdd65b", "packages/opencode/test/server/session-diff-scoped.test.ts": "20ef80b92ad872573b371540575f028be8f1ad3c9f34d72f814b621e14e7a59e", + "packages/opencode/test/server/summary-diff-wire.test.ts": "af750d297333225e2ee7342cab59b8e0131294157d742d8d35c62103691dc5d5", "packages/opencode/test/server/workspace-routing.test.ts": "2574731ac379c540f450edc15f8750ac5360e8c33b4a0dad7c61e8eb686137bb", "packages/opencode/test/session/adversarial-e2e.test.ts": "1b471a69223d040c5d5f4f2ee99c2daa6df56bdf11679b65d902c15bedac3e80", "packages/opencode/test/session/business-record.test.ts": "7857094479408bcfd6ea8d9f5b425c6384bcf8490f930674bc1c203e82782d75", @@ -459,11 +463,14 @@ "packages/session-ui/src/components/session-review.css": "ec8c5b359dd4b7ff80d7fad8906359678adade9ce62c9d74aedc000b646db341", "packages/session-ui/src/components/session-review.tsx": "d2015c9f7419279f08e84523439a0b73e83bd5d06587ad56b85ea53b66ecaee4", "packages/session-ui/src/components/session-turn.css": "792915f3373134eadf3cd51e1d4bfa88ec347e6b037b6caf60b639ec5e74fbd8", - "packages/session-ui/src/components/session-turn.tsx": "9cbc0d7a35f42e32dad53ace7b3c7d92623fa2d3108edb49e0b7bc868f5c1966", + "packages/session-ui/src/components/session-turn.tsx": "42f247b0434a9f52d88d9fb96adff59afe6edeab84dd912b62eb153253b6696f", + "packages/session-ui/src/components/summary-patch.test.ts": "560272a942f1cdcaf78ea0cdc6ec25e49e3ddc8eef1c93d4cd6c728294b6ae40", + "packages/session-ui/src/components/summary-patch.ts": "ab086ea1bc741e15b94f6e9aedc4d934cac17597f4aa1392b146994020006170", "packages/session-ui/src/components/tool-count-label.test.ts": "c62bbe2c662f6740794d25bd7c33fb3b9b46394a1b41d01d6568a8a2ee2b4e75", "packages/session-ui/src/components/tool-count-label.tsx": "06c96479710314d443d42d4c29a29b46d58a55ee84a844a5b6865b49a0513340", "packages/session-ui/src/components/tool-error-card.tsx": "e27b89836b7c709120ac15fbffd2200038944058ea3506e61e5897ffb3537503", "packages/session-ui/src/components/tool-status-title.tsx": "ae4561b58f55985d2294a352802e723b18943ac9a6551d97f1c4a6cc95368c8a", + "packages/session-ui/src/context/data.tsx": "7b840b5bf6929445d21da2cbd8e8006316c6950a09d559ea77e8c397e728cd99", "packages/session-ui/src/pierre/worker.ts": "7f210f931777956567eaeea15f2f5478befc7874a64a4b685504c59c66503fee", "packages/session-ui/src/util/clipboard.ts": "c37ba53914714ed04135e5a254e8ea90186f097d7470fa0873a83de82f3fef2d", "packages/ui/src/amicode/amico-presence.css": "b3f345b607e318dcfbefcdc677fe21cc7d387b9946b0f9d625b0a842fe7e31b9", @@ -949,11 +956,11 @@ "packages/ui/src/v2/components/text-shimmer-v2.tsx" ], "exceptions": [], - "extracted_at": "2026-10-12T21:54:35.577Z", + "extracted_at": "2026-10-12T22:06:29.073Z", "per_package": { "packages/app": { "A": 237, - "M": 158, + "M": 161, "D": 0 }, "packages/core": { @@ -967,7 +974,7 @@ "D": 0 }, "packages/opencode": { - "A": 79, + "A": 80, "M": 92, "D": 0 }, @@ -987,8 +994,8 @@ "D": 0 }, "packages/session-ui": { - "A": 42, - "M": 31, + "A": 44, + "M": 32, "D": 0 }, "packages/ui": { @@ -998,7 +1005,7 @@ } }, "counts": { - "overlay_total": 927, + "overlay_total": 934, "deletions": 7, "server_coupled": 0 }, @@ -1228,6 +1235,7 @@ "packages/app/src/i18n/uk.ts": "M", "packages/app/src/i18n/zh.ts": "M", "packages/app/src/i18n/zht.ts": "M", + "packages/app/src/pages/directory-layout.tsx": "M", "packages/app/src/pages/empty-workspace-landing.tsx": "A", "packages/app/src/pages/error.tsx": "M", "packages/app/src/pages/hold-debug-badge-gate.test.ts": "A", @@ -1260,6 +1268,8 @@ "packages/app/src/utils/amicode-workspace-projects.test.ts": "A", "packages/app/src/utils/amicode-workspace-projects.ts": "A", "packages/app/src/utils/chrome-dropdown.ts": "A", + "packages/app/src/utils/diffs.test.ts": "M", + "packages/app/src/utils/diffs.ts": "M", "packages/app/src/utils/global-clipboard.test.ts": "A", "packages/app/src/utils/global-clipboard.ts": "A", "packages/app/src/utils/pane-bridge.ts": "A", @@ -1399,6 +1409,7 @@ "packages/opencode/test/server/session-compact-errors.test.ts": "A", "packages/opencode/test/server/session-diff-missing-patch.test.ts": "M", "packages/opencode/test/server/session-diff-scoped.test.ts": "A", + "packages/opencode/test/server/summary-diff-wire.test.ts": "A", "packages/opencode/test/server/workspace-routing.test.ts": "M", "packages/opencode/test/session/adversarial-e2e.test.ts": "A", "packages/opencode/test/session/business-record.test.ts": "A", @@ -1455,10 +1466,13 @@ "packages/session-ui/src/components/session-review.tsx": "M", "packages/session-ui/src/components/session-turn.css": "M", "packages/session-ui/src/components/session-turn.tsx": "M", + "packages/session-ui/src/components/summary-patch.test.ts": "A", + "packages/session-ui/src/components/summary-patch.ts": "A", "packages/session-ui/src/components/tool-count-label.test.ts": "A", "packages/session-ui/src/components/tool-count-label.tsx": "M", "packages/session-ui/src/components/tool-error-card.tsx": "M", "packages/session-ui/src/components/tool-status-title.tsx": "M", + "packages/session-ui/src/context/data.tsx": "M", "packages/session-ui/src/pierre/worker.ts": "M", "packages/session-ui/src/util/clipboard.ts": "A", "packages/ui/src/amicode/amico-presence.css": "A", diff --git a/packages/app-bundle/overlay/packages/app/src/pages/directory-layout.tsx b/packages/app-bundle/overlay/packages/app/src/pages/directory-layout.tsx new file mode 100644 index 000000000..7120fd4cc --- /dev/null +++ b/packages/app-bundle/overlay/packages/app/src/pages/directory-layout.tsx @@ -0,0 +1,135 @@ +import { DataProvider } from "@opencode-ai/session-ui/context" +import { showToast } from "@/utils/toast" +import { base64Encode } from "@opencode-ai/core/util/encode" +import { useLocation, useNavigate, useParams } from "@solidjs/router" +import { type Accessor, createEffect, createMemo, createResource, onCleanup, type ParentProps, Show } from "solid-js" +import { useLanguage } from "@/context/language" +import { LocalProvider } from "@/context/local" +import { SDKProvider, useSDK } from "@/context/sdk" +import { useSync } from "@/context/sync" +import { decode64 } from "@/utils/base64" +import { Schema } from "effect" +import type { ServerConnection } from "@/context/server" +import { sessionHref } from "@/utils/session-route" +import { useServerSync } from "@/context/server-sync" + +export function DirectoryDataProvider( + props: ParentProps<{ + directory: string | Accessor + draftID?: string + server?: Accessor + }>, +) { + const location = useLocation() + const navigate = useNavigate() + const params = useParams() + const sdk = useSDK() + const sync = useSync() + const serverSync = useServerSync() + const directory = () => (typeof props.directory === "function" ? props.directory() : props.directory) + const slug = createMemo(() => base64Encode(directory())) + const href = (sessionID: string) => { + const server = props.server?.() + if (server) return sessionHref(server, sessionID) + return `/${slug()}/session/${sessionID}` + } + // #1778 (amicode overlay): live summaries carry no patch bodies; the per-turn + // diff view fetches a turn's stored diffs (patches included) when a file is + // expanded — the stored message is the source. + const loadSummaryDiffs = ({ sessionID, messageID }: { sessionID: string; messageID: string }) => + sdk() + .client.session.message({ sessionID, messageID }) + .then((response) => { + const info = response.data?.info + return info?.role === "user" ? info.summary?.diffs : undefined + }) + + createEffect(() => { + // A draft lives at /new-session?draftId=… and has no directory segment to normalize. + if (props.draftID || props.server?.()) return + const next = sync().data.path.directory + if (!next || next === directory()) return + const path = location.pathname.slice(slug().length + 1) + navigate(`/${base64Encode(next)}${path}${location.search}${location.hash}`, { replace: true }) + }) + + createResource( + () => params.id, + (id) => + sync() + .session.sync(id) + .catch(() => {}), + ) + + createEffect(() => { + const sessionID = params.id + if (!sessionID) return + serverSync().session.pin(sessionID) + onCleanup(() => serverSync().session.unpin(sessionID)) + }) + + return ( + + {(directory) => ( + navigate(href(sessionID))} + onSessionHref={href} + onLoadSummaryDiffs={loadSummaryDiffs} + > + {props.children} + + )} + + ) +} + +export const ProjectDirString = Schema.String.pipe(Schema.brand("ProjectDirString")) +export type ProjectDirString = Schema.Schema.Type + +export function decodeDirectory(dir: string): ProjectDirString | undefined { + const decoded = decode64(dir) + if (!decoded) return + return ProjectDirString.make(decoded) +} + +export default function Layout(props: ParentProps) { + const params = useParams() + const language = useLanguage() + const navigate = useNavigate() + let invalid = "" + + const resolved = createMemo(() => { + if (!params.dir) return "" + return decodeDirectory(params.dir) ?? "" + }) + + createEffect(() => { + const dir = params.dir + if (!dir) return + if (resolved()) { + invalid = "" + return + } + if (invalid === dir) return + invalid = dir + showToast({ + variant: "error", + title: language.t("common.requestFailed"), + description: language.t("directory.error.invalidUrl"), + }) + navigate("/", { replace: true }) + }) + + return ( + + {(resolved) => ( + + {props.children} + + )} + + ) +} diff --git a/packages/app-bundle/overlay/packages/app/src/utils/diffs.test.ts b/packages/app-bundle/overlay/packages/app/src/utils/diffs.test.ts new file mode 100644 index 000000000..bfa354df6 --- /dev/null +++ b/packages/app-bundle/overlay/packages/app/src/utils/diffs.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, test } from "bun:test" +import { readFileSync } from "node:fs" +import { resolve } from "node:path" +import type { SnapshotFileDiff } from "@opencode-ai/sdk/v2" +import type { FileDiffInfo } from "@opencode-ai/client/promise" +import type { Message } from "@opencode-ai/sdk/v2/client" +import { diffs, message } from "./diffs" + +const item = { + file: "src/app.ts", + patch: "@@ -1 +1 @@\n-old\n+new\n", + additions: 1, + deletions: 1, + status: "modified", +} satisfies FileDiffInfo & SnapshotFileDiff + +describe("diffs", () => { + test("keeps valid arrays", () => { + expect(diffs([item])).toEqual([item]) + }) + + test("wraps a single diff object", () => { + expect(diffs(item)).toEqual([item]) + }) + + test("reads keyed diff objects", () => { + expect(diffs({ a: item })).toEqual([item]) + }) + + test("drops invalid entries", () => { + expect( + diffs([ + item, + { file: "src/bad.ts", patch: 42, additions: 1, deletions: 1 }, + { file: "src/bad.ts", additions: "1", deletions: 1 }, + { patch: item.patch, additions: 1, deletions: 1 }, + ]), + ).toEqual([item]) + }) + + // #1778 (amicode overlay): a live summary's diffs carry no patch body — the + // engine strips it on the wire, and the per-turn diff view fetches it on + // demand. A patch-less diff (file, status, counts) is valid, not dropped: + // before, the whole live file list vanished. + test("keeps a patch-less diff — the live wire form", () => { + const { patch: _patch, ...wire } = item + expect(diffs([item, wire, { file: "src/new.ts", additions: 4, deletions: 0 }])).toEqual([ + item, + wire, + { file: "src/new.ts", additions: 4, deletions: 0 }, + ]) + }) +}) + +describe("message", () => { + test("normalizes user summaries with object diffs", () => { + const input = { + id: "msg_1", + sessionID: "ses_1", + role: "user", + time: { created: 1 }, + agent: "build", + model: { providerID: "openai", modelID: "gpt-5" }, + summary: { + title: "Edit", + diffs: { a: item }, + }, + } as unknown as Message + + expect(message(input)).toMatchObject({ + summary: { + title: "Edit", + diffs: [item], + }, + }) + }) + + test("drops invalid user summaries", () => { + const input = { + id: "msg_1", + sessionID: "ses_1", + role: "user", + time: { created: 1 }, + agent: "build", + model: { providerID: "openai", modelID: "gpt-5" }, + summary: true, + } as unknown as Message + + expect(message(input)).toMatchObject({ summary: undefined }) + }) + + test("a live message.updated summary keeps its file list and counts without patch bodies (#1778)", () => { + const wire = [ + { file: "src/app.ts", additions: 1, deletions: 1, status: "modified" }, + { file: "src/new.ts", additions: 4, deletions: 0, status: "added" }, + ] + const input = { + id: "msg_1", + sessionID: "ses_1", + role: "user", + time: { created: 1 }, + agent: "build", + model: { providerID: "openai", modelID: "gpt-5" }, + summary: { title: "Edit", diffs: wire }, + } as unknown as Message + + // unchanged: the normalizer returns the very same message + expect(message(input)).toBe(input) + }) +}) + +describe("directory data provider — on-demand summary patches (#1778)", () => { + const source = readFileSync(resolve(__dirname, "../pages/directory-layout.tsx"), "utf8") + + test("supplies the per-turn diff view's patch source from the stored message", () => { + expect(source).toContain("onLoadSummaryDiffs={loadSummaryDiffs}") + expect(source).toContain(".client.session.message({ sessionID, messageID })") + expect(source).toContain("summary?.diffs") + }) +}) diff --git a/packages/app-bundle/overlay/packages/app/src/utils/diffs.ts b/packages/app-bundle/overlay/packages/app/src/utils/diffs.ts new file mode 100644 index 000000000..47415f871 --- /dev/null +++ b/packages/app-bundle/overlay/packages/app/src/utils/diffs.ts @@ -0,0 +1,53 @@ +import type { SnapshotFileDiff, VcsFileDiff } from "@opencode-ai/sdk/v2" +import type { FileDiffInfo } from "@opencode-ai/client/promise" +import type { Message } from "@opencode-ai/sdk/v2/client" + +type Diff = FileDiffInfo | SnapshotFileDiff | VcsFileDiff + +function diff(value: unknown): value is Diff { + if (!value || typeof value !== "object" || Array.isArray(value)) return false + if (!("file" in value) || typeof value.file !== "string") return false + // #1778 (amicode overlay): a live summary diff has no patch body (the engine + // strips it on the wire; it is fetched on demand) — absent is valid, a + // non-string patch is not. + if ("patch" in value && value.patch !== undefined && typeof value.patch !== "string") return false + if (!("additions" in value) || typeof value.additions !== "number") return false + if (!("deletions" in value) || typeof value.deletions !== "number") return false + if (!("status" in value) || value.status === undefined) return true + return value.status === "added" || value.status === "deleted" || value.status === "modified" +} + +function object(value: unknown): value is Record { + return !!value && typeof value === "object" && !Array.isArray(value) +} + +export function diffs(value: unknown): Diff[] { + if (Array.isArray(value) && value.every(diff)) return value + if (Array.isArray(value)) return value.filter(diff) + if (diff(value)) return [value] + if (!object(value)) return [] + return Object.values(value).filter(diff) +} + +export function message(value: Message): Message { + if (value.role !== "user") return value + + const raw = value.summary as unknown + if (raw === undefined) return value + if (!object(raw)) return { ...value, summary: undefined } + + const title = typeof raw.title === "string" ? raw.title : undefined + const body = typeof raw.body === "string" ? raw.body : undefined + const next = diffs(raw.diffs) + + if (title === raw.title && body === raw.body && next === raw.diffs) return value + + return { + ...value, + summary: { + ...(title === undefined ? {} : { title }), + ...(body === undefined ? {} : { body }), + diffs: next, + }, + } +} diff --git a/packages/app-bundle/overlay/packages/opencode/src/bus/global-replay.ts b/packages/app-bundle/overlay/packages/opencode/src/bus/global-replay.ts index 12bd3e2f3..1eefa5834 100644 --- a/packages/app-bundle/overlay/packages/opencode/src/bus/global-replay.ts +++ b/packages/app-bundle/overlay/packages/opencode/src/bus/global-replay.ts @@ -4,7 +4,7 @@ // every subscriber, bounded by bytes) and fans each frame out to the live // subscriptions. A connection that resumes from a servable cursor gets the // frames after it, then live frames; any other cursor gets an explicit reset. -import { GlobalBus } from "./global" +import { GlobalBus, type GlobalEvent } from "./global" export type Frame = { /** The SSE id: `:`. */ @@ -228,6 +228,44 @@ export const GlobalEventReplay = new ReplayBuffer({ start: GlobalBus.seq, }) +/** + * The wire form of a global event (#1778). A per-turn summary (`info.summary. + * diffs` of a message or session) carries the full patch text of every changed + * file — measured up to 12.3 MB in one event. On the wire each diff keeps its + * file, status and counts and drops its patch body; the surfaces that render a + * patch fetch the stored message on demand. Only the serialized frame changes: + * the event object (what in-process subscribers and storage see) is never + * mutated, and a sync copy is sent whole — its subscriber replays it into a + * store of its own, which must keep the full record. + */ +export function wireEvent(event: GlobalEvent): GlobalEvent { + const payload = event.payload + if (!payload || typeof payload !== "object" || payload.type === "sync") return event + const info = payload.properties?.info + const diffs = info?.summary?.diffs + if (!Array.isArray(diffs) || !diffs.some((diff) => diff && typeof diff === "object" && "patch" in diff)) return event + return { + ...event, + payload: { + ...payload, + properties: { + ...payload.properties, + info: { + ...info, + summary: { + ...info.summary, + diffs: diffs.map((diff) => { + if (!diff || typeof diff !== "object" || !("patch" in diff)) return diff + const { patch: _patch, ...rest } = diff + return rest + }), + }, + }, + }, + }, + } +} + GlobalBus.tap((event, seq) => { // A sync copy whose sequence was already handed out mirrors its source. One // that consumed a fresh sequence (no source framed nearby) is a frame of its @@ -235,7 +273,8 @@ GlobalBus.tap((event, seq) => { const syncCopy = event.payload?.type === "sync" && seq <= GlobalEventReplay.current let data: string try { - data = JSON.stringify(event) + // serialized once: every subscriber, live or replayed, gets this same frame + data = JSON.stringify(wireEvent(event)) } catch (error) { // An unserializable event cannot be framed; its sequence becomes a hole the // client detects (heartbeats carry the current sequence), never a dead stream. diff --git a/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts b/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts index e11a37eff..3c530f0d6 100644 --- a/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts +++ b/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts @@ -12,7 +12,7 @@ import * as TestClock from "effect/testing/TestClock" import { HttpRouter, HttpServer } from "effect/unstable/http" import { HttpApiBuilder } from "effect/unstable/httpapi" import { Auth } from "../../src/auth" -import { GlobalBus } from "../../src/bus/global" +import { GlobalBus, type GlobalEvent } from "../../src/bus/global" import { GlobalEventReplay } from "../../src/bus/global-replay" import { Config } from "../../src/config/config" import { Installation } from "../../src/installation" @@ -73,7 +73,8 @@ const eventually = (probe: () => boolean, ms = 3_000) => const ID = /^([A-Za-z0-9_-]+):(\d+)$/ -type Frame = { id: string | undefined; event: string | undefined; data: any } +/** One SSE frame; `raw` is its `data` exactly as sent, `bytes` the whole frame on the wire. */ +type Frame = { id: string | undefined; event: string | undefined; data: any; raw: string; bytes: number } /** Split an SSE id into its epoch and sequence (fails the test if malformed). */ function idOf(frame: Frame) { @@ -83,14 +84,15 @@ function idOf(frame: Frame) { } function parseFrame(block: string): Frame { - const frame: Frame = { id: undefined, event: undefined, data: undefined } + const frame: Frame = { id: undefined, event: undefined, data: undefined, raw: "", bytes: Buffer.byteLength(block) + 2 } const data: string[] = [] for (const line of block.split("\n")) { if (line.startsWith("id: ")) frame.id = line.slice(4) else if (line.startsWith("event: ")) frame.event = line.slice(7) else if (line.startsWith("data: ")) data.push(line.slice(6)) } - frame.data = data.length ? JSON.parse(data.join("\n")) : undefined + frame.raw = data.join("\n") + frame.data = data.length ? JSON.parse(frame.raw) : undefined return frame } @@ -274,6 +276,92 @@ describe("global event stream — sync copies are opt-in (#1778)", () => { ) }) +// A per-turn summary as the engine publishes it: every changed file with its +// full patch text (measured up to 12.3 MB in one event on the Studio). +const summaryDiffs = () => [ + { file: "src/app.ts", patch: "@@ -1,2 +1,3 @@\n-old\n+new\n+more\n", additions: 2, deletions: 1, status: "modified" }, + { file: "src/new.ts", patch: `@@ -0,0 +1,400 @@\n${"+line\n".repeat(400)}`, additions: 400, deletions: 0, status: "added" }, + { file: "src/gone.ts", patch: "@@ -1 +0,0 @@\n-gone\n", additions: 0, deletions: 1, status: "deleted" }, +] +const stripped = (diffs: ReturnType) => diffs.map(({ patch: _patch, ...rest }) => rest) + +/** A summary-bearing durable event and its sync copy, as the event-v2 bridge emits them. */ +const summaryTwin = (id: string, type: "message.updated" | "session.updated") => { + const info = + type === "message.updated" + ? { id: "msg_i1778", sessionID: "ses_i1778", role: "user", summary: { title: "Edit", diffs: summaryDiffs() } } + : { id: "ses_i1778", title: "Session", summary: { additions: 402, deletions: 2, files: 3, diffs: summaryDiffs() } } + const properties = { sessionID: "ses_i1778", info } + const source = { directory: "/tmp/i1778", payload: { id, type, properties } } + const copy = { + directory: "/tmp/i1778", + payload: { type: "sync", syncEvent: { id, type: `${type}.1`, seq: 3, aggregateID: "ses_i1778", data: properties } }, + } + return { source, copy } +} + +describe("global event stream — summary diffs carry no patch bodies on the wire (#1778)", () => { + for (const type of ["message.updated", "session.updated"] as const) { + it.live(`a live ${type} keeps file, status and counts but drops patch bodies; the bus payload keeps them`, () => + Effect.gen(function* () { + const plain = yield* connect() + const synced = yield* connect({ headers: { "x-opencode-sync-copies": "true" } }) + yield* plain.next + yield* synced.next + const seen: GlobalEvent[] = [] + const listener = (event: GlobalEvent) => seen.push(event) + GlobalBus.on("event", listener) + yield* Effect.addFinalizer(() => Effect.sync(() => GlobalBus.off("event", listener))) + + const { source, copy } = summaryTwin(`evt_i1778_${type}`, type) + GlobalBus.emit("event", source) + GlobalBus.emit("event", copy) + + const live = yield* plain.next + expect(live.data.payload.type).toBe(type) + const info = live.data.payload.properties.info + expect(info.summary.diffs).toEqual(stripped(summaryDiffs())) + expect(live.raw).not.toContain('"patch"') + // everything else in the event is as published + expect({ ...info, summary: { ...info.summary, diffs: undefined } }).toEqual({ + ...source.payload.properties.info, + summary: { ...source.payload.properties.info.summary, diffs: undefined }, + }) + // the projection is wire-only: in-process subscribers see the full event, untouched + expect(seen.find((event) => event.payload.type === type)?.payload.properties.info.summary.diffs).toEqual( + summaryDiffs(), + ) + + // the opt-in subscriber gets the same stripped frame, and a full sync copy + // (the replica replays it into its own store, so it keeps the patches) + const [same, sync] = yield* synced.take(2) + expect(same.raw).toBe(live.raw) + expect(same.id).toBe(live.id) + expect(sync.data.payload.type).toBe("sync") + expect(sync.data.payload.syncEvent.data.info.summary.diffs).toEqual(summaryDiffs()) + }), + ) + } + + it.live("a replayed summary frame is byte-identical to the live one", () => + Effect.gen(function* () { + const a = yield* connect() + const cursor = (yield* a.next).id! + const { source } = summaryTwin("evt_i1778_replayed", "message.updated") + GlobalBus.emit("event", source) + const live = yield* a.next + a.abort() + + const b = yield* connect({ headers: { "last-event-id": cursor } }) + yield* b.next + const replayed = yield* b.next + expect(replayed.id).toBe(live.id) + expect(replayed.raw).toBe(live.raw) + expect(replayed.raw).not.toContain('"patch"') + }), + ) +}) + describe("global event stream — resume from a cursor (#1776)", () => { for (const carrier of ["Last-Event-ID header", "lastEventID query"] as const) { it.live(`replays exactly the frames after a ${carrier} cursor, then continues live`, () => diff --git a/packages/app-bundle/overlay/packages/opencode/test/server/summary-diff-wire.test.ts b/packages/app-bundle/overlay/packages/opencode/test/server/summary-diff-wire.test.ts new file mode 100644 index 000000000..05ec718bc --- /dev/null +++ b/packages/app-bundle/overlay/packages/opencode/test/server/summary-diff-wire.test.ts @@ -0,0 +1,110 @@ +/** + * #1778 — summary patch bodies are cut on the wire only. A real user message + * with per-file summary diffs goes through the session store and the event-v2 + * bridge: its live global-stream frame carries file/status/counts and no patch + * text, while the stored message — what `GET /session/:id/message/:messageID` + * serves, the source the per-turn diff view fetches a patch from on demand — + * keeps every patch. The durable event's sync copy (sent only to subscribers + * that replay it into a store of their own) keeps them too. + */ +import { afterEach, describe, expect } from "bun:test" +import { LayerNode } from "@opencode-ai/core/effect/layer-node" +import { Database } from "@opencode-ai/core/database/database" +import { FSUtil } from "@opencode-ai/core/fs-util" +import { ProviderV2 } from "@opencode-ai/core/provider" +import { ModelV2 } from "@opencode-ai/core/model" +import { SessionV1 } from "@opencode-ai/core/v1/session" +import { Effect, Layer } from "effect" +import { GlobalEventReplay, type Frame } from "@/bus/global-replay" +import { Session } from "@/session/session" +import { Snapshot } from "@/snapshot" +import { Storage } from "@/storage/storage" +import { SessionPaths } from "@/server/routes/instance/httpapi/groups/session" +import { MessageID } from "@/session/schema" +import { resetDatabase } from "../fixture/db" +import { disposeAllInstances, TestInstance } from "../fixture/fixture" +import { testEffect } from "../lib/effect" +import { httpApiLayer, requestInDirectory } from "./httpapi-layer" + +const it = testEffect( + Layer.mergeAll( + LayerNode.compile(LayerNode.group([Session.node, Snapshot.node, Storage.node, FSUtil.node, Database.node])), + httpApiLayer, + ), +) + +afterEach(async () => { + await disposeAllInstances() + await resetDatabase() +}) + +const pathFor = (template: string, params: Record) => + Object.entries(params).reduce((result, [key, value]) => result.replace(`:${key}`, value), template) + +const diffs = [ + { file: "src/main.ts", patch: "@@ -1 +1,2 @@\n-a\n+b\n+c\n", additions: 2, deletions: 1, status: "modified" as const }, + { file: "src/new.ts", patch: `@@ -0,0 +1,300 @@\n${"+x\n".repeat(300)}`, additions: 300, deletions: 0, status: "added" as const }, +] + +describe("summary diffs — wire-only projection (#1778)", () => { + it.instance( + "the live frame drops patch bodies; the stored message and the sync copy keep them", + () => + Effect.gen(function* () { + const test = yield* TestInstance + const frames: Frame[] = [] + const subscription = GlobalEventReplay.subscribe(undefined, { sync: true }) + subscription.start((frame) => frames.push(frame)) + yield* Effect.addFinalizer(() => Effect.sync(() => subscription.close())) + + const session = yield* Effect.acquireRelease(Session.use.create({ title: "wire-projection" }), (created) => + Session.use.remove(created.id).pipe(Effect.ignore), + ) + const messageID = MessageID.ascending() + yield* Session.use.updateMessage({ + id: messageID, + sessionID: session.id, + role: "user", + time: { created: Date.now() }, + agent: "build", + model: { providerID: ProviderV2.ID.make("test"), modelID: ModelV2.ID.make("model") }, + summary: { title: "Edit", diffs }, + } satisfies SessionV1.User) + + const ours = () => + frames + .map((frame) => ({ frame, event: JSON.parse(frame.data) })) + .filter(({ event }) => + event.payload.type === "sync" + ? event.payload.syncEvent.data?.info?.id === messageID + : event.payload.type === "message.updated" && event.payload.properties.info.id === messageID, + ) + // the bridge emits synchronously inside publish: both frames exist now + const [live, copy, ...extra] = ours() + expect(extra).toEqual([]) + + expect(live.event.payload.type).toBe("message.updated") + expect(live.frame.syncCopy).toBe(false) + expect(live.frame.data).not.toContain('"patch"') + expect(live.event.payload.properties.info.summary).toEqual({ + title: "Edit", + diffs: diffs.map(({ patch: _patch, ...rest }) => rest), + }) + + expect(copy.event.payload.type).toBe("sync") + expect(copy.frame.syncCopy).toBe(true) + expect(copy.frame.seq).toBe(live.frame.seq) + expect(copy.event.payload.syncEvent.data.info.summary.diffs).toEqual(diffs) + + // the stored record is unchanged — and it is the on-demand patch source + const response = yield* requestInDirectory( + pathFor(SessionPaths.message, { sessionID: session.id, messageID }), + test.directory, + ) + expect(response.status).toBe(200) + const stored = (yield* response.json) as { info: { summary?: { diffs: unknown } } } + expect(stored.info.summary?.diffs).toEqual(diffs) + }), + { git: true, config: { formatter: false, lsp: false } }, + ) +}) diff --git a/packages/app-bundle/overlay/packages/session-ui/src/components/session-turn.tsx b/packages/app-bundle/overlay/packages/session-ui/src/components/session-turn.tsx index da8ed7d53..56028dd43 100644 --- a/packages/app-bundle/overlay/packages/session-ui/src/components/session-turn.tsx +++ b/packages/app-bundle/overlay/packages/session-ui/src/components/session-turn.tsx @@ -26,6 +26,7 @@ import { TextReveal } from "@opencode-ai/ui/text-reveal" import { createAutoScroll } from "@opencode-ai/ui/hooks" import { useI18n } from "@opencode-ai/ui/context/i18n" import { normalize } from "./session-diff" +import { hasBody } from "./summary-patch" function record(value: unknown): value is Record { return !!value && typeof value === "object" && !Array.isArray(value) @@ -470,9 +471,32 @@ export function SessionTurn( > {(diff) => { - const view = normalize(diff) const active = createMemo(() => expanded().includes(diff.file)) const [shown, setShown] = createSignal(false) + // #1778: a live summary carries no patch body — fetch the + // turn's stored diffs once this file is expanded and shown. + // A plain signal, not a resource: a pending fetch must not + // suspend the timeline around it. + const [fetched, setFetched] = createSignal() + createEffect( + on(shown, (value) => { + if (!value || hasBody(diff) || fetched() !== undefined) return + void data + .summaryPatch({ + sessionID: props.sessionID, + messageID: props.messageID, + file: diff.file, + summary: diffs(), + }) + .then((patch) => { + if (patch !== undefined) setFetched(patch) + }) + }), + ) + const view = createMemo(() => { + const patch = fetched() + return normalize(patch === undefined ? diff : { ...diff, patch }) + }) createEffect( on( @@ -519,7 +543,7 @@ export function SessionTurn(
- +
diff --git a/packages/app-bundle/overlay/packages/session-ui/src/components/summary-patch.test.ts b/packages/app-bundle/overlay/packages/session-ui/src/components/summary-patch.test.ts new file mode 100644 index 000000000..0f0b03623 --- /dev/null +++ b/packages/app-bundle/overlay/packages/session-ui/src/components/summary-patch.test.ts @@ -0,0 +1,124 @@ +import { describe, expect, test } from "bun:test" +import { readFileSync } from "node:fs" +import { resolve } from "node:path" +import { normalize } from "./session-diff" +import { createSummaryPatchLoader, hasBody, type SummaryDiffSource } from "./summary-patch" + +// #1778 — live per-turn summaries carry each file's status and counts but no +// patch body (the engine strips patches on the wire; storage keeps them). The +// per-turn diff view fetches a patch on demand when a file is expanded, from +// the stored message. The loader is the behavior; session-turn.tsx's wiring is +// asserted on its source below (bun's test transpiler does not run Solid's +// JSX transform — the repo's component-source-assertion convention). + +const PATCH = "@@ -1,2 +1,2 @@\n-old line\n+new line\n context\n" +const stored = [ + { file: "src/app.ts", patch: PATCH, additions: 1, deletions: 1, status: "modified" as const }, + { file: "src/other.ts", patch: "@@ -0,0 +1 @@\n+other\n", additions: 1, deletions: 0, status: "added" as const }, +] +// what a live message.updated carries after the wire projection +const live = stored.map(({ patch: _patch, ...rest }) => rest) +const turn = { sessionID: "ses_1", messageID: "msg_1" } + +function recordingSource(answer: () => Promise = async () => stored) { + const calls: { sessionID: string; messageID: string }[] = [] + const source: SummaryDiffSource = (input) => { + calls.push(input) + return answer() + } + return { calls, source } +} + +describe("per-turn diff view — patch bodies on demand (#1778)", () => { + test("a live summary's file list and counts need no fetch; only a patch-less diff needs one", () => { + const { calls, source } = recordingSource() + createSummaryPatchLoader(source) + expect(live.map((diff) => [diff.file, diff.additions, diff.deletions, diff.status])).toEqual([ + ["src/app.ts", 1, 1, "modified"], + ["src/other.ts", 1, 0, "added"], + ]) + expect(live.map(hasBody)).toEqual([false, false]) + expect(stored.map(hasBody)).toEqual([true, true]) + expect(hasBody({ file: "a", before: "x", after: "y" })).toBe(true) + expect(calls).toEqual([]) + }) + + test("expanding a file fetches the turn's stored diffs once and renders that file's patch", async () => { + const { calls, source } = recordingSource() + const loader = createSummaryPatchLoader(source) + + const patch = await loader.patch({ ...turn, file: "src/app.ts", summary: live }) + expect(patch).toBe(PATCH) + const view = normalize({ ...live[0], patch }) + expect(view.fileDiff.additionLines.join("")).toContain("new line") + expect(view.fileDiff.deletionLines.join("")).toContain("old line") + expect({ additions: view.additions, deletions: view.deletions, status: view.status }).toEqual({ + additions: 1, + deletions: 1, + status: "modified", + }) + + // a second file of the same turn — and concurrent expands — share the one fetch + const [other, again] = await Promise.all([ + loader.patch({ ...turn, file: "src/other.ts", summary: live }), + loader.patch({ ...turn, file: "src/app.ts", summary: live }), + ]) + expect(other).toBe(stored[1].patch) + expect(again).toBe(PATCH) + expect(calls).toEqual([turn]) + }) + + test("a failed fetch resolves to no patch and is retried on the next expand", async () => { + let fail = true + const { calls, source } = recordingSource(async () => { + if (fail) throw new Error("offline") + return stored + }) + const loader = createSummaryPatchLoader(source) + expect(await loader.patch({ ...turn, file: "src/app.ts", summary: live })).toBeUndefined() + fail = false + expect(await loader.patch({ ...turn, file: "src/app.ts", summary: live })).toBe(PATCH) + expect(calls).toHaveLength(2) + }) + + test("a turn whose summary changed (a later message.updated) is fetched afresh", async () => { + const { calls, source } = recordingSource() + const loader = createSummaryPatchLoader(source) + await loader.patch({ ...turn, file: "src/app.ts", summary: live }) + await loader.patch({ ...turn, file: "src/app.ts", summary: [{ ...live[0], additions: 7 }, live[1]] }) + expect(calls).toHaveLength(2) + }) + + test("without a source (a provider that cannot fetch) a patch-less diff resolves to no patch", async () => { + expect(await createSummaryPatchLoader(undefined).patch({ ...turn, file: "src/app.ts", summary: live })).toBeUndefined() + }) +}) + +describe("per-turn diff view — wiring (#1778)", () => { + const turnSource = readFileSync(resolve(__dirname, "session-turn.tsx"), "utf8") + const dataSource = readFileSync(resolve(__dirname, "../context/data.tsx"), "utf8") + + test("the data provider takes an on-demand summary-diff source and shares one loader", () => { + expect(dataSource).toContain("onLoadSummaryDiffs?: SummaryDiffSource") + expect(dataSource).toContain("createSummaryPatchLoader(props.onLoadSummaryDiffs)") + expect(dataSource).toContain("summaryPatch:") + }) + + test("the accordion fetches a patch only once a patch-less file is expanded, and renders the fetched patch", () => { + // the fetch fires from the expanded+shown edge, only for a diff without a body + expect(turnSource).toContain("on(shown, (value) => {") + expect(turnSource).toContain("if (!value || hasBody(diff) || fetched() !== undefined) return") + expect(turnSource).toContain(".summaryPatch({") + expect(turnSource).toContain("normalize(patch === undefined ? diff : { ...diff, patch })") + expect(turnSource).toContain("fileDiff={view().fileDiff}") + // the per-file view is no longer parsed eagerly for collapsed files + expect(turnSource).not.toContain("const view = normalize(diff)") + }) + + test("the file list and counts render from the summary itself, with no fetch", () => { + expect(turnSource).toContain("") + expect(turnSource).toContain("") + const memo = turnSource.slice(turnSource.indexOf("const diffs = createMemo"), turnSource.indexOf("const MAX_FILES")) + expect(memo).not.toContain("summaryPatch") + }) +}) diff --git a/packages/app-bundle/overlay/packages/session-ui/src/components/summary-patch.ts b/packages/app-bundle/overlay/packages/session-ui/src/components/summary-patch.ts new file mode 100644 index 000000000..ca250f917 --- /dev/null +++ b/packages/app-bundle/overlay/packages/session-ui/src/components/summary-patch.ts @@ -0,0 +1,77 @@ +// #1778 — per-turn summary patches on demand. The engine sends a live summary's +// diffs with file, status and counts but no patch body (patches stay in +// storage); the per-turn diff view fetches the turn's stored diffs only when a +// patch-less file is expanded. Plain TypeScript (no reactivity) so the +// behavior is testable directly; session-turn.tsx drives it from its expand +// state. + +/** A diff as a summary carries it — the patch body is optional on the wire. */ +export type SummaryDiffLike = { + file?: string + patch?: string + before?: string + after?: string + additions?: number + deletions?: number + status?: string +} + +/** Fetch a turn's stored summary diffs (with patch bodies), e.g. from `GET /session/:id/message/:messageID`. */ +export type SummaryDiffSource = (input: { + sessionID: string + messageID: string +}) => Promise + +export type SummaryPatchRequest = { + sessionID: string + messageID: string + /** The expanded file. */ + file: string + /** The turn's summary diffs as currently shown — a changed summary is fetched afresh. */ + summary: readonly SummaryDiffLike[] +} + +/** Whether a diff can render without a fetch: it carries a patch, or before/after contents. */ +export function hasBody(diff: SummaryDiffLike) { + return typeof diff.patch === "string" || typeof diff.before === "string" || typeof diff.after === "string" +} + +const signature = (summary: readonly SummaryDiffLike[]) => + JSON.stringify(summary.map((diff) => [diff.file, diff.status, diff.additions, diff.deletions])) + +/** + * One loader per data provider. A turn's stored diffs are fetched at most once + * per summary (every file of the turn, and concurrent expands, share it); a + * failed fetch is forgotten so the next expand retries it. + */ +export function createSummaryPatchLoader(source: SummaryDiffSource | undefined) { + const turns = new Map>() + const limit = 32 + + const fetchTurn = (input: SummaryPatchRequest) => { + const key = `${input.sessionID}\0${input.messageID}\0${signature(input.summary)}` + const hit = turns.get(key) + if (hit) return hit + const pending = Promise.resolve() + .then(() => source?.({ sessionID: input.sessionID, messageID: input.messageID })) + .catch(() => { + turns.delete(key) + return undefined + }) + turns.set(key, pending) + while (turns.size > limit) turns.delete(turns.keys().next().value!) + return pending + } + + return { + /** The expanded file's patch body, or `undefined` when it cannot be had. */ + async patch(input: SummaryPatchRequest): Promise { + if (!source) return undefined + const diffs = await fetchTurn(input) + const match = diffs?.findLast((diff) => diff.file === input.file) + return typeof match?.patch === "string" ? match.patch : undefined + }, + } +} + +export type SummaryPatchLoader = ReturnType diff --git a/packages/app-bundle/overlay/packages/session-ui/src/context/data.tsx b/packages/app-bundle/overlay/packages/session-ui/src/context/data.tsx new file mode 100644 index 000000000..4541bd681 --- /dev/null +++ b/packages/app-bundle/overlay/packages/session-ui/src/context/data.tsx @@ -0,0 +1,82 @@ +import type { Message, Session, Part, SnapshotFileDiff, SessionStatus, Provider } from "@opencode-ai/sdk/v2" +import type { FileDiffInfo } from "@opencode-ai/client/promise" +import { createSimpleContext } from "@opencode-ai/ui/context" +import { PreloadMultiFileDiffResult } from "@pierre/diffs/ssr" +import { createSummaryPatchLoader, type SummaryDiffSource } from "../components/summary-patch" + +export type NormalizedProviderListResponse = { + all: Map + defaultModel?: { + providerID: string + modelID: string + } | null + default: { + [key: string]: string + } + connected: Array +} + +type Data = { + agent?: { + name: string + color?: string + }[] + provider?: NormalizedProviderListResponse + session: Session[] + session_status: { + [sessionID: string]: SessionStatus + } + session_diff: { + [sessionID: string]: (SnapshotFileDiff | FileDiffInfo)[] + } + session_diff_preload?: { + [sessionID: string]: PreloadMultiFileDiffResult[] + } + message: { + [sessionID: string]: Message[] + } + part: { + [messageID: string]: Part[] + } + part_text_accum_delta?: { + [partID: string]: string + } +} + +export type NavigateToSessionFn = (sessionID: string) => void + +export type SessionHrefFn = (sessionID: string) => string + +export const { use: useData, provider: DataProvider } = createSimpleContext({ + name: "Data", + init: (props: { + data: Data + directory: string + sessionID?: string + onNavigateToSession?: NavigateToSessionFn + onSessionHref?: SessionHrefFn + /** + * #1778 (amicode overlay): fetch a turn's stored summary diffs, patch + * bodies included. Live summaries carry no patch bodies (the engine + * strips them on the wire), so the per-turn diff view asks for a patch + * when a file is expanded. Without it a patch-less diff renders empty. + */ + onLoadSummaryDiffs?: SummaryDiffSource + }) => { + const loader = createSummaryPatchLoader(props.onLoadSummaryDiffs) + return { + get store() { + return props.data + }, + get directory() { + return props.directory + }, + get sessionID() { + return props.sessionID + }, + navigateToSession: props.onNavigateToSession, + sessionHref: props.onSessionHref, + summaryPatch: loader.patch, + } + }, +}) From eb32273de0243d1374332aa366681ec688814473 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 18:09:11 -0400 Subject: [PATCH 24/60] =?UTF-8?q?test(fleet):=20fan-in=20suites=20feed=20e?= =?UTF-8?q?ngine-shaped=20frames=20=E2=80=94=20id-less,=20sequenced,=20syn?= =?UTF-8?q?c=20twin,=20heartbeat=20=E2=80=94=20not=20synthetic=20id:=201?= =?UTF-8?q?=20(#1771)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../test/sse_fanin_aggregator.test.ts | 131 +++++++++--------- .../test/sse_fanin_driver_pump.test.ts | 107 ++++++++------ 2 files changed, 129 insertions(+), 109 deletions(-) diff --git a/packages/extension/test/sse_fanin_aggregator.test.ts b/packages/extension/test/sse_fanin_aggregator.test.ts index bc81920c3..b2f12c4a9 100644 --- a/packages/extension/test/sse_fanin_aggregator.test.ts +++ b/packages/extension/test/sse_fanin_aggregator.test.ts @@ -41,7 +41,7 @@ import { import { SessionOwnerMap } from "../src/amicode_service/session_multiplexer"; import { peerAuthHeader } from "../src/amicode_service/merged_projection"; import type { PeerTokenRead } from "../src/amicode_service/fleet_peer_store"; -import { fakeEngine, sequencedEvent, LOCAL_EPOCH, PEER_EPOCH } from "./fixtures/engine_stream/engine_frames"; +import { fakeEngine, sequencedEvent, engineId, LOCAL_EPOCH, PEER_EPOCH } from "./fixtures/engine_stream/engine_frames"; // ── test helpers ────────────────────────────────────────────────────────────── @@ -70,6 +70,20 @@ function frame(...lines: string[]): string { return lines.join("\n") + "\n\n"; } +/** #1771: ENGINE-SHAPED arm frames (test/fixtures/engine_stream) — what each + * arm's upstream `/global/event` actually sends: a sequenced (#1776) bus event + * `id: :` + one `data:` line (no `event:` line), one engine epoch + * per namespace. `frame()` above stays for the pure id-parsing/relay units. */ +const MINI_EPOCH = "bWluaTAx"; +const EPOCH_OF: Record = { [LOCAL_NAMESPACE]: LOCAL_EPOCH, studio: PEER_EPOCH, mini: MINI_EPOCH }; +function nsFrame(ns: string, n: number): string { + return sequencedEvent(EPOCH_OF[ns], n, { ns, n }); +} +/** The engine id `nsFrame(ns, n)` carries. */ +const idIn = (ns: string, n: number) => engineId(EPOCH_OF[ns], n); +/** The seq of an `:` id. */ +const seqOf = (id: string | undefined) => Number(id?.split(":")[1]); + /** A frame source that yields a fixed list of raw frames, then ends. */ function fixedSource(frames: string[]): SseFrameSource { let i = 0; @@ -90,19 +104,25 @@ function fixedSource(frames: string[]): SseFrameSource { // ══════════════════════════════════════════════════════════════════════════════ describe("#1511 AC1 (D4) — fleet-of-one byte-identity is the #1264 regression guard", () => { it("with zero peers, the aggregate stream is frame-for-frame identical to the local source", () => { - const sink = collectingSink(); - const agg = new SseFanInAggregator({ sink }); - agg.connect(); // no cursor, no peers, no focus provider → pure passthrough - - const f1 = frame("event: message", "data: {\"a\":1}", "id: 1"); - const f2 = frame("event: message", "data: {\"b\":2}", "id: 2"); - const f3 = frame("data: {\"c\":3}", "id: 3"); // no event: line — still verbatim - agg.ingest(LOCAL_NAMESPACE, f1); - agg.ingest(LOCAL_NAMESPACE, f2); - agg.ingest(LOCAL_NAMESPACE, f3); - - // BYTE-IDENTICAL: no id rewrite, no namespacing, no reordering, no loss. - expect(sink.text()).toBe(f1 + f2 + f3); + // #1771: the local engine's REAL stream, from an old (id-less) engine and a + // #1776 (sequenced) one: connected, an event, the plain + sync twin of a + // stored event, a heartbeat. + for (const engine of [fakeEngine({ legacy: true }), fakeEngine({ epoch: LOCAL_EPOCH })]) { + const sink = collectingSink(); + const agg = new SseFanInAggregator({ sink }); + agg.connect(); // no cursor, no peers, no focus provider → pure passthrough + + const frames = [ + engine.connected(), + engine.event("session.status", { a: 1 }), + ...engine.stored("message.part.updated", { b: 2 }, { aggregateID: "ses_1" }), + engine.heartbeat(), + ]; + for (const f of frames) agg.ingest(LOCAL_NAMESPACE, f); + + // BYTE-IDENTICAL: no id rewrite, no namespacing, no reordering, no loss. + expect(sink.text()).toBe(frames.join("")); + } }); it("`?lastEventID=local=` AND bare `` both resume the local arm at ", () => { @@ -167,14 +187,18 @@ describe("#1511 AC4 (D3) — composite cursor", () => { agg.connect(); agg.setMembership([{ machineId: "studio", reachable: true, token: "tok-studio" }]); // studio advances to 9 - agg.ingest("studio", frame("event: message", "data: {}", "id: 9")); - expect(parseCompositeCursor(agg.cursor()).get("studio")).toBe("9"); - // an id-less studio frame (heartbeat/comment) must NOT move the cursor - agg.ingest("studio", frame(": keep-alive comment")); - expect(parseCompositeCursor(agg.cursor()).get("studio")).toBe("9"); + agg.ingest("studio", nsFrame("studio", 9)); + expect(parseCompositeCursor(agg.cursor()).get("studio")).toBe(idIn("studio", 9)); + // an id-less studio frame must NOT move the cursor: an old engine's + // heartbeat and event (#1771 — the id-less frames production sees), or a comment + const old = fakeEngine({ legacy: true }); + for (const f of [old.heartbeat(), old.event("session.status", { s: 1 }), frame(": keep-alive comment")]) { + agg.ingest("studio", f); + expect(parseCompositeCursor(agg.cursor()).get("studio")).toBe(idIn("studio", 9)); + } // and neither does a frame with an empty id value agg.ingest("studio", frame("data: {}", "id:")); - expect(parseCompositeCursor(agg.cursor()).get("studio")).toBe("9"); + expect(parseCompositeCursor(agg.cursor()).get("studio")).toBe(idIn("studio", 9)); }); }); @@ -205,14 +229,11 @@ describe("#1511 AC3 (D2) — namespaced relay + res adapter", () => { it("pipeToResponse writes well-formed namespaced SSE bytes and flushes per frame", async () => { const sink = collectingSink(); - const src = fixedSource([ - frame("event: message", "data: {\"a\":1}", "id: 1"), - frame("event: message", "data: {\"b\":2}", "id: 2"), - ]); + const src = fixedSource([nsFrame("studio", 1), nsFrame("studio", 2)]); await pipeToResponse(src, sink, { namespace: "studio" }); const text = sink.text(); - expect(text).toContain(`id: studio${NS_SEP}1`); - expect(text).toContain(`id: studio${NS_SEP}2`); + expect(text).toContain(`id: studio${NS_SEP}${idIn("studio", 1)}`); + expect(text).toContain(`id: studio${NS_SEP}${idIn("studio", 2)}`); // every frame ends with the SSE blank-line delimiter expect(text.endsWith("\n\n")).toBe(true); // one flush per frame (no head-of-line buffering) @@ -379,9 +400,9 @@ describe("#1511 AC6 (A) — focus-on-connect snapshot", () => { expect(sink.text()).toContain("\"focusedMachineId\":\"studio\""); // a following local event frame comes AFTER the focus seed - agg.ingest(LOCAL_NAMESPACE, frame("data: {}", "id: 1")); + agg.ingest(LOCAL_NAMESPACE, nsFrame(LOCAL_NAMESPACE, 1)); const idx = sink.text().indexOf("amicode.fleet.focus"); - const evtIdx = sink.text().indexOf("id: 1"); + const evtIdx = sink.text().indexOf(`id: ${idIn(LOCAL_NAMESPACE, 1)}`); expect(idx).toBeGreaterThanOrEqual(0); expect(evtIdx).toBeGreaterThan(idx); }); @@ -480,9 +501,6 @@ describe("#1511 AC7 (B) — per-namespace back-pressure", () => { }; } - function nsFrame(ns: string, n: number): string { - return frame("event: message", `data: {"ns":"${ns}","n":${n}}`, `id: ${n}`); - } it("one overflowing peer drops in ISOLATION; the other namespaces and local keep flowing", () => { const sink = controllableSink(); @@ -523,11 +541,11 @@ describe("#1511 AC7 (B) — per-namespace back-pressure", () => { // studio's cursor sits at the last DELIVERED id, never a dropped one — // so a reconnect with this composite replays the gap losslessly (D3 × B). - const studioCursor = Number(parseCompositeCursor(agg.cursor()).get("studio")); + const studioCursor = seqOf(parseCompositeCursor(agg.cursor()).get("studio")); expect(studioCursor).toBeLessThan(5); // mini + local cursors reflect full delivery - expect(parseCompositeCursor(agg.cursor()).get("mini")).toBe("2"); - expect(parseCompositeCursor(agg.cursor()).get("local")).toBe("1"); + expect(parseCompositeCursor(agg.cursor()).get("mini")).toBe(idIn("mini", 2)); + expect(parseCompositeCursor(agg.cursor()).get("local")).toBe(idIn(LOCAL_NAMESPACE, 1)); }); it("steady state (downstream flowing) never buffers — frames deliver directly", () => { @@ -573,9 +591,6 @@ describe("#1617 — backpressure re-arm + gap signal", () => { }, }; } - function nsFrame(ns: string, n: number): string { - return frame("event: message", `data: {"ns":"${ns}","n":${n}}`, `id: ${n}`); - } it("the WEDGE (no overflow): one trip-frame strands one frame behind it; resume() delivers BOTH in order", () => { const sink = controllableSink(); @@ -597,7 +612,7 @@ describe("#1617 — backpressure re-arm + gap signal", () => { // order preserved: frame 1 before frame 2 expect(sink.text().indexOf('"n":1')).toBeLessThan(sink.text().indexOf('"n":2')); // cursor advanced past both delivered frames - expect(parseCompositeCursor(agg.cursor()).get("studio")).toBe("2"); + expect(parseCompositeCursor(agg.cursor()).get("studio")).toBe(idIn("studio", 2)); }); it("onBackpressure fires exactly when a write() returns false (the driver's upstream-pause hook)", () => { @@ -690,8 +705,8 @@ describe("#1617 — backpressure re-arm + gap signal", () => { const agg = new SseFanInAggregator({ sink }); agg.connect(); // zero peers → the §D4 verbatim branch - const f1 = frame("event: message", 'data: {"big":"tool-output"}', "id: 7"); - const f2 = frame("event: session.status", 'data: {"status":"idle"}', "id: 8"); // the rail-clearing frame + const f1 = sequencedEvent(LOCAL_EPOCH, 7, { big: "tool-output" }); + const f2 = sequencedEvent(LOCAL_EPOCH, 8, { status: "idle" }, "session.status"); // the rail-clearing frame sink.block(); // the socket fills on the NEXT write agg.ingest(LOCAL_NAMESPACE, f1); // trip-frame: written, write()→false @@ -708,7 +723,7 @@ describe("#1617 — backpressure re-arm + gap signal", () => { // composite id, no namespacing, no gap frame (no overflow occurred). expect(sink.text()).toBe(f1 + f2); expect(sink.text()).not.toContain("local\u001f"); // no NS_SEP namespacing - expect(sink.text()).not.toContain("local=8"); // no composite-cursor id rewrite + expect(sink.text()).not.toContain(`local=${idIn(LOCAL_NAMESPACE, 8)}`); // no composite-cursor id rewrite expect(sink.text()).not.toContain("amicode.sync.gap"); }); @@ -716,8 +731,8 @@ describe("#1617 — backpressure re-arm + gap signal", () => { const sink = controllableSink(); const agg = new SseFanInAggregator({ sink }); agg.connect(); - const f1 = frame("event: message", 'data: {"a":1}', "id: 1"); - const f2 = frame("data: {\"c\":3}", "id: 3"); // no event: line + // an old engine's id-less stream: the plain + sync twin of a stored event + const [f1, f2] = fakeEngine({ legacy: true }).stored("message.updated", { a: 1 }, { aggregateID: "ses_1" }); agg.ingest(LOCAL_NAMESPACE, f1); agg.ingest(LOCAL_NAMESPACE, f2); expect(sink.text()).toBe(f1 + f2); // untouched fast path @@ -733,9 +748,6 @@ describe("#1617 — backpressure re-arm + gap signal", () => { // the verbatim stream would break #1264, so NO gap is emitted there. // ══════════════════════════════════════════════════════════════════════════════ describe("#1638 — per-namespace cursor accessor (cursorFor)", () => { - function nsFrame(ns: string, n: number): string { - return frame("event: message", `data: {"ns":"${ns}","n":${n}}`, `id: ${n}`); - } it("cursorFor(ns) returns the live last-DELIVERED id for that namespace, advancing per delivered frame", () => { const sink = collectingSink(); @@ -747,18 +759,18 @@ describe("#1638 — per-namespace cursor accessor (cursorFor)", () => { expect(agg.cursorFor("studio")).toBeUndefined(); agg.ingest("studio", nsFrame("studio", 1)); - expect(agg.cursorFor("studio")).toBe("1"); // advanced to the delivered id + expect(agg.cursorFor("studio")).toBe(idIn("studio", 1)); // advanced to the delivered id agg.ingest("studio", nsFrame("studio", 2)); - expect(agg.cursorFor("studio")).toBe("2"); // advanced again + expect(agg.cursorFor("studio")).toBe(idIn("studio", 2)); // advanced again }); it("cursorFor tracks the LOCAL namespace on the verbatim (fleet-of-one) path too", () => { const sink = collectingSink(); const agg = new SseFanInAggregator({ sink }); agg.connect(); // zero peers → verbatim - agg.ingest(LOCAL_NAMESPACE, nsFrame("local", 5)); - expect(agg.cursorFor(LOCAL_NAMESPACE)).toBe("5"); + agg.ingest(LOCAL_NAMESPACE, nsFrame(LOCAL_NAMESPACE, 5)); + expect(agg.cursorFor(LOCAL_NAMESPACE)).toBe(idIn(LOCAL_NAMESPACE, 5)); }); it("an id-less frame does NOT advance cursorFor (matches the composite-cursor invariant)", () => { @@ -767,15 +779,12 @@ describe("#1638 — per-namespace cursor accessor (cursorFor)", () => { agg.connect(); agg.setMembership([{ machineId: "studio", reachable: true, token: "tok-studio" }]); agg.ingest("studio", nsFrame("studio", 1)); - agg.ingest("studio", frame(": heartbeat")); // id-less - expect(agg.cursorFor("studio")).toBe("1"); // unchanged by the id-less frame + agg.ingest("studio", fakeEngine({ legacy: true }).heartbeat()); // id-less (an old engine's heartbeat) + expect(agg.cursorFor("studio")).toBe(idIn("studio", 1)); // unchanged by the id-less frame }); }); describe("#1638 — composite-mode reopenArm emits exactly one id-less gap; fleet-of-one emits none", () => { - function nsFrame(ns: string, n: number): string { - return frame("event: message", `data: {"ns":"${ns}","n":${n}}`, `id: ${n}`); - } it("COMPOSITE MODE: a non-initial re-open emits EXACTLY ONE id-less amicode.sync.gap naming that namespace", () => { const sink = collectingSink(); @@ -803,7 +812,7 @@ describe("#1638 — composite-mode reopenArm emits exactly one id-less gap; flee const sink = collectingSink(); const agg = new SseFanInAggregator({ sink }); agg.connect(); // zero peers → verbatim, no focus provider - const f1 = frame("event: message", 'data: {"a":1}', "id: 1"); + const f1 = nsFrame(LOCAL_NAMESPACE, 1); agg.ingest(LOCAL_NAMESPACE, f1); agg.reopenArm(LOCAL_NAMESPACE); // the common #1601 local re-open in solo mode @@ -849,9 +858,6 @@ describe("#1638 — resume() no-op WHEN ALREADY FLOWING (not when the sink is fu }, }; } - function nsFrame(ns: string, n: number): string { - return frame("event: message", `data: {"ns":"${ns}","n":${n}}`, `id: ${n}`); - } it("resume() while ALREADY FLOWING is a no-op: it does not re-flush or duplicate already-delivered frames", () => { const sink = controllableSink(); @@ -872,8 +878,8 @@ describe("#1638 — resume() no-op WHEN ALREADY FLOWING (not when the sink is fu const sink = controllableSink(); const agg = new SseFanInAggregator({ sink }); agg.connect(); // fleet-of-one verbatim path - const f1 = frame("event: message", 'data: {"big":"tool-output"}', "id: 7"); - const f2 = frame("event: session.status", 'data: {"status":"idle"}', "id: 8"); // the rail-clearing idle frame + const f1 = sequencedEvent(LOCAL_EPOCH, 7, { big: "tool-output" }); + const f2 = sequencedEvent(LOCAL_EPOCH, 8, { status: "idle" }, "session.status"); // the rail-clearing idle frame sink.block(); agg.ingest(LOCAL_NAMESPACE, f1); // trip-frame, flowing→false agg.ingest(LOCAL_NAMESPACE, f2); // stranded in the verbatim buffer @@ -906,9 +912,6 @@ describe("#1638 — control/gap/focus writes are return-checked and drain in tru }, }; } - function nsFrame(ns: string, n: number): string { - return frame("event: message", `data: {"ns":"${ns}","n":${n}}`, `id: ${n}`); - } it("a control frame (gap) written DURING backpressure emerges AFTER the data frames that preceded it", () => { const sink = controllableSink(); diff --git a/packages/extension/test/sse_fanin_driver_pump.test.ts b/packages/extension/test/sse_fanin_driver_pump.test.ts index ce649bcb0..ec1d3f358 100644 --- a/packages/extension/test/sse_fanin_driver_pump.test.ts +++ b/packages/extension/test/sse_fanin_driver_pump.test.ts @@ -15,6 +15,8 @@ import { startEngineSseServer, writeChunks, largeSummaryFrame, + sequencedEvent, + engineId, chunkBytes, CHUNK_64K, TEN_MB, @@ -24,9 +26,15 @@ import { // ── helpers ────────────────────────────────────────────────────────────────── -function sseFrame(...lines: string[]): string { - return lines.join("\n") + "\n\n"; -} +/** #1771: ENGINE-SHAPED upstream frames (test/fixtures/engine_stream) — the + * wire the engine's `/global/event` actually writes: a sequenced (#1776) bus + * event `id: :` + one `data:` line of the serialized global event, + * no `event:` line. These replace the synthetic `event: message` / `id: l1` + * frames the suite used to feed (a wire production never sees). */ +const localEvt = (seq: number, properties: Record, type?: string) => sequencedEvent(LOCAL_EPOCH, seq, properties, type); +const peerEvt = (seq: number, properties: Record, type?: string) => sequencedEvent(PEER_EPOCH, seq, properties, type); +const localId = (seq: number) => engineId(LOCAL_EPOCH, seq); +const peerId = (seq: number) => engineId(PEER_EPOCH, seq); /** A controllable frame source: push frames into it, end it on demand. */ interface Ctl { @@ -227,8 +235,8 @@ describe("#1566 — dead arm zombie: pump exit cleans up source so reconcile re- { id: "ses-studio", amicode_owner: { owner_machine_id: "studio", owner_name: "studio", is_local: false } }, ]); - const F1 = sseFrame("event: message", 'data: {"from":"studio","n":1}', "id: p1"); - const F2 = sseFrame("event: message", 'data: {"from":"studio","n":2}', "id: p2"); + const F1 = peerEvt(1, { from: "studio", n: 1 }); + const F2 = peerEvt(2, { from: "studio", n: 2 }); // Track every openUpstream call and the controllable sources they return. const openCalls: string[] = []; @@ -370,8 +378,8 @@ describe("#1601 — dead LOCAL arm: reconcile re-opens the local arm", () => { { id: "ses-studio", amicode_owner: { owner_machine_id: "studio", owner_name: "studio", is_local: false } }, ]); - const L1 = sseFrame("event: message", 'data: {"from":"macbook","n":1}', "id: l1"); - const L2 = sseFrame("event: message", 'data: {"from":"macbook","n":2}', "id: l2"); + const L1 = localEvt(1, { from: "macbook", n: 1 }); + const L2 = localEvt(2, { from: "macbook", n: 2 }); // Track every openUpstream call; hand out a FRESH local source per open so we // can distinguish the re-opened arm from the original. @@ -510,8 +518,8 @@ describe("#1617 — driver wires drain→resume + upstream pause/resume", () => { id: "ses-studio", amicode_owner: { owner_machine_id: "studio", owner_name: "studio", is_local: false } }, ]) - const F1 = sseFrame("event: message", 'data: {"from":"macbook","n":1}', "id: l1") - const F2 = sseFrame("event: message", 'data: {"from":"macbook","n":2}', "id: l2") // the stranded idle-equivalent + const F1 = localEvt(1, { from: "macbook", n: 1 }) + const F2 = localEvt(2, { from: "macbook", n: 2 }) // the stranded idle-equivalent const localCtl = pausableSource() const peerCtl = pausableSource() @@ -568,8 +576,8 @@ describe("#1617 — driver wires drain→resume + upstream pause/resume", () => // the solo-machine wedge the composite test above does NOT exercise. const ownerMap = new SessionOwnerMap() - const F1 = sseFrame("event: message", 'data: {"big":"tool-output"}', "id: 7") - const F2 = sseFrame("event: session.status", 'data: {"status":"idle"}', "id: 8") // clears the rail + const F1 = localEvt(7, { big: "tool-output" }) + const F2 = localEvt(8, { status: "idle" }, "session.status") // clears the rail const localCtl = pausableSource() @@ -606,7 +614,7 @@ describe("#1617 — driver wires drain→resume + upstream pause/resume", () => expect(res.text()).toContain('"status":"idle"') expect(res.text()).toBe(F1 + F2) expect(res.text()).not.toContain("local\u001f") - expect(res.text()).not.toContain("local=8") + expect(res.text()).not.toContain(`local=${localId(8)}`) expect(localCtl.resumeCalls).toBeGreaterThan(0) localCtl.end() @@ -630,8 +638,8 @@ describe("#1638 — re-open resumes from last-delivered id + composite-mode reco { id: "ses-studio", amicode_owner: { owner_machine_id: "studio", owner_name: "studio", is_local: false } }, ]) - const L1 = sseFrame("event: message", 'data: {"from":"macbook","n":1}', "id: l1") - const L2 = sseFrame("event: message", 'data: {"from":"macbook","n":2}', "id: l2") + const L1 = localEvt(1, { from: "macbook", n: 1 }) + const L2 = localEvt(2, { from: "macbook", n: 2 }) const opened: Array<{ namespace: string; lastEventId?: string }> = [] const localCtls: Ctl[] = [] @@ -659,12 +667,12 @@ describe("#1638 — re-open resumes from last-delivered id + composite-mode reco }) const res = mockRes() - driver.handle(mockReq("/event?lastEventID=local=l0;studio=p0"), res) + driver.handle(mockReq(`/event?lastEventID=local=${localId(0)};studio=${peerId(0)}`), res) await tick() // initial local open used the connect cursor l0 const initialLocalOpen = opened.find((o) => o.namespace === LOCAL_NAMESPACE) - expect(initialLocalOpen?.lastEventId).toBe("l0") + expect(initialLocalOpen?.lastEventId).toBe(localId(0)) // deliver two local frames — the aggregator's live local cursor advances to l2 localCtls[0].push(L1) @@ -683,7 +691,7 @@ describe("#1638 — re-open resumes from last-delivered id + composite-mode reco const reopenLocal = opened.filter((o) => o.namespace === LOCAL_NAMESPACE) expect(reopenLocal.length).toBe(2) - expect(reopenLocal[1].lastEventId).toBe("l2") // <── last-delivered, not l0 + expect(reopenLocal[1].lastEventId).toBe(localId(2)) // <── last-delivered, not the connect-time id localCtls.forEach((c) => c.end()) peerCtl.end() @@ -696,7 +704,7 @@ describe("#1638 — re-open resumes from last-delivered id + composite-mode reco { id: "ses-studio", amicode_owner: { owner_machine_id: "studio", owner_name: "studio", is_local: false } }, ]) - const P1 = sseFrame("event: message", 'data: {"from":"studio","n":1}', "id: p1") + const P1 = peerEvt(1, { from: "studio", n: 1 }) const localCtl = controllableSource() const peerCtls: Ctl[] = [] @@ -751,8 +759,8 @@ describe("#1638 — re-open resumes from last-delivered id + composite-mode reco it("FLEET-OF-ONE (zero peers): a re-opened LOCAL arm emits NO gap and stays byte-identical (#1264)", async () => { const ownerMap = new SessionOwnerMap() // no owners → fleet-of-one - const L1 = sseFrame("event: message", 'data: {"from":"macbook","n":1}', "id: l1") - const L2 = sseFrame("event: message", 'data: {"from":"macbook","n":2}', "id: l2") + const L1 = localEvt(1, { from: "macbook", n: 1 }) + const L2 = localEvt(2, { from: "macbook", n: 2 }) const localCtls: Ctl[] = [] @@ -797,7 +805,7 @@ describe("#1638 — re-open resumes from last-delivered id + composite-mode reco it("a re-opened LOCAL arm in fleet-of-one resumes from the last-delivered id too", async () => { const ownerMap = new SessionOwnerMap() - const L1 = sseFrame("event: message", 'data: {"from":"macbook","n":1}', "id: l1") + const L1 = localEvt(1, { from: "macbook", n: 1 }) const opened: Array<{ namespace: string; lastEventId?: string }> = [] const localCtls: Ctl[] = [] @@ -818,9 +826,9 @@ describe("#1638 — re-open resumes from last-delivered id + composite-mode reco }) const res = mockRes() - driver.handle(mockReq("/event?lastEventID=l0"), res) + driver.handle(mockReq(`/event?lastEventID=${localId(0)}`), res) await tick() - expect(opened[0].lastEventId).toBe("l0") // initial: connect cursor + expect(opened[0].lastEventId).toBe(localId(0)) // initial: connect cursor localCtls[0].push(L1) await tick() @@ -830,7 +838,7 @@ describe("#1638 — re-open resumes from last-delivered id + composite-mode reco await tick() const localOpens = opened.filter((o) => o.namespace === LOCAL_NAMESPACE) - expect(localOpens[1].lastEventId).toBe("l1") // re-open resumes from last-delivered + expect(localOpens[1].lastEventId).toBe(localId(1)) // re-open resumes from last-delivered localCtls.forEach((c) => c.end()) res.fireClose() @@ -919,8 +927,8 @@ describe("#1772 AC1 — two concurrent listeners both stay connected and both re expect(driver.handle(mockReq("/global/event"), resB)).toBe(true); await tick(); - origins.broadcast(LOCAL_NAMESPACE, sseFrame("event: message", 'data: {"from":"macbook","n":1}', "id: l1")); - origins.broadcast("studio", sseFrame("event: message", 'data: {"from":"studio","n":2}', "id: p1")); + origins.broadcast(LOCAL_NAMESPACE, localEvt(1, { from: "macbook", n: 1 })); + origins.broadcast("studio", peerEvt(1, { from: "studio", n: 2 })); await tick(); for (const res of [resA, resB]) { @@ -951,7 +959,7 @@ describe("#1772 AC2 — closing one listener closes ONLY that listener's upstrea expect(origins.live(LOCAL_NAMESPACE)).toBe(2); expect(origins.live("studio")).toBe(2); - origins.broadcast(LOCAL_NAMESPACE, sseFrame("event: message", 'data: {"n":"before"}', "id: l1")); + origins.broadcast(LOCAL_NAMESPACE, localEvt(1, { n: "before" })); await tick(); // Listener A departs (its socket closes). @@ -967,8 +975,8 @@ describe("#1772 AC2 — closing one listener closes ONLY that listener's upstrea // B keeps streaming on the SAME arms (no re-open), from both origins. driver.reconcile(); await tick(); - origins.broadcast(LOCAL_NAMESPACE, sseFrame("event: message", 'data: {"n":"after-local"}', "id: l2")); - origins.broadcast("studio", sseFrame("event: message", 'data: {"n":"after-peer"}', "id: p2")); + origins.broadcast(LOCAL_NAMESPACE, localEvt(2, { n: "after-local" })); + origins.broadcast("studio", peerEvt(2, { n: "after-peer" })); await tick(); expect(resB.ended).toBe(false); expect(resB.text()).toContain('"n":"before"'); @@ -1011,7 +1019,7 @@ describe("#1772 AC3 — reconcile updates EVERY live connection's membership", ( driver.reconcile(); await tick(); expect(origins.live("studio")).toBe(2); // one peer arm per listener - origins.broadcast("studio", sseFrame("event: message", 'data: {"from":"studio"}', "id: p1")); + origins.broadcast("studio", peerEvt(1, { from: "studio" })); await tick(); expect(resA.text()).toContain('"from":"studio"'); expect(resB.text()).toContain('"from":"studio"'); @@ -1039,7 +1047,7 @@ describe("#1772 AC3 — reconcile updates EVERY live connection's membership", ( ownerMap.update(STUDIO_OWNED); expect(await until(() => origins.live("studio") === 2)).toBe(true); - origins.broadcast("studio", sseFrame("event: message", 'data: {"from":"studio"}', "id: p1")); + origins.broadcast("studio", peerEvt(1, { from: "studio" })); await tick(); expect(resA.text()).toContain('"from":"studio"'); expect(resB.text()).toContain('"from":"studio"'); @@ -1089,7 +1097,7 @@ describe("#1772 AC4 — a listener beyond the cap is refused (retryable), never expect(origins.opens(LOCAL_NAMESPACE)).toBe(16); expect(origins.closes(LOCAL_NAMESPACE)).toBe(0); expect(origins.live("studio")).toBe(16); - origins.broadcast(LOCAL_NAMESPACE, sseFrame("event: message", 'data: {"n":"after-refusal"}', "id: l1")); + origins.broadcast(LOCAL_NAMESPACE, localEvt(1, { n: "after-refusal" })); await tick(); for (const res of accepted) { expect(res.ended).toBe(false); @@ -1168,7 +1176,7 @@ describe("#1772 AC5 — two listeners: one local arm each, kept open — no repe expect(origins.closes("studio")).toBe(0); // Both views are live on those same arms. - origins.broadcast(LOCAL_NAMESPACE, sseFrame("event: message", 'data: {"n":"steady"}', "id: l1")); + origins.broadcast(LOCAL_NAMESPACE, localEvt(1, { n: "steady" })); await tick(); expect(viewA.current().text()).toContain('"n":"steady"'); expect(viewB.current().text()).toContain('"n":"steady"'); @@ -1189,13 +1197,22 @@ describe("#1772 invariants — #1264 / #1601 / #1617 hold PER LISTENER", () => { driver.handle(mockReq("/global/event"), resB); await tick(); - const F1 = sseFrame("event: message", 'data: {"a":1}', "id: 1"); - const F2 = sseFrame("event: message", 'data: {"b":2}', "id: 2"); - origins.broadcast(LOCAL_NAMESPACE, F1); - origins.broadcast(LOCAL_NAMESPACE, F2); - await tick(); - expect(resA.text()).toBe(F1 + F2); - expect(resB.text()).toBe(F1 + F2); + // #1771: the engine's real stream — an old (id-less) engine and a #1776 + // (sequenced) one — connected, an event, the plain + sync twin of a stored + // event, a heartbeat. Every byte relays verbatim to every listener. + for (const engine of [fakeEngine({ legacy: true }), fakeEngine({ epoch: LOCAL_EPOCH })]) { + const frames = [ + engine.connected(), + engine.event("session.status", { a: 1 }), + ...engine.stored("message.updated", { b: 2 }, { aggregateID: "ses_1" }), + engine.heartbeat(), + ]; + const before = [resA.text(), resB.text()]; + for (const f of frames) origins.broadcast(LOCAL_NAMESPACE, f); + await tick(); + expect(resA.text()).toBe(before[0] + frames.join("")); + expect(resB.text()).toBe(before[1] + frames.join("")); + } resA.fireClose(); resB.fireClose(); @@ -1225,8 +1242,8 @@ describe("#1772 invariants — #1264 / #1601 / #1617 hold PER LISTENER", () => { await tick(); expect(localCtls.length).toBe(3); // exactly one re-open — A's - localCtls[2].push(sseFrame("event: message", 'data: {"to":"A"}', "id: a1")); - localCtls[1].push(sseFrame("event: message", 'data: {"to":"B"}', "id: b1")); + localCtls[2].push(localEvt(2, { to: "A" })); + localCtls[1].push(localEvt(1, { to: "B" })); await tick(); expect(resA.text()).toContain('"to":"A"'); expect(resA.text()).not.toContain('"to":"B"'); @@ -1258,16 +1275,16 @@ describe("#1772 invariants — #1264 / #1601 / #1617 hold PER LISTENER", () => { const [aLocal, aPeer, bLocal, bPeer] = ctls; resA.block(); - aLocal.push(sseFrame("event: message", 'data: {"n":1}', "id: l1")); // trips A's backpressure + aLocal.push(localEvt(1, { n: 1 })); // trips A's backpressure await tick(); - aLocal.push(sseFrame("event: message", 'data: {"n":2}', "id: l2")); // stranded behind it + aLocal.push(localEvt(2, { n: 2 })); // stranded behind it await tick(); expect(aLocal.pauseCalls).toBeGreaterThan(0); expect(aPeer.pauseCalls).toBeGreaterThan(0); expect(bLocal.pauseCalls).toBe(0); // B is not throttled by A's slow socket expect(bPeer.pauseCalls).toBe(0); - bLocal.push(sseFrame("event: message", 'data: {"to":"B"}', "id: b1")); + bLocal.push(localEvt(1, { to: "B" })); await tick(); expect(resB.text()).toContain('"to":"B"'); // B streams on while A is backpressured expect(resA.text()).not.toContain('"n":2'); From 6fdcbb77b4d5731bf83a68b5b887620bbc9faae3 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 18:11:06 -0400 Subject: [PATCH 25/60] feat(engine): throttle shell output updates to <= 1 per 200 ms with a guaranteed final flush (#1778) --- packages/app-bundle/manifest.json | 14 +- .../src/tool/shell-output-throttle.ts | 63 ++ .../packages/opencode/src/tool/shell.ts | 646 ++++++++++++++++++ .../test/tool/shell-output-throttle.test.ts | 192 ++++++ 4 files changed, 911 insertions(+), 4 deletions(-) create mode 100644 packages/app-bundle/overlay/packages/opencode/src/tool/shell-output-throttle.ts create mode 100644 packages/app-bundle/overlay/packages/opencode/src/tool/shell.ts create mode 100644 packages/app-bundle/overlay/packages/opencode/test/tool/shell-output-throttle.test.ts diff --git a/packages/app-bundle/manifest.json b/packages/app-bundle/manifest.json index 60a61ff6a..bae1cfad4 100644 --- a/packages/app-bundle/manifest.json +++ b/packages/app-bundle/manifest.json @@ -359,6 +359,8 @@ "packages/opencode/src/tool/edit.ts": "38aa4ce0751ff99bbb0807bcda52f8b9a1d32b6e342df4cc2aedab1f03d342b1", "packages/opencode/src/tool/question.txt": "49bbecb33a825620c2ed14ec9b61faa294359c3673c33277e32f57c1070d2806", "packages/opencode/src/tool/registry.ts": "0097ab8a7904559d0354d165245caff6bf0e887c99ce267be77fa4ee4b66ec3c", + "packages/opencode/src/tool/shell-output-throttle.ts": "00933a5c28bf4f3026541b1ca226678778177146b5b0b56bb5f189d837be9067", + "packages/opencode/src/tool/shell.ts": "09808bb3865574853d162bfd7e1492a618c230828b95ec3989e9e0afa4acb97c", "packages/opencode/src/tool/task.ts": "ddeb07a1cf009dcafc1d1d1c86b9d507676a8991db33a4632c8f29ae0e70e654", "packages/opencode/src/tool/truncate.ts": "92c59d83c4a660116d6ecc48e900aeb16bde93962bfae9f520193fda18273379", "packages/opencode/src/tool/widget-author.ts": "912be8636b162adca8796a501e927019c2449393c14bb5f3ff478b02083b73a9", @@ -428,6 +430,7 @@ "packages/opencode/test/tool/edit.test.ts": "ec1ed35002b1af4e2ff508aeac456c3edbb2e1dde3f6df4abd957eda9ef6d056", "packages/opencode/test/tool/question.test.ts": "75760ba6ed98f2fdfd75945c3b492342dfe91026f99cde86db19cf52444260c2", "packages/opencode/test/tool/registry-schema.test.ts": "053cb9732299ea6554db007bad873d8d61cd3769aa9514d66c70c3bf0d405b90", + "packages/opencode/test/tool/shell-output-throttle.test.ts": "9fed6f4ea8f7c62941280ab4f973848ea4184274830a153fbea83aae2c028576", "packages/opencode/test/tool/task.test.ts": "fb0276cc33add808c669a4f43649d189d8f62c75974f3092a5f05e45cf176486", "packages/opencode/test/tool/truncation.test.ts": "582ef7199b3f294b1c1588fadda8a1c0ea43ca9af0424a97f48747ade1654ad1", "packages/opencode/test/tool/write.test.ts": "820d36fde2314abbd939ce43ca5bcc4cfa0139d12a4c669716249060d9a955ff", @@ -956,7 +959,7 @@ "packages/ui/src/v2/components/text-shimmer-v2.tsx" ], "exceptions": [], - "extracted_at": "2026-10-12T22:06:29.073Z", + "extracted_at": "2026-10-12T22:11:06.337Z", "per_package": { "packages/app": { "A": 237, @@ -974,8 +977,8 @@ "D": 0 }, "packages/opencode": { - "A": 80, - "M": 92, + "A": 82, + "M": 93, "D": 0 }, "packages/schema": { @@ -1005,7 +1008,7 @@ } }, "counts": { - "overlay_total": 934, + "overlay_total": 937, "deletions": 7, "server_coupled": 0 }, @@ -1361,6 +1364,8 @@ "packages/opencode/src/tool/edit.ts": "M", "packages/opencode/src/tool/question.txt": "M", "packages/opencode/src/tool/registry.ts": "M", + "packages/opencode/src/tool/shell-output-throttle.ts": "A", + "packages/opencode/src/tool/shell.ts": "M", "packages/opencode/src/tool/task.ts": "M", "packages/opencode/src/tool/truncate.ts": "M", "packages/opencode/src/tool/widget-author.ts": "A", @@ -1430,6 +1435,7 @@ "packages/opencode/test/tool/edit.test.ts": "M", "packages/opencode/test/tool/question.test.ts": "M", "packages/opencode/test/tool/registry-schema.test.ts": "A", + "packages/opencode/test/tool/shell-output-throttle.test.ts": "A", "packages/opencode/test/tool/task.test.ts": "M", "packages/opencode/test/tool/truncation.test.ts": "M", "packages/opencode/test/tool/write.test.ts": "M", diff --git a/packages/app-bundle/overlay/packages/opencode/src/tool/shell-output-throttle.ts b/packages/app-bundle/overlay/packages/opencode/src/tool/shell-output-throttle.ts new file mode 100644 index 000000000..69cd1c6fe --- /dev/null +++ b/packages/app-bundle/overlay/packages/opencode/src/tool/shell-output-throttle.ts @@ -0,0 +1,63 @@ +// #1778 — throttle a running tool's output preview. Each preview update is a +// `message.part.updated` on the live stream carrying up to 30 KB of output, and +// a chatty command produces one per output chunk. The first update goes out at +// once; after that at most one per interval, a trailing update carrying the +// latest value; `flush` delivers whatever is still unsent (on completion). +// Time is effect's Clock, so TestClock drives it deterministically. +import { Clock, Duration, Effect, Fiber } from "effect" +import type * as Scope from "effect/Scope" + +export const OUTPUT_THROTTLE_MS = 200 + +export const makeOutputThrottle = Effect.fnUntraced(function* ( + send: (output: string) => Effect.Effect, + interval: Duration.Input = Duration.millis(OUTPUT_THROTTLE_MS), +) { + const scope: Scope.Scope = yield* Effect.scope + const every = Duration.toMillis(Duration.fromInputUnsafe(interval)) + let latest: string | undefined + let sent: string | undefined + let sentAt = Number.NEGATIVE_INFINITY + let trailing: Fiber.Fiber | undefined + + // Sends the latest value unless it already went out. + const deliver = Effect.suspend(() => { + if (latest === undefined || latest === sent) return Effect.void + const value = latest + sent = value + return Clock.currentTimeMillis.pipe( + Effect.flatMap((now) => { + sentAt = now + return send(value) + }), + ) + }) + + /** A new preview value: sent now if the interval has passed, else by the trailing update. */ + const offer = (value: string) => + Effect.gen(function* () { + latest = value + if (trailing) return + const wait = sentAt + every - (yield* Clock.currentTimeMillis) + if (wait <= 0) return yield* deliver + trailing = yield* Effect.sleep(Duration.millis(wait)).pipe( + Effect.andThen( + Effect.suspend(() => { + trailing = undefined + return deliver + }), + ), + Effect.forkIn(scope), + ) + }) + + /** Deliver anything unsent now (the guaranteed final update) and drop the pending trailing one. */ + const flush = Effect.gen(function* () { + const pending = trailing + trailing = undefined + if (pending) yield* Fiber.interrupt(pending) + yield* deliver + }) + + return { offer, flush } +}) diff --git a/packages/app-bundle/overlay/packages/opencode/src/tool/shell.ts b/packages/app-bundle/overlay/packages/opencode/src/tool/shell.ts new file mode 100644 index 000000000..4d80b0929 --- /dev/null +++ b/packages/app-bundle/overlay/packages/opencode/src/tool/shell.ts @@ -0,0 +1,646 @@ +import { Effect, Fiber, Stream } from "effect" +import os from "os" +import { createWriteStream } from "node:fs" +import * as Tool from "./tool" +import path from "path" +import { containsPath, type InstanceContext } from "../project/instance-context" +import { InstanceState } from "@/effect/instance-state" +import { lazy } from "@/util/lazy" +import { Language, type Node } from "web-tree-sitter" + +import { FSUtil } from "@opencode-ai/core/fs-util" +import { fileURLToPath } from "url" +import { Config } from "@/config/config" +import { RuntimeFlags } from "@/effect/runtime-flags" +import { Shell } from "@opencode-ai/core/shell" +import { ShellID } from "./shell/id" + +import * as Truncate from "./truncate" +import { Plugin } from "@/plugin" +import { ChildProcess } from "effect/unstable/process" +import { ChildProcessSpawner } from "effect/unstable/process/ChildProcessSpawner" +import { ShellPrompt, type Parameters } from "./shell/prompt" +import { makeOutputThrottle } from "./shell-output-throttle" +import { BashArity } from "@/permission/arity" + +export { Parameters } from "./shell/prompt" + +const MAX_METADATA_LENGTH = 30_000 +const CWD = new Set(["cd", "chdir", "popd", "pushd", "push-location", "set-location"]) +const FILES = new Set([ + ...CWD, + "rm", + "cp", + "mv", + "mkdir", + "touch", + "chmod", + "chown", + "cat", + // Leave PowerShell aliases out for now. Common ones like cat/cp/mv/rm/mkdir + // already hit the entries above, and alias normalization should happen in one + // place later so we do not risk double-prompting. + "get-content", + "set-content", + "add-content", + "copy-item", + "move-item", + "remove-item", + "new-item", + "rename-item", +]) +const CMD_FILES = new Set([ + "copy", + "del", + "dir", + "erase", + "md", + "mkdir", + "move", + "rd", + "ren", + "rename", + "rmdir", + "type", +]) +const FLAGS = new Set(["-destination", "-literalpath", "-path"]) +const SWITCHES = new Set(["-confirm", "-debug", "-force", "-nonewline", "-recurse", "-verbose", "-whatif"]) + +type Part = { + type: string + text: string +} + +type Scan = { + dirs: Set + patterns: Set + always: Set +} + +type Chunk = { + text: string + size: number +} + +const resolveWasm = (asset: string) => { + if (asset.startsWith("file://")) return fileURLToPath(asset) + if (asset.startsWith("/") || /^[a-z]:/i.test(asset)) return asset + const url = new URL(asset, import.meta.url) + return fileURLToPath(url) +} + +function parts(node: Node) { + const out: Part[] = [] + for (let i = 0; i < node.childCount; i++) { + const child = node.child(i) + if (!child) continue + if (child.type === "command_elements") { + for (let j = 0; j < child.childCount; j++) { + const item = child.child(j) + if (!item || item.type === "command_argument_sep" || item.type === "redirection") continue + out.push({ type: item.type, text: item.text }) + } + continue + } + if ( + child.type !== "command_name" && + child.type !== "command_name_expr" && + child.type !== "word" && + child.type !== "string" && + child.type !== "raw_string" && + child.type !== "concatenation" + ) { + continue + } + out.push({ type: child.type, text: child.text }) + } + return out +} + +function source(node: Node) { + return (node.parent?.type === "redirected_statement" ? node.parent.text : node.text).trim() +} + +function commands(node: Node) { + return node.descendantsOfType("command").filter((child): child is Node => Boolean(child)) +} + +function unquote(text: string) { + if (text.length < 2) return text + const first = text[0] + const last = text[text.length - 1] + if ((first === '"' || first === "'") && first === last) return text.slice(1, -1) + return text +} + +function home(text: string) { + if (text === "~") return os.homedir() + if (text.startsWith("~/") || text.startsWith("~\\")) return path.join(os.homedir(), text.slice(2)) + return text +} + +function envValue(key: string) { + if (process.platform !== "win32") return process.env[key] + const name = Object.keys(process.env).find((item) => item.toLowerCase() === key.toLowerCase()) + return name ? process.env[name] : undefined +} + +function auto(key: string, cwd: string, shell: string) { + const name = key.toUpperCase() + if (name === "HOME") return os.homedir() + if (name === "PWD") return cwd + if (name === "PSHOME") return path.dirname(shell) +} + +function expand(text: string, cwd: string, shell: string) { + const out = unquote(text) + .replace(/\$\{env:([^}]+)\}/gi, (_, key: string) => envValue(key) || "") + .replace(/\$env:([A-Za-z_][A-Za-z0-9_]*)/gi, (_, key: string) => envValue(key) || "") + .replace(/\$(HOME|PWD|PSHOME)(?=$|[\\/])/gi, (_, key: string) => auto(key, cwd, shell) || "") + return home(out) +} + +function provider(text: string) { + const match = text.match(/^([A-Za-z]+)::(.*)$/) + if (match) { + if (match[1].toLowerCase() !== "filesystem") return + return match[2] + } + const prefix = text.match(/^([A-Za-z]+):(.*)$/) + if (!prefix) return text + if (prefix[1].length === 1) return text + return +} + +function dynamic(text: string, ps: boolean) { + if (text.startsWith("(") || text.startsWith("@(")) return true + if (text.includes("$(") || text.includes("${") || text.includes("`")) return true + if (ps) return /\$(?!env:)/i.test(text) + return text.includes("$") +} + +function prefix(text: string) { + const match = /[?*[]/.exec(text) + if (!match) return text + if (match.index === 0) return + return text.slice(0, match.index) +} + +function pathArgs(list: Part[], ps: boolean, cmd = false) { + if (!ps) { + return list + .slice(1) + .filter( + (item) => + !item.text.startsWith("-") && + !(cmd && item.text.startsWith("/")) && + !(list[0]?.text === "chmod" && item.text.startsWith("+")), + ) + .map((item) => item.text) + } + + const out: string[] = [] + let want = false + for (const item of list.slice(1)) { + if (want) { + out.push(item.text) + want = false + continue + } + if (item.type === "command_parameter") { + const flag = item.text.toLowerCase() + if (SWITCHES.has(flag)) continue + want = FLAGS.has(flag) + continue + } + out.push(item.text) + } + return out +} + +function preview(text: string) { + if (text.length <= MAX_METADATA_LENGTH) return text + return "...\n\n" + text.slice(-MAX_METADATA_LENGTH) +} + +function tail(text: string, maxLines: number, maxBytes: number) { + const lines = text.split("\n") + if (lines.length <= maxLines && Buffer.byteLength(text, "utf-8") <= maxBytes) { + return { + text, + cut: false, + } + } + + const out: string[] = [] + let bytes = 0 + for (let i = lines.length - 1; i >= 0 && out.length < maxLines; i--) { + const size = Buffer.byteLength(lines[i], "utf-8") + (out.length > 0 ? 1 : 0) + if (bytes + size > maxBytes) { + if (out.length === 0) { + const buf = Buffer.from(lines[i], "utf-8") + let start = buf.length - maxBytes + if (start < 0) start = 0 + while (start < buf.length && (buf[start] & 0xc0) === 0x80) start++ + out.unshift(buf.subarray(start).toString("utf-8")) + } + break + } + out.unshift(lines[i]) + bytes += size + } + return { + text: out.join("\n"), + cut: true, + } +} + +const parse = Effect.fn("ShellTool.parse")(function* (command: string, ps: boolean) { + const tree = yield* Effect.promise(() => parser().then((p) => (ps ? p.ps : p.bash).parse(command))) + if (!tree) throw new Error("Failed to parse command") + return tree +}) + +const ask = Effect.fn("ShellTool.ask")(function* (ctx: Tool.Context, scan: Scan, input: { command: string }) { + if (scan.dirs.size > 0) { + const directories = Array.from(scan.dirs) + const globs = directories.map((dir) => { + if (process.platform === "win32") return FSUtil.normalizePathPattern(path.join(dir, "*")) + return path.join(dir, "*") + }) + yield* ctx.ask({ + permission: "external_directory", + patterns: globs, + always: globs, + metadata: { + command: input.command, + directories, + patterns: globs, + }, + }) + } + + if (scan.patterns.size === 0) return + yield* ctx.ask({ + permission: ShellID.ToolID, + patterns: Array.from(scan.patterns), + always: Array.from(scan.always), + metadata: { + command: input.command, + }, + }) +}) + +function cmd(shell: string, command: string, cwd: string, env: NodeJS.ProcessEnv) { + if (process.platform === "win32" && Shell.ps(shell)) { + return ChildProcess.make(shell, ["-NoLogo", "-NoProfile", "-NonInteractive", "-Command", command], { + cwd, + env, + stdin: "ignore", + detached: false, + }) + } + + return ChildProcess.make(command, [], { + shell, + cwd, + env, + stdin: "ignore", + detached: process.platform !== "win32", + }) +} +const parser = lazy(async () => { + const { Parser } = await import("web-tree-sitter") + const { default: treeWasm } = await import("web-tree-sitter/tree-sitter.wasm" as string, { + with: { type: "wasm" }, + }) + const treePath = resolveWasm(treeWasm) + await Parser.init({ + locateFile() { + return treePath + }, + }) + const { default: bashWasm } = await import("tree-sitter-bash/tree-sitter-bash.wasm" as string, { + with: { type: "wasm" }, + }) + const { default: psWasm } = await import("tree-sitter-powershell/tree-sitter-powershell.wasm" as string, { + with: { type: "wasm" }, + }) + const bashPath = resolveWasm(bashWasm) + const psPath = resolveWasm(psWasm) + const [bashLanguage, psLanguage] = await Promise.all([Language.load(bashPath), Language.load(psPath)]) + const bash = new Parser() + bash.setLanguage(bashLanguage) + const ps = new Parser() + ps.setLanguage(psLanguage) + return { bash, ps } +}) + +export const ShellTool = Tool.define( + ShellID.ToolID, + Effect.gen(function* () { + const config = yield* Config.Service + const spawner = yield* ChildProcessSpawner + const fs = yield* FSUtil.Service + const trunc = yield* Truncate.Service + const plugin = yield* Plugin.Service + const flags = yield* RuntimeFlags.Service + const defaultTimeoutMs = flags.bashDefaultTimeoutMs ?? 2 * 60 * 1000 + + const cygpath = Effect.fn("ShellTool.cygpath")(function* (shell: string, text: string) { + const lines = yield* spawner + .lines(ChildProcess.make(shell, ["-lc", 'cygpath -w -- "$1"', "_", text])) + .pipe(Effect.catch(() => Effect.succeed([] as string[]))) + const file = lines[0]?.trim() + if (!file) return + return FSUtil.normalizePath(file) + }) + + const resolvePath = Effect.fn("ShellTool.resolvePath")(function* (text: string, root: string, shell: string) { + if (process.platform === "win32") { + if (Shell.posix(shell) && text.startsWith("/") && FSUtil.windowsPath(text) === text) { + const file = yield* cygpath(shell, text) + if (file) return file + } + return FSUtil.normalizePath(path.resolve(root, FSUtil.windowsPath(text))) + } + return path.resolve(root, text) + }) + + const argPath = Effect.fn("ShellTool.argPath")(function* (arg: string, cwd: string, ps: boolean, shell: string) { + const text = ps ? expand(arg, cwd, shell) : home(unquote(arg)) + const file = text && prefix(text) + if (!file || dynamic(file, ps)) return + const next = ps ? provider(file) : file + if (!next) return + return yield* resolvePath(next, cwd, shell) + }) + + const collect = Effect.fn("ShellTool.collect")(function* ( + root: Node, + cwd: string, + ps: boolean, + shell: string, + instance: InstanceContext, + ) { + const scan: Scan = { + dirs: new Set(), + patterns: new Set(), + always: new Set(), + } + const shellKind = ShellID.toKind(Shell.name(shell)) + + for (const node of commands(root)) { + const command = parts(node) + const tokens = command.map((item) => item.text) + const cmd = ps || shellKind === "cmd" ? tokens[0]?.toLowerCase() : tokens[0] + + if (cmd && (FILES.has(cmd) || (shellKind === "cmd" && CMD_FILES.has(cmd)))) { + for (const arg of pathArgs(command, ps, shellKind === "cmd")) { + const resolved = yield* argPath(arg, cwd, ps, shell) + yield* Effect.logInfo("resolved path", { arg, resolved }) + if (!resolved || containsPath(resolved, instance)) continue + const dir = (yield* fs.isDir(resolved)) ? resolved : path.dirname(resolved) + scan.dirs.add(dir) + } + } + + if (tokens.length && (!cmd || !CWD.has(cmd))) { + scan.patterns.add(source(node)) + scan.always.add(BashArity.prefix(tokens).join(" ") + " *") + } + } + + return scan + }) + + const shellEnv = Effect.fn("ShellTool.shellEnv")(function* (ctx: Tool.Context, cwd: string) { + const extra = yield* plugin.trigger( + "shell.env", + { cwd, sessionID: ctx.sessionID, callID: ctx.callID }, + { env: {} }, + ) + return { + ...process.env, + ...extra.env, + } + }) + + const run = Effect.fn("ShellTool.run")(function* ( + input: { + shell: string + command: string + cwd: string + env: NodeJS.ProcessEnv + timeout: number + }, + ctx: Tool.Context, + ) { + const limits = yield* trunc.limits() + const keep = limits.maxBytes * 2 + let full = "" + let last = "" + const list: Chunk[] = [] + let used = 0 + let file = "" + let sink: ReturnType | undefined + let cut = false + let expired = false + let aborted = false + + const closeSink = Effect.fnUntraced(function* () { + const stream = sink + if (!stream) return + sink = undefined + if (stream.destroyed || stream.closed) return + yield* Effect.promise( + () => + new Promise((resolve) => { + let settled = false + const done = () => { + if (settled) return + settled = true + stream.off("close", done) + stream.off("error", done) + stream.off("finish", done) + resolve() + } + stream.once("close", done) + stream.once("error", done) + stream.once("finish", done) + stream.end(done) + }), + ).pipe(Effect.catch(() => Effect.void)) + }) + + yield* ctx.metadata({ + metadata: { + output: "", + }, + }) + + // #1778 (amicode overlay): every preview update is a live-stream frame of + // up to 30 KB, and a chatty command makes one per output chunk. Throttle + // them (first at once, then <= 1 per 200 ms, trailing) and flush the final + // output once the command is done. + const code: number | null = yield* Effect.scoped( + Effect.gen(function* () { + yield* Effect.addFinalizer(closeSink) + const throttle = yield* makeOutputThrottle((output) => ctx.metadata({ metadata: { output } })) + const handle = yield* spawner.spawn(cmd(input.shell, input.command, input.cwd, input.env)) + + const reader = yield* Effect.forkScoped( + Stream.runForEach(Stream.decodeText(handle.all), (chunk) => { + const size = Buffer.byteLength(chunk, "utf-8") + list.push({ text: chunk, size }) + used += size + while (used > keep && list.length > 1) { + const item = list.shift() + if (!item) break + used -= item.size + cut = true + } + + last = preview(last + chunk) + + if (file) { + sink?.write(chunk) + } else { + full += chunk + if (Buffer.byteLength(full, "utf-8") > limits.maxBytes) { + return trunc.write(full).pipe( + Effect.andThen((next) => + Effect.sync(() => { + file = next + cut = true + sink = createWriteStream(next, { flags: "a" }) + full = "" + }), + ), + Effect.andThen(throttle.offer(last)), + ) + } + } + + return throttle.offer(last) + }), + ) + + const abort = Effect.callback((resume) => { + if (ctx.abort.aborted) return resume(Effect.void) + const handler = () => resume(Effect.void) + ctx.abort.addEventListener("abort", handler, { once: true }) + return Effect.sync(() => ctx.abort.removeEventListener("abort", handler)) + }) + + const timeout = Effect.sleep(`${input.timeout + 100} millis`) + + const exit = yield* Effect.raceAll([ + handle.exitCode.pipe(Effect.map((code) => ({ kind: "exit" as const, code }))), + abort.pipe(Effect.map(() => ({ kind: "abort" as const, code: null }))), + timeout.pipe(Effect.map(() => ({ kind: "timeout" as const, code: null }))), + ]) + + if (exit.kind === "abort") { + aborted = true + yield* handle.kill({ forceKillAfter: "3 seconds" }).pipe(Effect.orDie) + } + if (exit.kind === "timeout") { + expired = true + yield* handle.kill({ forceKillAfter: "3 seconds" }).pipe(Effect.orDie) + } + + // Stop the reader (as the scope's close would), then the final output + // always goes out: no update can race behind the flush. + yield* Fiber.interrupt(reader) + yield* throttle.flush + + return exit.kind === "exit" ? exit.code : null + }), + ).pipe(Effect.orDie) + + const meta: string[] = [] + if (expired) { + meta.push( + `shell tool terminated command after exceeding timeout ${input.timeout} ms. If this command is expected to take longer and is not waiting for interactive input, retry with a larger timeout value in milliseconds.`, + ) + } + if (aborted) meta.push("User aborted the command") + const raw = list.map((item) => item.text).join("") + const end = tail(raw, limits.maxLines, limits.maxBytes) + if (end.cut) cut = true + if (!file && end.cut) { + file = yield* trunc.write(raw) + } + + let output = end.text + if (!output) output = "(no output)" + + if (cut && file) { + output = `...output truncated...\n\nFull output saved to: ${file}\n\n` + output + } + + if (meta.length > 0) { + output += "\n\n\n" + meta.join("\n") + "\n" + } + return { + title: input.command, + metadata: { + output: last || preview(output), + exit: code, + truncated: cut, + ...(cut && file ? { outputPath: file } : {}), + }, + output, + } + }) + + return () => + Effect.gen(function* () { + const cfg = yield* config.get() + const shell = Shell.acceptable(cfg.shell) + const name = Shell.name(shell) + const limits = yield* trunc.limits() + const prompt = ShellPrompt.render(name, process.platform, limits, defaultTimeoutMs) + yield* Effect.logInfo("shell tool using shell", { shell }) + + return { + description: prompt.description, + parameters: prompt.parameters, + execute: (params: Parameters, ctx: Tool.Context) => + Effect.gen(function* () { + const instanceCtx = yield* InstanceState.context + const cwd = params.workdir + ? yield* resolvePath(params.workdir, instanceCtx.directory, shell) + : instanceCtx.directory + if (params.timeout !== undefined && params.timeout < 0) { + throw new Error(`Invalid timeout value: ${params.timeout}. Timeout must be a positive number.`) + } + const timeout = params.timeout ?? defaultTimeoutMs + const ps = Shell.ps(shell) + yield* Effect.scoped( + Effect.gen(function* () { + const tree = yield* Effect.acquireRelease(parse(params.command, ps), (tree) => + Effect.sync(() => tree.delete()), + ) + const scan = yield* collect(tree.rootNode, cwd, ps, shell, instanceCtx) + if (!containsPath(cwd, instanceCtx)) scan.dirs.add(cwd) + yield* ask(ctx, scan, params) + }), + ) + + return yield* run( + { + shell, + command: params.command, + cwd, + env: yield* shellEnv(ctx, cwd), + timeout, + }, + ctx, + ) + }), + } + }) + }), +) diff --git a/packages/app-bundle/overlay/packages/opencode/test/tool/shell-output-throttle.test.ts b/packages/app-bundle/overlay/packages/opencode/test/tool/shell-output-throttle.test.ts new file mode 100644 index 000000000..a09e07acc --- /dev/null +++ b/packages/app-bundle/overlay/packages/opencode/test/tool/shell-output-throttle.test.ts @@ -0,0 +1,192 @@ +// #1778 — a running shell tool's output preview is throttled: the first update +// goes out at once, then at most one per 200 ms (a trailing update carries the +// latest output), and the final output is always delivered when the command +// completes. Every update is a `message.part.updated` on the live stream +// carrying up to 30 KB of preview, so an unthrottled chatty command floods it. +// The throttle itself runs on effect's Clock (driven by TestClock here); the +// shell tool is then exercised with real chatty commands. Harness trimmed from +// shell.test.ts (upstream). +import { describe, expect } from "bun:test" +import { LayerNode } from "@opencode-ai/core/effect/layer-node" +import { Clock, Effect, Layer } from "effect" +import * as TestClock from "effect/testing/TestClock" +import path from "path" +import { Config } from "@/config/config" +import { Shell } from "@opencode-ai/core/shell" +import { ShellTool } from "../../src/tool/shell" +import { OUTPUT_THROTTLE_MS, makeOutputThrottle } from "../../src/tool/shell-output-throttle" +import { provideInstance, testInstanceStoreLayer } from "../fixture/fixture" +import { Agent } from "../../src/agent/agent" +import { Truncate } from "@/tool/truncate" +import { SessionID, MessageID } from "../../src/session/schema" +import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner" +import { FSUtil } from "@opencode-ai/core/fs-util" +import { Plugin } from "../../src/plugin" +import { testEffect } from "../lib/effect" +import type { Tool } from "@/tool/tool" +import { RuntimeFlags } from "@/effect/runtime-flags" + +const it = testEffect( + Layer.mergeAll( + LayerNode.compile( + LayerNode.group([ + CrossSpawnSpawner.node, + FSUtil.node, + Plugin.node, + Truncate.node, + Config.node, + Agent.node, + RuntimeFlags.node, + ]), + ), + testInstanceStoreLayer, + ), +) + +Shell.acceptable.reset() +const projectRoot = path.join(__dirname, "../..") +const bin = `"${process.execPath.replaceAll("\\", "/")}"` +const script = (code: string) => { + const shell = Shell.name(Shell.acceptable()) + const text = `${bin} -e ${shell === "cmd" ? `"${code}"` : `'${code}'`}` + return shell === "pwsh" || shell === "powershell" ? `& ${text}` : text +} + +/** Run the shell tool, recording every output preview update it publishes. */ +const runRecording = (command: string) => + Effect.gen(function* () { + const updates: { at: number; output: string }[] = [] + const ctx: Tool.Context = { + sessionID: SessionID.make("ses_i1778"), + messageID: MessageID.make("msg_i1778"), + callID: "", + agent: "build", + abort: AbortSignal.any([]), + messages: [], + metadata: (input) => + Effect.sync(() => { + const output = (input.metadata as { output?: string })?.output + if (output) updates.push({ at: performance.now(), output }) + }), + ask: () => Effect.void, + } + const tool = yield* (yield* ShellTool).init() + const started = performance.now() + const result = yield* tool.execute({ command }, ctx) + return { updates, result, elapsed: performance.now() - started } + }).pipe(provideInstance(projectRoot)) + +describe("shell output throttle (#1778)", () => { + it.effect("the first update goes out at once; updates inside 200 ms coalesce into one trailing update", () => + Effect.gen(function* () { + const sent: { at: number; output: string }[] = [] + const throttle = yield* makeOutputThrottle((output) => + Effect.map(Clock.currentTimeMillis, (at) => { + sent.push({ at, output }) + }), + ) + expect(OUTPUT_THROTTLE_MS).toBe(200) + + yield* throttle.offer("a") + expect(sent).toEqual([{ at: 0, output: "a" }]) + + yield* throttle.offer("ab") + yield* TestClock.adjust("50 millis") + yield* throttle.offer("abc") + yield* TestClock.adjust("149 millis") + expect(sent).toHaveLength(1) + yield* TestClock.adjust("1 millis") + expect(sent).toEqual([ + { at: 0, output: "a" }, + { at: 200, output: "abc" }, + ]) + + // quiet afterwards: nothing new, nothing sent + yield* TestClock.adjust("1 second") + expect(sent).toHaveLength(2) + // after a quiet spell the next update goes out at once again + yield* throttle.offer("abcd") + expect(sent.at(-1)).toEqual({ at: 1200, output: "abcd" }) + }), + ) + + it.effect("a steady stream of output yields at most one update per 200 ms, the latest each time", () => + Effect.gen(function* () { + const sent: { at: number; output: string }[] = [] + const throttle = yield* makeOutputThrottle((output) => + Effect.map(Clock.currentTimeMillis, (at) => { + sent.push({ at, output }) + }), + ) + let output = "" + for (let i = 0; i < 200; i++) { + output += `${i}\n` + yield* throttle.offer(output) + yield* TestClock.adjust("10 millis") + } + expect(sent.length).toBeLessThanOrEqual(1 + 2000 / 200) + expect(sent.length).toBeGreaterThanOrEqual(2000 / 200) + for (let i = 1; i < sent.length; i++) expect(sent[i].at - sent[i - 1].at).toBeGreaterThanOrEqual(200) + expect(sent.map((s) => s.output.length)).toEqual([...sent.map((s) => s.output.length)].sort((a, b) => a - b)) + }), + ) + + it.effect("flush delivers unsent output at once, cancels the pending trailing update, and is idempotent", () => + Effect.gen(function* () { + const sent: { at: number; output: string }[] = [] + const throttle = yield* makeOutputThrottle((output) => + Effect.map(Clock.currentTimeMillis, (at) => { + sent.push({ at, output }) + }), + ) + yield* throttle.offer("a") + yield* throttle.offer("ab") + yield* TestClock.adjust("20 millis") + yield* throttle.flush + expect(sent).toEqual([ + { at: 0, output: "a" }, + { at: 20, output: "ab" }, + ]) + yield* TestClock.adjust("1 second") + yield* throttle.flush + expect(sent).toHaveLength(2) + }), + ) +}) + +describe("tool.shell — throttled output updates (#1778)", () => { + it.live( + "a command printing continuously publishes at most about five output updates per second", + () => + Effect.gen(function* () { + const { updates, result, elapsed } = yield* runRecording( + script( + "for (let i = 0; i < 150; i++) { process.stdout.write(`line-${i}\\n`); await Bun.sleep(6) } process.stdout.write(`line-END\\n`)", + ), + ) + expect(result.metadata.exit).toBe(0) + // first update at once, then <= 1 per 200 ms, plus the final flush on completion + expect(updates.length).toBeLessThanOrEqual(2 + Math.ceil(elapsed / OUTPUT_THROTTLE_MS)) + expect(updates.length).toBeGreaterThan(1) + // the final output is delivered on completion: the last update is the full output + expect(updates.at(-1)!.output).toContain("line-END") + expect(updates.at(-1)!.output).toBe(result.metadata.output) + }), + 20_000, + ) + + it.live( + "output that lands inside the throttle window just before exit is still delivered", + () => + Effect.gen(function* () { + const { updates, result } = yield* runRecording( + script("process.stdout.write(`first\\n`); await Bun.sleep(30); process.stdout.write(`last\\n`)"), + ) + expect(result.metadata.exit).toBe(0) + expect(updates[0].output).toContain("first") + expect(updates.at(-1)!.output).toContain("last") + expect(updates.at(-1)!.output).toBe(result.metadata.output) + }), + 20_000, + ) +}) From 7d40fecf563fb62e23232d617eb4329ebf9bce5d Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 18:11:08 -0400 Subject: [PATCH 26/60] test(fleet): both fan-in boot branches log owner-map refresh outcomes and the census; tidy census line builder (#1771) --- .../src/amicode_service/fanin_census.ts | 20 +++++----- ...service_fleet_session_status_route.test.ts | 37 ++++++++++++++++++- 2 files changed, 46 insertions(+), 11 deletions(-) diff --git a/packages/extension/src/amicode_service/fanin_census.ts b/packages/extension/src/amicode_service/fanin_census.ts index 16046ef47..ecb0fa620 100644 --- a/packages/extension/src/amicode_service/fanin_census.ts +++ b/packages/extension/src/amicode_service/fanin_census.ts @@ -142,37 +142,37 @@ export class FanInCensus { listenerAccepted(conn: CensusConnection): void { this.accepted += 1; this.live.set(conn.id, conn); - this.log(`listener-accepted conn=${conn.id} listeners=${this.live.size}`); + this.log("listener-accepted", `conn=${conn.id} listeners=${this.live.size}`); } listenerClosed(connId: string, reason: FanInCloseReason): void { if (!this.live.delete(connId)) return; this.closed += 1; if (reason === "evicted") this.evictions += 1; - this.log(`listener-closed conn=${connId} reason=${reason} listeners=${this.live.size}`); + this.log("listener-closed", `conn=${connId} reason=${reason} listeners=${this.live.size}`); } listenerRefused(cap: number): void { this.refusals += 1; - this.log(`listener-refused listeners=${this.live.size} cap=${cap}`); + this.log("listener-refused", `listeners=${this.live.size} cap=${cap}`); } armOpened(connId: string, arm: string, opens: number): void { - this.log(`arm-opened conn=${connId} arm=${arm} opens=${opens}`); + this.log("arm-opened", `conn=${connId} arm=${arm} opens=${opens}`); } armClosed(connId: string, arm: string, reason: FanInCloseReason): void { - this.log(`arm-closed conn=${connId} arm=${arm} reason=${reason}`); + this.log("arm-closed", `conn=${connId} arm=${arm} reason=${reason}`); } gapEmitted(connId: string, arms: string[], cause: GapCause): void { - this.log(`gap-emitted conn=${connId} arm=${arms.join(",")} cause=${cause}`); + this.log("gap-emitted", `conn=${connId} arm=${arms.join(",")} cause=${cause}`); } /** The FIRST drop of an overflow episode (the census counts every drop; the * line is per episode — a drop storm never becomes a per-frame log). */ overflowDrop(connId: string, arm: string, pending: number): void { - this.log(`overflow-drop conn=${connId} arm=${arm} pending=${pending}`); + this.log("overflow-drop", `conn=${connId} arm=${arm} pending=${pending}`); } // ── the census ──────────────────────────────────────────────────────────── @@ -225,9 +225,9 @@ export class FanInCensus { this.write(this.line()); } - private log(event: string): void { - const [kind, ...rest] = event.split(" "); - this.write(`[fanin] ${kind} plane=${this.plane} ${rest.join(" ")}`); + /** One transition line: `[fanin] plane= `. */ + private log(kind: string, fields: string): void { + this.write(`[fanin] ${kind} plane=${this.plane} ${fields}`); } private write(line: string): void { diff --git a/packages/extension/test/amicode_service_fleet_session_status_route.test.ts b/packages/extension/test/amicode_service_fleet_session_status_route.test.ts index 9bbb641a8..a4b86851b 100644 --- a/packages/extension/test/amicode_service_fleet_session_status_route.test.ts +++ b/packages/extension/test/amicode_service_fleet_session_status_route.test.ts @@ -11,7 +11,7 @@ // - served: it reads the LIVE owner map of whichever boot branch built one // (observation-only base peer AND the premium staged plane), fanning out to // the owner in the session's exact directory with the owner's own credential. -import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { describe, it, expect, beforeAll, afterAll, vi } from "vitest"; import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -313,6 +313,41 @@ describe("#1766 — GET /amicode/fleet/session-status (route-level)", () => { } }); + // ── #1771: both boot branches log the owner-map refresh + the fan-in census ── + it.each([ + { name: "observation-only base peer", premium: false, plane: "observe" }, + { name: "premium staged plane", premium: true, plane: "premium" }, + ])("#1771 $name: the service log gets each refresh's per-peer outcome and the fan-in census", async ({ premium, plane }) => { + vi.useFakeTimers({ toFake: ["setInterval", "clearInterval"] }); + const lines: string[] = []; + let notify: (() => void) | undefined; + const svc = createAmicodeService({ + password: "service-own-mint", + engine: engine(), + fleet: { + hub: { getUrl: () => undefined }, + fleetPeers: servingPeerProvider(), + ...(premium ? { entitlements: ["amicissimo"], overlaySource } : { observationOnly: true }), + }, + log: (l) => { + lines.push(l); + notify?.(); + }, + }); + try { + // the feed's immediate refresh reaches the peer: ONE line for the peer + const isPeerLine = (l: string) => l.startsWith(`[owner-map] refresh peer=${PEER_ID} `); + if (!lines.some(isPeerLine)) await new Promise((r) => (notify = () => lines.some(isPeerLine) && r())); + expect(lines.find(isPeerLine)).toMatch(new RegExp(`^\\[owner-map\\] refresh peer=${PEER_ID} outcome=ok duration_ms=\\d+ entries=1$`)); + expect(lines.some((l) => l.includes(`peer=${LOCAL_ID} `))).toBe(false); // the local engine is not a peer + vi.advanceTimersByTime(30_000); + expect(lines.filter((l) => l.startsWith(`[fanin] census plane=${plane} listeners=0 `))).toHaveLength(1); + } finally { + await svc.stop(); + vi.useRealTimers(); + } + }); + it("an empty / missing ids parameter is an empty 200 map with no upstream status call", async () => { const svc = createAmicodeService({ password: "service-own-mint", From 725257329771ebebf6a3d1cc37ad7b372d5a3f43 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 18:16:34 -0400 Subject: [PATCH 27/60] =?UTF-8?q?test(engine):=20fixture-turn=20byte=20rep?= =?UTF-8?q?ort=20=E2=80=94=20largest=20live=20frame=20under=20256=20KB,=20?= =?UTF-8?q?total=20stream=20bytes=20before=20and=20after=20(#1778)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/app-bundle/manifest.json | 4 +- .../test/server/httpapi-global-event.test.ts | 159 ++++++++++++++++++ 2 files changed, 161 insertions(+), 2 deletions(-) diff --git a/packages/app-bundle/manifest.json b/packages/app-bundle/manifest.json index bae1cfad4..b233d15b6 100644 --- a/packages/app-bundle/manifest.json +++ b/packages/app-bundle/manifest.json @@ -400,7 +400,7 @@ "packages/opencode/test/server/httpapi-compression.test.ts": "5f7603d7018e0b95f1c2f36c01dd297c01770bfcdd91ce29778e03b7d6143f75", "packages/opencode/test/server/httpapi-file-write-gate.test.ts": "0c91b7417e615850edfde3f2878207dab6b6624d4cfbb6e6783b2e6d9aa4dff0", "packages/opencode/test/server/httpapi-file.test.ts": "d3386129bc81467ecbcc57a3861f53767ee56ac41b4ff6ad58d8b168fb3b3373", - "packages/opencode/test/server/httpapi-global-event.test.ts": "4c02060af1d2d9585f41293f2d62560de9df6b85b542c93ed6e2aa4171c145fc", + "packages/opencode/test/server/httpapi-global-event.test.ts": "9fc2b195d2784ca2f51e75b68f31c0f73aa69afdc7ca2bd704ac581971c20b30", "packages/opencode/test/server/httpapi-instance-route-auth.test.ts": "4b73a89d2431c02bc3b665d93f9002914e51eb1add979dc1867efbcbc0c36e25", "packages/opencode/test/server/httpapi-provider.test.ts": "9e8b3e085b478e3f99b689ffff600e10c0b4b50b3cabdbd749f88dae19bf5904", "packages/opencode/test/server/httpapi-session.test.ts": "eda6fd763ac4c17891cedaa4ccd49597c03e7227fcfdcf0f94a1b4545ba623ed", @@ -959,7 +959,7 @@ "packages/ui/src/v2/components/text-shimmer-v2.tsx" ], "exceptions": [], - "extracted_at": "2026-10-12T22:11:06.337Z", + "extracted_at": "2026-10-12T22:16:34.724Z", "per_package": { "packages/app": { "A": 237, diff --git a/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts b/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts index 3c530f0d6..26001fb5f 100644 --- a/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts +++ b/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts @@ -26,6 +26,7 @@ import { globalHandlers } from "../../src/server/routes/instance/httpapi/handler import { authorizationLayer } from "../../src/server/routes/instance/httpapi/middleware/authorization" import { schemaErrorLayer } from "../../src/server/routes/instance/httpapi/middleware/schema-error" import { testEffect } from "../lib/effect" +import { OUTPUT_THROTTLE_MS, makeOutputThrottle } from "../../src/tool/shell-output-throttle" const apiLayer = HttpRouter.serve( HttpApiBuilder.layer(RootHttpApi).pipe( @@ -499,3 +500,161 @@ describe("global event stream — client abort (#1776)", () => { }), ) }) + +// ── #1778 fixture turn: a multi-file edit plus a chatty shell command ────────── +// +// The same turn, two ways. BEFORE is what #1776's engine put on every +// subscriber's wire: each event serialized whole, each durable event followed by +// its sync copy, one shell preview update per output chunk. AFTER is measured +// on the live wire of a default subscriber of this engine: no sync copies, +// summary patch bodies stripped, shell updates through the real throttle +// (driven by TestClock). Sizes mirror the Studio measurement: one summary of +// 88 files carrying ~2.6 MB of patch text. +const FIXTURE = { files: 88, patchBytes: 30_000, chunks: 300, chunkEveryMs: 10, chunkBytes: 160, deltas: 120 } +const PREVIEW_MAX = 30_000 // the shell tool's MAX_METADATA_LENGTH +const KB = 1024 + +// the engine's SSE framing (the encoder omits the default `event: message` line) +const sseBytes = (id: string, data: string) => Buffer.byteLength(`id: ${id}\ndata: ${data}\n\n`) + +const patchFor = (file: string, bytes: number) => { + const line = `+ export const value_${file.length} = "${"x".repeat(60)}"\n` + return `--- a/${file}\n+++ b/${file}\n@@ -1,0 +1,${Math.ceil(bytes / line.length)} @@\n${line.repeat(Math.ceil(bytes / line.length))}` +} + +describe("global event stream — fixture turn byte report (#1778)", () => { + it.effect("a multi-file edit plus a chatty shell command keeps every live frame under 256 KB", () => + Effect.gen(function* () { + const plain = yield* connect() + yield* plain.next + const before: { type: string; bytes: number; sync: boolean }[] = [] + let liveEvents = 0 + const base = { directory: "/tmp/i1778-fixture", project: "prj_i1778" } + const sessionID = "ses_i1778_fixture" + + // Emit like the event-v2 bridge (the event, then for a durable one its sync + // copy) and record the frame #1776's tap would have serialized for each. + const publish = (type: string, properties: Record, options: { durable?: boolean; record?: boolean } = {}) => { + const id = `evt_i1778_fx_${GlobalBus.seq + 1}` + const event = { ...base, payload: { id, type, properties } } + GlobalBus.emit("event", event) + liveEvents++ + const record = options.record !== false + if (record) before.push({ type, sync: false, bytes: sseBytes(`${GlobalBus.epoch}:${GlobalBus.seq}`, JSON.stringify(event)) }) + if (options.durable === false) return + const copy = { + ...base, + payload: { type: "sync", syncEvent: { id, type: `${type}.1`, seq: before.length, aggregateID: sessionID, data: properties } }, + } + GlobalBus.emit("event", copy) + if (record) before.push({ type, sync: true, bytes: sseBytes(`${GlobalBus.epoch}:${GlobalBus.seq}`, JSON.stringify(copy)) }) + } + const user = { id: "msg_i1778_user", sessionID, role: "user", time: { created: 1 }, agent: "build", model: { providerID: "p", modelID: "m" } } + const assistant = { id: "msg_i1778_asst", sessionID, role: "assistant", parentID: user.id, time: { created: 2 } } + const tool = (callID: string, toolName: string, state: Record) => ({ + sessionID, + part: { id: `prt_${callID}`, sessionID, messageID: assistant.id, type: "tool", callID, tool: toolName, state }, + time: 3, + }) + + publish("message.updated", { sessionID, info: user }) + publish("message.updated", { sessionID, info: assistant }) + for (let i = 0; i < FIXTURE.deltas; i++) + publish("message.part.delta", { sessionID, messageID: assistant.id, partID: "prt_text", field: "text", delta: "Editing the module now. " }, { durable: false }) + + // the multi-file edit: each edit tool part runs then completes (its own + // metadata keeps its patch — other tools' metadata is out of scope) + const files = Array.from({ length: FIXTURE.files }, (_, i) => `packages/app/src/feature/module-${i}.ts`) + const diffs = files.map((file) => ({ file, patch: patchFor(file, FIXTURE.patchBytes), additions: 400, deletions: 12, status: "modified" })) + for (const [i, diff] of diffs.entries()) { + const input = { filePath: diff.file, oldString: "a", newString: "b" } + publish("message.part.updated", tool(`edit_${i}`, "edit", { status: "pending", input, raw: "" })) + const filediff = { file: diff.file, patch: diff.patch, additions: diff.additions, deletions: diff.deletions } + publish( + "message.part.updated", + tool(`edit_${i}`, "edit", { + status: "completed", + input, + output: "Edit applied successfully.", + title: diff.file, + metadata: { diff: diff.patch, filediff, diagnostics: {} }, + time: { start: 3, end: 4 }, + }), + ) + } + + // the chatty shell command: a running update per output chunk BEFORE, the + // throttled updates AFTER (first at once, <= 1 per 200 ms, final flush) + const command = { command: "bun run build --verbose" } + const running = (output: string) => tool("shell_0", "bash", { status: "running", input: command, metadata: { output }, time: { start: 5 } }) + const previews: string[] = [] + let output = "" + for (let i = 0; i < FIXTURE.chunks; i++) { + output += `[build] step ${String(i).padStart(4, "0")} ${"·".repeat(FIXTURE.chunkBytes - 24)}\n` + previews.push(output.length <= PREVIEW_MAX ? output : "...\n\n" + output.slice(-PREVIEW_MAX)) + } + // BEFORE: one running update (and its sync copy) per output chunk + const frameBytes = (payload: Record) => + sseBytes(`${GlobalBus.epoch}:${GlobalBus.seq + 1}`, JSON.stringify({ ...base, payload })) + for (const preview of previews) { + const id = `evt_i1778_fx_shell_${before.length}` + const data = running(preview) + before.push({ type: "shell", sync: false, bytes: frameBytes({ id, type: "message.part.updated", properties: data }) }) + before.push({ + type: "shell", + sync: true, + bytes: frameBytes({ id, type: "sync", syncEvent: { id, type: "message.part.updated.1", seq: 0, aggregateID: sessionID, data } }), + }) + } + // AFTER: the same chunks, 10 ms apart, through the shell tool's throttle + let shellUpdates = 0 + const throttle = yield* makeOutputThrottle((preview) => + Effect.sync(() => { + shellUpdates++ + publish("message.part.updated", running(preview), { record: false }) + }), + ) + for (const preview of previews) { + yield* throttle.offer(preview) + yield* TestClock.adjust(`${FIXTURE.chunkEveryMs} millis`) + } + yield* throttle.flush + publish("message.part.updated", tool("shell_0", "bash", { status: "completed", input: command, output: previews.at(-1), title: command.command, metadata: { output: previews.at(-1), exit: 0 }, time: { start: 5, end: 6 } })) + + // the per-turn summary: every changed file with its full patch + publish("message.updated", { sessionID, info: { ...user, summary: { title: "Edit 88 modules", diffs } } }) + publish("session.updated", { sessionID, info: { id: sessionID, title: "fixture", summary: { additions: 400 * 88, deletions: 12 * 88, files: 88 } } }) + + const frames = yield* plain.take(liveEvents) + // the wire carries exactly the turn's events — no sync copy, no hole + expect(frames.filter((f) => f.data.payload.type === "sync")).toEqual([]) + expect(seqsOf(frames)).toEqual(frames.map((_, i) => seqsOf(frames)[0] + i)) + // the BEFORE formula is the wire's own framing + for (const frame of frames) expect(sseBytes(frame.id!, frame.raw)).toBe(frame.bytes) + + const after = frames.map((frame) => ({ type: frame.data.payload.type as string, bytes: frame.bytes })) + const total = (rows: { bytes: number }[]) => rows.reduce((sum, row) => sum + row.bytes, 0) + const largest = (rows: { bytes: number }[]) => Math.max(...rows.map((row) => row.bytes)) + const summaryAfter = frames.find((f) => f.data.payload.properties?.info?.summary?.diffs)! + const report = { + before: { frames: before.length, totalBytes: total(before), largestFrameBytes: largest(before), syncCopies: before.filter((r) => r.sync).length, shellUpdates: previews.length }, + after: { frames: after.length, totalBytes: total(after), largestFrameBytes: largest(after), syncCopies: 0, shellUpdates }, + summaryFrame: { beforeBytes: before.filter((r) => r.type === "message.updated" && !r.sync).at(-1)!.bytes, afterBytes: summaryAfter.bytes }, + } + console.log(`[#1778 fixture-turn byte report] ${JSON.stringify(report)}`) + console.log( + `[#1778 fixture-turn byte report] total ${(report.before.totalBytes / KB / KB).toFixed(2)} MiB -> ${(report.after.totalBytes / KB / KB).toFixed(2)} MiB` + + ` (${(100 * (1 - report.after.totalBytes / report.before.totalBytes)).toFixed(1)}% cut); largest frame ${(report.before.largestFrameBytes / KB).toFixed(1)} KiB -> ${(report.after.largestFrameBytes / KB).toFixed(1)} KiB;` + + ` frames ${report.before.frames} -> ${report.after.frames}; shell updates ${report.before.shellUpdates} -> ${report.after.shellUpdates}`, + ) + + expect(report.before.largestFrameBytes).toBeGreaterThan(256 * KB) // the fixture is honest: BEFORE breaks the bound + expect(report.after.largestFrameBytes).toBeLessThan(256 * KB) + expect(report.after.totalBytes).toBeLessThan(report.before.totalBytes) + expect(shellUpdates).toBeLessThanOrEqual(1 + (FIXTURE.chunks * FIXTURE.chunkEveryMs) / OUTPUT_THROTTLE_MS + 1) + // the final shell output reached the wire before completion + const lastRunning = frames.filter((f) => f.data.payload.properties?.part?.state?.status === "running").at(-1)! + expect(lastRunning.data.payload.properties.part.state.metadata.output).toBe(previews.at(-1)) + }), + ) +}) From 4f12b10fb323369061d18143713d62e63846667d Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 18:40:39 -0400 Subject: [PATCH 28/60] style(engine): prettier-format the #1778 tests (#1778) --- packages/app-bundle/manifest.json | 10 +- .../workspace-sync-optin.test.ts | 17 +- .../test/server/httpapi-global-event.test.ts | 301 ++++++++++++------ .../test/server/summary-diff-wire.test.ts | 16 +- .../src/components/summary-patch.test.ts | 4 +- 5 files changed, 240 insertions(+), 108 deletions(-) diff --git a/packages/app-bundle/manifest.json b/packages/app-bundle/manifest.json index b233d15b6..735a70e53 100644 --- a/packages/app-bundle/manifest.json +++ b/packages/app-bundle/manifest.json @@ -368,7 +368,7 @@ "packages/opencode/src/tool/write.ts": "094661a3bdd3fab2cd439c0b6cb942b6a2653d3e9451420523b33acbc762d1a7", "packages/opencode/src/worktree/index.ts": "a86f542b579ef40f2940d36e2c8011ce5592d4811cd5ed437bafe614789fb87e", "packages/opencode/test/acp/service-session.test.ts": "5810c4a0c935f06389c5f01df791e4ae82a4c32268407baf32dc4502d487f282", - "packages/opencode/test/control-plane/workspace-sync-optin.test.ts": "ba35d79f25bf2d9215c60dfba1d34746684fd22b4a6d2870cc7d81de1b57ec2d", + "packages/opencode/test/control-plane/workspace-sync-optin.test.ts": "dd77386e26d2d5b357f22e19ae384f35befe08421556f84215a5ac6678d78ac8", "packages/opencode/test/effect/runner-queue.test.ts": "9ce3074dc4a86f620381e2c7ada08b9034b500a62b5da1a7d63896f2519085ca", "packages/opencode/test/effect/runner.test.ts": "6be02ad6a7d173642100a9e17b323410bad7ece842a00018d2170b91d4894c5a", "packages/opencode/test/fixture/fixture.ts": "357bdaa06c6233a91069db20cc5f1cbb2d6eafa3bc7ba3e89b019859d195d6ef", @@ -400,7 +400,7 @@ "packages/opencode/test/server/httpapi-compression.test.ts": "5f7603d7018e0b95f1c2f36c01dd297c01770bfcdd91ce29778e03b7d6143f75", "packages/opencode/test/server/httpapi-file-write-gate.test.ts": "0c91b7417e615850edfde3f2878207dab6b6624d4cfbb6e6783b2e6d9aa4dff0", "packages/opencode/test/server/httpapi-file.test.ts": "d3386129bc81467ecbcc57a3861f53767ee56ac41b4ff6ad58d8b168fb3b3373", - "packages/opencode/test/server/httpapi-global-event.test.ts": "9fc2b195d2784ca2f51e75b68f31c0f73aa69afdc7ca2bd704ac581971c20b30", + "packages/opencode/test/server/httpapi-global-event.test.ts": "0f08bc78a1ba2e3069e7fb292202db88fad532a8603653e4e8593e4dd95ab066", "packages/opencode/test/server/httpapi-instance-route-auth.test.ts": "4b73a89d2431c02bc3b665d93f9002914e51eb1add979dc1867efbcbc0c36e25", "packages/opencode/test/server/httpapi-provider.test.ts": "9e8b3e085b478e3f99b689ffff600e10c0b4b50b3cabdbd749f88dae19bf5904", "packages/opencode/test/server/httpapi-session.test.ts": "eda6fd763ac4c17891cedaa4ccd49597c03e7227fcfdcf0f94a1b4545ba623ed", @@ -409,7 +409,7 @@ "packages/opencode/test/server/session-compact-errors.test.ts": "2ae2044d66db1ba41a5e294afedea5271f11d5bf384685a06ea05ac515b16fb2", "packages/opencode/test/server/session-diff-missing-patch.test.ts": "70526ca7fb86741843fd167f8f2f9c6f6d7e5e415a20a5448d17d24229bdd65b", "packages/opencode/test/server/session-diff-scoped.test.ts": "20ef80b92ad872573b371540575f028be8f1ad3c9f34d72f814b621e14e7a59e", - "packages/opencode/test/server/summary-diff-wire.test.ts": "af750d297333225e2ee7342cab59b8e0131294157d742d8d35c62103691dc5d5", + "packages/opencode/test/server/summary-diff-wire.test.ts": "3b1653608a5c292074a46ea6a9cc0f680e03a3b65bc10c6b499250a5f761c170", "packages/opencode/test/server/workspace-routing.test.ts": "2574731ac379c540f450edc15f8750ac5360e8c33b4a0dad7c61e8eb686137bb", "packages/opencode/test/session/adversarial-e2e.test.ts": "1b471a69223d040c5d5f4f2ee99c2daa6df56bdf11679b65d902c15bedac3e80", "packages/opencode/test/session/business-record.test.ts": "7857094479408bcfd6ea8d9f5b425c6384bcf8490f930674bc1c203e82782d75", @@ -467,7 +467,7 @@ "packages/session-ui/src/components/session-review.tsx": "d2015c9f7419279f08e84523439a0b73e83bd5d06587ad56b85ea53b66ecaee4", "packages/session-ui/src/components/session-turn.css": "792915f3373134eadf3cd51e1d4bfa88ec347e6b037b6caf60b639ec5e74fbd8", "packages/session-ui/src/components/session-turn.tsx": "42f247b0434a9f52d88d9fb96adff59afe6edeab84dd912b62eb153253b6696f", - "packages/session-ui/src/components/summary-patch.test.ts": "560272a942f1cdcaf78ea0cdc6ec25e49e3ddc8eef1c93d4cd6c728294b6ae40", + "packages/session-ui/src/components/summary-patch.test.ts": "27ee37939eabcfd00dc3a7eb3da95d2179dd222d4193e9f71f7a840f6fd8579e", "packages/session-ui/src/components/summary-patch.ts": "ab086ea1bc741e15b94f6e9aedc4d934cac17597f4aa1392b146994020006170", "packages/session-ui/src/components/tool-count-label.test.ts": "c62bbe2c662f6740794d25bd7c33fb3b9b46394a1b41d01d6568a8a2ee2b4e75", "packages/session-ui/src/components/tool-count-label.tsx": "06c96479710314d443d42d4c29a29b46d58a55ee84a844a5b6865b49a0513340", @@ -959,7 +959,7 @@ "packages/ui/src/v2/components/text-shimmer-v2.tsx" ], "exceptions": [], - "extracted_at": "2026-10-12T22:16:34.724Z", + "extracted_at": "2026-10-12T22:40:33.106Z", "per_package": { "packages/app": { "A": 237, diff --git a/packages/app-bundle/overlay/packages/opencode/test/control-plane/workspace-sync-optin.test.ts b/packages/app-bundle/overlay/packages/opencode/test/control-plane/workspace-sync-optin.test.ts index 34074ab41..de2e8c254 100644 --- a/packages/app-bundle/overlay/packages/opencode/test/control-plane/workspace-sync-optin.test.ts +++ b/packages/app-bundle/overlay/packages/opencode/test/control-plane/workspace-sync-optin.test.ts @@ -138,7 +138,12 @@ const insertWorkspace = (info: Workspace.Info) => const attachSessionToWorkspace = (sessionID: SessionID, workspaceID: WorkspaceV2.ID) => Database.Service.use(({ db }) => - db.update(SessionTable).set({ workspace_id: workspaceID }).where(eq(SessionTable.id, sessionID)).run().pipe(Effect.orDie), + db + .update(SessionTable) + .set({ workspace_id: workspaceID }) + .where(eq(SessionTable.id, sessionID)) + .run() + .pipe(Effect.orDie), ) const sessionSequence = (sessionID: SessionID) => @@ -236,12 +241,14 @@ describe("workspace control plane — sync copies are opt-in (#1778)", () => { expect(requests.length).toBeGreaterThan(0) expect(requests.every((request) => request.syncCopies === "true")).toBe(true) // the opt-in rides a header: the event URL keeps its routed form - expect(requests.every((request) => request.url.pathname === "/optin/global/event" && !request.url.search)).toBe( - true, - ) + expect( + requests.every((request) => request.url.pathname === "/optin/global/event" && !request.url.search), + ).toBe(true) // and the remote's frames are still forwarded onto the local bus expect( - captured.events.some((event) => event.workspace === info.id && event.payload.type === "session.updated"), + captured.events.some( + (event) => event.workspace === info.id && event.payload.type === "session.updated", + ), ).toBe(true) yield* workspace.remove(info.id) } finally { diff --git a/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts b/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts index 26001fb5f..970d82786 100644 --- a/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts +++ b/packages/app-bundle/overlay/packages/opencode/test/server/httpapi-global-event.test.ts @@ -80,12 +80,19 @@ type Frame = { id: string | undefined; event: string | undefined; data: any; raw /** Split an SSE id into its epoch and sequence (fails the test if malformed). */ function idOf(frame: Frame) { const match = ID.exec(frame.id ?? "") - if (!match) throw new Error(`frame id ${JSON.stringify(frame.id)} is not : (${JSON.stringify(frame.data)})`) + if (!match) + throw new Error(`frame id ${JSON.stringify(frame.id)} is not : (${JSON.stringify(frame.data)})`) return { epoch: match[1], seq: Number(match[2]) } } function parseFrame(block: string): Frame { - const frame: Frame = { id: undefined, event: undefined, data: undefined, raw: "", bytes: Buffer.byteLength(block) + 2 } + const frame: Frame = { + id: undefined, + event: undefined, + data: undefined, + raw: "", + bytes: Buffer.byteLength(block) + 2, + } const data: string[] = [] for (const line of block.split("\n")) { if (line.startsWith("id: ")) frame.id = line.slice(4) @@ -167,68 +174,82 @@ const quiet = (stream: { next: Effect.Effect }, ms = 200) => stream.next.pipe(Effect.timeoutOrElse({ duration: ms, orElse: () => Effect.succeed(undefined) })) describe("global event stream — sequenced ids (#1776)", () => { - it.live("every frame carries an : id shared by all subscribers; a sync copy carries its source's id", () => - Effect.gen(function* () { - // sync copies are opt-in (#1778): both subscribers request them - const a = yield* connect({ query: "?sync=true" }) - const b = yield* connect({ query: "?sync=true" }) - const connectedA = yield* a.next - const connectedB = yield* b.next - expect(connectedA.data.payload.type).toBe("server.connected") - const start = idOf(connectedA) - expect(idOf(connectedB)).toEqual(start) - - emit("test.first") - // the event-v2 bridge's twin emission: the durable event, then its sync copy - GlobalBus.emit("event", { directory: "/tmp/i1776", payload: { id: "evt_i1776_durable", type: "test.durable", properties: {} } }) - GlobalBus.emit("event", { - directory: "/tmp/i1776", - payload: { - type: "sync", - syncEvent: { id: "evt_i1776_durable", type: "test.durable.1", seq: 1, aggregateID: "agg", data: {} }, - }, - }) - emit("test.last") - - const framesA = yield* a.take(4) - const framesB = yield* b.take(4) - expect(framesA.map((f) => f.data.payload.type)).toEqual(["test.first", "test.durable", "sync", "test.last"]) - expect(framesA.map((f) => f.id)).toEqual(framesB.map((f) => f.id)) - const seqs = framesA.map((f) => idOf(f).seq) - expect(framesA.every((f) => idOf(f).epoch === start.epoch)).toBe(true) - expect(seqs).toEqual([start.seq + 1, start.seq + 2, start.seq + 2, start.seq + 3]) - }), + it.live( + "every frame carries an : id shared by all subscribers; a sync copy carries its source's id", + () => + Effect.gen(function* () { + // sync copies are opt-in (#1778): both subscribers request them + const a = yield* connect({ query: "?sync=true" }) + const b = yield* connect({ query: "?sync=true" }) + const connectedA = yield* a.next + const connectedB = yield* b.next + expect(connectedA.data.payload.type).toBe("server.connected") + const start = idOf(connectedA) + expect(idOf(connectedB)).toEqual(start) + + emit("test.first") + // the event-v2 bridge's twin emission: the durable event, then its sync copy + GlobalBus.emit("event", { + directory: "/tmp/i1776", + payload: { id: "evt_i1776_durable", type: "test.durable", properties: {} }, + }) + GlobalBus.emit("event", { + directory: "/tmp/i1776", + payload: { + type: "sync", + syncEvent: { id: "evt_i1776_durable", type: "test.durable.1", seq: 1, aggregateID: "agg", data: {} }, + }, + }) + emit("test.last") + + const framesA = yield* a.take(4) + const framesB = yield* b.take(4) + expect(framesA.map((f) => f.data.payload.type)).toEqual(["test.first", "test.durable", "sync", "test.last"]) + expect(framesA.map((f) => f.id)).toEqual(framesB.map((f) => f.id)) + const seqs = framesA.map((f) => idOf(f).seq) + expect(framesA.every((f) => idOf(f).epoch === start.epoch)).toBe(true) + expect(seqs).toEqual([start.seq + 1, start.seq + 2, start.seq + 2, start.seq + 3]) + }), ) }) describe("global event stream — sync copies are opt-in (#1778)", () => { - it.live("a default subscriber gets no sync copies and its sequence stays gap-free; an opt-in subscriber gets them all", () => - Effect.gen(function* () { - const plain = yield* connect() - // the header carrier (what the workspace control plane sends); `?sync=true` is the query twin - const synced = yield* connect({ headers: { "x-opencode-sync-copies": "true" } }) - const start = idOf(yield* plain.next) - expect(idOf(yield* synced.next)).toEqual(start) - - emit("test.first") - twin("evt_i1778_optin_a") - twin("evt_i1778_optin_b") - emit("test.last") - - const plainFrames = yield* plain.take(4) - expect(typesOf(plainFrames)).toEqual(["test.first", "test.durable", "test.durable", "test.last"]) - // filtering the copies punches no hole: a copy shares its source's sequence - expect(seqsOf(plainFrames)).toEqual([1, 2, 3, 4].map((n) => start.seq + n)) - expect(yield* quiet(plain)).toBeUndefined() - - const syncedFrames = yield* synced.take(6) - expect(typesOf(syncedFrames)).toEqual(["test.first", "test.durable", "sync", "test.durable", "sync", "test.last"]) - expect(seqsOf(syncedFrames)).toEqual([1, 2, 2, 3, 3, 4].map((n) => start.seq + n)) - // the frames both subscribers get are the same frames - expect(syncedFrames.filter((f) => f.data.payload.type !== "sync").map((f) => f.id)).toEqual( - plainFrames.map((f) => f.id), - ) - }), + it.live( + "a default subscriber gets no sync copies and its sequence stays gap-free; an opt-in subscriber gets them all", + () => + Effect.gen(function* () { + const plain = yield* connect() + // the header carrier (what the workspace control plane sends); `?sync=true` is the query twin + const synced = yield* connect({ headers: { "x-opencode-sync-copies": "true" } }) + const start = idOf(yield* plain.next) + expect(idOf(yield* synced.next)).toEqual(start) + + emit("test.first") + twin("evt_i1778_optin_a") + twin("evt_i1778_optin_b") + emit("test.last") + + const plainFrames = yield* plain.take(4) + expect(typesOf(plainFrames)).toEqual(["test.first", "test.durable", "test.durable", "test.last"]) + // filtering the copies punches no hole: a copy shares its source's sequence + expect(seqsOf(plainFrames)).toEqual([1, 2, 3, 4].map((n) => start.seq + n)) + expect(yield* quiet(plain)).toBeUndefined() + + const syncedFrames = yield* synced.take(6) + expect(typesOf(syncedFrames)).toEqual([ + "test.first", + "test.durable", + "sync", + "test.durable", + "sync", + "test.last", + ]) + expect(seqsOf(syncedFrames)).toEqual([1, 2, 2, 3, 3, 4].map((n) => start.seq + n)) + // the frames both subscribers get are the same frames + expect(syncedFrames.filter((f) => f.data.payload.type !== "sync").map((f) => f.id)).toEqual( + plainFrames.map((f) => f.id), + ) + }), ) it.live("a resumed connection replays sync copies only when it requests them", () => @@ -252,28 +273,37 @@ describe("global event stream — sync copies are opt-in (#1778)", () => { expect(yield* quiet(plain)).toBeUndefined() const syncedFrames = yield* synced.take(6) - expect(typesOf(syncedFrames)).toEqual(["test.durable", "sync", "test.between", "test.durable", "sync", "test.live"]) + expect(typesOf(syncedFrames)).toEqual([ + "test.durable", + "sync", + "test.between", + "test.durable", + "sync", + "test.live", + ]) expect(seqsOf(syncedFrames)).toEqual([1, 1, 2, 3, 3, 4].map((n) => base + n)) }), ) - it.live("a sync copy with no framed source keeps its own sequence and reaches every subscriber, so no hole opens", () => - Effect.gen(function* () { - const plain = yield* connect() - const start = idOf(yield* plain.next) - emit("test.before") - GlobalBus.emit("event", { - directory: "/tmp/i1778", - payload: { - type: "sync", - syncEvent: { id: "evt_i1778_orphan", type: "test.orphan.1", seq: 1, aggregateID: "agg", data: {} }, - }, - }) - emit("test.after") - const frames = yield* plain.take(3) - expect(typesOf(frames)).toEqual(["test.before", "sync", "test.after"]) - expect(seqsOf(frames)).toEqual([1, 2, 3].map((n) => start.seq + n)) - }), + it.live( + "a sync copy with no framed source keeps its own sequence and reaches every subscriber, so no hole opens", + () => + Effect.gen(function* () { + const plain = yield* connect() + const start = idOf(yield* plain.next) + emit("test.before") + GlobalBus.emit("event", { + directory: "/tmp/i1778", + payload: { + type: "sync", + syncEvent: { id: "evt_i1778_orphan", type: "test.orphan.1", seq: 1, aggregateID: "agg", data: {} }, + }, + }) + emit("test.after") + const frames = yield* plain.take(3) + expect(typesOf(frames)).toEqual(["test.before", "sync", "test.after"]) + expect(seqsOf(frames)).toEqual([1, 2, 3].map((n) => start.seq + n)) + }), ) }) @@ -281,7 +311,13 @@ describe("global event stream — sync copies are opt-in (#1778)", () => { // full patch text (measured up to 12.3 MB in one event on the Studio). const summaryDiffs = () => [ { file: "src/app.ts", patch: "@@ -1,2 +1,3 @@\n-old\n+new\n+more\n", additions: 2, deletions: 1, status: "modified" }, - { file: "src/new.ts", patch: `@@ -0,0 +1,400 @@\n${"+line\n".repeat(400)}`, additions: 400, deletions: 0, status: "added" }, + { + file: "src/new.ts", + patch: `@@ -0,0 +1,400 @@\n${"+line\n".repeat(400)}`, + additions: 400, + deletions: 0, + status: "added", + }, { file: "src/gone.ts", patch: "@@ -1 +0,0 @@\n-gone\n", additions: 0, deletions: 1, status: "deleted" }, ] const stripped = (diffs: ReturnType) => diffs.map(({ patch: _patch, ...rest }) => rest) @@ -291,7 +327,11 @@ const summaryTwin = (id: string, type: "message.updated" | "session.updated") => const info = type === "message.updated" ? { id: "msg_i1778", sessionID: "ses_i1778", role: "user", summary: { title: "Edit", diffs: summaryDiffs() } } - : { id: "ses_i1778", title: "Session", summary: { additions: 402, deletions: 2, files: 3, diffs: summaryDiffs() } } + : { + id: "ses_i1778", + title: "Session", + summary: { additions: 402, deletions: 2, files: 3, diffs: summaryDiffs() }, + } const properties = { sessionID: "ses_i1778", info } const source = { directory: "/tmp/i1778", payload: { id, type, properties } } const copy = { @@ -534,22 +574,46 @@ describe("global event stream — fixture turn byte report (#1778)", () => { // Emit like the event-v2 bridge (the event, then for a durable one its sync // copy) and record the frame #1776's tap would have serialized for each. - const publish = (type: string, properties: Record, options: { durable?: boolean; record?: boolean } = {}) => { + const publish = ( + type: string, + properties: Record, + options: { durable?: boolean; record?: boolean } = {}, + ) => { const id = `evt_i1778_fx_${GlobalBus.seq + 1}` const event = { ...base, payload: { id, type, properties } } GlobalBus.emit("event", event) liveEvents++ const record = options.record !== false - if (record) before.push({ type, sync: false, bytes: sseBytes(`${GlobalBus.epoch}:${GlobalBus.seq}`, JSON.stringify(event)) }) + if (record) + before.push({ + type, + sync: false, + bytes: sseBytes(`${GlobalBus.epoch}:${GlobalBus.seq}`, JSON.stringify(event)), + }) if (options.durable === false) return const copy = { ...base, - payload: { type: "sync", syncEvent: { id, type: `${type}.1`, seq: before.length, aggregateID: sessionID, data: properties } }, + payload: { + type: "sync", + syncEvent: { id, type: `${type}.1`, seq: before.length, aggregateID: sessionID, data: properties }, + }, } GlobalBus.emit("event", copy) - if (record) before.push({ type, sync: true, bytes: sseBytes(`${GlobalBus.epoch}:${GlobalBus.seq}`, JSON.stringify(copy)) }) + if (record) + before.push({ + type, + sync: true, + bytes: sseBytes(`${GlobalBus.epoch}:${GlobalBus.seq}`, JSON.stringify(copy)), + }) + } + const user = { + id: "msg_i1778_user", + sessionID, + role: "user", + time: { created: 1 }, + agent: "build", + model: { providerID: "p", modelID: "m" }, } - const user = { id: "msg_i1778_user", sessionID, role: "user", time: { created: 1 }, agent: "build", model: { providerID: "p", modelID: "m" } } const assistant = { id: "msg_i1778_asst", sessionID, role: "assistant", parentID: user.id, time: { created: 2 } } const tool = (callID: string, toolName: string, state: Record) => ({ sessionID, @@ -560,12 +624,22 @@ describe("global event stream — fixture turn byte report (#1778)", () => { publish("message.updated", { sessionID, info: user }) publish("message.updated", { sessionID, info: assistant }) for (let i = 0; i < FIXTURE.deltas; i++) - publish("message.part.delta", { sessionID, messageID: assistant.id, partID: "prt_text", field: "text", delta: "Editing the module now. " }, { durable: false }) + publish( + "message.part.delta", + { sessionID, messageID: assistant.id, partID: "prt_text", field: "text", delta: "Editing the module now. " }, + { durable: false }, + ) // the multi-file edit: each edit tool part runs then completes (its own // metadata keeps its patch — other tools' metadata is out of scope) const files = Array.from({ length: FIXTURE.files }, (_, i) => `packages/app/src/feature/module-${i}.ts`) - const diffs = files.map((file) => ({ file, patch: patchFor(file, FIXTURE.patchBytes), additions: 400, deletions: 12, status: "modified" })) + const diffs = files.map((file) => ({ + file, + patch: patchFor(file, FIXTURE.patchBytes), + additions: 400, + deletions: 12, + status: "modified", + })) for (const [i, diff] of diffs.entries()) { const input = { filePath: diff.file, oldString: "a", newString: "b" } publish("message.part.updated", tool(`edit_${i}`, "edit", { status: "pending", input, raw: "" })) @@ -586,7 +660,8 @@ describe("global event stream — fixture turn byte report (#1778)", () => { // the chatty shell command: a running update per output chunk BEFORE, the // throttled updates AFTER (first at once, <= 1 per 200 ms, final flush) const command = { command: "bun run build --verbose" } - const running = (output: string) => tool("shell_0", "bash", { status: "running", input: command, metadata: { output }, time: { start: 5 } }) + const running = (output: string) => + tool("shell_0", "bash", { status: "running", input: command, metadata: { output }, time: { start: 5 } }) const previews: string[] = [] let output = "" for (let i = 0; i < FIXTURE.chunks; i++) { @@ -599,11 +674,19 @@ describe("global event stream — fixture turn byte report (#1778)", () => { for (const preview of previews) { const id = `evt_i1778_fx_shell_${before.length}` const data = running(preview) - before.push({ type: "shell", sync: false, bytes: frameBytes({ id, type: "message.part.updated", properties: data }) }) + before.push({ + type: "shell", + sync: false, + bytes: frameBytes({ id, type: "message.part.updated", properties: data }), + }) before.push({ type: "shell", sync: true, - bytes: frameBytes({ id, type: "sync", syncEvent: { id, type: "message.part.updated.1", seq: 0, aggregateID: sessionID, data } }), + bytes: frameBytes({ + id, + type: "sync", + syncEvent: { id, type: "message.part.updated.1", seq: 0, aggregateID: sessionID, data }, + }), }) } // AFTER: the same chunks, 10 ms apart, through the shell tool's throttle @@ -619,11 +702,24 @@ describe("global event stream — fixture turn byte report (#1778)", () => { yield* TestClock.adjust(`${FIXTURE.chunkEveryMs} millis`) } yield* throttle.flush - publish("message.part.updated", tool("shell_0", "bash", { status: "completed", input: command, output: previews.at(-1), title: command.command, metadata: { output: previews.at(-1), exit: 0 }, time: { start: 5, end: 6 } })) + publish( + "message.part.updated", + tool("shell_0", "bash", { + status: "completed", + input: command, + output: previews.at(-1), + title: command.command, + metadata: { output: previews.at(-1), exit: 0 }, + time: { start: 5, end: 6 }, + }), + ) // the per-turn summary: every changed file with its full patch publish("message.updated", { sessionID, info: { ...user, summary: { title: "Edit 88 modules", diffs } } }) - publish("session.updated", { sessionID, info: { id: sessionID, title: "fixture", summary: { additions: 400 * 88, deletions: 12 * 88, files: 88 } } }) + publish("session.updated", { + sessionID, + info: { id: sessionID, title: "fixture", summary: { additions: 400 * 88, deletions: 12 * 88, files: 88 } }, + }) const frames = yield* plain.take(liveEvents) // the wire carries exactly the turn's events — no sync copy, no hole @@ -637,9 +733,24 @@ describe("global event stream — fixture turn byte report (#1778)", () => { const largest = (rows: { bytes: number }[]) => Math.max(...rows.map((row) => row.bytes)) const summaryAfter = frames.find((f) => f.data.payload.properties?.info?.summary?.diffs)! const report = { - before: { frames: before.length, totalBytes: total(before), largestFrameBytes: largest(before), syncCopies: before.filter((r) => r.sync).length, shellUpdates: previews.length }, - after: { frames: after.length, totalBytes: total(after), largestFrameBytes: largest(after), syncCopies: 0, shellUpdates }, - summaryFrame: { beforeBytes: before.filter((r) => r.type === "message.updated" && !r.sync).at(-1)!.bytes, afterBytes: summaryAfter.bytes }, + before: { + frames: before.length, + totalBytes: total(before), + largestFrameBytes: largest(before), + syncCopies: before.filter((r) => r.sync).length, + shellUpdates: previews.length, + }, + after: { + frames: after.length, + totalBytes: total(after), + largestFrameBytes: largest(after), + syncCopies: 0, + shellUpdates, + }, + summaryFrame: { + beforeBytes: before.filter((r) => r.type === "message.updated" && !r.sync).at(-1)!.bytes, + afterBytes: summaryAfter.bytes, + }, } console.log(`[#1778 fixture-turn byte report] ${JSON.stringify(report)}`) console.log( diff --git a/packages/app-bundle/overlay/packages/opencode/test/server/summary-diff-wire.test.ts b/packages/app-bundle/overlay/packages/opencode/test/server/summary-diff-wire.test.ts index 05ec718bc..3fcc85a00 100644 --- a/packages/app-bundle/overlay/packages/opencode/test/server/summary-diff-wire.test.ts +++ b/packages/app-bundle/overlay/packages/opencode/test/server/summary-diff-wire.test.ts @@ -42,8 +42,20 @@ const pathFor = (template: string, params: Record) => Object.entries(params).reduce((result, [key, value]) => result.replace(`:${key}`, value), template) const diffs = [ - { file: "src/main.ts", patch: "@@ -1 +1,2 @@\n-a\n+b\n+c\n", additions: 2, deletions: 1, status: "modified" as const }, - { file: "src/new.ts", patch: `@@ -0,0 +1,300 @@\n${"+x\n".repeat(300)}`, additions: 300, deletions: 0, status: "added" as const }, + { + file: "src/main.ts", + patch: "@@ -1 +1,2 @@\n-a\n+b\n+c\n", + additions: 2, + deletions: 1, + status: "modified" as const, + }, + { + file: "src/new.ts", + patch: `@@ -0,0 +1,300 @@\n${"+x\n".repeat(300)}`, + additions: 300, + deletions: 0, + status: "added" as const, + }, ] describe("summary diffs — wire-only projection (#1778)", () => { diff --git a/packages/app-bundle/overlay/packages/session-ui/src/components/summary-patch.test.ts b/packages/app-bundle/overlay/packages/session-ui/src/components/summary-patch.test.ts index 0f0b03623..6b8941b7e 100644 --- a/packages/app-bundle/overlay/packages/session-ui/src/components/summary-patch.test.ts +++ b/packages/app-bundle/overlay/packages/session-ui/src/components/summary-patch.test.ts @@ -90,7 +90,9 @@ describe("per-turn diff view — patch bodies on demand (#1778)", () => { }) test("without a source (a provider that cannot fetch) a patch-less diff resolves to no patch", async () => { - expect(await createSummaryPatchLoader(undefined).patch({ ...turn, file: "src/app.ts", summary: live })).toBeUndefined() + expect( + await createSummaryPatchLoader(undefined).patch({ ...turn, file: "src/app.ts", summary: live }), + ).toBeUndefined() }) }) From 925900ee3f5a046180c227e7db0b8cd9d95889a1 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 19:28:59 -0400 Subject: [PATCH 29/60] =?UTF-8?q?fix(fleet):=20any=20data=20frame=20counts?= =?UTF-8?q?=20as=20delivered=20=E2=80=94=20a=20re-opened=20arm=20from=20an?= =?UTF-8?q?=20id-less=20engine=20announces=20its=20gap=20(#1773)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../amicode_service/sse_fanin_aggregator.ts | 60 +++++++++++------ .../test/sse_fanin_aggregator.test.ts | 67 +++++++++++++++++++ .../test/sse_fanin_driver_pump.test.ts | 8 ++- 3 files changed, 114 insertions(+), 21 deletions(-) diff --git a/packages/extension/src/amicode_service/sse_fanin_aggregator.ts b/packages/extension/src/amicode_service/sse_fanin_aggregator.ts index 9857e416c..d5f784afd 100644 --- a/packages/extension/src/amicode_service/sse_fanin_aggregator.ts +++ b/packages/extension/src/amicode_service/sse_fanin_aggregator.ts @@ -165,6 +165,16 @@ export function frameRawId(rawFrame: string): string | undefined { return id === undefined || id === "" ? undefined : id; } +/** #1773 — does the frame carry a `data` field (an event the client dispatches)? + * This is what "delivered" means for the reconnect gap: ANY data frame, with or + * without an `id:` line. A comment-only frame (`: keep-alive`) is not data. */ +export function isDataFrame(rawFrame: string): boolean { + for (const line of rawFrame.split(/\r?\n/)) { + if (line === "data" || line.startsWith("data:")) return true; + } + return false; +} + /** Rebuild a frame with its `id:` line set to `newId` (or removed when null), * preserving every other line (`event:`/`data:`/`retry:`/comments) verbatim. * The single canonical `id:` line is placed just before the trailing blank so @@ -234,6 +244,16 @@ export interface ArmTraffic { * (#1638), or frames were dropped on overflow (#1617). */ export type GapCause = "reopen" | "overflow"; +/** #1773 — what a re-open of an arm must do (`reopenArm`'s verdict): + * - `open` — the arm never delivered a data frame (an initial open): nothing + * was lost, open it silently; + * - `gap` — composite mode: the gap frame naming it was written; open it; + * - `sever` — fleet-of-one: a synthetic frame is not allowed in the verbatim + * stream (#1264), so the caller must END THE LISTENER WITH A + * TRANSPORT ERROR instead of re-opening — the client registers the + * disconnect and resyncs. Nothing was written. */ +export type ReopenVerdict = "open" | "gap" | "sever"; + /** #1771 — the aggregator's transitions, for the census. Observation only: * callbacks run after the aggregator decided, and a throwing observer is * swallowed. */ @@ -304,7 +324,9 @@ export class SseFanInAggregator { /** #1638 — namespaces whose arm has EVER delivered a frame this connection. * A re-open of an arm not in this set is the INITIAL open (no gap); a re-open * of one already here is a NON-INITIAL re-open (the #1601 path → gap in - * composite mode). */ + * composite mode, a severed listener in fleet-of-one). #1773: ANY data frame + * counts (an older engine sends no `id:` lines at all), counted as the arm + * hands it over — whether it is then delivered, buffered, or dropped. */ private readonly everDelivered = new Set(); /** #1617 (fleet-of-one) — frames stranded on the zero-peer VERBATIM path while * the downstream is backpressured. Kept SEPARATE from the composite `buffers` @@ -380,6 +402,7 @@ export class SseFanInAggregator { * arm; a machineId for a peer arm. */ ingest(namespace: string, rawFrame: string): void { this.countFrame(namespace, rawFrame); // #1771 census — observation only + if (isDataFrame(rawFrame)) this.everDelivered.add(namespace); // #1773 — any data frame const rawId = frameRawId(rawFrame); const fleetOfOne = this.peerArms.size === 0; @@ -387,10 +410,7 @@ export class SseFanInAggregator { // §D4 / AC1 byte-identity: with no peer arms the local frame is written // VERBATIM (bare id, every line untouched). The composite still tracks the // local position silently, so a peer that joins mid-stream inherits it. - if (rawId !== undefined) { - this.composite.set(LOCAL_NAMESPACE, rawId); - this.everDelivered.add(LOCAL_NAMESPACE); - } + if (rawId !== undefined) this.composite.set(LOCAL_NAMESPACE, rawId); // #1617 (fleet-of-one) — honor backpressure on the verbatim path too. When // the downstream is already backpressured, hold the frame VERBATIM in the // verbatim buffer (never namespaced) so it is not fired into an over-full @@ -500,19 +520,24 @@ export class SseFanInAggregator { /** #1638 — a peer/local arm is being RE-OPENED (the #1601 reconcile path). In * composite mode (≥1 peer arm) a NON-INITIAL re-open forces a client refetch * via exactly one id-less `amicode.sync.gap` naming that namespace, because - * the upstream `/global/event` route does NOT replay on `?lastEventID` (Step 0 - * = no replay) so the cursor-advance alone cannot recover the reconnect gap. - * In FLEET-OF-ONE (zero peers) NO gap is emitted — a synthetic frame in the - * verbatim stream would violate the #1264 byte-identity guard; the solo-mode - * arm re-opens silently (its verbatim stream stays frame-for-frame identical). - * The INITIAL open of an arm (never delivered a frame) emits no gap either — - * only a re-open of an arm that has already delivered. */ - reopenArm(namespace: string): void { - if (this.peerArms.size === 0) return; // fleet-of-one → NO synthetic frame (#1264) - if (!this.everDelivered.has(namespace)) return; // initial open → no gap + * the cursor-advance alone cannot recover the reconnect gap (an older engine + * has no ids to resume from; replay is #1777). + * + * #1773 — every loss is loud. "Non-initial" means the arm delivered ANY data + * frame (id'd or not). In FLEET-OF-ONE (zero peers) a synthetic frame in the + * verbatim stream would violate the #1264 byte-identity guard, so instead of + * re-opening silently the verdict is `sever`: the caller ends the listener + * with a transport error, and the client's disconnect is the gap signal. + * Nothing is written for `sever`. The INITIAL open of an arm (never delivered + * a data frame) is `open` in both modes — nothing was lost. A gap never + * advances any cursor. */ + reopenArm(namespace: string): ReopenVerdict { + if (!this.everDelivered.has(namespace)) return "open"; // initial open → nothing lost + if (this.peerArms.size === 0) return "sever"; // fleet-of-one → NO synthetic frame (#1264) const data = JSON.stringify({ type: "amicode.sync.gap", namespaces: [namespace] }); this.writeControl(`event: amicode.sync.gap\ndata: ${data}\n\n`); this.noteGap([namespace], "reopen"); + return "gap"; } /** #1617 — the overflow defense-in-depth. When resume() observes a nonzero @@ -611,10 +636,7 @@ export class SseFanInAggregator { * downstream (subsequent frames enqueue onto the ordered pending stream). */ private deliver(namespace: string, rawFrame: string): void { const rawId = frameRawId(rawFrame); - if (rawId !== undefined) { - this.composite.set(namespace, rawId); - this.everDelivered.add(namespace); - } + if (rawId !== undefined) this.composite.set(namespace, rawId); const wire = rewriteFrameId(rawFrame, formatCompositeCursor(this.composite)); const accepted = this.sink.write(wire); this.sink.flush?.(); diff --git a/packages/extension/test/sse_fanin_aggregator.test.ts b/packages/extension/test/sse_fanin_aggregator.test.ts index b2f12c4a9..e27c3d727 100644 --- a/packages/extension/test/sse_fanin_aggregator.test.ts +++ b/packages/extension/test/sse_fanin_aggregator.test.ts @@ -834,6 +834,73 @@ describe("#1638 — composite-mode reopenArm emits exactly one id-less gap; flee }); }); +// ══════════════════════════════════════════════════════════════════════════════ +// #1773 AC1 — "delivered" means ANY data frame, not only an id'd one. The re-open +// gap used to fire only for arms that had delivered a frame carrying an SSE id — +// which a peer running an older engine never sends — so a re-opened arm never +// announced its gap. Engine-shaped fixtures in BOTH modes: legacy (id-less) and +// sequenced (#1776, `id: :`). +// ══════════════════════════════════════════════════════════════════════════════ +describe("#1773 AC1 — a re-open of an arm that delivered ANY data frame emits exactly one gap (composite mode)", () => { + const gapFrames = (text: string) => text.split("\n\n").filter((f) => f.includes("amicode.sync.gap")); + const gapOf = (f: string) => JSON.parse(f.split("\n").find((l) => l.startsWith("data:"))!.slice(5).trim()); + + for (const mode of ["legacy", "sequenced"] as const) { + it(`${mode} engine: a non-initial re-open emits exactly one id-less gap naming that namespace, and never advances a cursor`, () => { + const sink = collectingSink(); + const agg = new SseFanInAggregator({ sink }); + agg.connect(`local=${LOCAL_EPOCH}:3`); + agg.setMembership([{ machineId: "studio", reachable: true, token: "tok-studio" }]); + const studio = mode === "legacy" ? fakeEngine({ legacy: true }) : fakeEngine({ epoch: PEER_EPOCH }); + + // the initial open: no gap + expect(agg.reopenArm("studio")).toBe("open"); + // the arm delivers the engine's real stream: connected, an event, a stored + // event's plain + sync twin, a heartbeat + for (const f of [ + studio.connected(), + studio.event("session.status", { sessionID: "ses_1", status: { type: "busy" } }), + ...studio.stored("message.part.updated", { part: { id: "prt_1" } }, { aggregateID: "ses_1" }), + studio.heartbeat(), + ]) + agg.ingest("studio", f); + const cursorBefore = agg.cursor(); + expect(sink.text()).not.toContain("amicode.sync.gap"); + + // the arm drops and the driver re-opens it + expect(agg.reopenArm("studio")).toBe("gap"); + + const gaps = gapFrames(sink.text()); + expect(gaps).toHaveLength(1); + expect(gapOf(gaps[0])).toEqual({ type: "amicode.sync.gap", namespaces: ["studio"] }); + expect(gaps[0].split("\n").some((l) => l.startsWith("id:"))).toBe(false); // id-less + expect(agg.cursor()).toBe(cursorBefore); // a gap signal never advances a cursor + }); + } + + it("an arm that delivered only a comment frame (no data line) has not delivered — its re-open is still an initial open", () => { + const sink = collectingSink(); + const agg = new SseFanInAggregator({ sink }); + agg.connect(); + agg.setMembership([{ machineId: "studio", reachable: true, token: "tok-studio" }]); + agg.ingest("studio", frame(": keep-alive")); + expect(agg.reopenArm("studio")).toBe("open"); + expect(sink.text()).not.toContain("amicode.sync.gap"); + }); + + it("the LOCAL arm (legacy engine) in composite mode gaps on re-open too, naming `local`", () => { + const sink = collectingSink(); + const agg = new SseFanInAggregator({ sink }); + agg.connect(); + agg.setMembership([{ machineId: "studio", reachable: true, token: "tok-studio" }]); + agg.ingest(LOCAL_NAMESPACE, fakeEngine({ legacy: true }).connected()); + expect(agg.reopenArm(LOCAL_NAMESPACE)).toBe("gap"); + const gaps = gapFrames(sink.text()); + expect(gaps).toHaveLength(1); + expect(gapOf(gaps[0]).namespaces).toEqual([LOCAL_NAMESPACE]); + }); +}); + describe("#1638 — resume() no-op WHEN ALREADY FLOWING (not when the sink is full); #1617 drain-flush preserved", () => { function controllableSink() { const chunks: string[] = []; diff --git a/packages/extension/test/sse_fanin_driver_pump.test.ts b/packages/extension/test/sse_fanin_driver_pump.test.ts index ec1d3f358..27a047dc4 100644 --- a/packages/extension/test/sse_fanin_driver_pump.test.ts +++ b/packages/extension/test/sse_fanin_driver_pump.test.ts @@ -698,13 +698,17 @@ describe("#1638 — re-open resumes from last-delivered id + composite-mode reco res.fireClose() }) - it("COMPOSITE MODE: re-opening a dropped peer arm emits exactly one amicode.sync.gap naming it", async () => { + // #1773 AC1: in BOTH engine modes — an older peer engine sends no `id:` lines, + // and its re-open must gap all the same ("delivered" = any data frame). + for (const mode of ["sequenced", "legacy"] as const) + it(`COMPOSITE MODE (${mode} peer engine): re-opening a dropped peer arm emits exactly one amicode.sync.gap naming it`, async () => { const ownerMap = new SessionOwnerMap() ownerMap.update([ { id: "ses-studio", amicode_owner: { owner_machine_id: "studio", owner_name: "studio", is_local: false } }, ]) - const P1 = peerEvt(1, { from: "studio", n: 1 }) + const studio = mode === "legacy" ? fakeEngine({ legacy: true }) : fakeEngine({ epoch: PEER_EPOCH }) + const P1 = studio.event("message.part.updated", { from: "studio", n: 1 }) const localCtl = controllableSource() const peerCtls: Ctl[] = [] From 7034b062e1316d6f59bcd34c039e6a78af6c017d Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 19:31:35 -0400 Subject: [PATCH 30/60] =?UTF-8?q?fix(fleet):=20a=20failed=20peer=20refresh?= =?UTF-8?q?=20keeps=20that=20peer's=20owner=20entries=20=E2=80=94=20routin?= =?UTF-8?q?g=20and=20its=20event=20arm=20unchanged=20(#1774)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../amicode_service/session_multiplexer.ts | 154 ++++++++++++++- ...de_service_multiplex_owner_routing.test.ts | 182 +++++++++++++++++- ...icode_service_observe_read_routing.test.ts | 34 ++++ ...code_service_observe_write_routing.test.ts | 35 ++++ 4 files changed, 397 insertions(+), 8 deletions(-) diff --git a/packages/extension/src/amicode_service/session_multiplexer.ts b/packages/extension/src/amicode_service/session_multiplexer.ts index 3cb9fd755..a98486944 100644 --- a/packages/extension/src/amicode_service/session_multiplexer.ts +++ b/packages/extension/src/amicode_service/session_multiplexer.ts @@ -77,6 +77,12 @@ export function extractSessionIdFromReadPath(pathname: string): string | undefin * never really created, or was deleted) and drops back to local. */ export const PENDING_OWNER_TTL_MS = 60_000; +/** #1774: how long a peer's last-good owner entries survive refreshes in which + * that peer's fetch FAILED (timeout / network error / 5xx). A peer's session set + * changes slowly relative to this; past it the entries expire. Configurable per + * map (`retentionMs`). */ +export const OWNER_RETENTION_MS = 60_000; + /** A session entry with owner tag (the fleet-wide projection's shape). */ export interface SessionEntry { id: string; @@ -84,6 +90,38 @@ export interface SessionEntry { [key: string]: unknown; } +/** #1774: does a failed peer refresh RETAIN the peer's last-good entries? Decided + * from #1771's per-peer outcome: a timeout, or a transport failure (a network + * error, a 5xx, the peer's upstream momentarily absent) retains. A credential or + * trust problem — unauthorized, untrusted, credential missing, identity + * conflict — does not: it is not a blip, and it surfaces as unavailable. */ +export function refreshRetains(peer: Pick): boolean { + if (peer.outcome === "timeout") return true; + if (peer.outcome !== "error") return false; + return peer.reason === "fetch-failed" || peer.reason === "no-upstream"; +} + +/** #1774: what one refresh did to ONE peer's owner entries (for the refresh log). + * - retained — the fetch failed within the window: `held` last-good entries + * kept, last confirmed `ageMs` ago; + * - expired — the window lapsed: `dropped` entries expired, the peer is now + * named unavailable; + * - dropped — a non-retaining failure (unauthorized, …): `dropped` entries + * removed at once, the peer is now named unavailable; + * - unavailable — the peer holds nothing and is still named unavailable. + * A successful refresh (or a failure with nothing ever held) has no record. */ +export type OwnerRetention = + | { kind: "retained"; held: number; ageMs: number } + | { kind: "expired"; dropped: number } + | { kind: "dropped"; dropped: number } + | { kind: "unavailable" }; + +/** One owned entry as the map holds it (the owner is the map key's peer). */ +interface HeldEntry { + id: string; + directory?: string; +} + /** The per-session owner binding — maps session IDs to their owner machine_id. * Populated from the fleet-wide projection's amicode_owner overlay (slice 2). * A miss returns undefined, which the multiplexer interprets as "local". @@ -92,7 +130,16 @@ export interface SessionEntry { * a peer is not yet in the pull-only projection, so a PENDING overlay bridges * the gap — `recordPendingOwner` binds the new id at create time, the binding * survives projection refreshes that don't yet include it, is superseded once - * the projection does, and expires after a TTL. */ + * the projection does, and expires after a TTL. + * + * #1774: when the projection carries #1771's PER-PEER refresh outcomes, the + * update is authoritative only for the peers that answered. A peer whose fetch + * failed (an empty slice, not an error) keeps its LAST-GOOD entries for + * `retentionMs` — its sessions keep routing to it and its fan-in arm stays open + * — then they expire and the peer is NAMED unavailable (still an owner in + * `ownerMachineIds()`, never a reachable arm — see `fanInReachable`) until a + * successful refresh. A non-retaining failure (unauthorized, …) drops the + * entries at once and names the peer unavailable the same way. */ export class SessionOwnerMap { private readonly owners = new Map(); /** #1766: each OWNED session's working directory (id → directory), recorded @@ -101,34 +148,107 @@ export class SessionOwnerMap { private readonly directories = new Map(); /** Create-time bindings awaiting projection confirmation (id → {owner, at}). */ private readonly pending = new Map(); + /** #1774: each peer's entries as of its last SUCCESSFUL refresh (peer → {at, entries}). */ + private readonly lastGood = new Map(); + /** #1774: peers whose entries expired or were dropped — named unavailable. */ + private readonly unavailablePeers = new Set(); private readonly now: () => number; + private readonly retentionMs: number; - constructor(opts: { now?: () => number } = {}) { + constructor(opts: { now?: () => number; retentionMs?: number } = {}) { this.now = opts.now ?? Date.now; + this.retentionMs = opts.retentionMs ?? OWNER_RETENTION_MS; } /** Rebuild the map from a fleet-wide projection's session list. The projection * is authoritative: a pending binding now present in it is superseded (dropped - * from the overlay); expired pending bindings are pruned. */ - update(sessions: SessionEntry[]): void { + * from the overlay); expired pending bindings are pruned. + * + * #1774: `peers` (the projection's per-peer refresh outcomes) scopes that + * authority — see the class comment. Returns what the update did to each + * failed peer's entries (absent for a peer that answered). Without `peers` + * every owner is replaced, exactly as before. */ + update( + sessions: SessionEntry[], + peers?: ReadonlyArray>, + ): Map { this.owners.clear(); this.directories.clear(); + const fresh = new Map(); for (const s of sessions) { if (typeof s.id === "string" && s.amicode_owner?.owner_machine_id) { - this.owners.set(s.id, s.amicode_owner.owner_machine_id); + const owner = s.amicode_owner.owner_machine_id; // #1766: the session record's own `directory` (the cross-project list // carries it verbatim); the owner tag mirrors the same value, so it is // the fallback when an entry carries no top-level field. const directory = typeof s.directory === "string" && s.directory !== "" ? s.directory : s.amicode_owner.directory; - if (typeof directory === "string" && directory !== "") this.directories.set(s.id, directory); + const held: HeldEntry = { id: s.id, ...(typeof directory === "string" && directory !== "" ? { directory } : {}) }; + this.hold(owner, held); + const list = fresh.get(owner); + if (list) list.push(held); + else fresh.set(owner, [held]); } } + const retention = peers === undefined ? this.replaceAll(fresh) : this.applyPeerOutcomes(fresh, peers); // Prune the pending overlay: drop entries the authoritative projection now // carries (superseded), and any that have outlived the TTL. const cutoff = this.now() - PENDING_OWNER_TTL_MS; for (const [id, entry] of this.pending) { if (this.owners.has(id) || entry.at < cutoff) this.pending.delete(id); } + return retention; + } + + private hold(owner: string, e: HeldEntry): void { + this.owners.set(e.id, owner); + if (e.directory !== undefined) this.directories.set(e.id, e.directory); + } + + /** No per-peer outcomes: the projection is authoritative for every owner. */ + private replaceAll(fresh: Map): Map { + this.lastGood.clear(); + this.unavailablePeers.clear(); + const at = this.now(); + for (const [owner, entries] of fresh) this.lastGood.set(owner, { at, entries }); + return new Map(); + } + + /** #1774: per peer — an answer replaces, a retaining failure keeps the + * last-good entries inside the window, anything else drops them. */ + private applyPeerOutcomes( + fresh: Map, + peers: ReadonlyArray>, + ): Map { + const now = this.now(); + const out = new Map(); + const refreshed = new Set(peers.map((p) => p.machineId)); + // A peer no longer refreshed at all left the fleet's serving set: nothing + // of it is retained, and it is not unavailable — it is gone. + for (const id of [...this.lastGood.keys()]) if (!refreshed.has(id)) this.lastGood.delete(id); + for (const id of [...this.unavailablePeers]) if (!refreshed.has(id)) this.unavailablePeers.delete(id); + for (const peer of peers) { + const id = peer.machineId; + if (peer.outcome === "ok") { + const entries = fresh.get(id) ?? []; + if (entries.length > 0) this.lastGood.set(id, { at: now, entries }); + else this.lastGood.delete(id); + this.unavailablePeers.delete(id); + continue; + } + const last = this.lastGood.get(id); + if (last && refreshRetains(peer) && now - last.at <= this.retentionMs) { + // keep what this refresh did not re-confirm; never shadow a fresh entry + for (const e of last.entries) if (!this.owners.has(e.id)) this.hold(id, e); + out.set(id, { kind: "retained", held: last.entries.length, ageMs: now - last.at }); + } else if (last) { + this.lastGood.delete(id); + this.unavailablePeers.add(id); + out.set(id, refreshRetains(peer) ? { kind: "expired", dropped: last.entries.length } : { kind: "dropped", dropped: last.entries.length }); + } else if (this.unavailablePeers.has(id)) { + out.set(id, { kind: "unavailable" }); + } + } + return out; } /** Bind a just-created session to its owner peer BEFORE the projection catches @@ -170,6 +290,12 @@ export class SessionOwnerMap { return [...new Set(this.owners.values())]; } + /** #1774: is this peer NAMED unavailable — its retention window lapsed, or its + * refresh was refused — with no successful refresh since? */ + isUnavailable(machineId: string): boolean { + return this.unavailablePeers.has(machineId); + } + /** The number of tracked sessions (for diagnostics). */ get size(): number { return this.owners.size; @@ -178,6 +304,18 @@ export class SessionOwnerMap { // ── owner-map feed (#1449, W1b) ────────────────────────────────────────────── +/** #1774: the fan-in driver's per-owner reachability, composed over the owner + * map — an owner-peer the map NAMES unavailable is never an active arm, whatever + * its transport says, so the aggregator closes its arm with the existing named + * unavailable comment instead of dropping it in silence. Both fan-in drivers in + * index.ts are wired with this. */ +export function fanInReachable( + ownerMap: SessionOwnerMap, + transportUp: (machineId: string) => boolean, +): (machineId: string) => boolean { + return (machineId) => !ownerMap.isUnavailable(machineId) && transportUp(machineId); +} + /** The live loop that keeps a SessionOwnerMap populated (#1449, W1b, AC1). * * The fleet-wide projection is PULL-ONLY — it is rebuilt per request at the @@ -238,7 +376,9 @@ export class OwnerMapFeed { return; // last-good map intact — never clear on a transient failure } if (this.stopped) return; // a stop() raced the in-flight build - this.opts.ownerMap.update(projection.sessions ?? []); + // #1774: the per-peer outcomes scope the update — a failed peer keeps its + // last-good entries for the retention window instead of vanishing. + this.opts.ownerMap.update(projection.sessions ?? [], projection.peers); for (const peer of projection.peers ?? []) this.log(ownerRefreshLine(peer)); } diff --git a/packages/extension/test/amicode_service_multiplex_owner_routing.test.ts b/packages/extension/test/amicode_service_multiplex_owner_routing.test.ts index e55fc8889..b3888d569 100644 --- a/packages/extension/test/amicode_service_multiplex_owner_routing.test.ts +++ b/packages/extension/test/amicode_service_multiplex_owner_routing.test.ts @@ -26,8 +26,11 @@ import { SessionOwnerMap, SessionMultiplexProxy, PENDING_OWNER_TTL_MS, + fanInReachable, type SessionEntry, } from "../src/amicode_service/session_multiplexer"; +import { SseFanInDriver, type FanInUpstreamRequest } from "../src/amicode_service/sse_fanin_driver"; +import type { SseFrameSource } from "../src/amicode_service/sse_fanin_aggregator"; import { AmicodeServiceServer, fleetMultiplexEnabled, @@ -40,7 +43,7 @@ import { EngineProxy } from "../src/amicode_service/engine_proxy"; import { writeAttachmentPointerFile } from "../src/amicode_service/attachment_pointer"; import { writeKeeperPointerFile } from "../src/amicode_service/keeper_pointer"; import { writeHubCredential, readHubCredential } from "../src/amicode_service/hub_credential"; -import { buildOwnerRoutingProjection } from "../src/amicode_service/merged_projection"; +import { buildOwnerRoutingProjection, type OwnerRefreshPeer } from "../src/amicode_service/merged_projection"; import { serverAuthHeader } from "../src/server_auth"; // ── a marker stub: returns a distinct marker for every request; records paths ── @@ -229,6 +232,183 @@ describe("#1771 — each ownership refresh logs a per-peer outcome record", () = }); }); +// ══════════════════════════════════════════════════════════════════════════════ +// #1774 — the ownership map keeps a peer's sessions across a failed refresh. A +// failed peer fetch is an EMPTY slice in the projection (the builder never +// throws), so before this the peer's sessions vanished from the map on one blip: +// its event arm closed without notice and its sessions routed local (404/empty). +// Retention is decided PER PEER from #1771's refresh outcome — timeouts, network +// errors and 5xx retain; an unauthorized answer does not (a credential problem). +// ══════════════════════════════════════════════════════════════════════════════ +const ownedBy = (id: string, owner: string, directory?: string): SessionEntry => ({ + id, + ...(directory !== undefined ? { directory } : {}), + amicode_owner: { owner_machine_id: owner, owner_name: owner, is_local: false }, +}); +const peerOk = (machineId: string, entries = 1): OwnerRefreshPeer => ({ machineId, outcome: "ok", durationMs: 40, entries }); +const peerTimeout = (machineId: string): OwnerRefreshPeer => ({ + machineId, + outcome: "timeout", + durationMs: 10_004, + entries: 0, + reason: "fetch-failed", + detail: "The operation was aborted due to timeout", +}); +const peer5xx = (machineId: string): OwnerRefreshPeer => ({ machineId, outcome: "error", durationMs: 12, entries: 0, reason: "fetch-failed", detail: "HTTP 502" }); +const peerNetErr = (machineId: string): OwnerRefreshPeer => ({ machineId, outcome: "error", durationMs: 3, entries: 0, reason: "fetch-failed", detail: "fetch failed" }); +const peerUnauthorized = (machineId: string): OwnerRefreshPeer => ({ machineId, outcome: "unauthorized", durationMs: 9, entries: 0, reason: "unauthorized" }); + +describe("#1774 AC1 — a refresh in which one peer times out or errors keeps that peer's previous owner entries", () => { + for (const [label, failed] of [ + ["a timeout", peerTimeout("studio")], + ["a 5xx answer", peer5xx("studio")], + ["a network error", peerNetErr("studio")], + ] as const) { + it(`${label}: the peer's sessions still resolve to it (owner + directory); the healthy peer is replaced as usual`, async () => { + const ownerMap = new SessionOwnerMap(); + let next: { sessions: SessionEntry[]; peers: OwnerRefreshPeer[] } = { + sessions: [ownedBy("ses-s1", "studio", "/srv/a"), ownedBy("ses-s2", "studio"), ownedBy("ses-m1", "mini")], + peers: [peerOk("studio", 2), peerOk("mini")], + }; + const feed = new OwnerMapFeed({ ownerMap, buildProjection: async () => next }); + await feed.refreshOnce(); + // studio's fetch fails: its slice is EMPTY in this projection; mini answers. + next = { sessions: [ownedBy("ses-m2", "mini")], peers: [failed, peerOk("mini")] }; + await feed.refreshOnce(); + expect(ownerMap.resolveOwner("ses-s1")).toBe("studio"); + expect(ownerMap.resolveOwner("ses-s2")).toBe("studio"); + expect(ownerMap.resolveDirectory("ses-s1")).toBe("/srv/a"); + expect(ownerMap.ownerMachineIds().sort()).toEqual(["mini", "studio"]); + // the healthy peer's refresh is still authoritative + expect(ownerMap.resolveOwner("ses-m1")).toBeUndefined(); + expect(ownerMap.resolveOwner("ses-m2")).toBe("mini"); + }); + } + + it("through the REAL routing projection: a peer whose session list answers 502 keeps its owner entries", async () => { + let studioDown = false; + const fetchImpl = (async (input: RequestInfo | URL) => { + const url = String(input); + if (url.startsWith("http://studio") && studioDown) return new Response("bad gateway", { status: 502 }); + if (url.endsWith("/global/health")) return new Response(JSON.stringify({ version: "t" }), { status: 200 }); + if (url.startsWith("http://studio")) return new Response(JSON.stringify(/archived=true/.test(url) ? [] : [{ id: "ses-s1" }]), { status: 200 }); + return new Response(JSON.stringify([]), { status: 200 }); // local + }) as unknown as typeof fetch; + const ownerMap = new SessionOwnerMap(); + const feed = new OwnerMapFeed({ + ownerMap, + buildProjection: async () => { + const p = await buildOwnerRoutingProjection({ + localMachineId: "macbook", + local: { getUrl: () => "http://local" }, + peers: [{ machineId: "studio", getUrl: () => "http://studio", token: "tok" }], + rosterLookup: () => undefined, + fetchImpl, + }); + return { sessions: p.sessions as SessionEntry[], peers: p.peers }; + }, + }); + await feed.refreshOnce(); + expect(ownerMap.resolveOwner("ses-s1")).toBe("studio"); + studioDown = true; + await feed.refreshOnce(); + expect(ownerMap.resolveOwner("ses-s1")).toBe("studio"); // retained, not dropped to local + }); + + it("its event arm stays OPEN across the failed refresh — no close, no unavailable comment", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update([ownedBy("ses-s1", "studio")], [peerOk("studio")]); + const origins = armOrigins(); + const driver = retentionDriver(origins, ownerMap); + const res = sseRes(); + driver.handle(sseReq(), res); + await settle(); + expect(origins.opens("studio")).toBe(1); + + ownerMap.update([], [peerTimeout("studio")]); // studio's slice is empty this refresh + driver.reconcile(); + await settle(); + expect(origins.closes("studio")).toBe(0); + expect(origins.opens("studio")).toBe(1); // the same arm, never re-opened + expect(res.text()).not.toContain("unavailable"); + res.fireClose(); + }); +}); + +// ── a deterministic fan-in harness (the driver + aggregator are used as-is) ─── +interface SseRes extends http.ServerResponse { + text(): string; + fireClose(): void; +} +function sseRes(): SseRes { + const chunks: string[] = []; + const closers: Array<() => void> = []; + const res = { + headersSent: false, + write(c: string) { + chunks.push(c); + return true; + }, + writeHead() { + res.headersSent = true; + return res; + }, + flushHeaders() {}, + on(event: string, cb: () => void) { + if (event === "close") closers.push(cb); + return res; + }, + end() {}, + text: () => chunks.join(""), + fireClose: () => closers.forEach((cb) => cb()), + }; + return res as unknown as SseRes; +} +const sseReq = (): http.IncomingMessage => ({ url: "/global/event", headers: {} }) as unknown as http.IncomingMessage; +/** Let the arm pumps drain (microtasks + one macrotask tick). */ +const settle = () => new Promise((r) => setTimeout(r, 20)); + +/** Upstream arms that stay open until closed, counted per namespace. */ +function armOrigins() { + const opens = new Map(); + const closes = new Map(); + const bump = (m: Map, k: string) => m.set(k, (m.get(k) ?? 0) + 1); + const openUpstream = (r: FanInUpstreamRequest): SseFrameSource => { + bump(opens, r.namespace); + const waiters: Array<(v: string | null) => void> = []; + let ended = false; + return { + next: () => (ended ? Promise.resolve(null) : new Promise((resolve) => waiters.push(resolve))), + close: () => { + if (ended) return; + ended = true; + bump(closes, r.namespace); + waiters.splice(0).forEach((w) => w(null)); + }, + }; + }; + return { openUpstream, opens: (ns: string) => opens.get(ns) ?? 0, closes: (ns: string) => closes.get(ns) ?? 0 }; +} + +/** The fan-in driver as index.ts wires it: reachability composed over the owner + * map (`fanInReachable`), so a peer the map names unavailable is never an arm. */ +function retentionDriver(origins: ReturnType, ownerMap: SessionOwnerMap): SseFanInDriver { + const peerBaseUrl = (id: string) => (id === "studio" ? "http://studio.invalid" : undefined); + return new SseFanInDriver({ + ownerMap, + localMachineId: "macbook", + localEventUrl: () => "http://local.invalid", + peerBaseUrl, + peerToken: (id) => + id === "studio" + ? { ok: true as const, credential: { baseUrl: "http://studio.invalid", token: "tok-studio" } } + : { ok: false as const, reason: "absent" as const }, + reachable: fanInReachable(ownerMap, (id) => peerBaseUrl(id) !== undefined), + reconcileMs: 999_999, + openUpstream: origins.openUpstream, + }); +} + // ══════════════════════════════════════════════════════════════════════════════ // FREE-TIER REMOTE CREATE (feature/free-tier-fleet) — pending create-time owner // bindings. A session just created on a peer is not yet in the pull-only fleet diff --git a/packages/extension/test/amicode_service_observe_read_routing.test.ts b/packages/extension/test/amicode_service_observe_read_routing.test.ts index 795c25362..2835f50bf 100644 --- a/packages/extension/test/amicode_service_observe_read_routing.test.ts +++ b/packages/extension/test/amicode_service_observe_read_routing.test.ts @@ -279,6 +279,40 @@ describe("#1537 AC5 — a known-but-unreachable owner answers honestly, never lo }); }); +// ══════════════════════════════════════════════════════════════════════════════ +// #1774 AC1 — a refresh in which the owner peer timed out keeps its sessions +// routed to it: the read is proxied to the peer, never to the local engine. +// ══════════════════════════════════════════════════════════════════════════════ +describe("#1774 AC1 — a peer-owned read keeps routing to the owner across a failed refresh", () => { + it("GET for a session whose owner's last refresh timed out → still the PEER; local NOT dialed", async () => { + const ownerMap = new SessionOwnerMap(); + const studioOwned = { id: "ses-studio", amicode_owner: { owner_machine_id: "studio", owner_name: "studio", is_local: false } }; + ownerMap.update([studioOwned], [{ machineId: "studio", outcome: "ok" }]); + ownerMap.update([], [{ machineId: "studio", outcome: "timeout", reason: "fetch-failed" }]); // the slice is empty + const server = new AmicodeServiceServer({ password: PW }); + server.attachEngineProxy(new EngineProxy({ getUrl: () => localStub.url })); + server.attachObservationReadPlane( + createObservationReadPlane({ + ownerMap, + localMachineId: "macbook", + peer: (id) => (id === "studio" ? { getUrl: () => peerStub.url, token: "tok-studio" } : undefined), + }), + ); + const origin = (await server.start()).toString().replace(/\/$/, ""); + const localBefore = localStub.requests.length; + const peerBefore = peerStub.requests.length; + try { + const res = await fetch(`${origin}/api/session/ses-studio`, { headers: authed }); + expect(res.status).toBe(200); + expect(((await res.json()) as { marker: string }).marker).toBe(peerStub.marker); + expect(peerStub.requests.length).toBe(peerBefore + 1); + expect(localStub.requests.length).toBe(localBefore); + } finally { + await server.stop(); + } + }); +}); + // ══════════════════════════════════════════════════════════════════════════════ // AC6 — production wiring: createAmicodeService on the OBSERVATION-ONLY path // wires the seam (a peer-owned read routes to the peer); the armed path stays diff --git a/packages/extension/test/amicode_service_observe_write_routing.test.ts b/packages/extension/test/amicode_service_observe_write_routing.test.ts index ecd481666..f6d372b3e 100644 --- a/packages/extension/test/amicode_service_observe_write_routing.test.ts +++ b/packages/extension/test/amicode_service_observe_write_routing.test.ts @@ -747,6 +747,41 @@ describe("#1542 AC4 — local-owned / unowned writes fall through byte-identical }); }); +// ══════════════════════════════════════════════════════════════════════════════ +// #1774 AC1 — a refresh in which the owner peer errored (5xx) keeps its sessions +// routed to it: the authorized write is proxied to the peer, never run locally. +// ══════════════════════════════════════════════════════════════════════════════ +describe("#1774 AC1 — a peer-owned write keeps routing to the owner across a failed refresh", () => { + it("POST for a session whose owner's last refresh answered 502 → still the PEER; local NOT dialed", async () => { + const ownerMap = new SessionOwnerMap(); + const studioOwned = { id: "ses-studio", amicode_owner: { owner_machine_id: "studio", owner_name: "studio", is_local: false } }; + ownerMap.update([studioOwned], [{ machineId: "studio", outcome: "ok" }]); + ownerMap.update([], [{ machineId: "studio", outcome: "error", reason: "fetch-failed" }]); // HTTP 502: an empty slice + const peer = (id: string): PeerTransport | undefined => (id === "studio" ? { getUrl: () => peerStub.url, token: "tok-studio" } : undefined); + const server = new AmicodeServiceServer({ password: PW }); + server.attachEngineProxy(new EngineProxy({ getUrl: () => localStub.url })); + server.attachObservationReadPlane(createObservationReadPlane({ ownerMap, localMachineId: "macbook", peer })); + server.attachObservationWritePlane( + createObservationWritePlane({ ownerMap, localMachineId: "macbook", peer, grantReader: (o) => (o === "studio" ? { scope: "control", state: "active" } : undefined) }), + ); + const origin = (await server.start()).toString().replace(/\/$/, ""); + const localBefore = localStub.requests.length; + const peerBefore = peerStub.requests.length; + try { + const res = await fetch(`${origin}/api/session/ses-studio/message`, { + method: "POST", + headers: { ...authed, "content-type": "application/json" }, + body: JSON.stringify({ text: "hi" }), + }); + expect(((await res.json()) as { marker: string }).marker).toBe(peerStub.marker); + expect(peerStub.requests.slice(peerBefore).some((r) => r.method === "POST" && r.path === "/api/session/ses-studio/message")).toBe(true); + expect(localStub.requests.length).toBe(localBefore); + } finally { + await server.stop(); + } + }); +}); + // ══════════════════════════════════════════════════════════════════════════════ // Byte-identity when UNATTACHED — a server with NO write plane consults nothing: // a POST for a peer-owned session falls through to the local engine (the From ac38b7476054d26550916ce073bb98761a1f0837 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 19:35:57 -0400 Subject: [PATCH 31/60] fix(fleet): fleet-of-one local-arm loss severs the listener with a transport error, never a synthetic frame (#1773) --- .../src/amicode_service/sse_fanin_driver.ts | 53 ++-- .../test/sse_fanin_driver_pump.test.ts | 251 ++++++++++++++++-- 2 files changed, 270 insertions(+), 34 deletions(-) diff --git a/packages/extension/src/amicode_service/sse_fanin_driver.ts b/packages/extension/src/amicode_service/sse_fanin_driver.ts index 9b7c37a67..b15faee76 100644 --- a/packages/extension/src/amicode_service/sse_fanin_driver.ts +++ b/packages/extension/src/amicode_service/sse_fanin_driver.ts @@ -167,7 +167,10 @@ export function openHttpSseSource(r: FanInUpstreamRequest): SseFrameSource { * aggregator on this response, the open upstream sources keyed by namespace, * and this connection's own reconcile timer. Closing it closes only its arms. * #1771: it is also the census's view of this listener — per-arm opens and - * closes-by-reason live here, traffic/pending/drops/gaps in the aggregator. */ + * closes-by-reason live here, traffic/pending/drops/gaps in the aggregator. + * #1773: it never ends its listener silently — every close except a graceful + * service shutdown is an ERROR close (the listener sees a transport error and + * resyncs), and a fleet-of-one local-arm loss severs the listener that way. */ class FanInConnection implements CensusConnection { private readonly agg: SseFanInAggregator; private readonly sources = new Map(); @@ -178,6 +181,9 @@ class FanInConnection implements CensusConnection { private closed = false; /** #1771 — per arm: how often it opened, and why it closed. */ private readonly armLife = new Map }>(); + /** #1773 — per arm: the reason of its most recent close (a severed + * listener's close is named after the loss that caused it). */ + private readonly lastClose = new Map(); /** #1771 — sources THIS connection closed (their reason is already recorded; * the pump's exit must not re-count them as an upstream end). */ private readonly closedByUs = new WeakSet(); @@ -188,6 +194,8 @@ class FanInConnection implements CensusConnection { private readonly census: FanInCensus, private readonly req: http.IncomingMessage, private readonly res: http.ServerResponse, + /** Invoked exactly once, when this connection closes (for any reason). */ + private readonly onClosed: (reason: FanInCloseReason) => void, ) { this.open = deps.openUpstream ?? openHttpSseSource; const a = req.headers.authorization; @@ -240,6 +248,7 @@ class FanInConnection implements CensusConnection { /** #1771 — record (and log) an arm's close, once. */ private noteArmClosed(namespace: string, reason: FanInCloseReason): void { this.lifeOf(namespace).closes[reason] += 1; + this.lastClose.set(namespace, reason); this.census.armClosed(this.id, namespace, reason); } @@ -376,13 +385,20 @@ class FanInConnection implements CensusConnection { /** #1638 — re-open an arm from the aggregator's LIVE last-delivered cursor (not * the stale connect-time id) and, on a NON-INITIAL re-open, force a client * refetch via the aggregator's composite-mode gap. `agg.reopenArm` is a no-op - * for an arm that has never delivered (an initial open) and in fleet-of-one - * (where a synthetic frame would break the #1264 byte-identity guard), so it - * is safe to call at every reconcile-time open. Falls back to the connect-time - * resume id when the arm has delivered nothing yet. */ + * for an arm that has never delivered (an initial open), so it is safe to call + * at every reconcile-time open. Falls back to the connect-time resume id when + * the arm has delivered nothing yet. + * + * #1773 — in fleet-of-one the verdict for an arm that delivered is `sever`: a + * synthetic gap frame may not enter the verbatim stream (#1264), so instead of + * re-opening silently (losing the stretch unannounced) the listener is ended + * with a transport error, named after the loss that caused it. */ private reopenAndTrack(namespace: string, url: string | undefined, authHeader: string | undefined): void { if (this.sources.has(namespace)) return; - this.agg.reopenArm(namespace); + if (this.agg.reopenArm(namespace) === "sever") { + this.close(this.lastClose.get(namespace) ?? "upstream-end"); + return; + } const lastEventId = this.agg.cursorFor(namespace) ?? this.resume.get(namespace); this.openArm(namespace, url, authHeader, lastEventId); } @@ -443,6 +459,11 @@ class FanInConnection implements CensusConnection { this.sources.delete(namespace); } + /** Close this listener's connection: stop its reconcile, close its arms, and + * end its response. #1773 — only a graceful service SHUTDOWN ends the stream + * cleanly; every other reason ERRORS it (`res.destroy(err)`), so the listener + * — or the relay in front of it — sees a transport error, never a clean end + * that reads as "nothing was lost". Idempotent: the first reason wins. */ close(reason: FanInCloseReason = "listener-closed"): void { if (this.closed) return; this.closed = true; @@ -453,10 +474,12 @@ class FanInConnection implements CensusConnection { for (const [namespace, s] of this.sources) this.closeSource(namespace, s, reason); this.sources.clear(); try { - this.res.end(); + if (reason === "shutdown") this.res.end(); + else this.res.destroy(new Error(`amicode fan-in: listener stream closed (${reason}); resync required`)); } catch { /* already torn down */ } + this.onClosed(reason); } } @@ -516,20 +539,20 @@ export class SseFanInDriver implements EventFanInDriver { } // #1565: always accept on the observation path. Zero non-local owners starts // in fleet-of-one mode (the aggregator's §D4 byte-identity relay); reconcile - // opens peer arms as the OwnerMapFeed discovers them. - const conn = new FanInConnection(`c${++this.nextConnection}`, this.deps, this.census, req, res); + // opens peer arms as the OwnerMapFeed discovers them. #1773: a connection + // can also close ITSELF (a fleet-of-one local-arm loss severs the listener); + // either way it leaves the live set and the census once, under its reason. + const conn: FanInConnection = new FanInConnection(`c${++this.nextConnection}`, this.deps, this.census, req, res, (reason) => { + this.connections.delete(conn); + this.census.listenerClosed(conn.id, reason); + }); this.connections.add(conn); this.census.listenerAccepted(conn); conn.start(); // A departing listener closes ONLY its own connection (and so only its own // upstream arms); every other listener keeps streaming. #1771: a close the // service's own teardown caused is attributed to `shutdown`. - res.on("close", () => { - this.connections.delete(conn); - const reason: FanInCloseReason = this.shuttingDown ? "shutdown" : "listener-closed"; - conn.close(reason); - this.census.listenerClosed(conn.id, reason); - }); + res.on("close", () => conn.close(this.shuttingDown ? "shutdown" : "listener-closed")); return true; } diff --git a/packages/extension/test/sse_fanin_driver_pump.test.ts b/packages/extension/test/sse_fanin_driver_pump.test.ts index 27a047dc4..4e42c5d58 100644 --- a/packages/extension/test/sse_fanin_driver_pump.test.ts +++ b/packages/extension/test/sse_fanin_driver_pump.test.ts @@ -36,24 +36,28 @@ const peerEvt = (seq: number, properties: Record, type?: string const localId = (seq: number) => engineId(LOCAL_EPOCH, seq); const peerId = (seq: number) => engineId(PEER_EPOCH, seq); -/** A controllable frame source: push frames into it, end it on demand. */ +/** A controllable frame source: push frames into it, end it on demand. #1773: + * `end("error")` models an upstream that FAILED (the real reader's + * `endReason()`); a bare `end()` is a clean upstream end. */ interface Ctl { push(frame: string): void; - end(): void; + end(why?: "end" | "error"): void; source: SseFrameSource; } function controllableSource(): Ctl { const queue: string[] = []; const waiters: Array<(v: string | null) => void> = []; let ended = false; + let reason: "end" | "error" | undefined; const push = (f: string): void => { const w = waiters.shift(); if (w) w(f); else queue.push(f); }; - const end = (): void => { + const end = (why: "end" | "error" = "end"): void => { if (ended) return; ended = true; + reason = why; let w: ((v: string | null) => void) | undefined; while ((w = waiters.shift())) w(null); }; @@ -70,17 +74,23 @@ function controllableSource(): Ctl { close(): void { end(); }, + endReason: () => reason, }, }; } /** A mock ServerResponse that collects writes. #1772 — also records whether the * response was ENDED (an evicted listener is ended by the driver) and the - * status/headers/body of a non-stream answer (the beyond-cap refusal). */ + * status/headers/body of a non-stream answer (the beyond-cap refusal). #1773 — + * and whether it was DESTROYED (an error close: the listener sees a transport + * error, not a clean end), with the error; like a real socket teardown, a + * destroy fires the response's `close` listeners. */ type MockRes = http.ServerResponse & { text(): string; fireClose(): void; ended: boolean; + destroyed: boolean; + destroyError?: Error; status?: number; headerMap: Record; body: string; @@ -91,6 +101,8 @@ function mockRes(): MockRes { const res = { headersSent: false, ended: false, + destroyed: false, + destroyError: undefined as Error | undefined, status: undefined as number | undefined, headerMap: {} as Record, body: "", @@ -123,6 +135,13 @@ function mockRes(): MockRes { if (typeof chunk === "string") res.body += chunk; res.ended = true; }, + destroy(err?: Error) { + if (res.destroyed) return res; + res.destroyed = true; + res.destroyError = err; + for (const cb of closeListeners) cb(); + return res; + }, text() { return chunks.join(""); }, @@ -760,11 +779,14 @@ describe("#1638 — re-open resumes from last-delivered id + composite-mode reco res.fireClose() }) - it("FLEET-OF-ONE (zero peers): a re-opened LOCAL arm emits NO gap and stays byte-identical (#1264)", async () => { + // #1773 AC2 supersedes #1638's "the solo-mode arm re-opens silently": a silent + // re-open loses the stretch between the arm's end and its re-subscription with + // nothing to announce it. The #1264 guards stay: no synthetic frame, and the + // stream is byte-identical to what the local source delivered while connected. + it("FLEET-OF-ONE (zero peers): a re-opened LOCAL arm emits NO gap, stays byte-identical (#1264), and severs the listener (#1773)", async () => { const ownerMap = new SessionOwnerMap() // no owners → fleet-of-one const L1 = localEvt(1, { from: "macbook", n: 1 }) - const L2 = localEvt(2, { from: "macbook", n: 2 }) const localCtls: Ctl[] = [] @@ -793,20 +815,22 @@ describe("#1638 — re-open resumes from last-delivered id + composite-mode reco await tick() driver.reconcile() await tick() - // the re-opened arm delivers another frame - localCtls[localCtls.length - 1].push(L2) - await tick() - // NO synthetic frame anywhere; the verbatim stream is exactly the two frames. + // NO synthetic frame anywhere; the verbatim stream is exactly what was delivered. expect(res.text()).not.toContain("amicode.sync.gap") expect(res.text()).not.toContain("local\u001f") - expect(res.text()).toBe(L1 + L2) // byte-identical, in order + expect(res.text()).toBe(L1) // byte-identical + // the loss is announced by severing: a transport error, not a clean end + expect(res.destroyed).toBe(true) + expect(res.destroyError).toBeInstanceOf(Error) + expect(res.ended).toBe(false) + expect(localCtls).toHaveLength(1) // and the severed listener opens nothing more localCtls.forEach((c) => c.end()) res.fireClose() }) - it("a re-opened LOCAL arm in fleet-of-one resumes from the last-delivered id too", async () => { + it("fleet-of-one cursor bookkeeping: a never-delivered arm re-opens from the connect cursor; a severed listener's reconnect resumes from the last-delivered id", async () => { const ownerMap = new SessionOwnerMap() const L1 = localEvt(1, { from: "macbook", n: 1 }) @@ -834,19 +858,203 @@ describe("#1638 — re-open resumes from last-delivered id + composite-mode reco await tick() expect(opened[0].lastEventId).toBe(localId(0)) // initial: connect cursor - localCtls[0].push(L1) + // the arm fails before delivering anything: nothing was lost → a plain re-open, + // still resumed from the connect-time cursor (the #1638 fallback) + localCtls[0].end("error") await tick() - localCtls[0].end() + driver.reconcile() + await tick() + expect(opened).toHaveLength(2) + expect(opened[1].lastEventId).toBe(localId(0)) + expect(res.destroyed).toBe(false) + + // the re-opened arm delivers, then drops → the listener is severed + localCtls[1].push(L1) + await tick() + localCtls[1].end() await tick() driver.reconcile() await tick() + expect(res.destroyed).toBe(true) + expect(opened).toHaveLength(2) + + // the client's cursor is the last SSE id it saw on the verbatim stream … + const lastSeen = [...res.text().matchAll(/^id: (.*)$/gm)].at(-1)?.[1] + expect(lastSeen).toBe(localId(1)) + // … and its reconnect resumes the local arm from exactly that id + const again = mockRes() + driver.handle(mockReq(`/event?lastEventID=${lastSeen}`), again) + await tick() + expect(opened[2].lastEventId).toBe(localId(1)) // last-delivered, not the stale connect id + + localCtls.forEach((c) => c.end()) + again.fireClose() + }) +}) - const localOpens = opened.filter((o) => o.namespace === LOCAL_NAMESPACE) - expect(localOpens[1].lastEventId).toBe(localId(1)) // re-open resumes from last-delivered +// ══════════════════════════════════════════════════════════════════════════════ +// #1773 AC2 — FLEET-OF-ONE loss is signalled by SEVERING, never by a frame. A +// synthetic gap frame may not enter the solo pass-through stream (#1264), so a +// re-opened local arm (one that delivered data) ends the listener's stream with a +// transport error: the client registers the disconnect and resyncs. While +// connected, the stream stays byte-identical to the local source. Engine-shaped +// fixtures in both modes (legacy id-less, sequenced). +// ══════════════════════════════════════════════════════════════════════════════ +describe("#1773 AC2 — fleet-of-one: a re-opened local arm severs the listener with a transport error", () => { + for (const mode of ["legacy", "sequenced"] as const) { + for (const why of ["end", "error"] as const) { + it(`${mode} engine, upstream ${why}: no synthetic frame, byte-identical while connected, then an error close named ${why === "end" ? "upstream-end" : "upstream-error"}`, async () => { + const { census, lines } = testCensus() + const localCtls: Ctl[] = [] + const driver = twoListenerDriver(fakeOrigins(), new SessionOwnerMap(), { + census, + openUpstream: () => { + const ctl = controllableSource() + localCtls.push(ctl) + return ctl.source + }, + }) + const engine = mode === "legacy" ? fakeEngine({ legacy: true }) : fakeEngine({ epoch: LOCAL_EPOCH }) + const res = mockRes() + driver.handle(mockReq("/global/event"), res) + const frames = [ + engine.connected(), + engine.event("session.status", { sessionID: "ses_1", status: { type: "busy" } }), + ...engine.stored("message.part.updated", { part: { id: "prt_1", text: "ciao" } }, { aggregateID: "ses_1" }), + engine.heartbeat(), + ] + for (const f of frames) localCtls[0].push(f) + await flush() + expect(res.text()).toBe(frames.join("")) // connected: byte-identical (#1264) + + localCtls[0].end(why) // the local upstream is lost … + await flush() + expect(res.destroyed).toBe(false) // … nothing is decided until the re-open + driver.reconcile() // the re-open: sever, never re-open silently + await flush() + + expect(res.text()).toBe(frames.join("")) // no synthetic frame was injected + expect(res.destroyed).toBe(true) + expect(res.destroyError).toBeInstanceOf(Error) + expect(res.ended).toBe(false) // an error close, never a clean end + expect(localCtls).toHaveLength(1) // the severed listener re-opened nothing + + // the census names the listener's close by its cause (#1771's set) + const reason = why === "end" ? "upstream-end" : "upstream-error" + expect(lines).toContain(`[fanin] arm-closed plane=observe conn=c1 arm=local reason=${reason}`) + expect(lines).toContain(`[fanin] listener-closed plane=observe conn=c1 reason=${reason} listeners=0`) + expect(census.snapshot()).toMatchObject({ listeners: 0, accepted: 1, closed: 1, evictions: 0 }) + }) + } + } + it("a local arm that never delivered a data frame re-opens normally — no sever (nothing was lost)", async () => { + const localCtls: Ctl[] = [] + const driver = twoListenerDriver(fakeOrigins(), new SessionOwnerMap(), { + openUpstream: () => { + const ctl = controllableSource() + localCtls.push(ctl) + return ctl.source + }, + }) + const res = mockRes() + driver.handle(mockReq("/global/event"), res) + localCtls[0].push(": keep-alive\n\n") // a comment is not data + await flush() + localCtls[0].end("error") + await flush() + driver.reconcile() + await flush() + expect(res.destroyed).toBe(false) + expect(localCtls).toHaveLength(2) + const f = fakeEngine({ legacy: true }).connected() + localCtls[1].push(f) + await flush() + expect(res.text()).toBe(": keep-alive\n\n" + f) localCtls.forEach((c) => c.end()) res.fireClose() }) + + it("severing one listener leaves another listener's stream untouched", async () => { + const localCtls: Ctl[] = [] + const driver = twoListenerDriver(fakeOrigins(), new SessionOwnerMap(), { + openUpstream: () => { + const ctl = controllableSource() + localCtls.push(ctl) + return ctl.source + }, + }) + const engine = fakeEngine({ legacy: true }) + const resA = mockRes() + const resB = mockRes() + driver.handle(mockReq("/global/event"), resA) // localCtls[0] + driver.handle(mockReq("/global/event"), resB) // localCtls[1] + const c = engine.connected() + localCtls[0].push(c) + localCtls[1].push(c) + await flush() + localCtls[0].end() // only A's upstream is lost + await flush() + driver.reconcile() + await flush() + expect(resA.destroyed).toBe(true) + expect(resB.destroyed).toBe(false) + const h = engine.heartbeat() + localCtls[1].push(h) + await flush() + expect(resB.text()).toBe(c + h) + localCtls.forEach((x) => x.end()) + resB.fireClose() + }) + + it("over REAL sockets: the listener's response errors (no clean end) after exactly the frames the local engine sent", async () => { + const engine = fakeEngine({ legacy: true }) // an old engine: id-less frames + const sent = [engine.connected(), engine.event("session.status", { sessionID: "ses_1", status: { type: "idle" } })] + const upstream = await startEngineSseServer((res) => { + for (const f of sent) res.write(f) + setImmediate(() => res.end()) // the local engine ends its stream (a restart) + }) + const lines: string[] = [] + const w = lineWaiter(lines) + const driver = new SseFanInDriver({ + ownerMap: new SessionOwnerMap(), + localMachineId: "macbook", + localEventUrl: () => upstream.url, + peerBaseUrl: () => undefined, + peerToken: () => ({ ok: false as const, reason: "absent" as const }), + reconcileMs: 999_999, + census: new FanInCensus({ plane: "observe", sink: w.sink }), + }) + const front = http.createServer((req, res) => void driver.handle(req, res)) + await new Promise((r) => front.listen(0, "127.0.0.1", r)) + const { port } = front.address() as import("node:net").AddressInfo + try { + let body = "" + let haveAll: () => void = () => undefined + const received = new Promise((r) => (haveAll = r)) + const outcome = new Promise<"end" | "error">((resolve) => { + http.get(`http://127.0.0.1:${port}/global/event`, (res) => { + res.setEncoding("utf8") + res.on("data", (c: string) => { + body += c + if (body === sent.join("")) haveAll() + }) + res.on("end", () => resolve("end")) + res.on("error", () => resolve("error")) + }) + }) + await w.waitFor((l) => l.includes("arm=local reason=upstream-end")) + await received // the listener holds every frame (production's reconcile is a later timer tick) + driver.reconcile() // the re-open → sever + expect(await outcome).toBe("error") // a transport error, never a clean end of stream + expect(body).toBe(sent.join("")) // byte-identical while connected, nothing injected + expect(await w.waitFor((l) => l.startsWith("[fanin] listener-closed"))).toContain("reason=upstream-end") + } finally { + front.closeAllConnections?.() + await new Promise((r) => front.close(() => r())) + await upstream.stop() + } + }) }) // ══════════════════════════════════════════════════════════════════════════════ @@ -1718,12 +1926,17 @@ describe("#1771 AC5 — the census adds no frames to any stream and changes no d vi.advanceTimersByTime(FANIN_CENSUS_INTERVAL_MS); // a census line mid-stream localCtls[0].end(); // the arm drops … await flush(); - driver.reconcile(); // … and re-opens (no gap in fleet-of-one) + driver.reconcile(); // … and the re-open severs the listener (#1773 — no gap frame in fleet-of-one) + expect(res.text()).toBe(first.join("")); + expect(res.destroyed).toBe(true); + // the client reconnects; the stream resumes on the new listener + const again = mockRes(); + driver.handle(mockReq("/global/event"), again); const second = [engine.heartbeat(), engine.event("session.status", { status: "idle" })]; for (const f of second) localCtls[1].push(f); await flush(); - expect(res.text()).toBe([...first, ...second].join("")); - res.fireClose(); + expect(again.text()).toBe(second.join("")); + again.fireClose(); } expect(lines.some((l) => l.startsWith("[fanin] census "))).toBe(true); expect(lines.some((l) => l.includes("arm-closed"))).toBe(true); From 71dd6d2fc2bb7a92e5679f3b4675296b362b295d Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 19:37:11 -0400 Subject: [PATCH 32/60] fix(fleet): past the retention window a peer's owner entries expire and its arm closes with the named unavailable comment (#1774) --- .../extension/src/amicode_service/index.ts | 27 ++- .../amicode_service/session_multiplexer.ts | 7 +- ...de_service_multiplex_owner_routing.test.ts | 215 +++++++++++++++++- 3 files changed, 237 insertions(+), 12 deletions(-) diff --git a/packages/extension/src/amicode_service/index.ts b/packages/extension/src/amicode_service/index.ts index 275fc989b..d942e133d 100644 --- a/packages/extension/src/amicode_service/index.ts +++ b/packages/extension/src/amicode_service/index.ts @@ -40,6 +40,7 @@ import { SessionMultiplexProxy, SessionOwnerMap, OwnerMapFeed, + fanInReachable, type PeerTransport, type MultiplexResolver, type SessionEntry, @@ -1219,14 +1220,18 @@ export function createAmicodeService( // `shutdown`) with the server. const census = new FanInCensus({ plane: "premium", ...(opts.log ? { sink: opts.log } : {}) }); census.start(); + const premiumPeerBaseUrl = (machineId: string): string | undefined => { + const r = peers.readPeerToken(machineId); + return r.ok ? r.credential.baseUrl : undefined; + }; const driver = new SseFanInDriver({ ownerMap, localMachineId: peers.localMachineId, localEventUrl: opts.engine?.getUrl ?? ((): string | undefined => undefined), - peerBaseUrl: (machineId) => { - const r = peers.readPeerToken(machineId); - return r.ok ? r.credential.baseUrl : undefined; - }, + peerBaseUrl: premiumPeerBaseUrl, + // #1774: a peer the owner map names unavailable (retention lapsed, or + // its refresh was refused) is never an arm — the aggregator announces it. + reachable: fanInReachable(ownerMap, (machineId) => premiumPeerBaseUrl(machineId) !== undefined), peerToken: (machineId) => { const r = peers.readPeerToken(machineId); return r.ok ? { ok: true, credential: r.credential } : { ok: false, reason: "absent" }; @@ -1456,15 +1461,19 @@ export function createAmicodeService( // #1771: with its census (30 s census + transition lines → the log). const observeCensus = new FanInCensus({ plane: "observe", ...(opts.log ? { sink: opts.log } : {}) }); observeCensus.start(); + const observePeerBaseUrl = (machineId: string): string | undefined => { + if (!fleetPeers.getServingPeers().some((p) => p.machineId === machineId)) return undefined; + const r = fleetPeers.readPeerToken(machineId); + return r.ok ? r.credential.baseUrl : undefined; + }; const observeDriver = new SseFanInDriver({ ownerMap, localMachineId: fleetPeers.localMachineId, localEventUrl: opts.engine?.getUrl ?? ((): string | undefined => undefined), - peerBaseUrl: (machineId) => { - if (!fleetPeers.getServingPeers().some((p) => p.machineId === machineId)) return undefined; - const r = fleetPeers.readPeerToken(machineId); - return r.ok ? r.credential.baseUrl : undefined; - }, + peerBaseUrl: observePeerBaseUrl, + // #1774: a peer the owner map names unavailable (retention lapsed, or + // its refresh was refused) is never an arm — the aggregator announces it. + reachable: fanInReachable(ownerMap, (machineId) => observePeerBaseUrl(machineId) !== undefined), peerToken: (machineId) => { const r = fleetPeers.readPeerToken(machineId); return r.ok ? { ok: true, credential: r.credential } : { ok: false, reason: "absent" }; diff --git a/packages/extension/src/amicode_service/session_multiplexer.ts b/packages/extension/src/amicode_service/session_multiplexer.ts index a98486944..c11571165 100644 --- a/packages/extension/src/amicode_service/session_multiplexer.ts +++ b/packages/extension/src/amicode_service/session_multiplexer.ts @@ -285,9 +285,12 @@ export class SessionOwnerMap { /** The DISTINCT owner machine_ids currently holding ≥1 owned session (#1511, * §D1) — the fan-in membership's owner set. Order is unspecified; callers - * exclude `local` themselves. */ + * exclude `local` themselves. #1774: plus every peer NAMED unavailable (its + * entries expired or were dropped), so the fan-in keeps it as a member and + * announces it unavailable instead of dropping its arm in silence — the + * driver's `fanInReachable` keeps such a member from ever being an arm. */ ownerMachineIds(): string[] { - return [...new Set(this.owners.values())]; + return [...new Set([...this.owners.values(), ...this.unavailablePeers])]; } /** #1774: is this peer NAMED unavailable — its retention window lapsed, or its diff --git a/packages/extension/test/amicode_service_multiplex_owner_routing.test.ts b/packages/extension/test/amicode_service_multiplex_owner_routing.test.ts index b3888d569..08fb4a024 100644 --- a/packages/extension/test/amicode_service_multiplex_owner_routing.test.ts +++ b/packages/extension/test/amicode_service_multiplex_owner_routing.test.ts @@ -26,11 +26,14 @@ import { SessionOwnerMap, SessionMultiplexProxy, PENDING_OWNER_TTL_MS, + OWNER_RETENTION_MS, + OWNER_MAP_REFRESH_MS, fanInReachable, type SessionEntry, } from "../src/amicode_service/session_multiplexer"; +import { createAmicodeService } from "../src/amicode_service"; import { SseFanInDriver, type FanInUpstreamRequest } from "../src/amicode_service/sse_fanin_driver"; -import type { SseFrameSource } from "../src/amicode_service/sse_fanin_aggregator"; +import { honestSourceComment, type SseFrameSource } from "../src/amicode_service/sse_fanin_aggregator"; import { AmicodeServiceServer, fleetMultiplexEnabled, @@ -335,6 +338,216 @@ describe("#1774 AC1 — a refresh in which one peer times out or errors keeps th }); }); +describe("#1774 AC2 — past the retention window the entries expire, the arm closes, and listeners get the named unavailable comment", () => { + it(`the default window is ${OWNER_RETENTION_MS} ms since the peer's last GOOD refresh: retained through it, expired past it`, () => { + let t = 1_000_000; + const m = new SessionOwnerMap({ now: () => t }); + m.update([ownedBy("ses-s1", "studio")], [peerOk("studio")]); + t += 30_000; + m.update([], [peerTimeout("studio")]); + expect(m.resolveOwner("ses-s1")).toBe("studio"); + t += 30_000; // exactly the window since the last good refresh + m.update([], [peer5xx("studio")]); + expect(m.resolveOwner("ses-s1")).toBe("studio"); + t += 1; // past it + m.update([], [peerTimeout("studio")]); + expect(m.resolveOwner("ses-s1")).toBeUndefined(); + expect(m.size).toBe(0); + expect(m.isUnavailable("studio")).toBe(true); + // still a NAMED owner of the fan-in membership (so the stream says so) … + expect(m.ownerMachineIds()).toEqual(["studio"]); + // … and never an active arm, whatever its transport says + expect(fanInReachable(m, () => true)("studio")).toBe(false); + }); + + it("the window is configurable per map (retentionMs)", () => { + let t = 0; + const m = new SessionOwnerMap({ now: () => t, retentionMs: 5_000 }); + m.update([ownedBy("ses-s1", "studio")], [peerOk("studio")]); + t = 5_000; + m.update([], [peerTimeout("studio")]); + expect(m.resolveOwner("ses-s1")).toBe("studio"); + t = 5_001; + m.update([], [peerTimeout("studio")]); + expect(m.resolveOwner("ses-s1")).toBeUndefined(); + }); + + it("expiry closes the peer's arm and the listener receives the named unavailable comment — not silence", async () => { + let t = 0; + const ownerMap = new SessionOwnerMap({ now: () => t }); + ownerMap.update([ownedBy("ses-s1", "studio")], [peerOk("studio")]); + const origins = armOrigins(); + const driver = retentionDriver(origins, ownerMap); + const res = sseRes(); + driver.handle(sseReq(), res); + await settle(); + expect(origins.opens("studio")).toBe(1); + + t = OWNER_RETENTION_MS + 1; + ownerMap.update([], [peerTimeout("studio")]); + driver.reconcile(); + await settle(); + expect(origins.closes("studio")).toBe(1); + expect(res.text()).toContain(honestSourceComment("studio", "peer-unreachable")); + // announced once, not on every reconcile + driver.reconcile(); + await settle(); + expect(res.text().split("source studio unavailable").length - 1).toBe(1); + res.fireClose(); + }); + + it("an UNAUTHORIZED refresh does not retain: the entries drop at once and the peer is named unavailable (Key Decision)", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update([ownedBy("ses-s1", "studio")], [peerOk("studio")]); + const origins = armOrigins(); + const driver = retentionDriver(origins, ownerMap); + const res = sseRes(); + driver.handle(sseReq(), res); + await settle(); + + ownerMap.update([], [peerUnauthorized("studio")]); // well inside the window + expect(ownerMap.resolveOwner("ses-s1")).toBeUndefined(); + driver.reconcile(); + await settle(); + expect(origins.closes("studio")).toBe(1); + expect(res.text()).toContain(honestSourceComment("studio", "peer-unreachable")); + res.fireClose(); + }); + + it("a trust or credential problem drops like unauthorized; a transport-down peer retains like a timeout", () => { + const m = new SessionOwnerMap(); + for (const reason of ["untrusted", "credential-missing", "identity-conflict"] as const) { + m.update([ownedBy("ses-s1", "studio")], [peerOk("studio")]); + m.update([], [{ machineId: "studio", outcome: "error", reason }]); + expect(m.resolveOwner("ses-s1")).toBeUndefined(); + } + m.update([ownedBy("ses-s1", "studio")], [peerOk("studio")]); + m.update([], [{ machineId: "studio", outcome: "error", reason: "no-upstream" }]); + expect(m.resolveOwner("ses-s1")).toBe("studio"); + }); + + it("a peer that LEAVES the refreshed set is gone, not retained and not named unavailable", () => { + const m = new SessionOwnerMap(); + m.update([ownedBy("ses-s1", "studio")], [peerOk("studio")]); + m.update([], []); // studio is no longer a serving peer + expect(m.resolveOwner("ses-s1")).toBeUndefined(); + expect(m.isUnavailable("studio")).toBe(false); + expect(m.ownerMachineIds()).toEqual([]); + }); + + it("production wiring (observation boot): a peer whose refresh turns unauthorized is announced unavailable on the live /event", async () => { + vi.useFakeTimers({ toFake: ["setInterval", "clearInterval"] }); // the feed + reconcile cadences; HTTP stays real + let studioDenies = false; + const local = await startFleetStub(() => false, []); + const studio = await startFleetStub(() => studioDenies, [{ id: "ses-s1" }]); + const root = mkdtempSync(join(tmpdir(), "amicode-1774-wire-")); + const savedHub = process.env.AMICO_FLEET_HUB_FILE; + process.env.AMICO_FLEET_HUB_FILE = join(root, "hub-cred-absent.json"); + const svc = createAmicodeService({ + password: "svc-1774", + engine: { password: "engine-1774", getUrl: () => local.url }, + fleet: { + hub: { getUrl: () => undefined }, + observationOnly: true, + fleetPeers: { + localMachineId: "macbook", + getServingPeers: () => [{ machineId: "studio" }], + readPeerToken: (id) => (id === "studio" ? { ok: true, credential: { baseUrl: studio.url, token: "tok-studio" } } : { ok: false }), + rosterLookup: (id) => ({ name: id }), + }, + }, + }); + const origin = (await svc.start()).toString().replace(/\/$/, ""); + const stream = new AbortController(); + try { + expect(await until(() => studio.listReads() >= 2)).toBe(true); // the boot refresh (active + archived) + await settle(); + const res = await fetch(`${origin}/event`, { headers: { Authorization: serverAuthHeader("engine-1774") }, signal: stream.signal }); + const text = streamText(res); + expect(await until(() => studio.eventOpens() === 1)).toBe(true); // the peer arm is open + + studioDenies = true; + const reads = studio.listReads(); + await vi.advanceTimersByTimeAsync(OWNER_MAP_REFRESH_MS); // the next owner-map refresh + expect(await until(() => studio.listReads() >= reads + 2)).toBe(true); + await settle(); + await vi.advanceTimersByTimeAsync(1_000); // the listener's reconcile tick + expect(await until(() => text().includes(honestSourceComment("studio", "peer-unreachable")))).toBe(true); + } finally { + stream.abort(); + await svc.stop(); + vi.useRealTimers(); + await Promise.all([local.stop(), studio.stop()]); + if (savedHub === undefined) delete process.env.AMICO_FLEET_HUB_FILE; + else process.env.AMICO_FLEET_HUB_FILE = savedHub; + rmSync(root, { recursive: true, force: true }); + } + }); +}); + +/** A fleet engine stub: its cross-project session list (401 while `denies()`), + * its health, and a /global/event stream that stays open. */ +async function startFleetStub(denies: () => boolean, sessions: Array>) { + let listReads = 0; + let eventOpens = 0; + const server = http.createServer((req, res) => { + const u = new URL(req.url ?? "/", "http://stub"); + if (u.pathname === "/experimental/session") { + listReads++; + if (denies()) return void res.writeHead(401).end("no"); + res.writeHead(200, { "content-type": "application/json" }); + return void res.end(JSON.stringify(u.searchParams.get("archived") === "true" ? [] : sessions)); + } + if (u.pathname === "/global/health") { + res.writeHead(200, { "content-type": "application/json" }); + return void res.end(JSON.stringify({ version: "stub-1774" })); + } + if (u.pathname === "/global/event") { + eventOpens++; + res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" }); + return void res.write(": open\n\n"); + } + res.writeHead(404).end(); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + return { + url: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, + listReads: () => listReads, + eventOpens: () => eventOpens, + stop: () => + new Promise((r) => { + server.closeAllConnections(); + server.close(() => r()); + }), + }; +} + +/** Accumulate a streaming response's text in the background. */ +function streamText(res: Response): () => string { + let text = ""; + const reader = res.body!.getReader(); + const decoder = new TextDecoder(); + void (async () => { + try { + for (;;) { + const { value, done } = await reader.read(); + if (done) return; + text += decoder.decode(value, { stream: true }); + } + } catch { + /* aborted at teardown */ + } + })(); + return () => text; +} + +/** Poll (real timers) until `cond` holds or the deadline passes. */ +async function until(cond: () => boolean, timeoutMs = 4_000): Promise { + const start = Date.now(); + while (!cond() && Date.now() - start < timeoutMs) await new Promise((r) => setTimeout(r, 10)); + return cond(); +} + // ── a deterministic fan-in harness (the driver + aggregator are used as-is) ─── interface SseRes extends http.ServerResponse { text(): string; From 8bdaa1a5340f4911c0fab7f3da533444b81b57d2 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 19:38:06 -0400 Subject: [PATCH 33/60] test(fleet): a successful refresh replaces a peer's owner entries authoritatively, and ends its unavailability (#1774) --- ...de_service_multiplex_owner_routing.test.ts | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/packages/extension/test/amicode_service_multiplex_owner_routing.test.ts b/packages/extension/test/amicode_service_multiplex_owner_routing.test.ts index 08fb4a024..3bf1e46cf 100644 --- a/packages/extension/test/amicode_service_multiplex_owner_routing.test.ts +++ b/packages/extension/test/amicode_service_multiplex_owner_routing.test.ts @@ -485,6 +485,55 @@ describe("#1774 AC2 — past the retention window the entries expire, the arm cl }); }); +describe("#1774 AC3 — a successful refresh replaces the peer's entries authoritatively", () => { + it("after a retained failure, the next good refresh drops the sessions that ended and adds the new ones", () => { + let t = 0; + const m = new SessionOwnerMap({ now: () => t }); + m.update([ownedBy("ses-ended", "studio"), ownedBy("ses-kept", "studio", "/old")], [peerOk("studio", 2)]); + t = 10_000; + m.update([], [peerTimeout("studio")]); + expect(m.resolveOwner("ses-ended")).toBe("studio"); // retained through the blip + t = 20_000; + m.update([ownedBy("ses-kept", "studio", "/new"), ownedBy("ses-born", "studio")], [peerOk("studio", 2)]); + expect(m.resolveOwner("ses-ended")).toBeUndefined(); // it ended — gone + expect(m.resolveOwner("ses-kept")).toBe("studio"); + expect(m.resolveDirectory("ses-kept")).toBe("/new"); + expect(m.resolveOwner("ses-born")).toBe("studio"); + expect(m.size).toBe(2); + // and the good refresh restarts the window: a failure 59 s later still retains + t = 20_000 + OWNER_RETENTION_MS - 1_000; + m.update([], [peerTimeout("studio")]); + expect(m.resolveOwner("ses-born")).toBe("studio"); + expect(m.resolveOwner("ses-ended")).toBeUndefined(); // never resurrected from the stale snapshot + }); + + it("a good refresh after expiry ends the unavailability: the peer is an ordinary owner again and its arm re-opens", async () => { + let t = 0; + const ownerMap = new SessionOwnerMap({ now: () => t }); + ownerMap.update([ownedBy("ses-s1", "studio")], [peerOk("studio")]); + const origins = armOrigins(); + const driver = retentionDriver(origins, ownerMap); + const res = sseRes(); + driver.handle(sseReq(), res); + await settle(); + t = OWNER_RETENTION_MS + 1; + ownerMap.update([], [peerTimeout("studio")]); + driver.reconcile(); + await settle(); + expect(origins.closes("studio")).toBe(1); + + t += 5_000; + ownerMap.update([ownedBy("ses-s2", "studio")], [peerOk("studio")]); + expect(ownerMap.isUnavailable("studio")).toBe(false); + expect(ownerMap.resolveOwner("ses-s2")).toBe("studio"); + expect(ownerMap.resolveOwner("ses-s1")).toBeUndefined(); + driver.reconcile(); + await settle(); + expect(origins.opens("studio")).toBe(2); // the arm is back + res.fireClose(); + }); +}); + /** A fleet engine stub: its cross-project session list (401 while `denies()`), * its health, and a /global/event stream that stays open. */ async function startFleetStub(denies: () => boolean, sessions: Array>) { From da2f958f3daffed71a2008364379fdad61daecfe Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 19:39:07 -0400 Subject: [PATCH 34/60] =?UTF-8?q?fix(fleet):=20fan-in=20and=20proxies=20er?= =?UTF-8?q?ror-close=20a=20lost=20stream=20=E2=80=94=20only=20graceful=20s?= =?UTF-8?q?hutdown=20ends=20a=20listener=20cleanly=20(#1773)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/amicode_service/engine_proxy.ts | 50 +++++---- .../src/amicode_service/hub_proxy.ts | 60 +++++----- .../test/amicode_service_engine_proxy.test.ts | 103 ++++++++++++++++++ .../test/sse_fanin_driver_pump.test.ts | 51 +++++++++ 4 files changed, 217 insertions(+), 47 deletions(-) diff --git a/packages/extension/src/amicode_service/engine_proxy.ts b/packages/extension/src/amicode_service/engine_proxy.ts index a037b7a6f..255822df2 100644 --- a/packages/extension/src/amicode_service/engine_proxy.ts +++ b/packages/extension/src/amicode_service/engine_proxy.ts @@ -25,6 +25,19 @@ export interface EngineProxyOptions { * (Authorization included) rides unchanged. */ const HOP_BY_HOP = ["host", "connection"] as const; +/** #1773 — an upstream lost AFTER the response headers went out: ERROR-close + * the downstream (destroy it with the error) so the listener — or the next + * relay — sees a transport error. A clean `end()` here read as "the stream + * ended normally, nothing was lost" and the client reconnected without a + * resync. Never throws. */ +function errorClose(res: http.ServerResponse, err: unknown): void { + try { + res.destroy(err instanceof Error ? err : new Error(`engine upstream failed: ${String(err)}`)); + } catch { + /* already gone */ + } +} + export class EngineProxy { constructor(private readonly opts: EngineProxyOptions) {} @@ -46,13 +59,8 @@ export class EngineProxy { const upstream = http.request(target, { method: req.method, headers }, (up) => { res.writeHead(up.statusCode ?? 502, up.headers); up.pipe(res); - up.on("error", () => { - try { - res.end(); - } catch { - /* already gone */ - } - }); + // #1773: the engine's stream died mid-body — an error close, not an end. + up.on("error", (err) => errorClose(res, err)); }); // #1261 (AC5): exactly-once reattach — a client disconnect (a reloaded // window closing its stream) tears down the engine upstream so a reload @@ -64,14 +72,14 @@ export class EngineProxy { }); upstream.on("error", (err) => { // The engine went away mid-flight (restart gap, crash): honest 502 - // JSON when headers aren't sent yet, else just close the stream. + // JSON when headers aren't sent yet, else (#1773) an error close. + if (res.headersSent) { + errorClose(res, err); + return; + } try { - if (!res.headersSent) { - res.writeHead(502, { "content-type": "application/json" }); - res.end(JSON.stringify({ ok: false, error: `engine upstream failed: ${err}` })); - } else { - res.end(); - } + res.writeHead(502, { "content-type": "application/json" }); + res.end(JSON.stringify({ ok: false, error: `engine upstream failed: ${err}` })); } catch { /* response already torn down — never throw into the server */ } @@ -80,14 +88,14 @@ export class EngineProxy { // (file uploads) pass through with no service-side memory cost. req.pipe(upstream); return true; - } catch { + } catch (err) { + if (res.headersSent) { + errorClose(res, err); + return true; + } try { - if (!res.headersSent) { - res.writeHead(502, { "content-type": "application/json" }); - res.end(JSON.stringify({ ok: false, error: "engine upstream unavailable" })); - } else { - res.end(); - } + res.writeHead(502, { "content-type": "application/json" }); + res.end(JSON.stringify({ ok: false, error: "engine upstream unavailable" })); } catch { /* never throw into the server */ } diff --git a/packages/extension/src/amicode_service/hub_proxy.ts b/packages/extension/src/amicode_service/hub_proxy.ts index 76fba2fb9..82d733769 100644 --- a/packages/extension/src/amicode_service/hub_proxy.ts +++ b/packages/extension/src/amicode_service/hub_proxy.ts @@ -63,6 +63,18 @@ const DROPPED_HEADERS = ["host", "connection", "authorization"] as const; * `?ticket=`/`?cursor=` query all ride through verbatim. */ const UPGRADE_DROPPED_HEADERS = ["host", "authorization"] as const; +/** #1773 — the hub stream was lost AFTER the response headers went out: + * ERROR-close the downstream (destroy it with the error) so the listener — or + * the next relay — sees a transport error. A clean `end()` here read as "the + * stream ended normally, nothing was lost". Never throws. */ +function errorClose(res: http.ServerResponse, err: unknown): void { + try { + res.destroy(err instanceof Error ? err : new Error(`hub upstream failed: ${String(err)}`)); + } catch { + /* already gone */ + } +} + export class HubProxy { constructor(private readonly opts: HubProxyOptions) {} @@ -104,13 +116,13 @@ export class HubProxy { } else { const cred = this.opts.credential(); if (!cred.ok) { + if (res.headersSent) { + errorClose(res, new Error(`hub-credential-missing (${cred.reason})`)); + return true; + } try { - if (!res.headersSent) { - res.writeHead(503, { "content-type": "application/json" }); - res.end(JSON.stringify({ ok: false, error: "hub-credential-missing", reason: cred.reason })); - } else { - res.end(); - } + res.writeHead(503, { "content-type": "application/json" }); + res.end(JSON.stringify({ ok: false, error: "hub-credential-missing", reason: cred.reason })); } catch { /* never throw into the server */ } @@ -200,13 +212,9 @@ export class HubProxy { } else { up.pipe(res); } - up.on("error", () => { - try { - res.end(); - } catch { - /* already gone */ - } - }); + // #1773: the hub's stream died mid-body — an error close, not an end + // (covers the resume-filtered per-session path and the verbatim pipe). + up.on("error", (err) => errorClose(res, err)); }); // #1261 (AC5): exactly-once reattach. When the CLIENT disconnects (a // reloaded window closing its stream), tear down the host upstream so a @@ -228,27 +236,27 @@ export class HubProxy { detail: err instanceof Error ? err.message : String(err), }); } + if (res.headersSent) { + errorClose(res, err); // #1773: lost after headers → an error close + return; + } try { - if (!res.headersSent) { - res.writeHead(502, { "content-type": "application/json" }); - res.end(JSON.stringify({ ok: false, error: `hub upstream failed: ${err}` })); - } else { - res.end(); - } + res.writeHead(502, { "content-type": "application/json" }); + res.end(JSON.stringify({ ok: false, error: `hub upstream failed: ${err}` })); } catch { /* response already torn down — never throw into the server */ } }); req.pipe(upstream); return true; - } catch { + } catch (err) { + if (res.headersSent) { + errorClose(res, err); + return true; + } try { - if (!res.headersSent) { - res.writeHead(502, { "content-type": "application/json" }); - res.end(JSON.stringify({ ok: false, error: "hub upstream unavailable" })); - } else { - res.end(); - } + res.writeHead(502, { "content-type": "application/json" }); + res.end(JSON.stringify({ ok: false, error: "hub upstream unavailable" })); } catch { /* never throw into the server */ } diff --git a/packages/extension/test/amicode_service_engine_proxy.test.ts b/packages/extension/test/amicode_service_engine_proxy.test.ts index 8ac22cc3b..8957598f8 100644 --- a/packages/extension/test/amicode_service_engine_proxy.test.ts +++ b/packages/extension/test/amicode_service_engine_proxy.test.ts @@ -13,6 +13,10 @@ import * as http from "node:http"; import { AddressInfo } from "node:net"; import { createAmicodeService } from "../src/amicode_service"; import { serverAuthHeader } from "../src/server_auth"; +import { EngineProxy } from "../src/amicode_service/engine_proxy"; +import { HubProxy } from "../src/amicode_service/hub_proxy"; +import type { HubCredentialRead } from "../src/amicode_service/hub_credential"; +import { fakeEngine, startEngineSseServer, LOCAL_EPOCH } from "./fixtures/engine_stream/engine_frames"; interface RecordedHit { method: string; @@ -187,3 +191,102 @@ describe("amicode service — engine proxy (transparent passthrough to the spawn expect(last - first, `first@${first}ms last@${last}ms — buffered, not streamed`).toBeGreaterThanOrEqual(180); }); }); + +// ══════════════════════════════════════════════════════════════════════════════ +// #1773 AC3 — a proxied stream lost AFTER its response headers were sent is an +// ERROR close, never a clean end. The proxies used to answer an upstream failure +// mid-stream with `res.end()`, which a listener (or the next relay) reads as +// "the stream ended normally, nothing was lost". Both the engine proxy and the +// hub proxy now destroy the downstream with the error, so it sees a transport +// error. A clean upstream end still passes through as a clean end. +// ══════════════════════════════════════════════════════════════════════════════ +describe("#1773 AC3 — engine and hub proxies error-close a stream lost after headers", () => { + /** Serve `handler` on 127.0.0.1:0 and dial it at its own address. */ + async function front(handler: (req: http.IncomingMessage, res: http.ServerResponse) => void) { + const server = http.createServer(handler); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + const { port } = server.address() as AddressInfo; + return { + url: `http://127.0.0.1:${port}`, + stop: () => + new Promise((r) => { + server.close(() => r()); + server.closeAllConnections?.(); + }), + }; + } + + /** Read a whole response: its status, body, and whether it ENDED cleanly or ERRORED. */ + function read(url: string): Promise<{ status: number; body: string; how: "end" | "error" }> { + return new Promise((resolve, reject) => { + const req = http.get(url, (res) => { + let body = ""; + res.setEncoding("utf8"); + res.on("data", (c: string) => (body += c)); + res.on("end", () => resolve({ status: res.statusCode ?? 0, body, how: "end" })); + res.on("error", () => resolve({ status: res.statusCode ?? 0, body, how: "error" })); + }); + req.on("error", reject); + }); + } + + const hubCredential = (): HubCredentialRead => ({ + ok: true, + mint: "hub", + credential: { baseUrl: "http://hub.invalid", token: "hub-tok" }, + }); + + const proxies: Array<{ name: string; handler: (upstreamUrl: string) => (req: http.IncomingMessage, res: http.ServerResponse) => void }> = [ + { + name: "engine proxy", + handler: (upstreamUrl) => { + const proxy = new EngineProxy({ getUrl: () => upstreamUrl }); + return (req, res) => void proxy.handle(req, res); + }, + }, + { + name: "hub proxy", + handler: (upstreamUrl) => { + const proxy = new HubProxy({ getUrl: () => upstreamUrl, credential: hubCredential }); + return (req, res) => void proxy.handle(req, res); + }, + }, + ]; + + for (const { name, handler } of proxies) { + it(`${name}: the upstream dies mid-stream → the listener's response ERRORS after the frames it got (no clean end)`, async () => { + const engine = fakeEngine({ legacy: true }); + const sent = [engine.connected(), engine.event("session.status", { sessionID: "ses_1", status: { type: "busy" } })]; + const upstream = await startEngineSseServer((res) => { + for (const f of sent) res.write(f); + setTimeout(() => res.destroy(), 20); // the link dies after the headers and two frames + }); + const relay = await front(handler(upstream.url)); + try { + const { status, body, how } = await read(`${relay.url}/global/event`); + expect(status).toBe(200); // the headers were already relayed … + expect(body).toBe(sent.join("")); // … with every frame the upstream sent … + expect(how).toBe("error"); // … so the loss must surface as a transport error + } finally { + await relay.stop(); + await upstream.stop(); + } + }); + + it(`${name}: a clean upstream end still relays as a clean end (pass-through unchanged)`, async () => { + const engine = fakeEngine({ epoch: LOCAL_EPOCH }); + const sent = [engine.connected(), engine.heartbeat()]; + const upstream = await startEngineSseServer((res) => { + for (const f of sent) res.write(f); + res.end(); + }); + const relay = await front(handler(upstream.url)); + try { + expect(await read(`${relay.url}/global/event`)).toEqual({ status: 200, body: sent.join(""), how: "end" }); + } finally { + await relay.stop(); + await upstream.stop(); + } + }); + } +}); diff --git a/packages/extension/test/sse_fanin_driver_pump.test.ts b/packages/extension/test/sse_fanin_driver_pump.test.ts index 4e42c5d58..300ecb834 100644 --- a/packages/extension/test/sse_fanin_driver_pump.test.ts +++ b/packages/extension/test/sse_fanin_driver_pump.test.ts @@ -1812,6 +1812,57 @@ describe("#1771 AC2 — close reasons are an enumerated set", () => { // shutdown() closed nothing itself — B's stream ended only with its socket. expect(lines.some((l) => l.includes("reason=evicted"))).toBe(false); }); + + // #1773 AC3 — a fan-in connection never ends its listener's stream cleanly + // except on a graceful service shutdown: every other close reason is an ERROR + // close (the listener, or the relay in front of it, sees a transport error). + it("#1773 AC3: a listener-closed connection is an ERROR close; a shutdown is the only clean end", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const driver = twoListenerDriver(fakeOrigins(), ownerMap); + const resA = mockRes(); + const resB = mockRes(); + driver.handle(mockReq("/global/event"), resA); + driver.handle(mockReq("/global/event"), resB); + await flush(); + + resA.fireClose(); // listener-closed + expect(resA.destroyed).toBe(true); + expect(resA.destroyError).toBeInstanceOf(Error); + expect(resA.destroyError?.message).toContain("listener-closed"); + expect(resA.ended).toBe(false); + + driver.shutdown(); + resB.fireClose(); // shutdown + expect(resB.ended).toBe(true); + expect(resB.destroyed).toBe(false); + }); + + it("#1773 AC3: the severed listener's error close names its cause (upstream-error) — census and error agree", async () => { + const { census, lines } = testCensus(); + const localCtls: Ctl[] = []; + const driver = twoListenerDriver(fakeOrigins(), new SessionOwnerMap(), { + census, + openUpstream: () => { + const ctl = controllableSource(); + localCtls.push(ctl); + return ctl.source; + }, + }); + const res = mockRes(); + driver.handle(mockReq("/global/event"), res); + localCtls[0].push(fakeEngine({ epoch: LOCAL_EPOCH }).connected()); + await flush(); + localCtls[0].end("error"); + await flush(); + driver.reconcile(); + expect(res.destroyError?.message).toContain("upstream-error"); + expect(lines.at(-1)).toBe("[fanin] listener-closed plane=observe conn=c1 reason=upstream-error listeners=0"); + // the socket's own close event afterwards changes nothing (closed once, first reason wins) + res.fireClose(); + expect(lines.filter((l) => l.includes("listener-closed")).length).toBe(1); + expect(census.snapshot()).toMatchObject({ listeners: 0, closed: 1 }); + }); }); describe("#1771 AC3 — each transition logs one line immediately, with its connection and arm", () => { From c10c399fb5b377dcbe3791fec5ea9e09fa488dc3 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 19:40:56 -0400 Subject: [PATCH 35/60] fix(fleet): the arm reader treats a non-200 upstream as an error, never a clean end or frames (#1773) --- .../src/amicode_service/sse_fanin_driver.ts | 12 ++- .../fixtures/engine_stream/engine_frames.ts | 16 ++- .../test/sse_fanin_driver_pump.test.ts | 99 ++++++++++++++++++- 3 files changed, 122 insertions(+), 5 deletions(-) diff --git a/packages/extension/src/amicode_service/sse_fanin_driver.ts b/packages/extension/src/amicode_service/sse_fanin_driver.ts index b15faee76..98f61793b 100644 --- a/packages/extension/src/amicode_service/sse_fanin_driver.ts +++ b/packages/extension/src/amicode_service/sse_fanin_driver.ts @@ -105,7 +105,9 @@ export const FANIN_LISTENER_CAP_ERROR = "fan-in listener cap reached"; * aggregator's §D2 relay. Never throws — a connection error just ends the * source (the aggregator names the gap; D3 replays it on reconnect). #1771: * the source remembers WHICH ended it — a clean upstream end or an error — for - * the census's close reason; both still end it identically. */ + * the census's close reason; both still end it identically. #1773: a non-200 + * answer (an auth failure, an overloaded peer, a beyond-cap 503) is an ERROR — + * a loss, not a clean end — and its body is never parsed as frames. */ export function openHttpSseSource(r: FanInUpstreamRequest): SseFrameSource { const queue: string[] = []; const waiters: Array<(v: string | null) => void> = []; @@ -130,6 +132,14 @@ export function openHttpSseSource(r: FanInUpstreamRequest): SseFrameSource { const headers: Record = {}; if (r.authHeader) headers.authorization = r.authHeader; clientReq = http.get(u, { headers }, (res) => { + if (res.statusCode !== 200) { + // #1773 — not a stream: an upstream ERROR. Discard the body (never read + // it as SSE frames) and let the socket go. + res.on("error", () => undefined); // a reset mid-drain must not throw + res.resume(); + end("error"); + return; + } let buf = ""; res.setEncoding("utf8"); res.on("data", (chunk: string) => { diff --git a/packages/extension/test/fixtures/engine_stream/engine_frames.ts b/packages/extension/test/fixtures/engine_stream/engine_frames.ts index 1321e2ed5..3dcdb0c24 100644 --- a/packages/extension/test/fixtures/engine_stream/engine_frames.ts +++ b/packages/extension/test/fixtures/engine_stream/engine_frames.ts @@ -248,12 +248,22 @@ export interface EngineSseServer { * frames, or raw chunks) and decides how it ends (`res.end()` = a clean * upstream end; `res.destroy()` = an upstream error). Bound to 127.0.0.1 on an * ephemeral port and dialed at its own address, so no foreign listener can - * share it. */ -export function startEngineSseServer(script: (res: http.ServerResponse, req: http.IncomingMessage) => void | Promise): Promise { + * share it. #1773: `opts.status` answers a non-200 instead (an auth failure, an + * overloaded peer) with a JSON content type; the script writes its body. */ +export function startEngineSseServer( + script: (res: http.ServerResponse, req: http.IncomingMessage) => void | Promise, + opts: { status?: number } = {}, +): Promise { const requests: string[] = []; + const status = opts.status ?? 200; const server = http.createServer((req, res) => { requests.push(req.url ?? "/"); - res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache, no-transform" }); + res.writeHead( + status, + status === 200 + ? { "content-type": "text/event-stream", "cache-control": "no-cache, no-transform" } + : { "content-type": "application/json" }, + ); res.flushHeaders(); void script(res, req); }); diff --git a/packages/extension/test/sse_fanin_driver_pump.test.ts b/packages/extension/test/sse_fanin_driver_pump.test.ts index 300ecb834..65d1f025e 100644 --- a/packages/extension/test/sse_fanin_driver_pump.test.ts +++ b/packages/extension/test/sse_fanin_driver_pump.test.ts @@ -5,7 +5,7 @@ // reconnection indefinitely. import { describe, it, expect, vi, afterEach } from "vitest"; import * as http from "node:http"; -import { SseFanInDriver } from "../src/amicode_service/sse_fanin_driver"; +import { SseFanInDriver, openHttpSseSource } from "../src/amicode_service/sse_fanin_driver"; import type { SseFrameSource } from "../src/amicode_service/sse_fanin_aggregator"; import { LOCAL_NAMESPACE } from "../src/amicode_service/sse_fanin_aggregator"; import { SessionOwnerMap } from "../src/amicode_service/session_multiplexer"; @@ -1784,6 +1784,103 @@ describe("#1771 AC2 — close reasons are an enumerated set", () => { } }); + // ──────────────────────────────────────────────────────────────────────────── + // #1773 AC4 — the arm reader (openHttpSseSource) reports upstream END and + // upstream ERROR as distinct reasons. A non-200 upstream (an auth failure, an + // overloaded peer, the beyond-cap 503) is an ERROR, not a clean end — and its + // body is never parsed as SSE frames. + // ──────────────────────────────────────────────────────────────────────────── + describe("#1773 AC4 — the arm reader's end-vs-error", () => { + /** Drain a source to its end: the frames it yielded, and its endReason. */ + async function drain(source: SseFrameSource): Promise<{ frames: string[]; reason: string | undefined }> { + const frames: string[] = []; + for (;;) { + const f = await source.next(); + if (f === null) break; + frames.push(f); + } + return { frames, reason: source.endReason?.() }; + } + const arm = (url: string) => openHttpSseSource({ namespace: "studio", url, path: "/global/event" }); + + it("a clean upstream end reads as `end`, after every frame", async () => { + const engine = fakeEngine({ epoch: PEER_EPOCH }); + const sent = [engine.connected(), engine.event("session.status", { s: 1 })]; + const server = await startEngineSseServer((res) => { + for (const f of sent) res.write(f); + res.end(); + }); + try { + expect(await drain(arm(server.url))).toEqual({ frames: sent, reason: "end" }); + } finally { + await server.stop(); + } + }); + + it("an upstream that dies mid-stream reads as `error`, after the frames it sent", async () => { + const engine = fakeEngine({ legacy: true }); + const sent = [engine.connected()]; + const server = await startEngineSseServer((res) => { + for (const f of sent) res.write(f); + setTimeout(() => res.destroy(), 20); + }); + try { + expect(await drain(arm(server.url))).toEqual({ frames: sent, reason: "error" }); + } finally { + await server.stop(); + } + }); + + for (const status of [401, 500, 503]) { + it(`a ${status} upstream is an \`error\` — never a clean end, and its body is not read as frames`, async () => { + const server = await startEngineSseServer( + (res) => res.end(`{"ok":false,"error":"refused"}\n\ndata: {"not":"a frame"}\n\n`), + { status }, + ); + try { + expect(await drain(arm(server.url))).toEqual({ frames: [], reason: "error" }); + } finally { + await server.stop(); + } + }); + } + + it("a refused connection is an `error`", async () => { + // port 1 (tcpmux) on loopback: nothing listens there, the dial is refused + expect(await drain(arm("http://127.0.0.1:1"))).toEqual({ frames: [], reason: "error" }); + }); + + it("through the driver: a peer answering 401 closes its arm as upstream-error in the census", async () => { + const local = await startEngineSseServer((res) => res.write(fakeEngine({ legacy: true }).connected())); + const peer = await startEngineSseServer((res) => res.end(`{"ok":false}`), { status: 401 }); + const lines: string[] = []; + const w = lineWaiter(lines); + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const driver = new SseFanInDriver({ + ownerMap, + localMachineId: "macbook", + localEventUrl: () => local.url, + peerBaseUrl: (id) => (id === "studio" ? peer.url : undefined), + peerToken: (id) => + id === "studio" + ? { ok: true as const, credential: { baseUrl: peer.url, token: "tok-studio" } } + : { ok: false as const, reason: "absent" as const }, + reconcileMs: 999_999, + census: new FanInCensus({ plane: "observe", sink: w.sink }), + }); + const res = mockRes(); + try { + driver.handle(mockReq("/global/event"), res); + expect(await w.waitFor((l) => l.includes("arm=studio reason="))).toMatch(/reason=upstream-error$/); + } finally { + res.fireClose(); + await local.stop(); + await peer.stop(); + } + }); + }); + it("a departing listener's arms close as listener-closed; after shutdown() the teardown's closes are shutdown", async () => { const ownerMap = new SessionOwnerMap(); ownerMap.update(STUDIO_OWNED); From 7011432b11a91ac94fd1cf0326f505d4e75cc551 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 19:44:17 -0400 Subject: [PATCH 36/60] fix(fleet): an unknown-owner session request is tried locally first; a local miss is a retryable owner-unknown 503, not a 404 (#1774) --- .../extension/src/amicode_service/index.ts | 4 + .../amicode_service/local_session_probe.ts | 62 +++++++++ .../amicode_service/observation_read_plane.ts | 11 +- .../observation_write_plane.ts | 11 +- .../extension/src/amicode_service/server.ts | 51 +++++++- .../amicode_service/session_multiplexer.ts | 33 +++-- ...icode_service_observe_read_routing.test.ts | 123 +++++++++++++++++- ...code_service_observe_write_routing.test.ts | 101 +++++++++++++- 8 files changed, 373 insertions(+), 23 deletions(-) create mode 100644 packages/extension/src/amicode_service/local_session_probe.ts diff --git a/packages/extension/src/amicode_service/index.ts b/packages/extension/src/amicode_service/index.ts index d942e133d..770d36423 100644 --- a/packages/extension/src/amicode_service/index.ts +++ b/packages/extension/src/amicode_service/index.ts @@ -1413,6 +1413,8 @@ export function createAmicodeService( const r = fleetPeers.readPeerToken(machineId); return r.ok ? { getUrl: () => r.credential.baseUrl, token: r.credential.token } : { getUrl: () => undefined }; }, + // #1774: the local-first probe for an unknown-owner read. + localEngineUrl: opts.engine?.getUrl ?? ((): string | undefined => undefined), ...(opts.fleet.dataPlaneTimeoutMs !== undefined ? { timeoutMs: opts.fleet.dataPlaneTimeoutMs } : {}), }), ); @@ -1444,6 +1446,8 @@ export function createAmicodeService( const g = findControlGrantByTarget(owner); return g ? { scope: g.scope, state: g.state } : undefined; }, + // #1774: the local-first probe for an unknown-owner write. + localEngineUrl: opts.engine?.getUrl ?? ((): string | undefined => undefined), ...(opts.fleet.dataPlaneTimeoutMs !== undefined ? { timeoutMs: opts.fleet.dataPlaneTimeoutMs } : {}), }), ); diff --git a/packages/extension/src/amicode_service/local_session_probe.ts b/packages/extension/src/amicode_service/local_session_probe.ts new file mode 100644 index 000000000..178a67467 --- /dev/null +++ b/packages/extension/src/amicode_service/local_session_probe.ts @@ -0,0 +1,62 @@ +// LOCAL SESSION PROBE (#1774): "does the LOCAL engine hold this session?" — the +// local-first half of the unknown-owner outcome. A session-scoped request whose +// id has NO known owner (the owner map has not caught up — a just-created local +// session, or a peer whose entries expired) is tried on the local engine first; +// only the engine's own 404 for the session detail read means "not here", and +// the caller then answers a retryable owner-unknown 503 instead of letting the +// request become a 404 or an empty result. +// +// The probe is the engine's own session detail read (`GET /session/{id}` → 404 +// on a storage miss), sent with the request's OWN credential and directory scope +// (the same Authorization the engine proxy would forward), never a body. Any +// other answer — 2xx, an auth refusal, a 5xx, a transport error, a timeout — is +// not a verdict: the request falls through to the local engine, which answers +// for itself exactly as before. +import type * as http from "node:http"; + +/** present → the local engine holds it; absent → its own 404; unknown → no verdict. */ +export type LocalSessionPresence = "present" | "absent" | "unknown"; + +/** The probe's bound when the caller sets none — short: it sits in front of a request. */ +export const LOCAL_SESSION_PROBE_TIMEOUT_MS = 3_000; + +/** The directory/workspace scope carriers a request may hold (header or query). */ +const SCOPE_HEADERS = ["x-opencode-directory", "x-opencode-workspace"] as const; +const SCOPE_PARAMS = ["directory", "workspace", "auth_token"] as const; + +export async function probeLocalSession(opts: { + /** The local engine origin, read per request; undefined → no verdict. */ + localUrl: string | undefined; + sessionId: string; + /** The incoming request: its credential and scope ride on the probe. */ + req: Pick; + fetchImpl?: typeof fetch; + timeoutMs?: number; +}): Promise { + if (!opts.localUrl) return "unknown"; + try { + const target = new URL(`/session/${encodeURIComponent(opts.sessionId)}`, opts.localUrl); + const incoming = new URL(opts.req.url ?? "/", "http://origin"); + for (const p of SCOPE_PARAMS) { + const v = incoming.searchParams.get(p); + if (v !== null) target.searchParams.set(p, v); + } + const headers: Record = {}; + const auth = opts.req.headers.authorization; + if (typeof auth === "string") headers.authorization = auth; + for (const h of SCOPE_HEADERS) { + const v = opts.req.headers[h]; + if (typeof v === "string") headers[h] = v; + } + const res = await (opts.fetchImpl ?? fetch)(target.toString(), { + method: "GET", + headers, + signal: AbortSignal.timeout(opts.timeoutMs ?? LOCAL_SESSION_PROBE_TIMEOUT_MS), + }); + await res.body?.cancel().catch(() => undefined); + if (res.status === 404) return "absent"; + return res.ok ? "present" : "unknown"; + } catch { + return "unknown"; + } +} diff --git a/packages/extension/src/amicode_service/observation_read_plane.ts b/packages/extension/src/amicode_service/observation_read_plane.ts index b8453ef98..dc0a0df96 100644 --- a/packages/extension/src/amicode_service/observation_read_plane.ts +++ b/packages/extension/src/amicode_service/observation_read_plane.ts @@ -22,17 +22,24 @@ import { type ObservationReadRouterOpts, type ObservationReadPeerTarget, } from "./session_multiplexer"; +import { probeLocalSession } from "./local_session_probe"; import type { ObservationReadPlane } from "./server"; /** Build the observation-only read plane: a resolver (path→owner peer) plus the * peer-token proxy. `timeoutMs` bounds the headers wait (SSE bodies ride past - * it), mirroring the hub/attached proxies. */ + * it), mirroring the hub/attached proxies. #1774: `localEngineUrl` (the local + * engine origin, read per request) arms the local-first probe for an + * unknown-owner read; absent → no verdict, the read falls through local. */ export function createObservationReadPlane( - opts: ObservationReadRouterOpts & { timeoutMs?: number }, + opts: ObservationReadRouterOpts & { timeoutMs?: number; localEngineUrl?: () => string | undefined }, ): ObservationReadPlane { const router = new ObservationReadRouter(opts); + const localEngineUrl = opts.localEngineUrl; return { resolve: (method, pathname) => router.resolve(method, pathname), + ...(localEngineUrl + ? { probeLocal: (req: http.IncomingMessage, sessionId: string) => probeLocalSession({ localUrl: localEngineUrl(), sessionId, req }) } + : {}), proxyToPeer(req: http.IncomingMessage, res: http.ServerResponse, target: ObservationReadPeerTarget): boolean { // A synthetic per-peer hub credential → HubProxy attaches // serverAuthHeader(target.token) === peerAuthHeader(target.token). diff --git a/packages/extension/src/amicode_service/observation_write_plane.ts b/packages/extension/src/amicode_service/observation_write_plane.ts index 2c8954380..f08ca7696 100644 --- a/packages/extension/src/amicode_service/observation_write_plane.ts +++ b/packages/extension/src/amicode_service/observation_write_plane.ts @@ -40,6 +40,7 @@ import { type ObservationWriteRouterOpts, type ObservationWritePeerTarget, } from "./session_multiplexer"; +import { probeLocalSession } from "./local_session_probe"; import type { ObservationWritePlane } from "./server"; /** Does the owning peer host a project whose worktree equals `dir`? Probes the @@ -69,14 +70,20 @@ async function peerHasProjectWorktree(peerUrl: string, token: string, dir: strin * ignored) that AUTHORIZES through the pure write gate, plus the peer proxy * presenting the credential the owner accepts (the peer reader token). * `timeoutMs` bounds the headers wait (SSE bodies ride past it), mirroring the - * read plane / hub / attached proxies. */ + * read plane / hub / attached proxies. #1774: `localEngineUrl` (the local + * engine origin, read per request) arms the local-first probe for an + * unknown-owner write; absent → no verdict, the write falls through local. */ export function createObservationWritePlane( - opts: ObservationWriteRouterOpts & { timeoutMs?: number }, + opts: ObservationWriteRouterOpts & { timeoutMs?: number; localEngineUrl?: () => string | undefined }, ): ObservationWritePlane { const router = new ObservationWriteRouter(opts); + const localEngineUrl = opts.localEngineUrl; return { resolve: (method, pathname) => router.resolve(method, pathname), resolveCreate: (method, pathname, headers) => router.resolveCreate(method, pathname, headers), + ...(localEngineUrl + ? { probeLocal: (req: http.IncomingMessage, sessionId: string) => probeLocalSession({ localUrl: localEngineUrl(), sessionId, req }) } + : {}), proxyToPeer(req: http.IncomingMessage, res: http.ServerResponse, target: ObservationWritePeerTarget): boolean { // A synthetic per-peer hub credential → HubProxy attaches // serverAuthHeader(target.token) === peerAuthHeader(target.token). The diff --git a/packages/extension/src/amicode_service/server.ts b/packages/extension/src/amicode_service/server.ts index 1cdf9c2ee..8136b12c9 100644 --- a/packages/extension/src/amicode_service/server.ts +++ b/packages/extension/src/amicode_service/server.ts @@ -34,6 +34,8 @@ import type { ObservationWriteResolution, ObservationWritePeerTarget, } from "./session_multiplexer"; +import { OWNER_MAP_REFRESH_MS } from "./session_multiplexer"; +import type { LocalSessionPresence } from "./local_session_probe"; import type { EventFanInDriver } from "./sse_fanin_driver"; import { BOUND_NONCE_HEADER, BOUND_IDENTITY_HEADER } from "./fleet_bootstrap_headers"; @@ -183,6 +185,9 @@ export interface ObservationReadPlane { * true when it owns the response; false when no upstream is bound (→ the * caller answers a named 503, never local). */ proxyToPeer(req: http.IncomingMessage, res: http.ServerResponse, target: ObservationReadPeerTarget): boolean; + /** #1774: does the LOCAL engine hold this session? Consulted for an + * unknown-owner request (local first). Absent → no verdict (fall through). */ + probeLocal?(req: http.IncomingMessage, sessionId: string): Promise; } /** #1542 (Fleet Studio B2b, WRITE seam): the OBSERVATION-ONLY per-session @@ -224,6 +229,9 @@ export interface ObservationWritePlane { res: http.ServerResponse, target: ObservationWritePeerTarget, ): Promise; + /** #1774: does the LOCAL engine hold this session? Consulted for an + * unknown-owner write (local first). Absent → no verdict (fall through). */ + probeLocal?(req: http.IncomingMessage, sessionId: string): Promise; } /** #1261 (AC6): a client's own named hub-down state — distinct from the base @@ -241,6 +249,28 @@ export const FLEET_PEER_UNREACHABLE_POINTER = "the attached server is unreachable — check the peer's tunnel / service, " + "or detach to work locally"; +/** #1774: the NAMED retryable answer for a session-scoped request whose id has + * no known owner and that the LOCAL engine does not hold — the owner map may + * simply not have caught up (a just-created session, a peer whose entries + * expired), so it is a 503 with a Retry-After of one owner-map refresh, never a + * 404 or an empty result. */ +export const OWNER_UNKNOWN_ERROR = "session owner unknown"; +export const OWNER_UNKNOWN_RETRY_AFTER_S = Math.ceil(OWNER_MAP_REFRESH_MS / 1000); + +function ownerUnknown(sessionId: string): AmicodeHandlerResult { + return { + status: 503, + headers: { "Retry-After": String(OWNER_UNKNOWN_RETRY_AFTER_S) }, + body: JSON.stringify({ + ok: false, + error: OWNER_UNKNOWN_ERROR, + reason: "owner-unknown", + session_id: sessionId, + retry_after_s: OWNER_UNKNOWN_RETRY_AFTER_S, + }), + }; +} + /** #1448 (W1a): the session-multiplexer feature-flag env var. Default ON. */ export const FLEET_MULTIPLEX_FLAG = "AMICO_FLEET_MULTIPLEX"; @@ -805,7 +835,15 @@ export class AmicodeServiceServer { // peer-unreachable 503, NEVER the local engine (the #1382 invariant). if (this.observeRead) { const decision = this.observeRead.resolve(req.method ?? "GET", url.pathname); - if (decision) { + if (decision?.kind === "unknown-owner") { + // #1774: no known owner → the LOCAL engine first; only its own 404 + // is "not here" → the retryable owner-unknown 503. Otherwise the read + // falls through to the engine proxy below, byte-identical. + if ((await this.observeRead.probeLocal?.(req, decision.sessionId)) === "absent") { + send(ownerUnknown(decision.sessionId)); + return; + } + } else if (decision) { if (decision.kind === "peer") { if (this.observeRead.proxyToPeer(req, res, decision)) return; } @@ -843,7 +881,16 @@ export class AmicodeServiceServer { // pathed write streams (proxyToPeer). const createDecision = this.observeWrite.resolveCreate(req.method ?? "GET", url.pathname, req.headers); const decision = createDecision ?? this.observeWrite.resolve(req.method ?? "GET", url.pathname); - if (decision) { + if (decision?.kind === "unknown-owner") { + // #1774: no known owner → the LOCAL engine first (an existence probe, + // never the write itself); only its own 404 is "not here" → the + // retryable owner-unknown 503, and the write runs nowhere. Otherwise + // it falls through to the engine proxy below, byte-identical. + if ((await this.observeWrite.probeLocal?.(req, decision.sessionId)) === "absent") { + send(ownerUnknown(decision.sessionId)); + return; + } + } else if (decision) { if (decision.kind === "peer") { const handled = createDecision ? await this.observeWrite.proxyCreateToPeer(req, res, decision) diff --git a/packages/extension/src/amicode_service/session_multiplexer.ts b/packages/extension/src/amicode_service/session_multiplexer.ts index c11571165..368234eaf 100644 --- a/packages/extension/src/amicode_service/session_multiplexer.ts +++ b/packages/extension/src/amicode_service/session_multiplexer.ts @@ -430,16 +430,26 @@ export interface PeerTransport { /** A REACHABLE peer read target — proxy the read to `url` with `token`. */ export type ObservationReadPeerTarget = { kind: "peer"; machineId: string; url: string; token: string }; +/** #1774: a session-scoped request whose id has NO known owner (not in the owner + * map, no pending binding). The caller tries the LOCAL engine first; if the + * local engine does not hold the session it answers a retryable owner-unknown + * 503 — never a 404 or an empty result (the map may simply not have caught up). */ +export type ObservationUnknownOwner = { kind: "unknown-owner"; sessionId: string }; + /** The resolution of an observation-mode read: * - a REACHABLE peer target → proxy to the owner with its reader token; * - the honest DEGRADED variant (owner is a KNOWN peer, but its transport is * down — getUrl() undefined / no token) → the caller answers a named 503, * NEVER local-dressed-as-peer (the #1382 invariant); + * - #1774 the unknown-owner outcome → local first, then a retryable 503; * - `undefined` (NOT a target) → LOCAL, the fail-safe: a write, a keyless / - * non-session path, /amicode/*, a local-owned or unowned session, or an - * owner that is not a known peer. The read falls through BYTE-IDENTICAL to - * the local engine. */ -export type ObservationReadResolution = ObservationReadPeerTarget | { kind: "degraded"; machineId: string }; + * non-session path, /amicode/*, a local-owned session, or an owner that is + * not a known peer. The read falls through BYTE-IDENTICAL to the local + * engine. */ +export type ObservationReadResolution = + | ObservationReadPeerTarget + | { kind: "degraded"; machineId: string } + | ObservationUnknownOwner; /** Options for the ObservationReadRouter. */ export interface ObservationReadRouterOpts { @@ -480,7 +490,7 @@ export class ObservationReadRouter { * 0. NON-read (POST/PATCH/DELETE/…) → local (writes NEVER route) * 1. /amicode/* (the machine's own honesty + local surface) → local * 2. no session id extractable from the path → local - * 3. owner unknown (unowned) → local + * 3. owner unknown (unowned) → #1774 the unknown-owner outcome (local first) * 4. owner == localMachineId (local-owned) → local * 5. owner is not a known peer → local * 6. owner is a known peer, transport down (no url/token) → DEGRADED (503) @@ -494,7 +504,7 @@ export class ObservationReadRouter { const sessionId = extractSessionIdFromReadPath(pathname); // (2) if (!sessionId) return undefined; const owner = this.ownerMap.resolveOwner(sessionId); // (3) - if (!owner) return undefined; + if (!owner) return { kind: "unknown-owner", sessionId }; if (owner === this.localMachineId) return undefined; // (4) const peer = this.peer(owner); // (5) if (!peer) return undefined; @@ -524,11 +534,12 @@ export type ObservationWriteDenied = { kind: "denied"; machineId: string; reason * credential (the reader token); * - a NAMED deny → the caller answers the gate's honest deny (an auth denial, * or the peer-unreachable 503 for `transport-down`), NEVER local (#1382); + * - #1774 the unknown-owner outcome → local first, then a retryable 503; * - `undefined` (NOT a target) → LOCAL, the fail-safe: a GET/HEAD (the read * plane owns those — the INVERTED decision), a keyless / non-session path, - * /amicode/*, a local-owned or unowned session, or an owner that is not a - * serving peer. The write falls through BYTE-IDENTICAL to the local engine. */ -export type ObservationWriteResolution = ObservationWritePeerTarget | ObservationWriteDenied; + * /amicode/*, a local-owned session, or an owner that is not a serving + * peer. The write falls through BYTE-IDENTICAL to the local engine. */ +export type ObservationWriteResolution = ObservationWritePeerTarget | ObservationWriteDenied | ObservationUnknownOwner; /** Options for the ObservationWriteRouter. */ export interface ObservationWriteRouterOpts { @@ -591,7 +602,7 @@ export class ObservationWriteRouter { * 0. GET/HEAD → local (the read plane owns reads — writes NEVER touch GET) * 1. /amicode/* (the machine's own surface) → local * 2. no session id extractable from the path → local - * 3. owner unknown (unowned) → local + * 3. owner unknown (unowned) → #1774 the unknown-owner outcome (local first) * 4. owner == localMachineId (local-owned) → local * 5. owner is not a serving peer → local * 6. owner is a serving peer → AUTHORIZE via the pure write gate: @@ -605,7 +616,7 @@ export class ObservationWriteRouter { const sessionId = extractSessionIdFromReadPath(pathname); // (2) if (!sessionId) return undefined; const owner = this.ownerMap.resolveOwner(sessionId); // (3) - if (!owner) return undefined; + if (!owner) return { kind: "unknown-owner", sessionId }; if (owner === this.localMachineId) return undefined; // (4) const peer = this.peer(owner); // (5) if (!peer) return undefined; // owner is not a serving peer → local fall-through diff --git a/packages/extension/test/amicode_service_observe_read_routing.test.ts b/packages/extension/test/amicode_service_observe_read_routing.test.ts index 2835f50bf..92eae6ac9 100644 --- a/packages/extension/test/amicode_service_observe_read_routing.test.ts +++ b/packages/extension/test/amicode_service_observe_read_routing.test.ts @@ -21,7 +21,7 @@ import { join } from "node:path"; import * as http from "node:http"; import { AddressInfo } from "node:net"; -import { AmicodeServiceServer } from "../src/amicode_service/server"; +import { AmicodeServiceServer, OWNER_UNKNOWN_ERROR, OWNER_UNKNOWN_RETRY_AFTER_S } from "../src/amicode_service/server"; import { createAmicodeService } from "../src/amicode_service"; import { EngineProxy } from "../src/amicode_service/engine_proxy"; import { @@ -148,11 +148,14 @@ describe("#1537 — ObservationReadRouter resolution order", () => { expect(r.resolve("PATCH", "/session/ses-studio")).toBeUndefined(); expect(r.resolve("DELETE", "/api/session/ses-studio")).toBeUndefined(); }); - it("local-owned / unowned / not-a-known-peer → undefined (local)", () => { + it("local-owned / not-a-known-peer → undefined (local)", () => { expect(r.resolve("GET", "/session/ses-local")).toBeUndefined(); // local-owned - expect(r.resolve("GET", "/api/session/ses-unknown")).toBeUndefined(); // unowned expect(r.resolve("GET", "/api/session/ses-stranger")).toBeUndefined(); // owner not a known peer }); + it("#1774: an UNOWNED session (no known owner) → the unknown-owner outcome: the caller tries local first", () => { + expect(r.resolve("GET", "/api/session/ses-unknown")).toEqual({ kind: "unknown-owner", sessionId: "ses-unknown" }); + expect(r.resolve("HEAD", "/session/ses-unknown/message")).toEqual({ kind: "unknown-owner", sessionId: "ses-unknown" }); + }); it("/amicode/* (own honesty + local surface) → undefined (never proxied)", () => { expect(r.resolve("GET", "/amicode/fleet/status")).toBeUndefined(); expect(r.resolve("GET", "/amicode/fleet/sessions")).toBeUndefined(); @@ -313,6 +316,101 @@ describe("#1774 AC1 — a peer-owned read keeps routing to the owner across a fa }); }); +// ══════════════════════════════════════════════════════════════════════════════ +// #1774 AC4 — a session-scoped read for an id with NO known owner is tried on the +// LOCAL engine first; if the local engine does not have it (its own 404), the +// answer is a RETRYABLE 503 + Retry-After naming `owner-unknown` — not a 404 and +// not an empty result (the owner map may simply not have caught up). +// ══════════════════════════════════════════════════════════════════════════════ + +/** An engine-like local stub: `GET /session/{id}` (the existence probe the + * service makes) answers 200 for a session it holds and the engine's own 404 + * otherwise; every other session path answers a marker for a held session and + * an EMPTY list for a missing one (the "empty result" a miss must not become). */ +function startEngineLikeStub(held: Set): Promise { + const requests: Stub["requests"] = []; + const server = http.createServer((req, res) => { + const u = new URL(req.url ?? "/", "http://stub"); + requests.push({ method: req.method ?? "GET", path: u.pathname, auth: typeof req.headers.authorization === "string" ? req.headers.authorization : undefined }); + const id = extractSessionIdFromReadPath(u.pathname); + res.setHeader("content-type", "application/json"); + if (id !== undefined && !held.has(id)) { + if (/^\/(api\/)?session\/[^/]+$/.test(u.pathname)) return void res.writeHead(404).end(JSON.stringify({ name: "NotFoundError", data: { message: `Session not found: ${id}` } })); + return void res.writeHead(200).end("[]"); + } + res.writeHead(200).end(JSON.stringify({ ok: true, marker: "ENGINE-LIKE" })); + }); + return new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => { + const port = (server.address() as AddressInfo).port; + resolve({ url: `http://127.0.0.1:${port}`, marker: "ENGINE-LIKE", requests, stop: () => new Promise((r) => server.close(() => r())) }); + }); + }); +} + +describe("#1774 AC4 — an unknown-owner read is tried locally; a local miss is a retryable owner-unknown 503", () => { + let engine: Stub; + beforeAll(async () => { + engine = await startEngineLikeStub(new Set(["ses-here"])); + }); + afterAll(async () => { + await engine?.stop(); + }); + const noPeers = (): PeerTransport | undefined => undefined; + + function boot(): AmicodeServiceServer { + const server = new AmicodeServiceServer({ password: PW }); + server.attachEngineProxy(new EngineProxy({ getUrl: () => engine.url })); + server.attachObservationReadPlane( + createObservationReadPlane({ ownerMap: new SessionOwnerMap(), localMachineId: "macbook", peer: noPeers, localEngineUrl: () => engine.url }), + ); + return server; + } + + for (const path of ["/api/session/ses-ghost", "/session/ses-ghost/message", "/api/session/ses-ghost/message?limit=50"]) { + it(`GET ${path} (not on the local engine) → 503 owner-unknown + Retry-After, never the engine's 404 or an empty list`, async () => { + const server = boot(); + const origin = (await server.start()).toString().replace(/\/$/, ""); + try { + const res = await fetch(`${origin}${path}`, { headers: authed }); + expect(res.status).toBe(503); + expect(res.headers.get("retry-after")).toBe(String(OWNER_UNKNOWN_RETRY_AFTER_S)); + const body = (await res.json()) as { ok: boolean; error: string; reason: string; session_id: string; retry_after_s: number }; + expect(body).toEqual({ ok: false, error: OWNER_UNKNOWN_ERROR, reason: "owner-unknown", session_id: "ses-ghost", retry_after_s: OWNER_UNKNOWN_RETRY_AFTER_S }); + } finally { + await server.stop(); + } + }); + } + + it("the local engine is probed with the request's OWN credential and directory scope", async () => { + const server = boot(); + const origin = (await server.start()).toString().replace(/\/$/, ""); + const before = engine.requests.length; + try { + await fetch(`${origin}/session/ses-ghost/message?directory=${encodeURIComponent("/srv/proj")}`, { headers: authed }); + const probe = engine.requests.slice(before); + expect(probe).toEqual([{ method: "GET", path: "/session/ses-ghost", auth: authed.Authorization }]); + } finally { + await server.stop(); + } + }); + + it("an unowned id the LOCAL engine holds → served by the local engine (local first)", async () => { + const server = boot(); + const origin = (await server.start()).toString().replace(/\/$/, ""); + const before = engine.requests.length; + try { + const res = await fetch(`${origin}/api/session/ses-here/message`, { headers: authed }); + expect(res.status).toBe(200); + expect(((await res.json()) as { marker: string }).marker).toBe("ENGINE-LIKE"); + expect(engine.requests.slice(before).map((r) => r.path)).toEqual(["/session/ses-here", "/api/session/ses-here/message"]); + } finally { + await server.stop(); + } + }); +}); + // ══════════════════════════════════════════════════════════════════════════════ // AC6 — production wiring: createAmicodeService on the OBSERVATION-ONLY path // wires the seam (a peer-owned read routes to the peer); the armed path stays @@ -385,4 +483,23 @@ describe("#1537 AC6 — production wiring (createAmicodeService observation-only await svc.stop(); } }); + + it("#1774: the observation boot wires the local-first probe — an id no one owns and the local engine lacks → 503 owner-unknown", async () => { + const engine = await startEngineLikeStub(new Set()); + const svc = createAmicodeService({ + password: PW, + engine: { password: "engine-mint", getUrl: () => engine.url }, + fleet: { hub: { getUrl: () => undefined }, observationOnly: true, fleetPeers: servingPeerProvider() }, + }); + const origin = (await svc.start()).toString().replace(/\/$/, ""); + try { + const res = await fetch(`${origin}/api/session/ses-ghost`, { headers: { Authorization: serverAuthHeader("engine-mint") } }); + expect(res.status).toBe(503); + expect(res.headers.get("retry-after")).toBe(String(OWNER_UNKNOWN_RETRY_AFTER_S)); + expect(((await res.json()) as { reason: string }).reason).toBe("owner-unknown"); + } finally { + await svc.stop(); + await engine.stop(); + } + }); }); diff --git a/packages/extension/test/amicode_service_observe_write_routing.test.ts b/packages/extension/test/amicode_service_observe_write_routing.test.ts index f6d372b3e..3290e7667 100644 --- a/packages/extension/test/amicode_service_observe_write_routing.test.ts +++ b/packages/extension/test/amicode_service_observe_write_routing.test.ts @@ -28,7 +28,7 @@ import { join } from "node:path"; import * as http from "node:http"; import { AddressInfo } from "node:net"; -import { AmicodeServiceServer } from "../src/amicode_service/server"; +import { AmicodeServiceServer, OWNER_UNKNOWN_ERROR, OWNER_UNKNOWN_RETRY_AFTER_S } from "../src/amicode_service/server"; import { createAmicodeService } from "../src/amicode_service"; import { EngineProxy } from "../src/amicode_service/engine_proxy"; import { @@ -147,12 +147,16 @@ describe("#1542 — ObservationWriteRouter resolution order (GET/non-GET INVERTE expect(r.resolve("HEAD", "/api/session/ses-studio")).toBeUndefined(); }); - it("local-owned / unowned / not-a-known-peer → undefined (BYTE-IDENTICAL local fall-through)", () => { + it("local-owned / not-a-known-peer → undefined (BYTE-IDENTICAL local fall-through)", () => { expect(r.resolve("POST", "/session/ses-local")).toBeUndefined(); // local-owned - expect(r.resolve("POST", "/api/session/ses-unknown/message")).toBeUndefined(); // unowned expect(r.resolve("POST", "/api/session/ses-stranger/message")).toBeUndefined(); // owner not a serving peer }); + it("#1774: an UNOWNED session (no known owner) → the unknown-owner outcome: the caller tries local first", () => { + expect(r.resolve("POST", "/api/session/ses-unknown/message")).toEqual({ kind: "unknown-owner", sessionId: "ses-unknown" }); + expect(r.resolve("DELETE", "/session/ses-unknown")).toEqual({ kind: "unknown-owner", sessionId: "ses-unknown" }); + }); + it("/amicode/* and non-session paths → undefined (never a write target)", () => { expect(r.resolve("POST", "/amicode/fleet/status")).toBeUndefined(); expect(r.resolve("POST", "/amicode/profile")).toBeUndefined(); @@ -782,6 +786,97 @@ describe("#1774 AC1 — a peer-owned write keeps routing to the owner across a f }); }); +// ══════════════════════════════════════════════════════════════════════════════ +// #1774 AC4 — a session-scoped WRITE for an id with NO known owner is tried on the +// LOCAL engine first (an existence probe — never the write itself); if the local +// engine does not have it, the answer is a retryable owner-unknown 503 and the +// write is NOT executed anywhere. +// ══════════════════════════════════════════════════════════════════════════════ +describe("#1774 AC4 — an unknown-owner write is tried locally; a local miss is a retryable owner-unknown 503", () => { + let engine: Stub; + beforeAll(async () => { + // engine-like: GET /session/{id} → 200 for a held session, the engine's 404 otherwise + const requests: Stub["requests"] = []; + const server = http.createServer((req, res) => { + const u = new URL(req.url ?? "/", "http://stub"); + requests.push({ method: req.method ?? "GET", path: u.pathname }); + res.setHeader("content-type", "application/json"); + const held = /^\/(api\/)?session\/ses-here(\/|$)/.test(u.pathname); + if (!held) return void res.writeHead(404).end('{"name":"NotFoundError"}'); + res.writeHead(200).end(JSON.stringify({ ok: true, marker: "ENGINE-LIKE" })); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + engine = { + url: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, + marker: "ENGINE-LIKE", + requests, + stop: () => new Promise((r) => server.close(() => r())), + }; + }); + afterAll(async () => { + await engine?.stop(); + }); + + function boot(): AmicodeServiceServer { + const ownerMap = new SessionOwnerMap(); + const noPeers = (): PeerTransport | undefined => undefined; + const server = new AmicodeServiceServer({ password: PW }); + server.attachEngineProxy(new EngineProxy({ getUrl: () => engine.url })); + server.attachObservationReadPlane(createObservationReadPlane({ ownerMap, localMachineId: "macbook", peer: noPeers, localEngineUrl: () => engine.url })); + server.attachObservationWritePlane( + createObservationWritePlane({ ownerMap, localMachineId: "macbook", peer: noPeers, grantReader: () => undefined, localEngineUrl: () => engine.url }), + ); + return server; + } + + for (const [method, path] of [ + ["POST", "/api/session/ses-ghost/message"], + ["DELETE", "/session/ses-ghost"], + ["PATCH", "/api/session/ses-ghost"], + ] as const) { + it(`${method} ${path} (not on the local engine) → 503 owner-unknown + Retry-After; the write is never executed`, async () => { + const server = boot(); + const origin = (await server.start()).toString().replace(/\/$/, ""); + const before = engine.requests.length; + try { + const res = await fetch(`${origin}${path}`, { + method, + headers: { ...authed, "content-type": "application/json" }, + ...(method === "DELETE" ? {} : { body: JSON.stringify({ text: "hi" }) }), + }); + expect(res.status).toBe(503); + expect(res.headers.get("retry-after")).toBe(String(OWNER_UNKNOWN_RETRY_AFTER_S)); + expect(await res.json()).toEqual({ ok: false, error: OWNER_UNKNOWN_ERROR, reason: "owner-unknown", session_id: "ses-ghost", retry_after_s: OWNER_UNKNOWN_RETRY_AFTER_S }); + // only the existence probe reached the local engine — never the write + expect(engine.requests.slice(before)).toEqual([{ method: "GET", path: "/session/ses-ghost" }]); + } finally { + await server.stop(); + } + }); + } + + it("an unowned id the LOCAL engine holds → the write runs on the local engine (local first)", async () => { + const server = boot(); + const origin = (await server.start()).toString().replace(/\/$/, ""); + const before = engine.requests.length; + try { + const res = await fetch(`${origin}/api/session/ses-here/message`, { + method: "POST", + headers: { ...authed, "content-type": "application/json" }, + body: JSON.stringify({ text: "hi" }), + }); + expect(res.status).toBe(200); + expect(((await res.json()) as { marker: string }).marker).toBe("ENGINE-LIKE"); + expect(engine.requests.slice(before)).toEqual([ + { method: "GET", path: "/session/ses-here" }, + { method: "POST", path: "/api/session/ses-here/message" }, + ]); + } finally { + await server.stop(); + } + }); +}); + // ══════════════════════════════════════════════════════════════════════════════ // Byte-identity when UNATTACHED — a server with NO write plane consults nothing: // a POST for a peer-owned session falls through to the local engine (the From e5186c65cc408d0d724e41a0e547f4fa27bb60a9 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 19:45:00 -0400 Subject: [PATCH 37/60] fix(fleet): the client treats an unexpected end of the global stream as a gap and forces a resync (#1773) --- packages/app-bundle/manifest.json | 10 ++-- .../packages/app/src/context/server-sdk.tsx | 19 ++++++- .../app/src/context/sse-gap-frame.test.ts | 34 +++++++++++ .../app/src/context/stream-liveness.test.ts | 57 ++++++++++++++++++- .../app/src/context/stream-liveness.ts | 37 ++++++++++++ 5 files changed, 150 insertions(+), 7 deletions(-) diff --git a/packages/app-bundle/manifest.json b/packages/app-bundle/manifest.json index 735a70e53..b7d7c66c2 100644 --- a/packages/app-bundle/manifest.json +++ b/packages/app-bundle/manifest.json @@ -183,7 +183,7 @@ "packages/app/src/context/models.tsx": "8904525767b36ed5ba93e85a0fddb9bf737a8d066dfff084380ce1a67182f535", "packages/app/src/context/platform.tsx": "edd9b1773feb5c084b922a57b6914f6a473eed558b44f9de0eeb1c4d4e2b49da", "packages/app/src/context/server-sdk.test.ts": "833920190c25089a5c3ed9849a62b021d2e0e2e4fb76196af8ddc9e6aa9817ba", - "packages/app/src/context/server-sdk.tsx": "36cfd55e634dbb974d4d6785281fedc80f4ddd539090f859fa7156f959ab500d", + "packages/app/src/context/server-sdk.tsx": "c1d26fbeaa37549ce17b348ea2fa61f424e730caa4518146542c23123a1c1390", "packages/app/src/context/server-session-child-active.test.ts": "de8c467f67f88b6cce351c1805ec90728cf238eb1ef32f39ffdbbf3de4dcec22", "packages/app/src/context/server-session-orphan-part.test.ts": "1d635374c0bfff25a01825061aabac7334e5cdaedca2dab9f3d9467ff543caea", "packages/app/src/context/server-session.test.ts": "30cd24bb1345376fd1a165d65677dee5c36d586efa7969d3b4844967040fda59", @@ -200,14 +200,14 @@ "packages/app/src/context/session-working.ts": "aa3f70d326c481524920502aae5fc0cdc4dc2c5067dd94969242629154358e9c", "packages/app/src/context/settings.tsx": "19e078e81620ffa21a92797335009b84d4311485a37c5ede2f76906b734292ad", "packages/app/src/context/split.tsx": "891aeb290b369b327a63ce548dcaab1daca08e41fab430648787be7f84e53ec0", - "packages/app/src/context/sse-gap-frame.test.ts": "cf93c169015d66cebbf25752bc5a99296af80a7253bde8b6e565d2c0e50410f3", + "packages/app/src/context/sse-gap-frame.test.ts": "4b76900f9e28680ea1aaed8b391518ce08d0f04e7538a336b584834b913fa62b", "packages/app/src/context/sse-gap-frame.ts": "2f64fde540a305579762c5276721d4bbfbf7c862131aaf4462e8a4e8f32c5960", "packages/app/src/context/sse-origin.test.ts": "71cd3aa996a51d7e99236a49cede4769e520e20e53f57c6920fe441f2abb0c7d", "packages/app/src/context/sse-origin.ts": "da9a231907dc50d87cf01758177bee0d4a0b0165d8adb5e152db8eca29dfd9f5", "packages/app/src/context/stream-gap.test.ts": "b7203445564f44fd1d966ef684a11cb9b5ec567d9ab54f1b7617726fdf44d3f5", "packages/app/src/context/stream-gap.ts": "2fd8eba30a6112cebf38e33c9418ab2b7d9c9d466f76729d828d5ab52d56c5ec", - "packages/app/src/context/stream-liveness.test.ts": "63a06ca7fef3e38c0c9abfa23df3d9043b1a804f3c527400d08ed9eb32c466b0", - "packages/app/src/context/stream-liveness.ts": "94fe6fe8a776eb0de6903546f653c89f4577cf7628e7f051afa83280e391d16f", + "packages/app/src/context/stream-liveness.test.ts": "2759eb2233f8ff6ced02957f971e756fe525575b2c8aefcbd7749470dec960f2", + "packages/app/src/context/stream-liveness.ts": "875fe871312ad15f6376b933659c152f2241b8101e70de97ed410a720cbbafaa", "packages/app/src/context/tab-migration.ts": "569e69cd3425917c7e195f91c3a838abea68b616267ea7511f3ccf01d94ac01e", "packages/app/src/context/tabs.tsx": "06d6ddb6d50727a4e0a01ac1ca4a70b06bf63a4e5f881526aead54ccde62472b", "packages/app/src/context/vault-panel.ts": "f91cb5fb49a9f0d46bd2a1b131f9a218b30147e2ad3617d2a2dc4f1e14599a53", @@ -959,7 +959,7 @@ "packages/ui/src/v2/components/text-shimmer-v2.tsx" ], "exceptions": [], - "extracted_at": "2026-10-12T22:40:33.106Z", + "extracted_at": "2026-10-12T23:44:37.584Z", "per_package": { "packages/app": { "A": 237, diff --git a/packages/app-bundle/overlay/packages/app/src/context/server-sdk.tsx b/packages/app-bundle/overlay/packages/app/src/context/server-sdk.tsx index 65559c790..a1825006c 100644 --- a/packages/app-bundle/overlay/packages/app/src/context/server-sdk.tsx +++ b/packages/app-bundle/overlay/packages/app/src/context/server-sdk.tsx @@ -15,7 +15,7 @@ import { useGlobal } from "./global" import { ServerScope } from "@/utils/server-scope" import { attachOwnerHeaderIfArmed } from "@/components/remote-create-arm" import { parseCursorNamespaces, resolveEventOrigin } from "./sse-origin" -import { shouldReconnectIdleStream } from "./stream-liveness" +import { createStreamEndResync, shouldReconnectIdleStream } from "./stream-liveness" import { detectServerProtocol, type ServerProtocol } from "@/utils/server-protocol" import { createCompatibleApi, type CompatibleApi } from "@/utils/server-compat" @@ -475,6 +475,12 @@ function createServerSdkContextBase(server: ServerConnection.Any, scope: ServerS // nominally connected, so the disconnect→connect edge never fires and the rail // would otherwise never self-heal a stranded view). const [resyncCount, setResyncCount] = createSignal(0) + // #1773: an unexpected NORMAL end of the global stream is a loss (the relays + // never end a listener cleanly except on shutdown). The loop arms this latch + // on such an end; the reconnected stream's first frame advances resyncCount, + // so the viewed transcript and the entity rail refetch once the stream is live + // again. A user stop, a page hide, or the client's own abort arm nothing. + const streamEndResync = createStreamEndResync() // #1584 client-side liveness watchdog. The v1 event iterator neither throws // nor completes when the socket goes silently half-open (sleep/wake, Wi-Fi @@ -560,6 +566,10 @@ function createServerSdkContextBase(server: ServerConnection.Any, scope: ServerS if (!streamGeneration.accepts(active)) return streamErrorLogged = false lastFrameAt = Date.now() + // #1773: the first frame after an unexpected end — the stream is live + // again, so force the resync now (the refetch cannot miss the events + // between the end and this re-subscription). + if (streamEndResync.frame()) setResyncCount((n) => n + 1) const legacy = "payload" in event if (legacy && event.payload.type === "sync") continue const directory = legacy ? (event.directory ?? "global") : (event.location?.directory ?? "global") @@ -629,6 +639,13 @@ function createServerSdkContextBase(server: ServerConnection.Any, scope: ServerS yielded = Date.now() await wait(0) } + // #1773: the iterator COMPLETED — a clean end of the global stream. If + // the client neither aborted this attempt nor stopped the loop, it is + // an unexpected end: a loss. Arm the resync for the reconnected stream. + streamEndResync.ended({ + attemptAborted: currentAttempt.signal.aborted, + stopped: abort.signal.aborted || !started || !streamGeneration.accepts(active), + }) } catch (error) { if (streamGeneration.accepts(active) && !isStreamClosed(error, currentAttempt.signal)) setStreamStatus("disconnected") diff --git a/packages/app-bundle/overlay/packages/app/src/context/sse-gap-frame.test.ts b/packages/app-bundle/overlay/packages/app/src/context/sse-gap-frame.test.ts index b4ea7025c..f7b7e585c 100644 --- a/packages/app-bundle/overlay/packages/app/src/context/sse-gap-frame.test.ts +++ b/packages/app-bundle/overlay/packages/app/src/context/sse-gap-frame.test.ts @@ -60,3 +60,37 @@ describe("#1617 — gap interception is wired at both client seams", () => { expect(syncSource).toMatch(/server\.connected"\s*&&\s*Date\.now\(\)\s*-\s*lastConnectedQueueAt/) }) }) + +// #1773 — an unexpected NORMAL end of the global stream is a gap too (defense in +// depth: kept even once replay exists). The relay chain now error-closes every +// loss, but a clean end the client did not ask for still means events may have +// been dropped. The decision (`createStreamEndResync`, stream-liveness.ts) is +// unit-tested there; these pins assert the loop actually wires it — falsifiable: +// deleting either call, or reporting the end without the abort/stop state, reds. +describe("#1773 — an unexpected end of the global stream advances the forced-resync signal", () => { + const sdkSource = readFileSync(join(import.meta.dir, "server-sdk.tsx"), "utf8") + const loop = sdkSource.indexOf("for await (const event of events)") + const loopCatch = sdkSource.indexOf("} catch (error) {", loop) + + test("the loop's NORMAL completion (the iterator ended without throwing) reports the end", () => { + expect(loop).toBeGreaterThan(0) + const body = sdkSource.slice(loop, loopCatch) + // after the for-await body's last statement and before the catch: the normal-end path + const afterLoop = body.slice(body.lastIndexOf("await wait(0)")) + expect(afterLoop).toContain("streamEndResync.ended(") + }) + + test("a user stop / page hide / the client's own abort are NOT a loss: the end carries both states", () => { + const body = sdkSource.slice(loop, loopCatch) + expect(body).toMatch(/attemptAborted:\s*currentAttempt\.signal\.aborted/) + // stopped = the root abort (teardown), the started latch (stop()/pagehide), the generation fence (reset) + expect(body).toMatch(/stopped:\s*abort\.signal\.aborted\s*\|\|\s*!started\s*\|\|\s*!streamGeneration\.accepts\(active\)/) + }) + + test("the reconnected stream's first frame advances the forced-resync signal (the transcript + entity rail refetch)", () => { + const body = sdkSource.slice(loop, loopCatch) + expect(body).toMatch(/if \(streamEndResync\.frame\(\)\) setResyncCount\(\(n\) => n \+ 1\)/) + // the signal the transcript (pages/session.tsx) and the entity rail read + expect(sdkSource).toMatch(/resyncCount,\s*\n/) + }) +}) diff --git a/packages/app-bundle/overlay/packages/app/src/context/stream-liveness.test.ts b/packages/app-bundle/overlay/packages/app/src/context/stream-liveness.test.ts index b2364a6f5..89984dbed 100644 --- a/packages/app-bundle/overlay/packages/app/src/context/stream-liveness.test.ts +++ b/packages/app-bundle/overlay/packages/app/src/context/stream-liveness.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test" -import { shouldReconnectIdleStream } from "./stream-liveness" +import { createStreamEndResync, isUnexpectedStreamEnd, shouldReconnectIdleStream } from "./stream-liveness" // #1584 — the client-side SSE liveness watchdog's DECISION, extracted as a // zero-dependency pure fn so it runs in-place (`bun test --conditions=solid`) @@ -37,3 +37,58 @@ describe("shouldReconnectIdleStream (#1584) — the liveness watchdog's decision ).toBe(false) }) }) + +// #1773 — an unexpected NORMAL end of the global stream is a loss. The global +// stream never ends on purpose except on shutdown, so a clean end the client did +// not ask for means events were dropped somewhere on the relay chain (a severed +// fan-in listener, a relay that ended instead of erroring). The loop used to +// reconnect 250 ms later without refetching anything. Now it forces a resync — +// but NOT for a user-initiated stop, a page hide, or the client's own abort. +describe("isUnexpectedStreamEnd (#1773) — which normal ends are a loss", () => { + test("a completed stream the client neither aborted nor stopped → unexpected (a loss)", () => { + expect(isUnexpectedStreamEnd({ attemptAborted: false, stopped: false })).toBe(true) + }) + + test("a user-initiated stop / page hide (the loop is stopped) → expected, no resync", () => { + expect(isUnexpectedStreamEnd({ attemptAborted: true, stopped: true })).toBe(false) + expect(isUnexpectedStreamEnd({ attemptAborted: false, stopped: true })).toBe(false) + }) + + test("the client's own abort (watchdog, error handling, reset) → expected, no resync here", () => { + expect(isUnexpectedStreamEnd({ attemptAborted: true, stopped: false })).toBe(false) + }) +}) + +describe("createStreamEndResync (#1773) — an unexpected end resyncs once the stream is back", () => { + test("after an unexpected end, the reconnected stream's FIRST frame fires the resync — exactly once", () => { + const resync = createStreamEndResync() + expect(resync.frame()).toBe(false) // a healthy stream never resyncs + resync.ended({ attemptAborted: false, stopped: false }) + expect(resync.pending()).toBe(true) + // the refetch waits for the re-subscription (the first frame on the new + // stream), so events between the end and the reconnect cannot slip past it + expect(resync.frame()).toBe(true) + expect(resync.frame()).toBe(false) + expect(resync.pending()).toBe(false) + }) + + test("a user stop or a page hide arms nothing — the next stream resyncs nothing", () => { + const resync = createStreamEndResync() + resync.ended({ attemptAborted: true, stopped: true }) // stop() / pagehide + expect(resync.pending()).toBe(false) + expect(resync.frame()).toBe(false) + }) + + test("the client's own abort arms nothing (its own path already heals: watchdog bump, disconnect edge)", () => { + const resync = createStreamEndResync() + resync.ended({ attemptAborted: true, stopped: false }) + expect(resync.frame()).toBe(false) + }) + + test("several unexpected ends before the stream is back still resync once", () => { + const resync = createStreamEndResync() + resync.ended({ attemptAborted: false, stopped: false }) + resync.ended({ attemptAborted: false, stopped: false }) + expect([resync.frame(), resync.frame()]).toEqual([true, false]) + }) +}) diff --git a/packages/app-bundle/overlay/packages/app/src/context/stream-liveness.ts b/packages/app-bundle/overlay/packages/app/src/context/stream-liveness.ts index 9fbd435ed..04fa5aa18 100644 --- a/packages/app-bundle/overlay/packages/app/src/context/stream-liveness.ts +++ b/packages/app-bundle/overlay/packages/app/src/context/stream-liveness.ts @@ -29,3 +29,40 @@ export function shouldReconnectIdleStream(input: { }): boolean { return input.status === "connected" && input.now - input.lastFrameAt > input.staleMs } + +// #1773 — an unexpected NORMAL end of the global stream is a loss. The global +// stream never ends on purpose except on shutdown, so a clean end the client did +// not ask for means events were dropped somewhere on the relay chain (a severed +// fan-in listener, a relay that ended instead of erroring). The loop used to +// reconnect 250 ms later without refetching anything: the viewed transcript and +// the entity rail stayed silently behind. Kept even once replay exists (defense +// in depth). + +/** Is this completion of the stream iterator a LOSS (a resync is owed)? Only + * when the client neither aborted the attempt (its watchdog, its error + * handling, a reset — those paths heal on their own) nor stopped the loop (a + * user-initiated stop, a page hide, teardown). */ +export function isUnexpectedStreamEnd(input: { attemptAborted: boolean; stopped: boolean }): boolean { + return !input.attemptAborted && !input.stopped +} + +/** The stream loop's end-of-stream resync latch. `ended()` is told every normal + * completion; an unexpected one ARMS a resync, which fires on the reconnected + * stream's FIRST frame (`frame()` returns true exactly once) — after the + * re-subscription is live, so a refetch cannot run ahead of the reconnect and + * miss the events in between. */ +export function createStreamEndResync() { + let armed = false + return { + ended(input: { attemptAborted: boolean; stopped: boolean }): void { + if (isUnexpectedStreamEnd(input)) armed = true + }, + /** A frame arrived: true when it is the first since an unexpected end. */ + frame(): boolean { + if (!armed) return false + armed = false + return true + }, + pending: (): boolean => armed, + } +} From 6668193fcd0c700f56990c752a3fcb23590e0e5f Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 19:45:42 -0400 Subject: [PATCH 38/60] test(fleet): a just-created local session routes locally, and a remote create's pending binding survives a failed peer refresh (#1774) --- ...code_service_observe_write_routing.test.ts | 115 ++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/packages/extension/test/amicode_service_observe_write_routing.test.ts b/packages/extension/test/amicode_service_observe_write_routing.test.ts index 3290e7667..de59b6cc3 100644 --- a/packages/extension/test/amicode_service_observe_write_routing.test.ts +++ b/packages/extension/test/amicode_service_observe_write_routing.test.ts @@ -877,6 +877,121 @@ describe("#1774 AC4 — an unknown-owner write is tried locally; a local miss is }); }); +// ══════════════════════════════════════════════════════════════════════════════ +// #1774 AC5 — the two "just created" bridges keep working under the unknown-owner +// outcome: a just-created LOCAL session (not yet in the owner map) is found by the +// local-first probe and served locally; a just-created REMOTE session's pending +// binding keeps routing to its peer — even across a refresh in which that peer +// failed — and never consults the local engine. +// ══════════════════════════════════════════════════════════════════════════════ +describe("#1774 AC5 — a just-created local session routes locally; the pending remote-create binding keeps working", () => { + /** An engine that CREATES sessions: POST /session mints ses-made-N and holds it. */ + async function startCreatingEngine() { + const held = new Set(); + const requests: Array<{ method: string; path: string }> = []; + let n = 0; + const server = http.createServer((req, res) => { + const u = new URL(req.url ?? "/", "http://stub"); + requests.push({ method: req.method ?? "GET", path: u.pathname }); + res.setHeader("content-type", "application/json"); + if (u.pathname === "/session" && req.method === "POST") { + const id = `ses-made-${++n}`; + held.add(id); + return void res.writeHead(200).end(JSON.stringify({ id })); + } + const id = /^\/(?:api\/)?session\/([^/]+)/.exec(u.pathname)?.[1]; + if (id !== undefined && !held.has(id)) return void res.writeHead(404).end('{"name":"NotFoundError"}'); + res.writeHead(200).end(JSON.stringify({ ok: true, marker: "LOCAL-CREATING", id })); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + return { + url: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, + requests, + stop: () => new Promise((r) => server.close(() => r())), + }; + } + + it("create locally, then read and write it at once (the owner map has not refreshed) → both served by the LOCAL engine", async () => { + const engine = await startCreatingEngine(); + const ownerMap = new SessionOwnerMap(); // the map knows nothing of the new session + const noPeers = (): PeerTransport | undefined => undefined; + const server = new AmicodeServiceServer({ password: PW }); + server.attachEngineProxy(new EngineProxy({ getUrl: () => engine.url })); + server.attachObservationReadPlane(createObservationReadPlane({ ownerMap, localMachineId: "macbook", peer: noPeers, localEngineUrl: () => engine.url })); + server.attachObservationWritePlane( + createObservationWritePlane({ ownerMap, localMachineId: "macbook", peer: noPeers, grantReader: () => undefined, localEngineUrl: () => engine.url }), + ); + const origin = (await server.start()).toString().replace(/\/$/, ""); + try { + const created = await fetch(`${origin}/session`, { method: "POST", headers: { ...authed, "content-type": "application/json" }, body: "{}" }); + const { id } = (await created.json()) as { id: string }; + expect(ownerMap.resolveOwner(id)).toBeUndefined(); + + const read = await fetch(`${origin}/session/${id}`, { headers: authed }); + expect(read.status).toBe(200); + expect(((await read.json()) as { marker: string }).marker).toBe("LOCAL-CREATING"); + + const write = await fetch(`${origin}/session/${id}/message`, { + method: "POST", + headers: { ...authed, "content-type": "application/json" }, + body: JSON.stringify({ text: "hi" }), + }); + expect(write.status).toBe(200); + expect(((await write.json()) as { marker: string }).marker).toBe("LOCAL-CREATING"); + expect(engine.requests.some((r) => r.method === "POST" && r.path === `/session/${id}/message`)).toBe(true); + } finally { + await server.stop(); + await engine.stop(); + } + }); + + it("a remote create's pending binding survives a refresh in which its peer FAILED: the follow-up read reaches the peer, local never consulted", async () => { + const engine = await startCreatingEngine(); + const peerSrv = http.createServer((req, res) => { + res.setHeader("content-type", "application/json"); + if (req.method === "POST") return void res.writeHead(200).end(JSON.stringify({ id: "ses-remote-pending" })); + res.writeHead(200).end(JSON.stringify({ ok: true, marker: "PEER-PENDING" })); + }); + await new Promise((r) => peerSrv.listen(0, "127.0.0.1", r)); + const peerUrl = `http://127.0.0.1:${(peerSrv.address() as AddressInfo).port}`; + const ownerMap = new SessionOwnerMap(); + const peer = (id: string): PeerTransport | undefined => (id === "studio" ? { getUrl: () => peerUrl, token: "tok-studio" } : undefined); + const server = new AmicodeServiceServer({ password: PW }); + server.attachEngineProxy(new EngineProxy({ getUrl: () => engine.url })); + server.attachObservationReadPlane(createObservationReadPlane({ ownerMap, localMachineId: "macbook", peer, localEngineUrl: () => engine.url })); + server.attachObservationWritePlane( + createObservationWritePlane({ + ownerMap, + localMachineId: "macbook", + peer, + grantReader: (o) => (o === "studio" ? { scope: "control", state: "active" } : undefined), + localEngineUrl: () => engine.url, + }), + ); + const origin = (await server.start()).toString().replace(/\/$/, ""); + try { + const created = await fetch(`${origin}/session`, { + method: "POST", + headers: { ...authed, [OWNER_ROUTING_HEADER]: "studio", "content-type": "application/json" }, + body: "{}", + }); + expect(((await created.json()) as { id: string }).id).toBe("ses-remote-pending"); + // the next owner-map refresh: studio's fetch timed out (an empty slice) + ownerMap.update([], [{ machineId: "studio", outcome: "timeout", reason: "fetch-failed" }]); + expect(ownerMap.resolveOwner("ses-remote-pending")).toBe("studio"); + + const read = await fetch(`${origin}/session/ses-remote-pending`, { headers: authed }); + expect(read.status).toBe(200); + expect(((await read.json()) as { marker: string }).marker).toBe("PEER-PENDING"); + expect(engine.requests).toEqual([]); // the local engine was never asked + } finally { + await server.stop(); + await new Promise((r) => peerSrv.close(() => r())); + await engine.stop(); + } + }); +}); + // ══════════════════════════════════════════════════════════════════════════════ // Byte-identity when UNATTACHED — a server with NO write plane consults nothing: // a POST for a peer-owned session falls through to the local engine (the From 6a13937c8ef5c5185c0213cb705be989900022f8 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 19:46:06 -0400 Subject: [PATCH 39/60] docs(fleet): fan-in comments name the #1773 sever path (#1773) --- .../extension/src/amicode_service/sse_fanin_aggregator.ts | 5 +++-- packages/extension/src/amicode_service/sse_fanin_driver.ts | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/extension/src/amicode_service/sse_fanin_aggregator.ts b/packages/extension/src/amicode_service/sse_fanin_aggregator.ts index d5f784afd..78ddcf939 100644 --- a/packages/extension/src/amicode_service/sse_fanin_aggregator.ts +++ b/packages/extension/src/amicode_service/sse_fanin_aggregator.ts @@ -50,8 +50,9 @@ export interface SseFrameSource { /** #1617 — resume pulling once the downstream has drained. */ resume?(): void; /** #1771 — once `next()` has resolved null: did the upstream END cleanly or - * fail with an ERROR? Census only (the close reason); both still end the - * source the same way. Absent / undefined ⇒ a clean end. */ + * fail with an ERROR? The census's close reason (and, #1773, the reason a + * fleet-of-one listener is severed under); both still end the source the + * same way. Absent / undefined ⇒ a clean end. */ endReason?(): "end" | "error" | undefined; } diff --git a/packages/extension/src/amicode_service/sse_fanin_driver.ts b/packages/extension/src/amicode_service/sse_fanin_driver.ts index 98f61793b..a4330fc7e 100644 --- a/packages/extension/src/amicode_service/sse_fanin_driver.ts +++ b/packages/extension/src/amicode_service/sse_fanin_driver.ts @@ -365,7 +365,8 @@ class FanInConnection implements CensusConnection { // namespace (its composite cursor), NOT the stale connect-time id. A // re-open of an arm that already delivered is a NON-INITIAL re-open — the // aggregator emits one composite-mode gap (reopenArm is a no-op for a - // never-delivered arm and in fleet-of-one, so calling it here is safe). + // never-delivered arm; a peer arm is only active in composite mode, so + // its verdict is never `sever`). this.reopenAndTrack(machineId, this.deps.peerBaseUrl(machineId), this.peerAuth(machineId)); } // Close the real upstream for any peer arm no longer active (dark peer, or @@ -387,6 +388,8 @@ class FanInConnection implements CensusConnection { // until reload"). Guard at the call site so a HEALTHY local arm is never // re-opened or duplicated (openArm early-returns on a present source too); // resume from LOCAL's live cursor for lossless replay — mirrors start(). + // #1773: in fleet-of-one a local arm that delivered is not re-opened at all — + // reopenAndTrack severs the listener (an error close) so the client resyncs. if (!this.sources.has(LOCAL_NAMESPACE)) { this.reopenAndTrack(LOCAL_NAMESPACE, this.deps.localEventUrl(), this.authHeader); } From 1c901c5c616c392557c536fe11a435dbd36d4af8 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 19:47:00 -0400 Subject: [PATCH 40/60] =?UTF-8?q?fix(fleet):=20the=20owner-map=20refresh?= =?UTF-8?q?=20line=20says=20what=20the=20map=20did=20with=20a=20failed=20p?= =?UTF-8?q?eer's=20entries=20=E2=80=94=20retained,=20expired,=20dropped,?= =?UTF-8?q?=20unavailable=20(#1774)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../amicode_service/session_multiplexer.ts | 25 ++++++++-- ...de_service_multiplex_owner_routing.test.ts | 50 +++++++++++++++++++ 2 files changed, 70 insertions(+), 5 deletions(-) diff --git a/packages/extension/src/amicode_service/session_multiplexer.ts b/packages/extension/src/amicode_service/session_multiplexer.ts index 368234eaf..a1dcd3f10 100644 --- a/packages/extension/src/amicode_service/session_multiplexer.ts +++ b/packages/extension/src/amicode_service/session_multiplexer.ts @@ -351,11 +351,26 @@ export interface OwnerMapFeedOpts { } /** #1771: one per-peer refresh line: outcome, duration, entry count, and for a - * failure its named reason and detail. */ -export function ownerRefreshLine(p: OwnerRefreshPeer): string { + * failure its named reason and detail. #1774: plus what the owner map did with + * a failed peer's entries (`retention=`), so a retained peer reads as retained — + * `entries` stays this refresh's own count. Absent for an answered peer. */ +export function ownerRefreshLine(p: OwnerRefreshPeer, retention?: OwnerRetention): string { const reason = p.reason !== undefined ? ` reason=${p.reason}` : ""; const detail = p.detail !== undefined ? ` detail=${JSON.stringify(p.detail)}` : ""; - return `[owner-map] refresh peer=${p.machineId} outcome=${p.outcome} duration_ms=${p.durationMs} entries=${p.entries}${reason}${detail}`; + return `[owner-map] refresh peer=${p.machineId} outcome=${p.outcome} duration_ms=${p.durationMs} entries=${p.entries}${reason}${detail}${retentionFields(retention)}`; +} + +function retentionFields(r: OwnerRetention | undefined): string { + if (r === undefined) return ""; + switch (r.kind) { + case "retained": + return ` retention=retained held=${r.held} age_ms=${r.ageMs}`; + case "expired": + case "dropped": + return ` retention=${r.kind} dropped=${r.dropped}`; + case "unavailable": + return " retention=unavailable"; + } } /** The default owner-map refresh cadence. */ @@ -381,8 +396,8 @@ export class OwnerMapFeed { if (this.stopped) return; // a stop() raced the in-flight build // #1774: the per-peer outcomes scope the update — a failed peer keeps its // last-good entries for the retention window instead of vanishing. - this.opts.ownerMap.update(projection.sessions ?? [], projection.peers); - for (const peer of projection.peers ?? []) this.log(ownerRefreshLine(peer)); + const retention = this.opts.ownerMap.update(projection.sessions ?? [], projection.peers); + for (const peer of projection.peers ?? []) this.log(ownerRefreshLine(peer, retention.get(peer.machineId))); } private log(line: string): void { diff --git a/packages/extension/test/amicode_service_multiplex_owner_routing.test.ts b/packages/extension/test/amicode_service_multiplex_owner_routing.test.ts index 3bf1e46cf..458204e73 100644 --- a/packages/extension/test/amicode_service_multiplex_owner_routing.test.ts +++ b/packages/extension/test/amicode_service_multiplex_owner_routing.test.ts @@ -233,6 +233,56 @@ describe("#1771 — each ownership refresh logs a per-peer outcome record", () = await expect(feed.refreshOnce()).resolves.toBeUndefined(); expect(lines).toEqual(['[owner-map] refresh failed error="peer fan-out failed"']); }); + + it("#1774: a failed peer's line says what the map did with its entries — retained, expired, dropped, still unavailable", async () => { + let t = 0; + const ownerMap = new SessionOwnerMap({ now: () => t }); + const lines: string[] = []; + let next: { sessions: SessionEntry[]; peers: OwnerRefreshPeer[] } = { + sessions: [ + { id: "ses-s1", amicode_owner: { owner_machine_id: "studio", owner_name: "Studio", is_local: false } }, + { id: "ses-s2", amicode_owner: { owner_machine_id: "studio", owner_name: "Studio", is_local: false } }, + { id: "ses-m1", amicode_owner: { owner_machine_id: "mini", owner_name: "Mini", is_local: false } }, + ], + peers: [ + { machineId: "studio", outcome: "ok", durationMs: 40, entries: 2 }, + { machineId: "mini", outcome: "ok", durationMs: 30, entries: 1 }, + ], + }; + const feed = new OwnerMapFeed({ ownerMap, log: (l) => lines.push(l), buildProjection: async () => next }); + await feed.refreshOnce(); + expect(lines.splice(0)).toEqual([ + "[owner-map] refresh peer=studio outcome=ok duration_ms=40 entries=2", + "[owner-map] refresh peer=mini outcome=ok duration_ms=30 entries=1", + ]); + + t = 15_000; + next = { + sessions: [], + peers: [ + { machineId: "studio", outcome: "timeout", durationMs: 10_004, entries: 0, reason: "fetch-failed", detail: "The operation was aborted due to timeout" }, + { machineId: "mini", outcome: "unauthorized", durationMs: 9, entries: 0, reason: "unauthorized" }, + ], + }; + await feed.refreshOnce(); + expect(lines.splice(0)).toEqual([ + '[owner-map] refresh peer=studio outcome=timeout duration_ms=10004 entries=0 reason=fetch-failed detail="The operation was aborted due to timeout" retention=retained held=2 age_ms=15000', + "[owner-map] refresh peer=mini outcome=unauthorized duration_ms=9 entries=0 reason=unauthorized retention=dropped dropped=1", + ]); + + t = OWNER_RETENTION_MS + 1; + next = { sessions: [], peers: [{ machineId: "studio", outcome: "error", durationMs: 12, entries: 0, reason: "fetch-failed", detail: "HTTP 502" }] }; + await feed.refreshOnce(); + expect(lines.splice(0)).toEqual([ + '[owner-map] refresh peer=studio outcome=error duration_ms=12 entries=0 reason=fetch-failed detail="HTTP 502" retention=expired dropped=2', + ]); + + t += 5_000; + await feed.refreshOnce(); + expect(lines.splice(0)).toEqual([ + '[owner-map] refresh peer=studio outcome=error duration_ms=12 entries=0 reason=fetch-failed detail="HTTP 502" retention=unavailable', + ]); + }); }); // ══════════════════════════════════════════════════════════════════════════════ From df193f3857542f57ed846e13330fd889ef940a7f Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 19:48:01 -0400 Subject: [PATCH 41/60] refactor(fleet): an outcome-less owner-map update just forgets peer retention; feed docs describe the per-peer contract (#1774) --- .../src/amicode_service/session_multiplexer.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/extension/src/amicode_service/session_multiplexer.ts b/packages/extension/src/amicode_service/session_multiplexer.ts index a1dcd3f10..ede8cb22c 100644 --- a/packages/extension/src/amicode_service/session_multiplexer.ts +++ b/packages/extension/src/amicode_service/session_multiplexer.ts @@ -189,7 +189,7 @@ export class SessionOwnerMap { else fresh.set(owner, [held]); } } - const retention = peers === undefined ? this.replaceAll(fresh) : this.applyPeerOutcomes(fresh, peers); + const retention = peers === undefined ? this.forgetPeers() : this.applyPeerOutcomes(fresh, peers); // Prune the pending overlay: drop entries the authoritative projection now // carries (superseded), and any that have outlived the TTL. const cutoff = this.now() - PENDING_OWNER_TTL_MS; @@ -204,12 +204,11 @@ export class SessionOwnerMap { if (e.directory !== undefined) this.directories.set(e.id, e.directory); } - /** No per-peer outcomes: the projection is authoritative for every owner. */ - private replaceAll(fresh: Map): Map { + /** No per-peer outcomes: the projection is authoritative for every owner, so + * there is nothing to retain and no one to name unavailable. */ + private forgetPeers(): Map { this.lastGood.clear(); this.unavailablePeers.clear(); - const at = this.now(); - for (const [owner, entries] of fresh) this.lastGood.set(owner, { at, entries }); return new Map(); } @@ -332,15 +331,18 @@ export function fanInReachable( * Honesty: a projection BUILD failure (a peer fan-out that threw) leaves the * LAST-GOOD map intact — it never clears the map on error, so a transient blip * does not silently reroute owned sessions to local. Only a SUCCESSFUL - * projection re-populates the map (clear + repopulate, the SessionOwnerMap.update - * contract). */ + * projection re-populates the map (the SessionOwnerMap.update contract). + * #1774: the builder itself never throws — a failed PEER is an empty slice plus + * its #1771 outcome in `peers`, which the feed hands to `update` so that peer + * keeps its last-good entries for the retention window. */ export interface OwnerMapFeedOpts { ownerMap: SessionOwnerMap; /** Build the fleet-wide projection whose `.sessions` carry the amicode_owner * overlay. Injected so the feed is unit-testable without a real fan-out. * #1771: `peers` (optional) carries each peer's refresh outcome — the feed * logs one line per peer, so a failed peer refresh is visible even though - * neither the builder nor the feed ever throws. */ + * neither the builder nor the feed ever throws. #1774: and scopes the map + * update per peer (retention); absent → every owner is replaced. */ buildProjection: () => Promise<{ sessions: SessionEntry[]; peers?: OwnerRefreshPeer[] }>; /** The refresh cadence in ms. Default 5000. */ intervalMs?: number; From edfa91799d35b5894a65b5b630fd4ab8e9985218 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 20:21:01 -0400 Subject: [PATCH 42/60] test(fleet): installer write-mode runs reach launchctl only through a recording seam (#1775) install.sh gains two machine-scoped seams, both defaulting to today's behavior: AMICO_FLEET_LAUNCH_AGENTS_DIR (the LaunchAgents dir) and AMICO_FLEET_LAUNCHCTL (the launchctl binary). Every installer run in fleet_scripts_projection now gets a recording stub at the seam plus a recording PATH backstop, so a write-mode test can never start a real ssh agent on the host (the macOS settings test previously reached the real /bin/launchctl with a temp tunnel plist). --- .../test/fleet_scripts_projection.test.ts | 49 +++++++++++++++++++ packages/extension/tools/fleet/install.sh | 19 ++++--- tools/fleet/install.sh | 19 ++++--- 3 files changed, 73 insertions(+), 14 deletions(-) diff --git a/packages/extension/test/fleet_scripts_projection.test.ts b/packages/extension/test/fleet_scripts_projection.test.ts index 3f98c5a4a..82ebc8a66 100644 --- a/packages/extension/test/fleet_scripts_projection.test.ts +++ b/packages/extension/test/fleet_scripts_projection.test.ts @@ -148,10 +148,54 @@ function fakeFrozenBinary(): void { chmodSync(join(dir, "opencode"), 0o755); } +/** #1775: NO run in this file may reach the real OS service manager — the + * installer writes launch agents and (re)loads them, and a real `launchctl + * load` of a fabricated plist would start a real ssh agent on the test host. + * Every run gets TWO recording stubs: + * - the SEAM: `$AMICO_FLEET_LAUNCHCTL` → `/stubs/launchctl` (OFF the + * PATH), appending one argv line per call to `/launchctl.argv`; + * - the BACKSTOP: a `launchctl` + `systemctl` FIRST on the child PATH that + * append to `/launchctl-path.argv` — reached only if the installer + * ever bypasses its seam (a test asserts it never does). + * `launchctl getenv ` prints `/stubs/getenv-` when present. */ +function writeLoaderStubs(): void { + const stubs = join(tmp, "stubs"); + const bin = join(tmp, "fakebin"); + mkdirSync(stubs, { recursive: true }); + mkdirSync(bin, { recursive: true }); + const recorder = (log: string): string => [ + "#!/usr/bin/env bash", + `printf '%s\\n' "$*" >> "${log}"`, + `if [[ "\${1:-}" == "getenv" && -f "${stubs}/getenv-\${2:-}" ]]; then cat "${stubs}/getenv-\${2:-}"; fi`, + "exit 0", + "", + ].join("\n"); + writeFileSync(join(stubs, "launchctl"), recorder(join(tmp, "launchctl.argv"))); + chmodSync(join(stubs, "launchctl"), 0o755); + for (const name of ["launchctl", "systemctl"]) { + writeFileSync(join(bin, name), recorder(join(tmp, "launchctl-path.argv"))); + chmodSync(join(bin, name), 0o755); + } +} + +/** The seam stub's recorded argv lines (empty when it was never called). */ +function launchctlCalls(): string[] { + const log = join(tmp, "launchctl.argv"); + return existsSync(log) ? readFileSync(log, "utf8").split("\n").filter((l) => l !== "") : []; +} + +/** Calls that bypassed the seam and hit the PATH backstop (must stay empty). */ +function backstopCalls(): string[] { + const log = join(tmp, "launchctl-path.argv"); + return existsSync(log) ? readFileSync(log, "utf8").split("\n").filter((l) => l !== "") : []; +} + function runScript(script: string, args: string[], opts: { path?: string } = {}): { code: number; out: string } { + writeLoaderStubs(); const env: Record = { HOME: tmp, PATH: opts.path ?? `${join(tmp, "fakebin")}:/usr/bin:/bin`, + AMICO_FLEET_LAUNCHCTL: join(tmp, "stubs", "launchctl"), }; // The guard/installer must never find the REAL machine's amico or node-installed fleet state. delete env.AMICISSIMO_ROOT; @@ -373,6 +417,11 @@ describe("the installer writes the platform-correct settings path (#1261 AC4)", const r = runScript(INSTALL, [], installEnv()); expect(r.code).toBe(0); expect(existsSync(join(tmp, "Library", "Application Support", "Code", "User", "settings.json"))).toBe(true); + // #1775: the tunnel (re)load went through the launchctl SEAM — never the + // PATH's launchctl (on a real host that would start a real ssh agent). + const tunnel = join(tmp, "Library", "LaunchAgents", "co.harmoniqs.amico-tunnel.plist"); + expect(launchctlCalls()).toContain(`load ${tunnel}`); + expect(backstopCalls()).toEqual([]); }); it("--check on LINUX catches settings drift (the check twin runs cross-platform, never 'skipped (not darwin)')", () => { diff --git a/packages/extension/tools/fleet/install.sh b/packages/extension/tools/fleet/install.sh index 171cd8449..49e8bd56d 100755 --- a/packages/extension/tools/fleet/install.sh +++ b/packages/extension/tools/fleet/install.sh @@ -17,8 +17,13 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" GUARD_SRC="$REPO_ROOT/tools/fleet/amico-opencode-fleet-guard" GUARD_DST="$HOME/.local/bin/amico-opencode-fleet-guard" +# #1775: the machine-scoped launchd seams — the LaunchAgents dir and the +# launchctl binary. Both default to today's behavior; tests point them at a +# temp dir and a recording stub so no run ever touches the real service manager. +LAUNCH_AGENTS_DIR="${AMICO_FLEET_LAUNCH_AGENTS_DIR:-$HOME/Library/LaunchAgents}" +LAUNCHCTL="${AMICO_FLEET_LAUNCHCTL:-launchctl}" PLIST_SRC="$REPO_ROOT/tools/fleet/co.harmoniqs.amico-tunnel.plist" -PLIST_DST="$HOME/Library/LaunchAgents/co.harmoniqs.amico-tunnel.plist" +PLIST_DST="$LAUNCH_AGENTS_DIR/co.harmoniqs.amico-tunnel.plist" SETTINGS="$HOME/Library/Application Support/Code/User/settings.json" CHECK=0 if [[ "${1:-}" == "--check" ]]; then CHECK=1; fi @@ -208,7 +213,7 @@ if [[ "$ROLE" != "client" ]]; then # A stale tunnel from a previous client enrollment must not linger on a # server: unload + remove it if present (idempotent, best-effort). if [[ "$(uname -s)" == "Darwin" && -f "$PLIST_DST" ]]; then - launchctl unload "$PLIST_DST" 2>/dev/null || true + "$LAUNCHCTL" unload "$PLIST_DST" 2>/dev/null || true rm -f "$PLIST_DST" say "removed stale tunnel plist $PLIST_DST (role is $ROLE, not client)" fi @@ -238,8 +243,8 @@ elif [[ "$(uname -s)" == "Darwin" ]]; then else cp "$_plist_tmp" "$PLIST_DST" # reload - launchctl unload "$PLIST_DST" 2>/dev/null || true - launchctl load "$PLIST_DST" 2>/dev/null || launchctl bootstrap "gui/$(id -u)" "$PLIST_DST" 2>/dev/null || true + "$LAUNCHCTL" unload "$PLIST_DST" 2>/dev/null || true + "$LAUNCHCTL" load "$PLIST_DST" 2>/dev/null || "$LAUNCHCTL" bootstrap "gui/$(id -u)" "$PLIST_DST" 2>/dev/null || true say "installed tunnel $PLIST_DST and (re)loaded (alias ${SSH_ALIAS}, port ${FLEET_PORT})" fi rm -f "$_plist_tmp" @@ -274,7 +279,7 @@ if [[ "$ROLE" == "server" ]]; then HUB_RUNNER="$HUB_EXT_ROOT/bin/dist/amicode-service-runner.mjs" HUB_APP_DIST="$HUB_EXT_ROOT/dist/app" # AMICODE_APP_DIST — the built app shelf the runner serves HUB_NODE="$(command -v node || true)" - HUB_PLIST_DST="$HOME/Library/LaunchAgents/${HUB_LABEL}.plist" + HUB_PLIST_DST="$LAUNCH_AGENTS_DIR/${HUB_LABEL}.plist" HUB_UNIT_DST="$HOME/.config/systemd/user/amico-hub.service" if [[ $CHECK -eq 1 ]]; then @@ -356,8 +361,8 @@ if [[ "$ROLE" == "server" ]]; then PLIST - launchctl unload "$HUB_PLIST_DST" 2>/dev/null || true - launchctl load "$HUB_PLIST_DST" 2>/dev/null || launchctl bootstrap "gui/$(id -u)" "$HUB_PLIST_DST" 2>/dev/null || true + "$LAUNCHCTL" unload "$HUB_PLIST_DST" 2>/dev/null || true + "$LAUNCHCTL" load "$HUB_PLIST_DST" 2>/dev/null || "$LAUNCHCTL" bootstrap "gui/$(id -u)" "$HUB_PLIST_DST" 2>/dev/null || true say "installed hub service $HUB_PLIST_DST and (re)loaded (RunAtLoad+KeepAlive, port ${FLEET_PORT}, runs the #955 runner)" else mkdir -p "$(dirname "$HUB_UNIT_DST")" "$(dirname "$HUB_DB")" diff --git a/tools/fleet/install.sh b/tools/fleet/install.sh index 171cd8449..49e8bd56d 100755 --- a/tools/fleet/install.sh +++ b/tools/fleet/install.sh @@ -17,8 +17,13 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" GUARD_SRC="$REPO_ROOT/tools/fleet/amico-opencode-fleet-guard" GUARD_DST="$HOME/.local/bin/amico-opencode-fleet-guard" +# #1775: the machine-scoped launchd seams — the LaunchAgents dir and the +# launchctl binary. Both default to today's behavior; tests point them at a +# temp dir and a recording stub so no run ever touches the real service manager. +LAUNCH_AGENTS_DIR="${AMICO_FLEET_LAUNCH_AGENTS_DIR:-$HOME/Library/LaunchAgents}" +LAUNCHCTL="${AMICO_FLEET_LAUNCHCTL:-launchctl}" PLIST_SRC="$REPO_ROOT/tools/fleet/co.harmoniqs.amico-tunnel.plist" -PLIST_DST="$HOME/Library/LaunchAgents/co.harmoniqs.amico-tunnel.plist" +PLIST_DST="$LAUNCH_AGENTS_DIR/co.harmoniqs.amico-tunnel.plist" SETTINGS="$HOME/Library/Application Support/Code/User/settings.json" CHECK=0 if [[ "${1:-}" == "--check" ]]; then CHECK=1; fi @@ -208,7 +213,7 @@ if [[ "$ROLE" != "client" ]]; then # A stale tunnel from a previous client enrollment must not linger on a # server: unload + remove it if present (idempotent, best-effort). if [[ "$(uname -s)" == "Darwin" && -f "$PLIST_DST" ]]; then - launchctl unload "$PLIST_DST" 2>/dev/null || true + "$LAUNCHCTL" unload "$PLIST_DST" 2>/dev/null || true rm -f "$PLIST_DST" say "removed stale tunnel plist $PLIST_DST (role is $ROLE, not client)" fi @@ -238,8 +243,8 @@ elif [[ "$(uname -s)" == "Darwin" ]]; then else cp "$_plist_tmp" "$PLIST_DST" # reload - launchctl unload "$PLIST_DST" 2>/dev/null || true - launchctl load "$PLIST_DST" 2>/dev/null || launchctl bootstrap "gui/$(id -u)" "$PLIST_DST" 2>/dev/null || true + "$LAUNCHCTL" unload "$PLIST_DST" 2>/dev/null || true + "$LAUNCHCTL" load "$PLIST_DST" 2>/dev/null || "$LAUNCHCTL" bootstrap "gui/$(id -u)" "$PLIST_DST" 2>/dev/null || true say "installed tunnel $PLIST_DST and (re)loaded (alias ${SSH_ALIAS}, port ${FLEET_PORT})" fi rm -f "$_plist_tmp" @@ -274,7 +279,7 @@ if [[ "$ROLE" == "server" ]]; then HUB_RUNNER="$HUB_EXT_ROOT/bin/dist/amicode-service-runner.mjs" HUB_APP_DIST="$HUB_EXT_ROOT/dist/app" # AMICODE_APP_DIST — the built app shelf the runner serves HUB_NODE="$(command -v node || true)" - HUB_PLIST_DST="$HOME/Library/LaunchAgents/${HUB_LABEL}.plist" + HUB_PLIST_DST="$LAUNCH_AGENTS_DIR/${HUB_LABEL}.plist" HUB_UNIT_DST="$HOME/.config/systemd/user/amico-hub.service" if [[ $CHECK -eq 1 ]]; then @@ -356,8 +361,8 @@ if [[ "$ROLE" == "server" ]]; then PLIST - launchctl unload "$HUB_PLIST_DST" 2>/dev/null || true - launchctl load "$HUB_PLIST_DST" 2>/dev/null || launchctl bootstrap "gui/$(id -u)" "$HUB_PLIST_DST" 2>/dev/null || true + "$LAUNCHCTL" unload "$HUB_PLIST_DST" 2>/dev/null || true + "$LAUNCHCTL" load "$HUB_PLIST_DST" 2>/dev/null || "$LAUNCHCTL" bootstrap "gui/$(id -u)" "$HUB_PLIST_DST" 2>/dev/null || true say "installed hub service $HUB_PLIST_DST and (re)loaded (RunAtLoad+KeepAlive, port ${FLEET_PORT}, runs the #955 runner)" else mkdir -p "$(dirname "$HUB_UNIT_DST")" "$(dirname "$HUB_DB")" From 3a459637169f3b3a2ed79b5f86e9f9a9e6c424a4 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 20:25:19 -0400 Subject: [PATCH 43/60] fix(fleet): the installer generates one SSH forward agent per attached peer (#1775) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each peer in the reader peer-token store whose base_url is a loopback http://127.0.0.1: gets co.harmoniqs.amico-peer--forward, rendered from the new co.harmoniqs.amico-peer-forward.plist template with the fleet transport's standard options (ServerAlive 15/2, TCPKeepAlive, ExitOnForwardFailure) forwarding the store's local port to the fleet's canonical hub port on the peer's roster sshAlias (else its machine id). Role-agnostic (a peer may be standalone), darwin only (#1260 owns linux/WSL); an unchanged agent is not reloaded, so a re-run is idempotent. Fleet — Repair runs the packaged copy, which ships the same template; the CI fleet gate checks the template's keepalives and packaged sync. --- .../extension/scripts/assert_fleet_guard.sh | 14 +- .../test/fleet_scripts_projection.test.ts | 129 +++++++++++++++++- .../co.harmoniqs.amico-peer-forward.plist | 36 +++++ packages/extension/tools/fleet/install.sh | 88 +++++++++++- .../co.harmoniqs.amico-peer-forward.plist | 36 +++++ tools/fleet/install.sh | 88 +++++++++++- 6 files changed, 383 insertions(+), 8 deletions(-) create mode 100644 packages/extension/tools/fleet/co.harmoniqs.amico-peer-forward.plist create mode 100644 tools/fleet/co.harmoniqs.amico-peer-forward.plist diff --git a/packages/extension/scripts/assert_fleet_guard.sh b/packages/extension/scripts/assert_fleet_guard.sh index 44f4a56f2..956d38d6a 100755 --- a/packages/extension/scripts/assert_fleet_guard.sh +++ b/packages/extension/scripts/assert_fleet_guard.sh @@ -45,6 +45,16 @@ grep -q "TCPKeepAlive=yes" "$PLIST" || fail "plist TCPKeepAlive yes missing" grep -q "127.0.0.1:4096:127.0.0.1:4096" "$PLIST" || fail "plist LocalForward 4096 missing" ok "tunnel plist $PLIST" +# #1775: the per-peer forward template carries the SAME standard options. +PEER_PLIST="$ROOT/tools/fleet/co.harmoniqs.amico-peer-forward.plist" +[[ -f "$PEER_PLIST" ]] || fail "peer-forward plist template missing at $PEER_PLIST" +grep -q "ServerAliveInterval=15" "$PEER_PLIST" || fail "peer-forward plist ServerAliveInterval 15 missing" +grep -q "ServerAliveCountMax=2" "$PEER_PLIST" || fail "peer-forward plist ServerAliveCountMax 2 missing" +grep -q "TCPKeepAlive=yes" "$PEER_PLIST" || fail "peer-forward plist TCPKeepAlive yes missing" +grep -q "ExitOnForwardFailure=yes" "$PEER_PLIST" || fail "peer-forward plist ExitOnForwardFailure yes missing" +grep -q "co.harmoniqs.amico-peer-PEER_MACHINE_ID-forward" "$PEER_PLIST" || fail "peer-forward plist label is not the co.harmoniqs.amico-peer--forward convention" +ok "peer-forward plist $PEER_PLIST" + # The extension's health checks consume the projection topology state # (fleet_topology.ts) — the consumer import must be present (#1106). grep -q "fleet_topology" "$ROOT/packages/extension/src/fleet_health.ts" || fail "fleet_health.ts missing the fleet_topology consumer import (#1106)" @@ -53,11 +63,13 @@ grep -q "fleet_topology" "$ROOT/packages/extension/src/fleet_health.ts" || fail PKG_GUARD="$ROOT/packages/extension/tools/fleet/amico-opencode-fleet-guard" PKG_PLIST="$ROOT/packages/extension/tools/fleet/co.harmoniqs.amico-tunnel.plist" PKG_INSTALL="$ROOT/packages/extension/tools/fleet/install.sh" -for f in "$PKG_GUARD" "$PKG_PLIST" "$PKG_INSTALL"; do +PKG_PEER_PLIST="$ROOT/packages/extension/tools/fleet/co.harmoniqs.amico-peer-forward.plist" +for f in "$PKG_GUARD" "$PKG_PLIST" "$PKG_INSTALL" "$PKG_PEER_PLIST"; do [[ -f "$f" ]] || fail "packaged fleet file missing at $f (run: cp tools/fleet/* packages/extension/tools/fleet/)" done cmp -s "$GUARD" "$PKG_GUARD" || fail "packaged guard drift — cp tools/fleet/amico-opencode-fleet-guard packages/extension/tools/fleet/" cmp -s "$PLIST" "$PKG_PLIST" || fail "packaged plist drift — cp tools/fleet/co.harmoniqs.amico-tunnel.plist packages/extension/tools/fleet/" +cmp -s "$PEER_PLIST" "$PKG_PEER_PLIST" || fail "packaged peer-forward plist drift — cp tools/fleet/co.harmoniqs.amico-peer-forward.plist packages/extension/tools/fleet/" cmp -s "$INSTALL" "$PKG_INSTALL" || fail "packaged installer drift — cp tools/fleet/install.sh packages/extension/tools/fleet/" echo "[fleet-gate] all fleet gates passed" diff --git a/packages/extension/test/fleet_scripts_projection.test.ts b/packages/extension/test/fleet_scripts_projection.test.ts index 82ebc8a66..e4bda3bd0 100644 --- a/packages/extension/test/fleet_scripts_projection.test.ts +++ b/packages/extension/test/fleet_scripts_projection.test.ts @@ -13,7 +13,7 @@ // failure, cache untouched. import { describe, it, expect, beforeEach, afterEach } from "vitest"; import { spawnSync } from "node:child_process"; -import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync, existsSync, readFileSync } from "node:fs"; +import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync, existsSync, readFileSync, readdirSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -530,11 +530,134 @@ describe.skipIf(!existsSync(HUB_RUNNER_BUNDLE))("#1258 the installer provisions // ── both copies ship byte-identical (the VSIX's packaged copy is the installer users run) ── describe("the packaged fleet scripts stay in sync with the repo copies", () => { - it("guard + installer: packages/extension/tools/fleet copies are byte-identical", () => { - for (const f of ["amico-opencode-fleet-guard", "install.sh"]) { + it("guard + installer + launchd templates: packages/extension/tools/fleet copies are byte-identical", () => { + for (const f of ["amico-opencode-fleet-guard", "install.sh", "co.harmoniqs.amico-tunnel.plist", "co.harmoniqs.amico-peer-forward.plist"]) { const pkg = join(REPO, "packages", "extension", "tools", "fleet", f); expect(existsSync(pkg)).toBe(true); expect(readFileSync(pkg, "utf8")).toBe(readFileSync(join(REPO, "tools", "fleet", f), "utf8")); } }); }); + +// ── #1775: installer-managed per-peer SSH forwards ───────────────────────────── +// The SSH forward a machine uses to reach a PEER's hub was a hand-installed +// launch agent with stale 30/3 keepalives. The installer now generates one +// agent per attached peer — label from the peer's machine id, local port from +// the peer token store's loopback base URL, remote port = the fleet's canonical +// hub port, ssh destination = the peer's roster sshAlias (else its machine id). +// Every run is hermetic: temp HOME, the launchctl seam stub, the PATH backstop. +const PEER_STORE_REL = [".amico", "fleet-peer-tokens-reader.json"]; +const peerForwardPlist = (id: string): string => join(tmp, "Library", "LaunchAgents", `co.harmoniqs.amico-peer-${id}-forward.plist`); + +/** The reader peer-token store (#1438) — { store_version, peers: { id: { base_url, token } } }. */ +function writePeerStore(peers: Record): void { + mkdirSync(join(tmp, ".amico"), { recursive: true }); + const doc = { store_version: 1, peers: Object.fromEntries(Object.entries(peers).map(([id, baseUrl]) => [id, { base_url: baseUrl, token: `tok-${id}` }])) }; + writeFileSync(join(tmp, ...PEER_STORE_REL), JSON.stringify(doc, null, 2)); +} + +/** The fleet roster cache — the peer's sshAlias lives on its row. */ +function writeRoster(rows: Array<{ machine_id: string; sshAlias: string }>): void { + const dir = join(tmp, ".amico", "ops", "fleet"); + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, "roster.json"), JSON.stringify({ schema_version: 1, rows: rows.map((r) => ({ ...r, transport: "ssh", server_mode: "server" })) }, null, 2)); +} + +/** A launchd plist's Label + ProgramArguments, read straight from the XML. */ +function readAgent(file: string): { label: string; args: string[] } { + const xml = readFileSync(file, "utf8"); + const label = /Label<\/key>\s*([^<]*)<\/string>/.exec(xml)?.[1] ?? ""; + const array = /ProgramArguments<\/key>\s*([\s\S]*?)<\/array>/.exec(xml)?.[1] ?? ""; + const args = [...array.matchAll(/([^<]*)<\/string>/g)].map((m) => m[1]); + return { label, args }; +} + +/** The standard forward argv (fleet_transport's options) for one peer. */ +const standardForward = (localPort: number, remotePort: number, alias: string): string[] => [ + "/usr/bin/ssh", "-N", + "-o", "ExitOnForwardFailure=yes", + "-o", "ServerAliveInterval=15", + "-o", "ServerAliveCountMax=2", + "-o", "TCPKeepAlive=yes", + "-o", "BatchMode=yes", + "-L", `127.0.0.1:${localPort}:127.0.0.1:${remotePort}`, + alias, +]; + +describe("#1775 the installer generates one SSH forward agent per attached peer", () => { + const installEnv = (): { path: string } => ({ + path: `${join(tmp, "fakebin")}:${process.env.PATH ?? ""}`, + }); + const verbFor = (role: string): string => + role === "standalone" ? verbJson() : verbJson(role, { host: "hq", port: 4096, sshAlias: "hq" }); + + it.each(["server", "client", "standalone"])( + "a %s attached to a peer gets that peer's forward agent — 15 s keepalive, 2 misses, TCPKeepAlive, ExitOnForwardFailure, the store's local port; a re-run is idempotent", + (role) => { + fakeAmico({ code: 0, stdout: verbFor(role) }); + fakeUname("Darwin"); + writePeerStore({ "jjs-mac-studio": "http://127.0.0.1:45096" }); + writeRoster([{ machine_id: "jjs-mac-studio", sshAlias: "studio-ssh" }]); + + const r = runScript(INSTALL, [], installEnv()); + expect(r.code, r.out).toBe(0); + const plist = peerForwardPlist("jjs-mac-studio"); + expect(existsSync(plist), r.out).toBe(true); + const agent = readAgent(plist); + expect(agent.label).toBe("co.harmoniqs.amico-peer-jjs-mac-studio-forward"); + expect(agent.args).toEqual(standardForward(45096, 4096, "studio-ssh")); + expect(readFileSync(plist, "utf8")).toMatch(/KeepAlive<\/key>\s*/); + expect(launchctlCalls()).toContain(`load ${plist}`); + + // Re-run: byte-identical agent, no reload, still exactly one forward agent. + const first = readFileSync(plist, "utf8"); + rmSync(join(tmp, "launchctl.argv"), { force: true }); + const again = runScript(INSTALL, [], installEnv()); + expect(again.code, again.out).toBe(0); + expect(readFileSync(plist, "utf8")).toBe(first); + expect(launchctlCalls().filter((l) => l.includes(plist))).toEqual([]); + expect(again.out).toMatch(/peer forward co\.harmoniqs\.amico-peer-jjs-mac-studio-forward unchanged/); + expect(readdirSync(join(tmp, "Library", "LaunchAgents")).filter((f) => f.endsWith("-forward.plist"))).toEqual([ + "co.harmoniqs.amico-peer-jjs-mac-studio-forward.plist", + ]); + expect(backstopCalls()).toEqual([]); + }, + ); + + it("the ssh destination falls back to the peer's machine id when the roster has no row for it", () => { + fakeAmico({ code: 0, stdout: verbFor("server") }); + fakeUname("Darwin"); + writePeerStore({ "jjs-mac-studio": "http://127.0.0.1:45096" }); + const r = runScript(INSTALL, [], installEnv()); + expect(r.code, r.out).toBe(0); + expect(readAgent(peerForwardPlist("jjs-mac-studio")).args).toEqual(standardForward(45096, 4096, "jjs-mac-studio")); + }); + + it("one agent per peer: each loopback peer gets its own label + port; a peer reached over https (tailscale/direct) gets no ssh forward", () => { + fakeAmico({ code: 0, stdout: verbFor("server") }); + fakeUname("Darwin"); + writePeerStore({ + "jjs-mac-studio": "http://127.0.0.1:45096", + "lab-box": "http://127.0.0.1:45097", + "far-node": "https://far-node.tailnet-1234.ts.net", + }); + writeRoster([{ machine_id: "lab-box", sshAlias: "lab" }]); + const r = runScript(INSTALL, [], installEnv()); + expect(r.code, r.out).toBe(0); + expect(readAgent(peerForwardPlist("jjs-mac-studio")).args).toEqual(standardForward(45096, 4096, "jjs-mac-studio")); + expect(readAgent(peerForwardPlist("lab-box")).args).toEqual(standardForward(45097, 4096, "lab")); + expect(existsSync(peerForwardPlist("far-node"))).toBe(false); + expect(r.out).toMatch(/far-node.*no SSH forward/); + }); + + it("Fleet — Repair runs the PACKAGED installer copy — it generates the same agent from the packaged template", () => { + fakeAmico({ code: 0, stdout: verbFor("server") }); + fakeUname("Darwin"); + writePeerStore({ "jjs-mac-studio": "http://127.0.0.1:45096" }); + const packaged = join(REPO, "packages", "extension", "tools", "fleet", "install.sh"); + const r = runScript(packaged, [], installEnv()); + expect(r.code, r.out).toBe(0); + expect(readAgent(peerForwardPlist("jjs-mac-studio")).args).toEqual(standardForward(45096, 4096, "jjs-mac-studio")); + expect(backstopCalls()).toEqual([]); + }); +}); diff --git a/packages/extension/tools/fleet/co.harmoniqs.amico-peer-forward.plist b/packages/extension/tools/fleet/co.harmoniqs.amico-peer-forward.plist new file mode 100644 index 000000000..6b07593c6 --- /dev/null +++ b/packages/extension/tools/fleet/co.harmoniqs.amico-peer-forward.plist @@ -0,0 +1,36 @@ + + + + + KeepAlive + + Label + co.harmoniqs.amico-peer-PEER_MACHINE_ID-forward + ProgramArguments + + /usr/bin/ssh + -N + -o + ExitOnForwardFailure=yes + -o + ServerAliveInterval=15 + -o + ServerAliveCountMax=2 + -o + TCPKeepAlive=yes + -o + BatchMode=yes + -L + 127.0.0.1:PEER_LOCAL_PORT:127.0.0.1:PEER_REMOTE_PORT + PEER_SSH_ALIAS + + RunAtLoad + + StandardErrorPath + /tmp/amico-peer-PEER_MACHINE_ID-forward.log + StandardOutPath + /tmp/amico-peer-PEER_MACHINE_ID-forward.log + ThrottleInterval + 10 + + diff --git a/packages/extension/tools/fleet/install.sh b/packages/extension/tools/fleet/install.sh index 49e8bd56d..f6bdd7a98 100755 --- a/packages/extension/tools/fleet/install.sh +++ b/packages/extension/tools/fleet/install.sh @@ -94,13 +94,94 @@ PORT_PARSED="$(printf '%s\n' "$PARSED" | sed -n 2p)" SSH_ALIAS="$(printf '%s\n' "$PARSED" | sed -n 3p)" if [[ "$PORT_PARSED" =~ ^[0-9]+$ ]]; then FLEET_PORT="$PORT_PARSED"; fi -# Standalone: nothing to install — the guard/tunnel are irrelevant. +# --- per-peer SSH forwards (#1775) --- +# One launchd agent per attached PEER (peer studios, ADR 0027 / ADR 0032): the +# SSH `-L` forward this machine uses to reach that peer's hub. Generated from +# the repo template with the fleet transport's standard options (15 s +# keepalive, 2 misses, TCPKeepAlive, ExitOnForwardFailure — sshForwardArgs), +# never hand-installed. Role-agnostic: a peer may be standalone, server, or +# client — the signal is the reader peer-token store (fleet_peer_store.ts), +# whose loopback base_url IS the forward's local end. Darwin only (the +# supervised linux/WSL link is #1260). +# label co.harmoniqs.amico-peer--forward +# local port the store's base_url port (http://127.0.0.1:) +# remote port the fleet's canonical hub port ($FLEET_PORT) +# destination the peer's roster sshAlias, else its machine id +PEER_PLIST_SRC="$REPO_ROOT/tools/fleet/co.harmoniqs.amico-peer-forward.plist" +PEER_STORE="${AMICO_FLEET_PEER_TOKEN_FILE:-$HOME/.amico/fleet-peer-tokens-reader.json}" +ROSTER_FILE="${AMICO_FLEET_ROSTER_FILE:-$HOME/.amico/ops/fleet/roster.json}" + +# One line per peer: `forwardidlocal_portalias`, or +# `notemessage` for a peer that rides no ssh forward. Never prints a token. +peer_forward_targets() { + node -e ' + const fs = require("fs"); + const doc = (p) => { try { const j = JSON.parse(fs.readFileSync(p, "utf8")); return j && typeof j === "object" && !Array.isArray(j) ? j : {}; } catch (e) { return {}; } }; + const store = doc(process.argv[1]); + const peers = store.peers && typeof store.peers === "object" && !Array.isArray(store.peers) ? store.peers : {}; + const roster = doc(process.argv[2]); + const rows = Array.isArray(roster.rows) ? roster.rows : []; + const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; + const SAFE_ALIAS = /^[A-Za-z0-9][A-Za-z0-9._@-]*$/; + const out = []; + for (const id of Object.keys(peers).sort()) { + const e = peers[id]; + const raw = e && typeof e === "object" && typeof e.base_url === "string" ? e.base_url.trim() : ""; + if (!SAFE_ID.test(id)) { out.push(["note", "peer id " + JSON.stringify(id) + " is not a safe launchd label fragment — no SSH forward generated"]); continue; } + let u = null; + try { u = new URL(raw); } catch (_) { u = null; } + const loopback = u !== null && u.protocol === "http:" && (u.hostname === "127.0.0.1" || u.hostname === "localhost"); + const port = u !== null ? Number(u.port) : NaN; + if (!loopback || !Number.isInteger(port) || port < 1 || port > 65535) { + out.push(["note", "peer " + id + " is reached at " + (raw || "(no base_url)") + " — no SSH forward (only a loopback http://127.0.0.1: base URL rides one)"]); + continue; + } + const row = rows.find((r) => r && r.machine_id === id); + const rowAlias = row && typeof row.sshAlias === "string" ? row.sshAlias.trim() : ""; + out.push(["forward", id, String(port), SAFE_ALIAS.test(rowAlias) ? rowAlias : id]); + } + process.stdout.write(out.map((f) => f.join("\t")).join("\n")); + ' "$PEER_STORE" "$ROSTER_FILE" +} + +peer_forwards() { + if [[ "$(uname -s)" != "Darwin" ]]; then return 0; fi + if [[ $CHECK -eq 1 ]]; then return 0; fi + local targets kind peer_id lport alias label dst tmpf + targets="$(peer_forward_targets)" || die "could not read the peer token store at $PEER_STORE (node failed)" + if [[ -z "$targets" ]]; then return 0; fi + if [[ ! -f "$PEER_PLIST_SRC" ]]; then + say "note: no peer-forward template at $PEER_PLIST_SRC — skipping peer forwards (git pull?)" + return 0 + fi + while IFS=$'\t' read -r kind peer_id lport alias; do + if [[ "$kind" == "note" ]]; then say "note: $peer_id"; continue; fi + label="co.harmoniqs.amico-peer-${peer_id}-forward" + dst="$LAUNCH_AGENTS_DIR/${label}.plist" + mkdir -p "$LAUNCH_AGENTS_DIR" + tmpf="$(mktemp)" + sed -e "s/PEER_MACHINE_ID/${peer_id}/g" -e "s/PEER_LOCAL_PORT/${lport}/g" -e "s/PEER_REMOTE_PORT/${FLEET_PORT}/g" -e "s/PEER_SSH_ALIAS/${alias}/g" "$PEER_PLIST_SRC" > "$tmpf" + if [[ -f "$dst" ]] && cmp -s "$tmpf" "$dst"; then + say "peer forward $label unchanged — skipping reload" + else + cp "$tmpf" "$dst" + "$LAUNCHCTL" unload "$dst" 2>/dev/null || true + "$LAUNCHCTL" load "$dst" 2>/dev/null || "$LAUNCHCTL" bootstrap "gui/$(id -u)" "$dst" 2>/dev/null || true + say "installed peer forward $dst and (re)loaded (127.0.0.1:${lport} → ${alias}:${FLEET_PORT}, 15/2 + TCPKeepAlive)" + fi + rm -f "$tmpf" + done <<< "$targets" +} + +# Standalone: no guard/tunnel/hub — but a standalone PEER still gets its +# per-peer forwards (#1775: peers stay standalone, ADR 0027). if [[ "$ROLE" == "standalone" ]]; then if [[ $CHECK -eq 1 ]]; then say "standalone (no fleet.json or role=standalone) — fleet checks skipped" else - say "standalone mode — nothing to install (use 'Amicode: Fleet — Enroll' to join a fleet)" + say "standalone mode — nothing to install for the fleet stance (use 'Amicode: Fleet — Enroll' to join a fleet)" fi + peer_forwards exit 0 fi @@ -252,6 +333,9 @@ elif [[ "$(uname -s)" == "Darwin" ]]; then fi fi +# --- per-peer SSH forwards (every role — #1775; see peer_forwards above) --- +peer_forwards + # --- canonical hub service (SERVER role only — reboot-survival, #1258) --- # The canonical hub must survive a REBOOT with NO editor ever opened. Today the # hub is the extension-spawned detached server (ADR 0020): survives a window diff --git a/tools/fleet/co.harmoniqs.amico-peer-forward.plist b/tools/fleet/co.harmoniqs.amico-peer-forward.plist new file mode 100644 index 000000000..6b07593c6 --- /dev/null +++ b/tools/fleet/co.harmoniqs.amico-peer-forward.plist @@ -0,0 +1,36 @@ + + + + + KeepAlive + + Label + co.harmoniqs.amico-peer-PEER_MACHINE_ID-forward + ProgramArguments + + /usr/bin/ssh + -N + -o + ExitOnForwardFailure=yes + -o + ServerAliveInterval=15 + -o + ServerAliveCountMax=2 + -o + TCPKeepAlive=yes + -o + BatchMode=yes + -L + 127.0.0.1:PEER_LOCAL_PORT:127.0.0.1:PEER_REMOTE_PORT + PEER_SSH_ALIAS + + RunAtLoad + + StandardErrorPath + /tmp/amico-peer-PEER_MACHINE_ID-forward.log + StandardOutPath + /tmp/amico-peer-PEER_MACHINE_ID-forward.log + ThrottleInterval + 10 + + diff --git a/tools/fleet/install.sh b/tools/fleet/install.sh index 49e8bd56d..f6bdd7a98 100755 --- a/tools/fleet/install.sh +++ b/tools/fleet/install.sh @@ -94,13 +94,94 @@ PORT_PARSED="$(printf '%s\n' "$PARSED" | sed -n 2p)" SSH_ALIAS="$(printf '%s\n' "$PARSED" | sed -n 3p)" if [[ "$PORT_PARSED" =~ ^[0-9]+$ ]]; then FLEET_PORT="$PORT_PARSED"; fi -# Standalone: nothing to install — the guard/tunnel are irrelevant. +# --- per-peer SSH forwards (#1775) --- +# One launchd agent per attached PEER (peer studios, ADR 0027 / ADR 0032): the +# SSH `-L` forward this machine uses to reach that peer's hub. Generated from +# the repo template with the fleet transport's standard options (15 s +# keepalive, 2 misses, TCPKeepAlive, ExitOnForwardFailure — sshForwardArgs), +# never hand-installed. Role-agnostic: a peer may be standalone, server, or +# client — the signal is the reader peer-token store (fleet_peer_store.ts), +# whose loopback base_url IS the forward's local end. Darwin only (the +# supervised linux/WSL link is #1260). +# label co.harmoniqs.amico-peer--forward +# local port the store's base_url port (http://127.0.0.1:) +# remote port the fleet's canonical hub port ($FLEET_PORT) +# destination the peer's roster sshAlias, else its machine id +PEER_PLIST_SRC="$REPO_ROOT/tools/fleet/co.harmoniqs.amico-peer-forward.plist" +PEER_STORE="${AMICO_FLEET_PEER_TOKEN_FILE:-$HOME/.amico/fleet-peer-tokens-reader.json}" +ROSTER_FILE="${AMICO_FLEET_ROSTER_FILE:-$HOME/.amico/ops/fleet/roster.json}" + +# One line per peer: `forwardidlocal_portalias`, or +# `notemessage` for a peer that rides no ssh forward. Never prints a token. +peer_forward_targets() { + node -e ' + const fs = require("fs"); + const doc = (p) => { try { const j = JSON.parse(fs.readFileSync(p, "utf8")); return j && typeof j === "object" && !Array.isArray(j) ? j : {}; } catch (e) { return {}; } }; + const store = doc(process.argv[1]); + const peers = store.peers && typeof store.peers === "object" && !Array.isArray(store.peers) ? store.peers : {}; + const roster = doc(process.argv[2]); + const rows = Array.isArray(roster.rows) ? roster.rows : []; + const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; + const SAFE_ALIAS = /^[A-Za-z0-9][A-Za-z0-9._@-]*$/; + const out = []; + for (const id of Object.keys(peers).sort()) { + const e = peers[id]; + const raw = e && typeof e === "object" && typeof e.base_url === "string" ? e.base_url.trim() : ""; + if (!SAFE_ID.test(id)) { out.push(["note", "peer id " + JSON.stringify(id) + " is not a safe launchd label fragment — no SSH forward generated"]); continue; } + let u = null; + try { u = new URL(raw); } catch (_) { u = null; } + const loopback = u !== null && u.protocol === "http:" && (u.hostname === "127.0.0.1" || u.hostname === "localhost"); + const port = u !== null ? Number(u.port) : NaN; + if (!loopback || !Number.isInteger(port) || port < 1 || port > 65535) { + out.push(["note", "peer " + id + " is reached at " + (raw || "(no base_url)") + " — no SSH forward (only a loopback http://127.0.0.1: base URL rides one)"]); + continue; + } + const row = rows.find((r) => r && r.machine_id === id); + const rowAlias = row && typeof row.sshAlias === "string" ? row.sshAlias.trim() : ""; + out.push(["forward", id, String(port), SAFE_ALIAS.test(rowAlias) ? rowAlias : id]); + } + process.stdout.write(out.map((f) => f.join("\t")).join("\n")); + ' "$PEER_STORE" "$ROSTER_FILE" +} + +peer_forwards() { + if [[ "$(uname -s)" != "Darwin" ]]; then return 0; fi + if [[ $CHECK -eq 1 ]]; then return 0; fi + local targets kind peer_id lport alias label dst tmpf + targets="$(peer_forward_targets)" || die "could not read the peer token store at $PEER_STORE (node failed)" + if [[ -z "$targets" ]]; then return 0; fi + if [[ ! -f "$PEER_PLIST_SRC" ]]; then + say "note: no peer-forward template at $PEER_PLIST_SRC — skipping peer forwards (git pull?)" + return 0 + fi + while IFS=$'\t' read -r kind peer_id lport alias; do + if [[ "$kind" == "note" ]]; then say "note: $peer_id"; continue; fi + label="co.harmoniqs.amico-peer-${peer_id}-forward" + dst="$LAUNCH_AGENTS_DIR/${label}.plist" + mkdir -p "$LAUNCH_AGENTS_DIR" + tmpf="$(mktemp)" + sed -e "s/PEER_MACHINE_ID/${peer_id}/g" -e "s/PEER_LOCAL_PORT/${lport}/g" -e "s/PEER_REMOTE_PORT/${FLEET_PORT}/g" -e "s/PEER_SSH_ALIAS/${alias}/g" "$PEER_PLIST_SRC" > "$tmpf" + if [[ -f "$dst" ]] && cmp -s "$tmpf" "$dst"; then + say "peer forward $label unchanged — skipping reload" + else + cp "$tmpf" "$dst" + "$LAUNCHCTL" unload "$dst" 2>/dev/null || true + "$LAUNCHCTL" load "$dst" 2>/dev/null || "$LAUNCHCTL" bootstrap "gui/$(id -u)" "$dst" 2>/dev/null || true + say "installed peer forward $dst and (re)loaded (127.0.0.1:${lport} → ${alias}:${FLEET_PORT}, 15/2 + TCPKeepAlive)" + fi + rm -f "$tmpf" + done <<< "$targets" +} + +# Standalone: no guard/tunnel/hub — but a standalone PEER still gets its +# per-peer forwards (#1775: peers stay standalone, ADR 0027). if [[ "$ROLE" == "standalone" ]]; then if [[ $CHECK -eq 1 ]]; then say "standalone (no fleet.json or role=standalone) — fleet checks skipped" else - say "standalone mode — nothing to install (use 'Amicode: Fleet — Enroll' to join a fleet)" + say "standalone mode — nothing to install for the fleet stance (use 'Amicode: Fleet — Enroll' to join a fleet)" fi + peer_forwards exit 0 fi @@ -252,6 +333,9 @@ elif [[ "$(uname -s)" == "Darwin" ]]; then fi fi +# --- per-peer SSH forwards (every role — #1775; see peer_forwards above) --- +peer_forwards + # --- canonical hub service (SERVER role only — reboot-survival, #1258) --- # The canonical hub must survive a REBOOT with NO editor ever opened. Today the # hub is the extension-spawned detached server (ADR 0020): survives a window From f3428471dffb27a5dfeab0c71565758972881646 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 20:27:00 -0400 Subject: [PATCH 44/60] fix(fleet): a hand-installed peer forward is replaced with a backup, never silently deleted (#1775) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before generating a peer's managed agent, the installer recognizes any legacy co.harmoniqs.amico--forward.plist (not the managed co.harmoniqs.amico-peer-*-forward shape) that binds the peer's local port or targets its ssh destination, unloads it (before the managed agent loads on the same port), and moves it byte-for-byte to a timestamped backup in ~/.amico/ops/fleet/launchagents-backup/ — outside LaunchAgents, so launchd never reloads it. Forwards for other peers are left untouched. --- .../test/fleet_scripts_projection.test.ts | 102 ++++++++++++++++++ packages/extension/tools/fleet/install.sh | 32 ++++++ tools/fleet/install.sh | 32 ++++++ 3 files changed, 166 insertions(+) diff --git a/packages/extension/test/fleet_scripts_projection.test.ts b/packages/extension/test/fleet_scripts_projection.test.ts index e4bda3bd0..6e48c6cb6 100644 --- a/packages/extension/test/fleet_scripts_projection.test.ts +++ b/packages/extension/test/fleet_scripts_projection.test.ts @@ -661,3 +661,105 @@ describe("#1775 the installer generates one SSH forward agent per attached peer" expect(backstopCalls()).toEqual([]); }); }); + +/** A hand-installed forward in the legacy `co.harmoniqs.amico--forward` + * shape — byte-for-byte the agent found on a real fleet laptop (stale 30/3, + * unbound `-L :127.0.0.1:`, BatchMode + accept-new). */ +function legacyForwardPlist(name: string, localPort: number, host: string): string { + return ` + + + + Label + co.harmoniqs.amico-${name}-forward + ProgramArguments + + /usr/bin/ssh + -N + -L + ${localPort}:127.0.0.1:4096 + -o + ExitOnForwardFailure=yes + -o + ServerAliveInterval=30 + -o + ServerAliveCountMax=3 + -o + BatchMode=yes + -o + StrictHostKeyChecking=accept-new + ${host} + + RunAtLoad + + KeepAlive + + StandardOutPath + /tmp/amico-${name}-forward.log + StandardErrorPath + /tmp/amico-${name}-forward.log + ThrottleInterval + 10 + + +`; +} + +describe("#1775 a hand-installed forward for the same peer is replaced — a backup is kept, never a silent delete", () => { + const installEnv = (): { path: string } => ({ + path: `${join(tmp, "fakebin")}:${process.env.PATH ?? ""}`, + }); + const launchAgents = (): string => join(tmp, "Library", "LaunchAgents"); + const backupDir = (): string => join(tmp, ".amico", "ops", "fleet", "launchagents-backup"); + const placeLegacy = (name: string, localPort: number, host: string): { file: string; content: string } => { + mkdirSync(launchAgents(), { recursive: true }); + const file = join(launchAgents(), `co.harmoniqs.amico-${name}-forward.plist`); + const content = legacyForwardPlist(name, localPort, host); + writeFileSync(file, content); + return { file, content }; + }; + + it.each([ + ["it binds the same local port", 45096, "jjs-mac-studio"], + ["it targets the same ssh destination", 47001, "jjs-mac-studio"], + ])("recognized because %s: unloaded, moved byte-for-byte to the backup dir, and the managed agent takes its place; a re-run makes no second backup", (_why, legacyPort, legacyHost) => { + fakeAmico({ code: 0, stdout: verbJson("server", { host: "hq", port: 4096 }) }); + fakeUname("Darwin"); + writePeerStore({ "jjs-mac-studio": "http://127.0.0.1:45096" }); + const legacy = placeLegacy("studio", legacyPort, legacyHost); + + const r = runScript(INSTALL, [], installEnv()); + expect(r.code, r.out).toBe(0); + expect(existsSync(legacy.file)).toBe(false); + const backups = readdirSync(backupDir()); + expect(backups).toHaveLength(1); + expect(backups[0].startsWith("co.harmoniqs.amico-studio-forward.plist.")).toBe(true); + expect(readFileSync(join(backupDir(), backups[0]), "utf8")).toBe(legacy.content); + expect(r.out).toMatch(/replaced hand-installed peer forward .*co\.harmoniqs\.amico-studio-forward\.plist.*backup/); + + // The legacy job is unloaded BEFORE the managed one loads (they bind the same port). + const managed = peerForwardPlist("jjs-mac-studio"); + const calls = launchctlCalls(); + expect(calls).toContain(`unload ${legacy.file}`); + expect(calls.indexOf(`unload ${legacy.file}`)).toBeLessThan(calls.indexOf(`load ${managed}`)); + expect(readAgent(managed).args).toEqual(standardForward(45096, 4096, "jjs-mac-studio")); + + const again = runScript(INSTALL, [], installEnv()); + expect(again.code, again.out).toBe(0); + expect(readdirSync(backupDir())).toEqual(backups); + expect(backstopCalls()).toEqual([]); + }); + + it("a forward for a DIFFERENT peer (another port, another host) is left exactly as it was — never unloaded, never moved", () => { + fakeAmico({ code: 0, stdout: verbJson("server", { host: "hq", port: 4096 }) }); + fakeUname("Darwin"); + writePeerStore({ "jjs-mac-studio": "http://127.0.0.1:45096" }); + const other = placeLegacy("lab", 47000, "lab-box"); + + const r = runScript(INSTALL, [], installEnv()); + expect(r.code, r.out).toBe(0); + expect(readFileSync(other.file, "utf8")).toBe(other.content); + expect(launchctlCalls().filter((l) => l.includes(other.file))).toEqual([]); + expect(existsSync(backupDir())).toBe(false); + }); +}); diff --git a/packages/extension/tools/fleet/install.sh b/packages/extension/tools/fleet/install.sh index f6bdd7a98..1afc2f81e 100755 --- a/packages/extension/tools/fleet/install.sh +++ b/packages/extension/tools/fleet/install.sh @@ -144,6 +144,37 @@ peer_forward_targets() { ' "$PEER_STORE" "$ROSTER_FILE" } +# A hand-installed forward predates the managed convention: any +# co.harmoniqs.amico--forward.plist that is NOT a managed +# co.harmoniqs.amico-peer-*-forward agent. It is "for the same peer" when it +# binds the peer's local port (it would block the managed agent's -L under +# ExitOnForwardFailure) or targets the peer's ssh destination. +PEER_FORWARD_BACKUP_DIR="$HOME/.amico/ops/fleet/launchagents-backup" +is_legacy_forward_for() { # + local f="$1" lport="$2" alias="$3" peer_id="$4" base + base="$(basename "$f")" + if [[ "$base" == co.harmoniqs.amico-peer-*-forward.plist ]]; then return 1; fi + if grep -qE "((127\.0\.0\.1|localhost):)?${lport}:[^<]*" "$f"; then return 0; fi + if grep -qF "${alias}" "$f" || grep -qF "${peer_id}" "$f"; then return 0; fi + return 1 +} + +# Replace (never silently delete): unload the legacy job, then MOVE its plist +# to a timestamped backup OUTSIDE LaunchAgents (so launchd never reloads it). +replace_legacy_forwards() { # + local f bk + for f in "$LAUNCH_AGENTS_DIR"/co.harmoniqs.amico-*-forward.plist; do + if [[ ! -f "$f" ]]; then continue; fi + if ! is_legacy_forward_for "$f" "$2" "$3" "$1"; then continue; fi + mkdir -p "$PEER_FORWARD_BACKUP_DIR" + bk="$PEER_FORWARD_BACKUP_DIR/$(basename "$f").$(date -u +%Y%m%dT%H%M%SZ)" + if [[ -e "$bk" ]]; then bk="$bk.$$"; fi + "$LAUNCHCTL" unload "$f" 2>/dev/null || true + mv "$f" "$bk" + say "replaced hand-installed peer forward $f with the managed co.harmoniqs.amico-peer-$1-forward (backup: $bk)" + done +} + peer_forwards() { if [[ "$(uname -s)" != "Darwin" ]]; then return 0; fi if [[ $CHECK -eq 1 ]]; then return 0; fi @@ -161,6 +192,7 @@ peer_forwards() { mkdir -p "$LAUNCH_AGENTS_DIR" tmpf="$(mktemp)" sed -e "s/PEER_MACHINE_ID/${peer_id}/g" -e "s/PEER_LOCAL_PORT/${lport}/g" -e "s/PEER_REMOTE_PORT/${FLEET_PORT}/g" -e "s/PEER_SSH_ALIAS/${alias}/g" "$PEER_PLIST_SRC" > "$tmpf" + replace_legacy_forwards "$peer_id" "$lport" "$alias" if [[ -f "$dst" ]] && cmp -s "$tmpf" "$dst"; then say "peer forward $label unchanged — skipping reload" else diff --git a/tools/fleet/install.sh b/tools/fleet/install.sh index f6bdd7a98..1afc2f81e 100755 --- a/tools/fleet/install.sh +++ b/tools/fleet/install.sh @@ -144,6 +144,37 @@ peer_forward_targets() { ' "$PEER_STORE" "$ROSTER_FILE" } +# A hand-installed forward predates the managed convention: any +# co.harmoniqs.amico--forward.plist that is NOT a managed +# co.harmoniqs.amico-peer-*-forward agent. It is "for the same peer" when it +# binds the peer's local port (it would block the managed agent's -L under +# ExitOnForwardFailure) or targets the peer's ssh destination. +PEER_FORWARD_BACKUP_DIR="$HOME/.amico/ops/fleet/launchagents-backup" +is_legacy_forward_for() { # + local f="$1" lport="$2" alias="$3" peer_id="$4" base + base="$(basename "$f")" + if [[ "$base" == co.harmoniqs.amico-peer-*-forward.plist ]]; then return 1; fi + if grep -qE "((127\.0\.0\.1|localhost):)?${lport}:[^<]*" "$f"; then return 0; fi + if grep -qF "${alias}" "$f" || grep -qF "${peer_id}" "$f"; then return 0; fi + return 1 +} + +# Replace (never silently delete): unload the legacy job, then MOVE its plist +# to a timestamped backup OUTSIDE LaunchAgents (so launchd never reloads it). +replace_legacy_forwards() { # + local f bk + for f in "$LAUNCH_AGENTS_DIR"/co.harmoniqs.amico-*-forward.plist; do + if [[ ! -f "$f" ]]; then continue; fi + if ! is_legacy_forward_for "$f" "$2" "$3" "$1"; then continue; fi + mkdir -p "$PEER_FORWARD_BACKUP_DIR" + bk="$PEER_FORWARD_BACKUP_DIR/$(basename "$f").$(date -u +%Y%m%dT%H%M%SZ)" + if [[ -e "$bk" ]]; then bk="$bk.$$"; fi + "$LAUNCHCTL" unload "$f" 2>/dev/null || true + mv "$f" "$bk" + say "replaced hand-installed peer forward $f with the managed co.harmoniqs.amico-peer-$1-forward (backup: $bk)" + done +} + peer_forwards() { if [[ "$(uname -s)" != "Darwin" ]]; then return 0; fi if [[ $CHECK -eq 1 ]]; then return 0; fi @@ -161,6 +192,7 @@ peer_forwards() { mkdir -p "$LAUNCH_AGENTS_DIR" tmpf="$(mktemp)" sed -e "s/PEER_MACHINE_ID/${peer_id}/g" -e "s/PEER_LOCAL_PORT/${lport}/g" -e "s/PEER_REMOTE_PORT/${FLEET_PORT}/g" -e "s/PEER_SSH_ALIAS/${alias}/g" "$PEER_PLIST_SRC" > "$tmpf" + replace_legacy_forwards "$peer_id" "$lport" "$alias" if [[ -f "$dst" ]] && cmp -s "$tmpf" "$dst"; then say "peer forward $label unchanged — skipping reload" else From ac340bc3b5630a7f471185e272709676de3e71a5 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 20:31:40 -0400 Subject: [PATCH 45/60] =?UTF-8?q?fix(fleet):=20a=20re-opened=20arm=20resum?= =?UTF-8?q?es=20from=20its=20last=20engine=20id=20=E2=80=94=20the=20upstre?= =?UTF-8?q?am=20replays=20the=20stretch,=20no=20gap,=20no=20resync=20(#177?= =?UTF-8?q?7)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fan-in's re-open of an arm that delivered #1776 engine ids now RESUMES: reopenArm returns `resume`, the driver re-opens from cursorFor (the last id accepted — a frame queued behind backpressure counts, one dropped on overflow does not), and nothing is written. The upstream's first answer decides: its server.connected confirms the replay and, in composite mode, is not forwarded (it is the upstream's handshake, not an event — the client would refetch on it); an id-less answer means an engine that cannot replay, so the #1773 fallback announces the loss (gap / sever). The #1773 gap and sever stay the fallback for id-less (pre-#1776) upstreams; the suites' sequenced re-open cases now assert the resume. --- .../amicode_service/sse_fanin_aggregator.ts | 215 ++++++++++++++---- .../src/amicode_service/sse_fanin_driver.ts | 33 ++- .../fixtures/engine_stream/engine_frames.ts | 135 +++++++++++ .../test/sse_fanin_aggregator.test.ts | 135 ++++++++++- .../test/sse_fanin_driver_pump.test.ts | 190 ++++++++++++++-- 5 files changed, 623 insertions(+), 85 deletions(-) diff --git a/packages/extension/src/amicode_service/sse_fanin_aggregator.ts b/packages/extension/src/amicode_service/sse_fanin_aggregator.ts index 78ddcf939..df12d69c4 100644 --- a/packages/extension/src/amicode_service/sse_fanin_aggregator.ts +++ b/packages/extension/src/amicode_service/sse_fanin_aggregator.ts @@ -176,6 +176,39 @@ export function isDataFrame(rawFrame: string): boolean { return false; } +/** #1777 — the engine's per-connection control frames on `/global/event`. They + * carry the connection's position as their id but consume no sequence (#1776). */ +export type EngineControlType = "server.connected" | "server.heartbeat" | "server.cursor.unservable"; + +/** The engine writes a control frame as a payload-only event whose payload + * starts `{"id":…,"type":…}` (handlers/global.ts `controlEvent`). A bus event + * serializes `{"directory":…` first, and quoted text inside any event is + * JSON-escaped, so this anchored match never mistakes one for a control frame. */ +const ENGINE_CONTROL = /^data: ?\{"payload":\{"id":"[^"\\]*","type":"(server\.(?:connected|heartbeat|cursor\.unservable))"/m; +/** Control frames are tiny; a big frame is never one (and is never scanned). */ +const ENGINE_CONTROL_MAX_BYTES = 2048; + +/** #1777 — which engine control frame this is, or undefined for any other frame + * (a bus event, a comment, a relay's own control frame). */ +export function engineControlType(rawFrame: string): EngineControlType | undefined { + if (rawFrame.length > ENGINE_CONTROL_MAX_BYTES) return undefined; + return ENGINE_CONTROL.exec(rawFrame)?.[1] as EngineControlType | undefined; +} + +/** #1777 — the engine's stream position `:` (#1776), or undefined + * for any other id (an older relay's, a composite). */ +export function parseEngineId(id: string | undefined): { epoch: string; seq: number } | undefined { + if (id === undefined) return undefined; + const m = /^([A-Za-z0-9_-]+):(\d+)$/.exec(id.trim()); + return m ? { epoch: m[1], seq: Number(m[2]) } : undefined; +} + +/** The id-less, payload-less gap frame the client resyncs on (#1617 shape). */ +function gapFrame(namespaces: string[]): string { + const data = JSON.stringify({ type: "amicode.sync.gap", namespaces }); + return `event: amicode.sync.gap\ndata: ${data}\n\n`; +} + /** Rebuild a frame with its `id:` line set to `newId` (or removed when null), * preserving every other line (`event:`/`data:`/`retry:`/comments) verbatim. * The single canonical `id:` line is placed just before the trailing blank so @@ -242,18 +275,29 @@ export interface ArmTraffic { } /** #1771 — why a gap frame was emitted: an arm re-opened after delivering - * (#1638), or frames were dropped on overflow (#1617). */ -export type GapCause = "reopen" | "overflow"; + * (#1638), frames were dropped on overflow (#1617), or (#1777) the upstream + * answered a resume with a reset — its replay could not serve the cursor. */ +export type GapCause = "reopen" | "overflow" | "reset"; /** #1773 — what a re-open of an arm must do (`reopenArm`'s verdict): - * - `open` — the arm never delivered a data frame (an initial open): nothing - * was lost, open it silently; - * - `gap` — composite mode: the gap frame naming it was written; open it; - * - `sever` — fleet-of-one: a synthetic frame is not allowed in the verbatim - * stream (#1264), so the caller must END THE LISTENER WITH A - * TRANSPORT ERROR instead of re-opening — the client registers the - * disconnect and resyncs. Nothing was written. */ -export type ReopenVerdict = "open" | "gap" | "sever"; + * - `open` — the arm never delivered a data frame (an initial open): nothing + * was lost, open it silently; + * - `resume` — #1777: the arm delivered engine-sequenced frames, so open it + * FROM ITS CURSOR (`cursorFor`) — the upstream replays what was + * missed. Nothing is written; whatever the replay cannot serve is + * announced when the upstream answers (a reset → one gap); + * - `gap` — composite mode, an upstream with no ids to resume from (an older + * engine): the gap frame naming it was written; open it; + * - `sever` — fleet-of-one, same upstream: a synthetic frame is not allowed in + * the verbatim stream (#1264), so the caller must END THE LISTENER + * WITH A TRANSPORT ERROR instead of re-opening — the client + * registers the disconnect and resyncs. Nothing was written. */ +export type ReopenVerdict = "open" | "resume" | "gap" | "sever"; + +/** #1777 — what the caller of `ingest` must do with the arm: carry on, or (a + * fleet-of-one resume the upstream could not honor — it answered without ids) + * sever the listener, since the verbatim stream may not carry a gap frame. */ +export type IngestVerdict = "ok" | "sever"; /** #1771 — the aggregator's transitions, for the census. Observation only: * callbacks run after the aggregator decided, and a throwing observer is @@ -329,6 +373,18 @@ export class SseFanInAggregator { * counts (an older engine sends no `id:` lines at all), counted as the arm * hands it over — whether it is then delivered, buffered, or dropped. */ private readonly everDelivered = new Set(); + /** #1777 — namespace → the engine id its arm RESUMES from: the last id'd frame + * this connection ACCEPTED from it (written, or queued behind backpressure — + * never one dropped on overflow), so a re-open's replay starts right after + * everything already on its way to the listener and never re-sends it. An + * upstream reset re-bases it in the new epoch. */ + private readonly resumeAt = new Map(); + /** #1777 — arms re-opened FROM A CURSOR whose upstream has not answered yet. + * The first data frame decides: the engine's `server.connected` confirms the + * replay (and is the upstream's handshake, not an event — it is not + * forwarded in composite mode); a reset frame becomes one gap; an id-less + * frame means an upstream that cannot resume at all (the #1773 fallback). */ + private readonly resuming = new Set(); /** #1617 (fleet-of-one) — frames stranded on the zero-peer VERBATIM path while * the downstream is backpressured. Kept SEPARATE from the composite `buffers` * because these flush byte-for-byte (raw `sink.write`, no composite-id rewrite) @@ -353,7 +409,11 @@ export class SseFanInAggregator { connect(cursor?: string): Map { const resume = parseCompositeCursor(cursor); this.composite.clear(); - for (const [k, v] of resume) this.composite.set(k, v); + this.resumeAt.clear(); + for (const [k, v] of resume) { + this.composite.set(k, v); + this.resumeAt.set(k, v); + } if (this.focusProvider) this.emitFocusFrame(); return new Map(resume); } @@ -400,18 +460,58 @@ export class SseFanInAggregator { } /** Ingest one raw SSE frame from an arm. `namespace === "local"` for the local - * arm; a machineId for a peer arm. */ - ingest(namespace: string, rawFrame: string): void { + * arm; a machineId for a peer arm. #1777: returns `sever` when the listener + * must be ended instead (see `IngestVerdict`); `ok` otherwise. */ + ingest(namespace: string, rawFrame: string): IngestVerdict { this.countFrame(namespace, rawFrame); // #1771 census — observation only - if (isDataFrame(rawFrame)) this.everDelivered.add(namespace); // #1773 — any data frame + const data = isDataFrame(rawFrame); + if (data) this.everDelivered.add(namespace); // #1773 — any data frame const rawId = frameRawId(rawFrame); const fleetOfOne = this.peerArms.size === 0; + const verbatim = namespace === LOCAL_NAMESPACE && fleetOfOne; + const control = data ? engineControlType(rawFrame) : undefined; + + // #1777 — the FIRST data frame of an arm re-opened from its cursor says + // whether the upstream replayed. An id-less one: an engine that cannot + // resume (older than #1776) ignored the cursor, so the stretch since the + // drop is lost — the #1773 fallback announces it (a gap; in fleet-of-one, + // where no synthetic frame may enter the verbatim stream, a sever). The + // engine's `server.connected` confirms the replay; in composite mode it is + // the upstream connection's handshake, not an event of this listener's + // stream, so it is not forwarded (the client would refetch on it). + if (data && this.resuming.delete(namespace)) { + if (rawId === undefined) { + if (verbatim) return "sever"; + this.writeControl(gapFrame([namespace])); + this.noteGap([namespace], "reopen"); + } else if (control === "server.connected" && !verbatim) { + this.advanceResume(namespace, rawId); + return "ok"; + } + } + + // #1777 — an upstream RESET: the engine could not serve the cursor the arm + // asked for (its process restarted, or the replay window moved past it). In + // composite mode that is exactly one gap naming this namespace — the reset + // frame itself is not forwarded (one signal, not two) — and the arm's resume + // point re-bases in the new epoch. In fleet-of-one the reset frame passes + // verbatim below: it IS the engine's own signal, and the client resyncs on + // it (#1264 — nothing synthetic enters the solo stream). + if (control === "server.cursor.unservable" && !verbatim) { + if (rawId !== undefined) this.advanceResume(namespace, rawId); + this.writeControl(gapFrame([namespace])); + this.noteGap([namespace], "reset"); + return "ok"; + } - if (namespace === LOCAL_NAMESPACE && fleetOfOne) { + if (verbatim) { // §D4 / AC1 byte-identity: with no peer arms the local frame is written // VERBATIM (bare id, every line untouched). The composite still tracks the // local position silently, so a peer that joins mid-stream inherits it. - if (rawId !== undefined) this.composite.set(LOCAL_NAMESPACE, rawId); + if (rawId !== undefined) { + this.composite.set(LOCAL_NAMESPACE, rawId); + this.advanceResume(LOCAL_NAMESPACE, rawId); + } // #1617 (fleet-of-one) — honor backpressure on the verbatim path too. When // the downstream is already backpressured, hold the frame VERBATIM in the // verbatim buffer (never namespaced) so it is not fired into an over-full @@ -421,7 +521,7 @@ export class SseFanInAggregator { // changes only TIMING — not a byte — so the #1264 guard holds. if (!this.flowing) { this.verbatimBuffer.push(rawFrame); - return; + return "ok"; } const accepted = this.sink.write(rawFrame); this.sink.flush?.(); @@ -429,7 +529,7 @@ export class SseFanInAggregator { this.flowing = false; this.onBackpressure?.(); } - return; + return "ok"; } // Composite mode (≥1 peer arm active): every arm's frames carry the full @@ -442,6 +542,7 @@ export class SseFanInAggregator { } else { this.enqueue(namespace, rawFrame); } + return "ok"; } /** Drain the ordered pending stream after the downstream drains (decision B / @@ -518,25 +619,33 @@ export class SseFanInAggregator { } } - /** #1638 — a peer/local arm is being RE-OPENED (the #1601 reconcile path). In - * composite mode (≥1 peer arm) a NON-INITIAL re-open forces a client refetch - * via exactly one id-less `amicode.sync.gap` naming that namespace, because - * the cursor-advance alone cannot recover the reconnect gap (an older engine - * has no ids to resume from; replay is #1777). + /** #1638 — a peer/local arm is being RE-OPENED (the #1601 reconcile path). * - * #1773 — every loss is loud. "Non-initial" means the arm delivered ANY data - * frame (id'd or not). In FLEET-OF-ONE (zero peers) a synthetic frame in the - * verbatim stream would violate the #1264 byte-identity guard, so instead of - * re-opening silently the verdict is `sever`: the caller ends the listener - * with a transport error, and the client's disconnect is the gap signal. - * Nothing is written for `sever`. The INITIAL open of an arm (never delivered - * a data frame) is `open` in both modes — nothing was lost. A gap never - * advances any cursor. */ + * #1777 — an arm that delivered engine-sequenced frames RESUMES: the caller + * re-opens it from `cursorFor(namespace)` and the upstream replays the frames + * the listener missed, so nothing is written and nothing is lost — no gap, no + * sever, in either mode. What the replay cannot serve is announced when the + * upstream answers (`ingest`: a reset → one gap; an id-less answer → the + * fallback below). + * + * #1773 — the FALLBACK, for an upstream with no ids to resume from (an engine + * older than #1776): every loss is loud. "Non-initial" means the arm + * delivered ANY data frame (id'd or not). In composite mode the re-open + * forces a client refetch via exactly one id-less `amicode.sync.gap` naming + * the namespace. In FLEET-OF-ONE (zero peers) a synthetic frame in the + * verbatim stream would violate the #1264 byte-identity guard, so the verdict + * is `sever`: the caller ends the listener with a transport error, and the + * client's disconnect is the gap signal. Nothing is written for `sever`. The + * INITIAL open of an arm (never delivered a data frame) is `open` in both + * modes — nothing was lost. A gap never advances any cursor. */ reopenArm(namespace: string): ReopenVerdict { if (!this.everDelivered.has(namespace)) return "open"; // initial open → nothing lost + if (parseEngineId(this.resumeAt.get(namespace)) !== undefined) { + this.resuming.add(namespace); // the upstream's first answer confirms or refuses the replay + return "resume"; + } if (this.peerArms.size === 0) return "sever"; // fleet-of-one → NO synthetic frame (#1264) - const data = JSON.stringify({ type: "amicode.sync.gap", namespaces: [namespace] }); - this.writeControl(`event: amicode.sync.gap\ndata: ${data}\n\n`); + this.writeControl(gapFrame([namespace])); this.noteGap([namespace], "reopen"); return "gap"; } @@ -553,8 +662,7 @@ export class SseFanInAggregator { const dropped: string[] = []; for (const [ns, n] of this.droppedCount) if (n > 0) dropped.push(ns); if (dropped.length === 0) return; - const data = JSON.stringify({ type: "amicode.sync.gap", namespaces: dropped }); - this.writeControl(`event: amicode.sync.gap\ndata: ${data}\n\n`); + this.writeControl(gapFrame(dropped)); for (const ns of dropped) this.droppedCount.set(ns, 0); this.noteGap(dropped, "overflow"); } @@ -570,14 +678,19 @@ export class SseFanInAggregator { return formatCompositeCursor(this.composite); } - /** #1638 — the live per-namespace last-DELIVERED id (the composite map's - * entry for `namespace`), or undefined if the arm has delivered no id'd frame - * yet. The driver reads this at re-open time to resume the arm from its - * last-delivered position rather than the stale connect-time cursor — honoring - * #1617's own cursor invariant. Advances per delivered frame on BOTH the - * composite and the verbatim path; an id-less frame never advances it. */ + /** #1638 — the live per-namespace resume position the driver re-opens an arm + * from (rather than the stale connect-time cursor — honoring #1617's own + * cursor invariant), or undefined if the arm has delivered no id'd frame and + * the client's cursor named none. Advances per frame on BOTH the composite + * and the verbatim path; an id-less frame never advances it. + * + * #1777 — it holds the engine's own `:` and counts a frame queued + * behind backpressure as delivered (it is on its way to the listener), so the + * upstream's replay after a re-open starts right after it and never re-sends + * a frame the listener will already get; a frame dropped on overflow does + * not count (that stretch is the overflow gap's, #1617). */ cursorFor(namespace: string): string | undefined { - return this.composite.get(namespace); + return this.resumeAt.get(namespace); } /** #1771 — one arm's traffic counters on this connection (a copy; zeros for @@ -637,7 +750,10 @@ export class SseFanInAggregator { * downstream (subsequent frames enqueue onto the ordered pending stream). */ private deliver(namespace: string, rawFrame: string): void { const rawId = frameRawId(rawFrame); - if (rawId !== undefined) this.composite.set(namespace, rawId); + if (rawId !== undefined) { + this.composite.set(namespace, rawId); + this.advanceResume(namespace, rawId); + } const wire = rewriteFrameId(rawFrame, formatCompositeCursor(this.composite)); const accepted = this.sink.write(wire); this.sink.flush?.(); @@ -670,6 +786,19 @@ export class SseFanInAggregator { return; } this.pending.push({ kind: "data", namespace, frame: rawFrame }); + const rawId = frameRawId(rawFrame); + if (rawId !== undefined) this.advanceResume(namespace, rawId); // #1777 — queued = on its way + } + + /** #1777 — move a namespace's resume point to `rawId`: never backwards within + * one engine epoch (a resume point only ever follows what was accepted); a + * new epoch (the upstream restarted and reset) or an id that is not an + * engine's replaces it. */ + private advanceResume(namespace: string, rawId: string): void { + const next = parseEngineId(rawId); + const prev = parseEngineId(this.resumeAt.get(namespace)); + if (next && prev && next.epoch === prev.epoch && next.seq < prev.seq) return; + this.resumeAt.set(namespace, rawId); } private emitFocusFrame(): void { diff --git a/packages/extension/src/amicode_service/sse_fanin_driver.ts b/packages/extension/src/amicode_service/sse_fanin_driver.ts index a4330fc7e..88a490cc5 100644 --- a/packages/extension/src/amicode_service/sse_fanin_driver.ts +++ b/packages/extension/src/amicode_service/sse_fanin_driver.ts @@ -395,17 +395,21 @@ class FanInConnection implements CensusConnection { } } - /** #1638 — re-open an arm from the aggregator's LIVE last-delivered cursor (not - * the stale connect-time id) and, on a NON-INITIAL re-open, force a client - * refetch via the aggregator's composite-mode gap. `agg.reopenArm` is a no-op - * for an arm that has never delivered (an initial open), so it is safe to call - * at every reconcile-time open. Falls back to the connect-time resume id when - * the arm has delivered nothing yet. + /** #1638 — re-open an arm from the aggregator's LIVE resume position (not the + * stale connect-time id). `agg.reopenArm` is a no-op for an arm that has + * never delivered (an initial open), so it is safe to call at every + * reconcile-time open. Falls back to the connect-time resume id when the arm + * has delivered nothing yet. * - * #1773 — in fleet-of-one the verdict for an arm that delivered is `sever`: a - * synthetic gap frame may not enter the verbatim stream (#1264), so instead of - * re-opening silently (losing the stretch unannounced) the listener is ended - * with a transport error, named after the loss that caused it. */ + * #1777 — an arm that delivered engine ids RESUMES: it re-opens from its last + * delivered id and the upstream replays the stretch it missed — no gap, no + * sever. An upstream reset is announced as the upstream answers. + * + * #1773 — the fallback for an upstream without ids: in composite mode one gap; + * in fleet-of-one the verdict is `sever` (a synthetic gap frame may not enter + * the verbatim stream, #1264), so instead of re-opening silently (losing the + * stretch unannounced) the listener is ended with a transport error, named + * after the loss that caused it. */ private reopenAndTrack(namespace: string, url: string | undefined, authHeader: string | undefined): void { if (this.sources.has(namespace)) return; if (this.agg.reopenArm(namespace) === "sever") { @@ -461,7 +465,14 @@ class FanInConnection implements CensusConnection { break; } if (frame === null || this.closed) break; - this.agg.ingest(namespace, frame); + // #1777 — a fleet-of-one arm resumed from its cursor whose upstream + // answered without ids (an engine that cannot replay) lost the stretch + // since the drop; the verbatim stream may carry no gap frame, so the + // listener is severed under the loss that caused the re-open (#1773). + if (this.agg.ingest(namespace, frame) === "sever") { + this.close(this.lastClose.get(namespace) ?? "upstream-end"); + return; + } } // #1771 — the upstream's OWN end (not one this connection caused): name it. if (!this.closed && !this.closedByUs.has(source)) { diff --git a/packages/extension/test/fixtures/engine_stream/engine_frames.ts b/packages/extension/test/fixtures/engine_stream/engine_frames.ts index 3dcdb0c24..4fc39b93e 100644 --- a/packages/extension/test/fixtures/engine_stream/engine_frames.ts +++ b/packages/extension/test/fixtures/engine_stream/engine_frames.ts @@ -179,6 +179,141 @@ export function fakeEngine( }; } +// ── #1777: an engine that REPLAYS (the #1776 subscribe contract) ──────────── + +/** One subscription to a replaying engine's `/global/event`: the arm source the + * fan-in pumps, plus the test's handle to end it (a link drop, an engine exit). */ +export interface EngineSubscription { + /** The cursor this subscription asked for (`undefined` = live only). */ + readonly cursor: string | undefined; + /** Whether the engine answered the cursor with a reset. */ + readonly reset: boolean; + source: { + next(): Promise; + close(): void; + endReason(): "end" | "error" | undefined; + }; + /** End the subscription: `"end"` = a clean upstream end, `"error"` = a failure. */ + end(why?: "end" | "error"): void; +} + +export interface ReplayingEngine { + /** The process epoch. */ + readonly epoch: string; + /** The current sequence (the last bus event's), 0 before any. */ + readonly seq: number; + /** The SSE id this engine stamps for `seq`. */ + idAt(seq: number): string; + /** Emit one bus event: framed ONCE (`id: :`), retained for replay, + * and delivered to every open subscription. Returns the frame. */ + publish(type: string, properties: Record): string; + /** A heartbeat on every open subscription (`{time, seq}`, the CURRENT seq). */ + heartbeat(): void; + /** Subscribe like `GET /global/event[?lastEventID=…]`: a servable cursor gets + * `server.connected` AT the cursor, then every retained frame after it, then + * live frames; an unservable one (wrong epoch, evicted, ahead, malformed) gets + * the `server.cursor.unservable` reset frame first, then `server.connected` at + * the current seq, then live. No cursor = `server.connected` + live. */ + subscribe(cursor?: string): EngineSubscription; + /** Drop retained frames at or below `seq` (the replay window moved past them). */ + evictThrough(seq: number): void; + /** Every subscription opened so far, in order. */ + readonly subscriptions: readonly EngineSubscription[]; + /** The subscriptions still open. */ + live(): EngineSubscription[]; +} + +/** A #1776 engine with a replay buffer, in-process: the fan-in suites drive a + * re-open against it and see exactly what a real engine would send. */ +export function replayingEngine(opts: { epoch: string; now?: () => number; directory?: string }): ReplayingEngine { + const epoch = opts.epoch; + const now = opts.now ?? (() => 0); + const directory = opts.directory ?? ENGINE_DIRECTORY; + let seq = 0; + let floor = 0; // the highest seq no longer retained + let evt = 0; + const retained: Array<{ seq: number; frame: string }> = []; + const subscriptions: Array = []; + const nextPayloadId = (): string => `evt_${epoch}_${String(++evt).padStart(6, "0")}`; + const control = (at: number, type: string, properties: Record = {}): string => + engineFrame(controlEvent(nextPayloadId(), type, properties), { id: engineId(epoch, at) }); + + const open = (cursor: string | undefined) => { + const queue: string[] = []; + const waiters: Array<(v: string | null) => void> = []; + let ended = false; + let reason: "end" | "error" | undefined; + const end = (why: "end" | "error" = "end"): void => { + if (ended) return; + ended = true; + reason = why; + sub.open = false; + let w: ((v: string | null) => void) | undefined; + while ((w = waiters.shift())) w(null); + }; + const push = (frame: string): void => { + if (ended) return; + const w = waiters.shift(); + if (w) w(frame); + else queue.push(frame); + }; + const parsed = cursor === undefined ? undefined : /^([A-Za-z0-9_-]+):(\d+)$/.exec(cursor.trim()); + const servable = + cursor === undefined || + (parsed !== null && parsed !== undefined && parsed[1] === epoch && Number(parsed[2]) <= seq && Number(parsed[2]) >= floor); + const position = cursor !== undefined && servable && parsed ? Number(parsed[2]) : seq; + const sub = { + cursor, + reset: !servable, + open: true, + push, + end, + source: { + next(): Promise { + const f = queue.shift(); + if (f !== undefined) return Promise.resolve(f); + if (ended) return Promise.resolve(null); + return new Promise((resolve) => waiters.push(resolve)); + }, + close: () => end("end"), + endReason: () => reason, + }, + }; + if (!servable) push(control(seq, "server.cursor.unservable", { epoch, seq })); + push(control(position, "server.connected")); + if (servable) for (const r of retained) if (r.seq > position) push(r.frame); + subscriptions.push(sub); + return sub; + }; + + return { + epoch, + get seq() { + return seq; + }, + idAt: (s) => engineId(epoch, s), + publish(type, properties) { + const s = ++seq; + const frame = engineFrame(busEvent({ id: nextPayloadId(), type, properties, directory }), { id: engineId(epoch, s) }); + retained.push({ seq: s, frame }); + for (const sub of subscriptions) if (sub.open) sub.push(frame); + return frame; + }, + heartbeat() { + for (const sub of subscriptions) if (sub.open) sub.push(control(seq, "server.heartbeat", { time: now(), seq })); + }, + subscribe: (cursor) => open(cursor), + evictThrough(s) { + while (retained.length > 0 && retained[0].seq <= s) retained.shift(); + floor = Math.max(floor, s); + }, + get subscriptions() { + return subscriptions; + }, + live: () => subscriptions.filter((s) => s.open), + }; +} + /** A one-off sequenced bus-event frame at an explicit `seq` — for suites that * script many namespaces' positions directly. `properties` ride verbatim. */ export function sequencedEvent(epoch: string, seq: number, properties: Record, type = "message.part.updated"): string { diff --git a/packages/extension/test/sse_fanin_aggregator.test.ts b/packages/extension/test/sse_fanin_aggregator.test.ts index e27c3d727..d6079b933 100644 --- a/packages/extension/test/sse_fanin_aggregator.test.ts +++ b/packages/extension/test/sse_fanin_aggregator.test.ts @@ -41,7 +41,7 @@ import { import { SessionOwnerMap } from "../src/amicode_service/session_multiplexer"; import { peerAuthHeader } from "../src/amicode_service/merged_projection"; import type { PeerTokenRead } from "../src/amicode_service/fleet_peer_store"; -import { fakeEngine, sequencedEvent, engineId, LOCAL_EPOCH, PEER_EPOCH } from "./fixtures/engine_stream/engine_frames"; +import { fakeEngine, sequencedEvent, legacyEvent, engineId, engineFrame, controlEvent, LOCAL_EPOCH, PEER_EPOCH } from "./fixtures/engine_stream/engine_frames"; // ── test helpers ────────────────────────────────────────────────────────────── @@ -83,6 +83,9 @@ function nsFrame(ns: string, n: number): string { const idIn = (ns: string, n: number) => engineId(EPOCH_OF[ns], n); /** The seq of an `:` id. */ const seqOf = (id: string | undefined) => Number(id?.split(":")[1]); +/** #1777: an engine per-connection control frame (connected / heartbeat / reset) at `id`. */ +const controlFrame = (id: string, type: string, properties: Record = {}) => + engineFrame(controlEvent(`evt_ctl_${id}`, type, properties), { id }); /** A frame source that yields a fixed list of raw frames, then ends. */ function fixedSource(frames: string[]): SseFrameSource { @@ -786,12 +789,15 @@ describe("#1638 — per-namespace cursor accessor (cursorFor)", () => { describe("#1638 — composite-mode reopenArm emits exactly one id-less gap; fleet-of-one emits none", () => { + // #1777: the gap is now the FALLBACK for an upstream with no ids to resume + // from (an engine older than #1776) — a sequenced arm resumes instead (see the + // #1777 suite below) — so these frames come from a legacy (id-less) engine. it("COMPOSITE MODE: a non-initial re-open emits EXACTLY ONE id-less amicode.sync.gap naming that namespace", () => { const sink = collectingSink(); const agg = new SseFanInAggregator({ sink }); agg.connect(); agg.setMembership([{ machineId: "studio", reachable: true, token: "tok-studio" }]); - agg.ingest("studio", nsFrame("studio", 1)); + agg.ingest("studio", legacyEvent({ ns: "studio", n: 1 })); agg.reopenArm("studio"); // the #1601 re-open path (arm dropped, reconcile re-opens) @@ -845,8 +851,11 @@ describe("#1773 AC1 — a re-open of an arm that delivered ANY data frame emits const gapFrames = (text: string) => text.split("\n\n").filter((f) => f.includes("amicode.sync.gap")); const gapOf = (f: string) => JSON.parse(f.split("\n").find((l) => l.startsWith("data:"))!.slice(5).trim()); + // #1777: in BOTH modes "delivered" still means any data frame — but only the + // legacy (id-less) arm gaps: the sequenced arm has engine ids to resume from, + // so its re-open RESUMES (the upstream replays) and writes nothing at all. for (const mode of ["legacy", "sequenced"] as const) { - it(`${mode} engine: a non-initial re-open emits exactly one id-less gap naming that namespace, and never advances a cursor`, () => { + it(`${mode} engine: a non-initial re-open ${mode === "legacy" ? "emits exactly one id-less gap naming that namespace" : "resumes — no gap"}, and never advances a cursor`, () => { const sink = collectingSink(); const agg = new SseFanInAggregator({ sink }); agg.connect(`local=${LOCAL_EPOCH}:3`); @@ -868,6 +877,14 @@ describe("#1773 AC1 — a re-open of an arm that delivered ANY data frame emits expect(sink.text()).not.toContain("amicode.sync.gap"); // the arm drops and the driver re-opens it + if (mode === "sequenced") { + const before = sink.text(); + expect(agg.reopenArm("studio")).toBe("resume"); + expect(agg.cursorFor("studio")).toBe(studio.idAt(studio.seq)); // … from the last delivered engine id + expect(sink.text()).toBe(before); // nothing written + expect(agg.cursor()).toBe(cursorBefore); + return; + } expect(agg.reopenArm("studio")).toBe("gap"); const gaps = gapFrames(sink.text()); @@ -987,8 +1004,9 @@ describe("#1638 — control/gap/focus writes are return-checked and drain in tru agg.setMembership([{ machineId: "studio", reachable: true, token: "tok-studio" }]); sink.block(); - agg.ingest("studio", nsFrame("studio", 1)); // trip-frame: written, flowing→false - agg.ingest("studio", nsFrame("studio", 2)); // buffered behind the trip-frame + // #1777: an id-less (legacy) peer, so the re-open below takes the gap fallback + agg.ingest("studio", legacyEvent({ ns: "studio", n: 1 })); // trip-frame: written, flowing→false + agg.ingest("studio", legacyEvent({ ns: "studio", n: 2 })); // buffered behind the trip-frame // a control frame arrives mid-backpressure — it MUST NOT jump ahead of the // buffered data frame, nor be swallowed into the full sink. agg.reopenArm("studio"); // emits a gap through the return-checked helper @@ -1095,7 +1113,12 @@ describe("#1771 — per-arm traffic counters, pending depth, and the drop/gap ob for (let n = 7; n <= 10; n++) agg.ingest("studio", sequencedEvent(PEER_EPOCH, n, { n })); expect(drops).toHaveLength(2); - agg.reopenArm("studio"); // composite mode, already delivered → a reopen gap + // #1777: a sequenced arm's re-open resumes (no gap); an upstream that then + // answers WITHOUT ids cannot have replayed — that is the reopen gap + expect(agg.reopenArm("studio")).toBe("resume"); + sink.unblock(); + agg.resume(); + agg.ingest("studio", legacyEvent({ n: 11 })); expect(gaps.at(-1)).toEqual([["studio"], "reopen"]); }); @@ -1111,7 +1134,7 @@ describe("#1771 — per-arm traffic counters, pending depth, and the drop/gap ob }); agg.connect(); agg.setMembership([{ machineId: "studio", reachable: true, token: "tok-studio" }]); - const f = sequencedEvent(PEER_EPOCH, 1, { n: 1 }); + const f = legacyEvent({ n: 1 }); // #1777: an id-less peer — its re-open gaps agg.ingest("studio", f); expect(() => agg.reopenArm("studio")).not.toThrow(); expect(sink.text()).toContain("amicode.sync.gap"); @@ -1148,3 +1171,101 @@ describe("#1771 AC5 — the census observer changes no delivered byte", () => { expect(plain).toContain("amicode.sync.gap"); // the sequence did exercise drop + gap }); }); + +// ══════════════════════════════════════════════════════════════════════════════ +// #1777 — LOSSLESS RESUME in the aggregator core. An arm that delivered engine +// ids (#1776 `:`) RESUMES on re-open: `cursorFor` is where its +// replay starts, nothing is written, and the upstream's first answer decides — +// its `server.connected` confirms the replay; an id-less answer means it could +// not resume (the #1773 fallback announces the loss). +// ══════════════════════════════════════════════════════════════════════════════ +describe("#1777 AC1 — the aggregator resumes a sequenced arm from its last id", () => { + function blockableSink() { + const chunks: string[] = []; + let blocked = false; + return { + write(chunk: string): boolean { + chunks.push(chunk); + return !blocked; + }, + flush() {}, + block() { + blocked = true; + }, + unblock() { + blocked = false; + }, + text: () => chunks.join(""), + }; + } + + it("a re-open while frames still queue behind backpressure resumes AFTER them — the replay never re-sends one", () => { + const sink = blockableSink(); + const agg = new SseFanInAggregator({ sink }); + agg.connect(); + agg.setMembership([{ machineId: "studio", reachable: true, token: "tok-studio" }]); + sink.block(); + for (const n of [1, 2, 3]) agg.ingest("studio", nsFrame("studio", n)); // 1 written (trips), 2 + 3 queued + expect(agg.pendingDepth("studio")).toBe(2); + expect(agg.cursorFor("studio")).toBe(idIn("studio", 3)); // queued = on its way to the listener + expect(agg.reopenArm("studio")).toBe("resume"); + sink.unblock(); + agg.resume(); + // the resumed upstream answers from 3: connected at 3, then 4 — each frame exactly once + agg.ingest("studio", controlFrame(idIn("studio", 3), "server.connected")); + agg.ingest("studio", nsFrame("studio", 4)); + const ns = [...sink.text().matchAll(/"n":(\d+)/g)].map((m) => Number(m[1])); + expect(ns).toEqual([1, 2, 3, 4]); + expect(sink.text()).not.toContain("amicode.sync.gap"); + expect(sink.text()).not.toContain("server.connected"); // the upstream's handshake is not forwarded + }); + + it("an overflow-dropped frame never advances the resume point (that stretch is the overflow gap's)", () => { + const sink = blockableSink(); + const agg = new SseFanInAggregator({ sink, bufferBound: 1 }); + agg.connect(); + agg.setMembership([{ machineId: "studio", reachable: true, token: "tok-studio" }]); + sink.block(); + for (const n of [1, 2, 3, 4]) agg.ingest("studio", nsFrame("studio", n)); // 1 written, 2 queued, 3 + 4 dropped + expect(agg.dropped("studio")).toBe(2); + expect(agg.cursorFor("studio")).toBe(idIn("studio", 2)); + }); + + it("composite: a resumed arm whose upstream answers WITHOUT ids (it cannot replay) gets one reopen gap, then its frames", () => { + const gaps: Array<[string[], string]> = []; + const sink = collectingSink(); + const agg = new SseFanInAggregator({ sink, observer: { onGap: (nss, cause) => gaps.push([nss, cause]) } }); + agg.connect(); + agg.setMembership([{ machineId: "studio", reachable: true, token: "tok-studio" }]); + agg.ingest("studio", nsFrame("studio", 1)); + expect(agg.reopenArm("studio")).toBe("resume"); + expect(sink.text()).not.toContain("amicode.sync.gap"); + expect(agg.ingest("studio", fakeEngine({ legacy: true }).connected())).toBe("ok"); + expect(gaps).toEqual([[["studio"], "reopen"]]); + const text = sink.text(); + expect(text.indexOf("amicode.sync.gap")).toBeLessThan(text.indexOf("server.connected")); // gap first, then the frame + }); + + it("fleet-of-one: the same id-less answer severs (ingest says so, nothing is written — #1264)", () => { + const sink = collectingSink(); + const agg = new SseFanInAggregator({ sink }); + agg.connect(); + const f1 = nsFrame(LOCAL_NAMESPACE, 1); + agg.ingest(LOCAL_NAMESPACE, f1); + expect(agg.reopenArm(LOCAL_NAMESPACE)).toBe("resume"); + expect(agg.ingest(LOCAL_NAMESPACE, fakeEngine({ legacy: true }).connected())).toBe("sever"); + expect(sink.text()).toBe(f1); + }); + + it("an arm with only id-less frames keeps the #1773 fallback (gap in composite, sever in fleet-of-one)", () => { + const composite = new SseFanInAggregator({ sink: collectingSink() }); + composite.connect(); + composite.setMembership([{ machineId: "studio", reachable: true, token: "tok-studio" }]); + composite.ingest("studio", legacyEvent({ n: 1 })); + expect(composite.reopenArm("studio")).toBe("gap"); + const solo = new SseFanInAggregator({ sink: collectingSink() }); + solo.connect(); + solo.ingest(LOCAL_NAMESPACE, legacyEvent({ n: 1 })); + expect(solo.reopenArm(LOCAL_NAMESPACE)).toBe("sever"); + }); +}); diff --git a/packages/extension/test/sse_fanin_driver_pump.test.ts b/packages/extension/test/sse_fanin_driver_pump.test.ts index 65d1f025e..14853cd4b 100644 --- a/packages/extension/test/sse_fanin_driver_pump.test.ts +++ b/packages/extension/test/sse_fanin_driver_pump.test.ts @@ -16,13 +16,17 @@ import { writeChunks, largeSummaryFrame, sequencedEvent, + legacyEvent, engineId, chunkBytes, CHUNK_64K, TEN_MB, LOCAL_EPOCH, PEER_EPOCH, + replayingEngine, + type ReplayingEngine, } from "./fixtures/engine_stream/engine_frames"; +import { parseCompositeCursor } from "../src/amicode_service/sse_composite_cursor"; // ── helpers ────────────────────────────────────────────────────────────────── @@ -719,8 +723,10 @@ describe("#1638 — re-open resumes from last-delivered id + composite-mode reco // #1773 AC1: in BOTH engine modes — an older peer engine sends no `id:` lines, // and its re-open must gap all the same ("delivered" = any data frame). + // #1777: the sequenced peer has engine ids to resume from, so its re-open + // RESUMES from the last delivered id instead — no gap (the replay covers it). for (const mode of ["sequenced", "legacy"] as const) - it(`COMPOSITE MODE (${mode} peer engine): re-opening a dropped peer arm emits exactly one amicode.sync.gap naming it`, async () => { + it(`COMPOSITE MODE (${mode} peer engine): re-opening a dropped peer arm ${mode === "legacy" ? "emits exactly one amicode.sync.gap naming it" : "resumes from its last delivered id, with no gap"}`, async () => { const ownerMap = new SessionOwnerMap() ownerMap.update([ { id: "ses-studio", amicode_owner: { owner_machine_id: "studio", owner_name: "studio", is_local: false } }, @@ -731,6 +737,7 @@ describe("#1638 — re-open resumes from last-delivered id + composite-mode reco const localCtl = controllableSource() const peerCtls: Ctl[] = [] + const peerOpens: Array = [] const driver = new SseFanInDriver({ ownerMap, @@ -744,6 +751,7 @@ describe("#1638 — re-open resumes from last-delivered id + composite-mode reco reconcileMs: 999_999, openUpstream: (r) => { if (r.namespace === LOCAL_NAMESPACE) return localCtl.source + peerOpens.push(r.lastEventId) const ctl = controllableSource() peerCtls.push(ctl) return ctl.source @@ -761,18 +769,24 @@ describe("#1638 — re-open resumes from last-delivered id + composite-mode reco const beforeGap = (res.text().match(/amicode\.sync\.gap/g) ?? []).length expect(beforeGap).toBe(0) // initial open emitted no gap - // drop the peer arm, then reconcile re-opens it → exactly one gap + // drop the peer arm, then reconcile re-opens it peerCtls[0].end() await tick() driver.reconcile() await tick() const gapMatches = res.text().match(/event: amicode\.sync\.gap/g) ?? [] - expect(gapMatches.length).toBe(1) - const gapFrame = res.text().split("\n\n").find((f) => f.includes("amicode.sync.gap"))! - const dataLine = gapFrame.split("\n").find((l) => l.startsWith("data:"))! - const parsed = JSON.parse(dataLine.slice("data:".length).trim()) - expect(parsed.namespaces).toContain("studio") + if (mode === "sequenced") { + expect(peerOpens).toEqual([undefined, studio.idAt(1)]) // resumed from the last delivered id + expect(gapMatches.length).toBe(0) + } else { + expect(peerOpens).toEqual([undefined, undefined]) // nothing to resume from + expect(gapMatches.length).toBe(1) + const gapFrame = res.text().split("\n\n").find((f) => f.includes("amicode.sync.gap"))! + const dataLine = gapFrame.split("\n").find((l) => l.startsWith("data:"))! + const parsed = JSON.parse(dataLine.slice("data:".length).trim()) + expect(parsed.namespaces).toContain("studio") + } localCtl.end() peerCtls.forEach((c) => c.end()) @@ -783,10 +797,12 @@ describe("#1638 — re-open resumes from last-delivered id + composite-mode reco // re-open loses the stretch between the arm's end and its re-subscription with // nothing to announce it. The #1264 guards stay: no synthetic frame, and the // stream is byte-identical to what the local source delivered while connected. + // #1777: the sever is the fallback for a local engine WITHOUT ids (older than + // #1776) — a sequenced one resumes instead (the #1777 suite) — so L1 is legacy. it("FLEET-OF-ONE (zero peers): a re-opened LOCAL arm emits NO gap, stays byte-identical (#1264), and severs the listener (#1773)", async () => { const ownerMap = new SessionOwnerMap() // no owners → fleet-of-one - const L1 = localEvt(1, { from: "macbook", n: 1 }) + const L1 = legacyEvent({ from: "macbook", n: 1 }) const localCtls: Ctl[] = [] @@ -830,7 +846,7 @@ describe("#1638 — re-open resumes from last-delivered id + composite-mode reco res.fireClose() }) - it("fleet-of-one cursor bookkeeping: a never-delivered arm re-opens from the connect cursor; a severed listener's reconnect resumes from the last-delivered id", async () => { + it("fleet-of-one cursor bookkeeping: a never-delivered arm re-opens from the connect cursor; a delivered one resumes from its last-delivered id, and so does a reconnecting listener", async () => { const ownerMap = new SessionOwnerMap() const L1 = localEvt(1, { from: "macbook", n: 1 }) @@ -868,24 +884,27 @@ describe("#1638 — re-open resumes from last-delivered id + composite-mode reco expect(opened[1].lastEventId).toBe(localId(0)) expect(res.destroyed).toBe(false) - // the re-opened arm delivers, then drops → the listener is severed + // #1777: the re-opened arm delivers, then drops → it RESUMES from the last + // delivered id (the engine replays the stretch) — the listener is not severed localCtls[1].push(L1) await tick() localCtls[1].end() await tick() driver.reconcile() await tick() - expect(res.destroyed).toBe(true) - expect(opened).toHaveLength(2) + expect(res.destroyed).toBe(false) + expect(opened).toHaveLength(3) + expect(opened[2].lastEventId).toBe(localId(1)) // last-delivered, not the stale connect id // the client's cursor is the last SSE id it saw on the verbatim stream … const lastSeen = [...res.text().matchAll(/^id: (.*)$/gm)].at(-1)?.[1] expect(lastSeen).toBe(localId(1)) // … and its reconnect resumes the local arm from exactly that id + res.fireClose() const again = mockRes() driver.handle(mockReq(`/event?lastEventID=${lastSeen}`), again) await tick() - expect(opened[2].lastEventId).toBe(localId(1)) // last-delivered, not the stale connect id + expect(opened[3].lastEventId).toBe(localId(1)) localCtls.forEach((c) => c.end()) again.fireClose() @@ -901,14 +920,23 @@ describe("#1638 — re-open resumes from last-delivered id + composite-mode reco // fixtures in both modes (legacy id-less, sequenced). // ══════════════════════════════════════════════════════════════════════════════ describe("#1773 AC2 — fleet-of-one: a re-opened local arm severs the listener with a transport error", () => { + // #1777: the sever is the fallback for a local engine without ids (legacy); a + // sequenced engine's arm RESUMES from its last delivered id instead — the + // replay announces nothing because nothing was lost, and the listener lives. for (const mode of ["legacy", "sequenced"] as const) { for (const why of ["end", "error"] as const) { - it(`${mode} engine, upstream ${why}: no synthetic frame, byte-identical while connected, then an error close named ${why === "end" ? "upstream-end" : "upstream-error"}`, async () => { + const title = + mode === "legacy" + ? `${mode} engine, upstream ${why}: no synthetic frame, byte-identical while connected, then an error close named ${why === "end" ? "upstream-end" : "upstream-error"}` + : `${mode} engine, upstream ${why}: no synthetic frame, and the arm resumes from its last id — no sever` + it(title, async () => { const { census, lines } = testCensus() const localCtls: Ctl[] = [] + const opened: Array = [] const driver = twoListenerDriver(fakeOrigins(), new SessionOwnerMap(), { census, - openUpstream: () => { + openUpstream: (r) => { + opened.push(r.lastEventId) const ctl = controllableSource() localCtls.push(ctl) return ctl.source @@ -930,18 +958,26 @@ describe("#1773 AC2 — fleet-of-one: a re-opened local arm severs the listener localCtls[0].end(why) // the local upstream is lost … await flush() expect(res.destroyed).toBe(false) // … nothing is decided until the re-open - driver.reconcile() // the re-open: sever, never re-open silently + driver.reconcile() // the re-open: resume, or sever — never re-open silently await flush() expect(res.text()).toBe(frames.join("")) // no synthetic frame was injected + const reason = why === "end" ? "upstream-end" : "upstream-error" + expect(lines).toContain(`[fanin] arm-closed plane=observe conn=c1 arm=local reason=${reason}`) + if (mode === "sequenced") { + expect(res.destroyed).toBe(false) + expect(opened).toEqual([undefined, engine.idAt(engine.seq)]) + expect(lines.at(-1)).toBe("[fanin] arm-opened plane=observe conn=c1 arm=local opens=2") + expect(census.snapshot()).toMatchObject({ listeners: 1, accepted: 1, closed: 0 }) + res.fireClose() + return + } expect(res.destroyed).toBe(true) expect(res.destroyError).toBeInstanceOf(Error) expect(res.ended).toBe(false) // an error close, never a clean end expect(localCtls).toHaveLength(1) // the severed listener re-opened nothing // the census names the listener's close by its cause (#1771's set) - const reason = why === "end" ? "upstream-end" : "upstream-error" - expect(lines).toContain(`[fanin] arm-closed plane=observe conn=c1 arm=local reason=${reason}`) expect(lines).toContain(`[fanin] listener-closed plane=observe conn=c1 reason=${reason} listeners=0`) expect(census.snapshot()).toMatchObject({ listeners: 0, accepted: 1, closed: 1, evictions: 0 }) }) @@ -1948,7 +1984,7 @@ describe("#1771 AC2 — close reasons are an enumerated set", () => { }); const res = mockRes(); driver.handle(mockReq("/global/event"), res); - localCtls[0].push(fakeEngine({ epoch: LOCAL_EPOCH }).connected()); + localCtls[0].push(fakeEngine({ legacy: true }).connected()); // #1777: an id-less engine — the sever fallback await flush(); localCtls[0].end("error"); await flush(); @@ -1979,7 +2015,10 @@ describe("#1771 AC3 — each transition logs one line immediately, with its conn res.fireClose(); }); - it("frames are never logged; an arm that drops and re-opens logs closed → gap emitted → opened", async () => { + // #1777: an id-less (legacy) arm's re-open logs its gap; a sequenced arm's + // re-open RESUMES — closed → opened, and no gap line at all. + for (const mode of ["legacy", "sequenced"] as const) + it(`frames are never logged; ${mode === "legacy" ? "an id-less arm that drops and re-opens logs closed → gap emitted → opened" : "a sequenced arm that drops and re-opens logs closed → opened (resumed, no gap)"}`, async () => { const ownerMap = new SessionOwnerMap(); ownerMap.update(STUDIO_OWNED); const { census, lines } = testCensus(); @@ -1992,7 +2031,7 @@ describe("#1771 AC3 — each transition logs one line immediately, with its conn return ctl.source; }, }); - const studio = fakeEngine({ epoch: PEER_EPOCH }); + const studio = mode === "legacy" ? fakeEngine({ legacy: true }) : fakeEngine({ epoch: PEER_EPOCH }); const res = mockRes(); driver.handle(mockReq("/global/event"), res); const before = lines.length; @@ -2006,7 +2045,7 @@ describe("#1771 AC3 — each transition logs one line immediately, with its conn driver.reconcile(); expect(lines.slice(before)).toEqual([ "[fanin] arm-closed plane=observe conn=c1 arm=studio reason=upstream-end", - "[fanin] gap-emitted plane=observe conn=c1 arm=studio cause=reopen", + ...(mode === "legacy" ? ["[fanin] gap-emitted plane=observe conn=c1 arm=studio cause=reopen"] : []), "[fanin] arm-opened plane=observe conn=c1 arm=studio opens=2", ]); peerCtls.forEach((c) => c.end()); @@ -2074,13 +2113,23 @@ describe("#1771 AC5 — the census adds no frames to any stream and changes no d vi.advanceTimersByTime(FANIN_CENSUS_INTERVAL_MS); // a census line mid-stream localCtls[0].end(); // the arm drops … await flush(); - driver.reconcile(); // … and the re-open severs the listener (#1773 — no gap frame in fleet-of-one) + driver.reconcile(); // … and the re-open severs the listener (#1773 — no gap frame in fleet-of-one) or, #1777, resumes it + const second = [engine.heartbeat(), engine.event("session.status", { status: "idle" })]; + if (engine.epoch !== undefined) { + // a sequenced engine: the arm resumed beneath the SAME listener — still + // the engine's frames, byte for byte + expect(res.destroyed).toBe(false); + for (const f of second) localCtls[1].push(f); + await flush(); + expect(res.text()).toBe(first.join("") + second.join("")); + res.fireClose(); + continue; + } expect(res.text()).toBe(first.join("")); expect(res.destroyed).toBe(true); // the client reconnects; the stream resumes on the new listener const again = mockRes(); driver.handle(mockReq("/global/event"), again); - const second = [engine.heartbeat(), engine.event("session.status", { status: "idle" })]; for (const f of second) localCtls[1].push(f); await flush(); expect(again.text()).toBe(second.join("")); @@ -2195,3 +2244,96 @@ describe("#1771 AC6 — a 10 MB engine frame delivered in 64 KB chunks (the #177 } }); }); + +// ══════════════════════════════════════════════════════════════════════════════ +// #1777 — LOSSLESS RESUME. The engine (#1776) sequences and replays its global +// stream; the fan-in now uses it. A re-opened arm resumes from its last delivered +// engine id, so the upstream REPLAYS what the listener missed — no gap, no resync. +// Only what the replay cannot serve is announced (an upstream reset → one gap; +// an old id-less engine → the #1773 gap / sever fallback). The upstream here is +// a REPLAYING engine (test/fixtures/engine_stream): it answers a cursor the way +// the #1776 route does — `server.connected` at the cursor, then the replay — or +// with the `server.cursor.unservable` reset first. +// ══════════════════════════════════════════════════════════════════════════════ + +/** The downstream stream as a client parses it: per frame, its `id:` and JSON data. */ +function sseFrames(text: string): Array<{ id?: string; data?: Record; type?: string }> { + const out: Array<{ id?: string; data?: Record; type?: string }> = []; + for (const block of text.split("\n\n")) { + if (block.trim() === "") continue; + const id = /^id: (.*)$/m.exec(block)?.[1]; + const dataLine = block.split("\n").find((l) => l.startsWith("data:")); + const data = dataLine ? (JSON.parse(dataLine.slice(5).trim()) as Record) : undefined; + const type: string | undefined = data ? (data.payload?.type ?? data.type) : undefined; + out.push({ ...(id !== undefined ? { id } : {}), ...(data ? { data } : {}), ...(type ? { type } : {}) }); + } + return out; +} + +/** A driver whose arms subscribe to in-process replaying engines. */ +function replayDriver( + engines: { local: ReplayingEngine; studio?: ReplayingEngine }, + ownerMap: SessionOwnerMap, + extra: Partial[0]> = {}, +): SseFanInDriver { + return new SseFanInDriver({ + ownerMap, + localMachineId: "macbook", + localEventUrl: () => "http://local.invalid", + peerBaseUrl: (id) => (id === "studio" ? "http://studio.invalid" : undefined), + peerToken: (id) => + id === "studio" + ? { ok: true as const, credential: { baseUrl: "http://studio.invalid", token: "tok-studio" } } + : { ok: false as const, reason: "absent" as const }, + reconcileMs: 999_999, + openUpstream: (r) => (r.namespace === LOCAL_NAMESPACE ? engines.local : engines.studio!).subscribe(r.lastEventId).source, + ...extra, + }); +} + +const partEvent = (n: number) => ["message.part.updated", { part: { id: `prt_${n}`, text: `line ${n}` }, n }] as const; + +describe("#1777 AC1 — a peer arm that drops and re-opens within the replay window resumes: every missed frame, no gap, no resync", () => { + for (const why of ["end", "error"] as const) { + it(`upstream ${why}: the re-open resumes from the last delivered id and the viewer gets every frame exactly once, in order`, async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const local = replayingEngine({ epoch: LOCAL_EPOCH }); + const studio = replayingEngine({ epoch: PEER_EPOCH }); + const { census, lines } = testCensus(); + const driver = replayDriver({ local, studio }, ownerMap, { census }); + const res = mockRes(); + driver.handle(mockReq("/global/event"), res); + await flush(); + + for (const n of [1, 2, 3]) studio.publish(...partEvent(n)); // mid-turn + await flush(); + studio.subscriptions[0].end(why); // the peer link drops … + await flush(); + for (const n of [4, 5, 6]) studio.publish(...partEvent(n)); // … while the turn goes on + driver.reconcile(); // the re-open + await flush(); + for (const n of [7, 8]) studio.publish(...partEvent(n)); + await flush(); + + // the re-open asked for the last delivered id, and the engine could serve it + expect(studio.subscriptions).toHaveLength(2); + expect(studio.subscriptions[1].cursor).toBe(studio.idAt(3)); + expect(studio.subscriptions[1].reset).toBe(false); + + const frames = sseFrames(res.text()); + // every frame the peer published, exactly once, in order + expect(frames.filter((f) => f.type === "message.part.updated").map((f) => f.data!.payload.properties.n)).toEqual([1, 2, 3, 4, 5, 6, 7, 8]); + // no gap signal … + expect(frames.some((f) => f.type === "amicode.sync.gap")).toBe(false); + expect(lines.some((l) => l.includes("gap-emitted"))).toBe(false); + // … and no resync cue: the resumed arm's own handshake is not an event of + // the listener's stream (only the two initial arms' connected frames show) + expect(frames.filter((f) => f.type === "server.connected")).toHaveLength(2); + // the listener stays connected, and its cursor carries the peer's position + expect(res.destroyed).toBe(false); + expect(parseCompositeCursor(frames.at(-1)!.id).get("studio")).toBe(studio.idAt(8)); + res.fireClose(); + }); + } +}); From 0bb0c61da2bf58559f82b25b02d5098f9ce6d6a9 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 20:33:36 -0400 Subject: [PATCH 46/60] fix(fleet): the healthcheck checks every per-peer forward agent the way it checks the hub tunnel (#1775) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit checkPeerForwards enumerates co.harmoniqs.amico-*-forward agents in an injected LaunchAgents dir by label convention: one check per attached loopback peer (from the reader peer-token store via the new listPeerBaseUrls, which never returns a token) — missing agent, stale or missing ServerAliveInterval/CountMax, TCPKeepAlive, ExitOnForwardFailure, or the 127.0.0.1::127.0.0.1: forward each named, with the install.sh fix line — plus one per other forward agent (a hand-installed legacy agent, flagged with the backup-keeping replace fix when it belongs to an attached peer). fleetHealthReport carries them on every role when given a LaunchAgents dir; both extension call sites pass ~/Library/LaunchAgents. install.sh --check is the CLI twin (report only — never calls launchctl); a test pins installer-rendered agents passing the TS check. --- .../src/amicode_service/fleet_peer_store.ts | 17 ++ packages/extension/src/extension.ts | 4 + packages/extension/src/fleet_health.ts | 174 +++++++++++++++++- packages/extension/test/fleet_health.test.ts | 148 ++++++++++++++- .../test/fleet_scripts_projection.test.ts | 84 +++++++++ packages/extension/tools/fleet/install.sh | 31 +++- tools/fleet/install.sh | 31 +++- 7 files changed, 479 insertions(+), 10 deletions(-) diff --git a/packages/extension/src/amicode_service/fleet_peer_store.ts b/packages/extension/src/amicode_service/fleet_peer_store.ts index 3b3d43341..3ecc94617 100644 --- a/packages/extension/src/amicode_service/fleet_peer_store.ts +++ b/packages/extension/src/amicode_service/fleet_peer_store.ts @@ -78,6 +78,23 @@ export function hasPeerToken(machineId: string, deps: PeerStoreDeps = {}): boole return readPeerToken(machineId, deps).ok; } +/** Every target this machine is attached to, with its base_url — NEVER the + * token (#1775: the per-peer SSH forward is keyed on the machine id and its + * local port is the loopback base_url's port). Sorted by machine id; an entry + * without a string base_url is skipped (absent/corrupt store → []). */ +export function listPeerBaseUrls(deps: PeerStoreDeps = {}): Array<{ machineId: string; baseUrl: string }> { + const peers = readKeyedCollection(peerTokenStorePath(deps), COLLECTION); + const out: Array<{ machineId: string; baseUrl: string }> = []; + for (const machineId of Object.keys(peers).sort()) { + const e = peers[machineId]; + if (typeof e !== "object" || e === null || Array.isArray(e)) continue; + const baseUrl = (e as Record).base_url; + if (typeof baseUrl !== "string" || baseUrl.trim() === "") continue; + out.push({ machineId, baseUrl: baseUrl.trim() }); + } + return out; +} + /** Remove the entire reader peer-store file; absent is a no-op. */ export function clearAllPeerTokens(deps: PeerStoreDeps = {}): void { clearKeyedStoreFile(peerTokenStorePath(deps)); diff --git a/packages/extension/src/extension.ts b/packages/extension/src/extension.ts index 01bc9a92c..2e2a664e8 100644 --- a/packages/extension/src/extension.ts +++ b/packages/extension/src/extension.ts @@ -2477,6 +2477,8 @@ export async function activate(ctx: vscode.ExtensionContext): Promise { configuredBinary: vscode.workspace.getConfiguration("amicode").get("opencodeBinary", ""), configuredPort: vscode.workspace.getConfiguration("amicode").get("opencodePort", 0), plistContent, + // #1775: every per-peer SSH forward agent, enumerated by label convention. + launchAgentsDir: path.join(os.homedir(), "Library", "LaunchAgents"), }); for (const c of fleetChecks) { results.push({ name: c.name, ok: c.ok, detail: c.ok ? c.detail : `${c.detail} → ${c.fix ?? ""}`.trim() }); @@ -2878,6 +2880,8 @@ export async function activate(ctx: vscode.ExtensionContext): Promise { configuredBinary: vscode.workspace.getConfiguration("amicode").get("opencodeBinary", ""), configuredPort: vscode.workspace.getConfiguration("amicode").get("opencodePort", 0), plistContent, + // #1775: per-peer forward agents (checkPeerForwards self-skips off darwin). + launchAgentsDir: path.join(os.homedir(), "Library", "LaunchAgents"), }); const failed = checks.filter((c) => !c.ok); if (failed.length === 0) return; diff --git a/packages/extension/src/fleet_health.ts b/packages/extension/src/fleet_health.ts index ca1d8d334..463d07246 100644 --- a/packages/extension/src/fleet_health.ts +++ b/packages/extension/src/fleet_health.ts @@ -20,6 +20,7 @@ import * as fs from "node:fs"; import * as path from "node:path"; import { homedir } from "node:os"; import { readFleetTopology, type FleetTopologyState } from "./fleet_topology"; +import { listPeerBaseUrls } from "./amicode_service/fleet_peer_store"; export const FLEET_GUARD_REL = "tools/fleet/amico-opencode-fleet-guard"; export const FLEET_GUARD_INSTALL = path.join(homedir(), ".local", "bin", "amico-opencode-fleet-guard"); @@ -162,6 +163,155 @@ export function checkFleetTunnel( return { name: "Fleet tunnel", ok: true, detail: `ServerAlive 15/2 + TCPKeepAlive, ${port} forward` }; } +// ── #1775: per-peer SSH forward agents ───────────────────────────────────────── +// The installer generates one launchd agent per attached peer (tools/fleet/ +// co.harmoniqs.amico-peer-forward.plist); this check enumerates every forward +// agent in the LaunchAgents dir BY LABEL CONVENTION and holds each to the hub +// tunnel's keepalive standard. The attached peers come from the reader +// peer-token store: a loopback base_url's port IS the forward's local end. + +/** The managed per-peer forward label — deterministic from the peer's machine id. */ +export function peerForwardLabel(machineId: string): string { + return `co.harmoniqs.amico-peer-${machineId}-forward`; +} + +/** Any `co.harmoniqs.amico--forward` agent file — managed or hand-installed. */ +const FORWARD_AGENT_FILE = /^co\.harmoniqs\.amico-(.+)-forward\.plist$/; +/** The managed shape inside it: `co.harmoniqs.amico-peer--forward`. */ +const MANAGED_FORWARD_FILE = /^co\.harmoniqs\.amico-peer-(.+)-forward\.plist$/; + +/** A peer's ssh-forwarded local port: only a loopback http base URL with an + * explicit port rides an SSH forward (https = tailscale/direct reach → none). */ +function loopbackForwardPort(baseUrl: string): number | null { + let u: URL; + try { + u = new URL(baseUrl); + } catch { + return null; + } + if (u.protocol !== "http:" || (u.hostname !== "127.0.0.1" && u.hostname !== "localhost")) return null; + const port = Number(u.port); + return Number.isInteger(port) && port >= 1 && port <= 65535 ? port : null; +} + +/** The keepalive standard (fleet_transport sshForwardArgs), value-exact. */ +function forwardKeepaliveIssues(content: string): string[] { + const issues: string[] = []; + const interval = /ServerAliveInterval=(\d+)/.exec(content)?.[1]; + if (interval === undefined) issues.push("ServerAliveInterval 15 missing"); + else if (interval !== "15") issues.push(`ServerAliveInterval is ${interval} (stale, should be 15)`); + const count = /ServerAliveCountMax=(\d+)/.exec(content)?.[1]; + if (count === undefined) issues.push("ServerAliveCountMax 2 missing"); + else if (count !== "2") issues.push(`ServerAliveCountMax is ${count} (stale, should be 2)`); + if (!content.includes("TCPKeepAlive=yes")) issues.push("TCPKeepAlive yes missing"); + if (!content.includes("ExitOnForwardFailure=yes")) issues.push("ExitOnForwardFailure yes missing"); + return issues; +} + +const PEER_FORWARD_REGENERATE = "bash tools/fleet/install.sh (Fleet — Repair regenerates it: 15/2 + TCPKeepAlive)"; +const PEER_FORWARD_REPLACE = "bash tools/fleet/install.sh (Fleet — Repair replaces it with the managed agent and keeps a backup)"; + +/** Per-peer forward check: one FleetCheck per attached loopback peer (its + * managed agent present, 15/2 + TCPKeepAlive + ExitOnForwardFailure, the + * `127.0.0.1::127.0.0.1:` forward) plus one per OTHER + * forward agent found by label convention (a hand-installed legacy agent, or + * a managed agent for a peer no longer attached). darwin only; [] when there + * is nothing to report. */ +export function checkPeerForwards(opts: { + launchAgentsDir: string; + /** The reader peer-token store (default: fleet_peer_store's resolution). */ + peerStoreFile?: string; + platform?: string; + topology?: FleetTopologyState; + listDir?: (dir: string) => string[]; + read?: (p: string) => string; +}): FleetCheck[] { + if ((opts.platform ?? process.platform) !== "darwin") return []; + const read = opts.read ?? ((p: string) => fs.readFileSync(p, "utf8")); + const listDir = opts.listDir ?? ((d: string) => fs.readdirSync(d)); + let files: string[] = []; + try { + files = listDir(opts.launchAgentsDir).filter((f) => FORWARD_AGENT_FILE.test(f)).sort(); + } catch { + files = []; + } + const attached = listPeerBaseUrls({ storeFile: opts.peerStoreFile }) + .map((p) => ({ machineId: p.machineId, port: loopbackForwardPort(p.baseUrl) })) + .filter((p): p is { machineId: string; port: number } => p.port !== null); + const remotePort = fleetPort(opts.topology ?? readFleetTopology()); + + const checks: FleetCheck[] = []; + const claimed = new Set(); + for (const peer of attached) { + const label = peerForwardLabel(peer.machineId); + const file = `${label}.plist`; + claimed.add(file); + const name = `Fleet peer forward (${peer.machineId})`; + const portForward = `127.0.0.1:${peer.port}:127.0.0.1:${remotePort}`; + const content = files.includes(file) ? readOrNull(path.join(opts.launchAgentsDir, file), read) : null; + if (content == null) { + checks.push({ + name, + ok: false, + detail: `launchd forward agent missing (${path.join(opts.launchAgentsDir, file)})`, + fix: "bash tools/fleet/install.sh (Fleet — Repair generates it)", + }); + continue; + } + const issues = forwardKeepaliveIssues(content); + if (!content.includes(portForward)) issues.push(`LocalForward ${portForward} missing`); + checks.push( + issues.length > 0 + ? { name, ok: false, detail: issues.join("; "), fix: PEER_FORWARD_REGENERATE } + : { name, ok: true, detail: `ServerAlive 15/2 + TCPKeepAlive, ${portForward} forward` }, + ); + } + + for (const file of files) { + if (claimed.has(file)) continue; + const content = readOrNull(path.join(opts.launchAgentsDir, file), read) ?? ""; + const issues = forwardKeepaliveIssues(content); + const managed = MANAGED_FORWARD_FILE.exec(file); + if (managed !== null) { + // A managed agent whose peer is no longer in the token store. + const name = `Fleet peer forward (${managed[1]})`; + const note = "no attached peer in the token store"; + checks.push( + issues.length > 0 + ? { name, ok: false, detail: `${issues.join("; ")} (${note})`, fix: PEER_FORWARD_REGENERATE } + : { name, ok: true, detail: `ServerAlive 15/2 + TCPKeepAlive (${note})` }, + ); + continue; + } + // A hand-installed `co.harmoniqs.amico--forward`. It belongs to an + // attached peer when it binds that peer's local port or names its machine + // id as the ssh destination — the installer replaces exactly those. + const label = file.replace(/\.plist$/, ""); + const name = `Fleet peer forward (hand-installed ${label})`; + const owner = attached.find( + (p) => new RegExp(`((127\\.0\\.0\\.1|localhost):)?${p.port}:`).test(content) || content.includes(`${p.machineId}`), + ); + if (owner !== undefined) { + checks.push({ + name, + ok: false, + detail: `hand-installed forward for peer ${owner.machineId}, not managed by the installer${issues.length > 0 ? ` — ${issues.join("; ")}` : ""}`, + fix: PEER_FORWARD_REPLACE, + }); + } else if (issues.length > 0) { + checks.push({ + name, + ok: false, + detail: `${issues.join("; ")} (no attached peer matches it, so the installer leaves it alone)`, + fix: `retune it to 15/2 + TCPKeepAlive, or attach its peer and run bash tools/fleet/install.sh (Fleet — Repair then replaces it and keeps a backup)`, + }); + } else { + checks.push({ name, ok: true, detail: "ServerAlive 15/2 + TCPKeepAlive (hand-installed; no attached peer matches it)" }); + } + } + return checks; +} + /** Fleet role check — surfaces the current mode from the projection topology. * Every state renders honestly (#1106): ok → the role + canonical target + * the D1 freshness verdict when present; absent → the base-default @@ -209,10 +359,13 @@ export function checkFleetRole( return { name: "Fleet role", ok: false, detail: `unknown role: ${role}`, fix: "refresh the projection: `amico fleet status --projection`" }; } -/** Aggregate helper — returns all fleet checks (role + guard + settings + tunnel). +/** Aggregate helper — returns all fleet checks (role + guard + settings + tunnel + * + per-peer forwards). * When role is standalone (or the projection is absent — the same base * default), guard/settings/tunnel checks are skipped (not relevant); a - * broken projection surfaces ONLY the role check's rendered fail. */ + * broken projection surfaces ONLY the role check's rendered fail. The + * per-peer forward checks (#1775) run on EVERY role — a peer may be + * standalone, server, or client — whenever a LaunchAgents dir is supplied. */ export function fleetHealthReport(args: { repoGuardPath: string; installedGuardPath?: string; @@ -224,9 +377,24 @@ export function fleetHealthReport(args: { platform?: string; /** #1106: the projection topology state (injectable; default: readFleetTopology()). */ topology?: FleetTopologyState; + /** #1775: the LaunchAgents dir the per-peer forward agents live in + * (production: ~/Library/LaunchAgents). Absent → no peer forward checks. */ + launchAgentsDir?: string; + /** #1775: the reader peer-token store (default: fleet_peer_store's resolution). */ + peerStoreFile?: string; }): FleetCheck[] { const topology = args.topology ?? readFleetTopology(); const role = topology.kind === "ok" ? topology.role : "standalone"; + // The agents are read from the injected dir itself — `read` above is the + // guard-file injection and never stands in for a plist read. + const peerForwards = (): FleetCheck[] => + args.launchAgentsDir === undefined + ? [] + : checkPeerForwards({ launchAgentsDir: args.launchAgentsDir, peerStoreFile: args.peerStoreFile, platform: args.platform, topology }); + + if (topology.kind === "broken") { + return [checkFleetRole({ topology, platform: args.platform })]; + } // Guard/settings/tunnel checks are only relevant for CLIENT machines — they // verify the engine-blocking guard, the binary redirect, and the SSH tunnel @@ -237,6 +405,7 @@ export function fleetHealthReport(args: { if (role !== "client" || topology.kind !== "ok") { return [ checkFleetRole({ topology, platform: args.platform }), + ...peerForwards(), ]; } @@ -245,5 +414,6 @@ export function fleetHealthReport(args: { checkFleetGuard(args.repoGuardPath, args.installedGuardPath, { read: args.read, isExecutable: args.isExecutable, platform: args.platform }), checkFleetSettings(args.configuredBinary, args.configuredPort, { platform: args.platform, topology }), checkFleetTunnel(args.plistContent, { platform: args.platform, topology }), + ...peerForwards(), ]; } diff --git a/packages/extension/test/fleet_health.test.ts b/packages/extension/test/fleet_health.test.ts index 4a0f6d8c8..f87320847 100644 --- a/packages/extension/test/fleet_health.test.ts +++ b/packages/extension/test/fleet_health.test.ts @@ -1,5 +1,8 @@ -import { describe, it, expect } from "vitest"; -import { checkFleetGuard, checkFleetSettings, checkFleetTunnel, checkFleetRole, fleetHealthReport, FLEET_GUARD_INSTALL } from "../src/fleet_health"; +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { checkFleetGuard, checkFleetSettings, checkFleetTunnel, checkFleetRole, checkPeerForwards, fleetHealthReport, FLEET_GUARD_INSTALL } from "../src/fleet_health"; import type { FleetTopologyState } from "../src/fleet_topology"; const REPO = "/repo/tools/fleet/amico-opencode-fleet-guard"; @@ -313,3 +316,144 @@ describe("fleet_health on linux — the guard backstop is wired cross-platform ( expect(r[0].detail).toMatch(/standalone/); }); }); + +// ── #1775: every per-peer SSH forward agent is checked the way the hub tunnel is ── +// One launchd agent per attached peer (co.harmoniqs.amico-peer-- +// forward), enumerated by label convention from an INJECTED LaunchAgents dir; +// the attached peers come from the reader peer-token store (loopback base_url +// = the forward's local end). Missing or stale keepalive values fail with the +// install.sh fix line — the tunnel check's exact shape. +describe("#1775 peer forward agents — every per-peer forward is checked like the hub tunnel", () => { + const TEMPLATE = readFileSync(join(__dirname, "..", "..", "..", "tools", "fleet", "co.harmoniqs.amico-peer-forward.plist"), "utf8"); + let root: string; + let launchAgentsDir: string; + let peerStoreFile: string; + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "fleet-health-peers-")); + launchAgentsDir = join(root, "LaunchAgents"); + mkdirSync(launchAgentsDir, { recursive: true }); + peerStoreFile = join(root, "fleet-peer-tokens-reader.json"); + }); + afterEach(() => rmSync(root, { recursive: true, force: true })); + + /** The agent exactly as the installer renders it from the repo template. */ + const rendered = (id: string, localPort: number, remotePort = 4096, alias = id): string => + TEMPLATE.replace(/PEER_MACHINE_ID/g, id) + .replace(/PEER_LOCAL_PORT/g, String(localPort)) + .replace(/PEER_REMOTE_PORT/g, String(remotePort)) + .replace(/PEER_SSH_ALIAS/g, alias); + const placeAgent = (file: string, content: string): void => writeFileSync(join(launchAgentsDir, file), content); + const managedFile = (id: string): string => `co.harmoniqs.amico-peer-${id}-forward.plist`; + const attach = (peers: Record): void => + writeFileSync(peerStoreFile, JSON.stringify({ store_version: 1, peers: Object.fromEntries(Object.entries(peers).map(([id, u]) => [id, { base_url: u, token: "t" }])) })); + const run = (topology: FleetTopologyState = okServer) => + checkPeerForwards({ launchAgentsDir, peerStoreFile, platform: "darwin", topology }); + + it("one check per attached peer: an installer-rendered agent passes with the tunnel's ok shape (15/2 + TCPKeepAlive, the forward)", () => { + attach({ "jjs-mac-studio": "http://127.0.0.1:45096" }); + placeAgent(managedFile("jjs-mac-studio"), rendered("jjs-mac-studio", 45096)); + const checks = run(); + expect(checks).toHaveLength(1); + expect(checks[0].name).toBe("Fleet peer forward (jjs-mac-studio)"); + expect(checks[0].ok).toBe(true); + expect(checks[0].detail).toMatch(/ServerAlive 15\/2 \+ TCPKeepAlive/); + expect(checks[0].detail).toContain("127.0.0.1:45096:127.0.0.1:4096"); + }); + + it("an attached peer with NO forward agent fails, naming the agent, with the install.sh fix line", () => { + attach({ "jjs-mac-studio": "http://127.0.0.1:45096" }); + const [c] = run(); + expect(c.ok).toBe(false); + expect(c.detail).toMatch(/missing/); + expect(c.detail).toContain(managedFile("jjs-mac-studio")); + expect(c.fix).toMatch(/bash tools\/fleet\/install\.sh/); + }); + + it("several agents, stale values: each is reported; each stale/missing keepalive value is named; the fix line is the tunnel's", () => { + attach({ + good: "http://127.0.0.1:45001", + stale: "http://127.0.0.1:45002", + "no-tcp": "http://127.0.0.1:45003", + "wrong-port": "http://127.0.0.1:45004", + }); + placeAgent(managedFile("good"), rendered("good", 45001)); + placeAgent(managedFile("stale"), rendered("stale", 45002).replace("ServerAliveInterval=15", "ServerAliveInterval=30").replace("ServerAliveCountMax=2", "ServerAliveCountMax=3")); + placeAgent(managedFile("no-tcp"), rendered("no-tcp", 45003).replace("-o\n\t\tTCPKeepAlive=yes\n", "").replace("-o\n\t\tExitOnForwardFailure=yes\n", "")); + placeAgent(managedFile("wrong-port"), rendered("wrong-port", 45999)); + const byName = Object.fromEntries(run().map((c) => [c.name, c])); + expect(Object.keys(byName).sort()).toEqual([ + "Fleet peer forward (good)", "Fleet peer forward (no-tcp)", "Fleet peer forward (stale)", "Fleet peer forward (wrong-port)", + ]); + expect(byName["Fleet peer forward (good)"].ok).toBe(true); + + const stale = byName["Fleet peer forward (stale)"]; + expect(stale.ok).toBe(false); + expect(stale.detail).toMatch(/ServerAliveInterval is 30 \(stale, should be 15\)/); + expect(stale.detail).toMatch(/ServerAliveCountMax is 3 \(stale, should be 2\)/); + expect(stale.fix).toBe("bash tools/fleet/install.sh (Fleet — Repair regenerates it: 15/2 + TCPKeepAlive)"); + + const noTcp = byName["Fleet peer forward (no-tcp)"]; + expect(noTcp.ok).toBe(false); + expect(noTcp.detail).toMatch(/TCPKeepAlive yes missing/); + expect(noTcp.detail).toMatch(/ExitOnForwardFailure yes missing/); + + const wrongPort = byName["Fleet peer forward (wrong-port)"]; + expect(wrongPort.ok).toBe(false); + expect(wrongPort.detail).toMatch(/LocalForward 127\.0\.0\.1:45004:127\.0\.0\.1:4096 missing/); + }); + + it("a hand-installed legacy forward is enumerated by label convention and its stale 30/3 flagged — the fix names the backup-keeping replace", () => { + attach({ "jjs-mac-studio": "http://127.0.0.1:45096" }); + // The real hand-installed shape: 30/3, no TCPKeepAlive, unbound -L. + placeAgent("co.harmoniqs.amico-studio-forward.plist", `Labelco.harmoniqs.amico-studio-forward +ProgramArguments/usr/bin/ssh-N-L45096:127.0.0.1:4096 +-oExitOnForwardFailure=yes-oServerAliveInterval=30 +-oServerAliveCountMax=3jjs-mac-studio`); + placeAgent("co.harmoniqs.amico-tunnel.plist", "not a forward agent — the hub tunnel has its own check"); + const byName = Object.fromEntries(run().map((c) => [c.name, c])); + expect(Object.keys(byName).sort()).toEqual([ + "Fleet peer forward (hand-installed co.harmoniqs.amico-studio-forward)", + "Fleet peer forward (jjs-mac-studio)", + ]); + const legacy = byName["Fleet peer forward (hand-installed co.harmoniqs.amico-studio-forward)"]; + expect(legacy.ok).toBe(false); + expect(legacy.detail).toMatch(/ServerAliveInterval is 30 \(stale, should be 15\)/); + expect(legacy.detail).toMatch(/ServerAliveCountMax is 3 \(stale, should be 2\)/); + expect(legacy.detail).toMatch(/TCPKeepAlive yes missing/); + expect(legacy.fix).toMatch(/bash tools\/fleet\/install\.sh .*replaces it .*backup/); + expect(byName["Fleet peer forward (jjs-mac-studio)"].ok).toBe(false); // the managed agent is not there yet + }); + + it("a peer reached over https (tailscale/direct) needs no forward — no check; no attached peers and no agents → no checks", () => { + attach({ far: "https://far.tailnet-1234.ts.net" }); + expect(run()).toEqual([]); + }); + + it("skips off darwin — launchd agents only (the supervised linux/WSL link is #1260)", () => { + attach({ "jjs-mac-studio": "http://127.0.0.1:45096" }); + expect(checkPeerForwards({ launchAgentsDir, peerStoreFile, platform: "linux", topology: okServer })).toEqual([]); + }); + + it("the aggregate report carries the peer forward checks on EVERY role (a peer may be server, standalone, or client); a broken projection still surfaces only the role fail", () => { + attach({ "jjs-mac-studio": "http://127.0.0.1:45096" }); + const base = { + repoGuardPath: REPO, + configuredBinary: INSTALLED, + configuredPort: 4096, + plistContent: `ServerAliveInterval=15 ServerAliveCountMax=2 TCPKeepAlive=yes 127.0.0.1:4096:127.0.0.1:4096`, + read: () => guardContent, + isExecutable: () => true, + platform: "darwin", + launchAgentsDir, + peerStoreFile, + }; + for (const [topology, n] of [[okServer, 2], [okStandalone, 2], [absent, 2], [okClient, 5]] as const) { + const r = fleetHealthReport({ ...base, topology }); + expect(r, `role ${topology.kind}`).toHaveLength(n); + const peer = r.find((c) => c.name === "Fleet peer forward (jjs-mac-studio)"); + expect(peer?.ok).toBe(false); + expect(peer?.fix).toMatch(/install\.sh/); + } + expect(fleetHealthReport({ ...base, topology: broken })).toHaveLength(1); + }); +}); diff --git a/packages/extension/test/fleet_scripts_projection.test.ts b/packages/extension/test/fleet_scripts_projection.test.ts index 6e48c6cb6..c7871b244 100644 --- a/packages/extension/test/fleet_scripts_projection.test.ts +++ b/packages/extension/test/fleet_scripts_projection.test.ts @@ -16,6 +16,7 @@ import { spawnSync } from "node:child_process"; import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync, existsSync, readFileSync, readdirSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { checkPeerForwards } from "../src/fleet_health"; const REPO = join(__dirname, "..", "..", ".."); const GUARD = join(REPO, "tools", "fleet", "amico-opencode-fleet-guard"); @@ -763,3 +764,86 @@ describe("#1775 a hand-installed forward for the same peer is replaced — a bac expect(existsSync(backupDir())).toBe(false); }); }); + +// ── #1775: `install.sh --check` is the CLI twin of the peer-forward healthcheck ── +describe("#1775 --check holds every peer forward agent to the standard (the healthcheck's CLI twin)", () => { + const installEnv = (): { path: string } => ({ + path: `${join(tmp, "fakebin")}:${process.env.PATH ?? ""}`, + }); + const launchAgents = (): string => join(tmp, "Library", "LaunchAgents"); + const attachStudio = (): void => { + fakeAmico({ code: 0, stdout: verbJson("client", { host: "hq", port: 4096, sshAlias: "hq" }) }); + fakeUname("Darwin"); + writePeerStore({ "jjs-mac-studio": "http://127.0.0.1:45096" }); + }; + const listing = (): string => readdirSync(launchAgents()).map((f) => `${f}:${readFileSync(join(launchAgents(), f), "utf8").length}`).sort().join("\n"); + + it("after an install, --check passes end to end and names the peer forward; --check itself never calls launchctl and writes nothing", () => { + attachStudio(); + expect(runScript(INSTALL, [], installEnv()).code).toBe(0); + rmSync(join(tmp, "launchctl.argv"), { force: true }); + const before = listing(); + const r = runScript(INSTALL, ["--check"], installEnv()); + expect(r.code, r.out).toBe(0); + expect(r.out).toMatch(/ok peer forward co\.harmoniqs\.amico-peer-jjs-mac-studio-forward \(15\/2 \+ TCPKeepAlive, 127\.0\.0\.1:45096 → jjs-mac-studio:4096\)/); + expect(r.out).toMatch(/fleet check: all ok/); + expect(launchctlCalls()).toEqual([]); + expect(backstopCalls()).toEqual([]); + expect(listing()).toBe(before); + }); + + it("an attached peer with no forward agent fails --check with the fix line", () => { + attachStudio(); + expect(runScript(INSTALL, [], installEnv()).code).toBe(0); + rmSync(join(launchAgents(), "co.harmoniqs.amico-peer-jjs-mac-studio-forward.plist")); + const r = runScript(INSTALL, ["--check"], installEnv()); + expect(r.code).toBe(1); + expect(r.out).toMatch(/FAIL peer forward co\.harmoniqs\.amico-peer-jjs-mac-studio-forward missing at .*\(run: bash tools\/fleet\/install\.sh\)/); + }); + + it("stale keepalive values fail --check, each value named (the tunnel check's lines)", () => { + attachStudio(); + expect(runScript(INSTALL, [], installEnv()).code).toBe(0); + const agent = join(launchAgents(), "co.harmoniqs.amico-peer-jjs-mac-studio-forward.plist"); + writeFileSync(agent, readFileSync(agent, "utf8").replace("ServerAliveInterval=15", "ServerAliveInterval=30").replace("ServerAliveCountMax=2", "ServerAliveCountMax=3")); + const r = runScript(INSTALL, ["--check"], installEnv()); + expect(r.code).toBe(1); + expect(r.out).toMatch(/FAIL peer forward co\.harmoniqs\.amico-peer-jjs-mac-studio-forward ServerAliveInterval 15 missing/); + expect(r.out).toMatch(/FAIL peer forward co\.harmoniqs\.amico-peer-jjs-mac-studio-forward ServerAliveCountMax 2 missing/); + expect(r.out).not.toMatch(/fleet check: all ok/); + }); + + it("a hand-installed forward for an attached peer fails --check — Fleet — Repair replaces it, keeping a backup", () => { + attachStudio(); + expect(runScript(INSTALL, [], installEnv()).code).toBe(0); + writeFileSync(join(launchAgents(), "co.harmoniqs.amico-studio-forward.plist"), legacyForwardPlist("studio", 45096, "jjs-mac-studio")); + rmSync(join(tmp, "launchctl.argv"), { force: true }); + const r = runScript(INSTALL, ["--check"], installEnv()); + expect(r.code).toBe(1); + expect(r.out).toMatch(/FAIL hand-installed peer forward .*co\.harmoniqs\.amico-studio-forward\.plist for jjs-mac-studio .*replaces it.*backup/); + // a check only reports: the legacy agent is neither unloaded nor moved + expect(launchctlCalls()).toEqual([]); + expect(existsSync(join(launchAgents(), "co.harmoniqs.amico-studio-forward.plist"))).toBe(true); + }); + + it("the installer and the healthcheck agree: every agent the installer renders passes checkPeerForwards", () => { + fakeAmico({ code: 0, stdout: verbJson("server", { host: "hq", port: 4096 }) }); + fakeUname("Darwin"); + writePeerStore({ "jjs-mac-studio": "http://127.0.0.1:45096", "lab-box": "http://127.0.0.1:45097" }); + writeRoster([{ machine_id: "lab-box", sshAlias: "lab" }]); + expect(runScript(INSTALL, [], installEnv()).code).toBe(0); + const checks = checkPeerForwards({ + launchAgentsDir: launchAgents(), + peerStoreFile: join(tmp, ...PEER_STORE_REL), + platform: "darwin", + topology: { + kind: "ok", role: "server", canonical: { host: "hq", port: 4096 }, mode: "fleet", posture: "ok", + freshness: {}, provenanceSource: "fleet.json", projection: { schema_version: 1, contract_version: 1, sections: {} }, + }, + }); + expect(checks.map((c) => [c.name, c.ok])).toEqual([ + ["Fleet peer forward (jjs-mac-studio)", true], + ["Fleet peer forward (lab-box)", true], + ]); + }); +}); diff --git a/packages/extension/tools/fleet/install.sh b/packages/extension/tools/fleet/install.sh index 1afc2f81e..e47754ed4 100755 --- a/packages/extension/tools/fleet/install.sh +++ b/packages/extension/tools/fleet/install.sh @@ -177,11 +177,10 @@ replace_legacy_forwards() { # peer_forwards() { if [[ "$(uname -s)" != "Darwin" ]]; then return 0; fi - if [[ $CHECK -eq 1 ]]; then return 0; fi - local targets kind peer_id lport alias label dst tmpf + local targets kind peer_id lport alias label dst tmpf f peer_fail=0 targets="$(peer_forward_targets)" || die "could not read the peer token store at $PEER_STORE (node failed)" if [[ -z "$targets" ]]; then return 0; fi - if [[ ! -f "$PEER_PLIST_SRC" ]]; then + if [[ $CHECK -eq 0 && ! -f "$PEER_PLIST_SRC" ]]; then say "note: no peer-forward template at $PEER_PLIST_SRC — skipping peer forwards (git pull?)" return 0 fi @@ -189,6 +188,31 @@ peer_forwards() { if [[ "$kind" == "note" ]]; then say "note: $peer_id"; continue; fi label="co.harmoniqs.amico-peer-${peer_id}-forward" dst="$LAUNCH_AGENTS_DIR/${label}.plist" + if [[ $CHECK -eq 1 ]]; then + # The healthcheck's CLI twin (fleet_health.ts checkPeerForwards) — report only. + for f in "$LAUNCH_AGENTS_DIR"/co.harmoniqs.amico-*-forward.plist; do + if [[ -f "$f" ]] && is_legacy_forward_for "$f" "$lport" "$alias" "$peer_id"; then + echo "[fleet] FAIL hand-installed peer forward $f for $peer_id — not managed by the installer (run: bash tools/fleet/install.sh — Fleet — Repair replaces it, keeping a backup)" + peer_fail=1 + fi + done + if [[ ! -f "$dst" ]]; then + echo "[fleet] FAIL peer forward $label missing at $dst (run: bash tools/fleet/install.sh)" + peer_fail=1 + continue + fi + local before=$peer_fail + grep -q "ServerAliveInterval=15" "$dst" || { echo "[fleet] FAIL peer forward $label ServerAliveInterval 15 missing"; peer_fail=1; } + grep -q "ServerAliveCountMax=2" "$dst" || { echo "[fleet] FAIL peer forward $label ServerAliveCountMax 2 missing"; peer_fail=1; } + grep -q "TCPKeepAlive=yes" "$dst" || { echo "[fleet] FAIL peer forward $label TCPKeepAlive yes missing"; peer_fail=1; } + grep -q "ExitOnForwardFailure=yes" "$dst" || { echo "[fleet] FAIL peer forward $label ExitOnForwardFailure yes missing"; peer_fail=1; } + grep -qF "127.0.0.1:${lport}:127.0.0.1:${FLEET_PORT}" "$dst" || { echo "[fleet] FAIL peer forward $label LocalForward 127.0.0.1:${lport}:127.0.0.1:${FLEET_PORT} missing"; peer_fail=1; } + grep -qF "${alias}" "$dst" || { echo "[fleet] FAIL peer forward $label destination is not ${alias}"; peer_fail=1; } + if [[ $peer_fail -eq $before ]]; then + say "ok peer forward $label (15/2 + TCPKeepAlive, 127.0.0.1:${lport} → ${alias}:${FLEET_PORT})" + fi + continue + fi mkdir -p "$LAUNCH_AGENTS_DIR" tmpf="$(mktemp)" sed -e "s/PEER_MACHINE_ID/${peer_id}/g" -e "s/PEER_LOCAL_PORT/${lport}/g" -e "s/PEER_REMOTE_PORT/${FLEET_PORT}/g" -e "s/PEER_SSH_ALIAS/${alias}/g" "$PEER_PLIST_SRC" > "$tmpf" @@ -203,6 +227,7 @@ peer_forwards() { fi rm -f "$tmpf" done <<< "$targets" + if [[ $peer_fail -eq 1 ]]; then exit 1; fi } # Standalone: no guard/tunnel/hub — but a standalone PEER still gets its diff --git a/tools/fleet/install.sh b/tools/fleet/install.sh index 1afc2f81e..e47754ed4 100755 --- a/tools/fleet/install.sh +++ b/tools/fleet/install.sh @@ -177,11 +177,10 @@ replace_legacy_forwards() { # peer_forwards() { if [[ "$(uname -s)" != "Darwin" ]]; then return 0; fi - if [[ $CHECK -eq 1 ]]; then return 0; fi - local targets kind peer_id lport alias label dst tmpf + local targets kind peer_id lport alias label dst tmpf f peer_fail=0 targets="$(peer_forward_targets)" || die "could not read the peer token store at $PEER_STORE (node failed)" if [[ -z "$targets" ]]; then return 0; fi - if [[ ! -f "$PEER_PLIST_SRC" ]]; then + if [[ $CHECK -eq 0 && ! -f "$PEER_PLIST_SRC" ]]; then say "note: no peer-forward template at $PEER_PLIST_SRC — skipping peer forwards (git pull?)" return 0 fi @@ -189,6 +188,31 @@ peer_forwards() { if [[ "$kind" == "note" ]]; then say "note: $peer_id"; continue; fi label="co.harmoniqs.amico-peer-${peer_id}-forward" dst="$LAUNCH_AGENTS_DIR/${label}.plist" + if [[ $CHECK -eq 1 ]]; then + # The healthcheck's CLI twin (fleet_health.ts checkPeerForwards) — report only. + for f in "$LAUNCH_AGENTS_DIR"/co.harmoniqs.amico-*-forward.plist; do + if [[ -f "$f" ]] && is_legacy_forward_for "$f" "$lport" "$alias" "$peer_id"; then + echo "[fleet] FAIL hand-installed peer forward $f for $peer_id — not managed by the installer (run: bash tools/fleet/install.sh — Fleet — Repair replaces it, keeping a backup)" + peer_fail=1 + fi + done + if [[ ! -f "$dst" ]]; then + echo "[fleet] FAIL peer forward $label missing at $dst (run: bash tools/fleet/install.sh)" + peer_fail=1 + continue + fi + local before=$peer_fail + grep -q "ServerAliveInterval=15" "$dst" || { echo "[fleet] FAIL peer forward $label ServerAliveInterval 15 missing"; peer_fail=1; } + grep -q "ServerAliveCountMax=2" "$dst" || { echo "[fleet] FAIL peer forward $label ServerAliveCountMax 2 missing"; peer_fail=1; } + grep -q "TCPKeepAlive=yes" "$dst" || { echo "[fleet] FAIL peer forward $label TCPKeepAlive yes missing"; peer_fail=1; } + grep -q "ExitOnForwardFailure=yes" "$dst" || { echo "[fleet] FAIL peer forward $label ExitOnForwardFailure yes missing"; peer_fail=1; } + grep -qF "127.0.0.1:${lport}:127.0.0.1:${FLEET_PORT}" "$dst" || { echo "[fleet] FAIL peer forward $label LocalForward 127.0.0.1:${lport}:127.0.0.1:${FLEET_PORT} missing"; peer_fail=1; } + grep -qF "${alias}" "$dst" || { echo "[fleet] FAIL peer forward $label destination is not ${alias}"; peer_fail=1; } + if [[ $peer_fail -eq $before ]]; then + say "ok peer forward $label (15/2 + TCPKeepAlive, 127.0.0.1:${lport} → ${alias}:${FLEET_PORT})" + fi + continue + fi mkdir -p "$LAUNCH_AGENTS_DIR" tmpf="$(mktemp)" sed -e "s/PEER_MACHINE_ID/${peer_id}/g" -e "s/PEER_LOCAL_PORT/${lport}/g" -e "s/PEER_REMOTE_PORT/${FLEET_PORT}/g" -e "s/PEER_SSH_ALIAS/${alias}/g" "$PEER_PLIST_SRC" > "$tmpf" @@ -203,6 +227,7 @@ peer_forwards() { fi rm -f "$tmpf" done <<< "$targets" + if [[ $peer_fail -eq 1 ]]; then exit 1; fi } # Standalone: no guard/tunnel/hub — but a standalone PEER still gets its From 8d67baa947e887701727fe6c98588908edf55722 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 20:35:28 -0400 Subject: [PATCH 47/60] fix(fleet): the installer retires the session-wide multiplex-flag agent and unsets the leaked variable (#1775) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit co.harmoniqs.amico-fleet-flag ran `launchctl setenv AMICO_FLEET_MULTIPLEX 1` at login, leaking the flag into every GUI process although it already defaults on (#1448). On every role (darwin), install mode unloads and removes the agent and unsets the session variable when it carries the agent's value — a deliberate 0 opt-out is left alone; a re-run is a no-op. --check fails on a present flag agent with the install.sh fix line. --- .../test/fleet_scripts_projection.test.ts | 93 +++++++++++++++++++ packages/extension/tools/fleet/install.sh | 33 ++++++- tools/fleet/install.sh | 33 ++++++- 3 files changed, 157 insertions(+), 2 deletions(-) diff --git a/packages/extension/test/fleet_scripts_projection.test.ts b/packages/extension/test/fleet_scripts_projection.test.ts index c7871b244..0c661daec 100644 --- a/packages/extension/test/fleet_scripts_projection.test.ts +++ b/packages/extension/test/fleet_scripts_projection.test.ts @@ -17,6 +17,7 @@ import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync, existsSync, r import { tmpdir } from "node:os"; import { join } from "node:path"; import { checkPeerForwards } from "../src/fleet_health"; +import { FLEET_MULTIPLEX_FLAG, fleetMultiplexEnabled } from "../src/amicode_service/server"; const REPO = join(__dirname, "..", "..", ".."); const GUARD = join(REPO, "tools", "fleet", "amico-opencode-fleet-guard"); @@ -847,3 +848,95 @@ describe("#1775 --check holds every peer forward agent to the standard (the heal ]); }); }); + +// ── #1775: the stray session-wide multiplex-flag agent is retired ────────────── +// `co.harmoniqs.amico-fleet-flag` ran `launchctl setenv AMICO_FLEET_MULTIPLEX 1` +// at login, leaking the flag into every GUI process — though the flag already +// defaults ON (#1448). The installer unloads + removes it and unsets the +// variable; asserted on the STUB's argv, never the real launchctl. +const FLAG_AGENT = ` + + + + Label + co.harmoniqs.amico-fleet-flag + ProgramArguments + + /bin/launchctl + setenv + AMICO_FLEET_MULTIPLEX + 1 + + RunAtLoad + + + +`; + +describe("#1775 the installer retires the session-wide multiplex-flag agent", () => { + const installEnv = (): { path: string } => ({ + path: `${join(tmp, "fakebin")}:${process.env.PATH ?? ""}`, + }); + const flagPlist = (): string => join(tmp, "Library", "LaunchAgents", "co.harmoniqs.amico-fleet-flag.plist"); + const placeFlagAgent = (): void => { + mkdirSync(join(tmp, "Library", "LaunchAgents"), { recursive: true }); + writeFileSync(flagPlist(), FLAG_AGENT); + }; + /** What the stubbed `launchctl getenv AMICO_FLEET_MULTIPLEX` prints (the session value). */ + const sessionFlag = (value: string | null): void => { + mkdirSync(join(tmp, "stubs"), { recursive: true }); + const f = join(tmp, "stubs", "getenv-AMICO_FLEET_MULTIPLEX"); + if (value === null) rmSync(f, { force: true }); + else writeFileSync(f, `${value}\n`); + }; + const verbFor = (role: string): string => + role === "standalone" ? verbJson() : verbJson(role, { host: "hq", port: 4096, sshAlias: "hq" }); + + it.each(["server", "client", "standalone"])("on a %s: the flag agent is unloaded + removed and the variable unset; a re-run is a no-op; the flag still defaults ON", (role) => { + fakeAmico({ code: 0, stdout: verbFor(role) }); + fakeUname("Darwin"); + placeFlagAgent(); + sessionFlag("1"); + + const r = runScript(INSTALL, [], installEnv()); + expect(r.code, r.out).toBe(0); + expect(existsSync(flagPlist())).toBe(false); + expect(launchctlCalls()).toContain(`unload ${flagPlist()}`); + expect(launchctlCalls()).toContain("unsetenv AMICO_FLEET_MULTIPLEX"); + expect(r.out).toMatch(/removed the session-wide multiplex-flag agent/); + expect(backstopCalls()).toEqual([]); + + // Fleet behavior is unchanged: with the variable gone the multiplexer is ON by default. + delete process.env[FLEET_MULTIPLEX_FLAG]; + expect(fleetMultiplexEnabled()).toBe(true); + + // Re-run after the unset: nothing left to retire — no unload, no unsetenv. + sessionFlag(null); + rmSync(join(tmp, "launchctl.argv"), { force: true }); + const again = runScript(INSTALL, [], installEnv()); + expect(again.code, again.out).toBe(0); + expect(launchctlCalls().filter((l) => l.includes("fleet-flag") || l.startsWith("unsetenv"))).toEqual([]); + }); + + it("a deliberate session opt-out (AMICO_FLEET_MULTIPLEX=0, no flag agent) is left alone — only the agent's leaked value is unset", () => { + fakeAmico({ code: 0, stdout: verbFor("server") }); + fakeUname("Darwin"); + sessionFlag("0"); + const r = runScript(INSTALL, [], installEnv()); + expect(r.code, r.out).toBe(0); + expect(launchctlCalls().filter((l) => l.startsWith("unsetenv"))).toEqual([]); + }); + + it("--check fails on a present flag agent with the fix line — and never calls launchctl", () => { + fakeAmico({ code: 0, stdout: verbFor("client") }); + fakeUname("Darwin"); + expect(runScript(INSTALL, [], installEnv()).code).toBe(0); + placeFlagAgent(); + rmSync(join(tmp, "launchctl.argv"), { force: true }); + const r = runScript(INSTALL, ["--check"], installEnv()); + expect(r.code).toBe(1); + expect(r.out).toMatch(/FAIL session-wide multiplex-flag agent present at .*co\.harmoniqs\.amico-fleet-flag\.plist.*run: bash tools\/fleet\/install\.sh/); + expect(existsSync(flagPlist())).toBe(true); + expect(launchctlCalls()).toEqual([]); + }); +}); diff --git a/packages/extension/tools/fleet/install.sh b/packages/extension/tools/fleet/install.sh index e47754ed4..c92152cf2 100755 --- a/packages/extension/tools/fleet/install.sh +++ b/packages/extension/tools/fleet/install.sh @@ -230,6 +230,35 @@ peer_forwards() { if [[ $peer_fail -eq 1 ]]; then exit 1; fi } +# --- the session-wide multiplex-flag agent (#1775) --- +# co.harmoniqs.amico-fleet-flag ran `launchctl setenv AMICO_FLEET_MULTIPLEX 1` +# at login, leaking the flag into EVERY GUI process — and the flag already +# defaults ON (#1448, fleetMultiplexEnabled), so fleet behavior is unchanged +# without it. Retire it: unload + remove the agent, and unset the variable when +# it carries the agent's leaked value (a deliberate `0` opt-out is left alone). +FLAG_PLIST="$LAUNCH_AGENTS_DIR/co.harmoniqs.amico-fleet-flag.plist" +retire_flag_agent() { + if [[ "$(uname -s)" != "Darwin" ]]; then return 0; fi + if [[ $CHECK -eq 1 ]]; then + if [[ -f "$FLAG_PLIST" ]]; then + echo "[fleet] FAIL session-wide multiplex-flag agent present at $FLAG_PLIST (it leaks AMICO_FLEET_MULTIPLEX=1 into every GUI process; the flag already defaults on — run: bash tools/fleet/install.sh)" + exit 1 + fi + return 0 + fi + local session + if [[ -f "$FLAG_PLIST" ]]; then + "$LAUNCHCTL" unload "$FLAG_PLIST" 2>/dev/null || true + rm -f "$FLAG_PLIST" + say "removed the session-wide multiplex-flag agent $FLAG_PLIST (AMICO_FLEET_MULTIPLEX already defaults on — #1448)" + fi + session="$("$LAUNCHCTL" getenv AMICO_FLEET_MULTIPLEX 2>/dev/null || true)" + if [[ "$session" == "1" ]]; then + "$LAUNCHCTL" unsetenv AMICO_FLEET_MULTIPLEX 2>/dev/null || true + say "unset the session-wide AMICO_FLEET_MULTIPLEX (the multiplexer defaults on without it)" + fi +} + # Standalone: no guard/tunnel/hub — but a standalone PEER still gets its # per-peer forwards (#1775: peers stay standalone, ADR 0027). if [[ "$ROLE" == "standalone" ]]; then @@ -238,6 +267,7 @@ if [[ "$ROLE" == "standalone" ]]; then else say "standalone mode — nothing to install for the fleet stance (use 'Amicode: Fleet — Enroll' to join a fleet)" fi + retire_flag_agent peer_forwards exit 0 fi @@ -390,7 +420,8 @@ elif [[ "$(uname -s)" == "Darwin" ]]; then fi fi -# --- per-peer SSH forwards (every role — #1775; see peer_forwards above) --- +# --- per-peer SSH forwards + the retired flag agent (every role — #1775) --- +retire_flag_agent peer_forwards # --- canonical hub service (SERVER role only — reboot-survival, #1258) --- diff --git a/tools/fleet/install.sh b/tools/fleet/install.sh index e47754ed4..c92152cf2 100755 --- a/tools/fleet/install.sh +++ b/tools/fleet/install.sh @@ -230,6 +230,35 @@ peer_forwards() { if [[ $peer_fail -eq 1 ]]; then exit 1; fi } +# --- the session-wide multiplex-flag agent (#1775) --- +# co.harmoniqs.amico-fleet-flag ran `launchctl setenv AMICO_FLEET_MULTIPLEX 1` +# at login, leaking the flag into EVERY GUI process — and the flag already +# defaults ON (#1448, fleetMultiplexEnabled), so fleet behavior is unchanged +# without it. Retire it: unload + remove the agent, and unset the variable when +# it carries the agent's leaked value (a deliberate `0` opt-out is left alone). +FLAG_PLIST="$LAUNCH_AGENTS_DIR/co.harmoniqs.amico-fleet-flag.plist" +retire_flag_agent() { + if [[ "$(uname -s)" != "Darwin" ]]; then return 0; fi + if [[ $CHECK -eq 1 ]]; then + if [[ -f "$FLAG_PLIST" ]]; then + echo "[fleet] FAIL session-wide multiplex-flag agent present at $FLAG_PLIST (it leaks AMICO_FLEET_MULTIPLEX=1 into every GUI process; the flag already defaults on — run: bash tools/fleet/install.sh)" + exit 1 + fi + return 0 + fi + local session + if [[ -f "$FLAG_PLIST" ]]; then + "$LAUNCHCTL" unload "$FLAG_PLIST" 2>/dev/null || true + rm -f "$FLAG_PLIST" + say "removed the session-wide multiplex-flag agent $FLAG_PLIST (AMICO_FLEET_MULTIPLEX already defaults on — #1448)" + fi + session="$("$LAUNCHCTL" getenv AMICO_FLEET_MULTIPLEX 2>/dev/null || true)" + if [[ "$session" == "1" ]]; then + "$LAUNCHCTL" unsetenv AMICO_FLEET_MULTIPLEX 2>/dev/null || true + say "unset the session-wide AMICO_FLEET_MULTIPLEX (the multiplexer defaults on without it)" + fi +} + # Standalone: no guard/tunnel/hub — but a standalone PEER still gets its # per-peer forwards (#1775: peers stay standalone, ADR 0027). if [[ "$ROLE" == "standalone" ]]; then @@ -238,6 +267,7 @@ if [[ "$ROLE" == "standalone" ]]; then else say "standalone mode — nothing to install for the fleet stance (use 'Amicode: Fleet — Enroll' to join a fleet)" fi + retire_flag_agent peer_forwards exit 0 fi @@ -390,7 +420,8 @@ elif [[ "$(uname -s)" == "Darwin" ]]; then fi fi -# --- per-peer SSH forwards (every role — #1775; see peer_forwards above) --- +# --- per-peer SSH forwards + the retired flag agent (every role — #1775) --- +retire_flag_agent peer_forwards # --- canonical hub service (SERVER role only — reboot-survival, #1258) --- From c42fe5c8074b160affd35e8f00594affcf6bc18d Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 20:36:41 -0400 Subject: [PATCH 48/60] docs(fleet): the installer header and fleet README describe the per-peer forwards (#1775) --- packages/extension/tools/fleet/install.sh | 2 ++ tools/fleet/README.md | 20 ++++++++++++++++++++ tools/fleet/install.sh | 2 ++ 3 files changed, 24 insertions(+) diff --git a/packages/extension/tools/fleet/install.sh b/packages/extension/tools/fleet/install.sh index c92152cf2..9471381bc 100755 --- a/packages/extension/tools/fleet/install.sh +++ b/packages/extension/tools/fleet/install.sh @@ -10,6 +10,8 @@ # branch (stated + the grant pointer), IDENTICAL to a CLI-absent machine; # other non-zero exits die honestly — never a silent raw-file fallthrough. # Installs/updates the fleet guard + tunnel on this host and fixes machine-scoped settings. +# #1775: also generates one SSH forward agent per attached peer (replacing a +# hand-installed one with a backup) and retires the session-wide multiplex-flag agent. # Usage: # bash tools/fleet/install.sh # install/repair (writes files, reloads launchd) # bash tools/fleet/install.sh --check # check only (no writes), exit 1 on drift diff --git a/tools/fleet/README.md b/tools/fleet/README.md index 363e61533..edb57925e 100644 --- a/tools/fleet/README.md +++ b/tools/fleet/README.md @@ -131,6 +131,26 @@ between them in near real time. Users can work with fleet sessions from either m making one peer the sole host. The Attach control selects a peer; it does not merge the peers' local databases into one shared database. +### Per-peer SSH forwards (#1775) + +When a peer is reached over SSH, this machine holds a loopback forward to that peer's hub. The +installer (and `Fleet — Repair`) generates it — never hand-install one: + +| | | +|---|---| +| Template | `co.harmoniqs.amico-peer-forward.plist` — the tunnel's standard options: `ServerAliveInterval 15 / CountMax 2 / TCPKeepAlive yes / ExitOnForwardFailure yes` (+ `BatchMode yes`) | +| Label | `co.harmoniqs.amico-peer--forward` — one agent per peer | +| Local port | the port of the peer's `base_url` in the reader peer-token store (`~/.amico/fleet-peer-tokens-reader.json`, `http://127.0.0.1:`) | +| Remote port / host | the fleet's canonical hub port, on the peer's roster `sshAlias` (else its machine id) | + +A peer reached over https (Tailscale / direct) gets no SSH forward. A hand-installed +`co.harmoniqs.amico--forward` agent for the same peer (same local port or ssh host) is +unloaded and moved to `~/.amico/ops/fleet/launchagents-backup/` — replaced, never silently +deleted. The healthcheck and `install.sh --check` report every forward agent and flag +missing or stale keepalives. The installer also retires the old `co.harmoniqs.amico-fleet-flag` +agent (it set `AMICO_FLEET_MULTIPLEX=1` session-wide; the flag defaults on). Darwin only — +the supervised Linux/WSL link is #1260. + ### Making two hub machines into peers (no single point of failure) There is **no `amico fleet` peer subcommand and no Command Palette wizard** yet — peer entry diff --git a/tools/fleet/install.sh b/tools/fleet/install.sh index c92152cf2..9471381bc 100755 --- a/tools/fleet/install.sh +++ b/tools/fleet/install.sh @@ -10,6 +10,8 @@ # branch (stated + the grant pointer), IDENTICAL to a CLI-absent machine; # other non-zero exits die honestly — never a silent raw-file fallthrough. # Installs/updates the fleet guard + tunnel on this host and fixes machine-scoped settings. +# #1775: also generates one SSH forward agent per attached peer (replacing a +# hand-installed one with a backup) and retires the session-wide multiplex-flag agent. # Usage: # bash tools/fleet/install.sh # install/repair (writes files, reloads launchd) # bash tools/fleet/install.sh --check # check only (no writes), exit 1 on drift From 1b9461884ee102d5cb019718fabb22e023313592 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 20:39:40 -0400 Subject: [PATCH 49/60] =?UTF-8?q?fix(fleet):=20the=20client=20resumes=20fr?= =?UTF-8?q?om=20the=20SSE=20id=20and=20checks=20contiguity=20per=20origin?= =?UTF-8?q?=20=E2=80=94=20a=20hole=20or=20a=20reset=20resyncs,=20a=20repla?= =?UTF-8?q?yed=20frame=20is=20never=20applied=20twice=20(#1777)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The global stream's cursor was the JSON payload id, which the #1776 engine rejects as malformed, so every reconnect was a reset. It is now the SSE id of the last applied frame (the engine's :, or the fan-in's composite), persisted in the existing session-storage key and sent on ONE carrier, ?lastEventID= (the SDK reader's Last-Event-ID header is stripped so two cursors can never disagree). stream-cursor.ts (pure, tested in place) checks each frame per origin, per epoch: a skipped bus seq, a heartbeat past the applied seq, or an unannounced new epoch is a hole; the engine's server.cursor.unservable is a reset; both force a full resync (resyncCount + the bootstrap gap event). A bus event at or behind the applied seq is dropped. A relay's gap re-bases the origins it names. Origin detection (#1539) now reads the SSE id it was designed for. --- packages/app-bundle/manifest.json | 12 +- .../packages/app/src/context/server-sdk.tsx | 132 +++++++----- .../app/src/context/stream-cursor.test.ts | 192 ++++++++++++++++++ .../packages/app/src/context/stream-cursor.ts | 176 ++++++++++++++++ 4 files changed, 452 insertions(+), 60 deletions(-) create mode 100644 packages/app-bundle/overlay/packages/app/src/context/stream-cursor.test.ts create mode 100644 packages/app-bundle/overlay/packages/app/src/context/stream-cursor.ts diff --git a/packages/app-bundle/manifest.json b/packages/app-bundle/manifest.json index b7d7c66c2..2df3b1f68 100644 --- a/packages/app-bundle/manifest.json +++ b/packages/app-bundle/manifest.json @@ -183,7 +183,7 @@ "packages/app/src/context/models.tsx": "8904525767b36ed5ba93e85a0fddb9bf737a8d066dfff084380ce1a67182f535", "packages/app/src/context/platform.tsx": "edd9b1773feb5c084b922a57b6914f6a473eed558b44f9de0eeb1c4d4e2b49da", "packages/app/src/context/server-sdk.test.ts": "833920190c25089a5c3ed9849a62b021d2e0e2e4fb76196af8ddc9e6aa9817ba", - "packages/app/src/context/server-sdk.tsx": "c1d26fbeaa37549ce17b348ea2fa61f424e730caa4518146542c23123a1c1390", + "packages/app/src/context/server-sdk.tsx": "bfa88452f7d42ffd0453425995af5294bfc34dade76cee3fbd627bb96051725a", "packages/app/src/context/server-session-child-active.test.ts": "de8c467f67f88b6cce351c1805ec90728cf238eb1ef32f39ffdbbf3de4dcec22", "packages/app/src/context/server-session-orphan-part.test.ts": "1d635374c0bfff25a01825061aabac7334e5cdaedca2dab9f3d9467ff543caea", "packages/app/src/context/server-session.test.ts": "30cd24bb1345376fd1a165d65677dee5c36d586efa7969d3b4844967040fda59", @@ -204,6 +204,8 @@ "packages/app/src/context/sse-gap-frame.ts": "2f64fde540a305579762c5276721d4bbfbf7c862131aaf4462e8a4e8f32c5960", "packages/app/src/context/sse-origin.test.ts": "71cd3aa996a51d7e99236a49cede4769e520e20e53f57c6920fe441f2abb0c7d", "packages/app/src/context/sse-origin.ts": "da9a231907dc50d87cf01758177bee0d4a0b0165d8adb5e152db8eca29dfd9f5", + "packages/app/src/context/stream-cursor.test.ts": "c9715bb28f576cdc00683b0bdc27bdc2a56a6f212969cc090f99b3dc8988e6a7", + "packages/app/src/context/stream-cursor.ts": "2711f85581cd86bb69b50a441b6a04c532553f61fa5e00cd1b9daa12193d3697", "packages/app/src/context/stream-gap.test.ts": "b7203445564f44fd1d966ef684a11cb9b5ec567d9ab54f1b7617726fdf44d3f5", "packages/app/src/context/stream-gap.ts": "2fd8eba30a6112cebf38e33c9418ab2b7d9c9d466f76729d828d5ab52d56c5ec", "packages/app/src/context/stream-liveness.test.ts": "2759eb2233f8ff6ced02957f971e756fe525575b2c8aefcbd7749470dec960f2", @@ -959,10 +961,10 @@ "packages/ui/src/v2/components/text-shimmer-v2.tsx" ], "exceptions": [], - "extracted_at": "2026-10-12T23:44:37.584Z", + "extracted_at": "2026-10-13T00:39:30.584Z", "per_package": { "packages/app": { - "A": 237, + "A": 239, "M": 161, "D": 0 }, @@ -1008,7 +1010,7 @@ } }, "counts": { - "overlay_total": 937, + "overlay_total": 939, "deletions": 7, "server_coupled": 0 }, @@ -1209,6 +1211,8 @@ "packages/app/src/context/sse-gap-frame.ts": "A", "packages/app/src/context/sse-origin.test.ts": "A", "packages/app/src/context/sse-origin.ts": "A", + "packages/app/src/context/stream-cursor.test.ts": "A", + "packages/app/src/context/stream-cursor.ts": "A", "packages/app/src/context/stream-gap.test.ts": "A", "packages/app/src/context/stream-gap.ts": "A", "packages/app/src/context/stream-liveness.test.ts": "A", diff --git a/packages/app-bundle/overlay/packages/app/src/context/server-sdk.tsx b/packages/app-bundle/overlay/packages/app/src/context/server-sdk.tsx index a1825006c..bb89bdf59 100644 --- a/packages/app-bundle/overlay/packages/app/src/context/server-sdk.tsx +++ b/packages/app-bundle/overlay/packages/app/src/context/server-sdk.tsx @@ -14,8 +14,8 @@ import { createRefCountMap } from "@/utils/refcount" import { useGlobal } from "./global" import { ServerScope } from "@/utils/server-scope" import { attachOwnerHeaderIfArmed } from "@/components/remote-create-arm" -import { parseCursorNamespaces, resolveEventOrigin } from "./sse-origin" import { createStreamEndResync, shouldReconnectIdleStream } from "./stream-liveness" +import { applyStreamCursor, createStreamCursor } from "./stream-cursor" import { detectServerProtocol, type ServerProtocol } from "@/utils/server-protocol" import { createCompatibleApi, type CompatibleApi } from "@/utils/server-compat" @@ -357,55 +357,43 @@ function createServerSdkContextBase(server: ServerConnection.Any, scope: ServerS } })() - // #1264 lossless reconnect: the frontdoor buffers recent events and - // replays the gap when the stream is re-opened with ?lastEventID=. - // Track the last payload id seen (the server ids every event; id-less - // events just don't advance the cursor — replay may over-deliver, which - // the reconcile-based reducers tolerate by design). + // #1264 lossless reconnect: the engine (#1776) sequences its global stream + // and replays the gap when the stream is re-opened with ?lastEventID=. + // #1777: the cursor is the SSE `id:` of the last applied frame — the engine's + // `:`, or the fan-in's composite of every origin's position — + // never the JSON payload id (the engine rejects that as malformed, so every + // reconnect used to be a reset). It also checks contiguity per origin: a + // hole, an engine reset, or an unannounced restart forces a resync, and a + // replayed frame already applied is dropped (stream-cursor.ts). // #1264: the cursor must survive RELOADS — module state died with every // document, so a drop right after a reload reconnected cursor-less and // the gap's events were lost (the harness caught it: the sse-gap flow // failed only in the post-reload context). sessionStorage is per-tab, // cheap, and exactly the lifetime the cursor wants. const CURSOR_KEY = "amicode.sse.lastEventID" - let lastEventID: string | undefined + let storedCursor: string | undefined try { - lastEventID = sessionStorage.getItem(CURSOR_KEY) ?? undefined + storedCursor = sessionStorage.getItem(CURSOR_KEY) ?? undefined } catch { /* private mode etc. — degrade to module state */ } - const trackEventID = (payload: unknown) => { - const id = (payload as { id?: unknown } | undefined)?.id - if (typeof id === "string" && id) { - lastEventID = id - try { - sessionStorage.setItem(CURSOR_KEY, id) - } catch { - /* best-effort */ - } + const cursor = createStreamCursor(storedCursor) + const persistCursor = () => { + const current = cursor.current() + if (current === undefined || current === storedCursor) return + storedCursor = current + try { + sessionStorage.setItem(CURSOR_KEY, current) + } catch { + /* best-effort */ } } - // #1539: track the parsed composite cursor to detect event origin (local vs - // remote peer). Initialized from the persisted cursor so the first event - // after reconnect diffs correctly. - let previousParsedCursor = parseCursorNamespaces(lastEventID) const sseFetch: FetchLike = (input, init) => { const base = eventFetch ?? globalThis.fetch - try { - if (lastEventID) { - const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url - const u = new URL(url, server.http.url) - if (u.pathname === "/event" || u.pathname === "/global/event") { - u.searchParams.set("lastEventID", lastEventID) - const modified = u.toString() - if (typeof input === "string" || input instanceof URL) return base(modified, init) - return base(new Request(modified, input), init) - } - } - } catch { - /* fall through unmodified */ - } - return base(input as Parameters[0], init as Parameters[1]) + // #1777: the ONE carrier is `?lastEventID=` (the SDK's own Last-Event-ID + // header is stripped so two cursors can never disagree). + const applied = applyStreamCursor(input, init, server.http.url, cursor.current()) + return base(applied.input as Parameters[0], applied.init as Parameters[1]) } const eventApi = createApiForServer({ server: server.http, fetch: sseFetch as typeof fetch }) const eventSdk = createSdkForServer({ @@ -475,6 +463,22 @@ function createServerSdkContextBase(server: ServerConnection.Any, scope: ServerS // nominally connected, so the disconnect→connect edge never fires and the rail // would otherwise never self-heal a stranded view). const [resyncCount, setResyncCount] = createSignal(0) + // #1617 / #1777: a FULL resync of the viewed state — the forced-resync signal + // (the transcript and the entity rail refetch, #1646) plus a synthetic GLOBAL + // `amicode.sync.gap` event, on which server-sync refetches the bootstrap + // unconditionally (past the #1289 `server.connected` debounce, which keys on + // a different type). Driven by a relay's gap frame, and by the cursor's + // verdicts: an engine reset, a sequence hole, an unannounced restart. + const forceResync = (namespaces: string[]) => { + setResyncCount((n) => n + 1) + if ( + enqueueServerEvent(queue, { + directory: "global", + payload: { type: SYNC_GAP_TYPE, properties: { namespaces } } as unknown as ServerEvent, + }) + ) + schedule() + } // #1773: an unexpected NORMAL end of the global stream is a loss (the relays // never end a listener cleanly except on shutdown). The loop arms this latch // on such an end; the reconnected stream's first frame advances resyncCount, @@ -551,9 +555,21 @@ function createServerSdkContextBase(server: ServerConnection.Any, scope: ServerS error, }) } + // #1777: the SSE id of the frame about to be yielded. The SDK reader + // calls onSseEvent with each parsed frame right before it yields that + // frame's data, so this is the id of the event the loop reads next. + let frameSseId: string | undefined const events = kind === "v1" - ? (await eventSdk.global.event({ signal: currentAttempt.signal, onSseError })).stream + ? ( + await eventSdk.global.event({ + signal: currentAttempt.signal, + onSseError, + onSseEvent: (frame: { id?: string }) => { + frameSseId = frame.id + }, + }) + ).stream : eventApi.event.subscribe({ signal: currentAttempt.signal }) if (!streamGeneration.accepts(active)) return setStreamStatus("connected") @@ -571,7 +587,6 @@ function createServerSdkContextBase(server: ServerConnection.Any, scope: ServerS // between the end and this re-subscription). if (streamEndResync.frame()) setResyncCount((n) => n + 1) const legacy = "payload" in event - if (legacy && event.payload.type === "sync") continue const directory = legacy ? (event.directory ?? "global") : (event.location?.directory ?? "global") const payload = legacy ? (event.payload as Event) : adaptServerEvent(event) // #1522 (ADR 0033 decision A): the SSE fan-in aggregator emits @@ -612,26 +627,31 @@ function createServerSdkContextBase(server: ServerConnection.Any, scope: ServerS ? (payload as Record) : ((payload as { properties?: Record }).properties ?? {}) const gap = parseSyncGapFrame({ type: SYNC_GAP_TYPE, ...gapData }) + // #1777: the named origins resync, so their next positions are new + // baselines (not a second "hole" against the lost stretch). + cursor.gap(gap?.namespaces ?? []) // Bump the forced-resync signal so the entity rail self-heals even // though the socket never crossed a disconnect→connect edge. - setResyncCount((n) => n + 1) - if (enqueueServerEvent(queue, { - directory: "global", - payload: { type: SYNC_GAP_TYPE, properties: { namespaces: gap?.namespaces ?? [] } } as unknown as ServerEvent, - })) - schedule() + forceResync(gap?.namespaces ?? []) continue } - trackEventID(legacy ? event.payload : (event as { id?: string })) - // #1539: resolve origin from the composite cursor diff. Fleet-of-one - // (bare scalar id) returns undefined immediately — zero overhead. - const sseId: string | undefined = legacy - ? (event.payload as { id?: string }).id - : (event as { id?: string }).id - const resolved = resolveEventOrigin(sseId, previousParsedCursor) - previousParsedCursor = resolved.cursor - const taggedPayload: ServerEvent = resolved.origin !== undefined - ? { ...payload, origin: resolved.origin } + // #1777: the cursor and the contiguity check, on the frame's SSE id + // (v1; the v2 stream at /api/event carries no resume cursor). A hole, + // a reset or an unannounced engine restart owes a full resync; a + // replayed frame already applied is dropped, never applied twice. + const verdict = cursor.frame({ + id: kind === "v1" ? frameSseId : undefined, + type: legacy ? event.payload.type : payload.type, + }) + if (verdict.resync) forceResync(verdict.resync.namespaces) + if (!verdict.apply) continue + persistCursor() + // a sync copy still moves the cursor (it holds a seq), but is no event + if (legacy && event.payload.type === "sync") continue + // #1539: the origin is the composite cursor's moving namespace. + // Fleet-of-one (a bare engine id) is always local. + const taggedPayload: ServerEvent = verdict.origin !== undefined + ? { ...payload, origin: verdict.origin } : payload if (enqueueServerEvent(queue, { directory, payload: taggedPayload })) schedule() @@ -675,8 +695,8 @@ function createServerSdkContextBase(server: ServerConnection.Any, scope: ServerS } const clearGlobalStreamCursor = () => { - lastEventID = undefined - previousParsedCursor = {} + cursor.clear() + storedCursor = undefined try { sessionStorage.removeItem(CURSOR_KEY) } catch { diff --git a/packages/app-bundle/overlay/packages/app/src/context/stream-cursor.test.ts b/packages/app-bundle/overlay/packages/app/src/context/stream-cursor.test.ts new file mode 100644 index 000000000..ddb19fa4b --- /dev/null +++ b/packages/app-bundle/overlay/packages/app/src/context/stream-cursor.test.ts @@ -0,0 +1,192 @@ +import { describe, expect, test } from "bun:test" +import { applyStreamCursor, createStreamCursor } from "./stream-cursor" + +// #1777 — the client's resume cursor and its contiguity check, extracted as a +// ZERO-DEPENDENCY pure module (like stream-liveness.ts) so it runs in place. +// server-sdk.tsx's global stream loop feeds it every data frame's SSE `id:` (the +// SDK reader's `onSseEvent`) and its payload type, and acts on the verdict. +// +// Wire facts (#1776, #1511): an engine stamps every global-stream frame +// `id: :` — bus events take strictly increasing seqs, and the +// per-connection control frames (server.connected / server.heartbeat / +// server.cursor.unservable) carry the CURRENT seq without consuming one. A +// fleet-of-one fan-in passes those ids through untouched; a fanned-in stream's +// id is the composite `local=;=` of every origin's position. + +const E = "bG9jYWwx" // the local engine's epoch +const P = "c3R1ZGlv" // a peer engine's epoch +const id = (epoch: string, seq: number) => `${epoch}:${seq}` +const composite = (local: number, studio: number) => `local=${id(E, local)};studio=${id(P, studio)}` +const event = (frameId: string | undefined, type = "message.part.updated") => ({ id: frameId, type }) + +describe("#1777 AC4 — the cursor is the SSE id, and it survives a reconnect and a reload", () => { + test("fleet-of-one: the cursor is the engine's bare id, verbatim", () => { + const cursor = createStreamCursor() + expect(cursor.current()).toBeUndefined() + cursor.frame(event(id(E, 0), "server.connected")) + cursor.frame(event(id(E, 1))) + expect(cursor.current()).toBe(id(E, 1)) + }) + + test("fanned in: the cursor is the composite id, verbatim", () => { + const cursor = createStreamCursor() + cursor.frame(event(composite(1, 7))) + cursor.frame(event(composite(2, 7))) + expect(cursor.current()).toBe(composite(2, 7)) + }) + + test("never the JSON payload id: a frame without an SSE id leaves the cursor where it was", () => { + const cursor = createStreamCursor() + cursor.frame(event(id(E, 4))) + expect(cursor.frame(event(undefined))).toEqual({ apply: true, origin: undefined }) + expect(cursor.current()).toBe(id(E, 4)) + }) + + test("after a reload: the stored cursor is resumed from, and frames are checked against it", () => { + const cursor = createStreamCursor(id(E, 9)) + expect(cursor.current()).toBe(id(E, 9)) + expect(cursor.frame(event(id(E, 9), "server.connected")).resync).toBeUndefined() // the engine resumed at it + expect(cursor.frame(event(id(E, 11))).resync).toEqual({ reason: "hole", namespaces: ["local"] }) // 10 never came + }) + + test("the ONE carrier: `?lastEventID=` on the global stream, and never a conflicting Last-Event-ID header", () => { + const base = "http://127.0.0.1:4096" + const req = new Request(`${base}/global/event`, { headers: { "Last-Event-ID": "stale:1", authorization: "Basic x" } }) + const out = applyStreamCursor(req, undefined, base, composite(3, 8)) + const sent = out.input as Request + expect(new URL(sent.url).searchParams.get("lastEventID")).toBe(composite(3, 8)) + expect(sent.headers.get("last-event-id")).toBeNull() + expect(sent.headers.get("authorization")).toBe("Basic x") + + // a string input, a stale header in init: the same single carrier + const str = applyStreamCursor(`${base}/global/event?lastEventID=old`, { headers: { "Last-Event-ID": "old" } }, base, id(E, 2)) + expect(new URL(str.input as string).searchParams.get("lastEventID")).toBe(id(E, 2)) + expect(new Headers(str.init?.headers).get("last-event-id")).toBeNull() + + // no cursor (a fresh or reset stream): no cursor at all + const fresh = applyStreamCursor(`${base}/global/event?lastEventID=old`, undefined, base, undefined) + expect(new URL(fresh.input as string).searchParams.has("lastEventID")).toBe(false) + + // any other request is untouched + const other = `${base}/session` + expect(applyStreamCursor(other, undefined, base, id(E, 2)).input).toBe(other) + }) +}) + +describe("#1777 AC4 — a sequence hole within one origin triggers a resync of that origin", () => { + test("fleet-of-one: a bus event that skips a seq is a hole — applied, and a resync is owed", () => { + const cursor = createStreamCursor() + cursor.frame(event(id(E, 1))) + expect(cursor.frame(event(id(E, 3)))).toEqual({ apply: true, origin: undefined, resync: { reason: "hole", namespaces: ["local"] } }) + expect(cursor.frame(event(id(E, 4))).resync).toBeUndefined() // contiguous again + }) + + test("an idle stream: a heartbeat carrying a seq the client never saw is a hole", () => { + const cursor = createStreamCursor() + cursor.frame(event(id(E, 5))) + expect(cursor.frame(event(id(E, 5), "server.heartbeat")).resync).toBeUndefined() // at the position: fine + expect(cursor.frame(event(id(E, 6), "server.heartbeat")).resync).toEqual({ reason: "hole", namespaces: ["local"] }) + }) + + test("fanned in: contiguity is per origin — a hole in the peer's seq resyncs the peer, the local arm's frames never count against it", () => { + const cursor = createStreamCursor() + cursor.frame(event(composite(5, 42))) + expect(cursor.frame(event(composite(6, 42))).resync).toBeUndefined() // local advanced by one + expect(cursor.frame(event(composite(7, 42))).resync).toBeUndefined() + expect(cursor.frame(event(composite(7, 43)))).toEqual({ apply: true, origin: "studio" }) // the peer, contiguous + expect(cursor.frame(event(composite(7, 45)))).toEqual({ apply: true, origin: "studio", resync: { reason: "hole", namespaces: ["studio"] } }) + }) + + test("per epoch: an engine restart nothing announced is a resync, and the new epoch is the new baseline", () => { + const cursor = createStreamCursor() + cursor.frame(event(id(E, 30))) + expect(cursor.frame(event(id("bG9jYWwy", 1))).resync).toEqual({ reason: "hole", namespaces: ["local"] }) + expect(cursor.frame(event(id("bG9jYWwy", 2))).resync).toBeUndefined() + }) + + test("a first sighting is a baseline, and ids that are not an engine's (an older relay) are never checked", () => { + const cursor = createStreamCursor() + expect(cursor.frame(event(id(E, 50))).resync).toBeUndefined() + const relay = createStreamCursor() + relay.frame(event("17")) + expect(relay.frame(event("40"))).toEqual({ apply: true, origin: undefined }) + }) +}) + +describe("#1777 AC2 — an upstream reset forces exactly one full resync", () => { + test("fleet-of-one: the engine's reset frame is a resync of `local`, and its new position is the baseline", () => { + const cursor = createStreamCursor(id(E, 12)) // a cursor from before the engine restarted + const reset = cursor.frame(event(id("bG9jYWwy", 3), "server.cursor.unservable")) + expect(reset).toEqual({ apply: true, origin: undefined, resync: { reason: "reset", namespaces: ["local"] } }) + expect(cursor.current()).toBe(id("bG9jYWwy", 3)) + expect(cursor.frame(event(id("bG9jYWwy", 3), "server.connected")).resync).toBeUndefined() + expect(cursor.frame(event(id("bG9jYWwy", 4))).resync).toBeUndefined() + }) + + test("a reset BEHIND the client's cursor (the engine's `ahead` reason, same epoch) is still the new baseline", () => { + const cursor = createStreamCursor(id(E, 40)) // a cursor past what the engine has + expect(cursor.frame(event(id(E, 7), "server.cursor.unservable")).resync).toEqual({ reason: "reset", namespaces: ["local"] }) + expect(cursor.current()).toBe(id(E, 7)) + expect(cursor.frame(event(id(E, 8)))).toEqual({ apply: true, origin: undefined }) // not mistaken for a duplicate + }) + + test("fanned in: after a gap names the peer, its next position (a new epoch) is a new baseline — one resync, not two", () => { + const cursor = createStreamCursor() + cursor.frame(event(composite(5, 42))) + cursor.gap(["studio"]) // the fan-in turned the peer's reset into a gap; the loop resyncs on it + const next = `local=${id(E, 5)};studio=${id("c3R1ZGlvMg", 7)}` + expect(cursor.frame(event(next, "server.connected"))).toEqual({ apply: true, origin: "studio" }) + expect(cursor.frame(event(`local=${id(E, 5)};studio=${id("c3R1ZGlvMg", 8)}`)).resync).toBeUndefined() + }) + + test("a gap naming no namespace re-bases every origin", () => { + const cursor = createStreamCursor() + cursor.frame(event(composite(5, 42))) + cursor.gap([]) + expect(cursor.frame(event(composite(9, 42))).resync).toBeUndefined() + expect(cursor.frame(event(composite(9, 50))).resync).toBeUndefined() + expect(cursor.frame(event(composite(9, 52))).resync).toEqual({ reason: "hole", namespaces: ["studio"] }) // checked again + }) + + test("clear() forgets the cursor (an attachment change opens the stream cursorless)", () => { + const cursor = createStreamCursor(id(E, 3)) + cursor.clear() + expect(cursor.current()).toBeUndefined() + expect(cursor.frame(event(id("bG9jYWwy", 90))).resync).toBeUndefined() + }) +}) + +describe("#1777 AC5 — replayed frames are never applied twice", () => { + test("fleet-of-one: a bus event at or behind the applied seq is dropped; the cursor never moves back", () => { + const cursor = createStreamCursor() + cursor.frame(event(id(E, 5))) + expect(cursor.frame(event(id(E, 5))).apply).toBe(false) // the last frame again + expect(cursor.frame(event(id(E, 3))).apply).toBe(false) // an older one + expect(cursor.current()).toBe(id(E, 5)) + expect(cursor.frame(event(id(E, 6)))).toEqual({ apply: true, origin: undefined }) + }) + + test("fanned in: a peer frame behind the peer's applied seq is dropped", () => { + const cursor = createStreamCursor() + cursor.frame(event(composite(5, 42))) + expect(cursor.frame(event(composite(5, 41))).apply).toBe(false) + expect(cursor.current()).toBe(composite(5, 42)) + expect(cursor.frame(event(composite(5, 43))).apply).toBe(true) + }) + + test("a connection resumed from an older cursor: its handshake still applies, but the replay of what was applied is dropped", () => { + const cursor = createStreamCursor() + for (const s of [1, 2, 3]) cursor.frame(event(id(E, s))) + expect(cursor.frame(event(id(E, 1), "server.connected")).apply).toBe(true) // control: delivered, cursor kept + expect(cursor.current()).toBe(id(E, 3)) + expect(cursor.frame(event(id(E, 2))).apply).toBe(false) + expect(cursor.frame(event(id(E, 3))).apply).toBe(false) + expect(cursor.frame(event(id(E, 4)))).toEqual({ apply: true, origin: undefined }) + }) + + test("the control frames at the position are never mistaken for duplicates", () => { + const cursor = createStreamCursor() + cursor.frame(event(id(E, 5))) + for (const type of ["server.connected", "server.heartbeat"]) expect(cursor.frame(event(id(E, 5), type)).apply).toBe(true) + }) +}) diff --git a/packages/app-bundle/overlay/packages/app/src/context/stream-cursor.ts b/packages/app-bundle/overlay/packages/app/src/context/stream-cursor.ts new file mode 100644 index 000000000..6cb2edb20 --- /dev/null +++ b/packages/app-bundle/overlay/packages/app/src/context/stream-cursor.ts @@ -0,0 +1,176 @@ +// #1777 — the client's resume cursor for the global event stream, and the +// contiguity check that makes every unreplayable loss loud. A ZERO-DEPENDENCY +// pure module (like stream-liveness.ts — no solid-js, no @opencode-ai/*) so its +// test runs in place; server-sdk.tsx's stream loop feeds it each data frame's +// SSE id and payload type, and acts on the verdict. +// +// The wire (#1776, #1511): an engine stamps every global-stream frame +// `id: :`. Bus events take strictly increasing seqs; the +// per-connection control frames (`server.connected`, `server.heartbeat`, the +// `server.cursor.unservable` reset) consume none and carry the CURRENT seq — so +// a heartbeat exposes a hole even on an idle stream. A fleet-of-one fan-in +// passes the engine's ids through untouched; a fanned-in stream's id is the +// composite `local=;=` of every origin's position, and each frame +// moves only its own origin's. +// +// The CURSOR is that SSE id — never the JSON payload id the client tracked +// before, which the engine rejects as malformed (every reconnect was a reset). +// Contiguity is checked per origin, per epoch: +// - the next bus event of an origin must be seq + 1; anything later is a HOLE +// (a resync of that origin is owed), anything at or behind it a DUPLICATE +// (a replayed frame already applied — dropped, never applied twice); +// - a control frame may sit at the position; past it is a hole; +// - a new epoch nothing announced is a hole (an engine restart the relay +// chain lost); a reset, or a gap the relay announced, re-bases the origin. +import { resolveEventOrigin } from "./sse-origin" + +/** The reserved namespace of the always-present local source (#1511 §D4). */ +export const LOCAL_ORIGIN = "local" + +const CONTROL_TYPES = new Set(["server.connected", "server.heartbeat", "server.cursor.unservable"]) +const RESET_TYPE = "server.cursor.unservable" + +type Position = { epoch: string; seq: number } + +function enginePosition(value: string | undefined): Position | undefined { + if (value === undefined) return undefined + const m = /^([A-Za-z0-9_-]+):(\d+)$/.exec(value) + return m ? { epoch: m[1], seq: Number(m[2]) } : undefined +} + +/** A stream id as per-origin positions: the composite `=;…`, or a bare + * id (no `=`, the fleet-of-one / direct-engine form) as the local origin's. */ +export function originPositions(id: string | undefined): Map { + const out = new Map() + const trimmed = id?.trim() + if (!trimmed) return out + if (!trimmed.includes("=")) { + out.set(LOCAL_ORIGIN, trimmed) + return out + } + for (const entry of trimmed.split(";")) { + const eq = entry.indexOf("=") + if (eq <= 0) continue + out.set(entry.slice(0, eq), entry.slice(eq + 1)) + } + return out +} + +export type StreamResync = { reason: "hole" | "reset"; namespaces: string[] } + +/** What the stream loop does with one data frame. */ +export type StreamCursorVerdict = { + /** false: a duplicate of a frame already applied — drop it. */ + apply: boolean + /** The origin that sent it: a peer's machine id, or undefined for local. */ + origin: string | undefined + /** Present when frames of these origins were lost: resync them. */ + resync?: StreamResync +} + +export function createStreamCursor(initial?: string) { + let raw = initial?.trim() || undefined + let applied = originPositions(raw) + // origins whose next position is a new baseline (after an announced gap) + const rebase = new Set() + + const frame = (input: { id: string | undefined; type: string | undefined }): StreamCursorVerdict => { + const id = input.id?.trim() + if (!id) return { apply: true, origin: undefined } // no SSE id (an older engine): nothing to check + const next = originPositions(id) + const bare = !id.includes("=") + const control = input.type !== undefined && CONTROL_TYPES.has(input.type) + const reset = input.type === RESET_TYPE + const changed = [...next.keys()].filter((ns) => next.get(ns) !== applied.get(ns)) + // #1539: the origin is the namespace the composite moved (fleet-of-one: local) + const origin = resolveEventOrigin(id, Object.fromEntries(applied)).origin + + const holes: string[] = [] + let behind = false + for (const ns of changed) { + const prev = enginePosition(applied.get(ns)) + const cur = enginePosition(next.get(ns)) + if (reset || rebase.has(ns) || prev === undefined || cur === undefined) continue // a baseline + if (cur.epoch !== prev.epoch) { + holes.push(ns) + continue + } + if (cur.seq <= prev.seq) { + behind = true + continue + } + if (cur.seq > prev.seq + (control ? 0 : 1)) holes.push(ns) + } + + // A frame behind what was applied: a replayed bus event is a duplicate + // (dropped); a control frame (a connection resumed from an older cursor) + // still applies, but never moves the cursor back. + if (behind) return control ? { apply: true, origin } : { apply: false, origin } + // Fleet-of-one: the last bus event again (same engine id) is a duplicate. + if (changed.length === 0 && bare && !control && enginePosition(id)) return { apply: false, origin } + + raw = id + if (bare) applied = next + else for (const [ns, value] of next) applied.set(ns, value) + for (const ns of changed) rebase.delete(ns) + + if (reset) return { apply: true, origin, resync: { reason: "reset", namespaces: changed.length > 0 ? changed : [...next.keys()] } } + if (holes.length > 0) return { apply: true, origin, resync: { reason: "hole", namespaces: holes } } + return { apply: true, origin } + } + + return { + /** The cursor to resume from: the last applied frame's SSE id, verbatim. */ + current: (): string | undefined => raw, + frame, + /** The relay announced a gap for these origins (none named = every origin); + * the loop resyncs them, so their next positions are new baselines. */ + gap(namespaces: string[]): void { + for (const ns of namespaces.length > 0 ? namespaces : [...applied.keys()]) rebase.add(ns) + }, + /** Forget the cursor: the next connection opens without one. */ + clear(): void { + raw = undefined + applied = new Map() + rebase.clear() + }, + } +} + +export type StreamCursor = ReturnType + +const STREAM_PATHS = new Set(["/event", "/global/event"]) + +const withoutLastEventIdHeader = (init: RequestInit | undefined): RequestInit | undefined => { + if (!init?.headers) return init + const headers = new Headers(init.headers) + headers.delete("last-event-id") + return { ...init, headers } +} + +/** The ONE cursor carrier on the event stream request: `?lastEventID=` + * — the parameter the fan-in, the proxies and the engine all read. The SDK + * reader would add its own `Last-Event-ID` header on an internal retry, which + * the engine prefers; it is removed, so two cursors can never disagree. No + * cursor: none is sent. Any other request passes through untouched. */ +export function applyStreamCursor( + input: RequestInfo | URL, + init: RequestInit | undefined, + base: string, + cursor: string | undefined, +): { input: RequestInfo | URL; init: RequestInit | undefined } { + let url: URL + try { + url = new URL(typeof input === "string" ? input : input instanceof URL ? input.href : input.url, base) + } catch { + return { input, init } + } + if (!STREAM_PATHS.has(url.pathname)) return { input, init } + if (cursor) url.searchParams.set("lastEventID", cursor) + else url.searchParams.delete("lastEventID") + if (typeof input === "string") return { input: url.toString(), init: withoutLastEventIdHeader(init) } + if (input instanceof URL) return { input: url, init: withoutLastEventIdHeader(init) } + const request = new Request(url.toString(), input) + request.headers.delete("last-event-id") + return { input: request, init: withoutLastEventIdHeader(init) } +} From ca96336f9fa776bee33ec4687016228d943638eb Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 20:39:48 -0400 Subject: [PATCH 50/60] test(fleet): an upstream reset on any arm is exactly one gap for its namespace; fleet-of-one passes the engine's reset frame verbatim (#1777) --- .../test/sse_fanin_driver_pump.test.ts | 145 ++++++++++++++++++ 1 file changed, 145 insertions(+) diff --git a/packages/extension/test/sse_fanin_driver_pump.test.ts b/packages/extension/test/sse_fanin_driver_pump.test.ts index 14853cd4b..b19add86f 100644 --- a/packages/extension/test/sse_fanin_driver_pump.test.ts +++ b/packages/extension/test/sse_fanin_driver_pump.test.ts @@ -2337,3 +2337,148 @@ describe("#1777 AC1 — a peer arm that drops and re-opens within the replay win }); } }); + +const RESTARTED_EPOCH = "c3R1ZGlvMg"; // the peer's engine after a restart: a new process epoch + +describe("#1777 AC2 — an upstream reset on any arm is exactly one gap for that namespace downstream", () => { + const gapsOf = (text: string) => sseFrames(text).filter((f) => f.type === "amicode.sync.gap").map((f) => f.data!.namespaces); + + it("composite: a peer engine that restarted answers the resume with a reset → one gap naming it; the reset frame itself is not forwarded; the arm re-bases in the new epoch", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const local = replayingEngine({ epoch: LOCAL_EPOCH }); + const studio = replayingEngine({ epoch: PEER_EPOCH }); + const engines: { local: ReplayingEngine; studio: ReplayingEngine } = { local, studio }; + const { census, lines } = testCensus(); + const driver = replayDriver(engines, ownerMap, { census }); + const res = mockRes(); + driver.handle(mockReq("/global/event"), res); + await flush(); + for (const n of [1, 2]) studio.publish(...partEvent(n)); + await flush(); + studio.subscriptions[0].end("error"); // the link drops — and the peer's engine restarts + await flush(); + const restarted = replayingEngine({ epoch: RESTARTED_EPOCH }); + engines.studio = restarted; + restarted.publish(...partEvent(3)); // lost with the old process's buffer: unrecoverable + driver.reconcile(); + await flush(); + + expect(restarted.subscriptions[0]).toMatchObject({ cursor: studio.idAt(2), reset: true }); + expect(gapsOf(res.text())).toEqual([["studio"]]); // exactly one gap, naming the peer + expect(sseFrames(res.text()).some((f) => f.type === "server.cursor.unservable")).toBe(false); + expect(lines).toContain("[fanin] gap-emitted plane=observe conn=c1 arm=studio cause=reset"); + expect(res.destroyed).toBe(false); + + // re-based: the next frames carry the new epoch, and a later drop resumes in it + restarted.publish(...partEvent(4)); + await flush(); + expect(parseCompositeCursor(sseFrames(res.text()).at(-1)!.id).get("studio")).toBe(restarted.idAt(2)); + restarted.subscriptions[0].end(); + await flush(); + driver.reconcile(); + await flush(); + expect(restarted.subscriptions[1]).toMatchObject({ cursor: restarted.idAt(2), reset: false }); + expect(gapsOf(res.text())).toHaveLength(1); // still one + res.fireClose(); + }); + + it("composite: a drop longer than the replay window (the cursor was evicted) → one gap naming the peer", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const local = replayingEngine({ epoch: LOCAL_EPOCH }); + const studio = replayingEngine({ epoch: PEER_EPOCH }); + const driver = replayDriver({ local, studio }, ownerMap); + const res = mockRes(); + driver.handle(mockReq("/global/event"), res); + await flush(); + studio.publish(...partEvent(1)); + await flush(); + studio.subscriptions[0].end("error"); + await flush(); + for (const n of [2, 3, 4]) studio.publish(...partEvent(n)); + studio.evictThrough(3); // the window moved past the cursor (seq 1) + driver.reconcile(); + await flush(); + expect(studio.subscriptions[1].reset).toBe(true); + expect(gapsOf(res.text())).toEqual([["studio"]]); + res.fireClose(); + }); + + it("composite: the LOCAL arm's reset (the local engine restarted) is one gap naming `local`", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const engines: { local: ReplayingEngine; studio: ReplayingEngine } = { + local: replayingEngine({ epoch: LOCAL_EPOCH }), + studio: replayingEngine({ epoch: PEER_EPOCH }), + }; + const driver = replayDriver(engines, ownerMap); + const res = mockRes(); + driver.handle(mockReq("/global/event"), res); + await flush(); + engines.local.publish(...partEvent(1)); + await flush(); + engines.local.subscriptions[0].end(); + await flush(); + engines.local = replayingEngine({ epoch: "bG9jYWwy" }); + driver.reconcile(); + await flush(); + expect(gapsOf(res.text())).toEqual([[LOCAL_NAMESPACE]]); + res.fireClose(); + }); + + it("composite: a listener reconnecting with a cursor its peer can no longer serve gets one gap for that peer — and a replay for the servable one", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const local = replayingEngine({ epoch: LOCAL_EPOCH }); + const studio = replayingEngine({ epoch: PEER_EPOCH }); + for (const n of [1, 2]) local.publish(...partEvent(n)); + const driver = replayDriver({ local, studio }, ownerMap); + const res = mockRes(); + // a cursor from before the peer's restart: `studio` names a dead epoch + driver.handle(mockReq(`/global/event?lastEventID=local=${local.idAt(1)};studio=${RESTARTED_EPOCH}:9`), res); + await flush(); + expect(local.subscriptions[0]).toMatchObject({ cursor: local.idAt(1), reset: false }); + expect(studio.subscriptions[0]).toMatchObject({ cursor: `${RESTARTED_EPOCH}:9`, reset: true }); + expect(gapsOf(res.text())).toEqual([["studio"]]); + // the local replay arrived: frame 2 (the one after the cursor) + expect(sseFrames(res.text()).filter((f) => f.type === "message.part.updated").map((f) => f.data!.payload.properties.n)).toEqual([2]); + res.fireClose(); + }); + + it("fleet-of-one: the engine's reset frame passes VERBATIM (it is the engine's own signal — #1264: nothing synthetic), and the listener lives", async () => { + const engines: { local: ReplayingEngine } = { local: replayingEngine({ epoch: LOCAL_EPOCH }) }; + const sent: string[] = []; + const driver = replayDriver(engines as { local: ReplayingEngine }, new SessionOwnerMap(), { + openUpstream: (r) => { + const sub = engines.local.subscribe(r.lastEventId); + const next = sub.source.next; + sub.source.next = async () => { + const f = await next(); + if (f !== null) sent.push(f); + return f; + }; + return sub.source; + }, + }); + const res = mockRes(); + driver.handle(mockReq(`/global/event?lastEventID=${LOCAL_EPOCH}:4`), res); // stale: the engine is at 0 + await flush(); + engines.local.publish(...partEvent(1)); + await flush(); + engines.local.subscriptions[0].end(); // mid-stream: the local engine restarts + await flush(); + engines.local = replayingEngine({ epoch: "bG9jYWwy" }); + driver.reconcile(); // resume → the new process resets + await flush(); + engines.local.publish(...partEvent(2)); + await flush(); + + expect(res.text()).toBe(sent.join("")); // byte-identical to what the engine sent, ids included + const types = sseFrames(res.text()).map((f) => f.type); + expect(types.filter((t) => t === "server.cursor.unservable")).toHaveLength(2); // the stale connect cursor, then the restart + expect(types).not.toContain("amicode.sync.gap"); + expect(res.destroyed).toBe(false); + res.fireClose(); + }); +}); From 4089797456a1b15340ba253bbbf288bc9c073947 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 20:41:44 -0400 Subject: [PATCH 51/60] =?UTF-8?q?test(fleet):=20fleet-of-one=20over=20real?= =?UTF-8?q?=20sockets=20=E2=80=94=20across=20a=20link=20loss=20and=20its?= =?UTF-8?q?=20replayed=20re-open=20the=20listener=20holds=20the=20engine's?= =?UTF-8?q?=20bytes,=20bare=20ids=20included,=20and=20nothing=20else=20(#1?= =?UTF-8?q?777)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../test/sse_fanin_driver_pump.test.ts | 115 ++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/packages/extension/test/sse_fanin_driver_pump.test.ts b/packages/extension/test/sse_fanin_driver_pump.test.ts index b19add86f..62d6adb90 100644 --- a/packages/extension/test/sse_fanin_driver_pump.test.ts +++ b/packages/extension/test/sse_fanin_driver_pump.test.ts @@ -2482,3 +2482,118 @@ describe("#1777 AC2 — an upstream reset on any arm is exactly one gap for that res.fireClose(); }); }); + +/** Serve a replaying engine's `/global/event` over a REAL socket (127.0.0.1, + * port 0, dialed at its own address): each request subscribes with its + * `?lastEventID=` and writes the subscription's frames as they come. Records + * every byte written per connection; `drop()` destroys the open connections + * (a link loss — the reader sees an error, not a clean end). */ +async function serveReplayingEngine(engine: ReplayingEngine) { + const written: string[][] = []; + const open = new Set(); + const server = http.createServer((req, res) => { + const cursor = new URL(req.url ?? "/", "http://engine").searchParams.get("lastEventID") ?? undefined; + const sub = engine.subscribe(cursor); + const mine: string[] = []; + written.push(mine); + res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache, no-transform" }); + res.flushHeaders(); + open.add(res); + res.on("close", () => { + open.delete(res); + sub.end("error"); + }); + void (async () => { + for (;;) { + const f = await sub.source.next(); + if (f === null) return; + mine.push(f); + res.write(f); + } + })(); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + const { port } = server.address() as import("node:net").AddressInfo; + return { + url: `http://127.0.0.1:${port}`, + written, + drop: () => { + for (const res of open) res.destroy(); + }, + stop: () => + new Promise((r) => { + server.close(() => r()); + server.closeAllConnections?.(); + }), + }; +} + +describe("#1777 AC3 — fleet-of-one: the listener's stream is byte-identical to the engine's, ids included", () => { + it("over REAL sockets, across a link loss and the replayed re-open: every byte the engine wrote, and nothing else", async () => { + const engine = replayingEngine({ epoch: LOCAL_EPOCH, now: () => 1_700_000_000_000 }); + const upstream = await serveReplayingEngine(engine); + const lines: string[] = []; + const w = lineWaiter(lines); + const driver = new SseFanInDriver({ + ownerMap: new SessionOwnerMap(), // fleet-of-one: the verbatim relay + localMachineId: "macbook", + localEventUrl: () => upstream.url, + peerBaseUrl: () => undefined, + peerToken: () => ({ ok: false as const, reason: "absent" as const }), + reconcileMs: 999_999, + census: new FanInCensus({ plane: "observe", sink: w.sink }), + }); + const front = http.createServer((req, res) => void driver.handle(req, res)); + await new Promise((r) => front.listen(0, "127.0.0.1", r)); + const { port } = front.address() as import("node:net").AddressInfo; + let body = ""; + let wake: () => void = () => undefined; + // event-driven (each listener chunk wakes it); the deadline only names a regression + const until = async (pred: () => boolean, what: string) => { + const deadline = Date.now() + 3_000; + while (!pred()) { + if (Date.now() > deadline) throw new Error(`timed out waiting for ${what}; the listener got:\n${body}`); + await new Promise((r) => { + wake = r; + setTimeout(r, 100); + }); + } + }; + const listener = http.get(`http://127.0.0.1:${port}/global/event`, (res) => { + res.setEncoding("utf8"); + res.on("data", (c: string) => { + body += c; + wake(); + }); + res.on("error", () => undefined); + }); + listener.on("error", () => undefined); + const engineBytes = () => upstream.written.flat().join(""); + try { + await until(() => body.includes("server.connected"), "the connected frame"); + for (const n of [1, 2]) engine.publish(...partEvent(n)); + engine.heartbeat(); + await until(() => body === engineBytes() && body.includes("server.heartbeat"), "the first connection's frames"); + + upstream.drop(); // the link dies mid-stream + await w.waitFor((l) => l.includes("arm=local reason=upstream-error")); + for (const n of [3, 4]) engine.publish(...partEvent(n)); // missed while down + driver.reconcile(); // resume from the last id the listener has + await until(() => upstream.written.length === 2 && body.includes('"n":4'), "the replay"); + engine.publish(...partEvent(5)); + await until(() => body.includes('"n":5'), "the live frame after the replay"); + + expect(upstream.written).toHaveLength(2); + expect(body).toBe(engineBytes()); // byte-identical, ids included, across both connections + const ids = [...body.matchAll(/^id: (.*)$/gm)].map((m) => m[1]); + expect(ids.every((i) => /^bG9jYWwx:\d+$/.test(i))).toBe(true); // the engine's own bare ids — never a composite + expect(body).not.toContain("amicode.sync.gap"); + expect(body).not.toContain("local="); + } finally { + listener.destroy(); + front.closeAllConnections?.(); + await new Promise((r) => front.close(() => r())); + await upstream.stop(); + } + }); +}); From 9c97fb95ccef6474d34507c6e3a9482ee534d4d8 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 20:42:39 -0400 Subject: [PATCH 52/60] =?UTF-8?q?test(fleet):=20a=20reconnecting=20listene?= =?UTF-8?q?r's=20last=20SSE=20id=20=E2=80=94=20bare=20engine=20id=20or=20c?= =?UTF-8?q?omposite=20=E2=80=94=20resumes=20every=20arm=20from=20its=20own?= =?UTF-8?q?=20engine=20id,=20and=20each=20engine=20replays=20(#1777)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../test/sse_fanin_aggregator.test.ts | 16 ++++++++ .../test/sse_fanin_driver_pump.test.ts | 40 +++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/packages/extension/test/sse_fanin_aggregator.test.ts b/packages/extension/test/sse_fanin_aggregator.test.ts index d6079b933..2c29dd316 100644 --- a/packages/extension/test/sse_fanin_aggregator.test.ts +++ b/packages/extension/test/sse_fanin_aggregator.test.ts @@ -175,6 +175,22 @@ describe("#1511 AC4 (D3) — composite cursor", () => { expect(parseCompositeCursor(undefined).size).toBe(0); }); + it("#1777: a bare ENGINE id `:` is the local position (the `:` is not a composite), and a composite of engine ids round-trips", () => { + // the cursor a fleet-of-one client resumes with: the engine's id, verbatim + expect([...parseCompositeCursor(idIn(LOCAL_NAMESPACE, 12))]).toEqual([[LOCAL_NAMESPACE, idIn(LOCAL_NAMESPACE, 12)]]); + // epochs are base64url (`-` and `_` included) — still one bare local id + expect([...parseCompositeCursor("a-b_c9Zx:7")]).toEqual([[LOCAL_NAMESPACE, "a-b_c9Zx:7"]]); + // the cursor a fanned-in client resumes with: every origin's engine id + const composite = `local=${idIn(LOCAL_NAMESPACE, 12)};studio=${idIn("studio", 40)}`; + expect(formatCompositeCursor(parseCompositeCursor(composite))).toBe(composite); + const agg = new SseFanInAggregator({ sink: collectingSink() }); + const resume = agg.connect(composite); + expect(resume.get(LOCAL_NAMESPACE)).toBe(idIn(LOCAL_NAMESPACE, 12)); + expect(resume.get("studio")).toBe(idIn("studio", 40)); + // and it is where each arm resumes from until the arm delivers + expect(agg.cursorFor("studio")).toBe(idIn("studio", 40)); + }); + it("connect() re-subscribes each upstream with ITS OWN resume id", () => { const sink = collectingSink(); const agg = new SseFanInAggregator({ sink }); diff --git a/packages/extension/test/sse_fanin_driver_pump.test.ts b/packages/extension/test/sse_fanin_driver_pump.test.ts index 62d6adb90..170ad7a42 100644 --- a/packages/extension/test/sse_fanin_driver_pump.test.ts +++ b/packages/extension/test/sse_fanin_driver_pump.test.ts @@ -2597,3 +2597,43 @@ describe("#1777 AC3 — fleet-of-one: the listener's stream is byte-identical to } }); }); + +describe("#1777 AC4 — a reconnecting listener resumes with the SSE id it last saw (composite when fanned in)", () => { + it("the last composite id a fanned-in listener saw resumes EVERY arm from its own engine id — each engine replays, no gap", async () => { + const ownerMap = new SessionOwnerMap(); + ownerMap.update(STUDIO_OWNED); + const local = replayingEngine({ epoch: LOCAL_EPOCH }); + const studio = replayingEngine({ epoch: PEER_EPOCH }); + const driver = replayDriver({ local, studio }, ownerMap); + const first = mockRes(); + driver.handle(mockReq("/global/event"), first); + await flush(); + local.publish(...partEvent(1)); + studio.publish(...partEvent(2)); + local.publish(...partEvent(3)); + await flush(); + const lastSeen = sseFrames(first.text()).at(-1)!.id!; // the client's cursor + expect(lastSeen).toBe(`local=${local.idAt(2)};studio=${studio.idAt(1)}`); + first.fireClose(); // the view reloads / its socket drops + + studio.publish(...partEvent(4)); // both origins move on while it is away + local.publish(...partEvent(5)); + const again = mockRes(); + driver.handle(mockReq(`/global/event?lastEventID=${encodeURIComponent(lastSeen)}`), again); + await flush(); + + expect(local.subscriptions.at(-1)).toMatchObject({ cursor: local.idAt(2), reset: false }); + expect(studio.subscriptions.at(-1)).toMatchObject({ cursor: studio.idAt(1), reset: false }); + const frames = sseFrames(again.text()); + expect(frames.filter((f) => f.type === "message.part.updated").map((f) => f.data!.payload.properties.n).sort()).toEqual([4, 5]); + expect(frames.some((f) => f.type === "amicode.sync.gap")).toBe(false); + // every frame's id carries both origins' positions, ending at the newest + expect(parseCompositeCursor(frames.at(-1)!.id)).toEqual( + new Map([ + [LOCAL_NAMESPACE, local.idAt(3)], + ["studio", studio.idAt(2)], + ]), + ); + again.fireClose(); + }); +}); From 56e00206468a569f0f5bcc947d2f58c0d9bfd624 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 20:50:32 -0400 Subject: [PATCH 53/60] =?UTF-8?q?test(fleet):=20two-peer=20E2E=20on=20two?= =?UTF-8?q?=20scratch=20engines=20=E2=80=94=20killing=20the=20peer=20link?= =?UTF-8?q?=20mid-turn=20and=20restoring=20it=20leaves=20the=20observer's?= =?UTF-8?q?=20transcript=20identical=20to=20a=20fresh=20load=20(#1777)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Opt-in (SCRATCH_ENGINE_E2E=1, after build:binary): engines A and B on 4191/4192 with isolated state, a TCP peer link on 4193 that is killed (all sockets destroyed, port closed) and restored mid shell turn, the real SseFanInDriver + reader at the production cadence, and an observer that applies every frame through the app's own stream-cursor. Asserts the peer engine replayed, no gap frame, no resync owed, and transcript == the peer's fresh load. --- .../slow/fleet_link_kill_resume_e2e.test.ts | 344 ++++++++++++++++++ 1 file changed, 344 insertions(+) create mode 100644 packages/extension/test/slow/fleet_link_kill_resume_e2e.test.ts diff --git a/packages/extension/test/slow/fleet_link_kill_resume_e2e.test.ts b/packages/extension/test/slow/fleet_link_kill_resume_e2e.test.ts new file mode 100644 index 000000000..3270947a3 --- /dev/null +++ b/packages/extension/test/slow/fleet_link_kill_resume_e2e.test.ts @@ -0,0 +1,344 @@ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { spawn, type ChildProcess } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import http from "node:http"; +import net from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { SseFanInDriver } from "../../src/amicode_service/sse_fanin_driver"; +import { SessionOwnerMap } from "../../src/amicode_service/session_multiplexer"; +import { FanInCensus } from "../../src/amicode_service/fanin_census"; +import { parseCompositeCursor } from "../../src/amicode_service/sse_composite_cursor"; +// The CLIENT's own resume cursor + contiguity check (the app overlay's pure +// module the webview's stream loop runs) — the observer applies every frame +// through it, exactly as the app does. +import { createStreamCursor } from "../../../app-bundle/overlay/packages/app/src/context/stream-cursor"; + +// ============================================================================ +// #1777 AC6 — TWO-PEER FLEET E2E WITH A LINK KILL, against two REAL compiled +// engines (the #1776 sequenced, replaying global stream). +// +// engine A (:4191) ── the observer's local engine ──────────┐ +// engine B (:4192) ── the owner peer ── link (:4193, TCP) ──┤ fan-in (in-test, +// │ SseFanInDriver + +// observer (SSE client, the app's │ the real reader) +// stream-cursor) ◀── /global/event ──────┘ +// +// A turn runs on B (a shell tool streaming output: user message, assistant +// message, a running tool part updated per chunk, completion). MID-TURN the +// link is KILLED — every proxied socket destroyed and the port closed (a dead +// SSH forward: connection refused) — and later RESTORED. The fan-in's peer arm +// fails, its re-opens are refused while the link is down, then one resumes from +// the last delivered engine id and B replays what the observer missed. After +// the turn the observer's transcript must equal a fresh load from B, with no +// gap frame and no resync owed (no hole, no reset). +// +// Opt-in: SCRATCH_ENGINE_E2E=1, after `pnpm --filter amicode run build:binary` +// (not an AMICODE_* name: test/setup_hermetic_env.ts deletes those before this +// module's gate is read). Both engines run the vendored binary of THIS checkout +// on spare ports in 4190–4199 (never a hub port), each with every state dir and +// its session DB in its own temp dir, and are killed afterwards. Live hubs and +// the live peer link are never touched. +// ============================================================================ + +const ENABLED = process.env.SCRATCH_ENGINE_E2E === "1"; +const BIN = + process.env.SCRATCH_ENGINE_BIN ?? join(__dirname, "..", "..", "vendor", "opencode", `${process.platform}-${process.arch}`, "opencode"); +const PORT_A = Number(process.env.SCRATCH_ENGINE_PORT_A ?? 4191); +const PORT_B = Number(process.env.SCRATCH_ENGINE_PORT_B ?? 4192); +const PORT_LINK = Number(process.env.SCRATCH_LINK_PORT ?? 4193); + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); +async function until(probe: () => T | undefined | false, what: string, ms = 15_000): Promise { + const deadline = Date.now() + ms; + for (;;) { + const value = probe(); + if (value !== undefined && value !== false) return value; + if (Date.now() > deadline) throw new Error(`timed out waiting for ${what}`); + await sleep(20); + } +} + +interface Engine { + proc: ChildProcess; + dir: string; + url: string; + log(): string; +} + +async function startEngine(port: number, name: string): Promise { + if (!(port >= 4190 && port <= 4199)) throw new Error(`scratch port ${port} outside 4190-4199`); + const dir = mkdtempSync(join(tmpdir(), `amicode-scratch-${name}-`)); + const proj = join(dir, "proj"); + mkdirSync(proj); + mkdirSync(join(dir, "home")); + const env: Record = {}; + for (const [k, v] of Object.entries(process.env)) if (v !== undefined && !/^(OPENCODE|AMICO)/.test(k)) env[k] = v; + Object.assign(env, { + HOME: join(dir, "home"), + XDG_DATA_HOME: join(dir, "data"), + XDG_CONFIG_HOME: join(dir, "config"), + XDG_STATE_HOME: join(dir, "state"), + XDG_CACHE_HOME: join(dir, "cache"), + OPENCODE_DB: join(dir, "db.sqlite"), + }); + let log = ""; + const proc = spawn(BIN, ["serve", "--port", String(port), "--hostname", "127.0.0.1", "--print-logs", "--log-level", "INFO", "--pure"], { + cwd: proj, + env, + stdio: ["ignore", "pipe", "pipe"], + }); + proc.stdout?.on("data", (d: Buffer) => (log += d)); + proc.stderr?.on("data", (d: Buffer) => (log += d)); + await until(() => log.includes(`listening on http://127.0.0.1:${port}`) || proc.exitCode !== null || undefined, `engine ${name} listen`, 60_000); + if (proc.exitCode !== null) throw new Error(`engine ${name} exited early:\n${log}`); + return { proc, dir, url: `http://127.0.0.1:${port}`, log: () => log }; +} + +async function stopEngine(e: Engine | undefined): Promise { + if (!e) return; + e.proc.kill("SIGTERM"); + for (let i = 0; i < 40 && e.proc.exitCode === null && e.proc.signalCode === null; i++) await sleep(50); + if (e.proc.exitCode === null && e.proc.signalCode === null) e.proc.kill("SIGKILL"); + rmSync(e.dir, { recursive: true, force: true }); +} + +/** The peer link: a TCP forward 127.0.0.1:PORT_LINK → engine B. `kill()` is a + * dead SSH forward — every proxied socket destroyed, the port closed + * (connection refused); `restore()` listens again. */ +function peerLink(target: number) { + const sockets = new Set(); + let server: net.Server | undefined; + let accepted = 0; + const listen = () => + new Promise((resolve, reject) => { + server = net.createServer((client) => { + accepted += 1; + const upstream = net.connect(target, "127.0.0.1"); + sockets.add(client); + sockets.add(upstream); + const drop = () => { + client.destroy(); + upstream.destroy(); + sockets.delete(client); + sockets.delete(upstream); + }; + client.on("error", drop).on("close", drop); + upstream.on("error", drop).on("close", drop); + client.pipe(upstream); + upstream.pipe(client); + }); + server.once("error", reject); + server.listen(PORT_LINK, "127.0.0.1", () => resolve()); + }); + return { + url: `http://127.0.0.1:${PORT_LINK}`, + restore: listen, + kill: () => + new Promise((resolve) => { + for (const s of sockets) s.destroy(); + sockets.clear(); + if (!server) return resolve(); + server.close(() => resolve()); + server = undefined; + }), + accepted: () => accepted, + }; +} + +async function post(url: string, body: unknown): Promise { + const res = await fetch(url, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) }); + if (!res.ok) throw new Error(`POST ${url} → ${res.status} ${await res.text()}`); + return res.json(); +} + +// ── the transcript a viewer holds: messages and parts of ONE session ──────── + +type Json = Record; +type Transcript = Map }>; + +function applyEvent(t: Transcript, sessionID: string, type: string, p: Json): void { + const msg = (id: string) => { + let m = t.get(id); + if (!m) t.set(id, (m = { parts: new Map() })); + return m; + }; + if (type === "message.updated" && p.info?.sessionID === sessionID) msg(p.info.id).info = p.info; + else if (type === "message.removed" && p.sessionID === sessionID) t.delete(p.messageID); + else if (type === "message.part.updated" && p.part?.sessionID === sessionID) msg(p.part.messageID).parts.set(p.part.id, p.part); + else if (type === "message.part.removed" && p.sessionID === sessionID) t.get(p.messageID)?.parts.delete(p.partID); + else if (type === "message.part.delta" && p.sessionID === sessionID) { + const part = t.get(p.messageID)?.parts.get(p.partID); + if (part) part[p.field] = String(part[p.field] ?? "") + p.delta; + } +} + +/** A comparable transcript. The wire drops summary patch bodies (#1778 — the + * stored record keeps them), so they are dropped from both sides. */ +function canonical(messages: Array<{ info?: Json; parts: Json[] }>): unknown { + const strip = (info: Json | undefined) => { + if (!info?.summary?.diffs) return info; + return { ...info, summary: { ...info.summary, diffs: info.summary.diffs.map(({ patch: _p, ...rest }: Json) => rest) } }; + }; + return [...messages] + .sort((a, b) => String(a.info?.id).localeCompare(String(b.info?.id))) + .map((m) => ({ info: strip(m.info), parts: [...m.parts].sort((a, b) => String(a.id).localeCompare(String(b.id))) })); +} + +describe.skipIf(!ENABLED)("#1777 AC6 — two-peer fleet E2E: a link kill mid-turn loses nothing (two scratch engines)", () => { + let a: Engine | undefined; + let b: Engine | undefined; + + beforeAll(async () => { + if (!existsSync(BIN)) throw new Error(`no compiled engine at ${BIN} — run build:binary first`); + if (!(PORT_LINK >= 4190 && PORT_LINK <= 4199)) throw new Error(`link port ${PORT_LINK} outside 4190-4199`); + [a, b] = await Promise.all([startEngine(PORT_A, "a"), startEngine(PORT_B, "b")]); + }, 120_000); + + afterAll(async () => { + await Promise.all([stopEngine(a), stopEngine(b)]); + }); + + it("killing the peer link mid-turn and restoring it leaves the observer's transcript identical to a fresh load", async () => { + const link = peerLink(PORT_B); + await link.restore(); + const ownerMap = new SessionOwnerMap(); + const census: string[] = []; + const driver = new SseFanInDriver({ + ownerMap, + localMachineId: "observer", + localEventUrl: () => a!.url, + peerBaseUrl: (id) => (id === "studio" ? link.url : undefined), + peerToken: (id) => + id === "studio" + ? { ok: true as const, credential: { baseUrl: link.url, token: "scratch-peer-token" } } + : { ok: false as const, reason: "absent" as const }, + reconcileMs: 200, // the production cadence drives every re-open + census: new FanInCensus({ plane: "observe", sink: (l) => census.push(l) }), + }); + const front = http.createServer((req, res) => void driver.handle(req, res)); + await new Promise((r) => front.listen(0, "127.0.0.1", r)); + const frontUrl = `http://127.0.0.1:${(front.address() as net.AddressInfo).port}`; + + // ── the observer: the app's cursor applied to every frame ──────────────── + const cursor = createStreamCursor(); + const transcript: Transcript = new Map(); + const seen: Array<{ id?: string; type?: string }> = []; + const resyncs: unknown[] = []; + const gaps: unknown[] = []; + let duplicates = 0; + let sessionID = ""; + const pendingBeforeSession: Array<{ type: string; p: Json }> = []; + let buffer = ""; + const observer = http.get(`${frontUrl}/global/event`, (res) => { + res.setEncoding("utf8"); + res.on("data", (chunk: string) => { + buffer += chunk; + let end: number; + while ((end = buffer.indexOf("\n\n")) >= 0) { + const block = buffer.slice(0, end); + buffer = buffer.slice(end + 2); + const id = /^id: (.*)$/m.exec(block)?.[1]; + const dataLine = block.split("\n").find((l) => l.startsWith("data:")); + if (!dataLine) continue; + const data = JSON.parse(dataLine.slice(5).trim()) as Json; + const type: string = data.payload?.type ?? data.type; + seen.push({ id, type }); + if (type === "amicode.sync.gap") { + gaps.push(data); + cursor.gap(data.namespaces ?? []); + continue; + } + const verdict = cursor.frame({ id, type }); + if (verdict.resync) resyncs.push(verdict.resync); + if (!verdict.apply) { + duplicates += 1; + continue; + } + const props = (data.payload?.properties ?? {}) as Json; + if (sessionID) applyEvent(transcript, sessionID, type, props); + else pendingBeforeSession.push({ type, p: props }); + } + }); + res.on("error", () => undefined); + }); + observer.on("error", () => undefined); + + try { + await until(() => seen.some((f) => f.type === "server.connected"), "the observer's stream"); + + // a session on the PEER, owned by it; the fan-in opens the peer arm + const session = (await post(`${b!.url}/session`, {})) as { id: string }; + sessionID = session.id; + for (const e of pendingBeforeSession) applyEvent(transcript, sessionID, e.type, e.p); + ownerMap.update([{ id: sessionID, amicode_owner: { owner_machine_id: "studio", owner_name: "studio", is_local: false } }]); + await until(() => seen.some((f) => f.type === "server.connected" && parseCompositeCursor(f.id).has("studio")), "the peer arm"); + + // ── the turn: a shell tool streaming output for ~3 s ──────────────────── + const ticks = Array.from({ length: 30 }, (_, i) => `t${i}`).join(" "); + const turn = post(`${b!.url}/session/${sessionID}/shell`, { + agent: "build", + model: { providerID: "scratch", modelID: "scratch" }, + command: `for x in ${ticks}; do echo tick; sleep 0.1; done`, + }); + const runningPart = () => + [...transcript.values()].flatMap((m) => [...m.parts.values()]).find((p) => p.type === "tool" && p.state?.status === "running" && String(p.state?.metadata?.output ?? "").includes("tick")); + await until(() => runningPart(), "the running tool part with output"); + + // ── KILL the link mid-turn ────────────────────────────────────────────── + const killedAt = seen.length; + await link.kill(); + await until(() => census.some((l) => l.includes("arm-closed") && l.includes("arm=studio") && l.includes("reason=upstream-")), "the peer arm to fail"); + await sleep(800); // the turn goes on while the link is down; re-opens are refused + const framesWhileDown = seen.length - killedAt; + const reopensRefused = census.filter((l) => l.includes("arm=studio") && l.includes("arm-closed")).length; + expect(runningPart()).toBeDefined(); // still mid-turn: the completion has not reached the observer + + // ── RESTORE it ────────────────────────────────────────────────────────── + await link.restore(); + await turn; // the turn completes on the peer + // the observer converges on the fresh load (it keeps streaming; poll the owner) + const freshLoad = async () => canonical((await (await fetch(`${b!.url}/session/${sessionID}/message`)).json()) as Array<{ info: Json; parts: Json[] }>); + const observed = () => canonical([...transcript.values()].map((m) => ({ info: m.info, parts: [...m.parts.values()] }))); + let fresh: unknown = await freshLoad(); + const deadline = Date.now() + 15_000; + while (JSON.stringify(observed()) !== JSON.stringify(fresh) && Date.now() < deadline) { + await sleep(100); + fresh = await freshLoad(); + } + + // the replay that recovered it, as the peer engine logged it + const replayLines = b!.log().split("\n").filter((l) => l.includes("global event connected") && l.includes("resume=replay")); + const replayed = replayLines.map((l) => Number(/replayed=(\d+)/.exec(l)?.[1] ?? 0)); + console.log( + "[#1777 AC6]", + JSON.stringify({ + observerFrames: seen.length, + framesWhileDown, + studioArmClosesWhileDown: reopensRefused, + linkConnections: link.accepted(), + peerReplays: replayed, + gaps: gaps.length, + resyncs, + duplicatesDropped: duplicates, + transcriptMessages: transcript.size, + cursor: cursor.current(), + }), + ); + + expect(observed()).toEqual(fresh); // identical to a fresh load + const parts = [...transcript.values()].flatMap((m) => [...m.parts.values()]); + expect(parts.find((p) => p.type === "tool")?.state?.status).toBe("completed"); + expect(gaps).toEqual([]); // no gap signal … + expect(resyncs).toEqual([]); // … and no hole or reset: nothing needed a resync + expect(census.some((l) => l.includes("gap-emitted"))).toBe(false); + expect(replayed.some((n) => n > 0)).toBe(true); // the peer REPLAYED the missed stretch + expect(parseCompositeCursor(cursor.current()).get("studio")).toMatch(/^[A-Za-z0-9_-]+:\d+$/); // the observer's cursor holds the peer's engine id + } finally { + observer.destroy(); + front.closeAllConnections?.(); + await new Promise((r) => front.close(() => r())); + await link.kill(); + } + }, 90_000); +}); From e9756d6ea697598e0b22e1cf5f8afda849cd9b01 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 20:51:23 -0400 Subject: [PATCH 54/60] docs(fleet): fan-in comments name the resume path and say what an overflow drop still costs (#1777) --- .../amicode_service/sse_fanin_aggregator.ts | 4 ++-- .../src/amicode_service/sse_fanin_driver.ts | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/extension/src/amicode_service/sse_fanin_aggregator.ts b/packages/extension/src/amicode_service/sse_fanin_aggregator.ts index df12d69c4..5e6caa966 100644 --- a/packages/extension/src/amicode_service/sse_fanin_aggregator.ts +++ b/packages/extension/src/amicode_service/sse_fanin_aggregator.ts @@ -38,8 +38,8 @@ export { LOCAL_NAMESPACE, NS_SEP } from "./sse_composite_cursor"; * #1617 — `pause()` / `resume()` are OPTIONAL flow control the driver drives * when the shared downstream backpressures: pausing every live upstream while * the downstream is stalled is what keeps the bounded per-namespace buffers - * from overflow-dropping (the overflow is otherwise UNRECOVERABLE — the live - * global event route has no replay cursor). A source that cannot pause simply + * from overflow-dropping (an overflow drop is never replayed — the arm stays + * open — so it can only be announced, by the overflow gap). A source that cannot pause simply * omits them; the aggregator falls back to buffer-and-drain (today's behavior * for injected test sources) — additive, no existing caller breaks. */ export interface SseFrameSource { diff --git a/packages/extension/src/amicode_service/sse_fanin_driver.ts b/packages/extension/src/amicode_service/sse_fanin_driver.ts index 88a490cc5..985778560 100644 --- a/packages/extension/src/amicode_service/sse_fanin_driver.ts +++ b/packages/extension/src/amicode_service/sse_fanin_driver.ts @@ -220,9 +220,9 @@ class FanInConnection implements CensusConnection { ...(deps.focusSnapshot ? { focusSnapshot: deps.focusSnapshot } : {}), // #1617 — re-derive the transparent proxy's backpressure by hand. When the // shared downstream `write()` returns false, pause EVERY live upstream so - // the bounded per-namespace buffers cannot overflow-drop (the overflow is - // otherwise unrecoverable — the live global event route has no replay - // cursor). The `drain` edge (wired in start()) resumes them and calls + // the bounded per-namespace buffers cannot overflow-drop (an overflow drop + // is not replayed — the arm stays open — so it can only be announced, by + // the overflow gap). The `drain` edge (wired in start()) resumes them and calls // agg.resume() to flush the buffers in order and re-arm `flowing`. onBackpressure: () => this.pauseUpstreams(), // #1771 — the census clock and its view of drops and gaps. @@ -361,12 +361,12 @@ class FanInConnection implements CensusConnection { // Open a real upstream for each newly-active peer arm. for (const machineId of active) { if (this.sources.has(machineId)) continue; - // #1638 — resume from the aggregator's LIVE last-delivered id for this - // namespace (its composite cursor), NOT the stale connect-time id. A - // re-open of an arm that already delivered is a NON-INITIAL re-open — the - // aggregator emits one composite-mode gap (reopenArm is a no-op for a - // never-delivered arm; a peer arm is only active in composite mode, so - // its verdict is never `sever`). + // #1638 — resume from the aggregator's LIVE resume position for this + // namespace, NOT the stale connect-time id. #1777: a re-open of an arm + // that delivered engine ids RESUMES there and the peer replays; one with + // no ids (an older engine) gets the composite-mode gap instead + // (reopenArm is a no-op for a never-delivered arm; a peer arm is only + // active in composite mode, so its verdict is never `sever`). this.reopenAndTrack(machineId, this.deps.peerBaseUrl(machineId), this.peerAuth(machineId)); } // Close the real upstream for any peer arm no longer active (dark peer, or From 22c4ad7e2b92d2b931db523cf944c77585b42e66 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 21:33:08 -0400 Subject: [PATCH 55/60] =?UTF-8?q?perf(fleet):=20the=20relay=20splits=20SSE?= =?UTF-8?q?=20frames=20in=20linear=20time=20=E2=80=94=20only=20the=20new?= =?UTF-8?q?=20chunk=20is=20scanned=20(#1779)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/amicode_service/sse_fanin_driver.ts | 52 ++++++++++++-- .../test/sse_fanin_driver_pump.test.ts | 72 ++++++++++++++++++- 2 files changed, 116 insertions(+), 8 deletions(-) diff --git a/packages/extension/src/amicode_service/sse_fanin_driver.ts b/packages/extension/src/amicode_service/sse_fanin_driver.ts index 985778560..d3fe6b8e8 100644 --- a/packages/extension/src/amicode_service/sse_fanin_driver.ts +++ b/packages/extension/src/amicode_service/sse_fanin_driver.ts @@ -99,6 +99,49 @@ export const FANIN_LISTENER_CAP_RETRY_AFTER_S = 5; /** #1772 — the named error on a beyond-cap refusal. */ export const FANIN_LISTENER_CAP_ERROR = "fan-in listener cap reached"; +/** The relay reader's frame splitter: decoded upstream text in, WHOLE frames out + * (delimiter-inclusive, verbatim). #1779: LINEAR in the text received. The old + * reader appended every chunk to the whole partial buffer and searched it from + * the start, so a multi-MB frame arriving in 64 KB chunks cost quadratic time. + * Now only the NEW chunk is scanned; a delimiter split across two chunks is + * caught by looking back one character into the partial frame, which is held + * as pieces and joined ONCE, when it completes. */ +export class SseFrameSplitter { + /** The partial frame, as received (never re-copied while it grows). */ + private parts: string[] = []; + /** The partial frame's last character — the lookback for a split delimiter. */ + private tail = ""; + + /** Feed one decoded chunk; returns the frames it completed, in order. */ + push(chunk: string): string[] { + const frames: string[] = []; + if (chunk === "") return frames; + const tail = this.tail; + let start = 0; // where the partial frame resumes in this chunk + let from = 0; + let lf: number; + while ((lf = chunk.indexOf("\n", from)) !== -1) { + from = lf + 1; + // the line feed ends a blank line iff the character before it — in this + // chunk, or the partial frame's tail when it is the chunk's first — is one too + const prev = lf > start ? chunk.charCodeAt(lf - 1) : lf === 0 ? tail.charCodeAt(0) : NaN; + if (prev !== LF_CODE) continue; + this.parts.push(chunk.slice(start, from)); + frames.push(this.parts.length === 1 ? this.parts[0] : this.parts.join("")); // whole frame, delimiter-inclusive + this.parts = []; + start = from; + } + if (start < chunk.length) { + this.parts.push(start === 0 ? chunk : chunk.slice(start)); + this.tail = chunk.charAt(chunk.length - 1); + } else { + this.tail = ""; // a frame ended exactly at the chunk's end + } + return frames; + } +} +const LF_CODE = 10; + /** The production upstream reader: a frame-preserving http SSE source. Splits the * byte stream on the SSE blank-line delimiter into WHOLE frames (delimiter- * inclusive) so `event:`/`data:`/`retry:`/`id:` lines survive intact for the @@ -140,16 +183,11 @@ export function openHttpSseSource(r: FanInUpstreamRequest): SseFrameSource { end("error"); return; } - let buf = ""; + const splitter = new SseFrameSplitter(); res.setEncoding("utf8"); res.on("data", (chunk: string) => { if (ended) return; - buf += chunk; - let idx: number; - while ((idx = buf.indexOf("\n\n")) >= 0) { - pushFrame(buf.slice(0, idx + 2)); // whole frame, delimiter-inclusive - buf = buf.slice(idx + 2); - } + for (const frame of splitter.push(chunk)) pushFrame(frame); }); res.on("end", () => end("end")); res.on("error", () => end("error")); diff --git a/packages/extension/test/sse_fanin_driver_pump.test.ts b/packages/extension/test/sse_fanin_driver_pump.test.ts index 170ad7a42..7548bb15b 100644 --- a/packages/extension/test/sse_fanin_driver_pump.test.ts +++ b/packages/extension/test/sse_fanin_driver_pump.test.ts @@ -5,7 +5,9 @@ // reconnection indefinitely. import { describe, it, expect, vi, afterEach } from "vitest"; import * as http from "node:http"; -import { SseFanInDriver, openHttpSseSource } from "../src/amicode_service/sse_fanin_driver"; +import { appendFileSync } from "node:fs"; +import { StringDecoder } from "node:string_decoder"; +import { SseFanInDriver, SseFrameSplitter, openHttpSseSource } from "../src/amicode_service/sse_fanin_driver"; import type { SseFrameSource } from "../src/amicode_service/sse_fanin_aggregator"; import { LOCAL_NAMESPACE } from "../src/amicode_service/sse_fanin_aggregator"; import { SessionOwnerMap } from "../src/amicode_service/session_multiplexer"; @@ -2245,6 +2247,74 @@ describe("#1771 AC6 — a 10 MB engine frame delivered in 64 KB chunks (the #177 }); }); +// ══════════════════════════════════════════════════════════════════════════════ +// #1779 — LINEAR-TIME FRAME SPLITTING in the relay's upstream reader. The reader +// (`openHttpSseSource`) decodes each network chunk (`res.setEncoding("utf8")`, a +// StringDecoder: a code point split across chunks is held back, never mangled) +// and hands the text to its frame splitter. `readFrames` is that data path minus +// the socket, so a test can choose the chunk boundaries exactly — the network +// cannot deliver a 10 MB frame in ONE chunk, which is the baseline AC1 compares to. +// ══════════════════════════════════════════════════════════════════════════════ + +/** The relay reader's data path: decode each byte chunk as `setEncoding("utf8")` + * does, split the text into whole frames. */ +function readFrames(chunks: Buffer[]): string[] { + const decoder = new StringDecoder("utf8"); + const splitter = new SseFrameSplitter(); + const frames: string[] = []; + for (const chunk of chunks) { + const text = decoder.write(chunk); + if (text !== "") frames.push(...splitter.push(text)); + } + return frames; +} + +const median = (xs: number[]): number => [...xs].sort((a, b) => a - b)[Math.floor(xs.length / 2)]; +const elapsedMs = (fn: () => unknown): number => { + const t0 = performance.now(); + fn(); + return performance.now() - t0; +}; + +/** #1779 AC1's assertion, made robust: a warm-up, then INTERLEAVED trials (both + * deliveries see the same machine noise), the median of each, and a generous + * factor. A quadratic splitter misses it by an order of magnitude. `SPLIT_RATIO_LOG` + * (a file path) records each run's ratio for a flake report. */ +function splitTimeRatio(read: (chunks: Buffer[]) => unknown, whole: Buffer[], chunked: Buffer[]): { ratio: number; wholeMs: number; chunkedMs: number } { + for (let i = 0; i < 2; i++) { + read(whole); + read(chunked); + } + const one: number[] = []; + const many: number[] = []; + for (let i = 0; i < 7; i++) { + one.push(elapsedMs(() => read(whole))); + many.push(elapsedMs(() => read(chunked))); + } + const wholeMs = median(one); + const chunkedMs = median(many); + return { ratio: chunkedMs / wholeMs, wholeMs, chunkedMs }; +} +const SPLIT_RATIO_FACTOR = 3; + +describe("#1779 AC1 — the relay reader splits a 10 MB frame in 64 KB chunks within a small constant factor of one chunk", () => { + it("chunked ÷ single-chunk split time (medians) stays ≤ 3×, and both yield the one frame byte-identical", () => { + const big = largeSummaryFrame({ epoch: LOCAL_EPOCH, seq: 1 }); + const whole = [Buffer.from(big, "utf8")]; + const chunked = chunkBytes(big, CHUNK_64K); + expect(Buffer.byteLength(big)).toBeGreaterThanOrEqual(TEN_MB); + expect(chunked.length).toBeGreaterThan(160); + for (const delivery of [whole, chunked]) { + const frames = readFrames(delivery); + expect(frames).toHaveLength(1); + expect(frames[0] === big).toBe(true); // byte-identical (#1264); no 10 MB diff on failure + } + const { ratio, wholeMs, chunkedMs } = splitTimeRatio(readFrames, whole, chunked); + if (process.env.SPLIT_RATIO_LOG) appendFileSync(process.env.SPLIT_RATIO_LOG, `relay ${ratio.toFixed(3)} ${wholeMs.toFixed(2)} ${chunkedMs.toFixed(2)}\n`); + expect(ratio, `chunked ${chunkedMs.toFixed(1)} ms vs whole ${wholeMs.toFixed(1)} ms`).toBeLessThanOrEqual(SPLIT_RATIO_FACTOR); + }, 60_000); +}); + // ══════════════════════════════════════════════════════════════════════════════ // #1777 — LOSSLESS RESUME. The engine (#1776) sequences and replays its global // stream; the fan-in now uses it. A re-opened arm resumes from its last delivered From da8344207c217ab16ac0e63474a41102f4fb38e8 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 21:35:31 -0400 Subject: [PATCH 56/60] =?UTF-8?q?perf(app):=20the=20global-stream=20reader?= =?UTF-8?q?=20splits=20SSE=20frames=20in=20linear=20time=20=E2=80=94=20onl?= =?UTF-8?q?y=20new=20text=20is=20normalized=20and=20searched=20(#1779)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/app-bundle/manifest.json | 12 +- .../app/src/context/sse-reader.test.ts | 152 ++++++++++ .../src/v2/gen/core/serverSentEvents.gen.ts | 268 ++++++++++++++++++ 3 files changed, 428 insertions(+), 4 deletions(-) create mode 100644 packages/app-bundle/overlay/packages/app/src/context/sse-reader.test.ts create mode 100644 packages/app-bundle/overlay/packages/sdk/js/src/v2/gen/core/serverSentEvents.gen.ts diff --git a/packages/app-bundle/manifest.json b/packages/app-bundle/manifest.json index 2df3b1f68..fd9ce9f46 100644 --- a/packages/app-bundle/manifest.json +++ b/packages/app-bundle/manifest.json @@ -204,6 +204,7 @@ "packages/app/src/context/sse-gap-frame.ts": "2f64fde540a305579762c5276721d4bbfbf7c862131aaf4462e8a4e8f32c5960", "packages/app/src/context/sse-origin.test.ts": "71cd3aa996a51d7e99236a49cede4769e520e20e53f57c6920fe441f2abb0c7d", "packages/app/src/context/sse-origin.ts": "da9a231907dc50d87cf01758177bee0d4a0b0165d8adb5e152db8eca29dfd9f5", + "packages/app/src/context/sse-reader.test.ts": "a6151ded71f97e911a84be946f43e5c20495512a4ff79361d7e5673d457dd0ca", "packages/app/src/context/stream-cursor.test.ts": "c9715bb28f576cdc00683b0bdc27bdc2a56a6f212969cc090f99b3dc8988e6a7", "packages/app/src/context/stream-cursor.ts": "2711f85581cd86bb69b50a441b6a04c532553f61fa5e00cd1b9daa12193d3697", "packages/app/src/context/stream-gap.test.ts": "b7203445564f44fd1d966ef684a11cb9b5ec567d9ab54f1b7617726fdf44d3f5", @@ -935,6 +936,7 @@ "packages/session-ui/src/v2/components/prompt-input/attachments.ts": "ff66078f2edc9a67a4df0d7dfd76a49be5eab27ac332b303f3412721ef7268ea", "packages/session-ui/src/v2/components/prompt-input/index.tsx": "194b1c6d0c99000579df78b47c7ff72fe34f62a92ecd21b2a9377133d6d27bb9", "packages/opencode/src/server/routes/instance/httpapi/server.ts": "a94becd6a5224fb5fc2778c1091733eca334b308253234948bcdcafdc709b021", + "packages/sdk/js/src/v2/gen/core/serverSentEvents.gen.ts": "7509e74c8ccd756bbe60598f38754775a62c36e82d8f5f59464fb18a49be4688", "packages/opencode/src/server/routes/instance/httpapi/groups/experimental.ts": "944a0aee215a917e867fd9482f3c2bd40f49082bd3174c8a7d967ca40e34ec15", "packages/opencode/src/server/routes/instance/httpapi/groups/file.ts": "a96fdc55ddb95ca7c481c86f67e37ed88d4cf2a190a482765ab55ce4409f3a59", "packages/opencode/src/server/routes/instance/httpapi/groups/mcp.ts": "048efa7b74d43c4ad4a75d0f8421c808760c8c0dcd593f0051a87044e40487d6", @@ -961,10 +963,10 @@ "packages/ui/src/v2/components/text-shimmer-v2.tsx" ], "exceptions": [], - "extracted_at": "2026-10-13T00:39:30.584Z", + "extracted_at": "2026-10-13T01:35:20.070Z", "per_package": { "packages/app": { - "A": 239, + "A": 240, "M": 161, "D": 0 }, @@ -990,7 +992,7 @@ }, "packages/sdk": { "A": 0, - "M": 3, + "M": 4, "D": 0 }, "packages/server": { @@ -1010,7 +1012,7 @@ } }, "counts": { - "overlay_total": 939, + "overlay_total": 941, "deletions": 7, "server_coupled": 0 }, @@ -1211,6 +1213,7 @@ "packages/app/src/context/sse-gap-frame.ts": "A", "packages/app/src/context/sse-origin.test.ts": "A", "packages/app/src/context/sse-origin.ts": "A", + "packages/app/src/context/sse-reader.test.ts": "A", "packages/app/src/context/stream-cursor.test.ts": "A", "packages/app/src/context/stream-cursor.ts": "A", "packages/app/src/context/stream-gap.test.ts": "A", @@ -1942,6 +1945,7 @@ "packages/session-ui/src/v2/components/prompt-input/attachments.ts": "M", "packages/session-ui/src/v2/components/prompt-input/index.tsx": "M", "packages/opencode/src/server/routes/instance/httpapi/server.ts": "M", + "packages/sdk/js/src/v2/gen/core/serverSentEvents.gen.ts": "M", "packages/opencode/src/server/routes/instance/httpapi/groups/experimental.ts": "M", "packages/opencode/src/server/routes/instance/httpapi/groups/file.ts": "M", "packages/opencode/src/server/routes/instance/httpapi/groups/mcp.ts": "M", diff --git a/packages/app-bundle/overlay/packages/app/src/context/sse-reader.test.ts b/packages/app-bundle/overlay/packages/app/src/context/sse-reader.test.ts new file mode 100644 index 000000000..dac409b97 --- /dev/null +++ b/packages/app-bundle/overlay/packages/app/src/context/sse-reader.test.ts @@ -0,0 +1,152 @@ +import { appendFileSync } from "node:fs" +import { describe, expect, test } from "bun:test" +import { createSdkForServer } from "@/utils/server" + +// #1779 — the app's global-stream READER: the SDK's SSE reader +// (`@opencode-ai/sdk/v2` core/serverSentEvents.gen.ts) that server-sdk.tsx reads +// the global stream through, driven exactly as the app drives it — +// `createSdkForServer(...).global.event({ onSseEvent })` — with a fetch that +// answers with chosen byte chunks, so a test can place every chunk boundary. +// (Each body chunk is one read: TextDecoderStream decodes chunk by chunk and +// holds back a code point split across two.) + +const SERVER = { url: "http://127.0.0.1:4096" } + +/** What one parsed frame reports through `onSseEvent` (the app reads `id`). */ +type Frame = { data: unknown; event?: string; id?: string; retry?: number } + +/** A fetch that answers the n-th request with the n-th delivery's chunks, one + * body read per chunk. A delivery with `fail` errors its body after its chunks + * (a dropped link). Every request is recorded. */ +function streamFetch(deliveries: Array<{ chunks: Uint8Array[]; fail?: boolean }>, requests: Request[] = []) { + return async (input: RequestInfo | URL, init?: RequestInit) => { + const request = input instanceof Request ? input : new Request(input, init) + const delivery = deliveries[Math.min(requests.length, deliveries.length - 1)] + requests.push(request) + let next = 0 + const body = new ReadableStream({ + pull(controller) { + if (next < delivery.chunks.length) controller.enqueue(delivery.chunks[next++]) + else if (delivery.fail) controller.error(new Error("link dropped")) + else controller.close() + }, + }) + return new Response(body, { headers: { "content-type": "text/event-stream" } }) + } +} + +/** Read the global stream to its end: every frame `onSseEvent` reported, and + * every event the stream yielded, in order. */ +async function readStream( + deliveries: Array<{ chunks: Uint8Array[]; fail?: boolean }>, + requests: Request[] = [], +): Promise<{ frames: Frame[]; events: unknown[] }> { + const sdk = createSdkForServer({ server: SERVER, fetch: streamFetch(deliveries, requests) as typeof fetch }) + const frames: Frame[] = [] + const result = await sdk.global.event({ + onSseEvent: (frame: Frame) => frames.push({ ...frame }), + sseSleepFn: async () => {}, + }) + const events: unknown[] = [] + for await (const event of result.stream) events.push(event) + return { frames, events } +} + +const encoder = new TextEncoder() +const bytes = (text: string) => encoder.encode(text) + +// ── #1771's chunked large-frame fixture ───────────────────────────────────── +// Ported verbatim from the extension's engine-shaped fixtures +// (test/fixtures/engine_stream/engine_frames.ts `largeSummaryFrame` / +// `chunkBytes`) — the app suite cannot import them: a `message.updated` summary +// frame whose `info.summary.diffs[].patch` text brings it to ≥ 10 MB of UTF-8, +// with non-ASCII text so a 64 KB byte boundary can fall inside a code point. + +const TEN_MB = 10 * 1024 * 1024 +const CHUNK_64K = 64 * 1024 +const EPOCH = "bG9jYWwx" + +function largeSummaryFrame(seq: number, targetBytes = TEN_MB): { frame: string; event: unknown } { + const lines: string[] = [] + for (let i = 0; i < 1400; i++) lines.push(`+ const row${i} = "résumé → ${"x".repeat(60)}";`) + const patch = `@@ -1,0 +1,1400 @@\n${lines.join("\n")}\n` + const perDiff = encoder.encode(JSON.stringify(patch)).length + 96 + const count = Math.max(1, Math.ceil(targetBytes / perDiff)) + const diffs = Array.from({ length: count }, (_, i) => ({ + file: `src/module_${i}.ts`, + patch, + additions: 1400, + deletions: 0, + status: "modified", + })) + const event = { + directory: "/Users/jj/harmoniqs/amicode", + project: "prj_0fixture", + payload: { + id: "evt_summary", + type: "message.updated", + properties: { info: { id: "msg_summary", sessionID: "ses_large", role: "user", summary: { diffs } } }, + }, + } + return { frame: `id: ${EPOCH}:${seq}\ndata: ${JSON.stringify(event)}\n\n`, event } +} + +/** Split text's UTF-8 BYTES into `size` pieces — how the network delivers it. */ +function chunkBytes(text: string, size = CHUNK_64K): Uint8Array[] { + const all = encoder.encode(text) + const out: Uint8Array[] = [] + for (let off = 0; off < all.length; off += size) out.push(all.subarray(off, off + size)) + return out +} + +const median = (xs: number[]) => [...xs].sort((a, b) => a - b)[Math.floor(xs.length / 2)] + +/** #1779 AC1's assertion, made robust: a warm-up, then INTERLEAVED trials (both + * deliveries see the same machine noise), the median of each, and a generous + * factor. A quadratic reader misses it by an order of magnitude. + * `SPLIT_RATIO_LOG` (a file path) records each run's ratio for a flake report. */ +async function readTimeRatio(whole: Uint8Array[], chunked: Uint8Array[]) { + const timed = async (chunks: Uint8Array[]) => { + const t0 = performance.now() + await readStream([{ chunks }]) + return performance.now() - t0 + } + for (let i = 0; i < 2; i++) { + await timed(whole) + await timed(chunked) + } + const one: number[] = [] + const many: number[] = [] + for (let i = 0; i < 7; i++) { + one.push(await timed(whole)) + many.push(await timed(chunked)) + } + const wholeMs = median(one) + const chunkedMs = median(many) + return { ratio: chunkedMs / wholeMs, wholeMs, chunkedMs } +} +const SPLIT_RATIO_FACTOR = 3 + +describe("#1779 AC1 — the app reader splits a 10 MB frame in 64 KB chunks within a small constant factor of one chunk", () => { + test( + "chunked ÷ single-chunk read time (medians) stays ≤ 3×, and both deliver the one event with its SSE id", + async () => { + const big = largeSummaryFrame(1) + const whole = [bytes(big.frame)] + const chunked = chunkBytes(big.frame) + expect(whole[0].length).toBeGreaterThanOrEqual(TEN_MB) + expect(chunked.length).toBeGreaterThan(160) + for (const chunks of [whole, chunked]) { + const { frames, events } = await readStream([{ chunks }]) + expect(frames.map((f) => f.id)).toEqual([`${EPOCH}:1`]) // #1777's cursor source + expect(events).toHaveLength(1) + expect(Bun.deepEquals(events[0], big.event)).toBe(true) // no 10 MB diff on failure + } + const { ratio, wholeMs, chunkedMs } = await readTimeRatio(whole, chunked) + if (process.env.SPLIT_RATIO_LOG) + appendFileSync(process.env.SPLIT_RATIO_LOG, `app ${ratio.toFixed(3)} ${wholeMs.toFixed(2)} ${chunkedMs.toFixed(2)}\n`) + expect(ratio).toBeLessThanOrEqual(SPLIT_RATIO_FACTOR) + }, + 120_000, + ) +}) diff --git a/packages/app-bundle/overlay/packages/sdk/js/src/v2/gen/core/serverSentEvents.gen.ts b/packages/app-bundle/overlay/packages/sdk/js/src/v2/gen/core/serverSentEvents.gen.ts new file mode 100644 index 000000000..27a8fc635 --- /dev/null +++ b/packages/app-bundle/overlay/packages/sdk/js/src/v2/gen/core/serverSentEvents.gen.ts @@ -0,0 +1,268 @@ +// This file is auto-generated by @hey-api/openapi-ts + +import type { Config } from "./types.gen.js" + +export type ServerSentEventsOptions = Omit & + Pick & { + /** + * Fetch API implementation. You can use this option to provide a custom + * fetch instance. + * + * @default globalThis.fetch + */ + fetch?: typeof fetch + /** + * Implementing clients can call request interceptors inside this hook. + */ + onRequest?: (url: string, init: RequestInit) => Promise + /** + * Callback invoked when a network or parsing error occurs during streaming. + * + * This option applies only if the endpoint returns a stream of events. + * + * @param error The error that occurred. + */ + onSseError?: (error: unknown) => void + /** + * Callback invoked when an event is streamed from the server. + * + * This option applies only if the endpoint returns a stream of events. + * + * @param event Event streamed from the server. + * @returns Nothing (void). + */ + onSseEvent?: (event: StreamEvent) => void + serializedBody?: RequestInit["body"] + /** + * Default retry delay in milliseconds. + * + * This option applies only if the endpoint returns a stream of events. + * + * @default 3000 + */ + sseDefaultRetryDelay?: number + /** + * Maximum number of retry attempts before giving up. + */ + sseMaxRetryAttempts?: number + /** + * Maximum retry delay in milliseconds. + * + * Applies only when exponential backoff is used. + * + * This option applies only if the endpoint returns a stream of events. + * + * @default 30000 + */ + sseMaxRetryDelay?: number + /** + * Optional sleep function for retry backoff. + * + * Defaults to using `setTimeout`. + */ + sseSleepFn?: (ms: number) => Promise + url: string + } + +export interface StreamEvent { + data: TData + event?: string + id?: string + retry?: number +} + +export type ServerSentEventsResult = { + stream: AsyncGenerator ? TData[keyof TData] : TData, TReturn, TNext> +} + +export const createSseClient = ({ + onRequest, + onSseError, + onSseEvent, + responseTransformer, + responseValidator, + sseDefaultRetryDelay, + sseMaxRetryAttempts, + sseMaxRetryDelay, + sseSleepFn, + url, + ...options +}: ServerSentEventsOptions): ServerSentEventsResult => { + let lastEventId: string | undefined + + const sleep = sseSleepFn ?? ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms))) + + const createStream = async function* () { + let retryDelay: number = sseDefaultRetryDelay ?? 3000 + let attempt = 0 + const signal = options.signal ?? new AbortController().signal + + while (true) { + if (signal.aborted) break + + attempt++ + + const headers = + options.headers instanceof Headers + ? options.headers + : new Headers(options.headers as Record | undefined) + + if (lastEventId !== undefined) { + headers.set("Last-Event-ID", lastEventId) + } + + try { + const requestInit: RequestInit = { + redirect: "follow", + ...options, + body: options.serializedBody, + headers, + signal, + } + let request = new Request(url, requestInit) + if (onRequest) { + request = await onRequest(url, requestInit) + } + // fetch must be assigned here, otherwise it would throw the error: + // TypeError: Failed to execute 'fetch' on 'Window': Illegal invocation + const _fetch = options.fetch ?? globalThis.fetch + const response = await _fetch(request) + + if (!response.ok) throw new Error(`SSE failed: ${response.status} ${response.statusText}`) + + if (!response.body) throw new Error("No body in SSE response") + + const reader = response.body.pipeThrough(new TextDecoderStream()).getReader() + + // amicode #1779: split frames in LINEAR time. The generated reader + // appended every chunk to the whole partial buffer, then re-normalized + // and re-split all of it — quadratic on a multi-MB frame arriving in + // 64 KB chunks, on the webview's main thread. Now only the NEW text is + // normalized and searched; a "\n\n" split across two chunks is caught by + // a one-character carry; the partial frame is held as pieces and joined + // once, when it completes. Frames come out exactly as before. + let pending: Array = [] + let pendingLast = "" + const splitFrames = (text: string): Array => { + const frames: Array = [] + if (!text) return frames + let start = 0 + if (pendingLast === "\n" && text.charCodeAt(0) === 10) { + // the delimiter straddles the chunk boundary + frames.push(pending.join("").slice(0, -1)) + pending = [] + start = 1 + } + let end: number + while ((end = text.indexOf("\n\n", start)) !== -1) { + pending.push(text.slice(start, end)) + frames.push(pending.join("")) + pending = [] + start = end + 2 + } + if (start < text.length) { + pending.push(start === 0 ? text : text.slice(start)) + pendingLast = text.charAt(text.length - 1) + } else { + pendingLast = "" + } + return frames + } + + const abortHandler = () => { + try { + reader.cancel() + } catch { + // noop + } + } + + signal.addEventListener("abort", abortHandler) + + try { + while (true) { + const { done, value } = await reader.read() + if (done) break + // Normalize line endings: CRLF -> LF, then CR -> LF + const chunks = splitFrames(value.replace(/\r\n/g, "\n").replace(/\r/g, "\n")) + + for (const chunk of chunks) { + const lines = chunk.split("\n") + const dataLines: Array = [] + let eventName: string | undefined + + for (const line of lines) { + if (line.startsWith("data:")) { + dataLines.push(line.replace(/^data:\s*/, "")) + } else if (line.startsWith("event:")) { + eventName = line.replace(/^event:\s*/, "") + } else if (line.startsWith("id:")) { + lastEventId = line.replace(/^id:\s*/, "") + } else if (line.startsWith("retry:")) { + const parsed = Number.parseInt(line.replace(/^retry:\s*/, ""), 10) + if (!Number.isNaN(parsed)) { + retryDelay = parsed + } + } + } + + let data: unknown + let parsedJson = false + + if (dataLines.length) { + const rawData = dataLines.join("\n") + try { + data = JSON.parse(rawData) + parsedJson = true + } catch { + data = rawData + } + } + + if (parsedJson) { + if (responseValidator) { + await responseValidator(data) + } + + if (responseTransformer) { + data = await responseTransformer(data) + } + } + + onSseEvent?.({ + data, + event: eventName, + id: lastEventId, + retry: retryDelay, + }) + + if (dataLines.length) { + yield data as any + } + } + } + } finally { + signal.removeEventListener("abort", abortHandler) + reader.releaseLock() + } + + break // exit loop on normal completion + } catch (error) { + // connection failed or aborted; retry after delay + onSseError?.(error) + + if (sseMaxRetryAttempts !== undefined && attempt >= sseMaxRetryAttempts) { + break // stop after firing error + } + + // exponential backoff: double retry each attempt, cap at 30s + const backoff = Math.min(retryDelay * 2 ** (attempt - 1), sseMaxRetryDelay ?? 30000) + await sleep(backoff) + } + } + } + + const stream = createStream() + + return { stream } +} From 3b901f0affd1cb409efe3319fc0fea5694ccc169 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 21:38:07 -0400 Subject: [PATCH 57/60] fix(fleet): the relay ends a frame at a CRLF blank line too, split across chunks or not, and relays it verbatim (#1779) --- .../src/amicode_service/sse_fanin_driver.ts | 31 +++++-- .../test/sse_fanin_driver_pump.test.ts | 92 +++++++++++++++++++ 2 files changed, 113 insertions(+), 10 deletions(-) diff --git a/packages/extension/src/amicode_service/sse_fanin_driver.ts b/packages/extension/src/amicode_service/sse_fanin_driver.ts index d3fe6b8e8..52f851853 100644 --- a/packages/extension/src/amicode_service/sse_fanin_driver.ts +++ b/packages/extension/src/amicode_service/sse_fanin_driver.ts @@ -104,12 +104,17 @@ export const FANIN_LISTENER_CAP_ERROR = "fan-in listener cap reached"; * reader appended every chunk to the whole partial buffer and searched it from * the start, so a multi-MB frame arriving in 64 KB chunks cost quadratic time. * Now only the NEW chunk is scanned; a delimiter split across two chunks is - * caught by looking back one character into the partial frame, which is held - * as pieces and joined ONCE, when it completes. */ + * caught by looking back into the partial frame (at most two characters), + * which is held as pieces and joined ONCE, when it completes. + * + * A frame ends at a blank line — a line feed whose line is empty: `\n\n`, or + * with CRLF line endings `\r\n\r\n` (any LF/CRLF mix: `\n\r\n`, `\r\n\n`). The + * frame is relayed exactly as received, CRs and all — never normalized (#1264 + * byte identity); the aggregator reads either line ending. */ export class SseFrameSplitter { /** The partial frame, as received (never re-copied while it grows). */ private parts: string[] = []; - /** The partial frame's last character — the lookback for a split delimiter. */ + /** The partial frame's last ≤2 characters — the lookback for a split delimiter. */ private tail = ""; /** Feed one decoded chunk; returns the frames it completed, in order. */ @@ -118,22 +123,27 @@ export class SseFrameSplitter { if (chunk === "") return frames; const tail = this.tail; let start = 0; // where the partial frame resumes in this chunk + let floor = -tail.length; // the current frame's first position (negative = in the tail) + /** The character at chunk position `i` (negative reaches into the tail), or + * NaN before the current frame — a lookback never crosses a frame boundary. */ + const at = (i: number): number => + i < floor ? NaN : i >= 0 ? chunk.charCodeAt(i) : tail.charCodeAt(tail.length + i); let from = 0; let lf: number; while ((lf = chunk.indexOf("\n", from)) !== -1) { from = lf + 1; - // the line feed ends a blank line iff the character before it — in this - // chunk, or the partial frame's tail when it is the chunk's first — is one too - const prev = lf > start ? chunk.charCodeAt(lf - 1) : lf === 0 ? tail.charCodeAt(0) : NaN; - if (prev !== LF_CODE) continue; + // the line this line feed ends is empty: it follows another line ending + const prev = at(lf - 1); + if (prev !== LF_CODE && !(prev === CR_CODE && at(lf - 2) === LF_CODE)) continue; this.parts.push(chunk.slice(start, from)); frames.push(this.parts.length === 1 ? this.parts[0] : this.parts.join("")); // whole frame, delimiter-inclusive this.parts = []; - start = from; + start = floor = from; } if (start < chunk.length) { - this.parts.push(start === 0 ? chunk : chunk.slice(start)); - this.tail = chunk.charAt(chunk.length - 1); + const rest = start === 0 ? chunk : chunk.slice(start); + this.parts.push(rest); + this.tail = rest.length >= 2 ? rest.slice(-2) : (start === 0 ? tail + rest : rest).slice(-2); } else { this.tail = ""; // a frame ended exactly at the chunk's end } @@ -141,6 +151,7 @@ export class SseFrameSplitter { } } const LF_CODE = 10; +const CR_CODE = 13; /** The production upstream reader: a frame-preserving http SSE source. Splits the * byte stream on the SSE blank-line delimiter into WHOLE frames (delimiter- diff --git a/packages/extension/test/sse_fanin_driver_pump.test.ts b/packages/extension/test/sse_fanin_driver_pump.test.ts index 7548bb15b..cb75c4887 100644 --- a/packages/extension/test/sse_fanin_driver_pump.test.ts +++ b/packages/extension/test/sse_fanin_driver_pump.test.ts @@ -2315,6 +2315,98 @@ describe("#1779 AC1 — the relay reader splits a 10 MB frame in 64 KB chunks wi }, 60_000); }); +/** Every way to cut `text`'s bytes into two chunks, and one byte at a time. */ +function everySplit(text: string): Buffer[][] { + const all = Buffer.from(text, "utf8"); + const deliveries: Buffer[][] = []; + for (let k = 1; k < all.length; k++) deliveries.push([all.subarray(0, k), all.subarray(k)]); + deliveries.push(Array.from({ length: all.length }, (_, i) => all.subarray(i, i + 1))); + return deliveries; +} + +/** CRLF framing (SSE allows it; a non-engine upstream or a proxy may use it). */ +const crlf = (frame: string): string => frame.replace(/\n/g, "\r\n"); + +describe("#1779 AC2 — the relay reader finds frame boundaries split across chunks, LF and CRLF alike, and relays each frame verbatim", () => { + const engine = fakeEngine({ epoch: LOCAL_EPOCH }); + const frames = [ + engine.connected(), // LF, the engine's own framing + crlf(engine.event("message.part.updated", { part: { id: "prt_1", text: "résumé → done" } })), // CRLF, non-ASCII + `event: message\nid: ${localId(2)}\r\ndata: {"n":2}\n\r\n`, // CRLF blank line after LF lines + `: keep-alive\r\n\n`, // LF blank line after a CRLF line + crlf(engine.heartbeat()), + engine.event("session.status", { status: "idle" }), + ]; + const stream = frames.join(""); + + it("cut at EVERY byte (the delimiter, a CRLF, a code point split between two chunks), and byte by byte: the same frames, verbatim", () => { + expect(readFrames([Buffer.from(stream, "utf8")])).toEqual(frames); + for (const delivery of everySplit(stream)) { + const got = readFrames(delivery); + if (got.length !== frames.length || got.some((f, i) => f !== frames[i])) { + expect({ cuts: delivery.map((c) => c.length), got }).toEqual({ cuts: delivery.map((c) => c.length), got: frames }); + } + } + }); + + it("through the real reader and a fleet-of-one driver: a CRLF stream whose delimiters straddle network chunks reaches the listener byte-identical (#1264)", async () => { + const all = Buffer.from(stream, "utf8"); + const startOf = (i: number): number => frames.slice(0, i).join("").length; + const byteAt = (charIndex: number): number => Buffer.byteLength(stream.slice(0, charIndex)); + const cuts = [ + byteAt(startOf(1) + frames[1].indexOf("\r\n") + 1), // a CRLF line ending: "\r" | "\n" + byteAt(startOf(1) + frames[1].indexOf("é")) + 1, // inside a code point + byteAt(startOf(2) - 1), // the CRLF delimiter: "\r\n\r" | "\n" + byteAt(startOf(3) - 2), // "\n" | "\r\n" (LF line, CRLF blank line) + byteAt(startOf(5) - 2), // the CRLF delimiter: "\r\n" | "\r\n" + ].sort((a, b) => a - b); + const chunks: Buffer[] = []; + let at = 0; + for (const end of cuts) { + chunks.push(all.subarray(at, end)); + at = end; + } + chunks.push(all.subarray(at)); + expect(Buffer.concat(chunks).equals(all)).toBe(true); + const server = await startEngineSseServer(async (res) => { + for (const c of chunks) { + res.write(c); + await new Promise((r) => setImmediate(r)); // separate network reads (best effort; the cut-everywhere test above is exact) + } + res.end(); + }); + const lines: string[] = []; + const w = lineWaiter(lines); + const census = new FanInCensus({ plane: "observe", sink: w.sink, now: () => 0 }); + const driver = new SseFanInDriver({ + ownerMap: new SessionOwnerMap(), // fleet-of-one: the verbatim relay + localMachineId: "macbook", + localEventUrl: () => server.url, + peerBaseUrl: () => undefined, + peerToken: () => ({ ok: false as const, reason: "absent" as const }), + reconcileMs: 999_999, + census, + }); + const res = mockRes(); + try { + driver.handle(mockReq("/global/event"), res); + await w.waitFor((l) => l.includes("arm=local reason=")); + expect(res.text()).toBe(stream); + // … and frame by frame: the census saw exactly these frames (a reader + // blind to CRLF would have merged them — the bytes alone cannot tell) + census.emit(); + const local = parseCensus(lines.at(-1)!).arms.find((a) => a.arm === "local")!; + expect(local).toMatchObject({ + frames: String(frames.length), + largest: String(Math.max(...frames.map((f) => Buffer.byteLength(f)))), + }); + } finally { + res.fireClose(); + await server.stop(); + } + }); +}); + // ══════════════════════════════════════════════════════════════════════════════ // #1777 — LOSSLESS RESUME. The engine (#1776) sequences and replays its global // stream; the fan-in now uses it. A re-opened arm resumes from its last delivered From bffbf18f19bdf4f012058964047120c846f7c888 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 21:40:27 -0400 Subject: [PATCH 58/60] fix(app): the global-stream reader holds a chunk's trailing CR for the next chunk, so a CRLF split across chunks never ends a frame early (#1779) --- packages/app-bundle/manifest.json | 6 +-- .../app/src/context/sse-reader.test.ts | 49 +++++++++++++++++++ .../src/v2/gen/core/serverSentEvents.gen.ts | 18 +++++-- 3 files changed, 66 insertions(+), 7 deletions(-) diff --git a/packages/app-bundle/manifest.json b/packages/app-bundle/manifest.json index fd9ce9f46..a62662745 100644 --- a/packages/app-bundle/manifest.json +++ b/packages/app-bundle/manifest.json @@ -204,7 +204,7 @@ "packages/app/src/context/sse-gap-frame.ts": "2f64fde540a305579762c5276721d4bbfbf7c862131aaf4462e8a4e8f32c5960", "packages/app/src/context/sse-origin.test.ts": "71cd3aa996a51d7e99236a49cede4769e520e20e53f57c6920fe441f2abb0c7d", "packages/app/src/context/sse-origin.ts": "da9a231907dc50d87cf01758177bee0d4a0b0165d8adb5e152db8eca29dfd9f5", - "packages/app/src/context/sse-reader.test.ts": "a6151ded71f97e911a84be946f43e5c20495512a4ff79361d7e5673d457dd0ca", + "packages/app/src/context/sse-reader.test.ts": "20ab571709f8e945bc655fbe684741c9dcce9f1ef42295fae6fd1d053ad74196", "packages/app/src/context/stream-cursor.test.ts": "c9715bb28f576cdc00683b0bdc27bdc2a56a6f212969cc090f99b3dc8988e6a7", "packages/app/src/context/stream-cursor.ts": "2711f85581cd86bb69b50a441b6a04c532553f61fa5e00cd1b9daa12193d3697", "packages/app/src/context/stream-gap.test.ts": "b7203445564f44fd1d966ef684a11cb9b5ec567d9ab54f1b7617726fdf44d3f5", @@ -936,7 +936,7 @@ "packages/session-ui/src/v2/components/prompt-input/attachments.ts": "ff66078f2edc9a67a4df0d7dfd76a49be5eab27ac332b303f3412721ef7268ea", "packages/session-ui/src/v2/components/prompt-input/index.tsx": "194b1c6d0c99000579df78b47c7ff72fe34f62a92ecd21b2a9377133d6d27bb9", "packages/opencode/src/server/routes/instance/httpapi/server.ts": "a94becd6a5224fb5fc2778c1091733eca334b308253234948bcdcafdc709b021", - "packages/sdk/js/src/v2/gen/core/serverSentEvents.gen.ts": "7509e74c8ccd756bbe60598f38754775a62c36e82d8f5f59464fb18a49be4688", + "packages/sdk/js/src/v2/gen/core/serverSentEvents.gen.ts": "9da41a900811e8526f3d6796341b787f7e3c64bdc9cc3a547977f05780c1f79f", "packages/opencode/src/server/routes/instance/httpapi/groups/experimental.ts": "944a0aee215a917e867fd9482f3c2bd40f49082bd3174c8a7d967ca40e34ec15", "packages/opencode/src/server/routes/instance/httpapi/groups/file.ts": "a96fdc55ddb95ca7c481c86f67e37ed88d4cf2a190a482765ab55ce4409f3a59", "packages/opencode/src/server/routes/instance/httpapi/groups/mcp.ts": "048efa7b74d43c4ad4a75d0f8421c808760c8c0dcd593f0051a87044e40487d6", @@ -963,7 +963,7 @@ "packages/ui/src/v2/components/text-shimmer-v2.tsx" ], "exceptions": [], - "extracted_at": "2026-10-13T01:35:20.070Z", + "extracted_at": "2026-10-13T01:40:27.081Z", "per_package": { "packages/app": { "A": 240, diff --git a/packages/app-bundle/overlay/packages/app/src/context/sse-reader.test.ts b/packages/app-bundle/overlay/packages/app/src/context/sse-reader.test.ts index dac409b97..955033d73 100644 --- a/packages/app-bundle/overlay/packages/app/src/context/sse-reader.test.ts +++ b/packages/app-bundle/overlay/packages/app/src/context/sse-reader.test.ts @@ -150,3 +150,52 @@ describe("#1779 AC1 — the app reader splits a 10 MB frame in 64 KB chunks with 120_000, ) }) + +/** Every way to cut `text`'s bytes into two chunks, and one byte at a time. */ +function everySplit(text: string): Uint8Array[][] { + const all = bytes(text) + const deliveries: Uint8Array[][] = [] + for (let k = 1; k < all.length; k++) deliveries.push([all.subarray(0, k), all.subarray(k)]) + deliveries.push(Array.from({ length: all.length }, (_, i) => all.subarray(i, i + 1))) + return deliveries +} + +describe("#1779 AC2 — the app reader finds frame boundaries split across chunks, CRLF line endings included", () => { + const stream = [ + `id: ${EPOCH}:0\ndata: {"payload":{"type":"server.connected"}}\n\n`, // LF, the engine's framing + `id: ${EPOCH}:1\r\ndata: {"n":1,"text":"résumé → done"}\r\n\r\n`, // CRLF, non-ASCII + `data: first line\r\ndata: second line\r\n\r\n`, // CRLF, multi-line data + `event: ping\rdata: {"n":3}\r\r`, // CR line endings + `data: {"n":4}\n\n`, + ].join("") + const expected: Frame[] = [ + { data: { payload: { type: "server.connected" } }, event: undefined, id: `${EPOCH}:0`, retry: 3000 }, + { data: { n: 1, text: "résumé → done" }, event: undefined, id: `${EPOCH}:1`, retry: 3000 }, + { data: "first line\nsecond line", event: undefined, id: `${EPOCH}:1`, retry: 3000 }, + { data: { n: 3 }, event: "ping", id: `${EPOCH}:1`, retry: 3000 }, + { data: { n: 4 }, event: undefined, id: `${EPOCH}:1`, retry: 3000 }, + ] + + test("in one chunk: each frame once, with its id", async () => { + const { frames, events } = await readStream([{ chunks: [bytes(stream)] }]) + expect(frames).toEqual(expected) + expect(events).toEqual(expected.map((f) => f.data)) + }) + + test("cut at EVERY byte — a delimiter, a CRLF, a code point split between two chunks — and byte by byte: the same frames, no extra, none merged", async () => { + for (const chunks of everySplit(stream)) { + const { frames, events } = await readStream([{ chunks }]) + const cuts = chunks.map((c) => c.length) + expect({ cuts, frames }).toEqual({ cuts, frames: expected }) + expect({ cuts, events }).toEqual({ cuts, events: expected.map((f) => f.data) }) + } + }) + + test("a stream that ENDS on a CR line ending still delivers its last frame (a held \\r is a line end at the end)", async () => { + const crOnly = `data: {"n":1}\r\rdata: {"n":2}\r\r` + for (const chunks of [[bytes(crOnly)], ...everySplit(crOnly)]) { + const { events } = await readStream([{ chunks }]) + expect({ cuts: chunks.map((c) => c.length), events }).toEqual({ cuts: chunks.map((c) => c.length), events: [{ n: 1 }, { n: 2 }] }) + } + }) +}) diff --git a/packages/app-bundle/overlay/packages/sdk/js/src/v2/gen/core/serverSentEvents.gen.ts b/packages/app-bundle/overlay/packages/sdk/js/src/v2/gen/core/serverSentEvents.gen.ts index 27a8fc635..2bd527c63 100644 --- a/packages/app-bundle/overlay/packages/sdk/js/src/v2/gen/core/serverSentEvents.gen.ts +++ b/packages/app-bundle/overlay/packages/sdk/js/src/v2/gen/core/serverSentEvents.gen.ts @@ -140,9 +140,19 @@ export const createSseClient = ({ // 64 KB chunks, on the webview's main thread. Now only the NEW text is // normalized and searched; a "\n\n" split across two chunks is caught by // a one-character carry; the partial frame is held as pieces and joined - // once, when it completes. Frames come out exactly as before. + // once, when it completes. A "\r" ending a chunk waits for the next one + // (it may be the first half of a CRLF — normalizing it alone made a CRLF + // split across chunks end its frame early). Frames come out as before. let pending: Array = [] let pendingLast = "" + let heldCR = false + // Normalize line endings: CRLF -> LF, then CR -> LF + const normalize = (value: string, final: boolean): string => { + let text = heldCR ? "\r" + value : value + heldCR = !final && text.endsWith("\r") + if (heldCR) text = text.slice(0, -1) + return text.includes("\r") ? text.replace(/\r\n/g, "\n").replace(/\r/g, "\n") : text + } const splitFrames = (text: string): Array => { const frames: Array = [] if (!text) return frames @@ -182,9 +192,8 @@ export const createSseClient = ({ try { while (true) { const { done, value } = await reader.read() - if (done) break - // Normalize line endings: CRLF -> LF, then CR -> LF - const chunks = splitFrames(value.replace(/\r\n/g, "\n").replace(/\r/g, "\n")) + // at the end of the stream, a held "\r" ends its line after all + const chunks = splitFrames(normalize(done ? "" : value, done)) for (const chunk of chunks) { const lines = chunk.split("\n") @@ -240,6 +249,7 @@ export const createSseClient = ({ yield data as any } } + if (done) break } } finally { signal.removeEventListener("abort", abortHandler) From ca888d9840abefc81593329b3b1c54d2ff9ddf88 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 21:43:32 -0400 Subject: [PATCH 59/60] =?UTF-8?q?test(fleet):=20both=20readers'=20framing?= =?UTF-8?q?=20pinned=20unchanged=20=E2=80=94=20multi-line=20data,=20event?= =?UTF-8?q?=20names,=20ids,=20retry=20fields,=20cut=20at=20every=20byte=20?= =?UTF-8?q?(#1779)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/app-bundle/manifest.json | 4 +- .../app/src/context/sse-reader.test.ts | 75 +++++++++++++++++-- .../test/sse_fanin_driver_pump.test.ts | 34 ++++++++- 3 files changed, 103 insertions(+), 10 deletions(-) diff --git a/packages/app-bundle/manifest.json b/packages/app-bundle/manifest.json index a62662745..fe78d3d03 100644 --- a/packages/app-bundle/manifest.json +++ b/packages/app-bundle/manifest.json @@ -204,7 +204,7 @@ "packages/app/src/context/sse-gap-frame.ts": "2f64fde540a305579762c5276721d4bbfbf7c862131aaf4462e8a4e8f32c5960", "packages/app/src/context/sse-origin.test.ts": "71cd3aa996a51d7e99236a49cede4769e520e20e53f57c6920fe441f2abb0c7d", "packages/app/src/context/sse-origin.ts": "da9a231907dc50d87cf01758177bee0d4a0b0165d8adb5e152db8eca29dfd9f5", - "packages/app/src/context/sse-reader.test.ts": "20ab571709f8e945bc655fbe684741c9dcce9f1ef42295fae6fd1d053ad74196", + "packages/app/src/context/sse-reader.test.ts": "3536b49e8c1f2b94accaabc0bff5348cc9e99dde93caccd33c5b80ae04f2eef0", "packages/app/src/context/stream-cursor.test.ts": "c9715bb28f576cdc00683b0bdc27bdc2a56a6f212969cc090f99b3dc8988e6a7", "packages/app/src/context/stream-cursor.ts": "2711f85581cd86bb69b50a441b6a04c532553f61fa5e00cd1b9daa12193d3697", "packages/app/src/context/stream-gap.test.ts": "b7203445564f44fd1d966ef684a11cb9b5ec567d9ab54f1b7617726fdf44d3f5", @@ -963,7 +963,7 @@ "packages/ui/src/v2/components/text-shimmer-v2.tsx" ], "exceptions": [], - "extracted_at": "2026-10-13T01:40:27.081Z", + "extracted_at": "2026-10-13T01:43:32.128Z", "per_package": { "packages/app": { "A": 240, diff --git a/packages/app-bundle/overlay/packages/app/src/context/sse-reader.test.ts b/packages/app-bundle/overlay/packages/app/src/context/sse-reader.test.ts index 955033d73..6ff72d9be 100644 --- a/packages/app-bundle/overlay/packages/app/src/context/sse-reader.test.ts +++ b/packages/app-bundle/overlay/packages/app/src/context/sse-reader.test.ts @@ -15,10 +15,14 @@ const SERVER = { url: "http://127.0.0.1:4096" } /** What one parsed frame reports through `onSseEvent` (the app reads `id`). */ type Frame = { data: unknown; event?: string; id?: string; retry?: number } -/** A fetch that answers the n-th request with the n-th delivery's chunks, one - * body read per chunk. A delivery with `fail` errors its body after its chunks - * (a dropped link). Every request is recorded. */ -function streamFetch(deliveries: Array<{ chunks: Uint8Array[]; fail?: boolean }>, requests: Request[] = []) { +/** One response body: its chunks, one body read each. With `failWhen`, the body + * errors (a dropped link) once that promise settles — never earlier, so no + * decoded chunk still queued in the pipe is discarded by the error. */ +type Delivery = { chunks: Uint8Array[]; failWhen?: Promise } + +/** A fetch that answers the n-th request with the n-th delivery. Every request + * is recorded. */ +function streamFetch(deliveries: Delivery[], requests: Request[] = []) { return async (input: RequestInfo | URL, init?: RequestInit) => { const request = input instanceof Request ? input : new Request(input, init) const delivery = deliveries[Math.min(requests.length, deliveries.length - 1)] @@ -27,7 +31,7 @@ function streamFetch(deliveries: Array<{ chunks: Uint8Array[]; fail?: boolean }> const body = new ReadableStream({ pull(controller) { if (next < delivery.chunks.length) controller.enqueue(delivery.chunks[next++]) - else if (delivery.fail) controller.error(new Error("link dropped")) + else if (delivery.failWhen) return delivery.failWhen.then(() => controller.error(new Error("link dropped"))) else controller.close() }, }) @@ -38,13 +42,17 @@ function streamFetch(deliveries: Array<{ chunks: Uint8Array[]; fail?: boolean }> /** Read the global stream to its end: every frame `onSseEvent` reported, and * every event the stream yielded, in order. */ async function readStream( - deliveries: Array<{ chunks: Uint8Array[]; fail?: boolean }>, + deliveries: Delivery[], requests: Request[] = [], + onFrame?: (frame: Frame) => void, ): Promise<{ frames: Frame[]; events: unknown[] }> { const sdk = createSdkForServer({ server: SERVER, fetch: streamFetch(deliveries, requests) as typeof fetch }) const frames: Frame[] = [] const result = await sdk.global.event({ - onSseEvent: (frame: Frame) => frames.push({ ...frame }), + onSseEvent: (frame: Frame) => { + frames.push({ ...frame }) + onFrame?.(frame) + }, sseSleepFn: async () => {}, }) const events: unknown[] = [] @@ -199,3 +207,56 @@ describe("#1779 AC2 — the app reader finds frame boundaries split across chunk } }) }) + +describe("#1779 AC3 — the app reader's framing is unchanged: multi-line data, event names, ids, retry fields", () => { + // Pinned against the generated reader as it was before #1779, in one chunk and + // cut at every byte (the parse of each frame must not depend on where it was cut). + const stream = [ + `: a comment line is ignored\nevent: message.updated\nid: ${EPOCH}:1\ndata: {"a":\ndata: 1}\n\n`, // multi-line data joins with \n + `data:no-space\nretry: 5000\n\n`, // a field without a space; a retry sets the delay + `data: padded\nretry: soon\n\n`, // the reader strips all leading whitespace; a bad retry is ignored + `: keep-alive\n\n`, // no data: reported, never yielded + `id: ${EPOCH}:2\nevent: x\nevent: y\ndata: {"b":2}\n\n`, // last event name wins + `data: {"c":3}\n\n`, // no id line: the last id stands (SSE last-event-ID) + `\ndata: {"d":4}\n\n`, // a stray blank line before a frame joins that frame (no empty dispatch) + `id:\ndata: {"e":5}\n\n`, // an empty id resets it + `data: {"lost":true}\n`, // incomplete at the end of the stream: discarded + ].join("") + const expected: Frame[] = [ + { data: { a: 1 }, event: "message.updated", id: `${EPOCH}:1`, retry: 3000 }, + { data: "no-space", event: undefined, id: `${EPOCH}:1`, retry: 5000 }, + { data: "padded", event: undefined, id: `${EPOCH}:1`, retry: 5000 }, + { data: undefined, event: undefined, id: `${EPOCH}:1`, retry: 5000 }, + { data: { b: 2 }, event: "y", id: `${EPOCH}:2`, retry: 5000 }, + { data: { c: 3 }, event: undefined, id: `${EPOCH}:2`, retry: 5000 }, + { data: { d: 4 }, event: undefined, id: `${EPOCH}:2`, retry: 5000 }, + { data: { e: 5 }, event: undefined, id: "", retry: 5000 }, + ] + const yielded = [{ a: 1 }, "no-space", "padded", { b: 2 }, { c: 3 }, { d: 4 }, { e: 5 }] + + test("in one chunk, and cut at every byte: the same frames, fields and events", async () => { + for (const chunks of [[bytes(stream)], ...everySplit(stream)]) { + const { frames, events } = await readStream([{ chunks }]) + const cuts = chunks.map((c) => c.length) + expect({ cuts, frames }).toEqual({ cuts, frames: expected }) + expect({ cuts, events }).toEqual({ cuts, events: yielded }) + } + }) + + test("ids: after a dropped link the SDK's own retry carries the last id it parsed (the Last-Event-ID header #1777's sseFetch strips)", async () => { + const requests: Request[] = [] + const first = `id: ${EPOCH}:6\ndata: {"n":6}\n\nid: ${EPOCH}:7\ndata: {"n":7}\n\n` + let parsedLast!: () => void + const dropLink = new Promise((resolve) => (parsedLast = resolve)) + const { frames, events } = await readStream( + [{ chunks: chunkBytes(first, 7), failWhen: dropLink }, { chunks: [bytes(`id: ${EPOCH}:8\ndata: {"n":8}\n\n`)] }], + requests, + (frame) => frame.id === `${EPOCH}:7` && parsedLast(), + ) + expect(requests).toHaveLength(2) + expect(requests[0].headers.get("last-event-id")).toBeNull() + expect(requests[1].headers.get("last-event-id")).toBe(`${EPOCH}:7`) + expect(frames.map((f) => f.id)).toEqual([`${EPOCH}:6`, `${EPOCH}:7`, `${EPOCH}:8`]) + expect(events).toEqual([{ n: 6 }, { n: 7 }, { n: 8 }]) + }) +}) diff --git a/packages/extension/test/sse_fanin_driver_pump.test.ts b/packages/extension/test/sse_fanin_driver_pump.test.ts index cb75c4887..0eb0d9b3e 100644 --- a/packages/extension/test/sse_fanin_driver_pump.test.ts +++ b/packages/extension/test/sse_fanin_driver_pump.test.ts @@ -9,7 +9,7 @@ import { appendFileSync } from "node:fs"; import { StringDecoder } from "node:string_decoder"; import { SseFanInDriver, SseFrameSplitter, openHttpSseSource } from "../src/amicode_service/sse_fanin_driver"; import type { SseFrameSource } from "../src/amicode_service/sse_fanin_aggregator"; -import { LOCAL_NAMESPACE } from "../src/amicode_service/sse_fanin_aggregator"; +import { LOCAL_NAMESPACE, frameRawId, isDataFrame } from "../src/amicode_service/sse_fanin_aggregator"; import { SessionOwnerMap } from "../src/amicode_service/session_multiplexer"; import { FanInCensus, FANIN_CENSUS_INTERVAL_MS, FANIN_CLOSE_REASONS } from "../src/amicode_service/fanin_census"; import { @@ -2407,6 +2407,38 @@ describe("#1779 AC2 — the relay reader finds frame boundaries split across chu }); }); +describe("#1779 AC3 — the relay reader's framing is unchanged: multi-line data, event names, ids, retry fields — each frame verbatim", () => { + // Pinned against the splitter as it was before #1779, whole and cut at every + // byte: where a frame was cut never changes which frames come out. + const engine = fakeEngine({ epoch: LOCAL_EPOCH }); + const connected = engine.connected(); + const [plain, twin] = engine.stored("message.updated", { info: { id: "msg_1" } }, { aggregateID: "ses_1" }); + const frames = [ + connected, + plain, // a durable event … + twin, // … and its sync twin: the same id + `: a comment\nevent: message.part.updated\nid: ${localId(2)}\nretry: 5000\ndata: {"a":\ndata: 1}\n\n`, // every field, multi-line data + `data:no-space\n\n`, + `id:\ndata: {"b":2}\n\n`, // an empty id (it advances nothing) + `: keep-alive\n\n`, // comment-only (not a data frame) + `\ndata: {"c":3}\n\n`, // a stray blank line joins the frame after it + `\n\n`, // … and alone it is an empty frame + engine.heartbeat(), + ]; + const stream = frames.join(""); + + it("whole and cut at every byte: the same frames, verbatim, with the ids and data the aggregator reads", () => { + for (const delivery of [[Buffer.from(stream, "utf8")], ...everySplit(stream)]) { + const got = readFrames(delivery); + if (got.length !== frames.length || got.some((f, i) => f !== frames[i])) { + expect({ cuts: delivery.map((c) => c.length), got }).toEqual({ cuts: delivery.map((c) => c.length), got: frames }); + } + } + expect(frames.map(frameRawId)).toEqual([localId(0), localId(1), localId(1), localId(2), undefined, undefined, undefined, undefined, undefined, localId(1)]); + expect(frames.map(isDataFrame)).toEqual([true, true, true, true, true, true, false, true, false, true]); + }); +}); + // ══════════════════════════════════════════════════════════════════════════════ // #1777 — LOSSLESS RESUME. The engine (#1776) sequences and replays its global // stream; the fan-in now uses it. A re-opened arm resumes from its last delivered From bb11b8f5e110289aab6c85b430a43135083eae83 Mon Sep 17 00:00:00 2001 From: amicode-ci Date: Mon, 12 Oct 2026 21:45:32 -0400 Subject: [PATCH 60/60] test(app): the reader test uses the SDK's typed retry knob, not an untyped pass-through (#1779) --- packages/app-bundle/manifest.json | 4 ++-- .../packages/app/src/context/sse-reader.test.ts | 11 ++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/app-bundle/manifest.json b/packages/app-bundle/manifest.json index fe78d3d03..459a66922 100644 --- a/packages/app-bundle/manifest.json +++ b/packages/app-bundle/manifest.json @@ -204,7 +204,7 @@ "packages/app/src/context/sse-gap-frame.ts": "2f64fde540a305579762c5276721d4bbfbf7c862131aaf4462e8a4e8f32c5960", "packages/app/src/context/sse-origin.test.ts": "71cd3aa996a51d7e99236a49cede4769e520e20e53f57c6920fe441f2abb0c7d", "packages/app/src/context/sse-origin.ts": "da9a231907dc50d87cf01758177bee0d4a0b0165d8adb5e152db8eca29dfd9f5", - "packages/app/src/context/sse-reader.test.ts": "3536b49e8c1f2b94accaabc0bff5348cc9e99dde93caccd33c5b80ae04f2eef0", + "packages/app/src/context/sse-reader.test.ts": "6429e846ef4ee2bccc25cdd7ee1915d787b557fb9355d6fbf2c99d15846337ac", "packages/app/src/context/stream-cursor.test.ts": "c9715bb28f576cdc00683b0bdc27bdc2a56a6f212969cc090f99b3dc8988e6a7", "packages/app/src/context/stream-cursor.ts": "2711f85581cd86bb69b50a441b6a04c532553f61fa5e00cd1b9daa12193d3697", "packages/app/src/context/stream-gap.test.ts": "b7203445564f44fd1d966ef684a11cb9b5ec567d9ab54f1b7617726fdf44d3f5", @@ -963,7 +963,7 @@ "packages/ui/src/v2/components/text-shimmer-v2.tsx" ], "exceptions": [], - "extracted_at": "2026-10-13T01:43:32.128Z", + "extracted_at": "2026-10-13T01:45:32.169Z", "per_package": { "packages/app": { "A": 240, diff --git a/packages/app-bundle/overlay/packages/app/src/context/sse-reader.test.ts b/packages/app-bundle/overlay/packages/app/src/context/sse-reader.test.ts index 6ff72d9be..72113cd37 100644 --- a/packages/app-bundle/overlay/packages/app/src/context/sse-reader.test.ts +++ b/packages/app-bundle/overlay/packages/app/src/context/sse-reader.test.ts @@ -40,20 +40,21 @@ function streamFetch(deliveries: Delivery[], requests: Request[] = []) { } /** Read the global stream to its end: every frame `onSseEvent` reported, and - * every event the stream yielded, in order. */ + * every event the stream yielded, in order. `retryDelayMs` sets the SDK's + * reconnect delay (its default is 3000 ms, reported as each frame's `retry`). */ async function readStream( deliveries: Delivery[], requests: Request[] = [], - onFrame?: (frame: Frame) => void, + opts: { onFrame?: (frame: Frame) => void; retryDelayMs?: number } = {}, ): Promise<{ frames: Frame[]; events: unknown[] }> { const sdk = createSdkForServer({ server: SERVER, fetch: streamFetch(deliveries, requests) as typeof fetch }) const frames: Frame[] = [] const result = await sdk.global.event({ onSseEvent: (frame: Frame) => { frames.push({ ...frame }) - onFrame?.(frame) + opts.onFrame?.(frame) }, - sseSleepFn: async () => {}, + ...(opts.retryDelayMs !== undefined ? { sseDefaultRetryDelay: opts.retryDelayMs } : {}), }) const events: unknown[] = [] for await (const event of result.stream) events.push(event) @@ -251,7 +252,7 @@ describe("#1779 AC3 — the app reader's framing is unchanged: multi-line data, const { frames, events } = await readStream( [{ chunks: chunkBytes(first, 7), failWhen: dropLink }, { chunks: [bytes(`id: ${EPOCH}:8\ndata: {"n":8}\n\n`)] }], requests, - (frame) => frame.id === `${EPOCH}:7` && parsedLast(), + { onFrame: (frame) => frame.id === `${EPOCH}:7` && parsedLast(), retryDelayMs: 0 }, ) expect(requests).toHaveLength(2) expect(requests[0].headers.get("last-event-id")).toBeNull()